diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e82c4b0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.22" + - name: format + run: test -z "$(gofmt -l cmd internal)" + - name: vet + run: go vet ./... + - name: test + run: go test ./... -race -count=1 + - name: failure + rebalance demos + run: | + ./scripts/run-cluster.sh + ./scripts/failure-demo.sh + ./scripts/rebalance-demo.sh + ./scripts/stop-cluster.sh diff --git a/Makefile b/Makefile index 2be8236..336b6ce 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test race bench cluster stop demo fmt +.PHONY: build test race bench cluster stop demo rebalance quickstart fmt ci build: mkdir -p bin @@ -23,5 +23,16 @@ stop: bench: build ./scripts/bench.sh +quickstart: + ./scripts/quickstart.sh + demo: build ./scripts/failure-demo.sh + +rebalance: build + ./scripts/rebalance-demo.sh + +ci: + test -z "$$(gofmt -l cmd internal)" + go vet ./... + go test ./... -race -count=1 diff --git a/README.md b/README.md index 3758045..21f628f 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ # ringcache -Distributed **in-memory** cache. One Go process per node, a consistent-hash ring with virtual nodes, per-node LRU+TTL, and synchronous replication (`R≥2`). +Distributed **in-memory** cache. One Go process per node, a consistent-hash ring with virtual nodes, per-node LRU+TTL, and synchronous replication (`R=2`). -This is a systems MVP, not a product: no persistence, no auth, no dynamic membership, no linearizability. Numbers below were **measured** on this machine, not invented. +**Consistency in one line:** a SET is successful when `acked ≥ 1`. That is **not** linearizable, **not** a quorum, and **not** durable. Replication is R=2 synchronous fan-out; last-writer-wins on coordinator `written_at`; nothing is written to disk. + +This is a systems MVP, not a product: no auth, no gossip membership, no WAL. Numbers below were **measured** on this machine, not invented. ```mermaid flowchart LR @@ -30,23 +32,38 @@ A client talks to **any** node. That node is the coordinator: it hashes the key, --- -## Run +## 60-second run -### Docker Compose (3 nodes) +Needs Go 1.22+ and `curl`. Docker is optional. ```bash -docker compose up --build +git clone https://github.com/hiroshi-os/ringcache && cd ringcache +./scripts/quickstart.sh ``` -Host ports: `node-a :8080`, `node-b :8081`, `node-c :8082`. Compose uses `network_mode: host` so the three processes can replicate over loopback (a Docker bridge on this nested CI VM drops 100% of veth-to-veth packets). On Docker Desktop (Mac/Windows), use `make cluster` instead. +That builds `bin/ringcache`, starts **node-a/b/c** on `127.0.0.1:8080-8082`, SETs `user:1`, and GETs it from another coordinator. -### Local processes (no Docker) +Same thing by hand: ```bash make cluster # 127.0.0.1:8080-8082 -# make stop # tear down +curl -sS -X PUT http://127.0.0.1:8080/v1/set \ + -H 'Content-Type: application/json' \ + -d '{"key":"user:1","value":"ok","ttl_ms":60000}' +curl -sS 'http://127.0.0.1:8081/v1/get?key=user:1' +./scripts/failure-demo.sh # A: kill replica, still hit; B: partial ACK then miss +./scripts/rebalance-demo.sh # join/leave remaps ~1/N primaries +make stop ``` +### Docker Compose (optional) + +```bash +docker compose up --build +``` + +Host ports: `node-a :8080`, `node-b :8081`, `node-c :8082`. Compose uses `network_mode: host` so the three processes can replicate over loopback (a Docker bridge on this nested CI VM drops 100% of veth-to-veth packets). On Docker Desktop (Mac/Windows), host networking is ignored — use `make cluster`. + Single node (self is implied if `-peers` is empty): ```bash @@ -60,7 +77,7 @@ go run ./cmd/ringcache -id node-a -listen :8080 JSON, no auth. `ttl_ms: 0` or omitted means **no expiry**. ```bash -# SET +# SET — HTTP 200 means acked≥1, not “all replicas applied” curl -sS -X PUT http://127.0.0.1:8080/v1/set \ -H 'Content-Type: application/json' \ -d '{"key":"user:1","value":"ok","ttl_ms":60000}' @@ -83,9 +100,10 @@ Aliases without the `/v1` prefix (`/get`, `/set`, `/delete`) do the same thing. | Method | Path | Notes | | --- | --- | --- | | `GET` | `/v1/get?key=` | 200 hit, 404 miss, 503 no replica reachable | -| `PUT`/`POST` | `/v1/set` | body `{key,value,ttl_ms}` — 200 if `acked≥1`, else 503 | +| `PUT`/`POST` | `/v1/set` | body `{key,value,ttl_ms}` — **200 if `acked≥1`**, else 503 | | `DELETE` | `/v1/delete?key=` | best-effort delete on all owners | | `GET` | `/internal/kv` | local store only (replication path) | +| `GET`/`POST`/`DELETE` | `/admin/members` | local ring membership only — **not gossip** | SET response (honest about partial writes): @@ -107,29 +125,34 @@ hash32(id + "#" + i) // FNV-1a 32-bit, i in [0, V) A key walks **clockwise** from `hash32(key)` and collects the first `R` **distinct** physical nodes. Those are the owners. -Why virtual nodes: one point per node makes arcs (and therefore key load) wildly uneven. `V=150` chops the circle so each node owns many small arcs. Adding a node remaps about `1/N` of primaries, not a whole neighbor's slice. Tests in `internal/ring` check determinism, uniqueness, spread, and join remap fraction. +Why virtual nodes: one point per node makes arcs (and therefore key load) wildly uneven. `V=150` chops the circle so each node owns many small arcs. Adding a node remaps about `1/N` of primaries, not a whole neighbor's slice. Tests in `internal/ring` check determinism, uniqueness, spread, join remap, and leave remap. Live join/leave: `./scripts/rebalance-demo.sh`. --- -## Replication and consistency (read this) +## Consistency model -**Default `R=2`.** SET is **synchronous fan-out** to all `R` owners. The coordinator returns as soon as those RPCs finish (or time out, default 200ms). Success means `acked ≥ 1`, **not** a quorum of `R`. +Read this before citing the demo as “HA cache.” -| What we are | What we are not | +| What a 200 SET means | What it does not mean | | --- | --- | -| Last-writer-wins on coordinator `written_at` (unix-nano) | Linearizable / sequential | -| Best-effort sync replication | Quorum R+W>N | -| Read: first live owner with a non-expired value | Read-your-writes from an arbitrary node | -| Opportunistic read-repair of other owners | Anti-entropy / hinted handoff | +| ≥1 of the R owners applied the write | Linearizable or sequential consistency | +| The coordinator finished (or timed out) a **sync fan-out to all R owners** | Quorum `R+W>N`, or “the cluster has the key” | +| Last-writer-wins on coordinator `written_at` (unix-nano) | Compare-and-swap / transactions | +| GET returns the first live owner with a non-expired value | Read-your-writes from an arbitrary node | +| Opportunistic read-repair of other owners | Anti-entropy, hinted handoff, or key migration on join | | 2s fail-open breaker after a peer error | Failure detection / membership | +**Defaults:** `R=2`, replica RPC timeout 200ms. The coordinator **always waits for every owner RPC** (or timeout). HTTP 200 is `acked ≥ 1`, **not** `acked == R`. + +**There is no persistence.** Restart = empty. No WAL, snapshot, or disk. + **Failure modes that actually happen:** -1. SET acks 1 of 2. Kill that replica → GET misses even though the cluster is “up”. +1. SET acks 1 of 2. Kill that replica → GET misses even though another node is “up.” 2. Two coordinators SET the same key concurrently → replicas can diverge until the next write or read-repair. 3. TTL is computed on the receiving node (`now + ttl_ms`). Clock skew moves expiry. -4. Membership is **static** (compose/env). A dead node stays on the ring; peers skip it briefly, then retry and eat the timeout. -5. Restart = empty. There is no WAL, snapshot, or disk. +4. Membership is **per process**. A dead node stays on the ring until someone `DELETE /admin/members`. Peers skip it briefly, then retry and eat the timeout. +5. Join/leave remaps ~`1/N` primaries. The newly responsible node starts **empty** — we do not move values. Caches hide this with TTL. Do not put a source of truth here. @@ -147,16 +170,27 @@ Caches hide this with TTL. Do not put a source of truth here. ## Failure demo ```bash -# local 3-process cluster -./scripts/failure-demo.sh - -# or, with compose already up: -./scripts/failure-demo.sh --compose +./scripts/failure-demo.sh # local 3-process cluster +./scripts/failure-demo.sh --compose # compose already up, or starts it ``` -What it does: pick a key whose owners include **node-b**, SET it via node-a, GET all nodes, **kill node-b**, GET node-a and node-c. +The key picker reads **`owners[]` only**. Grepping the whole `/ring` body matches `"node-b"` in the cluster node list and is wrong. + +**Scenario A — full ACK, kill one replica (expect HIT).** Pick a key whose `owners[]` include node-b. SET via node-a. Kill node-b. GET on :8080 and :8082 still hits the surviving owner. `:8081` stops accepting connections. This is **not** “the cluster heals”; the dead id remains on the ring. + +**Scenario B — partial ACK, then kill the only copy (expect MISS).** Kill node-b first. SET a key owned by `{node-a, node-b}` → `acked=1`, `failed=["node-b"]`, HTTP 200. Kill node-a. GET via node-c misses (both owners down). Success is `acked≥1`, not durability. -**Real result on this tree (2026-09-13):** script picked `demo:failure:1` with owners `[node-b, node-a]`. SET acked 2/2. Before the kill, `:8081` served from `node-b`. After `kill` of node-b, GET on :8080 and :8082 still returned `"found":true` with `served_by: node-a`. `:8081` stopped accepting connections. This is **not** “the cluster heals”; the dead id remains on the ring. If SET had only acked node-b, the same kill would miss. +**Real result on this tree (2026-09-13):** A picked `demo:failure-a:1` owners `[node-b, node-a]`, SET acked 2/2; after kill, :8080 and :8082 served `node-a`. B picked `demo:failure-b:1` owners `[node-b, node-a]`; SET while node-b down acked 1 (`failed=["node-b"]`); after killing node-a, GET :8082 returned **503** `all replicas unreachable`. + +--- + +## Join / leave rebalance + +```bash +./scripts/rebalance-demo.sh +``` + +Starts **node-d** on `:8083` and `POST /admin/members` on node-a/b/c (membership is local, not gossip). Samples 2000 primaries via `/ring?key=` before and after; expects ~`1/4` remapped (`N=3→4`). Leave restores the 3-node placement. Newly responsible nodes do **not** receive old values. --- @@ -172,7 +206,7 @@ Harness: `cmd/bench` — real HTTP PUT `/v1/set` and GET `/v1/get` against all t | | | | --- | --- | -| Date (UTC) | 2026-09-13T09:29:48Z | +| Date (UTC) | 2026-09-13T10:55:13Z | | Hardware | Linux 6.12.94+ x86_64, 4 vCPU, Intel Xeon, ~16 GiB RAM (Cursor Cloud Agent VM) | | Go | go1.22.2 linux/amd64 | | Cluster | 3 processes on loopback (`127.0.0.1:8080-8082`), `R=2`, `V=150`, cap 10000 | @@ -180,12 +214,10 @@ Harness: `cmd/bench` — real HTTP PUT `/v1/set` and GET `/v1/get` against all t | Phase | ok | errors | wall | ops/s | p50 | p95 | p99 | max | mean | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| SET | 4000 | 0 | 312ms | **12820** | 2.28ms | 4.76ms | 6.56ms | 10.18ms | 2.48ms | -| GET | 4000 | 0 | 129ms | **31027** | 0.82ms | 2.68ms | 3.91ms | 5.84ms | 1.02ms | - -These are **this VM, this commit, loopback**. Docker NAT, a laptop, or a noisy neighbor will differ. Do not cite them as product SLOs. +| SET | 4000 | 0 | 426ms | **9395** | 3.08ms | 6.72ms | 8.90ms | 18.17ms | 3.40ms | +| GET | 4000 | 0 | 378ms | **10583** | 2.33ms | 7.94ms | 11.31ms | 19.45ms | 3.01ms | -`docker compose up --build` was also run on this VM (host network). SET acked 2/2; after `compose stop node-b` on a key owned by node-b, GET on :8080 and :8082 still hit `node-a`. +These are **this VM, this commit, loopback**. A quieter slot on the same VM earlier today printed ~12.8k SET / ~31k GET ops/s — same harness, same flags. Shared-CPU noise is real; do not cite either row as a product SLO. Re-run `./scripts/bench.sh` and replace the table for your machine. --- @@ -193,10 +225,15 @@ These are **this VM, this commit, loopback**. Docker NAT, a laptop, or a noisy n ```bash go test ./... -race -count=1 +# or +make ci ``` -- `internal/ring` — deterministic owners, unique replicas, vnode count, key spread, join remaps ~1/N +- `internal/ring` — deterministic owners, unique replicas, vnode count, key spread, join/leave remap ~1/N - `internal/store` — LRU order, TTL expiry, LWW stale-write ignore, capacity-1 eviction +- `internal/node` — 3-node `httptest` SET/GET/DELETE, partial ACK when an owner is down, join/leave + +CI (GitHub Actions) runs `gofmt`, `go vet`, `-race` tests, then `failure-demo.sh` and `rebalance-demo.sh` against `make cluster`. --- @@ -230,5 +267,5 @@ go test ./... -race -count=1 > **DRAFT** — do not paste onto a résumé until you have run the demo yourself and can defend every number and failure mode. -- **DRAFT.** Built a 3-node Go in-memory cache with consistent hashing (`V` virtual nodes, FNV-1a), per-node LRU+TTL, and `R=2` synchronous fan-out; coordinators report `acked`/`failed` instead of pretending writes are atomic. -- **DRAFT.** Measured real HTTP SET/GET throughput and p99 on a 3-node loopback cluster and documented single-node-kill behavior: reads continue only when at least one surviving replica acked the write (last-writer-wins, no quorum). +- **DRAFT.** Built a 3-node Go in-memory cache with consistent hashing (`V` virtual nodes, FNV-1a), per-node LRU+TTL, and `R=2` synchronous fan-out; coordinators report `acked`/`failed` and treat success as `acked≥1` (not linearizable, not a quorum, no persistence). +- **DRAFT.** Measured real HTTP SET/GET throughput and p99 on a 3-node loopback cluster and documented single-replica-kill vs partial-ACK-then-kill: reads continue only when a surviving replica acked the write (last-writer-wins). diff --git a/cmd/ringcache/main.go b/cmd/ringcache/main.go index 8f75c02..09d9ddc 100644 --- a/cmd/ringcache/main.go +++ b/cmd/ringcache/main.go @@ -38,6 +38,9 @@ func main() { Addr: cfg.Listen, Handler: n.Handler(), ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 60 * time.Second, } errCh := make(chan error, 1) @@ -138,7 +141,8 @@ func runHealthcheck(listen string) error { if !strings.HasPrefix(port, ":") { port = ":8080" } - resp, err := http.Get("http://127.0.0.1" + port + "/health") + client := &http.Client{Timeout: 2 * time.Second} + resp, err := client.Get("http://127.0.0.1" + port + "/health") if err != nil { return err } diff --git a/internal/node/node.go b/internal/node/node.go index 4082c21..3cd9756 100644 --- a/internal/node/node.go +++ b/internal/node/node.go @@ -48,6 +48,7 @@ type Node struct { mu sync.Mutex downUntil map[string]time.Time + peers map[string]string } // New builds a node. The store janitor starts immediately. @@ -79,12 +80,14 @@ func New(cfg Config) (*Node, error) { if cfg.Replicas > len(cfg.Peers) { cfg.Replicas = len(cfg.Peers) } - r := ring.New(cfg.VNodes) + peers := make(map[string]string, len(cfg.Peers)) ids := make([]string, 0, len(cfg.Peers)) - for id := range cfg.Peers { + for id, u := range cfg.Peers { + peers[id] = u ids = append(ids, id) } sort.Strings(ids) + r := ring.New(cfg.VNodes) for _, id := range ids { r.Add(id) } @@ -95,6 +98,7 @@ func New(cfg Config) (*Node, error) { client: &http.Client{Timeout: cfg.ReplicaTimeout}, log: log.New(log.Writer(), "["+cfg.ID+"] ", log.LstdFlags|log.Lmicroseconds), downUntil: make(map[string]time.Time), + peers: peers, }, nil } @@ -116,6 +120,7 @@ func (n *Node) Handler() http.Handler { mux.HandleFunc("/v1/delete", n.handleDelete) mux.HandleFunc("/delete", n.handleDelete) mux.HandleFunc("/internal/kv", n.handleInternalKV) + mux.HandleFunc("/admin/members", n.handleMembers) return mux } @@ -193,7 +198,7 @@ func (n *Node) handleRing(w http.ResponseWriter, r *http.Request) { resp := ringResp{ ID: n.cfg.ID, Nodes: n.ring.Nodes(), - Peers: n.cfg.Peers, + Peers: n.snapshotPeers(), VNodes: n.cfg.VNodes, RingLen: n.ring.Len(), Replicas: n.cfg.Replicas, @@ -351,7 +356,11 @@ func (n *Node) readOwner(ctx context.Context, id, key string) (store.Entry, bool if n.isDown(id) { return store.Entry{}, false, fmt.Errorf("%s marked down", id) } - url := strings.TrimRight(n.cfg.Peers[id], "/") + "/internal/kv?key=" + urlQuery(key) + base := n.peerURL(id) + if base == "" { + return store.Entry{}, false, fmt.Errorf("unknown peer %s", id) + } + url := strings.TrimRight(base, "/") + "/internal/kv?key=" + urlQuery(key) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return store.Entry{}, false, err @@ -424,7 +433,10 @@ func (n *Node) writeOwner(ctx context.Context, method, id string, body kvBody) e return fmt.Errorf("bad method %s", method) } } - base := strings.TrimRight(n.cfg.Peers[id], "/") + base := strings.TrimRight(n.peerURL(id), "/") + if base == "" { + return fmt.Errorf("unknown peer %s", id) + } var req *http.Request var err error if method == http.MethodDelete { @@ -464,14 +476,16 @@ func (n *Node) readRepair(key string, ent store.Entry, owners []string, servedBy ttlMs = rem } body := kvBody{Key: key, Value: ent.Value, TTLMs: ttlMs, WrittenAt: ent.WrittenAt} - ctx, cancel := context.WithTimeout(context.Background(), n.cfg.ReplicaTimeout) - defer cancel() for _, id := range owners { if id == servedBy { continue } id := id go func() { + // Per-goroutine timeout: the parent must not cancel when handleGet + // returns (the previous shared ctx was canceled immediately). + ctx, cancel := context.WithTimeout(context.Background(), n.cfg.ReplicaTimeout) + defer cancel() if err := n.writeOwner(ctx, http.MethodPut, id, body); err != nil { n.log.Printf("read-repair %q → %s: %v", key, id, err) } @@ -479,6 +493,101 @@ func (n *Node) readRepair(key string, ent store.Entry, owners []string, servedBy } } +func (n *Node) handleMembers(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + writeJSON(w, http.StatusOK, map[string]any{ + "id": n.cfg.ID, + "nodes": n.ring.Nodes(), + "peers": n.snapshotPeers(), + }) + case http.MethodPut, http.MethodPost: + var body struct { + ID string `json:"id"` + URL string `json:"url"` + } + if err := readJSON(r, &body); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if err := n.join(body.ID, body.URL); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "ok": true, + "id": n.cfg.ID, + "nodes": n.ring.Nodes(), + "peers": n.snapshotPeers(), + }) + case http.MethodDelete: + id := r.URL.Query().Get("id") + if err := n.leave(id); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "ok": true, + "id": n.cfg.ID, + "nodes": n.ring.Nodes(), + "peers": n.snapshotPeers(), + }) + default: + http.Error(w, "method", http.StatusMethodNotAllowed) + } +} + +// join adds a physical node to this process's ring and peer map. +// Membership is not gossip: every live node must be told (see scripts/rebalance-demo.sh). +func (n *Node) join(id, rawURL string) error { + id = strings.TrimSpace(id) + rawURL = strings.TrimSpace(rawURL) + if id == "" || rawURL == "" { + return errors.New("id and url required") + } + if _, err := url.ParseRequestURI(rawURL); err != nil { + return fmt.Errorf("url: %w", err) + } + n.mu.Lock() + n.peers[id] = rawURL + n.mu.Unlock() + n.ring.Add(id) + return nil +} + +// leave removes a physical node from this process's ring. Self cannot leave. +func (n *Node) leave(id string) error { + id = strings.TrimSpace(id) + if id == "" { + return errors.New("id required") + } + if id == n.cfg.ID { + return errors.New("cannot remove self") + } + n.mu.Lock() + delete(n.peers, id) + delete(n.downUntil, id) + n.mu.Unlock() + n.ring.Remove(id) + return nil +} + +func (n *Node) snapshotPeers() map[string]string { + n.mu.Lock() + defer n.mu.Unlock() + out := make(map[string]string, len(n.peers)) + for k, v := range n.peers { + out[k] = v + } + return out +} + +func (n *Node) peerURL(id string) string { + n.mu.Lock() + defer n.mu.Unlock() + return n.peers[id] +} + func (n *Node) markDown(id string) { if id == n.cfg.ID { return diff --git a/internal/node/node_test.go b/internal/node/node_test.go index 7620fec..81b5614 100644 --- a/internal/node/node_test.go +++ b/internal/node/node_test.go @@ -3,6 +3,7 @@ package node import ( "bytes" "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" @@ -12,10 +13,14 @@ import ( func startTestNode(t *testing.T, id string, peers map[string]string) (*httptest.Server, *Node) { t.Helper() + copied := make(map[string]string, len(peers)) + for k, v := range peers { + copied[k] = v + } n, err := New(Config{ ID: id, Listen: "test", - Peers: peers, + Peers: copied, Replicas: 2, Capacity: 64, VNodes: 32, @@ -32,20 +37,48 @@ func startTestNode(t *testing.T, id string, peers map[string]string) (*httptest. return srv, n } -func TestReplicatedSetGetAcrossCoordinators(t *testing.T) { - peers := map[string]string{ - "a": "placeholder", - "b": "placeholder", - "c": "placeholder", +func startCluster(t *testing.T) (map[string]string, map[string]*Node) { + t.Helper() + placeholder := map[string]string{ + "a": "http://127.0.0.1:1", + "b": "http://127.0.0.1:2", + "c": "http://127.0.0.1:3", } - sa, _ := startTestNode(t, "a", peers) - sb, _ := startTestNode(t, "b", peers) - sc, _ := startTestNode(t, "c", peers) - // All nodes share this map (Config.Peers is not cloned). - peers["a"], peers["b"], peers["c"] = sa.URL, sb.URL, sc.URL + sa, na := startTestNode(t, "a", placeholder) + sb, nb := startTestNode(t, "b", placeholder) + sc, nc := startTestNode(t, "c", placeholder) + urls := map[string]string{"a": sa.URL, "b": sb.URL, "c": sc.URL} + nodes := map[string]*Node{"a": na, "b": nb, "c": nc} + for _, n := range nodes { + for id, u := range urls { + if err := n.join(id, u); err != nil { + t.Fatal(err) + } + } + } + return urls, nodes +} + +func pickKeyWithOwner(t *testing.T, n *Node, want string) (string, []string) { + t.Helper() + for i := 0; i < 4000; i++ { + k := fmt.Sprintf("pick:%d", i) + owners := n.ring.Owners(k, n.cfg.Replicas) + for _, o := range owners { + if o == want { + return k, owners + } + } + } + t.Fatalf("no key owned by %s", want) + return "", nil +} + +func TestReplicatedSetGetAcrossCoordinators(t *testing.T) { + urls, _ := startCluster(t) body, _ := json.Marshal(map[string]any{"key": "k1", "value": "v1", "ttl_ms": 5000}) - res, err := http.Post(sa.URL+"/v1/set", "application/json", bytes.NewReader(body)) + res, err := http.Post(urls["a"]+"/v1/set", "application/json", bytes.NewReader(body)) if err != nil { t.Fatal(err) } @@ -62,8 +95,7 @@ func TestReplicatedSetGetAcrossCoordinators(t *testing.T) { t.Fatalf("acked=%d failed=%v", wr.Acked, wr.Failed) } - // GET from a node that may not be the coordinator. - res, err = http.Get(sc.URL + "/v1/get?key=k1") + res, err = http.Get(urls["c"] + "/v1/get?key=k1") if err != nil { t.Fatal(err) } @@ -82,14 +114,10 @@ func TestReplicatedSetGetAcrossCoordinators(t *testing.T) { } func TestDeleteRemovesReplicas(t *testing.T) { - peers := map[string]string{"a": "", "b": "", "c": ""} - sa, _ := startTestNode(t, "a", peers) - sb, _ := startTestNode(t, "b", peers) - sc, _ := startTestNode(t, "c", peers) - peers["a"], peers["b"], peers["c"] = sa.URL, sb.URL, sc.URL + urls, _ := startCluster(t) body, _ := json.Marshal(map[string]any{"key": "gone", "value": "x", "ttl_ms": 0}) - res, err := http.Post(sb.URL+"/v1/set", "application/json", bytes.NewReader(body)) + res, err := http.Post(urls["b"]+"/v1/set", "application/json", bytes.NewReader(body)) if err != nil { t.Fatal(err) } @@ -99,7 +127,7 @@ func TestDeleteRemovesReplicas(t *testing.T) { t.Fatalf("set %d", res.StatusCode) } - req, _ := http.NewRequest(http.MethodDelete, sa.URL+"/v1/delete?key=gone", nil) + req, _ := http.NewRequest(http.MethodDelete, urls["a"]+"/v1/delete?key=gone", nil) res, err = http.DefaultClient.Do(req) if err != nil { t.Fatal(err) @@ -107,7 +135,7 @@ func TestDeleteRemovesReplicas(t *testing.T) { io.Copy(io.Discard, res.Body) res.Body.Close() - res, err = http.Get(sc.URL + "/v1/get?key=gone") + res, err = http.Get(urls["c"] + "/v1/get?key=gone") if err != nil { t.Fatal(err) } @@ -122,3 +150,147 @@ func TestDeleteRemovesReplicas(t *testing.T) { t.Fatalf("deleted key still found: %s", raw) } } + +func TestSetPartialAckWhenOwnerDown(t *testing.T) { + placeholder := map[string]string{ + "a": "http://127.0.0.1:1", + "b": "http://127.0.0.1:2", + "c": "http://127.0.0.1:3", + } + sa, na := startTestNode(t, "a", placeholder) + sb, nb := startTestNode(t, "b", placeholder) + sc, nc := startTestNode(t, "c", placeholder) + for _, n := range []*Node{na, nb, nc} { + _ = n.join("a", sa.URL) + _ = n.join("b", sb.URL) + _ = n.join("c", sc.URL) + } + sb.Close() + + key, owners := pickKeyWithOwner(t, na, "b") + body, _ := json.Marshal(map[string]any{"key": key, "value": "partial", "ttl_ms": 0}) + res, err := http.Post(sa.URL+"/v1/set", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + raw, _ := io.ReadAll(res.Body) + res.Body.Close() + if res.StatusCode != 200 { + t.Fatalf("set status %d %s (owners=%v)", res.StatusCode, raw, owners) + } + var wr writeResp + if err := json.Unmarshal(raw, &wr); err != nil { + t.Fatal(err) + } + if wr.Acked < 1 { + t.Fatalf("acked=%d (success is acked>=1) failed=%v", wr.Acked, wr.Failed) + } + sawB := false + for _, id := range wr.Failed { + if id == "b" { + sawB = true + } + } + if !sawB { + t.Fatalf("expected owner b in failed, got acked=%d failed=%v owners=%v", wr.Acked, wr.Failed, owners) + } +} + +func TestJoinLeaveUpdatesOwners(t *testing.T) { + n, err := New(Config{ + ID: "a", + Listen: "test", + Peers: map[string]string{"a": "http://127.0.0.1:1", "b": "http://127.0.0.1:2"}, + Replicas: 2, + Capacity: 8, + VNodes: 32, + ReplicaTimeout: time.Second, + }) + if err != nil { + t.Fatal(err) + } + defer n.Close() + + const samples = 400 + before := make([]string, samples) + for i := 0; i < samples; i++ { + before[i] = n.ring.Primary(fmt.Sprintf("m:%d", i)) + } + if err := n.join("c", "http://127.0.0.1:3"); err != nil { + t.Fatal(err) + } + changed := 0 + for i := 0; i < samples; i++ { + if n.ring.Primary(fmt.Sprintf("m:%d", i)) != before[i] { + changed++ + } + } + if changed == 0 { + t.Fatal("join did not remap any primaries") + } + if err := n.leave("c"); err != nil { + t.Fatal(err) + } + if err := n.leave("a"); err == nil { + t.Fatal("self leave should fail") + } + reverted := 0 + for i := 0; i < samples; i++ { + if n.ring.Primary(fmt.Sprintf("m:%d", i)) == before[i] { + reverted++ + } + } + if reverted != samples { + t.Fatalf("leave did not restore primaries: %d/%d", reverted, samples) + } +} + +func TestAdminMembersHTTP(t *testing.T) { + n, err := New(Config{ + ID: "a", + Listen: "test", + Peers: map[string]string{"a": "http://127.0.0.1:8080"}, + Replicas: 1, + Capacity: 8, + VNodes: 8, + ReplicaTimeout: time.Second, + }) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(n.Handler()) + t.Cleanup(func() { srv.Close(); n.Close() }) + + body, _ := json.Marshal(map[string]string{"id": "b", "url": "http://127.0.0.1:8081"}) + res, err := http.Post(srv.URL+"/admin/members", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + raw, _ := io.ReadAll(res.Body) + res.Body.Close() + if res.StatusCode != 200 { + t.Fatalf("join %d %s", res.StatusCode, raw) + } + res, err = http.Get(srv.URL + "/admin/members") + if err != nil { + t.Fatal(err) + } + raw, _ = io.ReadAll(res.Body) + res.Body.Close() + if !bytes.Contains(raw, []byte(`"b"`)) { + t.Fatalf("members after join: %s", raw) + } + req, _ := http.NewRequest(http.MethodDelete, srv.URL+"/admin/members?id=b", nil) + res, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + io.Copy(io.Discard, res.Body) + res.Body.Close() + if res.StatusCode != 200 { + t.Fatalf("leave %d", res.StatusCode) + } + if n.ring.Size() != 1 { + t.Fatalf("size=%d", n.ring.Size()) + } +} diff --git a/internal/ring/ring_test.go b/internal/ring/ring_test.go index 7d98726..537a470 100644 --- a/internal/ring/ring_test.go +++ b/internal/ring/ring_test.go @@ -126,6 +126,27 @@ func TestJoinRemapsAboutOneNth(t *testing.T) { } } +func TestLeaveRemapsAboutOneNth(t *testing.T) { + r := testRing(t, 150, "a", "b", "c", "d") + const n = 5000 + before := make([]string, n) + for i := 0; i < n; i++ { + before[i] = r.Primary("k:" + strconv.Itoa(i)) + } + r.Remove("d") + changed := 0 + for i := 0; i < n; i++ { + if r.Primary("k:"+strconv.Itoa(i)) != before[i] { + changed++ + } + } + frac := float64(changed) / float64(n) + // 4 → 3: keys whose primary was d (~1/4) should move. + if frac < 0.12 || frac > 0.40 { + t.Fatalf("leave remap fraction %.3f outside [0.12, 0.40] (changed=%d)", frac, changed) + } +} + func TestReplicaSetStableAfterUnrelatedJoin(t *testing.T) { r := testRing(t, 150, "a", "b") owners := r.Owners("sticky-key", 2) diff --git a/internal/store/store.go b/internal/store/store.go index bb2a056..859ac12 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -44,19 +44,20 @@ type Stats struct { // Store is a mutex-protected LRU+TTL map. type Store struct { - mu sync.Mutex - capacity int - ll *list.List // front = most recently used - items map[string]*list.Element - hits uint64 - misses uint64 - sets uint64 - deletes uint64 - evicts uint64 - expired uint64 - skipped uint64 - stop chan struct{} - stopped chan struct{} + mu sync.Mutex + capacity int + ll *list.List // front = most recently used + items map[string]*list.Element + hits uint64 + misses uint64 + sets uint64 + deletes uint64 + evicts uint64 + expired uint64 + skipped uint64 + stop chan struct{} + stopped chan struct{} + closeOnce sync.Once } // New creates a store. capacity < 1 means 1. @@ -75,15 +76,12 @@ func New(capacity int) *Store { return s } -// Close stops the janitor. Safe to call once. +// Close stops the janitor. Safe to call more than once. func (s *Store) Close() { - select { - case <-s.stop: - return - default: + s.closeOnce.Do(func() { close(s.stop) - } - <-s.stopped + <-s.stopped + }) } // Get returns (entry, true) on a live hit. Expired keys are deleted and miss. diff --git a/internal/store/store_test.go b/internal/store/store_test.go index ab868c6..a7070d1 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -155,6 +155,12 @@ func TestHitsAndMisses(t *testing.T) { } } +func TestCloseIdempotent(t *testing.T) { + s := New(2) + s.Close() + s.Close() +} + func TestConcurrentSetGet(t *testing.T) { s := New(256) defer s.Close() diff --git a/scripts/failure-demo.sh b/scripts/failure-demo.sh index f012506..6389a55 100755 --- a/scripts/failure-demo.sh +++ b/scripts/failure-demo.sh @@ -1,27 +1,17 @@ #!/usr/bin/env bash -# Kill one node and show that a replicated key is still readable. +# Failure modes, not a "the cluster heals" story. +# +# A) SET acked by both owners, then kill one replica → GET still hits +# the surviving owner. The dead id stays on the ring. +# B) SET while one owner is already down (acked=1), then kill the +# surviving replica → GET misses. Success is acked≥1, not durability. # -# Real behavior (not a guarantee of "HA"): -# - R=2, N=3: every key has two owners. Killing ONE node leaves at least -# one replica IF the original SET was acked by both owners. -# - If SET only acked the node you then kill, GET returns 404/503. -# - Membership is static: the dead node stays on the ring; peers skip it -# for ~2s via a fail-open breaker, then retry and time out again. set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=lib.sh +source "$ROOT/scripts/lib.sh" cd "$ROOT" -docker_cmd() { - if docker info >/dev/null 2>&1; then - docker "$@" - elif sudo docker info >/dev/null 2>&1; then - sudo docker "$@" - else - echo "docker is not usable" >&2 - return 1 - fi -} - USE_COMPOSE=0 if [[ "${1:-}" == "--compose" ]]; then USE_COMPOSE=1 @@ -29,80 +19,184 @@ elif command -v docker >/dev/null 2>&1 && docker_cmd compose ps --status running USE_COMPOSE=1 fi -if [[ "$USE_COMPOSE" -eq 0 ]]; then - if ! curl -sf http://127.0.0.1:8080/health >/dev/null 2>&1; then - ./scripts/run-cluster.sh +STARTED_CLUSTER=0 +kill_node() { + local id="$1" + if [[ "$USE_COMPOSE" -eq 1 ]]; then + docker_cmd compose stop "$id" >/dev/null + else + stop_local_node "$id" fi -fi +} -# Pick a key whose owner set includes node-b so killing that process is a -# real replica loss, not a no-op on a key that never lived there. -KEY="" -for i in $(seq 1 400); do - cand="demo:failure:$i" - owners="$(curl -sf "http://127.0.0.1:8080/ring?key=$cand" || true)" - if echo "$owners" | grep -q '"node-b"'; then - KEY="$cand" - break +start_node() { + local id="$1" + if [[ "$USE_COMPOSE" -eq 1 ]]; then + docker_cmd compose start "$id" >/dev/null + else + ensure_binary "$ROOT" + start_local_node "$ROOT" "$id" "127.0.0.1:$(port_for "$id")" + fi +} + +restore_all() { + start_node node-a || true + start_node node-b || true + start_node node-c || true + wait_health "8080 8081 8082" 80 || true +} + +if [[ "$USE_COMPOSE" -eq 1 ]]; then + if ! curl -sf --max-time 1 http://127.0.0.1:8080/health >/dev/null 2>&1; then + echo "starting compose cluster..." + docker_cmd compose up -d --build + STARTED_CLUSTER=1 + fi +else + if ! curl -sf --max-time 1 http://127.0.0.1:8080/health >/dev/null 2>&1; then + ./scripts/run-cluster.sh + STARTED_CLUSTER=1 fi -done -if [[ -z "$KEY" ]]; then - echo "could not find a key owned by node-b" >&2 - exit 1 fi +wait_health "8080 8081 8082" 80 + +# Always try to bring the 3-node cluster back so a failed scenario is not sticky. +trap restore_all EXIT -echo "=== 1. SET via node-a (key=$KEY, chosen so node-b is an owner) ===" -curl -sS -X PUT http://127.0.0.1:8080/v1/set \ - -H 'Content-Type: application/json' \ - -d "{\"key\":\"$KEY\",\"value\":\"still-here\",\"ttl_ms\":60000}" echo +echo "############################################################" +echo "# Scenario A — full ACK, then kill one replica (expect HIT)" +echo "############################################################" + +picked="$(pick_key_owned_by "node-b" "demo:failure-a" 8080)" || { + echo "could not find a key whose owners[] include node-b" >&2 + exit 1 +} +KEY="${picked%%$'\t'*}" +OWNERS="${picked#*$'\t'}" +echo "key=$KEY owners=[$OWNERS] (owners[] only; not the cluster node list)" -echo "=== 2. owners ===" -curl -sS "http://127.0.0.1:8080/ring?key=$KEY" echo +echo "=== A1. SET via node-a ===" +SETA="$(curl -sS --max-time 3 -X PUT http://127.0.0.1:8080/v1/set \ + -H 'Content-Type: application/json' \ + -d "{\"key\":\"$KEY\",\"value\":\"still-here\",\"ttl_ms\":60000}")" +echo "$SETA" +ACKA="$(printf '%s' "$SETA" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("acked",0))')" +FAILEDA="$(printf '%s' "$SETA" | python3 -c 'import json,sys; print(",".join(json.load(sys.stdin).get("failed") or []))')" +if [[ "$ACKA" -lt 1 ]]; then + echo "SET failed entirely (acked=$ACKA). Success requires acked≥1." >&2 + exit 1 +fi +if [[ ",$FAILEDA," == *",node-a,"* && ",$FAILEDA," == *",node-c,"* ]]; then + echo "SET only landed on node-b; killing node-b would miss (not scenario A)." >&2 + exit 1 +fi -echo "=== 3. GET from all live nodes ===" +echo +echo "=== A2. GET all live nodes (before kill) ===" for p in 8080 8081 8082; do - echo -n ":$p " - curl -sS "http://127.0.0.1:$p/v1/get?key=$KEY" || echo "(down)" + echo -n " :$p " + curl -sS --max-time 2 "http://127.0.0.1:$p/v1/get?key=$KEY" || echo "(down)" echo done -echo "=== 4. kill node-b ===" -if [[ "$USE_COMPOSE" -eq 1 ]]; then - docker_cmd compose stop node-b -else - if [[ -f /tmp/ringcache-b.pid ]]; then - kill "$(cat /tmp/ringcache-b.pid)" || true - rm -f /tmp/ringcache-b.pid - else - echo "no node-b pid; cannot kill" >&2 - exit 1 - fi -fi +echo +echo "=== A3. kill node-b (replica stays on the ring; peers skip it ~2s) ===" +kill_node node-b sleep 0.3 -echo "=== 5. GET after node-b is down (expect a hit if SET acked ≥1 surviving replica) ===" +echo +echo "=== A4. GET node-a and node-c (expect HIT if a surviving owner acked) ===" hit=0 for p in 8080 8082; do - echo -n ":$p " - body="$(curl -sS -w '\n%{http_code}' "http://127.0.0.1:$p/v1/get?key=$KEY" || true)" - echo "$body" - if echo "$body" | grep -q '"found":true'; then + echo -n " :$p " + body="$(curl -sS --max-time 2 -o /tmp/ringcache-get-a.json -w '%{http_code}' "http://127.0.0.1:$p/v1/get?key=$KEY" || true)" + echo "$(cat /tmp/ringcache-get-a.json 2>/dev/null) HTTP $body" + if grep -q '"found":true' /tmp/ringcache-get-a.json 2>/dev/null; then hit=1 fi done -echo "=== 6. GET node-b (should fail to connect) ===" +echo +echo "=== A5. GET node-b (expect connect failure) ===" if curl -sf --max-time 1 "http://127.0.0.1:8081/health"; then - echo "node-b still healthy (unexpected)" -else - echo "node-b unreachable (expected)" + echo "node-b still healthy (unexpected)" >&2 + exit 1 +fi +echo "node-b unreachable (expected)" + +if [[ "$hit" -ne 1 ]]; then + echo "RESULT A: miss after one death (partial write or both owners included the dead node)" >&2 + exit 1 +fi +echo "RESULT A: read succeeded after one replica death — not healing, just R=2" + +echo +echo "=== A6. restore node-b ===" +start_node node-b +wait_health "8080 8081 8082" 80 + +echo +echo "############################################################" +echo "# Scenario B — SET while owner down (acked=1), then kill it" +echo "# (expect MISS). acked≥1 is not durability." +echo "############################################################" + +picked="$(pick_key_owned_by "node-a,node-b" "demo:failure-b" 8080)" || { + echo "could not find a key owned by both node-a and node-b" >&2 + exit 1 +} +KEYB="${picked%%$'\t'*}" +OWNERSB="${picked#*$'\t'}" +echo "key=$KEYB owners=[$OWNERSB]" + +echo +echo "=== B1. kill node-b first, then SET via node-a ===" +kill_node node-b +sleep 0.2 +SETB="$(curl -sS --max-time 3 -X PUT http://127.0.0.1:8080/v1/set \ + -H 'Content-Type: application/json' \ + -d "{\"key\":\"$KEYB\",\"value\":\"only-on-survivor\",\"ttl_ms\":60000}")" +echo "$SETB" +ACKB="$(printf '%s' "$SETB" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("acked",0))')" +FAILEDB="$(printf '%s' "$SETB" | python3 -c 'import json,sys; print(",".join(json.load(sys.stdin).get("failed") or []))')" +if [[ "$ACKB" -lt 1 ]]; then + echo "SET acked=0; scenario B needs a partial success" >&2 + exit 1 fi +if [[ ",$FAILEDB," != *",node-b,"* ]]; then + echo "expected node-b in failed[], got failed=[$FAILEDB]" >&2 + exit 1 +fi +echo "SET acked=$ACKB failed=[$FAILEDB] — HTTP 200 because acked≥1" -if [[ "$hit" -eq 1 ]]; then - echo "RESULT: read succeeded after one node death" - exit 0 +echo +echo "=== B2. kill the surviving owner node-a (the copy we just wrote) ===" +kill_node node-a +sleep 0.3 + +echo +echo "=== B3. GET via node-c (not an owner; both owners are down) ===" +echo -n " :8082 " +body="$(curl -sS --max-time 2 -o /tmp/ringcache-get-b.json -w '%{http_code}' "http://127.0.0.1:8082/v1/get?key=$KEYB" || true)" +echo "$(cat /tmp/ringcache-get-b.json 2>/dev/null) HTTP $body" +if grep -q '"found":true' /tmp/ringcache-get-b.json 2>/dev/null; then + echo "RESULT B: unexpected hit — both owners should be gone" >&2 + exit 1 +fi +echo "RESULT B: miss (404/503). Killing the only replica that acked the write loses the key." + +echo +echo "=== B4. restore node-a and node-b ===" +start_node node-a +start_node node-b +wait_health "8080 8081 8082" 80 + +echo +echo "DONE. Consistency recap:" +echo " success = acked≥1 (not linearizable, not a quorum)" +echo " R=2 sync fan-out, last-writer-wins, no persistence" +if [[ "$STARTED_CLUSTER" -eq 1 && "$USE_COMPOSE" -eq 0 ]]; then + echo " cluster still running; make stop when finished" fi -echo "RESULT: no live replica had the key (partial write or both owners included node-b and the other failed)" >&2 -exit 1 diff --git a/scripts/lib.sh b/scripts/lib.sh new file mode 100755 index 0000000..86f0dc3 --- /dev/null +++ b/scripts/lib.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# Shared helpers for demo scripts. Source this file; do not execute it. +# Requires: curl, python3. + +RINGCACHE_PEERS_DEFAULT="${RINGCACHE_PEERS_DEFAULT:-node-a=http://127.0.0.1:8080,node-b=http://127.0.0.1:8081,node-c=http://127.0.0.1:8082}" + +docker_cmd() { + if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then + docker "$@" + elif command -v docker >/dev/null 2>&1 && sudo docker info >/dev/null 2>&1; then + sudo docker "$@" + else + echo "docker is not usable" >&2 + return 1 + fi +} + +port_for() { + case "$1" in + node-a) echo 8080 ;; + node-b) echo 8081 ;; + node-c) echo 8082 ;; + node-d) echo 8083 ;; + *) echo "unknown node $1" >&2; return 1 ;; + esac +} + +wait_health() { + local ports="${1:-8080 8081 8082}" + local tries="${2:-50}" + local i p + for i in $(seq 1 "$tries"); do + local ok=1 + for p in $ports; do + if ! curl -sf --max-time 1 "http://127.0.0.1:${p}/health" >/dev/null; then + ok=0 + break + fi + done + if [[ "$ok" -eq 1 ]]; then + return 0 + fi + sleep 0.1 + done + echo "nodes not healthy on ports: $ports" >&2 + return 1 +} + +# Print comma-separated owners for key. Parses the owners[] field only — +# grepping the whole /ring body matches the cluster node list and is wrong. +owners_of() { + local key="$1" + local port="${2:-8080}" + curl -sf --max-time 2 "http://127.0.0.1:${port}/ring?key=${key}" | python3 -c ' +import json, sys +d = json.load(sys.stdin) +print(",".join(d.get("owners") or [])) +' +} + +# Find a key whose owner set includes every id in WANT (comma-separated). +# Prints: KEYOWNERS +pick_key_owned_by() { + local want="$1" + local prefix="${2:-demo:key}" + local port="${3:-8080}" + python3 - "$want" "$prefix" "$port" <<'PY' +import json, sys, urllib.request +want = [x for x in sys.argv[1].split(",") if x] +prefix = sys.argv[2] +port = sys.argv[3] +need = set(want) +for i in range(1, 801): + key = f"{prefix}:{i}" + url = f"http://127.0.0.1:{port}/ring?key={key}" + try: + with urllib.request.urlopen(url, timeout=2) as r: + d = json.load(r) + except Exception: + continue + owners = d.get("owners") or [] + if need.issubset(owners): + print(f"{key}\t{','.join(owners)}") + sys.exit(0) +sys.exit(1) +PY +} + +json_get() { + python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get(sys.argv[1], ""))' "$1" +} + +ensure_binary() { + local root="$1" + mkdir -p "$root/bin" + if [[ ! -x "$root/bin/ringcache" ]]; then + (cd "$root" && go build -o bin/ringcache ./cmd/ringcache) + fi +} + +start_local_node() { + local root="$1" id="$2" listen="$3" + local letter="${id##*-}" + local pidf="/tmp/ringcache-${letter}.pid" + local logf="/tmp/ringcache-${letter}.log" + if [[ -f "$pidf" ]] && kill -0 "$(cat "$pidf")" 2>/dev/null; then + return 0 + fi + local peers="${4:-$RINGCACHE_PEERS_DEFAULT}" + "$root/bin/ringcache" \ + -id "$id" \ + -listen "$listen" \ + -replicas 2 \ + -capacity 10000 \ + -vnodes 150 \ + -replica-timeout-ms 200 \ + -peers "$peers" \ + >"$logf" 2>&1 & + echo $! >"$pidf" +} + +stop_local_node() { + local id="$1" + local letter="${id##*-}" + local pidf="/tmp/ringcache-${letter}.pid" + if [[ -f "$pidf" ]]; then + local pid + pid="$(cat "$pidf")" + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + fi + rm -f "$pidf" + fi +} + +members_join() { + local port="$1" id="$2" url="$3" + curl -sf --max-time 2 -X POST "http://127.0.0.1:${port}/admin/members" \ + -H 'Content-Type: application/json' \ + -d "{\"id\":\"${id}\",\"url\":\"${url}\"}" +} + +members_leave() { + local port="$1" id="$2" + curl -sf --max-time 2 -X DELETE "http://127.0.0.1:${port}/admin/members?id=${id}" +} diff --git a/scripts/quickstart.sh b/scripts/quickstart.sh new file mode 100755 index 0000000..e03227e --- /dev/null +++ b/scripts/quickstart.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# 60-second path: build, start 3 nodes, SET, GET. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +echo "==> 3-node cluster on 127.0.0.1:8080-8082 (R=2, V=150)" +./scripts/run-cluster.sh + +echo +echo "==> SET user:1 via node-a (200 means acked≥1, not a quorum)" +curl -sS --max-time 3 -X PUT http://127.0.0.1:8080/v1/set \ + -H 'Content-Type: application/json' \ + -d '{"key":"user:1","value":"ok","ttl_ms":60000}' +echo + +echo +echo "==> GET via node-b (any node coordinates; first live owner wins)" +curl -sS --max-time 3 'http://127.0.0.1:8081/v1/get?key=user:1' +echo + +echo +echo "==> owners for user:1" +curl -sS --max-time 3 'http://127.0.0.1:8080/ring?key=user:1' +echo + +echo +echo "Up. Next (optional):" +echo " ./scripts/failure-demo.sh # kill a replica; then partial-ack miss" +echo " ./scripts/rebalance-demo.sh # join/leave ~1/N remap" +echo " make stop" diff --git a/scripts/rebalance-demo.sh b/scripts/rebalance-demo.sh new file mode 100755 index 0000000..d0aacd4 --- /dev/null +++ b/scripts/rebalance-demo.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Join/leave remapping on a live cluster. +# +# Membership is NOT gossip. This script tells every live node about node-d +# via POST /admin/members (and DELETE on leave). The ring itself remaps +# about 1/N primaries — same math as internal/ring tests. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=lib.sh +source "$ROOT/scripts/lib.sh" +cd "$ROOT" + +SAMPLES="${SAMPLES:-2000}" +PEERS4="${RINGCACHE_PEERS_DEFAULT},node-d=http://127.0.0.1:8083" + +if ! curl -sf --max-time 1 http://127.0.0.1:8080/health >/dev/null 2>&1; then + ./scripts/run-cluster.sh +fi +wait_health "8080 8081 8082" 80 +ensure_binary "$ROOT" + +cleanup() { + members_leave 8080 node-d >/dev/null 2>&1 || true + members_leave 8081 node-d >/dev/null 2>&1 || true + members_leave 8082 node-d >/dev/null 2>&1 || true + stop_local_node node-d +} +trap cleanup EXIT + +echo "=== 1. sample $SAMPLES primaries on the 3-node ring (via node-a /ring?key=) ===" +python3 - "$SAMPLES" /tmp/ringcache-primaries-before.txt <<'PY' +import json, sys, urllib.request +n = int(sys.argv[1]) +out = sys.argv[2] +primaries = [] +for i in range(n): + key = f"rebalance:{i}" + with urllib.request.urlopen(f"http://127.0.0.1:8080/ring?key={key}", timeout=2) as r: + d = json.load(r) + owners = d.get("owners") or [] + primaries.append(owners[0] if owners else "") +open(out, "w").write("\n".join(primaries)) +print(f"wrote {n} primaries") +PY + +BEFORE_NODES="$(curl -sf http://127.0.0.1:8080/admin/members | python3 -c 'import json,sys; print(",".join(json.load(sys.stdin).get("nodes") or []))')" +echo "node-a members: [$BEFORE_NODES]" + +echo +echo "=== 2. start node-d on :8083 and join it on node-a/b/c (no gossip) ===" +start_local_node "$ROOT" node-d "127.0.0.1:8083" "$PEERS4" +wait_health "8083" 50 +for p in 8080 8081 8082; do + members_join "$p" node-d "http://127.0.0.1:8083" >/dev/null +done +AFTER_NODES="$(curl -sf http://127.0.0.1:8080/admin/members | python3 -c 'import json,sys; print(",".join(json.load(sys.stdin).get("nodes") or []))')" +echo "node-a members after join: [$AFTER_NODES]" +if [[ "$AFTER_NODES" != *node-d* ]]; then + echo "join did not stick on node-a" >&2 + exit 1 +fi + +echo +echo "=== 3. resample primaries — consistent hashing should move ~1/4 ===" +python3 - "$SAMPLES" /tmp/ringcache-primaries-before.txt /tmp/ringcache-primaries-after.txt <<'PY' +import json, sys, urllib.request +n = int(sys.argv[1]) +before = open(sys.argv[2]).read().splitlines() +after = [] +for i in range(n): + key = f"rebalance:{i}" + with urllib.request.urlopen(f"http://127.0.0.1:8080/ring?key={key}", timeout=2) as r: + d = json.load(r) + owners = d.get("owners") or [] + after.append(owners[0] if owners else "") +open(sys.argv[3], "w").write("\n".join(after)) +changed = sum(1 for a, b in zip(before, after) if a != b) +frac = changed / n +print(f"primaries remapped: {changed}/{n} = {frac:.3f}") +# 3 → 4 nodes: expect ~1/4. Wide band so a noisy hash still passes. +if frac < 0.10 or frac > 0.45: + raise SystemExit(f"remap fraction {frac:.3f} outside [0.10, 0.45]") +print("within expected band [0.10, 0.45] for N=3 → N=4") +PY + +echo +echo "=== 4. leave node-d on a/b/c, stop process, expect primaries to revert ===" +for p in 8080 8081 8082; do + members_leave "$p" node-d >/dev/null +done +stop_local_node node-d +python3 - "$SAMPLES" /tmp/ringcache-primaries-before.txt <<'PY' +import json, sys, urllib.request +n = int(sys.argv[1]) +before = open(sys.argv[2]).read().splitlines() +reverted = 0 +for i in range(n): + key = f"rebalance:{i}" + with urllib.request.urlopen(f"http://127.0.0.1:8080/ring?key={key}", timeout=2) as r: + d = json.load(r) + owners = d.get("owners") or [] + primary = owners[0] if owners else "" + if i < len(before) and primary == before[i]: + reverted += 1 +frac = reverted / n +print(f"primaries restored: {reverted}/{n} = {frac:.3f}") +if frac < 0.99: + raise SystemExit("leave did not restore the 3-node placement") +print("leave restored the original 3-node placement") +PY + +echo +echo "DONE. Join/leave remaps ~1/N primaries. Keys are not migrated;" +echo "a newly responsible node starts empty (no persistence, no hinted handoff)." diff --git a/scripts/stop-cluster.sh b/scripts/stop-cluster.sh index 429e1f9..130d716 100755 --- a/scripts/stop-cluster.sh +++ b/scripts/stop-cluster.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -u -for n in a b c; do +for n in a b c d; do f="/tmp/ringcache-${n}.pid" if [[ -f "$f" ]]; then pid="$(cat "$f")"