From 7e426f399a8e4ba8e877e832a2df90acf95a54c3 Mon Sep 17 00:00:00 2001 From: Gleb Linnik Date: Wed, 11 Nov 2026 01:27:08 +0300 Subject: [PATCH 01/54] refactor(notify): route streams through addressed store jobs Capture stream eligibility in host inputs and execute notification IO in the app runner, preserving same-call admission settlement and per-connection ordering. Retire the Notify worker, global-read and production-fork ledger entries. --- ci/allow/forks/data-notify.txt | 8 - ci/allow/globals/data-notify.txt | 13 - ci/allow/workers/data-notify.txt | 6 - ci/list-caps.ini | 2 +- docs/design-briefs/notify-stream.md | 96 +++++ docs/sans-io-guide.md | 20 ++ docs/sans-io.md | 20 +- rust-modules/data/src/stores/mod.rs | 42 ++- rust-modules/data/src/stores/notify.rs | 335 ++++++++---------- rust-modules/data/src/stores/notify_job.rs | 99 ++++++ rust-modules/data/src/stores/notify_tests.rs | 315 ++++++++++++++-- rust-modules/plex/src/plex/client_keys.rs | 25 +- .../plex/src/plex/client_keys_roster_tests.rs | 48 +++ .../src/app/adapters/notify_runner_tests.rs | 163 +++++++++ rust-modules/src/app/adapters/stores.rs | 138 +++++++- rust-modules/src/app/bridge.rs | 27 +- .../src/app/browse_hubs_drain_tests.rs | 47 +-- rust-modules/src/app/notify_bridge_tests.rs | 185 ++++++++++ tests/test_harness.py | 7 +- 19 files changed, 1304 insertions(+), 292 deletions(-) delete mode 100644 ci/allow/forks/data-notify.txt delete mode 100644 ci/allow/globals/data-notify.txt delete mode 100644 ci/allow/workers/data-notify.txt create mode 100644 docs/design-briefs/notify-stream.md create mode 100644 rust-modules/data/src/stores/notify_job.rs create mode 100644 rust-modules/src/app/adapters/notify_runner_tests.rs create mode 100644 rust-modules/src/app/notify_bridge_tests.rs diff --git a/ci/allow/forks/data-notify.txt b/ci/allow/forks/data-notify.txt deleted file mode 100644 index 5c1a8fb5d8..0000000000 --- a/ci/allow/forks/data-notify.txt +++ /dev/null @@ -1,8 +0,0 @@ -# count: 3 -# Test/production forks (docs/sans-io.md), area `data-notify`: one line per production-visible `#[cfg(..)]` / `cfg!(..)` whose predicate reads `test` or `feature = "test-support"` -# (`not(test)`, `not(any(test, feature = "test-support"))`, ...) in every machine crate (data/, plex/, media/, session/, screens/, ui/, appkit/ and machine/), keyed `::` (an attribute on an item), `::` (a statement or an inline `cfg!`) or `::` (a field). -# Logic is one thing in production and tests deliver it events: there is no cfg(test) fork. Spec: ci/check-deps.sh `forks`. -# The gate compares this list to the tree as an exact multiset: a new site fails it, and so does a line whose site is gone, so it only shrinks. A migration lands by deleting its lines; do not add one to make a new site pass. -rust-modules/data/src/stores/notify.rs::Spawn -rust-modules/data/src/stores/notify.rs::default_connector -rust-modules/data/src/stores/notify.rs::impl Connector for Spawn diff --git a/ci/allow/globals/data-notify.txt b/ci/allow/globals/data-notify.txt deleted file mode 100644 index f3da0c562b..0000000000 --- a/ci/allow/globals/data-notify.txt +++ /dev/null @@ -1,13 +0,0 @@ -# count: 8 -# Process-global reads inside machines (docs/sans-io.md), area `data-notify`: one line per production read of a registry in a machine — -# (the spellings are ci/check-deps.sh's `globals` rule) — keyed `::::`. -# A machine reads no registry: the value arrives in an event or an argument. Spec: ci/check-deps.sh `globals`. -# The gate compares this list to the tree as an exact multiset: a new site fails it, and so does a line whose site is gone, so it only shrinks. A migration lands by deleting its lines; do not add one to make a new site pass. -rust-modules/data/src/stores/notify.rs::connect::client_for -rust-modules/data/src/stores/notify.rs::pump::client_for -rust-modules/data/src/stores/notify.rs::sync_roster::client_for -rust-modules/data/src/stores/notify.rs::sync_roster::client_for -rust-modules/data/src/stores/notify.rs::sync_roster::server_facts_gen -rust-modules/data/src/stores/notify.rs::sync_roster::server_ids -rust-modules/data/src/stores/notify.rs::sync_roster::server_ids -rust-modules/data/src/stores/notify.rs::sync_roster::server_roster_gen diff --git a/ci/allow/workers/data-notify.txt b/ci/allow/workers/data-notify.txt deleted file mode 100644 index 72c87178fd..0000000000 --- a/ci/allow/workers/data-notify.txt +++ /dev/null @@ -1,6 +0,0 @@ -# count: 1 -# Worker starts inside machines (docs/sans-io.md), area `data-notify`: one line per production call that starts a worker in a machine — -# `task::spawn`, `spawn_small*`, `spawn_off_frame*`, `storage_worker::submit`, `submit_retained`, a command worker's `.submit_ticket` — keyed `::::`. -# A machine starts no worker: its work leaves as a typed effect the app's runners execute, and the result returns as one addressed event. Spec: ci/check-deps.sh `workers`. -# The gate compares this list to the tree as an exact multiset: a new site fails it, and so does a line whose site is gone, so it only shrinks. A migration lands by deleting its lines; do not add one to make a new site pass. -rust-modules/data/src/stores/notify.rs::connect::spawn_off_frame diff --git a/ci/list-caps.ini b/ci/list-caps.ini index 7c3959d461..34fffba4a9 100644 --- a/ci/list-caps.ini +++ b/ci/list-caps.ini @@ -20,7 +20,7 @@ # Do not class a site `work` or `preview` on a guess: read the code until you know whether content # past the bound is reachable. The list only shrinks: removing a cap removes its section. -[plex/src/plex/client_keys.rs::refill_roster] +[plex/src/plex/client_keys.rs::refill_roster_with_stream_policy] class = not-a-list reason = The truncate trims reusable metadata storage to the number of slots visited after the full roster walk, omitting no registered slot or reachable content. diff --git a/docs/design-briefs/notify-stream.md b/docs/design-briefs/notify-stream.md new file mode 100644 index 0000000000..812b4b8951 --- /dev/null +++ b/docs/design-briefs/notify-stream.md @@ -0,0 +1,96 @@ +# NEXT Lane A — Notify stream / addressed outbox + +## Status, authority, ownership + +Implementation written after the S4/shared-file release: typed Notify core and app runner/Bridge cursor integration. NOT compiled or verified acceptance. The frozen API is `.empryo/reviews/verdict-notify-core.md`; integration files, tests and limits are recorded in `.empryo/reviews/verdict-notify-integration.md`. Production selects real off-frame transport; test boot explicitly selects refusal or finite scripts. No fixture, transport codec, record schema or ordinal changes are part of integration. + +Authority: original `C:/Users/gleb_/verdict.md:13-42,109-118`; current `docs/design-briefs/verdict-remaining.md:39-65`; actual `.empryo/reviews/verdict-s4-core.md:9-46,86-97`. Attached `C:/Users/gleb_/stores-rest-design.md:209-234,611-638` supplies S7/S8, but its one-pump refusal concession at 632 and globals -4 estimate at 636 are SUPERSEDED. No new runtime, AppFx, recorder record or frozen schema. + +Source paths below use actual app location `rust-modules/src/app/`, not the historical `rust-modules/app/src/` spelling. Original line references in the preservation contract describe the pre-extraction source, not current line numbers. Written tests and source inspection are not compiler, replay or device evidence. + +## Existing contract to preserve + +`data/src/stores/notify.rs:364-420` increments frame once, reconciles roster, then visits conns in retained table order. For EACH conn it checks identity/permission, attempts launch, immediately backs off on false **before** draining that conn, and only then advances to the next conn. `Conn::drain:206-274` takes posts and end under one lock; posts in FIFO order can first mark Connecting live, accrue resync, append play/library output; stability accounting follows posts; terminal follows all posts. Ordinary terminal backoff uses the frame of observation, not worker wall time. No ordering by global worker-arrival time exists across connections. + +Keep `back_off:195-203` exactly: seconds `(2 << attempts.min(8)).min(60)`, frames seconds * 60, jitter `(frame + sid.raw()*7919) % max(frames/4,1)`, saturating attempts increment once, until frame+frames+jitter. Keep 25*60 stable frames, gap_owed, one resync per unstable escalation, deferred stable debt, 401 token park, other 4xx/Unsupported session park, status zero retry. First connection is not a resync. Notify hints never become stops, Offset publications or account writes. + +## S4 input: extend, do not replace + +The actual `plex/src/plex/client_keys.rs:49-59,106-120` has ONE reusable key Vec plus ONE all-slot facts Vec, current and two generations. Facts already contain machine_id, immutable optional described ServerFacts and coherent optional probe/tier. Reuse current/key/slots/facts/description/probe; no RosterView, extra roster buffer, per-frame collect/hash or fresh strings/Arc. + +The new consumed fact is `ClientFacts::stream_allowed: bool` and `ClientKeys::stream_allowed(sid)->bool` (false without key/row). Equality, Debug and retained refill include it. Key-only compatibility constructors remain fail-closed. The old `refill_roster` stays compatible; app host capture uses `refill_roster_with_stream_policy` with `c.has_token() && !c.data_io_disabled() && c.credential_transport_open()`. Every fill runs policy within the existing eligible/full-key, probe Acquire, tier, and second eligible/full-key fences, even without generation changes. Incoherence clears source/key and permission. No credential value, Client pointer, second registry buffer or profile-vector allocation enters logic. + +Notify input is `NotifyInput<'a> { keys: &'a ClientKeys, armed: bool }`; suspended/enabled/frame remain store-owned. Admission requires described.is_some_and(|d|d.owned), current key and stream_allowed, !suspended, !armed and enabled. UNKNOWN DESCRIPTION IS DISALLOWED. Browse's undescribed-owned=true fallback is not shared policy. Current server does not restrict Notify to a single server. Reconcile retained conns from slots/keys/described facts, retaining existing conn order and appending newly eligible slots in supplied slot order; do not sort existing conns on each fill. Eligibility/description changes must be checked even with unchanged generation hints. Missing key retires/cancels the old mailbox, never uses ambient fallback. + +Keep Conn's existing tuple identity if desired (grant moves token already), but every request and release uses full ClientKey; storing full key in Conn/Link is simpler and preferred. Identity replacement drops old Link, resets attempts, owes gap iff previously live and preserves Session park; old Token park clears. Input gate loss cancels, marks live gap and moves Connecting/Live to Idle exactly as existing pump. Suspend/network/reset keep their distinct existing deliberate-gap behavior (`notify.rs:467-515`); resume forgives Backoff only; network preserves parks; reset clears conns/resync/seen but must NOT recycle request identities. Armed and policy-disabled stand-down cancel/forget each Link and discard its mail; for each retained conn, Live sets gap_owed=true, Connecting/Live become Idle, and existing Backoff deadlines, Token/Session parks, attempts, live_since and resynced_unstable remain unchanged. Connecting retains any existing gap debt but adds none. This implements the current armed/gate-loss branch (`rust-modules/data/src/stores/notify.rs:400-407`), not merely close_all. Source distinction: current enabled=false early return (:367-369) only closes links and would strand Connecting/Live; the proposed disabled/reenabled policy explicitly applies the same state repair as armed rather than claiming that repair already exists. Identity/roster changes retain the rules above. No owed stream waiting under dump/replay; gap debt is not an active claim. + +## Exact typed lane + +Implemented typed lane (types remain Notify-local; the authoritative signatures are in the core verdict): + +```rust +pub struct NotifyStreamReq { pub sid: ServerId, pub key: ClientKey } +// Job.addr.to = MachineId::Store(StoreId::Notify.ord()) +// Job.addr.req = fresh monotonically allocated RequestId for each connection incarnation +StoreJob::NotifyStream(Job, NotifyStreamPort) +pub struct NotifyStreamPort { link: Link } // Clone; captured at emission +pub enum NotifyTerminal { End(NotifyEnd), NotStarted } +impl Port for NotifyStreamPort { /* done / refused(addr) */ } +// refused is initial rejection only; done(NotStarted) is ordinary release failure. +pub struct NotifyAdmission { pub addr: Addr, pub origin_frame: u64, + pub accepted: bool } +``` + +Bind Link to the complete Addr/full key; wrong-address post/end/refusal is ignored. Allocate requests store-wide, not per server; do not reset allocator on profile reset (checked exhaustion rather than wrap/reuse). Conn keeps the cancel-owning Link clone. Port adds addressed `post(addr, Notification)` forwarding to Link. No Notification cloning out of borrowed events; move batches from Inbox. Implement identity-only Debug/PartialEq for job/port as existing StoreJob derives require; never compare mutex payloads or log tokens. + +Worker entry in reviewed data port/body file, e.g. `data/src/stores/notify_job.rs`: +`finish_notify_stream_job(port: &NotifyStreamPort, addr: Addr, client: &'static Client, req: NotifyStreamReq, off: &OffFrame)` runs existing `client.notifications(off, port.link.cancel_flag(), &mut |n| port.post(addr,n))`, catches unwind and terminates Transport on panic. The app adapter alone resolves client and starts `spawn_off_frame("notify", ...)`. Full key and fresh has_token/data_io_disabled/credential_transport_open checks at launch AND held release, then again inside worker before opening; keep underlying http::stream's credential gate. Cancellation uses Acquire, store writes Release. No callback from the data step into resolver/launcher/transport. Preserve exact SSE path, identity headers, own-session filter, idle window and existing account/mock safety; do not duplicate transport. + +## SAME-CALL protocol: connection cursor, not bulk two-pump + +Choose a minimal split of the existing per-connection loop. A bulk begin-all/start-all/drain-all changes when earlier connections' ordinary mail is observed; reject it. A second `pump()` changes frame and retry/stability accounting; reject it. The following named pure entry points are the implementation contract: + +1. `Stores::notify_begin(&NotifyInput)->NotifyPass`: increment frame ONCE; reconcile once; capture frame and initial connection count/cursor in a private pass token; initialize NotifyOut. No network mailbox is taken. Armed/disabled applies the explicit per-conn stand-down transitions above before returning an empty completed pass, without taking orphan mail or emitting jobs. Preserve identity/roster reconciliation rules; close_all alone is insufficient. Do not expose a replay landing. +2. `notify_prepare_next(&mut pass, &NotifyInput)->Option` visits exactly the next conn. Perform its existing key/gate/drop/due logic. For due launch create fresh Link, retain pending incarnation and emit ONE NotifyStream job. Turn holds conn identity/Addr/frame, not a borrowed Client. Existing live conn emits no job. No attempts/backoff is changed speculatively for an attempted launch. +3. Exit data step. Bridge drains **only StoreId::Notify** immediately through existing `take_jobs_of` -> `start_jobs` -> `StoreRunner::start`. Runner synchronously reports addressed admission to a separate Link admission slot (not ordinary Inbox end), carrying the originating frame captured when emitted. Accepted means successfully scheduled OR deliberately held, not first SSE data. At INITIAL admission only, false launcher means work was never executed; key/gate mismatch rejects with NotStarted and prelaunch cancel settles Cancelled once. Initial rejection is settled before this conn's ordinary drain. Deliberately held acceptance is final single assignment: RELEASE is a distinct execution stage, not another admission. On release key/gate/OS-launch failure, deliver one ordinary terminal (NotStarted, retryable like Transport/status zero) to the accepted Link; never rewrite admission or reuse origin_frame. Owned current-conn drain backs off at observation frame. Cancelled/orphaned release terminalizes only its old port once and never touches any replacement/store state. The accepted receipt is not a terminal. Admission slot cannot be overwritten. Implement the receipt as Notify-local Link metadata read by the next pure entry, preserving `start`'s existing void API; no callback or new application runtime. +4. `notify_settle_admission(&mut pass, turn)` consumes only that turn's addressed admission. On accepted install/commit Connecting. On false reproduce `back_off(origin_frame)` NOW, drop pending Link, and consume the local NotStarted terminal so it cannot cause a second backoff. Prelaunch cancel of an already orphaned incarnation has no store effect. This occurs before ANY ordinary drain for that conn. Never read ordinary worker terminal to infer admission: an accepted worker can complete before `start` returns. +5. `notify_finish_turn(&mut pass, turn)` takes that conn's ordinary events/end once under its inbox lock, running existing `Conn::drain(pass.frame)` ordering and stability exactly once. Release input fence requires same incarnation/Addr/full key and allowed owned evidence; a lifecycle/profile change invalidates turn and discards mail. Advance cursor only here. Repeat 2-5 for next conn. No retry decrement or frame increment in settlement/finish. +6. `notify_end(pass)->NotifyOut` asserts every turn settled/drained; Bridge invokes notify_map ONCE on completed output, then drains NOTIFY_FILLS immediately before returning existing `(resynced, endpoints)`. + +Admission origin_frame can be private Link emission metadata rather than added to NotifyStreamReq: INITIAL admission rejection returns its captured Addr+frame, never reads a later store frame; an accepted-held RELEASE failure instead uses ordinary terminal observation frame. Generic start_store_jobs_of(Notify) can schedule a manually emitted job, but cannot finish a pass: each real caller must use the cursor driver; aggregate backstop remains an error signal, not an alternative delayed implementation. + +Ordering example: conn A old live mail, B refused launch, C accepted fast worker. A is prepared/drained before B is launched; B backs off using F before B drain; C then launches and drains at F. If worker B/C posts asynchronously before its own lock-take it is eligible this call, just as old connect-before-drain; after lock-take it waits for next frame. An A post arriving while C launches waits for next frame, not scooped up by a final aggregate drain. Within each Link posts before terminal are ordered by the same mutex. Across connections retained table order, not admission receipt arrival or network completion order, controls NotifyOut arrays. This is source-equivalent to `notify.rs:408-419` with IO extracted between pure phases, not a one-pump marker. Same-call refers to admission; ordinary network completion remains observed at the original connection drain point, never eagerly at runner start. + +## Terminal ownership and orphan discipline + +Harden existing Inbox in place with `terminal_sent: bool` that persists after `end.take()`. Under one mutex: post ignores terminal_sent; finish checks it, sets it and queues exactly one terminal, preserving previously queued posts. Never use `end.is_none()` as permission to finish again. A separate admission slot is single assignment, independent of terminal state: a fast accepted worker may finish before its accepted receipt is stored. `Port::refused` atomically commits NotStarted once; launch false promises closure never ran. No refusal enters recorded src_result, tape or landgate mailbox. + +Store cancellation sets cancel and finishes Cancelled once on the old Link, then forgets it. Worker completion racing cancellation loses harmlessly to terminal_sent; later pushes ignored. Orphan means old Conn no longer owns the Link: nothing from that Link (including terminal/admission) can land in a replacement Conn. Drop/reset closes all. Suspended/armed/removed/profile-reset incarnations are not owed; held-cancel is pruned/terminally settled without dialing. INITIAL prelaunch key/gate mismatch or launcher false: NotStarted plus rejected admission, backoff once at originating frame before drain. AFTER accepted-held admission, release key/gate mismatch or launcher false: one ordinary NotStarted terminal, admission remains accepted, and backoff once at observation frame only if this exact Addr/full-key incarnation still owns the Link. This follows existing ordinary retry semantics without resetting attempts on accepted/held/released scheduling; only existing identity replacement, stability, resume and network rules reset attempts. If gate/key reconciliation has already retired the Link, discard its terminal without attempts, retry, gap or resync changes to any newer/orphan connection. Accepted worker panic: Transport terminal, ordinary drain at observation frame. Actual accepted cancel: Cancelled once, with existing current-conn terminal semantics if still owned; deliberate lifecycle cancellation already removes ownership and clears gap where specified. Transport status zero remains ordinary retry, not Session park. + +No new reserving Fetch or landgate claim is needed: Conn's pending/Connecting/Live incarnation is the local claim. End/refusal/cancel retires it exactly once; stale/orphan has no owed bit. Armed path cannot retain an indefinite stream owed predicate, so dump never waits for SSE termination. Keep S7 many-post/one-terminal/cancel vocabulary; `machine/src/job.rs:53-58` only supplies terminal Port, not a shared stream implementation. Do not genericize all ports/channels now. Watchlist W2 can extract the proven minimal shared channel when its second concrete consumer lands. + +## Integration checklist and acceptance obligations + +- `data/src/stores/mod.rs`: append Notify AFTER Catalog (ordinal **12**); ALL length 13, name "notify", from_ord/ord round trips; old 0..11 unchanged. No StoreCmd/screen dispatch variant needed. Add NotifyStream variant, admission Direct, store() Notify, take_jobs_of and has_jobs_of Notify; aggregate take_jobs/has_jobs and leftover/backstop automatically traverse ALL. Update all exhaustive StoreId/StoreJob matches, ordinal-array assumptions and tests; no new recorded mailbox payload or accepted record kind. +- Runner `rust-modules/src/app/adapters/stores.rs` (central StoreJob is `rust-modules/data/src/stores/mod.rs`, not runner-local): start -> start_notify_stream; launch path -> off-frame worker; held park path -> accepted receipt; release -> current full-key/gate recheck, accepted receipt immutable, release failure ordinary terminal/observation-frame backoff only while still owned; run_held_where -> Notify-specific off-thread/scripted branch; take_held returns owned port, test fabrication must terminate or cancel it. retire_parked_details must KEEP Notify, never cancel it on Metadata replacement. admit_home non-Home passthrough includes NotifyStream; start_admitted routes Direct without tape/Home calls. All branches terminal or intentionally active/held; cancelled parked requests prune and settle once. Add Notify to job-kind classification where appropriate. +- Default Live uses real worker. Default Held and park_page_reads put NotifyStream in live-kind policy **only with boot-selected safe mock/no-network stream port**; never call indefinite SSE on test thread from run_held_where. This follows existing BrowseDiscovery/MetaAltSources off-thread exclusions (`adapters/stores.rs:216-250`) and replaces old default Refuse safety (`notify.rs:115-130`). Held-all parks Notify; explicit release runs finite scripted posts/terminal synchronously OR schedules live transport off-thread, never raw notifications on test thread. Default test boot must choose refusing/scripted stream transport so owned synthetic registry entries cannot accidentally open real SSE. This policy requires runner constructor/rig migration, not a new cfg fork in Notify logic. +- `rust-modules/src/app/bridge.rs:1859-1875`: move refill BEFORE notify_begin (today after notify_pump); use one captured keys input across cursor pass and map. Drain Notify per turn, then completed map, then current NOTIFY_FILLS `[Browse,Hubs]` in existing ordinal order. Do not add Notify to post-map fills as a substitute for pre-drain admission settlement. Watchlist delta is currently inline in notify_map; W2 must add Watchlist to NOTIFY_FILLS when it gains an outbox, not prematurely here. Preserve endpoint merge/logging/return and notify-map hint-only behavior. +- Caller ownership: current `Stores::notify_pump` -> NotifyStore pump becomes pure phase wrappers; Bridge's route-unconditional notify_pump is the sole app orchestration seam. Existing frame/update caller keeps one Bridge call, covered pages/shown inputs and resync reread behavior. Lifecycle Bridge notify_suspend/resume/network_changed/reset remain direct owner mutations, no new launch/drain on lifecycle-only calls. Migrate notify_tests Fake Connector rig to input+outbox+scripted runner using SAME phase driver; direct aggregate callers cannot retain the old un-drained pump. NotifyMap's Browse/Hubs fan-outs retain existing drains; future Watchlist second consumer stays separate. +- When S4 releases app files, ONE Lane A integration owner owns `rust-modules/plex/src/plex/client_keys.rs`, `rust-modules/src/app/adapters/stores.rs`, `rust-modules/src/app/bridge.rs`, central Stores/StoreJob `rust-modules/data/src/stores/mod.rs`, runner/Bridge/Notify tests and guidance/ledger/pins updates. Notify implementer owns store policy/body/channel. Serialize hub changes with S4 and Lane B; no concurrent app/Stores edits. No changes to M4, session runtime, transport body, fixture bytes, recorder schema or rank. + +## Required evidence before acceptance + +1. Characterize old connector false at frame F, including nonzero attempts: exact until/jitter, attempts +1 once, no launch before until, launch exactly at until. New runner false/key mismatch matches in SAME Bridge return, even when fast unrelated worker mail exists. Assert no jobs left and late_store_jobs=0. +2. Multi-conn A/B/C order above, mixed refused/accepted/old live, posts racing before/after each lock, posts+terminal same batch, ping-first live transition; no earlier normal completion take, no second pump, stable accounting once. Pin exact ordered plays/library/resynced and map fan-out launch order. +3. Unexpected-live loss/reconnect, rapid flaps, owed stable debt at 1500 frames, refusal during gap, 401/403/Unsupported/status0 parks, resume/network and cancel cadence; exact attempts and resync epochs across each boundary. +4. Live mock worker; Held accepted/no posts; Held release; held cancel before release; launch false; panic before/after posts; stale Addr/full-key instance/token/grant change; profile reset; owned->shared/unknown removal; old worker late posts/end to orphan. Each terminalized case ends once with no retained claim, dump wait or busy bit; intentionally active/held acceptance is not itself completion. Pin accepted-held at F with attempts a>0, release gate/key/OS-launch failure, ordinary observation at G>F: attempts stays a through acceptance/release, then a+1 once; retry until is G + D + ((G wrapping_add sid.raw()*7919) % max(D/4,1)), D=60*min(2< drain `end.take()` -> second finish/post must remain ignored. Wrong Addr cannot poison current channel. Admission accepted racing worker terminal cannot become NotStarted or double backoff. +6. Fresh credential gate expiry with unchanged roster/facts generation and unchanged key; has_token/data_io_disabled changes; held-release expiry; retoken/repoint/revoke capture races; probe Acquire-before-tier/full-key recheck; described owned/shared/unknown. Unknown never borrows Browse fallback. No credential/client pointer in logic; strings/Vec reuse unchanged. +7. Arm/unarm from BOTH Connecting and Live, and policy disable/reenable from BOTH, with queued old posts/end: no old output/job/owed claim, state Idle, attempts unchanged, Live gap=true and Connecting's prior debt unchanged. On reenable/unarm, Idle launches at the next eligible single pump; first post pays live gap once under existing unstable rules (deferred debt waits 1500 stable frames). Separately pin Backoff unchanged deadline/attempts (including unarm before/after deadline), Token/Session parks retained, no retry while blocked; identity changes follow normal reset/park rules. Characterize current disabled close-only behavior separately, not as desired equivalence. Boot-policy tests: default test store/Bridge never dials real network; default Held aggregate drain and held_all release never block test thread on SSE; scripted and mock-only finite/live streams; no actual account writes, stops or Offset. Preserve existing worker-side transport safeguards. +8. StoreId old ordinal/name round trips and new 12; no Notify admission/result recording under armed record/replay/dump; full canonical Linux gates, shipping check-nodef, relevant hostsim and six unchanged 1/6/12 Targets+Resolve replays **6/6** with zero differences. No rererecord or fixture/schema repair to hide a mismatch. + +Parent supplied an actual header-plus-entry ledger recount of **6 workers / 89 globals / 65 forks / 11 fxlanes** while the three Notify area files still exist. Integration leaves those files and pins unchanged for the next shell's guarded retirement. Their exact removal is expected to yield **5/81/62/11**; that post-retirement tree must be recounted before pin changes. No added/renamed sites or raised pins are authorized. The source extraction removes the old connect worker, eight registry reads and three connector/default forks; the retained ledger files deliberately make this an unverified source-gate checkpoint. + +## Handoff + +Core and app integration are handed to the shell compiler loop, not held waiting for tests. Stop acceptance on any next-pump initial refusal, bulk-drain timing change, accidental live test dial, orphan claim, second terminal after take, or fixture difference. No canonical snapshot, build, tests, ledger deletion or fixture operations ran in this integration node. Source-timing equivalence is the per-connection protocol above, not a claim of identical asynchronous network wall-time scheduling; disabled-state repair is an intentional repair, not historical close-only equivalence. diff --git a/docs/sans-io-guide.md b/docs/sans-io-guide.md index cd147d8c40..d3781ae3ab 100644 --- a/docs/sans-io-guide.md +++ b/docs/sans-io-guide.md @@ -7,6 +7,26 @@ process-global registry and has no `cfg(test)` fork. Its work leaves as a typed the app's runners execute, and the result returns as one addressed event. The reason is that a step then has no seam to fake: a test and a replay feed it the events the product feeds it. +## Notify stream integration (written; acceptance pending) + +Notify uses one captured `NotifyInput { keys, armed }` and the actual store's +`begin -> prepare_next -> settle_admission -> finish_turn -> end` cursor. Refill +keys with `refill_roster_with_stream_policy` before begin. For each connection, +exit data and drain only Notify jobs through the app runner before settlement; +finish that connection before preparing the next. Map the completed output once, +then drain the unchanged Browse/Hubs fill list and Notify in StoreId order. + +Initial refusal settles at the emission frame before ordinary mail. Accepted-held +release failure is an ordinary terminal at the observation frame, never a second +admission. Production streams always run off-frame, including held real release; +`Bridge::for_test` selects refusal, and finite scripts supply explicit eligibility. +No stream callback runs inside data or while borrowing Stores across the runner. +The coordinator reports post-retirement ledger totals of 5 workers / 81 globals / +62 forks / 11 fxlanes, down from the S4 pre-Notify 6/89/65/11. The source-only +stale-key test passed in orun-27; runtime tests, capture comparisons and full gates +remain pending. These contracts are not runtime or replay acceptance; see +`.empryo/reviews/verdict-notify-integration.md` for the handoff limits. + ## Which code is a machine `ci/check-deps.sh` (the `MACHINES` array) scans these for `workers` and `globals`: diff --git a/docs/sans-io.md b/docs/sans-io.md index 4d20be292d..045fc75456 100644 --- a/docs/sans-io.md +++ b/docs/sans-io.md @@ -7,6 +7,24 @@ player machine: `player/lifecycle.rs` is the machine alone (private state, one w `player/foreground.rs` its drivers and `player/transport.rs` the transport-pause contract; its shape is gated by `ci/machines.ini` (see "Machine shape" below). +## Notify integration checkpoint (not acceptance) + +Notify's typed Direct stream lane and app per-connection cursor are written; the +frozen API and integration evidence are in `.empryo/reviews/verdict-notify-core.md` +and `.empryo/reviews/verdict-notify-integration.md`. StoreId Notify is appended at +12; old ordinals, Home/tape records and NOTIFY_FILLS `[Browse, Hubs]` are unchanged. +Bridge captures fresh stream eligibility before the one-frame cursor, settles each +initial admission before that connection's ordinary drain, then maps once and +drains fan-out jobs. Accepted-held release failures use ordinary terminal timing. +Production opens off-frame; test boot refuses unless explicitly scripted. + +The three Notify allow-area files have been retired. The coordinator reports actual +ledger totals of 5 workers / 81 globals / 62 forks / 11 fxlanes, down from the +S4 pre-Notify checkpoint of 6/89/65/11; the pins now match. The source-only stale-key +check passed in orun-27; runtime tests, capture comparisons and full gates remain +pending. Per-connection source ordering is preserved; asynchronous worker wall-time +is not pinned, and disabled-state repair is explicitly intentional. + ## The rule Logic never performs IO. A machine (a screen, a `data/` store, the session owner, the player's @@ -221,7 +239,7 @@ headed "The listing lanes"): their two worker starts and two registry reads are `data/src/browse/mod.rs`. The Library's source discovery (the section list, the server's name and the item counts) is the twelfth (the bullet headed "The discovery lane"): its worker start and its one registry read are gone from `data/src/browse/mod.rs`, and its `data-browse` workers ledger file is -deleted. The Home pins take the session from the Bridge (the bullet headed "The pins' session input"), and so do the Watchlist and catalog stores, for the seated profile and its plex.tv identity (the bullet headed "The Watchlist and catalog session input"), which deletes three `data-watchlist` globals lines. S4 replaces Browse's roster/gate/current reads with explicit `BrowseRosterInput` and the existing host-filled `ClientKeys`; the obsolete 14-entry Browse globals file is pending guarded deletion. Expected pins after that deletion are **6 workers / 89 globals / 65 forks / 11 fxlanes** (recount before deletion: 6/103/65/11). This is not integrated verification evidence; legacy fixture/caller migration remains tracked in `.empryo/reviews/verdict-s4-integration.md`. Other stores' and Session's global sites remain unchanged. The Metadata store's +deleted. The Home pins take the session from the Bridge (the bullet headed "The pins' session input"), and so do the Watchlist and catalog stores, for the seated profile and its plex.tv identity (the bullet headed "The Watchlist and catalog session input"), which deletes three `data-watchlist` globals lines. S4 replaces Browse's roster/gate/current reads with explicit `BrowseRosterInput` and the existing host-filled `ClientKeys`; the obsolete 14-entry Browse globals file was retired. The S4 checkpoint was **6 workers / 89 globals / 65 forks / 11 fxlanes** (recount before deletion: 6/103/65/11); post-Notify retirement the coordinator reports **5/81/62/11**. Runtime and integrated verification remain pending; legacy fixture/caller migration remains tracked in `.empryo/reviews/verdict-s4-integration.md`. Other stores' and Session's global sites remain unchanged. The Metadata store's season switch and episode-page window are the tenth (the bullet headed "The Metadata reads"): their two worker starts and the `client_for` reads of `fetch_episodes` and `fetch_episode_page` are gone from `data/src/metadata.rs`; the Related row's tail window read joins them in the same bullet, and its diff --git a/rust-modules/data/src/stores/mod.rs b/rust-modules/data/src/stores/mod.rs index b21a75565e..bda309688d 100644 --- a/rust-modules/data/src/stores/mod.rs +++ b/rust-modules/data/src/stores/mod.rs @@ -67,6 +67,7 @@ pub use content_arg::ContentArg; pub mod hubs; pub mod metadata; pub mod notify; +pub mod notify_job; pub mod notify_map; pub mod person; pub mod collection; @@ -281,6 +282,7 @@ impl Stores { #[must_use] pub fn take_jobs_of(&mut self, id: StoreId) -> Vec { match id { + StoreId::Notify => self.notify.take_jobs(), StoreId::Metadata => self.metadata.take_jobs(), StoreId::Collection => self.collection.take_jobs(), StoreId::Person => self.person.take_jobs(), @@ -304,6 +306,7 @@ impl Stores { fn has_jobs_of(&self, id: StoreId) -> bool { match id { + StoreId::Notify => self.notify.has_jobs(), StoreId::Metadata => self.metadata.has_jobs(), StoreId::Collection => self.collection.has_jobs(), StoreId::Person => self.person.has_jobs(), @@ -444,9 +447,22 @@ impl Stores { /// The notification feed's per-frame pass. It stands down while the landing gate is armed (a /// recording, a replay or a dump owns the run), and never reads the registry's tokens. - pub fn notify_pump(&mut self) -> notify::NotifyOut { - self.notify.pump(self.landgate.is_armed()) + pub fn notify_begin(&mut self, input: ¬ify::NotifyInput<'_>) -> notify::NotifyPass { + assert_eq!(input.armed, self.landgate.is_armed()); + self.notify.begin(input) } + pub fn notify_prepare_next(&mut self, pass: &mut notify::NotifyPass, input: ¬ify::NotifyInput<'_>) -> Option { + assert_eq!(input.armed, self.landgate.is_armed()); + self.notify.prepare_next(pass, input) + } + pub fn notify_settle_admission(&mut self, pass: &mut notify::NotifyPass, turn: &mut notify::NotifyTurn) { + self.notify.settle_admission(pass, turn) + } + pub fn notify_finish_turn(&mut self, pass: &mut notify::NotifyPass, turn: notify::NotifyTurn, input: ¬ify::NotifyInput<'_>) { + assert_eq!(input.armed, self.landgate.is_armed()); + self.notify.finish_turn(pass, turn, input) + } + pub fn notify_end(&mut self, pass: notify::NotifyPass) -> notify::NotifyOut { self.notify.end(pass) } /// Hold (`true`) or release the view-state reconcile network work; see [`Stores::network_held`]. /// Patches are not held: what the stores already show keeps changing while the player is up. @@ -627,7 +643,7 @@ impl Stores { StoreId::Watchlist => self.watchlist.gen(), StoreId::Catalog => self.catalog.gen(), // A landing gate slot only: the stop mailbox is not a store and has no generation. - StoreId::ViewStop => 0, + StoreId::ViewStop | StoreId::Notify => 0, } } @@ -693,6 +709,8 @@ pub enum StoreId { Watchlist, /// Appended for the same reason: the catalog page's discover record (`crate::catalog`). Catalog, + /// Never recorded; appended without moving existing ordinals. + Notify, } /// Route-scoped background work, distinct from a user command. Polling an idle store must not @@ -724,6 +742,7 @@ impl StoreWork { /// lane a store is pumped on, a job carries the request itself and the port its answer goes to. #[derive(Debug, PartialEq)] pub enum StoreJob { + NotifyStream(plx_machine::job::Job, notify_job::NotifyStreamPort), /// One detail fetch (`MetadataCmd::RequestDetail`), addressed by its request generation. Detail(plx_machine::job::Job, crate::metadata::DetailPort), /// One collection page request (resolve, header or children), emitted by the Collection pump @@ -844,7 +863,8 @@ impl StoreJob { // `frame_ingest`, the Browse pump's own discovery step) starts it through // `StoreRunner::start` as `Direct`, byte for byte as its unrecorded spawn was. Self::BrowseDiscovery(..) => Admission::Home, - Self::Catalog(..) + Self::NotifyStream(..) + | Self::Catalog(..) | Self::SubtitleSearch(..) | Self::ViewReconcile(..) | Self::Search(..) @@ -865,6 +885,7 @@ impl StoreJob { Self::Detail(..) | Self::MetaSeason(..) | Self::MetaEpisodes(..) | Self::MetaRelated(..) | Self::MetaAltSources(..) => StoreId::Metadata, Self::Collection(..) => StoreId::Collection, Self::Person(..) => StoreId::Person, + Self::NotifyStream(..) => StoreId::Notify, Self::Catalog(..) => StoreId::Catalog, Self::SubtitleSearch(..) => StoreId::SubtitleSearch, Self::ViewReconcile(..) => StoreId::ViewReconcile, @@ -877,7 +898,7 @@ impl StoreJob { } impl StoreId { - pub const ALL: [StoreId; 12] = [ + pub const ALL: [StoreId; 13] = [ StoreId::Browse, StoreId::Hubs, StoreId::Metadata, @@ -890,6 +911,7 @@ impl StoreId { StoreId::ViewStop, StoreId::Watchlist, StoreId::Catalog, + StoreId::Notify, ]; /// The library's ordinal for this store (spec §5.1: the library never names `StoreId`). @@ -915,6 +937,7 @@ impl StoreId { StoreId::ViewStop => "viewstop", StoreId::Watchlist => "watchlist", StoreId::Catalog => "catalog", + StoreId::Notify => "notify", } } } @@ -1406,7 +1429,14 @@ mod tests { for id in StoreId::ALL { assert_eq!(StoreId::from_ord(id.ord()), Some(id)); } - assert_eq!(StoreId::from_ord(StoreOrd(12)), None); + assert_eq!(StoreId::Notify.ord(), StoreOrd(12)); + assert_eq!(StoreId::Notify.name(), "notify"); + assert_eq!(StoreId::from_ord(StoreOrd(13)), None); + let original = ["browse", "hubs", "metadata", "search", "person", "viewstate", + "collection", "subsearch", "viewreconcile", "viewstop", "watchlist", "catalog"]; + for (ord, name) in original.into_iter().enumerate() { + assert_eq!(StoreId::from_ord(StoreOrd(ord as u32)).unwrap().name(), name); + } } /// A fan-out call that can fill an outbox names it in a `*_FILLS` list the Bridge drains. A diff --git a/rust-modules/data/src/stores/notify.rs b/rust-modules/data/src/stores/notify.rs index 6ede5e1282..f7fb96156d 100644 --- a/rust-modules/data/src/stores/notify.rs +++ b/rust-modules/data/src/stores/notify.rs @@ -14,7 +14,7 @@ //! session must not scrobble our other servers (`fan_out_origin_watched`). The output //! ([`NotifyOut`]) is read by `stores::notify_map`, which turns it into read-backs of held items. //! -//! The store has no `StoreId` and no `StoreCmd`: nothing on a screen addresses it. It is pumped once +//! The store has an unrecorded job `StoreId` but no `StoreCmd`: no screen addresses it. It is pumped once //! a frame by the application (`Bridge::notify_pump`) and told about the lifecycle //! (`notify_suspend`, `notify_resume`, `notify_network_changed`) and a profile reset //! (`notify_reset`). Every deadline is counted in pumped frames, never read from a clock. @@ -29,8 +29,11 @@ use plx_plex::plex::notify::{LibraryEntry, Notification, NotifyEnd, PlaySession}; use plx_plex::plex::ServerId; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; +use plx_plex::plex::{ClientKey, ClientKeys}; +use plx_machine::machine::{Addr, MachineId, RequestId}; +use plx_machine::job::Job; +use super::{StoreId, StoreJob}; +use super::notify_job::{Link, NotifyStreamPort, NotifyStreamReq}; /// Ships `true`. `docs/pms-api.md` §9 lists what was checked on the mock TV and what is still owed. The owned-only rule is separate and /// unconditional; `notify_tests::enabled_const_connects_to_an_owned_server` and @@ -47,87 +50,31 @@ const BACKOFF_MAX_S: u64 = 60; /// starts over, and a resync it owed but skipped while flapping is delivered. const STABLE_FRAMES: u64 = 25 * FRAMES_PER_S; -/// `(instance_gen, token_gen)` of the client a connection was opened for. -type Key = (u32, u32); +type Key = ClientKey; -#[derive(Default)] -struct Inbox { - events: Vec<(Key, Notification)>, - end: Option, -} +/// Captured immutable input for a complete cursor pass. +pub struct NotifyInput<'a> { pub keys: &'a ClientKeys, pub armed: bool } -/// The worker's side of one connection: the cancel flag it watches, and the mailbox it fills. -/// Every connection gets a fresh one, so a worker that outlives its connection writes into an -/// orphan nobody reads. -#[derive(Clone)] -pub struct Link { - key: Key, - cancel: Arc, - inbox: Arc>, +/// Owned transaction token: no borrow of Stores crosses runner IO. +#[must_use] +pub struct NotifyPass { + owner: std::sync::Arc<()>, frame: u64, cursor: usize, count: usize, + pending: bool, out: NotifyOut, } - -impl Link { - fn new(key: Key) -> Link { - Link { key, cancel: Arc::new(AtomicBool::new(false)), inbox: Arc::default() } - } - fn lock(&self) -> std::sync::MutexGuard<'_, Inbox> { - self.inbox.lock().unwrap_or_else(|e| e.into_inner()) - } - pub fn cancel_flag(&self) -> &AtomicBool { - &self.cancel - } - pub fn is_cancelled(&self) -> bool { - self.cancel.load(Ordering::Acquire) - } - /// A notification arrived (pings included: the first frame of any kind is what marks the - /// connection live). - pub fn push(&self, n: Notification) { - self.lock().events.push((self.key, n)); - } - /// The connection ended. Anything pushed before this is still delivered. - pub fn finish(&self, end: NotifyEnd) { - self.lock().end = Some(end); - } -} - -/// Opens a connection for `sid` and returns at once. `false` = the connection could not be started -/// (the OS refused the worker thread); the store backs off. -pub trait Connector { - fn connect(&mut self, sid: ServerId, link: Link) -> bool; +#[must_use] +pub struct NotifyTurn { + owner: std::sync::Arc<()>, frame: u64, index: usize, sid: ServerId, key: Key, + addr: Option, launch: bool, settled: bool, } -/// The production connector: one small fire-and-forget worker per connection, resolving the -/// server's client at the spawn site. -#[cfg(not(any(test, feature = "test-support")))] -struct Spawn; - -#[cfg(not(any(test, feature = "test-support")))] -impl Connector for Spawn { - fn connect(&mut self, sid: ServerId, link: Link) -> bool { - let Some(client) = plx_plex::plex::client_for(sid) else { return false }; - plx_base::task::spawn_off_frame("notify", move |off| { - let end = client.notifications(off, link.cancel_flag(), &mut |n| link.push(n)); - link.finish(end); - }) - } +fn owned(keys: &ClientKeys, sid: ServerId) -> bool { + keys.key(sid).is_some() && keys.facts(sid) + .and_then(|f| f.described).is_some_and(|f| f.owned) } - -/// Under test only an injected connector can run: a default-built store never opens a stream. -#[cfg(any(test, feature = "test-support"))] -struct Refuse; - -#[cfg(any(test, feature = "test-support"))] -impl Connector for Refuse { - fn connect(&mut self, _sid: ServerId, _link: Link) -> bool { - false - } -} - -fn default_connector() -> Box { - #[cfg(not(any(test, feature = "test-support")))] - return Box::new(Spawn); - #[cfg(any(test, feature = "test-support"))] - return Box::new(Refuse); +fn stand_down(conn: &mut Conn) { + conn.drop_link(); + if conn.state == State::Live { conn.gap_owed = true; } + if matches!(conn.state, State::Connecting | State::Live) { conn.state = State::Idle; } } #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -188,7 +135,7 @@ impl Conn { /// Cancel and forget the open connection, if any. fn drop_link(&mut self) { if let Some(link) = self.link.take() { - link.cancel.store(true, Ordering::Release); + link.cancel(); } } @@ -307,20 +254,22 @@ pub struct NotifyPlays { pub struct NotifyStore { enabled: bool, - connector: Box, + jobs: Vec, + next_req: u32, + active: Option, + owner: std::sync::Arc<()>, conns: Vec, resync: Vec<(ServerId, u32)>, /// `(roster_gen, facts_gen)` the connection table was last built against. seen: (u32, u32), frame: u64, suspended: bool, - /// May this client's token ride its origin now? Production: the grant authority's live answer. - gate: fn(&plx_plex::plex::Client) -> bool, + } impl Default for NotifyStore { fn default() -> Self { - Self::new(NOTIFY_ENABLED, default_connector()) + Self::new(NOTIFY_ENABLED) } } @@ -331,21 +280,14 @@ impl Drop for NotifyStore { } impl NotifyStore { - fn new(enabled: bool, connector: Box) -> Self { - Self { enabled, connector, conns: Vec::new(), resync: Vec::new(), seen: (0, 0), frame: 0, suspended: false, gate: plx_plex::plex::Client::credential_transport_open } + fn new(enabled: bool) -> Self { + Self { enabled, jobs: Vec::new(), next_req: 0, active: None, + owner: std::sync::Arc::new(()), conns: Vec::new(), resync: Vec::new(), + seen: (0, 0), frame: 0, suspended: false } } - /// A store with `connector` and the enable rule overridden: the only way a test runs one. - #[cfg(any(test, feature = "test-support"))] - pub fn with_connector_for_test(connector: Box) -> Self { - Self::new(true, connector) - } - - /// Test seam: replace the credential-transport gate. - #[cfg(any(test, feature = "test-support"))] - pub fn set_gate_for_test(&mut self, gate: fn(&plx_plex::plex::Client) -> bool) { - self.gate = gate; - } + pub fn take_jobs(&mut self) -> Vec { std::mem::take(&mut self.jobs) } + pub fn has_jobs(&self) -> bool { !self.jobs.is_empty() } /// How many times `sid`'s feed came back after having been live. pub fn resync_epoch(&self, sid: ServerId) -> u32 { @@ -361,111 +303,126 @@ impl NotifyStore { /// One frame: reconcile the connection table with the registry, connect what is due and /// allowed, collect what the workers delivered. `armed` is the landing gate: a recording, a /// replay or a dump owns the run and a live stream would break its determinism. - pub fn pump(&mut self, armed: bool) -> NotifyOut { - self.frame += 1; - let mut out = NotifyOut::default(); - if !self.enabled { - self.close_all(); - return out; + pub fn begin(&mut self, input: &NotifyInput<'_>) -> NotifyPass { + assert!(self.active.is_none(), "Notify pass in progress"); + assert!(self.jobs.is_empty(), "undrained Notify jobs"); + self.frame = self.frame.checked_add(1).expect("Notify frame exhausted"); + self.sync_roster(input.keys); + let blocked = input.armed || !self.enabled || self.suspended; + if blocked { for conn in &mut self.conns { stand_down(conn); } } + self.active = Some(self.frame); + NotifyPass { owner: self.owner.clone(), frame: self.frame, cursor: 0, + count: if blocked { 0 } else { self.conns.len() }, pending: false, + out: NotifyOut::default() } + } + + fn check_pass(&self, pass: &NotifyPass) { + assert!(std::sync::Arc::ptr_eq(&self.owner, &pass.owner)); + assert_eq!(self.active, Some(pass.frame)); + } + + pub fn prepare_next(&mut self, pass: &mut NotifyPass, input: &NotifyInput<'_>) -> Option { + self.check_pass(pass); + assert!(!pass.pending, "finish current Notify turn first"); + if pass.cursor == pass.count { return None; } + let conn = &mut self.conns[pass.cursor]; + let allowed = !input.armed && self.enabled && !self.suspended + && owned(input.keys, conn.sid) && input.keys.key(conn.sid) == Some(conn.key) + && input.keys.stream_allowed(conn.sid); + let mut launch = false; + if !allowed { stand_down(conn); } + else if conn.link.is_none() && conn.due(pass.frame) { + self.next_req = self.next_req.checked_add(1).expect("Notify request ids exhausted"); + let addr = Addr { to: MachineId::Store(StoreId::Notify.ord()), req: RequestId(self.next_req) }; + let link = Link::new(addr, conn.key, pass.frame); + self.jobs.push(StoreJob::NotifyStream(Job { addr, + req: NotifyStreamReq { sid: conn.sid, key: conn.key } }, NotifyStreamPort::new(link.clone()))); + conn.link = Some(link); + launch = true; } - self.sync_roster(); - let frame = self.frame; - let Self { conns, connector, resync, suspended, gate, .. } = self; - for conn in conns.iter_mut() { - // `sync_roster` kept only servers with a client, but a sign-out can land between. - let Some(client) = plx_plex::plex::client_for(conn.sid) else { - conn.drop_link(); - conn.state = State::Idle; - continue; - }; - let key = (client.instance_gen(), client.token_gen()); - if key != conn.key { - // A re-pointed client or a new token: the old connection and its mailbox are - // forgotten. A 401 park was about the old token; any other park stays. - conn.drop_link(); - conn.key = key; - conn.attempts = 0; - if conn.state == State::Live { - conn.gap_owed = true; - } - if conn.state != State::Parked(Park::Session) { - conn.state = State::Idle; - } - } - let allowed = !*suspended - && !armed - && client.has_token() - && !client.data_io_disabled() - && gate(client); - if !allowed { + pass.pending = true; + Some(NotifyTurn { owner: self.owner.clone(), frame: pass.frame, index: pass.cursor, sid: conn.sid, key: conn.key, + addr: conn.link.as_ref().map(|l| l.addr), launch, settled: false }) + } + + fn check_turn(&self, pass: &NotifyPass, turn: &NotifyTurn) { + self.check_pass(pass); + assert!(pass.pending); + assert!(std::sync::Arc::ptr_eq(&self.owner, &turn.owner)); + assert_eq!((turn.frame, turn.index), (pass.frame, pass.cursor)); + let conn = &self.conns[turn.index]; + assert_eq!((turn.sid, turn.key), (conn.sid, conn.key)); + assert_eq!(turn.addr, conn.link.as_ref().map(|l| l.addr)); + } + + pub fn settle_admission(&mut self, pass: &mut NotifyPass, turn: &mut NotifyTurn) { + self.check_turn(pass, turn); + assert!(!turn.settled); + let conn = &mut self.conns[turn.index]; + if turn.launch { + let receipt = conn.link.as_ref().unwrap().take_admission().expect("runner must report Notify admission synchronously"); + assert_eq!(Some(receipt.addr), turn.addr); + assert_eq!(receipt.origin_frame, pass.frame); + if receipt.accepted { conn.state = State::Connecting; } + else { conn.drop_link(); - if conn.state == State::Live { - conn.gap_owed = true; - } - if matches!(conn.state, State::Connecting | State::Live) { - conn.state = State::Idle; - } - } else if conn.link.is_none() && conn.due(frame) { - let link = Link::new(conn.key); - if connector.connect(conn.sid, link.clone()) { - conn.link = Some(link); - conn.state = State::Connecting; - plx_base::eventlog::log(&format!("notify: connecting sid={}", conn.sid.raw())); - } else { - plx_base::eventlog::log(&format!("notify: sid={} could not start a worker", conn.sid.raw())); - conn.back_off(frame); - } + conn.back_off(receipt.origin_frame); + turn.addr = None; } - conn.drain(frame, resync, &mut out); } - out + turn.settled = true; } - /// Rebuild the connection table when the registry's roster or facts moved: one connection per - /// server this account OWNS, none for a server nothing has described (fail closed). - fn sync_roster(&mut self) { - let marker = (plx_plex::plex::server_roster_gen(), plx_plex::plex::server_facts_gen()); - if marker == self.seen { - return; - } - self.seen = marker; - let owned = |sid: ServerId| plx_plex::plex::server_facts(sid).is_some_and(|f| f.owned); - let (mut servers, mut described, mut owned_n) = (0, 0, 0); - for sid in plx_plex::plex::server_ids() { - servers += 1; - if let Some(facts) = plx_plex::plex::server_facts(sid) { - described += 1; - owned_n += usize::from(facts.owned); - } - } - plx_base::eventlog::log(&format!("notify: roster servers={servers} described={described} owned={owned_n}")); - self.conns.retain_mut(|c| { - let keep = owned(c.sid) && plx_plex::plex::client_for(c.sid).is_some(); - if !keep { - c.drop_link(); - } + pub fn finish_turn(&mut self, pass: &mut NotifyPass, turn: NotifyTurn, input: &NotifyInput<'_>) { + self.check_turn(pass, &turn); + assert!(turn.settled); + let conn = &mut self.conns[turn.index]; + if input.armed || !self.enabled || self.suspended || !owned(input.keys, conn.sid) + || input.keys.key(conn.sid) != Some(turn.key) || !input.keys.stream_allowed(conn.sid) { + stand_down(conn); + } else { conn.drain(pass.frame, &mut self.resync, &mut pass.out); } + pass.pending = false; + pass.cursor += 1; + } + + pub fn end(&mut self, pass: NotifyPass) -> NotifyOut { + self.check_pass(&pass); + assert!(!pass.pending && pass.cursor == pass.count); + assert!(self.jobs.is_empty(), "undrained Notify jobs"); + self.active = None; + pass.out + } + + /// Reconcile every captured fill, including eligibility changes without generation moves. + /// Retain connection order; append newly described owned slots in supplied slot order. + fn sync_roster(&mut self, keys: &ClientKeys) { + self.seen = (keys.roster_gen(), keys.facts_gen()); + self.conns.retain_mut(|conn| { + let keep = owned(keys, conn.sid); + if !keep { conn.drop_link(); } keep }); - for sid in plx_plex::plex::server_ids() { - if self.conns.iter().any(|c| c.sid == sid) || !owned(sid) { - continue; + for conn in &mut self.conns { + let key = keys.key(conn.sid).unwrap(); + if key != conn.key { + conn.drop_link(); + conn.key = key; + conn.attempts = 0; + if conn.state == State::Live { conn.gap_owed = true; } + if conn.state != State::Parked(Park::Session) { conn.state = State::Idle; } } - let Some(client) = plx_plex::plex::client_for(sid) else { continue }; - self.conns.push(Conn { - sid, - key: (client.instance_gen(), client.token_gen()), - link: None, - state: State::Idle, - gap_owed: false, - resynced_unstable: false, - live_since: 0, - attempts: 0, - }); + } + for sid in keys.slots() { + if !owned(keys, sid) || self.conns.iter().any(|c| c.sid == sid) { continue; } + self.conns.push(Conn { sid, key: keys.key(sid).unwrap(), link: None, + state: State::Idle, gap_owed: false, resynced_unstable: false, + live_since: 0, attempts: 0 }); } } /// WILL background: every connection is cancelled now and none opens until [`Self::resume`]. pub fn suspend(&mut self) { + assert!(self.active.is_none(), "Notify pass in progress"); plx_base::eventlog::log(&format!("notify: suspend conns={}", self.conns.len())); self.suspended = true; for conn in &mut self.conns { @@ -481,6 +438,7 @@ impl NotifyStore { /// DID foreground: connections may open again, and a pending backoff is forgiven. A parked /// server stays parked. pub fn resume(&mut self) { + assert!(self.active.is_none(), "Notify pass in progress"); plx_base::eventlog::log(&format!("notify: resume conns={}", self.conns.len())); self.suspended = false; for conn in &mut self.conns { @@ -494,6 +452,7 @@ impl NotifyStore { /// The network may have changed under every open connection: cancel them all and reconnect. /// Backoff is forgiven; a parked server stays parked (a refusal is not about the network). pub fn network_changed(&mut self) { + assert!(self.active.is_none(), "Notify pass in progress"); plx_base::eventlog::log(&format!("notify: network changed conns={}", self.conns.len())); for conn in &mut self.conns { conn.drop_link(); @@ -508,6 +467,8 @@ impl NotifyStore { /// A profile reset: every connection is closed and forgotten, parks included, and the table /// is rebuilt from the registry on the next pump. pub fn reset(&mut self) { + assert!(self.active.is_none(), "Notify pass in progress"); + self.jobs.clear(); self.close_all(); self.conns.clear(); self.resync.clear(); diff --git a/rust-modules/data/src/stores/notify_job.rs b/rust-modules/data/src/stores/notify_job.rs new file mode 100644 index 0000000000..bbfaeeaa45 --- /dev/null +++ b/rust-modules/data/src/stores/notify_job.rs @@ -0,0 +1,99 @@ +//! Addressed Notify stream channel and off-frame body. Scheduling belongs to the app. +use plx_machine::{job::Port, machine::Addr}; +use plx_plex::plex::{Client, ClientKey, ServerId, notify::{Notification, NotifyEnd}}; +use plx_base::task::OffFrame; +use std::sync::{Arc, Mutex, atomic::{AtomicBool, Ordering}}; + +#[derive(Clone, Debug, PartialEq)] +pub struct NotifyStreamReq { pub sid: ServerId, pub key: ClientKey } +#[derive(Debug)] +pub enum NotifyTerminal { End(NotifyEnd), NotStarted } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NotifyAdmission { pub addr: Addr, pub origin_frame: u64, pub accepted: bool } + +#[derive(Default)] +pub(super) struct Inbox { + pub events: Vec<(ClientKey, Notification)>, + pub end: Option, + terminal_sent: bool, + admission: Option, + admission_sent: bool, +} +#[derive(Clone)] +pub(super) struct Link { + pub addr: Addr, pub key: ClientKey, origin_frame: u64, + cancel: Arc, inbox: Arc>, +} +impl Link { + pub fn new(addr: Addr, key: ClientKey, origin_frame: u64) -> Self { + Self { addr, key, origin_frame, cancel: Arc::default(), inbox: Arc::default() } + } + pub fn lock(&self) -> std::sync::MutexGuard<'_, Inbox> { + self.inbox.lock().unwrap_or_else(|e| e.into_inner()) + } + pub fn take_admission(&self) -> Option { self.lock().admission.take() } + pub fn cancel(&self) { + self.cancel.store(true, Ordering::Release); + self.finish(NotifyEnd::Cancelled); + } + pub fn is_cancelled(&self) -> bool { self.cancel.load(Ordering::Acquire) } + pub fn push(&self, n: Notification) { + let mut inbox = self.lock(); + if !inbox.terminal_sent { inbox.events.push((self.key, n)); } + } + pub fn finish(&self, end: NotifyEnd) { Self::finish_locked(&mut self.lock(), end); } + fn finish_locked(inbox: &mut Inbox, end: NotifyEnd) { + if !inbox.terminal_sent { inbox.terminal_sent = true; inbox.end = Some(end); } + } +} +#[derive(Clone)] +pub struct NotifyStreamPort { pub(super) link: Link } +impl std::fmt::Debug for NotifyStreamPort { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NotifyStreamPort").field("addr", &self.link.addr).field("key", &self.link.key).finish() + } +} +impl PartialEq for NotifyStreamPort { + fn eq(&self, other: &Self) -> bool { self.link.addr == other.link.addr && self.link.key == other.link.key } +} +impl NotifyStreamPort { + pub(super) fn new(link: Link) -> Self { Self { link } } + pub fn is_cancelled(&self) -> bool { self.link.is_cancelled() } + pub fn cancel_flag(&self) -> &AtomicBool { &self.link.cancel } + pub fn post(&self, addr: Addr, n: Notification) { + if addr == self.link.addr { self.link.push(n); } + } + /// INITIAL scheduling receipt only. Held acceptance is final, even after take. + /// A later release failure must use done(NotStarted), never a second receipt. + pub fn admit(&self, addr: Addr, accepted: bool) { + if addr != self.link.addr { return; } + let mut inbox = self.link.lock(); + if inbox.admission_sent { return; } + inbox.admission_sent = true; + inbox.admission = Some(NotifyAdmission { addr, origin_frame: self.link.origin_frame, accepted }); + if !accepted { + Link::finish_locked(&mut inbox, if self.is_cancelled() { NotifyEnd::Cancelled } else { NotifyEnd::Transport }); + } + } +} +impl Port for NotifyStreamPort { + fn done(&self, addr: Addr, out: NotifyTerminal) { + if addr != self.link.addr { return; } + self.link.finish(match out { NotifyTerminal::End(end) => end, NotifyTerminal::NotStarted => NotifyEnd::Transport }); + } + fn refused(&self, addr: Addr) { self.admit(addr, false); } +} + +pub fn finish_notify_stream_job(port: &NotifyStreamPort, addr: Addr, client: &'static Client, + req: NotifyStreamReq, off: &OffFrame) { + if addr != port.link.addr || req.key != port.link.key { return; } + let end = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if port.is_cancelled() { return NotifyEnd::Cancelled; } + if client.id() != req.sid || ClientKey::of(client) != req.key || !client.has_token() + || client.data_io_disabled() || !client.credential_transport_open() { + return NotifyEnd::Transport; + } + client.notifications(off, port.cancel_flag(), &mut |n| port.post(addr, n)) + })).unwrap_or(NotifyEnd::Transport); + port.done(addr, NotifyTerminal::End(end)); +} diff --git a/rust-modules/data/src/stores/notify_tests.rs b/rust-modules/data/src/stores/notify_tests.rs index aafd0530a2..1db88e049e 100644 --- a/rust-modules/data/src/stores/notify_tests.rs +++ b/rust-modules/data/src/stores/notify_tests.rs @@ -1,8 +1,9 @@ -//! The notification store against a fake connector: when it connects, when it does not, what a -//! reconnect owes, and that nothing here ever opens a real stream. +//! Scripted addressed admission drives the actual pure cursor phases; no stream is opened. use super::*; -use crate::stores::Stores; +use plx_machine::job::Port; +use std::sync::{Arc, Mutex}; +use crate::stores::notify_job::NotifyTerminal; use plx_plex::plex::notify::PlayState; use plx_plex::plex::{self, GrantEvidence}; @@ -28,16 +29,36 @@ impl Fake { } } -impl Connector for Fake { - fn connect(&mut self, sid: ServerId, link: Link) -> bool { - let mut log = self.lock(); - if log.refuse { - return false; +impl Fake { + fn admit_jobs(&self, store: &mut NotifyStore) { + for job in store.take_jobs() { + let StoreJob::NotifyStream(job, port) = job else { panic!("not Notify") }; + let mut log = self.lock(); + if log.refuse { port.refused(job.addr); } + else { + log.links.push((job.req.sid, port.link.clone())); + port.admit(job.addr, true); + } } - log.links.push((sid, link)); - true } } +fn capture(gate: fn(&plex::Client) -> bool) -> ClientKeys { + let mut keys = ClientKeys::default(); + keys.refill_roster_with_stream_policy(plex::current_server(), plex::server_roster_gen(), + plex::server_facts_gen(), plex::server_ids(), plex::client_for, plex::server_facts, + plex::server_probe_result, |c| c.has_token() && !c.data_io_disabled() && gate(c)); + keys +} +fn drive(store: &mut NotifyStore, fake: &Fake, keys: &ClientKeys, armed: bool) -> NotifyOut { + let input = NotifyInput { keys, armed }; + let mut pass = store.begin(&input); + while let Some(mut turn) = store.prepare_next(&mut pass, &input) { + fake.admit_jobs(store); + store.settle_admission(&mut pass, &mut turn); + store.finish_turn(&mut pass, turn, &input); + } + store.end(pass) +} /// The registry is process-global: hold its lock for the test and empty it on the way out. struct Rig { @@ -45,6 +66,7 @@ struct Rig { store: NotifyStore, fake: Fake, sid: ServerId, + gate: fn(&plex::Client) -> bool, } impl Drop for Rig { @@ -62,8 +84,8 @@ fn rig_with(token: &str, evidence: Option) -> Rig { plex::describe_server(sid, "Server A", "", evidence); } let fake = Fake::default(); - let store = NotifyStore::with_connector_for_test(Box::new(fake.clone())); - Rig { _lock: lock, store, fake, sid } + let store = NotifyStore::default(); + Rig { _lock: lock, store, fake, sid, gate: open_gate } } fn rig() -> Rig { @@ -72,7 +94,8 @@ fn rig() -> Rig { impl Rig { fn pump(&mut self) -> NotifyOut { - self.store.pump(false) + let keys = capture(self.gate); + drive(&mut self.store, &self.fake, &keys, false) } /// Pump `n` frames, discarding the output. fn pump_n(&mut self, n: u64) { @@ -297,10 +320,10 @@ fn open_gate(_: &plex::Client) -> bool { #[test] fn a_refused_origin_is_not_dialled_and_reconnects_when_the_grant_returns() { let mut r = rig(); - r.store.set_gate_for_test(refuse_gate); + r.gate = refuse_gate; r.pump_n(70 * FRAMES_PER_S); assert_eq!(r.fake.connects(), 0, "a plaintext origin with no live grant is not dialled"); - r.store.set_gate_for_test(open_gate); + r.gate = open_gate; r.pump_n(1); assert_eq!(r.fake.connects(), 1, "discovery re-minting the grant brings the feed back"); } @@ -311,10 +334,10 @@ fn a_gate_refusal_mid_stream_cancels_and_recovers() { r.pump_n(1); r.fake.link(0).push(Notification::Ping); r.pump_n(1); - r.store.set_gate_for_test(refuse_gate); + r.gate = refuse_gate; r.pump_n(1); assert!(r.fake.link(0).is_cancelled()); - r.store.set_gate_for_test(open_gate); + r.gate = open_gate; r.pump_n(1); assert_eq!(r.fake.connects(), 2); r.fake.link(1).push(Notification::Ping); @@ -413,20 +436,15 @@ fn a_5xx_is_retried_with_backoff() { #[test] fn off_when_gate_armed() { - let r = rig(); - let mut stores = Stores::default(); - stores.notify = NotifyStore::with_connector_for_test(Box::new(r.fake.clone())); - stores.landgate.arm_recording(); - let _ = stores.notify_pump(); - let _ = stores.notify_pump(); + let mut r = rig(); + let keys = capture(open_gate); + drive(&mut r.store, &r.fake, &keys, true); + drive(&mut r.store, &r.fake, &keys, true); assert_eq!(r.fake.connects(), 0, "a recording owns the run: no live stream"); - stores.landgate.disarm(); - let _ = stores.notify_pump(); + r.pump(); assert_eq!(r.fake.connects(), 1); - stores.landgate.arm_replay(vec![vec![(1, 1)]]); - let _ = stores.notify_pump(); + drive(&mut r.store, &r.fake, &keys, true); assert!(r.fake.link(0).is_cancelled(), "arming mid-stream cancels the connection"); - stores.landgate.disarm(); } #[test] @@ -457,9 +475,10 @@ fn enabled_const_connects_to_an_owned_server() { let sid = plex::register_for_test("machine-a", "127.0.0.1", 9, "tok", "cid-own"); plex::describe_server(sid, "Server A", "", GrantEvidence::ours()); let fake = Fake::default(); - let mut store = NotifyStore::new(NOTIFY_ENABLED, Box::new(fake.clone())); + let mut store = NotifyStore::new(NOTIFY_ENABLED); for _ in 0..10 { - let _ = store.pump(false); + let keys = capture(open_gate); + let _ = drive(&mut store, &fake, &keys, false); } assert_eq!(fake.connects(), 1); plex::reset_servers_for_test(); @@ -502,7 +521,7 @@ fn drop_cancels() { r.pump_n(1); let link = r.fake.link(0); assert!(!link.is_cancelled()); - let old = std::mem::replace(&mut r.store, NotifyStore::new(true, Box::new(Fake::default()))); + let old = std::mem::replace(&mut r.store, NotifyStore::new(true)); drop(old); assert!(link.is_cancelled(), "dropping the store cancels every worker"); } @@ -584,3 +603,237 @@ fn a_worker_ending_after_a_cancel_is_never_read() { r.pump_n(1); assert_eq!(r.fake.connects(), 2, "the cancelled connection's end cannot park its successor"); } + +#[test] +fn initial_refusal_is_same_frame_and_exactly_one_backoff() { + let mut r = rig(); + r.fake.lock().refuse = true; + r.pump(); + let frame = r.store.frame; + let jitter = frame.wrapping_add(u64::from(r.sid.raw()) * 7919) % 30; + assert_eq!(r.store.conns[0].attempts, 1); + assert_eq!(r.store.conns[0].state, State::Backoff { until: frame + 120 + jitter }); + assert!(r.store.conns[0].link.is_none()); + assert!(!r.store.has_jobs()); + r.fake.lock().refuse = false; + r.pump_n(119 + jitter); + assert_eq!(r.fake.connects(), 0); + r.pump(); + assert_eq!(r.fake.connects(), 1); +} + +#[test] +fn terminal_and_admission_latches_survive_take_and_wrong_addresses() { + let mut r = rig(); + r.pump(); + let link = r.fake.link(0); + let port = NotifyStreamPort::new(link.clone()); + let bad = Addr { req: RequestId(link.addr.req.0 + 1), ..link.addr }; + port.post(bad, Notification::Ping); + port.done(bad, NotifyTerminal::End(NotifyEnd::Unsupported)); + port.refused(bad); + assert!(link.lock().events.is_empty()); + assert!(link.lock().end.is_none()); + // Accepted receipt was already taken by settlement; it cannot be overwritten. + port.refused(link.addr); + assert!(link.take_admission().is_none()); + port.post(link.addr, Notification::Ping); + port.done(link.addr, NotifyTerminal::End(NotifyEnd::Closed)); + assert_eq!(link.lock().events.len(), 1); + assert!(link.lock().end.take().is_some()); + port.done(link.addr, NotifyTerminal::NotStarted); + port.post(link.addr, Notification::Ping); + assert!(link.lock().end.is_none()); + assert_eq!(link.lock().events.len(), 1); +} + +#[test] +fn accepted_held_release_failure_uses_observation_frame_and_preserves_attempts() { + let mut r = rig(); + r.fake.lock().refuse = true; + r.pump(); + r.store.resume(); // existing deliberate forgiveness; seed a nonzero escalation explicitly. + r.store.conns[0].attempts = 3; + r.fake.lock().refuse = false; + r.pump(); + let link = r.fake.link(0); + assert_eq!(r.store.conns[0].attempts, 3); + r.pump_n(9); + let port = NotifyStreamPort::new(link.clone()); + port.done(link.addr, NotifyTerminal::NotStarted); + r.pump(); + let frame = r.store.frame; + let delay = 16 * FRAMES_PER_S; + let jitter = frame.wrapping_add(u64::from(r.sid.raw()) * 7919) % (delay / 4); + assert_eq!(r.store.conns[0].attempts, 4); + assert_eq!(r.store.conns[0].state, State::Backoff { until: frame + delay + jitter }); + port.done(link.addr, NotifyTerminal::NotStarted); + r.pump(); + assert_eq!(r.store.conns[0].attempts, 4); +} + +#[test] +fn armed_and_disabled_repair_connecting_and_live_without_losing_retry_or_gap_debt() { + for live in [false, true] { + for disable in [false, true] { + let mut r = rig(); + r.pump(); + let old = r.fake.link(0); + if live { old.push(Notification::Ping); r.pump(); } + r.store.conns[0].attempts = 2; + old.push(play("stale", PlayState::Playing)); + if disable { r.store.enabled = false; } + let keys = capture(open_gate); + let out = drive(&mut r.store, &r.fake, &keys, !disable); + assert_eq!(out, NotifyOut::default()); + assert!(old.is_cancelled()); + assert!(r.store.conns[0].link.is_none()); + assert_eq!(r.store.conns[0].state, State::Idle); + assert_eq!(r.store.conns[0].gap_owed, live); + assert_eq!(r.store.conns[0].attempts, 2); + r.store.enabled = true; + r.pump(); + assert_eq!(r.fake.connects(), 2); + r.fake.link(1).push(Notification::Ping); + assert_eq!(r.pump().resynced.len(), usize::from(live)); + } + } +} + +#[test] +fn cursor_drains_a_before_launching_b_and_never_scoops_a_twice() { + let mut r = rig(); + r.pump(); + let a = r.fake.link(0); + a.push(play("early", PlayState::Playing)); + let b = plex::register_for_test("machine-b", "127.0.0.2", 9, "tok", "cid"); + plex::describe_server(b, "B", "", GrantEvidence::ours()); + let keys = capture(open_gate); + let input = NotifyInput { keys: &keys, armed: false }; + let mut pass = r.store.begin(&input); + let mut turn = r.store.prepare_next(&mut pass, &input).unwrap(); + assert!(!r.store.has_jobs()); + r.store.settle_admission(&mut pass, &mut turn); + r.store.finish_turn(&mut pass, turn, &input); + assert_eq!(pass.out.plays[0].sid, r.sid); + let mut turn = r.store.prepare_next(&mut pass, &input).unwrap(); + a.push(play("late", PlayState::Paused)); + let StoreJob::NotifyStream(job, port) = r.store.take_jobs().pop().unwrap() else { panic!() }; + port.admit(job.addr, true); + port.post(job.addr, play("b", PlayState::Paused)); + port.done(job.addr, NotifyTerminal::End(NotifyEnd::Closed)); + r.store.settle_admission(&mut pass, &mut turn); + r.store.finish_turn(&mut pass, turn, &input); + assert!(r.store.prepare_next(&mut pass, &input).is_none()); + let out = r.store.end(pass); + assert_eq!(out.plays.iter().map(|p| p.sid).collect::>(), [r.sid, b]); + assert_eq!(out.plays[0].sessions.len(), 1); + assert_eq!(r.pump().plays[0].sessions[0].rating_key.as_deref(), Some("late")); +} + +#[test] +fn cancelled_reset_port_cannot_mutate_replacement_and_request_ids_never_recycle() { + let mut r = rig(); + r.pump(); + let old = r.fake.link(0); + r.store.reset(); + r.pump(); + let new = r.fake.link(1); + assert_ne!(old.addr, new.addr); + let port = NotifyStreamPort::new(old.clone()); + port.done(old.addr, NotifyTerminal::NotStarted); + port.post(old.addr, play("orphan", PlayState::Stopped)); + port.refused(old.addr); + assert_eq!(r.pump(), NotifyOut::default()); + assert_eq!(r.store.conns[0].attempts, 0); + assert_eq!(r.store.conns[0].state, State::Connecting); + assert!(old.lock().end.take().is_some()); + port.done(old.addr, NotifyTerminal::NotStarted); + assert!(old.lock().end.is_none()); +} + +#[test] +fn concurrent_terminal_cancel_and_post_remain_terminal_once() { + let mut r = rig(); r.pump(); + let link = r.fake.link(0); + let a = link.clone(); let b = link.clone(); + let one = std::thread::spawn(move || a.finish(NotifyEnd::Closed)); + let two = std::thread::spawn(move || { b.push(Notification::Ping); b.cancel(); }); + one.join().unwrap(); two.join().unwrap(); + assert!(link.lock().end.take().is_some()); + let before = link.lock().events.len(); + link.finish(NotifyEnd::Transport); link.push(Notification::Ping); + assert!(link.lock().end.is_none()); + assert_eq!(link.lock().events.len(), before); +} + +#[test] +fn stand_down_preserves_backoff_deadlines_and_both_parks() { + for state in [State::Backoff { until: 20 }, State::Parked(Park::Token), State::Parked(Park::Session)] { + let mut r = rig(); r.pump(); + r.store.conns[0].state = state; + r.store.conns[0].attempts = 4; + r.store.conns[0].live_since = 3; + r.store.conns[0].resynced_unstable = true; + let keys = capture(open_gate); + drive(&mut r.store, &r.fake, &keys, true); + assert_eq!(r.store.conns[0].state, state); + assert_eq!(r.store.conns[0].attempts, 4); + assert_eq!(r.store.conns[0].live_since, 3); + assert!(r.store.conns[0].resynced_unstable); + r.pump(); + assert_eq!(r.fake.connects(), 1); + if matches!(state, State::Backoff { .. }) { + r.pump_n(16); + assert_eq!(r.fake.connects(), 1); + r.pump(); // frame 20, not frame 21 or an extra settlement frame. + assert_eq!(r.fake.connects(), 2); + assert_eq!(r.store.conns[0].attempts, 4); + } + } +} + +#[test] +fn stable_deferred_gap_is_paid_once_before_terminal_in_same_batch() { + let mut r = rig(); r.pump(); + let link = r.fake.link(0); + link.push(Notification::Ping); r.pump(); + let live_since = r.store.conns[0].live_since; + r.store.conns[0].attempts = 5; + r.store.conns[0].gap_owed = true; + r.store.conns[0].resynced_unstable = true; + r.store.frame = live_since + STABLE_FRAMES - 1; + link.push(play("stable", PlayState::Playing)); + link.finish(NotifyEnd::Closed); + let out = r.pump(); + assert_eq!(out.resynced, [(r.sid, 1)]); + assert_eq!(out.plays.len(), 1); + assert_eq!(r.store.conns[0].attempts, 1, "stability resets before terminal backoff"); + assert!(r.store.conns[0].gap_owed, "terminal after stability owes the next gap"); +} + +#[test] +fn release_input_fence_discards_mail_and_classification_is_direct_notify_only() { + let mut r = rig(); + let keys = capture(open_gate); + let input = NotifyInput { keys: &keys, armed: false }; + let mut pass = r.store.begin(&input); + let mut turn = r.store.prepare_next(&mut pass, &input).unwrap(); + let job = r.store.take_jobs().pop().unwrap(); + assert_eq!(job.store(), StoreId::Notify); + assert_eq!(job.admission(), super::super::Admission::Direct); + let StoreJob::NotifyStream(job, port) = job else { panic!() }; + assert_eq!(job.addr.to, MachineId::Store(StoreId::Notify.ord())); + assert_eq!(job.req.key, keys.key(r.sid).unwrap()); + port.admit(job.addr, true); + port.post(job.addr, play("stale", PlayState::Playing)); + r.store.settle_admission(&mut pass, &mut turn); + r.client().set_token("changed-during-runner"); + let changed = capture(open_gate); + r.store.finish_turn(&mut pass, turn, &NotifyInput { keys: &changed, armed: false }); + assert_eq!(r.store.end(pass), NotifyOut::default()); + assert!(port.is_cancelled()); + assert_eq!(r.store.conns[0].state, State::Idle); + r.pump(); + assert_eq!(r.store.conns[0].attempts, 0); +} diff --git a/rust-modules/plex/src/plex/client_keys.rs b/rust-modules/plex/src/plex/client_keys.rs index 34d7bc19eb..50c2148fe9 100644 --- a/rust-modules/plex/src/plex/client_keys.rs +++ b/rust-modules/plex/src/plex/client_keys.rs @@ -56,11 +56,13 @@ pub struct ClientFacts { pub described: Option<&'static super::ServerFacts>, /// None rejects an incoherent observation; Some((None, tier)) means not probed. pub source: Option<(Option, Option)>, + pub stream_allowed: bool, } impl PartialEq for ClientFacts { fn eq(&self, other: &Self) -> bool { self.machine_id == other.machine_id && self.source == other.source + && self.stream_allowed == other.stream_allowed && match (self.described, other.described) { (None, None) => true, (Some(a), Some(b)) => a.name == b.name && a.handle == b.handle @@ -73,7 +75,8 @@ impl Eq for ClientFacts {} impl std::fmt::Debug for ClientFacts { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ClientFacts").field("machine_id", &self.machine_id) - .field("described", &self.described.is_some()).field("source", &self.source).finish() + .field("described", &self.described.is_some()).field("source", &self.source) + .field("stream_allowed", &self.stream_allowed).finish() } } impl Default for ClientKeys { @@ -120,11 +123,26 @@ impl ClientKeys { /// reader runs before tier and the second eligible lookup checks instance/token/grant again. /// A raced row keeps its registered slot and evidence but has no active key or source pair. pub fn refill_roster( + &mut self, current: ServerId, roster_gen: u32, facts_gen: u32, + slots: impl IntoIterator, + lookup: impl FnMut(ServerId) -> Option<&'static Client>, + describe: impl FnMut(ServerId) -> Option<&'static super::ServerFacts>, + probe: impl FnMut(ServerId) -> Option, + ) { + self.refill_roster_with_stream_policy(current, roster_gen, facts_gen, slots, lookup, describe, probe, |_| false); + } + + pub fn stream_allowed(&self, sid: ServerId) -> bool { + self.key(sid).is_some() && self.facts(sid).is_some_and(|f| f.stream_allowed) + } + + pub fn refill_roster_with_stream_policy( &mut self, current: ServerId, roster_gen: u32, facts_gen: u32, slots: impl IntoIterator, mut lookup: impl FnMut(ServerId) -> Option<&'static Client>, mut describe: impl FnMut(ServerId) -> Option<&'static super::ServerFacts>, mut probe: impl FnMut(ServerId) -> Option, + mut stream_allowed: impl FnMut(&Client) -> bool, ) { self.0.clear(); self.2 = current; @@ -133,7 +151,7 @@ impl ClientKeys { let mut count = 0; for sid in slots { if count == self.1.len() { - self.1.push((sid, ClientFacts { machine_id: String::new(), described: None, source: None })); + self.1.push((sid, ClientFacts { machine_id: String::new(), described: None, source: None, stream_allowed: false })); } let row = &mut self.1[count]; row.0 = sid; @@ -146,10 +164,12 @@ impl ClientKeys { }; if !unchanged { row.1.described = described; } row.1.source = None; + row.1.stream_allowed = false; if let Some(client) = lookup(sid) { let key = ClientKey::of(client); let outcome = probe(sid); let tier = client.link(); + let allowed = stream_allowed(client); if lookup(sid).is_some_and(|now| ClientKey::of(now) == key) && ClientKey::of(client) == key { self.0.push((sid, key)); @@ -158,6 +178,7 @@ impl ClientKeys { row.1.machine_id.push_str(client.machine_id()); } row.1.source = Some((outcome, tier)); + row.1.stream_allowed = allowed; } else { row.1.machine_id.clear(); } diff --git a/rust-modules/plex/src/plex/client_keys_roster_tests.rs b/rust-modules/plex/src/plex/client_keys_roster_tests.rs index 42c6afd782..90e98750c2 100644 --- a/rust-modules/plex/src/plex/client_keys_roster_tests.rs +++ b/rust-modules/plex/src/plex/client_keys_roster_tests.rs @@ -127,3 +127,51 @@ fn capture_remains_a_cloneable_send_sync_value_not_a_client_handle() { value::(); assert_eq!(ClientKeys::default().iter().count(), 0); } + +#[test] +fn stream_policy_is_fresh_each_fill_fail_closed_and_reuses_s4_buffers() { + let a = client(61); + let mut keys = ClientKeys::default(); + assert!(!ClientKeys::of([(a.id(), a)]).stream_allowed(a.id())); + keys.refill_roster(a.id(), 1, 1, [a.id()], |_| Some(a), |_| None, |_| None); + assert!(!keys.stream_allowed(a.id()), "compatibility capture cannot authorize streaming"); + let buffers = (keys.0.as_ptr(), keys.1.as_ptr()); + let string = keys.facts(a.id()).unwrap().machine_id.as_ptr(); + let mut previous = keys.clone(); + for allowed in [true, false, true, false] { + keys.refill_roster_with_stream_policy(a.id(), 1, 1, [a.id()], |_| Some(a), + |_| None, |_| None, |_| allowed); + assert_eq!(keys.stream_allowed(a.id()), allowed); + assert_eq!(keys.facts(a.id()).unwrap().stream_allowed, allowed); + assert_eq!((keys.0.as_ptr(), keys.1.as_ptr()), buffers); + assert_eq!(keys.facts(a.id()).unwrap().machine_id.as_ptr(), string); + assert_ne!(keys, previous, "stream policy participates in equality without generation moves"); + previous = keys.clone(); + } + assert!(!keys.stream_allowed(ServerId::UNSET)); +} + +#[test] +fn stream_policy_runs_after_probe_and_incoherent_capture_clears_permission() { + let a = client(62); + let stage = std::cell::Cell::new(0); + let mut reads = 0; + let mut keys = ClientKeys::default(); + keys.refill_roster_with_stream_policy(a.id(), 1, 1, [a.id()], |_| { + reads += 1; + (reads == 1).then_some(a) + }, |_| None, |_| { stage.set(1); None }, |_| { + assert_eq!(stage.get(), 1); + stage.set(2); + true + }); + assert_eq!(stage.get(), 2); + assert_eq!(keys.key(a.id()), None); + assert!(!keys.stream_allowed(a.id())); + assert!(!keys.facts(a.id()).unwrap().stream_allowed); + // A credential identity move inside the policy fence rejects the same row. + keys.refill_roster_with_stream_policy(a.id(), 1, 1, [a.id()], |_| Some(a), + |_| None, |_| None, |c| { c.set_token("changed-in-policy"); true }); + assert_eq!(keys.key(a.id()), None); + assert!(!keys.stream_allowed(a.id())); +} diff --git a/rust-modules/src/app/adapters/notify_runner_tests.rs b/rust-modules/src/app/adapters/notify_runner_tests.rs new file mode 100644 index 0000000000..53520b260e --- /dev/null +++ b/rust-modules/src/app/adapters/notify_runner_tests.rs @@ -0,0 +1,163 @@ +//! Finite runner ports drive the production cursor; no account, socket, or worker is needed. +use super::*; +use plx_data::stores::notify::{NotifyInput, NotifyOut, NotifyStore}; +use plx_plex::plex::{Origin, ServerFacts, ServerId}; +use plx_plex::plex::notify::{Notification, NotifyEnd}; + +struct Rig { store: NotifyStore, keys: ClientKeys, client: &'static Client, frame: u64 } +impl Rig { + fn new() -> Self { + let client: &'static Client = Box::leak(Box::new(Client::new(ServerId::from_raw(71), "notify-runner", + Origin::http("127.0.0.1", 9), "script-token", "script-client"))); + let facts: &'static ServerFacts = Box::leak(Box::new(ServerFacts { + name: "Script".into(), handle: String::new(), owned: true, home: false, owner_id: 0, + })); + let mut keys = ClientKeys::default(); + keys.refill_roster_with_stream_policy(client.id(), 1, 1, [client.id()], |_| Some(client), + |_| Some(facts), |_| None, |_| true); + Self { store: NotifyStore::default(), keys, client, frame: 0 } + } + fn tick(&mut self, mut start: impl FnMut(StoreJob)) -> NotifyOut { + self.frame += 1; + let input = NotifyInput { keys: &self.keys, armed: false }; + let mut pass = self.store.begin(&input); + while let Some(mut turn) = self.store.prepare_next(&mut pass, &input) { + for job in self.store.take_jobs() { start(job); } + self.store.settle_admission(&mut pass, &mut turn); + self.store.finish_turn(&mut pass, turn, &input); + } + let out = self.store.end(pass); + assert!(!self.store.has_jobs()); + out + } + fn quiet_until(&mut self, frame: u64) { + while self.frame + 1 < frame { self.tick(|_| panic!("retry before deadline")); } + } +} +fn delay(frame: u64, sid: ServerId, attempts: u32) -> u64 { + let d = (2u64 << attempts.min(8)).min(60) * 60; + frame + d + frame.wrapping_add(u64::from(sid.raw()) * 7919) % (d / 4).max(1) +} + +#[test] +fn initial_spawn_false_is_exactly_once_at_origin_frame_with_exact_jitter() { + let mut r = Rig::new(); + let c = r.client; + let mut runner = StoreRunner::live(); + let mut launches = 0; + r.tick(|job| { + let StoreJob::NotifyStream(job, port) = job else { panic!("Notify") }; + let late = port.clone(); let addr = job.addr; + runner.start_notify_stream_with(job, port, |_| Some(c), |_, _, _| { launches += 1; false }); + late.refused(addr); // repeated initial refusal cannot double-backoff + late.done(addr, NotifyTerminal::NotStarted); + }); + assert_eq!(launches, 1); + let until = delay(1, c.id(), 0); + r.quiet_until(until); + r.tick(|job| { + let StoreJob::NotifyStream(job, port) = job else { panic!("Notify") }; + runner.start_notify_stream_with(job, port, |_| Some(c), |_, _, _| { launches += 1; false }); + }); + assert_eq!(launches, 2); + r.quiet_until(delay(until, c.id(), 1)); +} + +#[test] +fn accepted_held_release_false_is_ordinary_current_frame_and_preserves_attempts() { + let mut r = Rig::new(); let c = r.client; + r.tick(|job| { let StoreJob::NotifyStream(job, port) = job else { panic!("Notify") }; port.refused(job.addr); }); + let first = delay(1, c.id(), 0); + r.quiet_until(first); + let mut held = StoreRunner::held_all(); + r.tick(|job| { + let StoreJob::NotifyStream(job, port) = job else { panic!("Notify") }; + held.start_notify_stream_with(job, port, |_| Some(c), |_, _, _| panic!("park, not launch")); + }); + for _ in 0..7 { r.tick(|_| panic!("held acceptance is not completion")); } + let jobs = held.take_held(); assert_eq!(jobs.len(), 1); + let StoreJob::NotifyStream(job, port) = jobs.into_iter().next().unwrap() else { panic!("Notify") }; + StoreRunner::release_notify_stream_with(job, port, |_| Some(c), |_, _, _| false); + r.tick(|_| panic!("terminal is observed, not retried in this frame")); + let observed = r.frame; + let until = delay(observed, c.id(), 1); + r.quiet_until(until); + let mut emitted = 0; + r.tick(|job| { + emitted += 1; + let StoreJob::NotifyStream(job, port) = job else { panic!("Notify") }; port.refused(job.addr); + }); + assert_eq!(emitted, 1); +} + +#[test] +fn cancellation_and_orphan_release_never_launch_or_poison_replacement() { + let mut r = Rig::new(); let c = r.client; + let mut held = StoreRunner::held_all(); + r.tick(|job| { + let StoreJob::NotifyStream(job, port) = job else { panic!("Notify") }; + held.start_notify_stream_with(job, port, |_| Some(c), |_, _, _| panic!("held")); + }); + let StoreJob::NotifyStream(old, port) = held.take_held().pop().unwrap() else { panic!("Notify") }; + let old_addr = old.addr; + r.store.reset(); + assert!(port.is_cancelled()); + StoreRunner::release_notify_stream_with(old, port.clone(), |_| panic!("cancel before lookup"), |_, _, _| panic!("cancel before launch")); + r.tick(|job| { + let StoreJob::NotifyStream(job, fresh) = job else { panic!("Notify") }; + assert!(job.addr.req.0 > old_addr.req.0); + fresh.admit(job.addr, true); + fresh.post(job.addr, Notification::Ping); + port.post(old_addr, Notification::Ping); + port.done(old_addr, NotifyTerminal::End(NotifyEnd::Closed)); + }); + r.tick(|_| panic!("old terminal cannot retry the new live connection")); + r.store.suspend(); +} + +#[test] +fn held_release_rechecks_full_key_even_when_generations_do_not_move() { + thread_local! { static CANDIDATE: std::cell::Cell> = const { std::cell::Cell::new(None) }; } + struct CandidateGuard(Option<&'static Client>); + impl Drop for CandidateGuard { + fn drop(&mut self) { CANDIDATE.with(|slot| slot.set(self.0)); } + } + let mut r = Rig::new(); let c = r.client; + let _candidate_guard = CandidateGuard(CANDIDATE.with(|slot| slot.replace(Some(c)))); + let captured_keys = r.keys.clone(); + let mut held = StoreRunner::held_all(); + r.tick(|job| { + let StoreJob::NotifyStream(job, port) = job else { panic!("Notify") }; + held.start_notify_stream_with(job, port, |_| Some(c), |_, _, _| panic!("held")); + }); + let StoreJob::NotifyStream(job, port) = held.take_held().pop().unwrap() else { panic!("Notify") }; + c.set_token("replacement"); + assert_eq!(r.keys, captured_keys); // Captured roster and its generations have not moved. + assert_ne!(job.req.key, ClientKey::of(c)); + let scripted = NotifyTransport::Script { + eligible: |_| CANDIDATE.with(|slot| slot.get()), + run: |_, _, _, _| panic!("expired"), + }; + let NotifyTransport::Script { eligible, .. } = scripted else { unreachable!() }; + assert!(eligible(&job.req).is_some_and(|candidate| std::ptr::eq(candidate, c))); + assert!(scripted.resolve(&job.req).is_none()); + let mut current_req = job.req.clone(); + current_req.key = ClientKey::of(c); + assert!(scripted.resolve(¤t_req).is_some_and(|candidate| std::ptr::eq(candidate, c))); + let mut launched = false; + StoreRunner::release_notify_stream_with(job, port, |req| scripted.resolve(req), + |_, _, _| { launched = true; false }); + assert!(!launched); + // Still-owned captured input observes ordinary failure; later host capture retires it. + r.tick(|_| panic!("no immediate retry")); + let until = delay(r.frame, c.id(), 0); + r.quiet_until(until); + let mut retries = 0; + r.tick(|job| { + retries += 1; + let StoreJob::NotifyStream(job, port) = job else { panic!("Notify") }; + port.refused(job.addr); + }); + assert_eq!(r.frame, until); + assert_eq!(retries, 1); +} diff --git a/rust-modules/src/app/adapters/stores.rs b/rust-modules/src/app/adapters/stores.rs index 546851201b..c560a0fb20 100644 --- a/rust-modules/src/app/adapters/stores.rs +++ b/rust-modules/src/app/adapters/stores.rs @@ -74,6 +74,7 @@ use plx_data::stores::catalog::{finish_catalog_job, CatalogPort, CatalogReq}; use plx_data::stores::viewreconcile::{finish_reconcile_job, ReconcilePort, ReconcileReq}; use plx_data::pms::{launch_hubs_job, HubRequest, HubsPort}; use plx_data::stores::{Admission, StoreJob}; +use plx_data::stores::notify_job::{finish_notify_stream_job, NotifyStreamReq, NotifyStreamPort, NotifyTerminal}; use plx_data::search::recents::{finish_save_recents_job, SaveRecentsPort, SaveRecentsReq}; use plx_data::search::{finish_search_job, SearchPort, SearchReq}; use plx_data::subsearch::{finish_subsearch_job, SubSearchPort, SubSearchReq}; @@ -96,7 +97,7 @@ fn os_zone() -> Zone { /// ones that answer, which makes `keys.key(sid)` equal `client_for(sid).map(ClientKey::of)` for /// every `sid`, a retired or unregistered one included. pub(crate) fn refill_client_keys(keys: &mut ClientKeys) { - keys.refill_roster( + keys.refill_roster_with_stream_policy( plx_plex::plex::current_server(), plx_plex::plex::server_roster_gen(), plx_plex::plex::server_facts_gen(), @@ -104,6 +105,7 @@ pub(crate) fn refill_client_keys(keys: &mut ClientKeys) { plx_plex::plex::client_for, plx_plex::plex::server_facts, plx_plex::plex::server_probe_result, + |c| c.has_token() && !c.data_io_disabled() && c.credential_transport_open(), ); } @@ -196,7 +198,53 @@ pub(crate) enum StoreRunner { /// (`run_held` resolves its client when it runs the job). A kind in `live` starts as `Live` /// starts it. #[cfg(test)] - Held { parked: Vec, live: &'static [LiveKind] }, + Held { parked: Vec, live: &'static [LiveKind], notify: NotifyTransport }, +} + +/// Boot-selected transport, not a second Notify state machine. Scripts must be finite; +/// false promises no script/work ran. Real streams always execute off-frame. +#[derive(Clone, Copy)] +#[allow(dead_code)] // Refusing/scripted boot ports belong to host tests, never store logic. +pub(crate) enum NotifyTransport { + Real, + Refuse, + Script { + eligible: fn(&NotifyStreamReq) -> Option<&'static Client>, + run: fn(&NotifyStreamPort, plx_machine::machine::Addr, &'static Client, NotifyStreamReq) -> bool, + }, +} + +fn notify_client(req: &NotifyStreamReq) -> Option<&'static Client> { + resolve(req.sid, req.key).filter(|c| c.has_token() && !c.data_io_disabled() && c.credential_transport_open()) +} + +impl NotifyTransport { + fn resolve(self, req: &NotifyStreamReq) -> Option<&'static Client> { + let client = match self { + Self::Script { eligible, .. } => eligible(req), + Self::Real | Self::Refuse => notify_client(req), + }?; + (client.id() == req.sid && ClientKey::of(client) == req.key).then_some(client) + } + + fn launch(self, job: Job, port: NotifyStreamPort, client: &'static Client) -> bool { + let Job { addr, req } = job; + match self { + Self::Refuse => false, + Self::Script { run, .. } => run(&port, addr, client, req), + Self::Real => plx_base::task::spawn_off_frame("notify", move |off| { + // Re-resolve at execution as well as admission: a held/scheduled request + // cannot open a replacement credential or a retired registry slot. + if port.is_cancelled() { + port.done(addr, NotifyTerminal::End(plx_plex::plex::notify::NotifyEnd::Cancelled)); + } else if let Some(current) = notify_client(&req) { + finish_notify_stream_job(&port, addr, current, req, off); + } else { + port.done(addr, NotifyTerminal::NotStarted); + } + }), + } + } } /// TEST ONLY: a kind of job a [`StoreRunner::Held`] starts on a worker instead of parking. It names @@ -222,6 +270,7 @@ pub(crate) enum LiveKind { BrowseDiscovery, /// Multi-server metadata reads stay off the ordinary held drain's test thread. MetaAltSources, + NotifyStream, } impl StoreRunner { @@ -238,7 +287,7 @@ impl StoreRunner { /// worker (the Related rigs land the window with `land_related_for_test`), so those are parked. #[cfg(test)] pub(crate) fn held() -> Self { - Self::Held { parked: Vec::new(), live: &[LiveKind::Hubs, LiveKind::MetaSeason, LiveKind::BrowsePage, LiveKind::BrowseDiscovery, LiveKind::MetaAltSources] } + Self::Held { parked: Vec::new(), live: &[LiveKind::Hubs, LiveKind::MetaSeason, LiveKind::BrowsePage, LiveKind::BrowseDiscovery, LiveKind::MetaAltSources, LiveKind::NotifyStream], notify: NotifyTransport::Refuse } } /// TEST ONLY: park the Library listing's page reads of a held runner too (the Home Hubs read @@ -247,7 +296,7 @@ impl StoreRunner { #[cfg(test)] pub(crate) fn park_page_reads(&mut self) { if let Self::Held { live, .. } = self { - *live = &[LiveKind::Hubs, LiveKind::MetaSeason, LiveKind::BrowseDiscovery, LiveKind::MetaAltSources]; + *live = &[LiveKind::Hubs, LiveKind::MetaSeason, LiveKind::BrowseDiscovery, LiveKind::MetaAltSources, LiveKind::NotifyStream]; } } @@ -256,7 +305,7 @@ impl StoreRunner { /// chooses to settle them. #[cfg(test)] pub(crate) fn held_all() -> Self { - Self::Held { parked: Vec::new(), live: &[] } + Self::Held { parked: Vec::new(), live: &[], notify: NotifyTransport::Refuse } } /// Start `job`, admitting it through the tape when its kind is taped (Detail, Collection, @@ -282,6 +331,7 @@ impl StoreRunner { StoreJob::MetaEpisodes(job, port) => self.start_meta_episodes(job, port), StoreJob::MetaRelated(job, port) => self.start_meta_related(job, port), StoreJob::MetaAltSources(job, port) => self.start_alt_sources(job, port), + StoreJob::NotifyStream(job, port) => self.start_notify_stream(job, port), } } @@ -334,7 +384,8 @@ impl StoreRunner { | StoreJob::MetaSeason(..) | StoreJob::MetaEpisodes(..) | StoreJob::MetaRelated(..) - | StoreJob::MetaAltSources(..)) => self.start(other), + | StoreJob::MetaAltSources(..) + | StoreJob::NotifyStream(..)) => self.start(other), } } @@ -363,7 +414,7 @@ impl StoreRunner { (_, HubsLaunch::Refused) => return launch_hubs_job(job, port, client, |_work| false), (_, HubsLaunch::Late { extra, items }) => return plx_data::pms::launch_late_hubs_job_for_test( job, port, client, |work| plx_base::task::spawn_small("hubs", work), extra, items), - (Self::Held { parked, live }, HubsLaunch::Real) if !live.contains(&LiveKind::Hubs) => { + (Self::Held { parked, live, .. }, HubsLaunch::Real) if !live.contains(&LiveKind::Hubs) => { port.claim(); parked.push(StoreJob::Hubs(job, port)); return true; @@ -433,6 +484,7 @@ impl StoreRunner { | StoreJob::MetaEpisodes(..) | StoreJob::MetaRelated(..) | StoreJob::MetaAltSources(..) + | StoreJob::NotifyStream(..) ) => held.push(kept), } } @@ -665,7 +717,7 @@ impl StoreRunner { match self { Self::Live => Self::spawn_meta_season(job, port), #[cfg(test)] - Self::Held { parked, live } => { + Self::Held { parked, live, .. } => { if live.contains(&LiveKind::MetaSeason) { return Self::spawn_meta_season(job, port); } @@ -751,7 +803,7 @@ impl StoreRunner { fn launch_alt_sources(&mut self, job: Job, port: MetaAltSourcesPort, launch: impl FnOnce(Box) -> bool) -> bool { #[cfg(test)] - if let Self::Held { parked, live } = self { + if let Self::Held { parked, live, .. } = self { if !live.contains(&LiveKind::MetaAltSources) { parked.push(StoreJob::MetaAltSources(job, port)); return true; @@ -926,7 +978,7 @@ impl StoreRunner { return false; // the OS refusing the thread } #[cfg(test)] - if let Self::Held { parked, live } = self { + if let Self::Held { parked, live, .. } = self { if !live.contains(&LiveKind::BrowsePage) { parked.push(StoreJob::BrowsePage(job, port)); return true; @@ -965,7 +1017,7 @@ impl StoreRunner { return false; // the OS refusing the thread } #[cfg(test)] - if let Self::Held { parked, live } = self { + if let Self::Held { parked, live, .. } = self { if !live.contains(&LiveKind::BrowseDiscovery) { parked.push(StoreJob::BrowseDiscovery(job, port)); return true; @@ -986,6 +1038,61 @@ impl StoreRunner { /// `finish_browse_hubs_job`, `finish_view_write_job`, `finish_browse_directory_job`, /// `finish_browse_page_job`, `finish_browse_discovery_job`) and return how many ran. /// Their landings are queued for the next pump. Under `Live` there is nothing parked. + fn notify_transport(&self) -> NotifyTransport { + match self { + Self::Live => NotifyTransport::Real, + #[cfg(test)] + Self::Held { notify, .. } => *notify, + } + } + + #[cfg(test)] + pub(crate) fn set_notify_transport(&mut self, transport: NotifyTransport) { + if let Self::Held { notify, .. } = self { *notify = transport; } + } + + fn start_notify_stream(&mut self, job: Job, port: NotifyStreamPort) { + let transport = self.notify_transport(); + self.start_notify_stream_with(job, port, |req| transport.resolve(req), + |job, port, client| transport.launch(job, port, client)); + } + + // Explicit resolver/launcher ports also let tests model an eligible capture without accounts. + fn start_notify_stream_with(&mut self, job: Job, port: NotifyStreamPort, + eligible: impl FnOnce(&NotifyStreamReq) -> Option<&'static Client>, + launch: impl FnOnce(Job, NotifyStreamPort, &'static Client) -> bool) { + let addr = job.addr; + if port.is_cancelled() { port.refused(addr); return; } + let Some(client) = eligible(&job.req) else { port.refused(addr); return; }; + #[cfg(test)] + if let Self::Held { parked, live, .. } = self { + if !live.contains(&LiveKind::NotifyStream) { + port.admit(addr, true); + parked.push(StoreJob::NotifyStream(job, port)); + return; + } + } + let receipt = port.clone(); + let accepted = launch(job, port, client); + receipt.admit(addr, accepted); + } + + #[cfg(test)] + fn release_notify_stream_with(job: Job, port: NotifyStreamPort, + eligible: impl FnOnce(&NotifyStreamReq) -> Option<&'static Client>, + launch: impl FnOnce(Job, NotifyStreamPort, &'static Client) -> bool) { + let addr = job.addr; + if port.is_cancelled() { + port.done(addr, NotifyTerminal::End(plx_plex::plex::notify::NotifyEnd::Cancelled)); + return; + } + let Some(client) = eligible(&job.req) else { + port.done(addr, NotifyTerminal::NotStarted); return; + }; + let terminal = port.clone(); + if !launch(job, port, client) { terminal.done(addr, NotifyTerminal::NotStarted); } + } + #[cfg(test)] pub(crate) fn run_held(&mut self) -> usize { self.run_held_where(|_| true) @@ -1012,6 +1119,7 @@ impl StoreRunner { #[cfg(test)] fn run_held_where(&mut self, wanted: impl Fn(&StoreJob) -> bool) -> usize { + let transport = self.notify_transport(); let Self::Held { parked, .. } = self else { return 0 }; let (run, keep): (Vec<_>, Vec<_>) = std::mem::take(parked).into_iter().partition(|job| wanted(job)); *parked = keep; @@ -1143,12 +1251,20 @@ impl StoreRunner { StoreJob::MetaAltSources(job, port) => { finish_alt_sources_job(&port, job.addr, plx_plex::plex::client_for, job.req); } + StoreJob::NotifyStream(job, port) => { + Self::release_notify_stream_with(job, port, |req| transport.resolve(req), + |job, port, client| transport.launch(job, port, client)); + } } } n } } +#[cfg(test)] +#[path = "notify_runner_tests.rs"] +mod notify_runner_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/rust-modules/src/app/bridge.rs b/rust-modules/src/app/bridge.rs index 3c3e4ad31b..47a4654612 100644 --- a/rust-modules/src/app/bridge.rs +++ b/rust-modules/src/app/bridge.rs @@ -46,6 +46,10 @@ mod metadata_alt_sources_bridge_tests; #[path = "browse_roster_bridge_tests.rs"] mod browse_roster_bridge_tests; +#[cfg(test)] +#[path = "notify_bridge_tests.rs"] +mod notify_bridge_tests; + use plx_screens::family::SettingsPage; use plx_screens::player::HudPolicy; use plx_screens::registry::{AppArg, AppFx, AppMounter, AppMsg, ConsentCmd, ContentArg, ContentReq, HomeCmd, HomeLike, HomeReq, HomeTab, ItemMenuKind, LibraryReq, LoopReq, PageMemory}; @@ -1955,10 +1959,31 @@ impl Bridge { /// and the endpoint refreshes the map's quiet refetches settled, for the caller to carry. pub(crate) fn notify_pump(&mut self, pages: &[ContentArg], shown: plx_data::stores::notify_map::Shown) -> (bool, plx_data::stores::EndpointRefreshSet) { - let out = self.stores.notify_pump(); super::adapters::stores::refill_client_keys(&mut self.client_keys); + self.notify_pump_captured(pages, shown) + } + + /// One captured input, one frame, and a runner boundary for EACH connection. This + /// helper is also the explicit-value test port; it never reads ambient keys. + fn notify_pump_captured(&mut self, pages: &[ContentArg], shown: plx_data::stores::notify_map::Shown) + -> (bool, plx_data::stores::EndpointRefreshSet) { + let input = plx_data::stores::notify::NotifyInput { + keys: &self.client_keys, armed: self.stores.landgate.is_armed(), + }; + let mut pass = self.stores.notify_begin(&input); + while let Some(mut turn) = self.stores.notify_prepare_next(&mut pass, &input) { + // No Stores borrow survives the IO boundary. Earlier connections have + // already drained before any later connection is prepared or launched. + for job in self.stores.take_jobs_of(StoreId::Notify) { + self.store_runner.start(job); + } + self.stores.notify_settle_admission(&mut pass, &mut turn); + self.stores.notify_finish_turn(&mut pass, turn, &input); + } + let out = self.stores.notify_end(pass); let mut spent = self.stores.notify_map(&out, pages, shown, self.directory.view(), &self.client_keys); self.start_store_jobs_in(plx_data::stores::Stores::NOTIFY_FILLS); + self.start_store_jobs_of(StoreId::Notify); if !spent.refetched.is_empty() || !spent.invalidated.is_empty() { plx_base::eventlog::log(&format!( "notify: refetched {} source(s) invalidated {} server(s)", spent.refetched.len(), spent.invalidated.len())); diff --git a/rust-modules/src/app/browse_hubs_drain_tests.rs b/rust-modules/src/app/browse_hubs_drain_tests.rs index 2e7a117282..561e4af227 100644 --- a/rust-modules/src/app/browse_hubs_drain_tests.rs +++ b/rust-modules/src/app/browse_hubs_drain_tests.rs @@ -13,12 +13,13 @@ use super::test_support::*; use super::*; use plx_data::stores::browse::{BrowseCmd, LibraryWork, SectionAddress}; use plx_data::stores::StoreJob; -use plx_data::stores::notify::{Connector, Link, NotifyStore}; +use plx_data::stores::notify_job::NotifyStreamReq; +use crate::app::adapters::stores::NotifyTransport; use plx_data::stores::notify_map::Shown; use plx_ui::dispatch::Rig as _; use plx_plex::plex::notify::{LibraryEntry, LibraryKind, LibraryState, Notification}; use plx_plex::plex::{GrantEvidence, ServerId}; -use std::sync::{Arc, Mutex}; + /// Joins a frame's effects and parts, as the dispatcher hands them to `Bridge::deliver`. fn deliver_browse_work(rig: &mut Bridge) { @@ -88,6 +89,7 @@ fn kind(job: &StoreJob) -> &'static str { StoreJob::BrowseHubs(..) => "BrowseHubs", StoreJob::BrowseDirectory(..) => "BrowseDirectory", StoreJob::BrowsePage(..) => "BrowsePage", + StoreJob::Hubs(..) => "Hubs", _ => "other", } } @@ -140,16 +142,6 @@ fn the_viewstate_pump_starts_the_read_its_burst_end_invalidation_asked_for() { rig.assert_one_read_started(); } -/// A connector that keeps the link it was given, for the test to push a notification down. -#[derive(Clone, Default)] -struct Wire(Arc>>); - -impl Connector for Wire { - fn connect(&mut self, _sid: ServerId, link: Link) -> bool { - self.0.lock().unwrap_or_else(|e| e.into_inner()).push(link); - true - } -} /// A library change on an owned server invalidates its sections' shelves /// (`BrowseCmd::HubsInvalidateServer`) through the notification map: `NOTIFY_FILLS` names Browse, @@ -159,16 +151,27 @@ fn the_notify_pump_starts_the_read_a_library_change_asked_for() { let _guard = plx_base::testlock::serial(); let mut rig = armed_rig(); plx_plex::plex::describe_server(rig.own, "Server A", "", GrantEvidence::ours()); - let wire = Wire::default(); - rig.bridge.stores.notify = NotifyStore::with_connector_for_test(Box::new(wire.clone())); - - let _ = rig.bridge.notify_pump(&[], Shown::ALL); - let link = wire.0.lock().unwrap_or_else(|e| e.into_inner()).first().cloned().expect("the feed connected"); - link.push(Notification::Library(vec![LibraryEntry { - item_id: "a".into(), kind: LibraryKind::Movie, state: LibraryState::Created, section: None }])); - let _ = rig.bridge.notify_pump(&[], Shown::ALL); - - rig.assert_one_read_started(); + if let crate::app::adapters::stores::StoreRunner::Held { live, .. } = &mut rig.bridge.store_runner { + // Keep Notify's finite script immediate, park BOTH map fan-out kinds. + *live = &[crate::app::adapters::stores::LiveKind::NotifyStream]; + } + rig.bridge.store_runner.set_notify_transport(NotifyTransport::Script { + eligible: |req: &NotifyStreamReq| plx_plex::plex::client_for(req.sid) + .filter(|c| plx_plex::plex::ClientKey::of(c) == req.key), + run: |port, addr, _, _| { + port.post(addr, Notification::Library(vec![LibraryEntry { + item_id: "a".into(), kind: LibraryKind::Movie, state: LibraryState::Created, section: None }])); + true + }, + }); + // Scripted eligible input is explicit, not an ambient credential-policy override. + rig.bridge.client_keys.refill_roster_with_stream_policy(rig.own, 1, 1, [rig.own], + plx_plex::plex::client_for, plx_plex::plex::server_facts, |_| None, |_| true); + let _ = rig.bridge.notify_pump_captured(&[], Shown::ALL); + + assert!(rig.bridge.stores.browse.borrow().hubs_read_out_for_test()); + rig.assert_started(&["BrowseHubs", "Hubs"]); + assert_eq!(rig.bridge.late_store_jobs, 0); } /// The membership re-query a page pushed over the Library page owes is pumped by diff --git a/rust-modules/src/app/notify_bridge_tests.rs b/rust-modules/src/app/notify_bridge_tests.rs new file mode 100644 index 0000000000..fe02c485ad --- /dev/null +++ b/rust-modules/src/app/notify_bridge_tests.rs @@ -0,0 +1,185 @@ +//! Notify's real Bridge cursor with boot-selected finite ports. No live account or SSE. +use super::*; +use plx_data::stores::{tape, notify_job::{NotifyStreamPort, NotifyStreamReq, NotifyTerminal}}; +use plx_machine::{job::Port, machine::Addr}; +use plx_plex::plex::{self, Client, ClientKey, GrantEvidence, ServerId}; +use plx_plex::plex::notify::{Notification, NotifyEnd, PlaySession, PlayState}; +use crate::app::adapters::stores::{StoreRunner, NotifyTransport, LiveKind}; +use std::cell::RefCell; + +#[derive(Default)] +struct Script { first: Option<(Addr, NotifyStreamPort)>, launches: Vec, eligible: bool } +thread_local! { static SCRIPT: RefCell