From ca59b5d2f9231ddc8137e21ded42ec76cbe72237 Mon Sep 17 00:00:00 2001 From: Wang Date: Wed, 2 Sep 2026 22:34:24 +0800 Subject: [PATCH 01/13] feat(peer): converge Mesh reachability leases Generated-by: Codex (gpt-5.6-sol) --- .../runtime-host-local-operator.test.ts | 2 +- .../runtime-host-ssh-terminal.test.ts | 2 +- .../main/runtime-host-peer-mesh-management.ts | 6 +- .../ui/runtime-host-peer-mesh-dialog.tsx | 19 +- .../renderer/styles/settings/runtime-host.css | 4 +- .../src/__tests__/peer-mesh.test.ts | 175 +- packages/runtime-host/src/peer-mesh/limits.ts | 2 +- packages/runtime-host/src/peer-mesh/model.ts | 132 +- packages/runtime-host/src/peer-mesh/node.ts | 1408 +++++++++++------ packages/runtime-host/src/peer-mesh/store.ts | 168 +- .../src/peer-reachability/index.ts | 1 + .../src/peer-reachability/publisher.ts | 21 + .../runtime-host/src/protocol/peer-mesh.ts | 56 +- .../src/server/peer-mesh-authority.ts | 2 +- 14 files changed, 1257 insertions(+), 741 deletions(-) diff --git a/apps/desktop/src/main/__tests__/runtime-host-local-operator.test.ts b/apps/desktop/src/main/__tests__/runtime-host-local-operator.test.ts index 26f95d8b5d..f32962bf26 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-local-operator.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-local-operator.test.ts @@ -224,7 +224,7 @@ test('local Peer Mesh join keeps invitations off argv and accepts bounded large peerId: `peer-${meshIndex}-${memberIndex}-${'x'.repeat(48)}`, endpointKind: 'client' as const, displayName: `Member ${meshIndex}-${memberIndex} ${'x'.repeat(60)}`, - state: 'route_available' as const, + state: 'reachable' as const, expiresAt: 4_000_000_000_000, })), pendingInvitationCount: 0, diff --git a/apps/desktop/src/main/__tests__/runtime-host-ssh-terminal.test.ts b/apps/desktop/src/main/__tests__/runtime-host-ssh-terminal.test.ts index 88413a47a4..7e9c87fc06 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-ssh-terminal.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-ssh-terminal.test.ts @@ -775,7 +775,7 @@ test('sends a Mesh invitation only after the authenticated remote operator reque revision: 2, closed: false, members: [ - { peerId: 'peer-a', state: 'route_available', expiresAt: Date.now() + 60_000 }, + { peerId: 'peer-a', state: 'reachable', expiresAt: Date.now() + 60_000 }, { peerId: 'peer-b', state: 'local' }, ], pendingInvitationCount: 0, diff --git a/apps/desktop/src/main/runtime-host-peer-mesh-management.ts b/apps/desktop/src/main/runtime-host-peer-mesh-management.ts index e321230229..675962f0fb 100644 --- a/apps/desktop/src/main/runtime-host-peer-mesh-management.ts +++ b/apps/desktop/src/main/runtime-host-peer-mesh-management.ts @@ -490,7 +490,11 @@ async function reconcileManagedTarget( function authorityRouteNeedsRecovery(mesh: PeerMeshQueryResult['meshes'][number]): boolean { const authority = mesh.members.find(({ peerId }) => peerId === mesh.authorityPeerId); - return authority === undefined || authority.state === 'unknown' || authority.state === 'stale'; + return ( + authority === undefined || + authority.state === 'needs_repair' || + authority.state === 'reconnecting' + ); } function requireQueryResult(result: PeerMeshResult): PeerMeshQueryResult { diff --git a/apps/desktop/src/renderer/features/runtime-host-management/ui/runtime-host-peer-mesh-dialog.tsx b/apps/desktop/src/renderer/features/runtime-host-management/ui/runtime-host-peer-mesh-dialog.tsx index ef0fee878c..49788c0f29 100644 --- a/apps/desktop/src/renderer/features/runtime-host-management/ui/runtime-host-peer-mesh-dialog.tsx +++ b/apps/desktop/src/renderer/features/runtime-host-management/ui/runtime-host-peer-mesh-dialog.tsx @@ -489,7 +489,8 @@ export function RuntimeHostPeerMeshDialog(props: { meshId, code: JSON.stringify(result.invitation), expiresAt: result.invitation.expiresAt, - hasCoordinationRelay: result.invitation.coordinationRelays.length > 0, + hasCoordinationRelay: + result.invitation.reachability.lease.coordinationRoutes.length > 0, }); setSnapshot(result.snapshot); }, @@ -1918,10 +1919,10 @@ function peerMeshCopy(locale: string) { circuits: '连接', routeState: { local: '本机', - route_available: '路径可用', - coordination_only: '仅协调路径', - stale: '路径已过期', - unknown: '路径未知', + connecting: '正在连接', + reachable: '可达', + reconnecting: '正在恢复连接', + needs_repair: '需要新邀请码修复', }, endpointKind: { client: 'Client', @@ -2078,10 +2079,10 @@ function peerMeshCopy(locale: string) { circuits: 'Circuits', routeState: { local: 'Local', - route_available: 'Route known', - coordination_only: 'Coordination only', - stale: 'Stale route', - unknown: 'Route unknown', + connecting: 'Connecting', + reachable: 'Reachable', + reconnecting: 'Reconnecting', + needs_repair: 'Needs a new invitation', }, endpointKind: { client: 'Client', diff --git a/apps/desktop/src/renderer/styles/settings/runtime-host.css b/apps/desktop/src/renderer/styles/settings/runtime-host.css index bd1baf8e0f..d4fd7d7d05 100644 --- a/apps/desktop/src/renderer/styles/settings/runtime-host.css +++ b/apps/desktop/src/renderer/styles/settings/runtime-host.css @@ -723,11 +723,11 @@ } .settingsPeerMeshMemberState-local, -.settingsPeerMeshMemberState-route_available { +.settingsPeerMeshMemberState-reachable { background: var(--success); } -.settingsPeerMeshMemberState-coordination_only { +.settingsPeerMeshMemberState-reconnecting { background: var(--warning); } diff --git a/packages/runtime-host/src/__tests__/peer-mesh.test.ts b/packages/runtime-host/src/__tests__/peer-mesh.test.ts index be45b83810..537f7b7364 100644 --- a/packages/runtime-host/src/__tests__/peer-mesh.test.ts +++ b/packages/runtime-host/src/__tests__/peer-mesh.test.ts @@ -31,7 +31,22 @@ import { peerMeshId, signPeerMeshRoster, } from '../peer-mesh/model.js'; -import { openPeerMeshNode, type PeerMeshNode, type PeerMeshTransport } from '../peer-mesh/node.js'; +import { + openPeerMeshNode as openPeerMeshNodeImpl, + type PeerMeshNode, + type PeerMeshTransport, +} from '../peer-mesh/node.js'; +import { + canonicalPeerReachabilityLease, + decodeSignedPeerReachabilityLease, + PEER_REACHABILITY_LEASE_TTL_MS, + PEER_REACHABILITY_REFRESH_LEAD_MS, + peerReachabilityLeaseSigningBytes, + samePeerReachabilityRoutes, + verifySignedPeerReachabilityLease, + type PeerReachabilityPublisher, + type SignedPeerReachabilityLeaseV1, +} from '../peer-reachability/index.js'; import { hasPeerMeshIdentityObligations, migrateLegacyPeerMeshState, @@ -40,6 +55,15 @@ import { } from '../peer-mesh/store.js'; import { createPeerMeshOperationHandlers } from '../server/peer-mesh-authority.js'; +function openPeerMeshNode( + input: Omit[0], 'peer' | 'reachability'> & { + readonly peer: MemoryPeerClient; + }, +): Promise { + input.peer.useClock(input.now ?? Date.now); + return openPeerMeshNodeImpl({ ...input, reachability: input.peer }); +} + test('preserves durable Mesh mutation outcomes and drains after an unknown commit', async () => { let drains = 0; const postCommit = createPeerMeshOperationHandlers( @@ -89,11 +113,11 @@ test('authenticates three peers, consumes invitations once, and keeps authority const [authority, memberB, memberC] = nodes as [PeerMeshNode, PeerMeshNode, PeerMeshNode]; await authority.setDisplayName('Alice Desktop'); const mesh = await authority.create(); - assert.deepEqual(mesh.authority.coordinationRelays, ['/memory/relay/peer-a']); + assert.equal(mesh.authorityPeerId, 'peer-a'); const serving = authority.serve(); const contested = await authority.invite(mesh.roster.roster.meshId); - assert.deepEqual(contested.coordinationRelays, mesh.authority.coordinationRelays); + assert.deepEqual(contested.reachability.lease.coordinationRoutes, ['/memory/relay/peer-a']); const attempts = await Promise.allSettled([memberB.join(contested), memberC.join(contested)]); assert.equal(attempts.filter(({ status }) => status === 'fulfilled').length, 1); assert.equal(attempts.filter(({ status }) => status === 'rejected').length, 1); @@ -282,29 +306,30 @@ test('reconciles changed routes, propagates removal, and recovers the verified c await authority.reconcile(); await memberC.setDisplayName('Peer C'); await memberC.reconcile(); - authorityPeer.stallNextControl(); - await memberB.reconcile(AbortSignal.timeout(1_000)); + authorityPeer.setReachable(false); + await memberB.reconcile(); assert.deepEqual(memberB.resolveRoutes('peer-a')?.routeHints, ['/memory/peer-a/p2p/peer-a']); - memberCPeer.setRouteHints(['/memory/peer-c-moved/p2p/peer-c']); + await memberCPeer.setRouteHints(['/memory/peer-c-moved/p2p/peer-c']); await memberC.reconcile(); await memberB.reconcile(); assert.deepEqual(memberB.resolveRoutes('peer-c')?.routeHints, [ '/memory/peer-c-moved/p2p/peer-c', ]); + authorityPeer.setReachable(true); await memberC.close(); await serving[2]; await rm(join(root, 'member-c'), { recursive: true, force: true }); now += 6 * 60 * 1_000; - authorityPeer.setRouteHints(['/memory/peer-a-moved/p2p/peer-a']); + await authorityPeer.setRouteHints(['/memory/peer-a-moved/p2p/peer-a']); await authority.reconcile(); await memberB.reconcile(); assert.deepEqual(memberB.resolveRoutes('peer-a')?.routeHints, [ '/memory/peer-a-moved/p2p/peer-a', ]); - memberCPeer.setRouteHints(['/memory/peer-c-rejoined/p2p/peer-c']); + await memberCPeer.setRouteHints(['/memory/peer-c-rejoined/p2p/peer-c']); memberC = await openPeerMeshNode({ dataRoot: join(root, 'member-c'), peer: memberCPeer, @@ -352,6 +377,43 @@ test('reconciles changed routes, propagates removal, and recovers the verified c } }); +test('repairs an existing membership with a fresh invitation after every locator is lost', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-peer-mesh-route-repair-')); + const network = new MemoryPeerNetwork(); + const authorityPeer = network.create('peer-a'); + const memberPeer = network.create('peer-b'); + const authority = await openPeerMeshNode({ + dataRoot: join(root, 'authority'), + peer: authorityPeer, + }); + const member = await openPeerMeshNode({ dataRoot: join(root, 'member'), peer: memberPeer }); + const serving = [authority.serve(), member.serve()]; + try { + const meshId = (await authority.create()).roster.roster.meshId; + await member.join(await authority.invite(meshId)); + + await authorityPeer.setCoordinationRelays([]); + await authorityPeer.setRouteHints([]); + await authority.reconcile(); + await member.reconcile(); + assert.equal( + member.status()[0]?.memberRoutes.find(({ peerId }) => peerId === 'peer-a')?.state, + 'needs_repair', + ); + + const recoveredRoute = '/memory/peer-a-recovered/p2p/peer-a'; + await authorityPeer.setRouteHints([recoveredRoute]); + const repaired = await member.join(await authority.invite(meshId)); + assert.equal(member.status().length, 1); + assert.deepEqual(repaired.roster.roster.members, ['peer-a', 'peer-b']); + assert.deepEqual(member.resolveRoutes('peer-a')?.routeHints, [recoveredRoute]); + } finally { + await Promise.allSettled([authority.close(), member.close()]); + await Promise.allSettled([authorityPeer.close(), memberPeer.close(), ...serving]); + await rm(root, { recursive: true, force: true }); + } +}); + test('publishes a changed local route promptly and refreshes a live cached peer route', { timeout: 10_000, }, async () => { @@ -384,8 +446,8 @@ test('publishes a changed local route promptly and refreshes a live cached peer const movedRoute = '/memory/peer-b-restarted/p2p/peer-b'; const movedRelay = '/memory/relay/peer-b-restarted'; - memberBPeer.setRouteHints([movedRoute]); - memberBPeer.setCoordinationRelays([movedRelay]); + await memberBPeer.setRouteHints([movedRoute]); + await memberBPeer.setCoordinationRelays([movedRelay]); await waitForRoutes(authority, 'peer-b', [movedRoute], [movedRelay]); assert.deepEqual(memberC.resolveRoutes('peer-b')?.routeHints, ['/memory/peer-b/p2p/peer-b']); @@ -423,9 +485,11 @@ test('reconciles one selected Mesh into signed transit routes and native policy' await memberC.join(await authority.invite(meshId)); await memberD.join(await authority.invite(meshId)); - authorityPeer.failNextTransitConfiguration(); + await authority.reconcile(); + authorityPeer.setTransitConfigurationFailure(true); await assert.rejects(authority.setTransitMesh(meshId), /transit configuration failure/u); assert.equal(authority.transitMeshId(), meshId); + authorityPeer.setTransitConfigurationFailure(false); await authority.reconcile(); await memberB.reconcile(); assert.equal(authority.transitMeshId(), meshId); @@ -441,7 +505,7 @@ test('reconciles one selected Mesh into signed transit routes and native policy' await memberD.setTransitMesh(meshId); await memberD.reconcile(); - memberDPeer.setRouteHints(['/memory/peer-c/p2p/peer-c']); + await memberDPeer.setRouteHints(['/memory/peer-c/p2p/peer-c']); await memberD.reconcile(); await memberB.reconcile(); assert.deepEqual(memberBPeer.transitPolicy.relayCandidates, [ @@ -709,12 +773,13 @@ test('does not redeem a prepared join after explicit cancellation', async () => join(memberRoot, 'peer-mesh.json'), `${JSON.stringify( { - version: 6, + version: 7, localPeerId: 'peer-b', displayName: null, meshes: [], pendingJoins: [{ invitation, phase: 'prepared' }], - routes: [], + reachability: [], + advertisements: [], transitMeshId: null, }, null, @@ -927,7 +992,7 @@ class MemoryPeerNetwork { } } -class MemoryPeerClient implements PeerMeshTransport { +class MemoryPeerClient implements PeerMeshTransport, PeerReachabilityPublisher { #meshServer: | { readonly onStream: (stream: RuntimeHostPeerNativeStream) => void; @@ -941,6 +1006,10 @@ class MemoryPeerClient implements PeerMeshTransport { #reachable = true; #routeHints: readonly string[]; #coordinationRelays: readonly string[]; + #reachability: SignedPeerReachabilityLeaseV1 | undefined; + #reachabilityRevision = 0; + #now: () => number = Date.now; + readonly #reachabilityListeners = new Set<(lease: SignedPeerReachabilityLeaseV1) => void>(); #nextConnectionBarrier: | { readonly started: () => void; @@ -956,7 +1025,7 @@ class MemoryPeerClient implements PeerMeshTransport { readonly coordinationRelays: readonly string[]; }[], }; - #failNextTransitConfiguration = false; + #transitConfigurationFailure = false; #nextTransitBarrier: | { readonly started: () => void; @@ -981,12 +1050,74 @@ class MemoryPeerClient implements PeerMeshTransport { } as const; } - setRouteHints(routeHints: readonly string[]): void { + useClock(now: () => number): void { + this.#now = now; + } + + current(): SignedPeerReachabilityLeaseV1 { + if (!this.#reachability) throw new Error('Test reachability is not initialized'); + return this.#reachability; + } + + async refresh(): Promise { + const identity = this.identity(); + const now = this.#now(); + if ( + this.#reachability && + this.#reachability.lease.expiresAt > now + PEER_REACHABILITY_REFRESH_LEAD_MS && + samePeerReachabilityRoutes(this.#reachability.lease, identity) + ) { + return this.#reachability; + } + this.#reachabilityRevision += 1; + const lease = canonicalPeerReachabilityLease({ + version: 1, + peerId: this.peerId, + revision: this.#reachabilityRevision, + issuedAt: now, + expiresAt: now + PEER_REACHABILITY_LEASE_TTL_MS, + directRoutes: identity.listenAddresses, + coordinationRoutes: identity.coordinationRelays, + }); + const proof = await this.signIdentity(peerReachabilityLeaseSigningBytes(lease)); + const signed = decodeSignedPeerReachabilityLease({ + lease, + publicKey: proof.publicKey.toString('base64url'), + signature: proof.signature.toString('base64url'), + }); + this.verify(signed, this.peerId); + this.#reachability = signed; + for (const listener of this.#reachabilityListeners) listener(signed); + return signed; + } + + verify( + value: unknown, + expectedPeerId: string, + options: { readonly allowExpired?: boolean } = {}, + ): SignedPeerReachabilityLeaseV1 { + return verifySignedPeerReachabilityLease({ + value, + expectedPeerId, + now: this.#now(), + verifyIdentity: this.verifyIdentity.bind(this), + ...(options.allowExpired === undefined ? {} : { allowExpired: options.allowExpired }), + }); + } + + subscribe(listener: (lease: SignedPeerReachabilityLeaseV1) => void): () => void { + this.#reachabilityListeners.add(listener); + return () => this.#reachabilityListeners.delete(listener); + } + + async setRouteHints(routeHints: readonly string[]): Promise { this.#routeHints = [...routeHints]; + await this.refresh(); } - setCoordinationRelays(coordinationRelays: readonly string[]): void { + async setCoordinationRelays(coordinationRelays: readonly string[]): Promise { this.#coordinationRelays = [...coordinationRelays]; + await this.refresh(); } setReachable(reachable: boolean): void { @@ -997,8 +1128,8 @@ class MemoryPeerClient implements PeerMeshTransport { this.#responseDelayMs = delayMs; } - failNextTransitConfiguration(): void { - this.#failNextTransitConfiguration = true; + setTransitConfigurationFailure(fail: boolean): void { + this.#transitConfigurationFailure = fail; } stallNextTransitConfiguration(): { @@ -1081,8 +1212,7 @@ class MemoryPeerClient implements PeerMeshTransport { readonly coordinationRelays: readonly string[]; }[]; }): Promise { - if (this.#failNextTransitConfiguration) { - this.#failNextTransitConfiguration = false; + if (this.#transitConfigurationFailure) { throw new Error('Injected transit configuration failure'); } const barrier = this.#nextTransitBarrier; @@ -1155,6 +1285,7 @@ class MemoryPeerClient implements PeerMeshTransport { close(): Promise { if (this.#closed) return Promise.resolve(); this.#closed = true; + this.#reachabilityListeners.clear(); this.#meshServer?.stop(); return Promise.resolve(); } diff --git a/packages/runtime-host/src/peer-mesh/limits.ts b/packages/runtime-host/src/peer-mesh/limits.ts index 95e1c77d5a..6eecb5b69f 100644 --- a/packages/runtime-host/src/peer-mesh/limits.ts +++ b/packages/runtime-host/src/peer-mesh/limits.ts @@ -24,4 +24,4 @@ export const PEER_MESH_MAX_INVITATION_RECORDS = PEER_MESH_MAX_PENDING_INVITATION export const PEER_MESH_MAX_ROUTE_HINTS = 16; export const PEER_MESH_MAX_TRANSIT_RELAY_ADDRESSES = 256; export const PEER_MESH_MAX_TRANSIT_ADDRESSES_PER_RELAY = 4; -export const PEER_MESH_ROUTE_RECORD_MAX_BYTES = 4 * 1024; +export const PEER_MESH_MEMBER_ADVERTISEMENT_MAX_BYTES = 2 * 1024; diff --git a/packages/runtime-host/src/peer-mesh/model.ts b/packages/runtime-host/src/peer-mesh/model.ts index 606afa8339..5339e07173 100644 --- a/packages/runtime-host/src/peer-mesh/model.ts +++ b/packages/runtime-host/src/peer-mesh/model.ts @@ -32,10 +32,10 @@ import { type PeerMeshInvitationV1, } from '../protocol/peer-mesh.js'; import { - PEER_MESH_MAX_MEMBERS, - PEER_MESH_MAX_ROUTE_HINTS, - PEER_MESH_ROUTE_RECORD_MAX_BYTES, -} from './limits.js'; + decodeSignedPeerReachabilityLease, + type SignedPeerReachabilityLeaseV1, +} from '../peer-reachability/index.js'; +import { PEER_MESH_MEMBER_ADVERTISEMENT_MAX_BYTES, PEER_MESH_MAX_MEMBERS } from './limits.js'; import { canonicalPeerMeshDisplayName } from './display-name.js'; export { @@ -46,7 +46,7 @@ export { PEER_MESH_MAX_ROUTE_HINTS, PEER_MESH_MAX_TRANSIT_ADDRESSES_PER_RELAY, PEER_MESH_MAX_TRANSIT_RELAY_ADDRESSES, - PEER_MESH_ROUTE_RECORD_MAX_BYTES, + PEER_MESH_MEMBER_ADVERTISEMENT_MAX_BYTES, } from './limits.js'; export interface PeerMeshRosterV1 { @@ -66,8 +66,7 @@ export interface SignedPeerMeshRosterV1 { export interface PeerMeshAuthorityTarget { readonly peerId: string; - readonly routeHints: readonly string[]; - readonly coordinationRelays: readonly string[]; + readonly reachability: SignedPeerReachabilityLeaseV1; } export interface PeerMeshAuthorityKeyPair { @@ -75,17 +74,18 @@ export interface PeerMeshAuthorityKeyPair { readonly privateKey: string; } -export interface PeerMeshRouteRecordV1 extends PeerMeshAuthorityTarget { +export interface PeerMeshMemberAdvertisementV1 { readonly version: 1; - readonly sequence: number; - readonly expiresAt: number; + readonly meshId: string; + readonly peerId: string; + readonly revision: number; readonly endpointKind?: 'client' | 'host'; readonly displayName?: string; - readonly transitMeshId?: string; + readonly offersTransit: boolean; } -export interface SignedPeerMeshRouteRecordV1 { - readonly route: PeerMeshRouteRecordV1; +export interface SignedPeerMeshMemberAdvertisementV1 { + readonly advertisement: PeerMeshMemberAdvertisementV1; readonly publicKey: string; readonly signature: string; } @@ -224,29 +224,30 @@ export function canonicalPeerMeshRoster(value: unknown): PeerMeshRosterV1 { } export function decodeAuthorityTarget(value: unknown): PeerMeshAuthorityTarget { - const record = exactObject(value, 'Peer Mesh authority target', [ - 'peerId', - 'routeHints', - 'coordinationRelays', - ]); + const record = exactObject(value, 'Peer Mesh authority target', ['peerId', 'reachability']); + const peerId = token(record.peerId, 'peerId', 256); + const reachability = decodeSignedPeerReachabilityLease(record.reachability); + if (reachability.lease.peerId !== peerId) { + throw new Error('Peer Mesh authority reachability belongs to a different peer'); + } return Object.freeze({ - peerId: token(record.peerId, 'peerId', 256), - routeHints: Object.freeze(addressArray(record.routeHints, 'routeHints')), - coordinationRelays: Object.freeze( - addressArray(record.coordinationRelays, 'coordinationRelays'), - ), + peerId, + reachability, }); } -export function canonicalPeerMeshRouteRecord(value: unknown): PeerMeshRouteRecordV1 { - const keys = ['version', 'peerId', 'sequence', 'expiresAt', 'routeHints', 'coordinationRelays']; +export function canonicalPeerMeshMemberAdvertisement( + value: unknown, +): PeerMeshMemberAdvertisementV1 { + const keys = ['version', 'meshId', 'peerId', 'revision', 'offersTransit']; if (value && typeof value === 'object' && !Array.isArray(value)) { if (Object.hasOwn(value, 'endpointKind')) keys.push('endpointKind'); if (Object.hasOwn(value, 'displayName')) keys.push('displayName'); - if (Object.hasOwn(value, 'transitMeshId')) keys.push('transitMeshId'); } - const record = exactObject(value, 'Peer Mesh route record', keys); - if (record.version !== 1) throw new Error('Unsupported Peer Mesh route record version'); + const record = exactObject(value, 'Peer Mesh member advertisement', keys); + if (record.version !== 1) { + throw new Error('Unsupported Peer Mesh member advertisement version'); + } if ( record.endpointKind !== undefined && record.endpointKind !== 'client' && @@ -254,56 +255,58 @@ export function canonicalPeerMeshRouteRecord(value: unknown): PeerMeshRouteRecor ) { throw new Error('Invalid Peer Mesh endpoint kind'); } - const route = Object.freeze({ + if (typeof record.offersTransit !== 'boolean') { + throw new Error('Invalid Peer Mesh transit advertisement'); + } + const advertisement = Object.freeze({ version: 1 as const, + meshId: string(record.meshId, 'meshId', 128), peerId: token(record.peerId, 'peerId', 256), - sequence: integer(record.sequence, 'route sequence', 1), - expiresAt: integer(record.expiresAt, 'route expiry', 1), - routeHints: Object.freeze(addressArray(record.routeHints, 'routeHints')), - coordinationRelays: Object.freeze( - addressArray(record.coordinationRelays, 'coordinationRelays'), - ), + revision: integer(record.revision, 'advertisement revision', 1), ...(record.endpointKind === undefined ? {} : { endpointKind: record.endpointKind }), ...(record.displayName === undefined ? {} : { displayName: canonicalPeerMeshDisplayName(record.displayName) }), - ...(record.transitMeshId === undefined - ? {} - : { transitMeshId: string(record.transitMeshId, 'transitMeshId', 128) }), + offersTransit: record.offersTransit, }); - if (peerMeshRouteRecordSigningBytes(route).byteLength > PEER_MESH_ROUTE_RECORD_MAX_BYTES) { - throw new Error('Peer Mesh route record is too large'); + if ( + peerMeshMemberAdvertisementSigningBytes(advertisement).byteLength > + PEER_MESH_MEMBER_ADVERTISEMENT_MAX_BYTES + ) { + throw new Error('Peer Mesh member advertisement is too large'); } - return route; + return advertisement; } -export function decodeSignedPeerMeshRouteRecord(value: unknown): SignedPeerMeshRouteRecordV1 { - const record = exactObject(value, 'signed Peer Mesh route record', [ - 'route', +export function decodeSignedPeerMeshMemberAdvertisement( + value: unknown, +): SignedPeerMeshMemberAdvertisementV1 { + const record = exactObject(value, 'signed Peer Mesh member advertisement', [ + 'advertisement', 'publicKey', 'signature', ]); - const publicKey = canonicalProof(record.publicKey, 'route public key', 256); - const signature = canonicalProof(record.signature, 'route signature', 256); + const publicKey = canonicalProof(record.publicKey, 'advertisement public key', 256); + const signature = canonicalProof(record.signature, 'advertisement signature', 256); return Object.freeze({ - route: canonicalPeerMeshRouteRecord(record.route), + advertisement: canonicalPeerMeshMemberAdvertisement(record.advertisement), publicKey, signature, }); } -export function peerMeshRouteRecordSigningBytes(route: PeerMeshRouteRecordV1): Buffer { +export function peerMeshMemberAdvertisementSigningBytes( + advertisement: PeerMeshMemberAdvertisementV1, +): Buffer { return Buffer.from( - `maka.peer-mesh.route.v1\n${JSON.stringify({ - coordinationRelays: route.coordinationRelays, - ...(route.displayName ? { displayName: route.displayName } : {}), - ...(route.endpointKind ? { endpointKind: route.endpointKind } : {}), - expiresAt: route.expiresAt, - peerId: route.peerId, - routeHints: route.routeHints, - sequence: route.sequence, - ...(route.transitMeshId ? { transitMeshId: route.transitMeshId } : {}), - version: route.version, + `maka.peer-mesh.member-advertisement.v1\n${JSON.stringify({ + ...(advertisement.displayName ? { displayName: advertisement.displayName } : {}), + ...(advertisement.endpointKind ? { endpointKind: advertisement.endpointKind } : {}), + meshId: advertisement.meshId, + offersTransit: advertisement.offersTransit, + peerId: advertisement.peerId, + revision: advertisement.revision, + version: advertisement.version, })}`, ); } @@ -418,16 +421,3 @@ function stringArray(value: unknown, label: string, maxItems: number, maxLength: } return value.map((item) => string(item, label, maxLength)); } - -function addressArray(value: unknown, label: string): string[] { - const addresses = stringArray(value, label, PEER_MESH_MAX_ROUTE_HINTS, 1024); - if ( - addresses.some( - (address) => !address.startsWith('/') || /\s|[\u0000-\u001f\u007f]/u.test(address), - ) || - new Set(addresses).size !== addresses.length - ) { - throw new Error(`Invalid Peer Mesh ${label}`); - } - return addresses; -} diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index 02315a67fc..19bf952074 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -25,12 +25,12 @@ import type { } from '../transport/peer-native.js'; import { setTimeout as delay } from 'node:timers/promises'; import { + canonicalPeerMeshMemberAdvertisement, canonicalPeerMeshRoster, - canonicalPeerMeshRouteRecord, createPeerMeshInvitationSecret, validatePeerMeshInvitation, + decodeSignedPeerMeshMemberAdvertisement, decodeSignedPeerMeshRoster, - decodeSignedPeerMeshRouteRecord, generatePeerMeshAuthorityKeyPair, matchesPeerMeshInvitationSecret, PEER_MESH_MAX_MEMBERS, @@ -39,18 +39,21 @@ import { PEER_MESH_MAX_PENDING_INVITATIONS, PEER_MESH_MAX_TRANSIT_ADDRESSES_PER_RELAY, PEER_MESH_MAX_TRANSIT_RELAY_ADDRESSES, - peerMeshRouteRecordSigningBytes, + peerMeshMemberAdvertisementSigningBytes, peerMeshId, peerMeshInvitationSecretDigest, signPeerMeshRoster, type PeerMeshAuthorityTarget, - type PeerMeshRouteRecordV1, + type SignedPeerMeshMemberAdvertisementV1, type SignedPeerMeshRosterV1, - type SignedPeerMeshRouteRecordV1, } from './model.js'; import { canonicalPeerMeshDisplayName } from './display-name.js'; import type { PeerMeshInvitationV1 } from '../protocol/peer-mesh.js'; -import type { PeerReachabilityPublisher } from '../peer-reachability/index.js'; +import { + decodeSignedPeerReachabilityLease, + type PeerReachabilityPublisher, + type SignedPeerReachabilityLeaseV1, +} from '../peer-reachability/index.js'; import { authorityKeys, isActivePeerMeshMembership as isActiveMembership, @@ -64,33 +67,32 @@ import { type PeerMeshStoredStateV1, } from './store.js'; -const CONTROL_FRAME_MAX_BYTES = 64 * 1024; +const CONTROL_FRAME_MAX_BYTES = 128 * 1024; const DEFAULT_INVITATION_TTL_MS = 15 * 60 * 1_000; const CONNECT_DEADLINE_MS = 30_000; const CONTROL_REQUEST_DEADLINE_MS = 10_000; const MAX_ACTIVE_CONTROL_STREAMS = 32; const MAX_ACTIVE_CONTROL_STREAMS_PER_PEER = 2; -const ROUTE_TTL_MS = 5 * 60 * 1_000; -const ROUTE_REFRESH_LEAD_MS = 60 * 1_000; -const ROUTE_MAX_FUTURE_MS = 10 * 60 * 1_000; -const ROUTE_PAGE_SIZE = 8; +const REACHABILITY_HISTORY_MS = 24 * 60 * 60 * 1_000; +const EVIDENCE_PAGE_SIZE = 2; const RECONCILE_CONCURRENCY = 4; const RECONCILE_DEADLINE_MS = 60 * 1_000; -const RECONCILE_INTERVAL_MS = 30 * 1_000; -const LOCAL_ROUTE_OBSERVATION_INTERVAL_MS = 1_000; +const RECONCILE_INTERVAL_MS = 5 * 60 * 1_000; interface RedeemInvitationRequest { readonly kind: 'redeem-invitation'; readonly meshId: string; readonly secret: string; - readonly route: SignedPeerMeshRouteRecordV1; + readonly reachability: SignedPeerReachabilityLeaseV1; + readonly advertisement: SignedPeerMeshMemberAdvertisementV1; } type RedeemInvitationResponse = | { readonly kind: 'invitation-redeemed'; readonly roster: SignedPeerMeshRosterV1; - readonly routes: readonly SignedPeerMeshRouteRecordV1[]; + readonly reachability: readonly SignedPeerReachabilityLeaseV1[]; + readonly advertisements: readonly SignedPeerMeshMemberAdvertisementV1[]; } | { readonly kind: 'invitation-rejected'; @@ -99,24 +101,31 @@ type RedeemInvitationResponse = type RedeemInvitationRejectionReason = 'invalid' | 'expired' | 'closed' | 'full'; -interface PeerMeshRouteSequence { +interface PeerMeshEvidenceRevision { readonly peerId: string; - readonly sequence: number; + readonly revision: number; +} + +interface PeerMeshAdvertisementRevision extends PeerMeshEvidenceRevision { + readonly meshId: string; } interface SyncPeerMeshRequest { readonly kind: 'sync'; readonly meshId: string; readonly roster: SignedPeerMeshRosterV1; - readonly route: SignedPeerMeshRouteRecordV1; - readonly knownRoutes: readonly PeerMeshRouteSequence[]; + readonly reachability: SignedPeerReachabilityLeaseV1; + readonly advertisement: SignedPeerMeshMemberAdvertisementV1; + readonly knownReachability: readonly PeerMeshEvidenceRevision[]; + readonly knownAdvertisements: readonly PeerMeshAdvertisementRevision[]; } type SyncPeerMeshResponse = | { readonly kind: 'sync-result'; readonly roster: SignedPeerMeshRosterV1; - readonly routes: readonly SignedPeerMeshRouteRecordV1[]; + readonly reachability: readonly SignedPeerReachabilityLeaseV1[]; + readonly advertisements: readonly SignedPeerMeshMemberAdvertisementV1[]; readonly more: boolean; } | { readonly kind: 'sync-rejected'; readonly reason: 'unknown' }; @@ -147,6 +156,11 @@ type PeerMeshControlRequest = | LeavePeerMeshRequest | AnnouncePeerMeshRosterRequest; +interface LocalPeerMeshEvidence { + readonly reachability: SignedPeerReachabilityLeaseV1; + readonly advertisements: readonly SignedPeerMeshMemberAdvertisementV1[]; +} + export interface PeerMeshNode { localPeerId(): string; displayName(): string | undefined; @@ -177,7 +191,7 @@ export interface PeerMeshNode { export interface PeerMeshStatus { readonly role: 'authority' | 'member'; - readonly authority: PeerMeshAuthorityTarget; + readonly authorityPeerId: string; readonly roster: SignedPeerMeshRosterV1; readonly pendingInvitationCount: number; readonly memberRoutes: readonly PeerMeshMemberRouteStatus[]; @@ -187,7 +201,7 @@ export interface PeerMeshMemberRouteStatus { readonly peerId: string; readonly endpointKind?: 'client' | 'host'; readonly displayName?: string; - readonly state: 'local' | 'route_available' | 'coordination_only' | 'stale' | 'unknown'; + readonly state: 'local' | 'connecting' | 'reachable' | 'reconnecting' | 'needs_repair'; readonly expiresAt?: number; } @@ -224,7 +238,7 @@ export interface PeerMeshTransport { export async function openPeerMeshNode(input: { readonly dataRoot: string; readonly peer: PeerMeshTransport; - readonly reachability?: PeerReachabilityPublisher; + readonly reachability: PeerReachabilityPublisher; readonly endpointKind?: 'client' | 'host'; readonly now?: () => number; readonly onBackgroundReconcileError?: (error: unknown) => void; @@ -243,7 +257,7 @@ export async function openPeerMeshNode(input: { class PeerMeshNodeImpl implements PeerMeshNode { readonly #store: PeerMeshStateStore; readonly #peer: PeerMeshTransport; - readonly #reachability: PeerReachabilityPublisher | undefined; + readonly #reachability: PeerReachabilityPublisher; readonly #endpointKind: 'client' | 'host' | undefined; readonly #now: () => number; readonly #onBackgroundReconcileError: ((error: unknown) => void) | undefined; @@ -251,16 +265,21 @@ class PeerMeshNodeImpl implements PeerMeshNode { readonly #lifetime = new AbortController(); #admissionTail = Promise.resolve(); #reconcileTail = Promise.resolve(); + #transitTail = Promise.resolve(); #reconcileCursor = 0; #gossipCursor = 0; - #routeRefreshTask: Promise | undefined; + #evidenceRefreshTask: Promise | undefined; + #unsubscribeReachability: (() => void) | undefined; + #reconcileGeneration = 0; + readonly #reconcileWaiters = new Set<() => void>(); + readonly #recentlyReached = new Set(); #serveTask: Promise | undefined; #closeTask: Promise | undefined; constructor(input: { readonly store: PeerMeshStateStore; readonly peer: PeerMeshTransport; - readonly reachability?: PeerReachabilityPublisher; + readonly reachability: PeerReachabilityPublisher; readonly endpointKind?: 'client' | 'host'; readonly now?: () => number; readonly onBackgroundReconcileError?: (error: unknown) => void; @@ -274,7 +293,17 @@ class PeerMeshNodeImpl implements PeerMeshNode { } async initialize(): Promise { - for (const route of this.#store.read().routes) this.#assertRouteSignature(route); + const stored = this.#store.read(); + for (const lease of stored.reachability) { + this.#validateReachability(lease, lease.lease.peerId, true); + } + for (const advertisement of stored.advertisements) { + this.#assertAdvertisementSignature(advertisement); + } + await this.#refreshLocalEvidence(); + this.#unsubscribeReachability = this.#reachability.subscribe(() => { + this.#triggerReconciliation(); + }); await this.#reconcileTransit(); } @@ -293,21 +322,17 @@ class PeerMeshNodeImpl implements PeerMeshNode { const canonical = displayName === null ? null : canonicalPeerMeshDisplayName(displayName); await this.#store.mutate(async (current) => { if (current.displayName === canonical) return { state: current, result: undefined }; - const state = { ...current, displayName: canonical }; - const localPeerId = this.#peer.identity().peerId; - if (!state.meshes.some((mesh) => isActiveMembership(mesh, localPeerId))) { - return { state, result: undefined }; - } - const route = await this.#signLocalRoute(state); + const next = { ...current, displayName: canonical }; + const advertisements = await this.#localAdvertisementsFor(next); return { state: { - ...state, - routes: mergeRoutes(state.routes, [route], this.#now()), + ...next, + advertisements: mergeAdvertisements(current.advertisements, advertisements), }, result: undefined, }; }); - void this.reconcile().catch(() => undefined); + this.#triggerReconciliation(); }); } @@ -335,8 +360,9 @@ class PeerMeshNodeImpl implements PeerMeshNode { roster, targets: rosterAnnouncementTargets( state.roster.roster.members, - current.routes, + current.reachability, this.#peer.identity().peerId, + this.#now(), ), }, }; @@ -347,8 +373,10 @@ class PeerMeshNodeImpl implements PeerMeshNode { requireAuthority(stored.meshes, meshId), this.#peer.identity(), this.#endpointKind, - stored.routes, + stored.reachability, + stored.advertisements, this.#now(), + this.#recentlyReached, ); }); } @@ -361,7 +389,15 @@ class PeerMeshNodeImpl implements PeerMeshNode { stored.meshes .filter((state) => state.roster.roster.closed || isActiveMembership(state, identity.peerId)) .map((state) => - peerMeshStatus(state, identity, this.#endpointKind, stored.routes, this.#now()), + peerMeshStatus( + state, + identity, + this.#endpointKind, + stored.reachability, + stored.advertisements, + this.#now(), + this.#recentlyReached, + ), ), ); } @@ -386,47 +422,45 @@ class PeerMeshNodeImpl implements PeerMeshNode { authorityPrivateKey: keys.privateKey, invitations: [], }; - const signedRoute = await this.#signLocalRoute(); + const reachability = await this.#reachability.refresh(); const now = this.#now(); await this.#store.mutate((current) => { assertMeshCapacity(current.meshes, identity.peerId, current.pendingJoins.length); - const currentLocalRoute = current.routes - .filter(({ route }) => route.peerId === identity.peerId) - .sort((left, right) => right.route.sequence - left.route.sequence)[0]; - const localRoute = - currentLocalRoute && currentLocalRoute.route.sequence >= signedRoute.route.sequence - ? currentLocalRoute - : signedRoute; return { state: { ...current, meshes: appendMesh(current.meshes, state, identity.peerId), - routes: mergeRoutes(current.routes, [localRoute], now), + reachability: mergeReachability(current.reachability, [reachability], now), }, result: undefined, }; }); + await this.#refreshLocalEvidence(); const stored = this.#store.read(); return peerMeshStatus( findMesh(stored.meshes, state.roster.roster.meshId)!, identity, this.#endpointKind, - stored.routes, + stored.reachability, + stored.advertisements, now, + this.#recentlyReached, ); }); } - invite(meshId: string, input: { readonly ttlMs?: number } = {}): Promise { - if (this.#lifetime.signal.aborted) return Promise.reject(new Error('Peer Mesh node is closed')); + async invite( + meshId: string, + input: { readonly ttlMs?: number } = {}, + ): Promise { + if (this.#lifetime.signal.aborted) throw new Error('Peer Mesh node is closed'); const now = this.#now(); const identity = this.#peer.identity(); const ttlMs = input.ttlMs ?? DEFAULT_INVITATION_TTL_MS; if (!Number.isSafeInteger(ttlMs) || ttlMs < 1_000 || ttlMs > 24 * 60 * 60 * 1_000) { - return Promise.reject( - new Error('Peer Mesh invitation TTL must be between 1 second and 1 day'), - ); + throw new Error('Peer Mesh invitation TTL must be between 1 second and 1 day'); } + const reachability = await this.#reachability.refresh(); return this.#store.mutate((current) => { const state = requireAuthority(current.meshes, meshId); if (state.roster.roster.closed) throw new Error('Peer Mesh is closed'); @@ -442,14 +476,13 @@ class PeerMeshNodeImpl implements PeerMeshNode { throw new Error('Peer Mesh has too many recent invitations'); const secret = createPeerMeshInvitationSecret(); const expiresAt = now + ttlMs; - const target = authorityTarget(identity); const invitation: PeerMeshInvitationV1 = { version: 1, meshId: state.roster.roster.meshId, authorityPublicKey: state.roster.authorityPublicKey, secret, expiresAt, - ...target, + reachability, }; return { state: { @@ -474,6 +507,11 @@ class PeerMeshNodeImpl implements PeerMeshNode { join(invitationValue: PeerMeshInvitationV1, signal?: AbortSignal): Promise { return this.#admitMesh(async () => { const invitation = validatePeerMeshInvitation(invitationValue); + const authorityReachability = this.#validateReachability( + invitation.reachability, + invitation.reachability.lease.peerId, + true, + ); const current = this.#store.read(); const existing = findMesh(current.meshes, invitation.meshId); const pending = current.pendingJoins.find( @@ -507,14 +545,13 @@ class PeerMeshNodeImpl implements PeerMeshNode { try { stream = await this.#peer.connectMeshControl( { - peerId: invitation.peerId, - routeHints: invitation.routeHints, - coordinationRelays: invitation.coordinationRelays, + ...dialTarget(authorityReachability), directDeadlineMs: CONNECT_DEADLINE_MS, }, operationSignal, ); - const localRoute = (await this.#refreshLocalRoute()) ?? (await this.#signLocalRoute()); + const localReachability = await this.#reachability.refresh(); + const localAdvertisement = await this.#signLocalAdvertisement(invitation.meshId); await this.#store.mutate((current) => { const existing = findMesh(current.meshes, invitation.meshId); if (existing?.role === 'authority') { @@ -549,7 +586,13 @@ class PeerMeshNodeImpl implements PeerMeshNode { }; }); joinIntentAdmitted = true; - return await this.#redeemPendingJoin(invitation, localRoute, stream, operationSignal); + return await this.#redeemPendingJoin( + invitation, + localReachability, + localAdvertisement, + stream, + operationSignal, + ); } catch (error) { if (signal?.aborted && joinIntentAdmitted) { await this.#cancelPendingJoin(invitation.meshId); @@ -565,7 +608,8 @@ class PeerMeshNodeImpl implements PeerMeshNode { async #redeemPendingJoin( invitation: PeerMeshInvitationV1, - localRoute: SignedPeerMeshRouteRecordV1, + localReachability: SignedPeerReachabilityLeaseV1, + localAdvertisement: SignedPeerMeshMemberAdvertisementV1, stream: RuntimeHostPeerNativeStream, signal: AbortSignal, ): Promise { @@ -599,7 +643,8 @@ class PeerMeshNodeImpl implements PeerMeshNode { kind: 'redeem-invitation', meshId: invitation.meshId, secret: invitation.secret, - route: localRoute, + reachability: localReachability, + advertisement: localAdvertisement, }, decodeRedeemResponse, signal, @@ -617,7 +662,8 @@ class PeerMeshNodeImpl implements PeerMeshNode { ) { throw new Error('Peer Mesh authority returned an unrelated roster'); } - const routes = await this.#validateRoutes(response.routes, roster, this.#now()); + const reachability = this.#validateReachabilityPage(response.reachability, roster, true); + const advertisements = this.#validateAdvertisementPage(response.advertisements, roster); signal.throwIfAborted(); await this.#store.mutate((current) => { const pending = current.pendingJoins.find( @@ -638,9 +684,8 @@ class PeerMeshNodeImpl implements PeerMeshNode { const state: PeerMeshReplicaStateV1 = { role: 'replica', authority: { - peerId: invitation.peerId, - routeHints: invitation.routeHints, - coordinationRelays: invitation.coordinationRelays, + peerId: invitation.reachability.lease.peerId, + reachability: authorityReachabilityFor(invitation), }, roster: selectedRoster, desiredMembership: pending.phase === 'leave_pending' ? 'left' : 'active', @@ -652,20 +697,36 @@ class PeerMeshNodeImpl implements PeerMeshNode { ? replaceMesh(current.meshes, state) : appendMesh(current.meshes, state, identity.peerId), pendingJoins: current.pendingJoins.filter((candidate) => candidate !== pending), - routes: mergeRoutes(current.routes, [...routes, localRoute], this.#now()), + reachability: mergeReachability( + current.reachability, + [...reachability, localReachability], + this.#now(), + ), + advertisements: mergeAdvertisements(current.advertisements, [ + ...advertisements, + localAdvertisement, + ]), }, result: undefined, }; }); signal.throwIfAborted(); - await this.#refreshLocalRoute(); + await this.#refreshLocalEvidence(); signal.throwIfAborted(); await this.#reconcileTransit(); signal.throwIfAborted(); const stored = this.#store.read(); const state = findMesh(stored.meshes, invitation.meshId); if (!state) throw new Error('Peer Mesh join was not retained'); - return peerMeshStatus(state, identity, this.#endpointKind, stored.routes, this.#now()); + return peerMeshStatus( + state, + identity, + this.#endpointKind, + stored.reachability, + stored.advertisements, + this.#now(), + this.#recentlyReached, + ); } #discardPendingJoin(invitation: PeerMeshInvitationV1): Promise { @@ -704,16 +765,21 @@ class PeerMeshNodeImpl implements PeerMeshNode { async #resumePendingJoin(pending: PendingPeerMeshJoin, signal: AbortSignal): Promise { const stream = await this.#peer.connectMeshControl( { - peerId: pending.invitation.peerId, - routeHints: pending.invitation.routeHints, - coordinationRelays: pending.invitation.coordinationRelays, + ...dialTarget(pending.invitation.reachability), directDeadlineMs: CONNECT_DEADLINE_MS, }, signal, ); try { - const localRoute = (await this.#refreshLocalRoute()) ?? (await this.#signLocalRoute()); - await this.#redeemPendingJoin(pending.invitation, localRoute, stream, signal); + const localReachability = await this.#reachability.refresh(); + const localAdvertisement = await this.#signLocalAdvertisement(pending.invitation.meshId); + await this.#redeemPendingJoin( + pending.invitation, + localReachability, + localAdvertisement, + stream, + signal, + ); } finally { await stream.close().catch(() => undefined); } @@ -772,7 +838,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { setTransitMesh(meshId: string | null): Promise { return this.#admitMesh(async () => { const localPeerId = this.#peer.identity().peerId; - await this.#store.mutate((current) => { + await this.#store.mutate(async (current) => { if ( meshId !== null && !current.meshes.some( @@ -781,16 +847,21 @@ class PeerMeshNodeImpl implements PeerMeshNode { ) { throw new Error('Transit requires an active Peer Mesh membership'); } + if (current.transitMeshId === meshId) return { state: current, result: undefined }; + const next = { ...current, transitMeshId: meshId }; + const advertisements = await this.#localAdvertisementsFor(next); return { - state: { ...current, transitMeshId: meshId }, + state: { + ...next, + advertisements: mergeAdvertisements(current.advertisements, advertisements), + }, result: undefined, }; }); try { await this.#reconcileTransit(); - await this.#refreshLocalRoute(); } finally { - void this.reconcile().catch(() => undefined); + this.#triggerReconciliation(); } }); } @@ -818,32 +889,17 @@ class PeerMeshNodeImpl implements PeerMeshNode { .map(({ roster }) => roster.roster.meshId); const visible = sharedMeshIds.length > 0; if (!visible) return undefined; - const route = stored.routes - .filter(({ route }) => route.peerId === peerId && route.expiresAt > now) - .sort((left, right) => right.route.sequence - left.route.sequence)[0]?.route; + const reachability = latestReachability(stored.reachability, peerId, now, true)?.lease; const localPeerId = this.#peer.identity().peerId; const transitRelayPeerIds = transitRelayCandidates( - stored.routes - .filter( - ({ route: candidate }) => - candidate.peerId !== localPeerId && - candidate.peerId !== peerId && - candidate.expiresAt > now && - candidate.transitMeshId !== undefined && - sharedMeshIds.includes(candidate.transitMeshId) && - isActiveMeshMember( - stored.meshes, - candidate.transitMeshId, - localPeerId, - candidate.peerId, - ), - ) - .sort((left, right) => left.route.peerId.localeCompare(right.route.peerId)), + eligibleTransitEvidence(stored, localPeerId, now).filter( + ({ meshId, lease }) => lease.lease.peerId !== peerId && sharedMeshIds.includes(meshId), + ), ).map(({ peerId: relayPeerId }) => relayPeerId); - if (!route && transitRelayPeerIds.length === 0) return undefined; + if (!reachability && transitRelayPeerIds.length === 0) return undefined; return Object.freeze({ - routeHints: route?.routeHints ?? [], - coordinationRelays: route?.coordinationRelays ?? [], + routeHints: reachability?.directRoutes ?? [], + coordinationRelays: reachability?.coordinationRoutes ?? [], transitRelayPeerIds: Object.freeze(transitRelayPeerIds), }); } @@ -912,16 +968,21 @@ class PeerMeshNodeImpl implements PeerMeshNode { async #close(): Promise { this.#lifetime.abort(); + this.#unsubscribeReachability?.(); + this.#unsubscribeReachability = undefined; + for (const wake of this.#reconcileWaiters) wake(); + this.#reconcileWaiters.clear(); await this.#serveTask?.catch(() => undefined); for (const stream of this.#activeControlStreams) stream.abort(); this.#activeControlStreams.clear(); - await Promise.all([this.#admissionTail, this.#reconcileTail]); + await Promise.all([this.#admissionTail, this.#reconcileTail, this.#transitTail]); return this.#store.close(); } async #runReconciliation(signal: AbortSignal): Promise { let failureReported = false; while (!signal.aborted) { + const observedGeneration = this.#reconcileGeneration; try { await this.reconcile(signal); failureReported = false; @@ -935,30 +996,32 @@ class PeerMeshNodeImpl implements PeerMeshNode { } } } - await this.#waitForReconciliationTrigger(signal).catch(() => undefined); + await this.#waitForReconciliationTrigger(observedGeneration, signal).catch(() => undefined); } } - async #waitForReconciliationTrigger(signal: AbortSignal): Promise { - let remainingMs = RECONCILE_INTERVAL_MS; - while (!signal.aborted && remainingMs > 0) { - const waitMs = Math.min(LOCAL_ROUTE_OBSERVATION_INTERVAL_MS, remainingMs); - await delay(waitMs, undefined, { signal }); - if (this.#localRouteRequiresRefresh()) return; - remainingMs -= waitMs; + async #waitForReconciliationTrigger( + observedGeneration: number, + signal: AbortSignal, + ): Promise { + if (this.#reconcileGeneration !== observedGeneration) return; + let wake!: () => void; + const triggered = new Promise((resolve) => { + wake = resolve; + this.#reconcileWaiters.add(wake); + }); + if (this.#reconcileGeneration !== observedGeneration) wake(); + try { + await Promise.race([triggered, delay(RECONCILE_INTERVAL_MS, undefined, { signal })]); + } finally { + this.#reconcileWaiters.delete(wake); } } - #localRouteRequiresRefresh(): boolean { - const identity = this.#peer.identity(); - const current = this.#store.read(); - if (!current.meshes.some((state) => isActiveMembership(state, identity.peerId))) { - return false; - } - const existing = current.routes - .filter(({ route }) => route.peerId === identity.peerId) - .sort((left, right) => right.route.sequence - left.route.sequence)[0]; - return !isCurrentLocalRoute(existing, identity, current, this.#endpointKind, this.#now()); + #triggerReconciliation(): void { + this.#reconcileGeneration += 1; + for (const wake of this.#reconcileWaiters) wake(); + this.#reconcileWaiters.clear(); } async #reconcile(signal?: AbortSignal): Promise { @@ -967,23 +1030,20 @@ class PeerMeshNodeImpl implements PeerMeshNode { : this.#lifetime.signal; lifetimeSignal.throwIfAborted(); await this.#reconcileTransit(); - await this.#refreshLocalRoute(); + await this.#refreshLocalEvidence(); const identity = this.#peer.identity(); const stored = this.#store.read(); const memberships = stored.meshes.filter( - (state): state is PeerMeshReplicaStateV1 => - state.role === 'replica' && - !state.roster.roster.closed && - state.roster.roster.members.includes(identity.peerId), + (state) => + !state.roster.roster.closed && state.roster.roster.members.includes(identity.peerId), ); const pending: Array< | { readonly kind: 'join'; readonly join: PendingPeerMeshJoin } | { readonly kind: 'membership'; readonly meshId: string; - readonly targets: readonly PeerMeshAuthorityTarget[]; - readonly authorityRouteExpired: boolean; - readonly desiredMembership: PeerMeshReplicaStateV1['desiredMembership']; + readonly target: PeerMeshAuthorityTarget; + readonly desiredMembership: 'active' | 'left'; readonly roster: SignedPeerMeshRosterV1; } > = stored.pendingJoins.map((join) => ({ kind: 'join', join })); @@ -991,26 +1051,38 @@ class PeerMeshNodeImpl implements PeerMeshNode { this.#gossipCursor = (this.#gossipCursor + 1) % PEER_MESH_MAX_MEMBERS; const now = this.#now(); for (const [index, state] of memberships.entries()) { - const authorityRoute = stored.routes.find( - ({ route }) => route.peerId === state.authority.peerId, - ); - const targets = [currentAuthorityTarget(state, stored.routes)]; - const gossipRoutes = state.roster.roster.members - .filter((peerId) => peerId !== identity.peerId && peerId !== state.authority.peerId) - .flatMap((peerId) => { - const route = stored.routes.find((candidate) => candidate.route.peerId === peerId)?.route; - return route ? [route] : []; + const meshId = state.roster.roster.meshId; + const desiredMembership = state.role === 'replica' ? state.desiredMembership : 'active'; + const authority = + state.role === 'replica' + ? currentAuthorityTarget(state, stored.reachability, now) + : undefined; + if (authority) { + pending.push({ + kind: 'membership', + meshId, + target: authority, + desiredMembership, + roster: state.roster, }); - if (gossipRoutes.length > 0) { - targets.push(gossipRoutes[(gossipCursor + index) % gossipRoutes.length]!); } + if (desiredMembership === 'left') continue; + const rotatingTargets = state.roster.roster.members + .filter( + (peerId) => + peerId !== identity.peerId && + (state.role === 'authority' || peerId !== state.authority.peerId), + ) + .flatMap((peerId) => { + const target = peerTarget(peerId, stored.reachability, now); + return target ? [target] : []; + }); + if (rotatingTargets.length === 0) continue; pending.push({ kind: 'membership', - meshId: state.roster.roster.meshId, - targets, - authorityRouteExpired: - authorityRoute !== undefined && authorityRoute.route.expiresAt <= now, - desiredMembership: state.desiredMembership, + meshId, + target: rotatingTargets[(gossipCursor + index) % rotatingTargets.length]!, + desiredMembership, roster: state.roster, }); } @@ -1032,21 +1104,21 @@ class PeerMeshNodeImpl implements PeerMeshNode { } else if (operation.desiredMembership === 'left') { await this.#notifyLeave( operation.meshId, - operation.targets[0]!, + operation.target, operation.roster, operationSignal, ); } else { - await this.#syncTargets( - operation.meshId, - operation.targets, - operation.authorityRouteExpired, - operationSignal, - ); + await this.#syncPeer(operation.meshId, operation.target, operationSignal); } } catch (error) { if (lifetimeSignal.aborted) lifetimeSignal.throwIfAborted(); - failures.push(error); + if (operation.kind === 'membership') { + this.#recentlyReached.delete(operation.target.peerId); + } + if (operation.kind === 'join' || operation.desiredMembership === 'left') { + failures.push(error); + } if (deadline.aborted) return; } } @@ -1072,7 +1144,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { signal: AbortSignal, ): Promise { const stream = await this.#peer.connectMeshControl( - { ...target, directDeadlineMs: CONNECT_DEADLINE_MS }, + { ...dialTarget(target.reachability), directDeadlineMs: CONNECT_DEADLINE_MS }, signal, ); try { @@ -1085,75 +1157,38 @@ class PeerMeshNodeImpl implements PeerMeshNode { if (response.kind === 'leave-rejected') { throw new Error('Peer Mesh authority rejected the leave request'); } - await this.#applySync(meshId, response.roster, []); + await this.#applySync(meshId, response.roster, [], []); + this.#recentlyReached.add(target.peerId); } finally { await stream.close().catch(() => undefined); } } - async #syncTargets( - meshId: string, - targets: readonly PeerMeshAuthorityTarget[], - authorityRouteExpired: boolean, - signal: AbortSignal, - ): Promise { - if (!authorityRouteExpired || targets.length === 1) { - try { - await this.#syncPeer(meshId, targets[0]!, signal); - } catch (authorityFailure) { - const fallbackTargets = targets.slice(1); - if (fallbackTargets.length === 0) throw authorityFailure; - await Promise.any(fallbackTargets.map((target) => this.#syncPeer(meshId, target, signal))); - } - return; - } - const controllers = targets.map(() => new AbortController()); - const authorityPeerId = targets[0]!.peerId; - const attempts = targets.map(async (target, index) => { - await this.#syncPeer(meshId, target, AbortSignal.any([signal, controllers[index]!.signal])); - if (index > 0 && !this.#hasFreshRoute(meshId, authorityPeerId)) { - throw new Error('Peer Mesh gossip did not recover the authority route'); - } - }); - try { - await Promise.any(attempts); - } finally { - for (const controller of controllers) controller.abort(); - await Promise.allSettled(attempts); - } - } - - #hasFreshRoute(meshId: string, peerId: string): boolean { - const stored = this.#store.read(); - const mesh = findMesh(stored.meshes, meshId); - return Boolean( - mesh && - isActiveMembership(mesh, this.#peer.identity().peerId) && - stored.routes.some(({ route }) => route.peerId === peerId && route.expiresAt > this.#now()), - ); - } - async #syncPeer( meshId: string, target: PeerMeshAuthorityTarget, signal: AbortSignal, ): Promise { - for (let page = 0; page <= Math.ceil(PEER_MESH_MAX_MEMBERS / ROUTE_PAGE_SIZE); page += 1) { - await this.#refreshLocalRoute(); + for (let page = 0; page <= PEER_MESH_MAX_MEMBERS; page += 1) { + await this.#refreshLocalEvidence(); const stored = this.#store.read(); const state = findMesh(stored.meshes, meshId); const localPeerId = this.#peer.identity().peerId; if (!state || !isActiveMembership(state, localPeerId)) return; - const route = stored.routes.find((candidate) => candidate.route.peerId === localPeerId); - if (!route) throw new Error('Peer Mesh local route is unavailable'); + const reachability = latestReachability(stored.reachability, localPeerId, this.#now(), true); + const advertisement = findAdvertisement(stored.advertisements, meshId, localPeerId); + if (!reachability || !advertisement) { + throw new Error('Peer Mesh local evidence is unavailable'); + } const discovered = this.resolveRoutes(target.peerId); + const targetRoutes = dialTarget(target.reachability); const stream = await this.#peer.connectMeshControl( { peerId: target.peerId, - routeHints: mergeAddresses(discovered?.routeHints ?? [], target.routeHints), + routeHints: mergeAddresses(discovered?.routeHints ?? [], targetRoutes.routeHints), coordinationRelays: mergeAddresses( discovered?.coordinationRelays ?? [], - target.coordinationRelays, + targetRoutes.coordinationRelays, ), transitRelayPeerIds: discovered?.transitRelayPeerIds, directDeadlineMs: CONNECT_DEADLINE_MS, @@ -1167,8 +1202,14 @@ class PeerMeshNodeImpl implements PeerMeshNode { kind: 'sync', meshId, roster: state.roster, - route, - knownRoutes: routeSequences(stored.routes, state.roster, this.#now()), + reachability, + advertisement, + knownReachability: reachabilityRevisions( + stored.reachability, + state.roster, + this.#now(), + ), + knownAdvertisements: advertisementRevisions(stored.advertisements, state.roster), }, decodeSyncResponse, signal, @@ -1176,7 +1217,13 @@ class PeerMeshNodeImpl implements PeerMeshNode { if (response.kind === 'sync-rejected') { throw new Error(`Peer Mesh synchronization was rejected: ${response.reason}`); } - await this.#applySync(meshId, response.roster, response.routes); + await this.#applySync( + meshId, + response.roster, + response.reachability, + response.advertisements, + ); + this.#recentlyReached.add(target.peerId); if (!response.more) return; } finally { await stream.close().catch(() => undefined); @@ -1185,120 +1232,178 @@ class PeerMeshNodeImpl implements PeerMeshNode { throw new Error('Peer Mesh synchronization exceeded its page bound'); } - #refreshLocalRoute(): Promise { - this.#routeRefreshTask ??= this.#refreshLocalRouteOnce().finally(() => { - this.#routeRefreshTask = undefined; + #refreshLocalEvidence(): Promise { + this.#evidenceRefreshTask ??= this.#refreshLocalEvidenceOnce().finally(() => { + this.#evidenceRefreshTask = undefined; }); - return this.#routeRefreshTask; + return this.#evidenceRefreshTask; } - async #refreshLocalRouteOnce(): Promise { - await this.#reachability?.refresh(); + async #refreshLocalEvidenceOnce(): Promise { + const reachability = await this.#reachability.refresh(); const identity = this.#peer.identity(); const now = this.#now(); return this.#store.mutate(async (current) => { - if (!current.meshes.some((state) => isActiveMembership(state, identity.peerId))) { + const active = current.meshes.filter((state) => isActiveMembership(state, identity.peerId)); + if (active.length === 0) { return { state: current, result: undefined }; } - const existing = current.routes - .filter(({ route }) => route.peerId === identity.peerId) - .sort((left, right) => right.route.sequence - left.route.sequence)[0]; - if (isCurrentLocalRoute(existing, identity, current, this.#endpointKind, now)) { - return { state: current, result: existing }; - } - const route = await this.#signLocalRoute(current); + const advertisements = await this.#localAdvertisementsFor(current); return { - state: { ...current, routes: mergeRoutes(current.routes, [route], now) }, - result: route, + state: { + ...current, + reachability: mergeReachability(current.reachability, [reachability], now), + advertisements: mergeAdvertisements(current.advertisements, advertisements), + }, + result: Object.freeze({ reachability, advertisements: Object.freeze(advertisements) }), }; }); } - async #signLocalRoute( + async #localAdvertisementsFor( + stored: PeerMeshStoredStateV1, + ): Promise { + const identity = this.#peer.identity(); + const advertisements: SignedPeerMeshMemberAdvertisementV1[] = []; + for (const state of stored.meshes) { + if (!isActiveMembership(state, identity.peerId)) continue; + const meshId = state.roster.roster.meshId; + const existing = findAdvertisement(stored.advertisements, meshId, identity.peerId); + advertisements.push( + isCurrentLocalAdvertisement(existing, meshId, identity.peerId, stored, this.#endpointKind) + ? existing + : await this.#signLocalAdvertisement(meshId, stored), + ); + } + return Object.freeze(advertisements); + } + + async #signLocalAdvertisement( + meshId: string, stored: PeerMeshStoredStateV1 = this.#store.read(), - ): Promise { + ): Promise { const identity = this.#peer.identity(); - const maxSequence = stored.routes - .filter(({ route }) => route.peerId === identity.peerId) - .reduce((maximum, { route }) => Math.max(maximum, route.sequence), 0); - const route = canonicalPeerMeshRouteRecord({ + const maxRevision = stored.advertisements + .filter( + ({ advertisement }) => + advertisement.meshId === meshId && advertisement.peerId === identity.peerId, + ) + .reduce((maximum, { advertisement }) => Math.max(maximum, advertisement.revision), 0); + const advertisement = canonicalPeerMeshMemberAdvertisement({ version: 1, + meshId, peerId: identity.peerId, - sequence: maxSequence + 1, - expiresAt: this.#now() + ROUTE_TTL_MS, - routeHints: identity.listenAddresses, - coordinationRelays: identity.coordinationRelays, + revision: maxRevision + 1, ...(this.#endpointKind ? { endpointKind: this.#endpointKind } : {}), ...(stored.displayName ? { displayName: stored.displayName } : {}), - ...(stored.transitMeshId ? { transitMeshId: stored.transitMeshId } : {}), + offersTransit: stored.transitMeshId === meshId, }); - const proof = await this.#peer.signIdentity(peerMeshRouteRecordSigningBytes(route)); - const signed = decodeSignedPeerMeshRouteRecord({ - route, + const proof = await this.#peer.signIdentity( + peerMeshMemberAdvertisementSigningBytes(advertisement), + ); + const signed = decodeSignedPeerMeshMemberAdvertisement({ + advertisement, publicKey: proof.publicKey.toString('base64url'), signature: proof.signature.toString('base64url'), }); - this.#assertRouteSignature(signed); + this.#assertAdvertisementSignature(signed); return signed; } - #validateRoutes( - values: readonly SignedPeerMeshRouteRecordV1[], + #validateReachabilityPage( + values: readonly SignedPeerReachabilityLeaseV1[], roster: SignedPeerMeshRosterV1, - now: number, - ): readonly SignedPeerMeshRouteRecordV1[] { - if (values.length > PEER_MESH_MAX_MEMBERS) throw new Error('Too many Peer Mesh routes'); - const routes = values.map(decodeSignedPeerMeshRouteRecord); - if (new Set(routes.map(({ route }) => route.peerId)).size !== routes.length) { - throw new Error('Duplicate Peer Mesh routes'); + allowExpired: boolean, + ): readonly SignedPeerReachabilityLeaseV1[] { + if (values.length > EVIDENCE_PAGE_SIZE) { + throw new Error('Too many Peer Mesh reachability leases'); } - for (const signed of routes) { - if (!roster.roster.members.includes(signed.route.peerId)) { - throw new Error('Peer Mesh route is outside the active roster or lifetime'); + const reachability = values.map((value) => { + const signed = decodeSignedPeerReachabilityLease(value); + if (!roster.roster.members.includes(signed.lease.peerId)) { + throw new Error('Peer Mesh reachability is outside the active roster'); } - this.#validateRemoteRoute(signed, signed.route.peerId, now); + return this.#validateReachability(signed, signed.lease.peerId, allowExpired); + }); + if (new Set(reachability.map(({ lease }) => lease.peerId)).size !== reachability.length) { + throw new Error('Duplicate Peer Mesh reachability leases'); } - return Object.freeze(routes); + return Object.freeze(reachability); } - #validateRemoteRoute( - value: SignedPeerMeshRouteRecordV1, + #validateReachability( + value: SignedPeerReachabilityLeaseV1, expectedPeerId: string, - now = this.#now(), - ): SignedPeerMeshRouteRecordV1 { - const signed = decodeSignedPeerMeshRouteRecord(value); - if ( - signed.route.peerId !== expectedPeerId || - signed.route.expiresAt <= now || - signed.route.expiresAt > now + ROUTE_MAX_FUTURE_MS - ) { - throw new Error('Peer Mesh route is outside the authenticated peer or lifetime'); + allowExpired = false, + ): SignedPeerReachabilityLeaseV1 { + const signed = this.#reachability.verify(value, expectedPeerId, { allowExpired }); + if (allowExpired && signed.lease.expiresAt + REACHABILITY_HISTORY_MS <= this.#now()) { + throw new Error('Peer Mesh reachability is outside the recovery horizon'); } - this.#assertRouteSignature(signed); return signed; } - #assertRouteSignature(signedValue: SignedPeerMeshRouteRecordV1): void { - const signed = decodeSignedPeerMeshRouteRecord(signedValue); + #validateAdvertisementPage( + values: readonly SignedPeerMeshMemberAdvertisementV1[], + roster: SignedPeerMeshRosterV1, + ): readonly SignedPeerMeshMemberAdvertisementV1[] { + if (values.length > EVIDENCE_PAGE_SIZE) { + throw new Error('Too many Peer Mesh member advertisements'); + } + const advertisements = values.map((value) => { + const signed = decodeSignedPeerMeshMemberAdvertisement(value); + if ( + signed.advertisement.meshId !== roster.roster.meshId || + !roster.roster.members.includes(signed.advertisement.peerId) + ) { + throw new Error('Peer Mesh member advertisement is outside the active roster'); + } + this.#assertAdvertisementSignature(signed); + return signed; + }); + const keys = advertisements.map(({ advertisement }) => advertisement.peerId); + if (new Set(keys).size !== keys.length) { + throw new Error('Duplicate Peer Mesh member advertisements'); + } + return Object.freeze(advertisements); + } + + #validateAdvertisementForPeer( + value: SignedPeerMeshMemberAdvertisementV1, + meshId: string, + peerId: string, + ): SignedPeerMeshMemberAdvertisementV1 { + const signed = decodeSignedPeerMeshMemberAdvertisement(value); + if (signed.advertisement.meshId !== meshId || signed.advertisement.peerId !== peerId) { + throw new Error('Peer Mesh member advertisement belongs to a different member'); + } + this.#assertAdvertisementSignature(signed); + return signed; + } + + #assertAdvertisementSignature(signedValue: SignedPeerMeshMemberAdvertisementV1): void { + const signed = decodeSignedPeerMeshMemberAdvertisement(signedValue); const valid = this.#peer.verifyIdentity( - signed.route.peerId, - peerMeshRouteRecordSigningBytes(signed.route), + signed.advertisement.peerId, + peerMeshMemberAdvertisementSigningBytes(signed.advertisement), { publicKey: Buffer.from(signed.publicKey, 'base64url'), signature: Buffer.from(signed.signature, 'base64url'), }, ); - if (!valid) throw new Error('Peer Mesh route signature is invalid'); + if (!valid) throw new Error('Peer Mesh member advertisement signature is invalid'); } async #applySync( meshId: string, rosterValue: SignedPeerMeshRosterV1, - routeValues: readonly SignedPeerMeshRouteRecordV1[], + reachabilityValues: readonly SignedPeerReachabilityLeaseV1[], + advertisementValues: readonly SignedPeerMeshMemberAdvertisementV1[], ): Promise { const roster = decodeSignedPeerMeshRoster(rosterValue); if (roster.roster.meshId !== meshId) throw new Error('Peer Mesh synchronization changed Mesh'); - const routes = this.#validateRoutes(routeValues, roster, this.#now()); + const reachability = this.#validateReachabilityPage(reachabilityValues, roster, true); + const advertisements = this.#validateAdvertisementPage(advertisementValues, roster); const localPeerId = this.#peer.identity().peerId; await this.#store.mutate((current) => { const state = findMesh(current.meshes, meshId); @@ -1314,14 +1419,17 @@ class PeerMeshNodeImpl implements PeerMeshNode { state: { ...current, meshes: replaceMesh(current.meshes, next), - routes: !isActiveMembership(next, localPeerId) - ? current.routes - : mergeRoutes(current.routes, routes, this.#now()), + reachability: !isActiveMembership(next, localPeerId) + ? current.reachability + : mergeReachability(current.reachability, reachability, this.#now()), + advertisements: !isActiveMembership(next, localPeerId) + ? current.advertisements + : mergeAdvertisements(current.advertisements, advertisements), }, result: undefined, }; }); - await this.#refreshLocalRoute(); + await this.#refreshLocalEvidence(); await this.#reconcileTransit(); } @@ -1378,8 +1486,9 @@ class PeerMeshNodeImpl implements PeerMeshNode { roster, targets: rosterAnnouncementTargets( state.roster.roster.members, - current.routes, + current.reachability, this.#peer.identity().peerId, + this.#now(), ), }, }; @@ -1397,7 +1506,10 @@ class PeerMeshNodeImpl implements PeerMeshNode { findMesh(stored.meshes, meshId)!, this.#peer.identity(), this.#endpointKind, - stored.routes, + stored.reachability, + stored.advertisements, + this.#now(), + this.#recentlyReached, ); } @@ -1410,7 +1522,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { void Promise.allSettled( targets.map(async (target) => { const stream = await this.#peer.connectMeshControl( - { ...target, directDeadlineMs: CONNECT_DEADLINE_MS }, + { ...dialTarget(target.reachability), directDeadlineMs: CONNECT_DEADLINE_MS }, signal, ); try { @@ -1427,6 +1539,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { if (response.kind === 'roster-rejected') { throw new Error('Peer Mesh roster announcement was rejected'); } + this.#recentlyReached.add(target.peerId); } finally { await stream.close().catch(() => undefined); } @@ -1435,7 +1548,8 @@ class PeerMeshNodeImpl implements PeerMeshNode { } #scheduleMaintenance(): void { - void this.#refreshLocalRoute().catch(() => undefined); + this.#triggerReconciliation(); + void this.#refreshLocalEvidence().catch(() => undefined); void this.#reconcileTransit().catch(() => undefined); } @@ -1481,11 +1595,12 @@ class PeerMeshNodeImpl implements PeerMeshNode { | LeavePeerMeshResponse | AnnouncePeerMeshRosterResponse; if (request.kind === 'redeem-invitation') { - await this.#refreshLocalRoute(); + await this.#refreshLocalEvidence(); response = await this.#redeem( request, stream.peerId, - this.#validateRemoteRoute(request.route, stream.peerId), + this.#validateReachability(request.reachability, stream.peerId), + this.#validateAdvertisementForPeer(request.advertisement, request.meshId, stream.peerId), ); } else if (request.kind === 'sync') { response = await this.#sync(request, stream.peerId); @@ -1531,7 +1646,8 @@ class PeerMeshNodeImpl implements PeerMeshNode { async #redeem( request: RedeemInvitationRequest, remotePeerId: string, - remoteRoute: SignedPeerMeshRouteRecordV1, + remoteReachability: SignedPeerReachabilityLeaseV1, + remoteAdvertisement: SignedPeerMeshMemberAdvertisementV1, ): Promise { const now = this.#now(); const response = await this.#store.mutate((current) => { @@ -1551,21 +1667,25 @@ class PeerMeshNodeImpl implements PeerMeshNode { ) { return { state: current, result: rejected('invalid') }; } - const updated = { - ...state, - }; - const routes = mergeAuthenticatedRoute(current.routes, remoteRoute, now); + const reachability = mergeAuthenticatedReachability( + current.reachability, + remoteReachability, + now, + ); + const advertisements = mergeAdvertisements(current.advertisements, [remoteAdvertisement]); + const evidence = initialEvidence( + state, + reachability, + advertisements, + this.#peer.identity().peerId, + now, + ); return { - state: { ...current, meshes: replaceMesh(current.meshes, updated), routes }, + state: { ...current, reachability, advertisements }, result: { kind: 'invitation-redeemed', - roster: updated.roster, - routes: responseRoutes( - updated, - routes, - [{ peerId: remotePeerId, sequence: Number.MAX_SAFE_INTEGER }], - now, - ).routes, + roster: state.roster, + ...evidence, }, }; } @@ -1612,40 +1732,17 @@ class PeerMeshNodeImpl implements PeerMeshNode { result: rejected('full'), }; } - if (state.roster.roster.members.includes(remotePeerId)) { - const updated = { - ...state, - invitations: [ - ...remaining.filter( - (record) => record.status === 'pending' || record.peerId !== remotePeerId, - ), - redeemedInvitation(invitation, remotePeerId), - ], - }; - const routes = mergeAuthenticatedRoute(current.routes, remoteRoute, now); - return { - state: { ...current, meshes: replaceMesh(current.meshes, updated), routes }, - result: { - kind: 'invitation-redeemed', - roster: state.roster, - routes: responseRoutes( - updated, - routes, - [{ peerId: remotePeerId, sequence: Number.MAX_SAFE_INTEGER }], - now, - ).routes, - }, - }; - } - const members = [...state.roster.roster.members, remotePeerId].sort(); - const roster = signPeerMeshRoster( - { - ...state.roster.roster, - revision: state.roster.roster.revision + 1, - members, - }, - authorityKeys(state), - ); + const existingMember = state.roster.roster.members.includes(remotePeerId); + const roster = existingMember + ? state.roster + : signPeerMeshRoster( + { + ...state.roster.roster, + revision: state.roster.roster.revision + 1, + members: [...state.roster.roster.members, remotePeerId].sort(), + }, + authorityKeys(state), + ); const updated = { ...state, roster, @@ -1656,18 +1753,30 @@ class PeerMeshNodeImpl implements PeerMeshNode { redeemedInvitation(invitation, remotePeerId), ], }; - const routes = mergeRoutes(current.routes, [remoteRoute], now); + const reachability = mergeAuthenticatedReachability( + current.reachability, + remoteReachability, + now, + ); + const advertisements = mergeAdvertisements(current.advertisements, [remoteAdvertisement]); + const evidence = initialEvidence( + updated, + reachability, + advertisements, + this.#peer.identity().peerId, + now, + ); return { - state: { ...current, meshes: replaceMesh(current.meshes, updated), routes }, + state: { + ...current, + meshes: replaceMesh(current.meshes, updated), + reachability, + advertisements, + }, result: { kind: 'invitation-redeemed', roster, - routes: responseRoutes( - updated, - routes, - [{ peerId: remotePeerId, sequence: Number.MAX_SAFE_INTEGER }], - now, - ).routes, + ...evidence, }, }; }); @@ -1679,11 +1788,13 @@ class PeerMeshNodeImpl implements PeerMeshNode { state.roster, rosterAnnouncementTargets( state.roster.roster.members, - stored.routes, + stored.reachability, this.#peer.identity().peerId, + this.#now(), ).filter(({ peerId }) => peerId !== remotePeerId), ); } + this.#recentlyReached.add(remotePeerId); } return response; } @@ -1739,8 +1850,9 @@ class PeerMeshNodeImpl implements PeerMeshNode { state.roster, rosterAnnouncementTargets( state.roster.roster.members, - stored.routes, + stored.reachability, this.#peer.identity().peerId, + this.#now(), ), ); } @@ -1749,8 +1861,13 @@ class PeerMeshNodeImpl implements PeerMeshNode { } async #sync(request: SyncPeerMeshRequest, remotePeerId: string): Promise { - const remoteRoute = this.#validateRemoteRoute(request.route, remotePeerId); - await this.#refreshLocalRoute(); + const remoteReachability = this.#validateReachability(request.reachability, remotePeerId); + const remoteAdvertisement = this.#validateAdvertisementForPeer( + request.advertisement, + request.meshId, + remotePeerId, + ); + await this.#refreshLocalEvidence(); const incomingRoster = decodeSignedPeerMeshRoster(request.roster); const response = await this.#store.mutate((current) => { const state = findMesh(current.meshes, request.meshId); @@ -1768,37 +1885,70 @@ class PeerMeshNodeImpl implements PeerMeshNode { }; const localMember = isActiveMembership(updated, localPeerId); const remoteMember = !roster.roster.closed && roster.roster.members.includes(remotePeerId); - const routes = + const reachability = + localMember && remoteMember + ? mergeAuthenticatedReachability(current.reachability, remoteReachability, this.#now()) + : current.reachability; + const advertisements = localMember && remoteMember - ? mergeRoutes(current.routes, [remoteRoute], this.#now()) - : current.routes; + ? mergeAdvertisements(current.advertisements, [remoteAdvertisement]) + : current.advertisements; if (!localMember || !remoteMember) { return { - state: { ...current, meshes: replaceMesh(current.meshes, updated), routes }, + state: { + ...current, + meshes: replaceMesh(current.meshes, updated), + reachability, + advertisements, + }, result: { kind: 'sync-result', roster, - routes: [], + reachability: [], + advertisements: [], more: false, } as const, }; } - const page = responseRoutes(updated, routes, request.knownRoutes, this.#now()); + const page = responseEvidence( + updated, + reachability, + advertisements, + request.knownReachability, + request.knownAdvertisements, + this.#now(), + ); return { - state: { ...current, meshes: replaceMesh(current.meshes, updated), routes }, + state: { + ...current, + meshes: replaceMesh(current.meshes, updated), + reachability, + advertisements, + }, result: { kind: 'sync-result', roster, - routes: page.routes, - more: page.more, + ...page, } as const, }; }); - if (response.kind === 'sync-result') await this.#reconcileTransit(); + if (response.kind === 'sync-result') { + this.#recentlyReached.add(remotePeerId); + await this.#reconcileTransit(); + } return response; } - async #reconcileTransit(): Promise { + #reconcileTransit(): Promise { + const task = this.#transitTail.then(() => this.#applyTransitSnapshot()); + this.#transitTail = task.then( + () => undefined, + () => undefined, + ); + return task; + } + + async #applyTransitSnapshot(): Promise { const stored = this.#store.read(); const localPeerId = this.#peer.identity().peerId; const now = this.#now(); @@ -1806,19 +1956,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { (mesh) => mesh.roster.roster.meshId === stored.transitMeshId && isActiveMembership(mesh, localPeerId), ); - const eligibleRelays = stored.routes - .filter(({ route }) => { - if ( - route.peerId === localPeerId || - route.expiresAt <= now || - route.transitMeshId === undefined || - route.routeHints.length + route.coordinationRelays.length === 0 - ) { - return false; - } - return isActiveMeshMember(stored.meshes, route.transitMeshId, localPeerId, route.peerId); - }) - .sort((left, right) => left.route.peerId.localeCompare(right.route.peerId)); + const eligibleRelays = eligibleTransitEvidence(stored, localPeerId, now); const relayCandidates = transitRelayCandidates(eligibleRelays); const approvedRelayPeerIds = [ ...new Set( @@ -1838,17 +1976,54 @@ class PeerMeshNodeImpl implements PeerMeshNode { } } +interface PeerMeshTransitEvidence { + readonly meshId: string; + readonly lease: SignedPeerReachabilityLeaseV1; +} + +function eligibleTransitEvidence( + stored: PeerMeshStoredStateV1, + localPeerId: string, + now: number, +): readonly PeerMeshTransitEvidence[] { + const evidence = new Map(); + for (const { advertisement } of stored.advertisements) { + if ( + advertisement.peerId === localPeerId || + !advertisement.offersTransit || + !isActiveMeshMember(stored.meshes, advertisement.meshId, localPeerId, advertisement.peerId) + ) { + continue; + } + const lease = latestReachability(stored.reachability, advertisement.peerId, now, false); + if (!lease || lease.lease.directRoutes.length + lease.lease.coordinationRoutes.length === 0) { + continue; + } + evidence.set(advertisement.peerId, { + meshId: advertisement.meshId, + lease, + }); + } + return Object.freeze( + [...evidence.values()].sort((left, right) => + left.lease.lease.peerId.localeCompare(right.lease.lease.peerId), + ), + ); +} + function transitRelayCandidates( - routes: readonly SignedPeerMeshRouteRecordV1[], + evidence: readonly PeerMeshTransitEvidence[], ): readonly RuntimeHostPeerTransitRelayCandidate[] { let remaining = PEER_MESH_MAX_TRANSIT_RELAY_ADDRESSES; const candidates: RuntimeHostPeerTransitRelayCandidate[] = []; - for (const { route } of routes) { + for (const { + lease: { lease }, + } of evidence) { if (remaining === 0) break; const routeHints = [ - ...new Set(route.routeHints.filter((address) => isBaseRelayFor(address, route.peerId))), + ...new Set(lease.directRoutes.filter((address) => isBaseRelayFor(address, lease.peerId))), ].slice(0, Math.min(PEER_MESH_MAX_TRANSIT_ADDRESSES_PER_RELAY, remaining)); - const coordinationRelays = [...new Set(route.coordinationRelays)].slice( + const coordinationRelays = [...new Set(lease.coordinationRoutes)].slice( 0, Math.min( PEER_MESH_MAX_TRANSIT_ADDRESSES_PER_RELAY - routeHints.length, @@ -1858,7 +2033,7 @@ function transitRelayCandidates( if (routeHints.length + coordinationRelays.length === 0) continue; candidates.push( Object.freeze({ - peerId: route.peerId, + peerId: lease.peerId, addresses: Object.freeze(routeHints), coordinationRelays: Object.freeze(coordinationRelays), }), @@ -1897,14 +2072,16 @@ function peerMeshStatus( state: PeerMeshStateV1, identity: ReturnType, endpointKind: 'client' | 'host' | undefined, - routes: readonly SignedPeerMeshRouteRecordV1[] = [], + reachability: readonly SignedPeerReachabilityLeaseV1[] = [], + advertisements: readonly SignedPeerMeshMemberAdvertisementV1[] = [], now = Date.now(), + recentlyReached: ReadonlySet = new Set(), ): PeerMeshStatus { - const localDisplayName = routes.find(({ route }) => route.peerId === identity.peerId)?.route - .displayName; + const meshId = state.roster.roster.meshId; + const localAdvertisement = findAdvertisement(advertisements, meshId, identity.peerId); return Object.freeze({ role: state.role === 'authority' ? 'authority' : 'member', - authority: state.role === 'authority' ? authorityTarget(identity) : state.authority, + authorityPeerId: state.role === 'authority' ? identity.peerId : state.authority.peerId, roster: state.roster, pendingInvitationCount: state.role === 'authority' @@ -1918,30 +2095,31 @@ function peerMeshStatus( return Object.freeze({ peerId, ...(endpointKind ? { endpointKind } : {}), - ...(localDisplayName ? { displayName: localDisplayName } : {}), + ...(localAdvertisement?.advertisement.displayName + ? { displayName: localAdvertisement.advertisement.displayName } + : {}), state: 'local' as const, }); } - const route = routes.find((candidate) => candidate.route.peerId === peerId)?.route; - if (!route) return Object.freeze({ peerId, state: 'unknown' as const }); - if (route.expiresAt <= now) { - return Object.freeze({ - peerId, - ...(route.endpointKind ? { endpointKind: route.endpointKind } : {}), - ...(route.displayName ? { displayName: route.displayName } : {}), - state: 'stale' as const, - expiresAt: route.expiresAt, - }); - } + const advertisement = findAdvertisement(advertisements, meshId, peerId)?.advertisement; + const signed = latestReachability(reachability, peerId, now, true); + const lease = signed?.lease; + const hasLocator = Boolean( + lease && lease.directRoutes.length + lease.coordinationRoutes.length > 0, + ); + const state = !hasLocator + ? ('needs_repair' as const) + : lease!.expiresAt <= now + ? ('reconnecting' as const) + : recentlyReached.has(peerId) + ? ('reachable' as const) + : ('connecting' as const); return Object.freeze({ peerId, - ...(route.endpointKind ? { endpointKind: route.endpointKind } : {}), - ...(route.displayName ? { displayName: route.displayName } : {}), - state: - route.routeHints.length > 0 - ? ('route_available' as const) - : ('coordination_only' as const), - expiresAt: route.expiresAt, + ...(advertisement?.endpointKind ? { endpointKind: advertisement.endpointKind } : {}), + ...(advertisement?.displayName ? { displayName: advertisement.displayName } : {}), + state, + ...(lease ? { expiresAt: lease.expiresAt } : {}), }); }), ), @@ -1950,26 +2128,38 @@ function peerMeshStatus( function currentAuthorityTarget( state: PeerMeshReplicaStateV1, - routes: readonly SignedPeerMeshRouteRecordV1[], -): PeerMeshAuthorityTarget { - const learned = routes.find(({ route }) => route.peerId === state.authority.peerId)?.route; - return learned ? mergeTargets(learned, state.authority) : state.authority; + reachability: readonly SignedPeerReachabilityLeaseV1[], + now: number, +): PeerMeshAuthorityTarget | undefined { + const learned = latestReachability(reachability, state.authority.peerId, now, true); + const fallback = usableHistoricalReachability(state.authority.reachability, now) + ? state.authority.reachability + : undefined; + const selected = selectReachability(learned, fallback); + return selected + ? Object.freeze({ peerId: state.authority.peerId, reachability: selected }) + : undefined; } function rosterAnnouncementTargets( memberPeerIds: readonly string[], - routes: readonly SignedPeerMeshRouteRecordV1[], + reachability: readonly SignedPeerReachabilityLeaseV1[], localPeerId: string, + now: number, ): readonly PeerMeshAuthorityTarget[] { const members = new Set(memberPeerIds); return Object.freeze( - routes - .filter(({ route }) => route.peerId !== localPeerId && members.has(route.peerId)) - .map(({ route }) => + reachability + .filter( + (signed) => + signed.lease.peerId !== localPeerId && + members.has(signed.lease.peerId) && + usableHistoricalReachability(signed, now), + ) + .map((signed) => Object.freeze({ - peerId: route.peerId, - routeHints: route.routeHints, - coordinationRelays: route.coordinationRelays, + peerId: signed.lease.peerId, + reachability: signed, }), ), ); @@ -2010,16 +2200,6 @@ function redeemedInvitation(invitation: { readonly secretDigest: string }, peerI }; } -function authorityTarget( - identity: ReturnType, -): PeerMeshAuthorityTarget { - return Object.freeze({ - peerId: identity.peerId, - routeHints: identity.listenAddresses, - coordinationRelays: identity.coordinationRelays, - }); -} - function assertMeshCapacity( states: readonly PeerMeshStateV1[], localPeerId: string, @@ -2074,112 +2254,275 @@ function selectRoster( return candidate; } -function mergeRoutes( - current: readonly SignedPeerMeshRouteRecordV1[], - candidates: readonly SignedPeerMeshRouteRecordV1[], +function mergeReachability( + current: readonly SignedPeerReachabilityLeaseV1[], + candidates: readonly SignedPeerReachabilityLeaseV1[], now: number, -): readonly SignedPeerMeshRouteRecordV1[] { - const routes = new Map(current.map((route) => [route.route.peerId, route] as const)); +): readonly SignedPeerReachabilityLeaseV1[] { + const leases = new Map( + current + .filter((signed) => usableHistoricalReachability(signed, now)) + .map((signed) => [signed.lease.peerId, signed] as const), + ); for (const candidate of candidates) { - if (candidate.route.expiresAt <= now) continue; - const existing = routes.get(candidate.route.peerId); - if ( - !existing || - existing.route.expiresAt <= now || - candidate.route.sequence > existing.route.sequence - ) { - routes.set(candidate.route.peerId, candidate); + if (!usableHistoricalReachability(candidate, now)) continue; + const existing = leases.get(candidate.lease.peerId); + if (!existing || candidate.lease.revision > existing.lease.revision) { + leases.set(candidate.lease.peerId, candidate); continue; } if ( - candidate.route.sequence === existing.route.sequence && + candidate.lease.revision === existing.lease.revision && JSON.stringify(candidate) !== JSON.stringify(existing) ) { - throw new Error('Peer Mesh route sequence identifies conflicting facts'); + throw new Error('Peer reachability revision identifies conflicting facts'); } } return Object.freeze( - [...routes.values()].sort((left, right) => left.route.peerId.localeCompare(right.route.peerId)), + [...leases.values()].sort((left, right) => left.lease.peerId.localeCompare(right.lease.peerId)), ); } -function mergeAuthenticatedRoute( - current: readonly SignedPeerMeshRouteRecordV1[], - candidate: SignedPeerMeshRouteRecordV1, +function mergeAuthenticatedReachability( + current: readonly SignedPeerReachabilityLeaseV1[], + candidate: SignedPeerReachabilityLeaseV1, now: number, -): readonly SignedPeerMeshRouteRecordV1[] { - const existing = current.find(({ route }) => route.peerId === candidate.route.peerId); - if ( - existing && - existing.route.expiresAt > now && - existing.route.sequence > candidate.route.sequence - ) { +): readonly SignedPeerReachabilityLeaseV1[] { + const existing = current.find(({ lease }) => lease.peerId === candidate.lease.peerId); + if (existing && existing.lease.revision > candidate.lease.revision) { return current; } - return mergeRoutes( - current.filter(({ route }) => route.peerId !== candidate.route.peerId), + return mergeReachability( + current.filter(({ lease }) => lease.peerId !== candidate.lease.peerId), [candidate], now, ); } -function routeSequences( - routes: readonly SignedPeerMeshRouteRecordV1[], +function mergeAdvertisements( + current: readonly SignedPeerMeshMemberAdvertisementV1[], + candidates: readonly SignedPeerMeshMemberAdvertisementV1[], +): readonly SignedPeerMeshMemberAdvertisementV1[] { + const advertisements = new Map( + current.map((signed) => [advertisementKey(signed.advertisement), signed] as const), + ); + for (const candidate of candidates) { + const key = advertisementKey(candidate.advertisement); + const existing = advertisements.get(key); + if (!existing || candidate.advertisement.revision > existing.advertisement.revision) { + advertisements.set(key, candidate); + continue; + } + if ( + candidate.advertisement.revision === existing.advertisement.revision && + JSON.stringify(candidate) !== JSON.stringify(existing) + ) { + throw new Error('Peer Mesh advertisement revision identifies conflicting facts'); + } + } + return Object.freeze( + [...advertisements.values()].sort((left, right) => + advertisementKey(left.advertisement).localeCompare(advertisementKey(right.advertisement)), + ), + ); +} + +function reachabilityRevisions( + reachability: readonly SignedPeerReachabilityLeaseV1[], roster: SignedPeerMeshRosterV1, now: number, -): readonly PeerMeshRouteSequence[] { +): readonly PeerMeshEvidenceRevision[] { + return Object.freeze( + reachability + .filter( + (signed) => + usableHistoricalReachability(signed, now) && + roster.roster.members.includes(signed.lease.peerId), + ) + .map(({ lease }) => Object.freeze({ peerId: lease.peerId, revision: lease.revision })) + .sort((left, right) => left.peerId.localeCompare(right.peerId)), + ); +} + +function advertisementRevisions( + advertisements: readonly SignedPeerMeshMemberAdvertisementV1[], + roster: SignedPeerMeshRosterV1, +): readonly PeerMeshAdvertisementRevision[] { return Object.freeze( - routes - .filter(({ route }) => route.expiresAt > now && roster.roster.members.includes(route.peerId)) - .map(({ route }) => Object.freeze({ peerId: route.peerId, sequence: route.sequence })) + advertisements + .filter( + ({ advertisement }) => + advertisement.meshId === roster.roster.meshId && + roster.roster.members.includes(advertisement.peerId), + ) + .map(({ advertisement }) => + Object.freeze({ + meshId: advertisement.meshId, + peerId: advertisement.peerId, + revision: advertisement.revision, + }), + ) .sort((left, right) => left.peerId.localeCompare(right.peerId)), ); } -function responseRoutes( +function responseEvidence( state: PeerMeshStateV1, - routes: readonly SignedPeerMeshRouteRecordV1[], - knownRoutes: readonly PeerMeshRouteSequence[], + reachability: readonly SignedPeerReachabilityLeaseV1[], + advertisements: readonly SignedPeerMeshMemberAdvertisementV1[], + knownReachability: readonly PeerMeshEvidenceRevision[], + knownAdvertisements: readonly PeerMeshAdvertisementRevision[], now: number, ): { - readonly routes: readonly SignedPeerMeshRouteRecordV1[]; + readonly reachability: readonly SignedPeerReachabilityLeaseV1[]; + readonly advertisements: readonly SignedPeerMeshMemberAdvertisementV1[]; readonly more: boolean; } { - const known = new Map(knownRoutes.map(({ peerId, sequence }) => [peerId, sequence])); - const missing = routes.filter( - ({ route }) => - route.expiresAt > now && - state.roster.roster.members.includes(route.peerId) && - route.sequence > (known.get(route.peerId) ?? 0), + const knownLeases = new Map(knownReachability.map(({ peerId, revision }) => [peerId, revision])); + const knownAds = new Map( + knownAdvertisements.map(({ meshId, peerId, revision }) => [`${meshId}\n${peerId}`, revision]), + ); + const missing = [ + ...reachability + .filter( + (signed) => + usableHistoricalReachability(signed, now) && + state.roster.roster.members.includes(signed.lease.peerId) && + signed.lease.revision > (knownLeases.get(signed.lease.peerId) ?? 0), + ) + .map((value) => ({ kind: 'reachability' as const, peerId: value.lease.peerId, value })), + ...advertisements + .filter( + ({ advertisement }) => + advertisement.meshId === state.roster.roster.meshId && + state.roster.roster.members.includes(advertisement.peerId) && + advertisement.revision > (knownAds.get(advertisementKey(advertisement)) ?? 0), + ) + .map((value) => ({ + kind: 'advertisement' as const, + peerId: value.advertisement.peerId, + value, + })), + ].sort((left, right) => + left.peerId === right.peerId + ? left.kind.localeCompare(right.kind) + : left.peerId.localeCompare(right.peerId), ); + const page = missing.slice(0, EVIDENCE_PAGE_SIZE); return Object.freeze({ - routes: Object.freeze(missing.slice(0, ROUTE_PAGE_SIZE)), - more: missing.length > ROUTE_PAGE_SIZE, + reachability: Object.freeze( + page.flatMap((entry) => (entry.kind === 'reachability' ? [entry.value] : [])), + ), + advertisements: Object.freeze( + page.flatMap((entry) => (entry.kind === 'advertisement' ? [entry.value] : [])), + ), + more: missing.length > EVIDENCE_PAGE_SIZE, }); } -function sameAddresses(left: readonly string[], right: readonly string[]): boolean { - return left.length === right.length && left.every((address, index) => address === right[index]); +function initialEvidence( + state: PeerMeshStateV1, + reachability: readonly SignedPeerReachabilityLeaseV1[], + advertisements: readonly SignedPeerMeshMemberAdvertisementV1[], + localPeerId: string, + now: number, +): { + readonly reachability: readonly SignedPeerReachabilityLeaseV1[]; + readonly advertisements: readonly SignedPeerMeshMemberAdvertisementV1[]; +} { + const lease = latestReachability(reachability, localPeerId, now, true); + const advertisement = findAdvertisement(advertisements, state.roster.roster.meshId, localPeerId); + if (!lease || !advertisement) throw new Error('Peer Mesh authority evidence is unavailable'); + return Object.freeze({ + reachability: Object.freeze([lease]), + advertisements: Object.freeze([advertisement]), + }); } -function isCurrentLocalRoute( - existing: SignedPeerMeshRouteRecordV1 | undefined, - identity: ReturnType, +function isCurrentLocalAdvertisement( + existing: SignedPeerMeshMemberAdvertisementV1 | undefined, + meshId: string, + peerId: string, current: Pick, endpointKind: 'client' | 'host' | undefined, - now: number, -): existing is SignedPeerMeshRouteRecordV1 { +): existing is SignedPeerMeshMemberAdvertisementV1 { return Boolean( existing && - existing.route.expiresAt > now + ROUTE_REFRESH_LEAD_MS && - sameAddresses(existing.route.routeHints, identity.listenAddresses) && - sameAddresses(existing.route.coordinationRelays, identity.coordinationRelays) && - existing.route.endpointKind === endpointKind && - existing.route.displayName === (current.displayName ?? undefined) && - existing.route.transitMeshId === current.transitMeshId, + existing.advertisement.meshId === meshId && + existing.advertisement.peerId === peerId && + existing.advertisement.endpointKind === endpointKind && + existing.advertisement.displayName === (current.displayName ?? undefined) && + existing.advertisement.offersTransit === (current.transitMeshId === meshId), + ); +} + +function latestReachability( + reachability: readonly SignedPeerReachabilityLeaseV1[], + peerId: string, + now: number, + includeHistorical: boolean, +): SignedPeerReachabilityLeaseV1 | undefined { + const signed = reachability.find(({ lease }) => lease.peerId === peerId); + if (!signed) return undefined; + if (signed.lease.expiresAt > now) return signed; + return includeHistorical && usableHistoricalReachability(signed, now) ? signed : undefined; +} + +function usableHistoricalReachability(signed: SignedPeerReachabilityLeaseV1, now: number): boolean { + return signed.lease.expiresAt + REACHABILITY_HISTORY_MS > now; +} + +function selectReachability( + first: SignedPeerReachabilityLeaseV1 | undefined, + second: SignedPeerReachabilityLeaseV1 | undefined, +): SignedPeerReachabilityLeaseV1 | undefined { + if (!first) return second; + if (!second || first.lease.revision >= second.lease.revision) return first; + return second; +} + +function peerTarget( + peerId: string, + reachability: readonly SignedPeerReachabilityLeaseV1[], + now: number, +): PeerMeshAuthorityTarget | undefined { + const signed = latestReachability(reachability, peerId, now, true); + return signed ? Object.freeze({ peerId, reachability: signed }) : undefined; +} + +function dialTarget(reachability: SignedPeerReachabilityLeaseV1): { + readonly peerId: string; + readonly routeHints: readonly string[]; + readonly coordinationRelays: readonly string[]; +} { + return Object.freeze({ + peerId: reachability.lease.peerId, + routeHints: reachability.lease.directRoutes, + coordinationRelays: reachability.lease.coordinationRoutes, + }); +} + +function authorityReachabilityFor(invitation: PeerMeshInvitationV1): SignedPeerReachabilityLeaseV1 { + return invitation.reachability; +} + +function findAdvertisement( + advertisements: readonly SignedPeerMeshMemberAdvertisementV1[], + meshId: string, + peerId: string, +): SignedPeerMeshMemberAdvertisementV1 | undefined { + return advertisements.find( + ({ advertisement }) => advertisement.meshId === meshId && advertisement.peerId === peerId, ); } +function advertisementKey(advertisement: { + readonly meshId: string; + readonly peerId: string; +}): string { + return `${advertisement.meshId}\n${advertisement.peerId}`; +} + function mergeAddresses( primary: readonly string[], fallback: readonly string[], @@ -2200,19 +2543,6 @@ function waitForTurn(previous: Promise, signal?: AbortSignal): Promise( stream: RuntimeHostPeerNativeStream, request: Request, @@ -2242,25 +2572,36 @@ function decodeControlRequest(value: unknown): PeerMeshControlRequest { const record = recordValue(value); if ( record.kind === 'redeem-invitation' && - hasExactKeys(record, ['kind', 'meshId', 'secret', 'route']) + hasExactKeys(record, ['kind', 'meshId', 'secret', 'reachability', 'advertisement']) ) { return { kind: 'redeem-invitation', meshId: requiredString(record.meshId, 128), secret: requiredString(record.secret, 64), - route: decodeSignedPeerMeshRouteRecord(record.route), + reachability: decodeSignedPeerReachabilityLease(record.reachability), + advertisement: decodeSignedPeerMeshMemberAdvertisement(record.advertisement), }; } if ( record.kind === 'sync' && - hasExactKeys(record, ['kind', 'meshId', 'roster', 'route', 'knownRoutes']) + hasExactKeys(record, [ + 'kind', + 'meshId', + 'roster', + 'reachability', + 'advertisement', + 'knownReachability', + 'knownAdvertisements', + ]) ) { return { kind: 'sync', meshId: requiredString(record.meshId, 128), roster: decodeSignedPeerMeshRoster(record.roster), - route: decodeSignedPeerMeshRouteRecord(record.route), - knownRoutes: decodeRouteSequences(record.knownRoutes), + reachability: decodeSignedPeerReachabilityLease(record.reachability), + advertisement: decodeSignedPeerMeshMemberAdvertisement(record.advertisement), + knownReachability: decodeEvidenceRevisions(record.knownReachability), + knownAdvertisements: decodeAdvertisementRevisions(record.knownAdvertisements), }; } if (record.kind === 'leave' && hasExactKeys(record, ['kind', 'meshId', 'roster'])) { @@ -2282,11 +2623,18 @@ function decodeControlRequest(value: unknown): PeerMeshControlRequest { function decodeRedeemResponse(value: unknown): RedeemInvitationResponse { const record = recordValue(value); - if (record.kind === 'invitation-redeemed' && hasExactKeys(record, ['kind', 'roster', 'routes'])) { + if ( + record.kind === 'invitation-redeemed' && + hasExactKeys(record, ['kind', 'roster', 'reachability', 'advertisements']) + ) { + const reachability = decodeReachabilityPage(record.reachability); + const advertisements = decodeAdvertisementPage(record.advertisements); + assertEvidencePageSize(reachability, advertisements); return { kind: 'invitation-redeemed', roster: decodeSignedPeerMeshRoster(record.roster), - routes: decodeRoutePage(record.routes), + reachability, + advertisements, }; } if ( @@ -2306,13 +2654,17 @@ function decodeSyncResponse(value: unknown): SyncPeerMeshResponse { const record = recordValue(value); if ( record.kind === 'sync-result' && - hasExactKeys(record, ['kind', 'roster', 'routes', 'more']) && + hasExactKeys(record, ['kind', 'roster', 'reachability', 'advertisements', 'more']) && typeof record.more === 'boolean' ) { + const reachability = decodeReachabilityPage(record.reachability); + const advertisements = decodeAdvertisementPage(record.advertisements); + assertEvidencePageSize(reachability, advertisements); return { kind: 'sync-result', roster: decodeSignedPeerMeshRoster(record.roster), - routes: decodeRoutePage(record.routes), + reachability, + advertisements, more: record.more, }; } @@ -2356,35 +2708,77 @@ function decodeAnnounceRosterResponse(value: unknown): AnnouncePeerMeshRosterRes throw new Error('Invalid Peer Mesh roster announcement response'); } -function decodeRoutePage(value: unknown): readonly SignedPeerMeshRouteRecordV1[] { - if (!Array.isArray(value) || value.length > ROUTE_PAGE_SIZE) { - throw new Error('Invalid Peer Mesh route page'); +function decodeReachabilityPage(value: unknown): readonly SignedPeerReachabilityLeaseV1[] { + if (!Array.isArray(value) || value.length > EVIDENCE_PAGE_SIZE) { + throw new Error('Invalid Peer Mesh reachability page'); } - return Object.freeze(value.map(decodeSignedPeerMeshRouteRecord)); + return Object.freeze(value.map(decodeSignedPeerReachabilityLease)); } -function decodeRouteSequences(value: unknown): readonly PeerMeshRouteSequence[] { +function decodeAdvertisementPage(value: unknown): readonly SignedPeerMeshMemberAdvertisementV1[] { + if (!Array.isArray(value) || value.length > EVIDENCE_PAGE_SIZE) { + throw new Error('Invalid Peer Mesh advertisement page'); + } + return Object.freeze(value.map(decodeSignedPeerMeshMemberAdvertisement)); +} + +function assertEvidencePageSize( + reachability: readonly SignedPeerReachabilityLeaseV1[], + advertisements: readonly SignedPeerMeshMemberAdvertisementV1[], +): void { + if (reachability.length + advertisements.length > EVIDENCE_PAGE_SIZE) { + throw new Error('Peer Mesh evidence page exceeds its bound'); + } +} + +function decodeEvidenceRevisions(value: unknown): readonly PeerMeshEvidenceRevision[] { + if (!Array.isArray(value) || value.length > PEER_MESH_MAX_MEMBERS) { + throw new Error('Invalid Peer Mesh reachability revisions'); + } + const revisions = value.map((entry) => { + const record = recordValue(entry); + if (!hasExactKeys(record, ['peerId', 'revision'])) { + throw new Error('Invalid Peer Mesh reachability revision'); + } + const revision = record.revision; + if (!Number.isSafeInteger(revision) || (revision as number) < 1) { + throw new Error('Invalid Peer Mesh reachability revision'); + } + return Object.freeze({ + peerId: requiredString(record.peerId, 256), + revision: revision as number, + }); + }); + if (new Set(revisions.map(({ peerId }) => peerId)).size !== revisions.length) { + throw new Error('Duplicate Peer Mesh reachability revision'); + } + return Object.freeze(revisions); +} + +function decodeAdvertisementRevisions(value: unknown): readonly PeerMeshAdvertisementRevision[] { if (!Array.isArray(value) || value.length > PEER_MESH_MAX_MEMBERS) { - throw new Error('Invalid Peer Mesh route sequences'); + throw new Error('Invalid Peer Mesh advertisement revisions'); } - const sequences = value.map((entry) => { + const revisions = value.map((entry) => { const record = recordValue(entry); - if (!hasExactKeys(record, ['peerId', 'sequence'])) { - throw new Error('Invalid Peer Mesh route sequence'); + if (!hasExactKeys(record, ['meshId', 'peerId', 'revision'])) { + throw new Error('Invalid Peer Mesh advertisement revision'); } - const sequence = record.sequence; - if (!Number.isSafeInteger(sequence) || (sequence as number) < 1) { - throw new Error('Invalid Peer Mesh route sequence'); + const revision = record.revision; + if (!Number.isSafeInteger(revision) || (revision as number) < 1) { + throw new Error('Invalid Peer Mesh advertisement revision'); } return Object.freeze({ + meshId: requiredString(record.meshId, 128), peerId: requiredString(record.peerId, 256), - sequence: sequence as number, + revision: revision as number, }); }); - if (new Set(sequences.map(({ peerId }) => peerId)).size !== sequences.length) { - throw new Error('Duplicate Peer Mesh route sequence'); + const keys = revisions.map(({ meshId, peerId }) => `${meshId}\n${peerId}`); + if (new Set(keys).size !== keys.length) { + throw new Error('Duplicate Peer Mesh advertisement revision'); } - return Object.freeze(sequences); + return Object.freeze(revisions); } async function writeFrame(stream: RuntimeHostPeerNativeStream, value: unknown): Promise { diff --git a/packages/runtime-host/src/peer-mesh/store.ts b/packages/runtime-host/src/peer-mesh/store.ts index 31afe60ec3..13681f8564 100644 --- a/packages/runtime-host/src/peer-mesh/store.ts +++ b/packages/runtime-host/src/peer-mesh/store.ts @@ -25,19 +25,23 @@ import { } from '@maka/storage/file-lifetime-owner'; import { decodeAuthorityTarget, + decodeSignedPeerMeshMemberAdvertisement, decodeSignedPeerMeshRoster, - decodeSignedPeerMeshRouteRecord, PEER_MESH_MAX_INVITATION_RECORDS, PEER_MESH_MAX_MEMBERS, PEER_MESH_MAX_MESHES, PEER_MESH_MAX_PENDING_INVITATIONS, type PeerMeshAuthorityKeyPair, type PeerMeshAuthorityTarget, + type SignedPeerMeshMemberAdvertisementV1, type SignedPeerMeshRosterV1, - type SignedPeerMeshRouteRecordV1, validatePeerMeshInvitation, validatePeerMeshAuthorityKeyPair, } from './model.js'; +import { + decodeSignedPeerReachabilityLease, + type SignedPeerReachabilityLeaseV1, +} from '../peer-reachability/index.js'; import { canonicalPeerMeshDisplayName } from './display-name.js'; import type { PeerMeshInvitationV1 } from '../protocol/peer-mesh.js'; @@ -86,7 +90,8 @@ export interface PeerMeshStoredStateV1 { readonly displayName: string | null; readonly meshes: readonly PeerMeshStateV1[]; readonly pendingJoins: readonly PendingPeerMeshJoin[]; - readonly routes: readonly SignedPeerMeshRouteRecordV1[]; + readonly reachability: readonly SignedPeerReachabilityLeaseV1[]; + readonly advertisements: readonly SignedPeerMeshMemberAdvertisementV1[]; readonly transitMeshId: string | null; } @@ -205,7 +210,7 @@ class PeerMeshStateStoreImpl implements PeerMeshStateStore { if (this.#failure) throw this.#failure; const updated = await operation(this.#state); if (updated.state === this.#state) return updated.result; - const candidate = pruneUnreferencedRoutes(updated.state, this.localPeerId); + const candidate = pruneUnreferencedEvidence(updated.state, this.localPeerId); const canonical = decodePeerMeshStoredState(candidate, this.localPeerId); assertStateAdvance(this.#state.meshes, canonical.meshes, this.localPeerId); try { @@ -246,11 +251,7 @@ class PeerMeshStateStoreImpl implements PeerMeshStateStore { } } -export function decodePeerMeshState( - value: unknown, - localPeerId: string, - legacyReplica = false, -): PeerMeshStateV1 { +export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMeshStateV1 { if (!value || typeof value !== 'object' || Array.isArray(value)) { throw new Error('Invalid Peer Mesh state'); } @@ -258,9 +259,7 @@ export function decodePeerMeshState( const expectedKeys = record.role === 'authority' ? ['role', 'roster', 'authorityPrivateKey', 'invitations'] - : legacyReplica - ? ['role', 'roster', 'authority'] - : ['role', 'roster', 'authority', 'desiredMembership']; + : ['role', 'roster', 'authority', 'desiredMembership']; if ( Object.keys(record).length !== expectedKeys.length || expectedKeys.some((key) => !Object.hasOwn(record, key)) @@ -295,19 +294,15 @@ export function decodePeerMeshState( role: 'replica', authority, roster, - desiredMembership: legacyReplica ? 'active' : decodeDesiredMembership(record.desiredMembership), + desiredMembership: decodeDesiredMembership(record.desiredMembership), }); } -function decodePeerMeshStates( - value: unknown, - localPeerId: string, - legacyReplica = false, -): readonly PeerMeshStateV1[] { +function decodePeerMeshStates(value: unknown, localPeerId: string): readonly PeerMeshStateV1[] { if (!Array.isArray(value) || value.length > PEER_MESH_MAX_MESHES) { throw new Error('Invalid Peer Mesh state collection'); } - const states = value.map((state) => decodePeerMeshState(state, localPeerId, legacyReplica)); + const states = value.map((state) => decodePeerMeshState(state, localPeerId)); const meshIds = states.map(({ roster }) => roster.roster.meshId); if (new Set(meshIds).size !== meshIds.length) { throw new Error('Duplicate Peer Mesh state'); @@ -384,42 +379,17 @@ async function readState( throw new Error('Invalid Peer Mesh state document'); } const record = document as Record; - const versionOne = - record.version === 1 && - Object.keys(record).length === 3 && - Object.hasOwn(record, 'localPeerId') && - Object.hasOwn(record, 'meshes'); - const versionTwo = - record.version === 2 && - Object.keys(record).length === 4 && - Object.hasOwn(record, 'localPeerId') && - Object.hasOwn(record, 'meshes') && - Object.hasOwn(record, 'routes'); - const versionThree = - record.version === 3 && - Object.keys(record).length === 5 && - Object.hasOwn(record, 'localPeerId') && - Object.hasOwn(record, 'meshes') && - Object.hasOwn(record, 'routes') && - Object.hasOwn(record, 'transitMeshId'); - const versionFour = - record.version === 4 && - Object.keys(record).length === 6 && - Object.hasOwn(record, 'localPeerId') && - Object.hasOwn(record, 'displayName') && - Object.hasOwn(record, 'meshes') && - Object.hasOwn(record, 'routes') && - Object.hasOwn(record, 'transitMeshId'); - const versionSix = - record.version === 6 && - Object.keys(record).length === 7 && - Object.hasOwn(record, 'localPeerId') && - Object.hasOwn(record, 'displayName') && - Object.hasOwn(record, 'meshes') && - Object.hasOwn(record, 'pendingJoins') && - Object.hasOwn(record, 'routes') && - Object.hasOwn(record, 'transitMeshId'); - if (!versionOne && !versionTwo && !versionThree && !versionFour && !versionSix) { + if ( + record.version !== 7 || + Object.keys(record).length !== 8 || + !Object.hasOwn(record, 'localPeerId') || + !Object.hasOwn(record, 'displayName') || + !Object.hasOwn(record, 'meshes') || + !Object.hasOwn(record, 'pendingJoins') || + !Object.hasOwn(record, 'reachability') || + !Object.hasOwn(record, 'advertisements') || + !Object.hasOwn(record, 'transitMeshId') + ) { throw new Error('Unsupported Peer Mesh state document'); } if (boundedString(record.localPeerId, 'localPeerId', 256) !== expectedLocalPeerId) { @@ -427,14 +397,14 @@ async function readState( } return decodePeerMeshStoredState( { - displayName: versionFour || versionSix ? record.displayName : null, + displayName: record.displayName, meshes: record.meshes, - pendingJoins: versionSix ? record.pendingJoins : [], - routes: versionOne ? [] : record.routes, - transitMeshId: versionThree || versionFour || versionSix ? record.transitMeshId : null, + pendingJoins: record.pendingJoins, + reachability: record.reachability, + advertisements: record.advertisements, + transitMeshId: record.transitMeshId, }, expectedLocalPeerId, - !versionSix, ); } catch (error) { if (isNodeError(error, 'ENOENT')) { @@ -442,7 +412,8 @@ async function readState( displayName: null, meshes: Object.freeze([]), pendingJoins: Object.freeze([]), - routes: Object.freeze([]), + reachability: Object.freeze([]), + advertisements: Object.freeze([]), transitMeshId: null, }); } @@ -465,7 +436,7 @@ async function writeState( localPeerId: string, state: PeerMeshStoredStateV1, ): Promise { - const document = `${JSON.stringify({ version: 6, localPeerId, ...state }, null, 2)}\n`; + const document = `${JSON.stringify({ version: 7, localPeerId, ...state }, null, 2)}\n`; if (Buffer.byteLength(document) > MAX_STATE_BYTES) throw new Error('Peer Mesh state is too large'); const temporary = `${path}.tmp`; @@ -558,15 +529,15 @@ function decodeInvitations(value: unknown): PeerMeshInvitationRecord[] { return invitations; } -function decodeRoutes( +function decodeReachability( value: unknown, meshes: readonly PeerMeshStateV1[], -): readonly SignedPeerMeshRouteRecordV1[] { +): readonly SignedPeerReachabilityLeaseV1[] { if (!Array.isArray(value) || value.length > PEER_MESH_MAX_MESHES * PEER_MESH_MAX_MEMBERS) { - throw new Error('Invalid Peer Mesh routes'); + throw new Error('Invalid Peer Mesh reachability'); } - const routes = value.map(decodeSignedPeerMeshRouteRecord); - const peerIds = routes.map(({ route }) => route.peerId); + const leases = value.map(decodeSignedPeerReachabilityLease); + const peerIds = leases.map(({ lease }) => lease.peerId); const knownPeers = new Set( meshes .filter(({ roster }) => !roster.roster.closed) @@ -576,31 +547,55 @@ function decodeRoutes( new Set(peerIds).size !== peerIds.length || peerIds.some((peerId) => !knownPeers.has(peerId)) ) { - throw new Error('Invalid Peer Mesh routes'); + throw new Error('Invalid Peer Mesh reachability'); } - return Object.freeze(routes); + return Object.freeze(leases); } -function decodePeerMeshStoredState( +function decodeAdvertisements( value: unknown, - localPeerId: string, - legacyReplica = false, -): PeerMeshStoredStateV1 { + meshes: readonly PeerMeshStateV1[], +): readonly SignedPeerMeshMemberAdvertisementV1[] { + if (!Array.isArray(value) || value.length > PEER_MESH_MAX_MESHES * PEER_MESH_MAX_MEMBERS) { + throw new Error('Invalid Peer Mesh member advertisements'); + } + const advertisements = value.map(decodeSignedPeerMeshMemberAdvertisement); + const keys = advertisements.map( + ({ advertisement }) => `${advertisement.meshId}\n${advertisement.peerId}`, + ); + if ( + new Set(keys).size !== keys.length || + advertisements.some(({ advertisement }) => { + const mesh = meshes.find(({ roster }) => roster.roster.meshId === advertisement.meshId); + return ( + !mesh || + mesh.roster.roster.closed || + !mesh.roster.roster.members.includes(advertisement.peerId) + ); + }) + ) { + throw new Error('Invalid Peer Mesh member advertisements'); + } + return Object.freeze(advertisements); +} + +function decodePeerMeshStoredState(value: unknown, localPeerId: string): PeerMeshStoredStateV1 { if (!value || typeof value !== 'object' || Array.isArray(value)) { throw new Error('Invalid Peer Mesh state document'); } const record = value as Record; if ( - Object.keys(record).length !== 5 || + Object.keys(record).length !== 6 || !Object.hasOwn(record, 'displayName') || !Object.hasOwn(record, 'meshes') || !Object.hasOwn(record, 'pendingJoins') || - !Object.hasOwn(record, 'routes') || + !Object.hasOwn(record, 'reachability') || + !Object.hasOwn(record, 'advertisements') || !Object.hasOwn(record, 'transitMeshId') ) { throw new Error('Invalid Peer Mesh state document'); } - const meshes = decodePeerMeshStates(record.meshes, localPeerId, legacyReplica); + const meshes = decodePeerMeshStates(record.meshes, localPeerId); const pendingJoins = decodePendingJoins(record.pendingJoins, meshes, localPeerId); const displayName = record.displayName === null ? null : canonicalPeerMeshDisplayName(record.displayName); @@ -622,7 +617,8 @@ function decodePeerMeshStoredState( displayName, meshes, pendingJoins, - routes: decodeRoutes(record.routes, meshes), + reachability: decodeReachability(record.reachability, meshes), + advertisements: decodeAdvertisements(record.advertisements, meshes), transitMeshId, }); } @@ -688,7 +684,7 @@ function decodePendingJoinPhase(value: unknown): PendingPeerMeshJoin['phase'] { return value; } -function pruneUnreferencedRoutes( +function pruneUnreferencedEvidence( state: PeerMeshStoredStateV1, localPeerId: string, ): PeerMeshStoredStateV1 { @@ -697,7 +693,15 @@ function pruneUnreferencedRoutes( .filter(({ roster }) => !roster.roster.closed) .flatMap(({ roster }) => roster.roster.members), ); - const routes = state.routes.filter(({ route }) => knownPeers.has(route.peerId)); + const reachability = state.reachability.filter(({ lease }) => knownPeers.has(lease.peerId)); + const advertisements = state.advertisements.filter(({ advertisement }) => + state.meshes.some( + ({ roster }) => + !roster.roster.closed && + roster.roster.meshId === advertisement.meshId && + roster.roster.members.includes(advertisement.peerId), + ), + ); const transitMeshId = state.meshes.some( (mesh) => mesh.roster.roster.meshId === state.transitMeshId && @@ -705,9 +709,11 @@ function pruneUnreferencedRoutes( ) ? state.transitMeshId : null; - return routes.length === state.routes.length && transitMeshId === state.transitMeshId + return reachability.length === state.reachability.length && + advertisements.length === state.advertisements.length && + transitMeshId === state.transitMeshId ? state - : { ...state, routes, transitMeshId }; + : { ...state, reachability, advertisements, transitMeshId }; } function boundedString(value: unknown, label: string, max: number): string { diff --git a/packages/runtime-host/src/peer-reachability/index.ts b/packages/runtime-host/src/peer-reachability/index.ts index b47c6a7758..82444ad154 100644 --- a/packages/runtime-host/src/peer-reachability/index.ts +++ b/packages/runtime-host/src/peer-reachability/index.ts @@ -27,6 +27,7 @@ export { PEER_REACHABILITY_MAX_ROUTES_PER_CLASS, PEER_REACHABILITY_REFRESH_LEAD_MS, peerReachabilityLeaseSigningBytes, + samePeerReachabilityRoutes, verifySignedPeerReachabilityLease, type PeerReachabilityIdentity, type PeerReachabilityLeaseV1, diff --git a/packages/runtime-host/src/peer-reachability/publisher.ts b/packages/runtime-host/src/peer-reachability/publisher.ts index abe478a7cf..c0d49f4ed5 100644 --- a/packages/runtime-host/src/peer-reachability/publisher.ts +++ b/packages/runtime-host/src/peer-reachability/publisher.ts @@ -42,6 +42,7 @@ const MAX_STATE_BYTES = 64 * 1_024; export interface PeerReachabilityPublisher { current(): SignedPeerReachabilityLeaseV1; refresh(): Promise; + subscribe(listener: (lease: SignedPeerReachabilityLeaseV1) => void): () => void; close(): Promise; } @@ -74,6 +75,7 @@ class PeerReachabilityPublisherImpl implements PeerReachabilityPublisher { #failure: Error | undefined; #closed = false; #closeTask: Promise | undefined; + readonly #listeners = new Set<(lease: SignedPeerReachabilityLeaseV1) => void>(); constructor( private readonly path: string, @@ -98,6 +100,12 @@ class PeerReachabilityPublisherImpl implements PeerReachabilityPublisher { return this.#current; } + subscribe(listener: (lease: SignedPeerReachabilityLeaseV1) => void): () => void { + this.#assertOpen(); + this.#listeners.add(listener); + return () => this.#listeners.delete(listener); + } + refresh(): Promise { this.#assertOpen(); const task = this.#tail.then(async () => { @@ -147,11 +155,13 @@ class PeerReachabilityPublisherImpl implements PeerReachabilityPublisher { } catch (error) { if (error instanceof PeerReachabilityPostCommitError) { this.#adopt(signed, now, monotonicNow); + this.#notify(signed); this.#failure = error; throw error; } throw new PeerReachabilityPersistenceError(error); } + this.#notify(signed); return signed; }); this.#tail = task.then( @@ -169,6 +179,17 @@ class PeerReachabilityPublisherImpl implements PeerReachabilityPublisher { async #close(): Promise { this.#closed = true; await this.#tail; + this.#listeners.clear(); + } + + #notify(lease: SignedPeerReachabilityLeaseV1): void { + for (const listener of this.#listeners) { + try { + listener(lease); + } catch { + // Reachability publication remains authoritative even if an observer fails. + } + } } #assertOpen(): void { diff --git a/packages/runtime-host/src/protocol/peer-mesh.ts b/packages/runtime-host/src/protocol/peer-mesh.ts index 9fae971d6f..523c147044 100644 --- a/packages/runtime-host/src/protocol/peer-mesh.ts +++ b/packages/runtime-host/src/protocol/peer-mesh.ts @@ -26,15 +26,14 @@ import { } from './codec.js'; import { defineOperation } from './operation-spec.js'; import { canonicalPeerMeshDisplayName } from '../peer-mesh/display-name.js'; +import { PEER_MESH_MAX_MEMBERS, PEER_MESH_MAX_MESHES } from '../peer-mesh/limits.js'; import { - PEER_MESH_MAX_MEMBERS, - PEER_MESH_MAX_MESHES, - PEER_MESH_MAX_ROUTE_HINTS, -} from '../peer-mesh/limits.js'; + decodeSignedPeerReachabilityLease, + type SignedPeerReachabilityLeaseV1, +} from '../peer-reachability/index.js'; const PEER_ID_MAX_BYTES = 256; const MESH_ID_MAX_BYTES = 128; -const MESH_ADDRESS_MAX_LENGTH = 1024; export interface PeerMeshInvitationV1 { readonly version: 1; @@ -42,9 +41,7 @@ export interface PeerMeshInvitationV1 { readonly authorityPublicKey: string; readonly secret: string; readonly expiresAt: number; - readonly peerId: string; - readonly routeHints: readonly string[]; - readonly coordinationRelays: readonly string[]; + readonly reachability: SignedPeerReachabilityLeaseV1; } export interface PeerMeshProjection { @@ -62,7 +59,7 @@ export interface PeerMeshMemberProjection { readonly peerId: string; readonly endpointKind?: 'client' | 'host'; readonly displayName?: string; - readonly state: 'local' | 'route_available' | 'coordination_only' | 'stale' | 'unknown'; + readonly state: 'local' | 'connecting' | 'reachable' | 'reconnecting' | 'needs_repair'; readonly expiresAt?: number; } @@ -250,9 +247,7 @@ export function decodePeerMeshInvitation(value: unknown): PeerMeshInvitationV1 { 'authorityPublicKey', 'secret', 'expiresAt', - 'peerId', - 'routeHints', - 'coordinationRelays', + 'reachability', ]); if (record.version !== 1) throw new Error('Unsupported Peer Mesh invitation version'); return { @@ -265,37 +260,10 @@ export function decodePeerMeshInvitation(value: unknown): PeerMeshInvitationV1 { ), secret: requireString(record.secret, 'Peer Mesh invitation secret', 64), expiresAt: requireCount(record.expiresAt, 'Peer Mesh invitation expiry'), - peerId: requireMeshToken(record.peerId, 'Peer Mesh authority peerId'), - routeHints: requireMeshAddresses(record.routeHints, 'Peer Mesh route hints'), - coordinationRelays: requireMeshAddresses( - record.coordinationRelays, - 'Peer Mesh coordination relays', - ), + reachability: decodeSignedPeerReachabilityLease(record.reachability), }; } -function requireMeshToken(value: unknown, label: string): string { - const result = requireString(value, label, PEER_ID_MAX_BYTES); - if (/\s|[\u0000-\u001f\u007f]/u.test(result)) throw new Error(`Invalid ${label}`); - return result; -} - -function requireMeshAddresses(value: unknown, label: string): readonly string[] { - if (!Array.isArray(value) || value.length > PEER_MESH_MAX_ROUTE_HINTS) { - throw new Error(`Invalid ${label}`); - } - const addresses = value.map((address) => requireString(address, label, MESH_ADDRESS_MAX_LENGTH)); - if ( - addresses.some( - (address) => !address.startsWith('/') || /\s|[\u0000-\u001f\u007f]/u.test(address), - ) || - new Set(addresses).size !== addresses.length - ) { - throw new Error(`Invalid ${label}`); - } - return Object.freeze(addresses); -} - function decodePeerMeshRemoveInput(value: unknown): PeerMeshRemoveInput { const record = requireExactRecord(value, 'Peer Mesh remove input', ['meshId', 'peerId']); return { @@ -442,10 +410,10 @@ function decodePeerMeshMemberProjection(value: unknown): PeerMeshMemberProjectio ]); if ( record.state !== 'local' && - record.state !== 'route_available' && - record.state !== 'coordination_only' && - record.state !== 'stale' && - record.state !== 'unknown' + record.state !== 'connecting' && + record.state !== 'reachable' && + record.state !== 'reconnecting' && + record.state !== 'needs_repair' ) { throw new Error('Invalid Peer Mesh member route state'); } diff --git a/packages/runtime-host/src/server/peer-mesh-authority.ts b/packages/runtime-host/src/server/peer-mesh-authority.ts index da00a5358b..342d9c576a 100644 --- a/packages/runtime-host/src/server/peer-mesh-authority.ts +++ b/packages/runtime-host/src/server/peer-mesh-authority.ts @@ -169,7 +169,7 @@ export function projectPeerMeshStatus(status: PeerMeshStatus): PeerMeshProjectio meshId: status.roster.roster.meshId, ...(status.roster.roster.displayName ? { displayName: status.roster.roster.displayName } : {}), role: status.role, - authorityPeerId: status.authority.peerId, + authorityPeerId: status.authorityPeerId, revision: status.roster.roster.revision, closed: status.roster.roster.closed, members: Object.freeze(status.memberRoutes.map((member) => Object.freeze({ ...member }))), From b949b8b0cfc11c9b95be59d277f3aa14fd465157 Mon Sep 17 00:00:00 2001 From: Wang Date: Thu, 3 Sep 2026 01:07:57 +0800 Subject: [PATCH 02/13] fix(peer): converge reachability recovery state Generated-by: Codex (gpt-5.6-sol) --- .../src/__tests__/peer-mesh.test.ts | 111 ++++++-- .../src/__tests__/peer-native.test.ts | 13 +- .../runtime-host/src/client/peer-client.ts | 19 +- packages/runtime-host/src/peer-mesh/model.ts | 20 +- packages/runtime-host/src/peer-mesh/node.ts | 262 +++++++++++++----- packages/runtime-host/src/peer-mesh/store.ts | 2 +- .../src/peer-reachability/index.ts | 3 + packages/runtime-host/src/protocol/index.ts | 5 +- 8 files changed, 329 insertions(+), 106 deletions(-) diff --git a/packages/runtime-host/src/__tests__/peer-mesh.test.ts b/packages/runtime-host/src/__tests__/peer-mesh.test.ts index 537f7b7364..bc195c081f 100644 --- a/packages/runtime-host/src/__tests__/peer-mesh.test.ts +++ b/packages/runtime-host/src/__tests__/peer-mesh.test.ts @@ -143,7 +143,11 @@ test('authenticates three peers, consumes invitations once, and keeps authority displayName, })), [ - { peerId: 'peer-a', endpointKind: 'client', displayName: 'Alice Desktop' }, + { + peerId: 'peer-a', + endpointKind: 'client', + displayName: 'Alice Desktop', + }, { peerId: 'peer-b', endpointKind: 'host', displayName: 'Build Host' }, { peerId: 'peer-c', endpointKind: 'host', displayName: undefined }, ], @@ -180,7 +184,10 @@ test('commits an offline leave locally and reconciles it after restart', async ( peer: authorityPeer, }); const memberRoot = join(root, 'member'); - let member = await openPeerMeshNode({ dataRoot: memberRoot, peer: memberPeer }); + let member = await openPeerMeshNode({ + dataRoot: memberRoot, + peer: memberPeer, + }); const serving = [authority.serve(), member.serve()]; try { const mesh = await authority.create(); @@ -220,7 +227,10 @@ test('announces authority commits without coupling success to delivery', async ( dataRoot: join(root, 'authority'), peer: authorityPeer, }); - const member = await openPeerMeshNode({ dataRoot: join(root, 'member'), peer: memberPeer }); + const member = await openPeerMeshNode({ + dataRoot: join(root, 'member'), + peer: memberPeer, + }); const serving = [authority.serve(), member.serve()]; try { const mesh = await authority.create(); @@ -353,7 +363,7 @@ test('reconciles changed routes, propagates removal, and recovers the verified c await memberC.reconcile(); authorityPeer.setReachable(false); await memberB.reconcile(); - assert.equal(memberB.resolveRoutes('peer-c'), undefined); + assert.equal(memberB.resolveRoutes('peer-c').state, 'exhausted'); assert.deepEqual(memberC.status()[0]?.roster.roster.members, ['peer-a', 'peer-c']); assert.deepEqual(memberC.resolveRoutes('peer-a')?.routeHints, [ '/memory/peer-a-moved/p2p/peer-a', @@ -386,7 +396,10 @@ test('repairs an existing membership with a fresh invitation after every locator dataRoot: join(root, 'authority'), peer: authorityPeer, }); - const member = await openPeerMeshNode({ dataRoot: join(root, 'member'), peer: memberPeer }); + const member = await openPeerMeshNode({ + dataRoot: join(root, 'member'), + peer: memberPeer, + }); const serving = [authority.serve(), member.serve()]; try { const meshId = (await authority.create()).roster.roster.meshId; @@ -396,6 +409,11 @@ test('repairs an existing membership with a fresh invitation after every locator await authorityPeer.setRouteHints([]); await authority.reconcile(); await member.reconcile(); + assert.equal( + member.status()[0]?.memberRoutes.find(({ peerId }) => peerId === 'peer-a')?.state, + 'reconnecting', + ); + await member.prepareRoutes('peer-a', AbortSignal.timeout(4_000)); assert.equal( member.status()[0]?.memberRoutes.find(({ peerId }) => peerId === 'peer-a')?.state, 'needs_repair', @@ -474,10 +492,24 @@ test('reconciles one selected Mesh into signed transit routes and native policy' const memberBPeer = network.create('peer-b'); const memberCPeer = network.create('peer-c'); const memberDPeer = network.create('peer-d'); - const authority = await openPeerMeshNode({ dataRoot: join(root, 'a'), peer: authorityPeer }); - const memberB = await openPeerMeshNode({ dataRoot: join(root, 'b'), peer: memberBPeer }); - const memberC = await openPeerMeshNode({ dataRoot: join(root, 'c'), peer: memberCPeer }); - const memberD = await openPeerMeshNode({ dataRoot: join(root, 'd'), peer: memberDPeer }); + await memberCPeer.setRouteHints([]); + await memberCPeer.setCoordinationRelays([]); + const authority = await openPeerMeshNode({ + dataRoot: join(root, 'a'), + peer: authorityPeer, + }); + const memberB = await openPeerMeshNode({ + dataRoot: join(root, 'b'), + peer: memberBPeer, + }); + const memberC = await openPeerMeshNode({ + dataRoot: join(root, 'c'), + peer: memberCPeer, + }); + const memberD = await openPeerMeshNode({ + dataRoot: join(root, 'd'), + peer: memberDPeer, + }); const serving = [authority.serve(), memberB.serve(), memberC.serve(), memberD.serve()]; try { const meshId = (await authority.create()).roster.roster.meshId; @@ -501,6 +533,7 @@ test('reconciles one selected Mesh into signed transit routes and native policy' coordinationRelays: ['/memory/relay/peer-a'], }, ]); + assert.equal(memberB.resolveRoutes('peer-c').state, 'available'); assert.deepEqual(memberB.resolveRoutes('peer-c')?.transitRelayPeerIds, ['peer-a']); await memberD.setTransitMesh(meshId); @@ -585,7 +618,10 @@ test('reserves capacity for an offline leave until its authority obligation reti dataRoot: join(root, 'authority'), peer: authorityPeer, }); - const member = await openPeerMeshNode({ dataRoot: join(root, 'member'), peer: memberPeer }); + const member = await openPeerMeshNode({ + dataRoot: join(root, 'member'), + peer: memberPeer, + }); const serving = authority.serve(); try { const first = await authority.create(); @@ -663,11 +699,16 @@ test('retries a committed invitation redemption for the same authenticated peer' now: () => now, }); const memberRoot = join(root, 'member'); - let member = await openPeerMeshNode({ dataRoot: memberRoot, peer: memberPeer }); + let member = await openPeerMeshNode({ + dataRoot: memberRoot, + peer: memberPeer, + }); let serving = authority.serve(); try { const mesh = await authority.create(); - const invitation = await authority.invite(mesh.roster.roster.meshId, { ttlMs: 1_000 }); + const invitation = await authority.invite(mesh.roster.roster.meshId, { + ttlMs: 1_000, + }); authorityPeer.failNextResponse(); await assert.rejects(member.join(invitation)); @@ -722,13 +763,19 @@ test('cancels a recovered join while its authority is reconnecting', async () => const serving = authority.serve(); try { const mesh = await authority.create(); - const invitation = await authority.invite(mesh.roster.roster.meshId, { ttlMs: 1_000 }); + const invitation = await authority.invite(mesh.roster.roster.meshId, { + ttlMs: 1_000, + }); authorityPeer.failNextResponse(); await assert.rejects(member.join(invitation)); await member.close(); now += 2_000; - member = await openPeerMeshNode({ dataRoot: memberRoot, peer: memberPeer, now: () => now }); + member = await openPeerMeshNode({ + dataRoot: memberRoot, + peer: memberPeer, + now: () => now, + }); const connection = memberPeer.stallNextConnection(); const abort = new AbortController(); const retry = member.join(invitation, abort.signal); @@ -763,7 +810,10 @@ test('does not redeem a prepared join after explicit cancellation', async () => }); const memberRoot = join(root, 'cancelled'); let cancelled: PeerMeshNode | undefined; - const joining = await openPeerMeshNode({ dataRoot: join(root, 'joining'), peer: joiningPeer }); + const joining = await openPeerMeshNode({ + dataRoot: join(root, 'joining'), + peer: joiningPeer, + }); const serving = authority.serve(); try { const mesh = await authority.create(); @@ -787,7 +837,10 @@ test('does not redeem a prepared join after explicit cancellation', async () => )}\n`, { mode: 0o600 }, ); - cancelled = await openPeerMeshNode({ dataRoot: memberRoot, peer: cancelledPeer }); + cancelled = await openPeerMeshNode({ + dataRoot: memberRoot, + peer: cancelledPeer, + }); const connection = cancelledPeer.stallNextConnection(); const reconciliation = cancelled.reconcile(); @@ -820,7 +873,10 @@ test('cancels a redemption stalled after the control connection opens', async () dataRoot: join(root, 'authority'), peer: authorityPeer, }); - const member = await openPeerMeshNode({ dataRoot: join(root, 'member'), peer: memberPeer }); + const member = await openPeerMeshNode({ + dataRoot: join(root, 'member'), + peer: memberPeer, + }); const serving = authority.serve(); try { const mesh = await authority.create(); @@ -856,7 +912,10 @@ test('preserves an invitation when join is cancelled before redemption', async ( dataRoot: join(root, 'cancelled'), peer: cancelledPeer, }); - const joining = await openPeerMeshNode({ dataRoot: join(root, 'joining'), peer: joiningPeer }); + const joining = await openPeerMeshNode({ + dataRoot: join(root, 'joining'), + peer: joiningPeer, + }); const serving = authority.serve(); try { const mesh = await authority.create(); @@ -888,7 +947,10 @@ test('rejoins a Mesh after completed leave or stale authority removal', async () dataRoot: join(root, 'authority'), peer: authorityPeer, }); - const member = await openPeerMeshNode({ dataRoot: join(root, 'member'), peer: memberPeer }); + const member = await openPeerMeshNode({ + dataRoot: join(root, 'member'), + peer: memberPeer, + }); const serving = authority.serve(); try { const mesh = await authority.create(); @@ -924,7 +986,10 @@ test('turns a committed join into leave when cancellation arrives during transit dataRoot: join(root, 'authority'), peer: authorityPeer, }); - const member = await openPeerMeshNode({ dataRoot: join(root, 'member'), peer: memberPeer }); + const member = await openPeerMeshNode({ + dataRoot: join(root, 'member'), + peer: memberPeer, + }); const serving = authority.serve(); try { const mesh = await authority.create(); @@ -960,7 +1025,10 @@ test('preserves a committed join when its peer endpoint shuts down', async () => peer: authorityPeer, }); const memberRoot = join(root, 'member'); - let member = await openPeerMeshNode({ dataRoot: memberRoot, peer: memberPeer }); + let member = await openPeerMeshNode({ + dataRoot: memberRoot, + peer: memberPeer, + }); const serving = authority.serve(); try { const mesh = await authority.create(); @@ -1335,6 +1403,7 @@ async function waitForRoutes( await delay(25); } assert.deepEqual(node.resolveRoutes(peerId), { + state: 'available', routeHints: expectedRouteHints, coordinationRelays: expectedCoordinationRelays, transitRelayPeerIds: [], diff --git a/packages/runtime-host/src/__tests__/peer-native.test.ts b/packages/runtime-host/src/__tests__/peer-native.test.ts index 935c3f332f..ebd0698167 100644 --- a/packages/runtime-host/src/__tests__/peer-native.test.ts +++ b/packages/runtime-host/src/__tests__/peer-native.test.ts @@ -121,11 +121,17 @@ module.exports = { resolveRoutes: (peerId) => routesPrepared && peerId !== 'observed' ? { + state: 'available', routeHints: ['/memory/discovered'], coordinationRelays: ['/memory/relay'], transitRelayPeerIds: ['transit-peer'], } - : undefined, + : { + state: 'recovering', + routeHints: [], + coordinationRelays: [], + transitRelayPeerIds: [], + }, }, }); const native = await import(nativePath); @@ -327,7 +333,10 @@ module.exports = { const native = await import(modulePath); assert.deepEqual(native.default.starts, [{ keyPath: 'unused', webRtcStunUrls: [] }]); assert.equal( - await ensureRuntimeHostPeerIdentity({ nativePath: modulePath, keyPath: 'unused' }), + await ensureRuntimeHostPeerIdentity({ + nativePath: modulePath, + keyPath: 'unused', + }), 'peer', ); } finally { diff --git a/packages/runtime-host/src/client/peer-client.ts b/packages/runtime-host/src/client/peer-client.ts index 172e2f7974..06ed0fb79d 100644 --- a/packages/runtime-host/src/client/peer-client.ts +++ b/packages/runtime-host/src/client/peer-client.ts @@ -43,14 +43,19 @@ export interface RuntimeHostPeerConnectInput { export type RuntimeHostPeerConnectionPhase = 'discovering' | 'connecting'; +interface RuntimeHostPeerRouteCandidateSnapshot { + readonly routeHints: readonly string[]; + readonly coordinationRelays: readonly string[]; + readonly transitRelayPeerIds?: readonly string[]; +} + +export type RuntimeHostPeerRouteResolution = + | (RuntimeHostPeerRouteCandidateSnapshot & { readonly state: 'available' }) + | (RuntimeHostPeerRouteCandidateSnapshot & { readonly state: 'recovering' }) + | (RuntimeHostPeerRouteCandidateSnapshot & { readonly state: 'exhausted' }); + export interface RuntimeHostPeerRouteResolver { - resolveRoutes(peerId: string): - | { - readonly routeHints: readonly string[]; - readonly coordinationRelays: readonly string[]; - readonly transitRelayPeerIds?: readonly string[]; - } - | undefined; + resolveRoutes(peerId: string): RuntimeHostPeerRouteResolution; prepareRoutes?(peerId: string, signal: AbortSignal): Promise; } diff --git a/packages/runtime-host/src/peer-mesh/model.ts b/packages/runtime-host/src/peer-mesh/model.ts index 5339e07173..97f11ac89b 100644 --- a/packages/runtime-host/src/peer-mesh/model.ts +++ b/packages/runtime-host/src/peer-mesh/model.ts @@ -65,7 +65,6 @@ export interface SignedPeerMeshRosterV1 { } export interface PeerMeshAuthorityTarget { - readonly peerId: string; readonly reachability: SignedPeerReachabilityLeaseV1; } @@ -135,12 +134,17 @@ export function validatePeerMeshAuthorityKeyPair(keys: PeerMeshAuthorityKeyPair) type: 'pkcs8', }); } catch (error) { - throw new Error('Invalid Peer Mesh authority private key', { cause: error }); + throw new Error('Invalid Peer Mesh authority private key', { + cause: error, + }); } if (privateKey.asymmetricKeyType !== 'ed25519') { throw new Error('Peer Mesh authority key must be Ed25519'); } - const derived = createPublicKey(privateKey).export({ format: 'der', type: 'spki' }); + const derived = createPublicKey(privateKey).export({ + format: 'der', + type: 'spki', + }); if (derived.toString('base64url') !== keys.publicKey) { throw new Error('Peer Mesh authority private key does not match its public key'); } @@ -224,15 +228,9 @@ export function canonicalPeerMeshRoster(value: unknown): PeerMeshRosterV1 { } export function decodeAuthorityTarget(value: unknown): PeerMeshAuthorityTarget { - const record = exactObject(value, 'Peer Mesh authority target', ['peerId', 'reachability']); - const peerId = token(record.peerId, 'peerId', 256); - const reachability = decodeSignedPeerReachabilityLease(record.reachability); - if (reachability.lease.peerId !== peerId) { - throw new Error('Peer Mesh authority reachability belongs to a different peer'); - } + const record = exactObject(value, 'Peer Mesh authority target', ['reachability']); return Object.freeze({ - peerId, - reachability, + reachability: decodeSignedPeerReachabilityLease(record.reachability), }); } diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index 19bf952074..86548f268c 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -17,6 +17,7 @@ * under the License. */ +import type { RuntimeHostPeerRouteResolution } from '../client/peer-client.js'; import type { RuntimeHostPeerIdentityProof, RuntimeHostPeerNativeStream, @@ -24,6 +25,7 @@ import type { RuntimeHostPeerTransitSnapshot, } from '../transport/peer-native.js'; import { setTimeout as delay } from 'node:timers/promises'; +import { performance } from 'node:perf_hooks'; import { canonicalPeerMeshMemberAdvertisement, canonicalPeerMeshRoster, @@ -51,7 +53,10 @@ import { canonicalPeerMeshDisplayName } from './display-name.js'; import type { PeerMeshInvitationV1 } from '../protocol/peer-mesh.js'; import { decodeSignedPeerReachabilityLease, + isPeerReachabilityLeaseCurrent, + peerReachabilityLeaseReceipt, type PeerReachabilityPublisher, + type PeerReachabilityLeaseReceipt, type SignedPeerReachabilityLeaseV1, } from '../peer-reachability/index.js'; import { @@ -176,13 +181,7 @@ export interface PeerMeshNode { setTransitMesh(meshId: string | null): Promise; transitMeshId(): string | null; transitSnapshot(): RuntimeHostPeerTransitSnapshot; - resolveRoutes(peerId: string): - | { - readonly routeHints: readonly string[]; - readonly coordinationRelays: readonly string[]; - readonly transitRelayPeerIds: readonly string[]; - } - | undefined; + resolveRoutes(peerId: string): RuntimeHostPeerRouteResolution; prepareRoutes(peerId: string, signal: AbortSignal): Promise; reconcile(signal?: AbortSignal): Promise; serve(): Promise; @@ -241,6 +240,7 @@ export async function openPeerMeshNode(input: { readonly reachability: PeerReachabilityPublisher; readonly endpointKind?: 'client' | 'host'; readonly now?: () => number; + readonly monotonicNow?: () => number; readonly onBackgroundReconcileError?: (error: unknown) => void; }): Promise { const store = await openPeerMeshStateStore(input.dataRoot, input.peer.identity().peerId); @@ -260,6 +260,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { readonly #reachability: PeerReachabilityPublisher; readonly #endpointKind: 'client' | 'host' | undefined; readonly #now: () => number; + readonly #monotonicNow: () => number; readonly #onBackgroundReconcileError: ((error: unknown) => void) | undefined; readonly #activeControlStreams = new Set(); readonly #lifetime = new AbortController(); @@ -273,6 +274,8 @@ class PeerMeshNodeImpl implements PeerMeshNode { #reconcileGeneration = 0; readonly #reconcileWaiters = new Set<() => void>(); readonly #recentlyReached = new Set(); + readonly #reachabilityReceipts = new Map(); + readonly #completedRecoverySweeps = new Set(); #serveTask: Promise | undefined; #closeTask: Promise | undefined; @@ -282,6 +285,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { readonly reachability: PeerReachabilityPublisher; readonly endpointKind?: 'client' | 'host'; readonly now?: () => number; + readonly monotonicNow?: () => number; readonly onBackgroundReconcileError?: (error: unknown) => void; }) { this.#store = input.store; @@ -289,6 +293,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { this.#reachability = input.reachability; this.#endpointKind = input.endpointKind; this.#now = input.now ?? Date.now; + this.#monotonicNow = input.monotonicNow ?? (input.now ? input.now : () => performance.now()); this.#onBackgroundReconcileError = input.onBackgroundReconcileError; } @@ -355,7 +360,10 @@ class PeerMeshNodeImpl implements PeerMeshNode { authorityKeys(state), ); return { - state: { ...current, meshes: replaceMesh(current.meshes, { ...state, roster }) }, + state: { + ...current, + meshes: replaceMesh(current.meshes, { ...state, roster }), + }, result: { roster, targets: rosterAnnouncementTargets( @@ -377,6 +385,8 @@ class PeerMeshNodeImpl implements PeerMeshNode { stored.advertisements, this.#now(), this.#recentlyReached, + (peerId) => this.resolveRoutes(peerId), + (signed) => this.#isReachabilityCurrent(signed), ); }); } @@ -397,6 +407,8 @@ class PeerMeshNodeImpl implements PeerMeshNode { stored.advertisements, this.#now(), this.#recentlyReached, + (peerId) => this.resolveRoutes(peerId), + (signed) => this.#isReachabilityCurrent(signed), ), ), ); @@ -445,6 +457,8 @@ class PeerMeshNodeImpl implements PeerMeshNode { stored.advertisements, now, this.#recentlyReached, + (peerId) => this.resolveRoutes(peerId), + (signed) => this.#isReachabilityCurrent(signed), ); }); } @@ -684,7 +698,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { const state: PeerMeshReplicaStateV1 = { role: 'replica', authority: { - peerId: invitation.reachability.lease.peerId, reachability: authorityReachabilityFor(invitation), }, roster: selectedRoster, @@ -726,6 +739,8 @@ class PeerMeshNodeImpl implements PeerMeshNode { stored.advertisements, this.#now(), this.#recentlyReached, + (peerId) => this.resolveRoutes(peerId), + (signed) => this.#isReachabilityCurrent(signed), ); } @@ -750,7 +765,10 @@ class PeerMeshNodeImpl implements PeerMeshNode { ...current, meshes: existing?.role === 'replica' - ? replaceMesh(current.meshes, { ...existing, desiredMembership: 'left' }) + ? replaceMesh(current.meshes, { + ...existing, + desiredMembership: 'left', + }) : current.meshes, pendingJoins: current.pendingJoins.flatMap((pending) => { if (pending.invitation.meshId !== meshId) return [pending]; @@ -814,7 +832,10 @@ class PeerMeshNodeImpl implements PeerMeshNode { return { state: { ...current, - meshes: replaceMesh(current.meshes, { ...state, desiredMembership: 'left' }), + meshes: replaceMesh(current.meshes, { + ...state, + desiredMembership: 'left', + }), }, result: undefined, }; @@ -880,26 +901,38 @@ class PeerMeshNodeImpl implements PeerMeshNode { this.#assertOpen(); const now = this.#now(); const stored = this.#store.read(); + const localPeerId = this.#peer.identity().peerId; const sharedMeshIds = stored.meshes .filter( (state) => - isActiveMembership(state, this.#peer.identity().peerId) && - state.roster.roster.members.includes(peerId), + isActiveMembership(state, localPeerId) && state.roster.roster.members.includes(peerId), ) .map(({ roster }) => roster.roster.meshId); const visible = sharedMeshIds.length > 0; - if (!visible) return undefined; + if (!visible) return emptyRouteResolution('exhausted'); const reachability = latestReachability(stored.reachability, peerId, now, true)?.lease; - const localPeerId = this.#peer.identity().peerId; const transitRelayPeerIds = transitRelayCandidates( - eligibleTransitEvidence(stored, localPeerId, now).filter( + eligibleTransitEvidence(stored, localPeerId, now, (signed) => + this.#isReachabilityCurrent(signed), + ).filter( ({ meshId, lease }) => lease.lease.peerId !== peerId && sharedMeshIds.includes(meshId), ), ).map(({ peerId: relayPeerId }) => relayPeerId); - if (!reachability && transitRelayPeerIds.length === 0) return undefined; + const routeHints = reachability?.directRoutes ?? []; + const coordinationRelays = reachability?.coordinationRoutes ?? []; + const hasCandidates = + routeHints.length + coordinationRelays.length + transitRelayPeerIds.length > 0; + const state = hasCandidates + ? 'available' + : hasPeerRecoverySource(stored, sharedMeshIds, peerId, localPeerId, now) + ? 'recovering' + : this.#completedRecoverySweeps.has(peerId) + ? 'exhausted' + : 'recovering'; return Object.freeze({ - routeHints: reachability?.directRoutes ?? [], - coordinationRelays: reachability?.coordinationRoutes ?? [], + state, + routeHints, + coordinationRelays, transitRelayPeerIds: Object.freeze(transitRelayPeerIds), }); } @@ -907,19 +940,24 @@ class PeerMeshNodeImpl implements PeerMeshNode { async prepareRoutes(peerId: string, signal: AbortSignal): Promise { this.#assertOpen(); signal.throwIfAborted(); + this.#completedRecoverySweeps.delete(peerId); const localPeerId = this.#peer.identity().peerId; const stored = this.#store.read(); const visible = stored.meshes.some( (mesh) => isActiveMembership(mesh, localPeerId) && mesh.roster.roster.members.includes(peerId), ); - if (!visible) return; + if (!visible) { + this.#completedRecoverySweeps.add(peerId); + return; + } // A signed route can remain within its TTL after a peer restarted or // rotated Relay reservations. Every connection establishment therefore // asks the Mesh control plane for its newest record. Callers with a // self-contained invitation run this reconciliation in parallel with the // first dial; callers without usable routes wait for it. await this.reconcile(signal); + this.#completedRecoverySweeps.add(peerId); } reconcile(signal?: AbortSignal): Promise { @@ -1071,7 +1109,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { .filter( (peerId) => peerId !== identity.peerId && - (state.role === 'authority' || peerId !== state.authority.peerId), + (state.role === 'authority' || peerId !== state.authority.reachability.lease.peerId), ) .flatMap((peerId) => { const target = peerTarget(peerId, stored.reachability, now); @@ -1114,7 +1152,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { } catch (error) { if (lifetimeSignal.aborted) lifetimeSignal.throwIfAborted(); if (operation.kind === 'membership') { - this.#recentlyReached.delete(operation.target.peerId); + this.#recentlyReached.delete(operation.target.reachability.lease.peerId); } if (operation.kind === 'join' || operation.desiredMembership === 'left') { failures.push(error); @@ -1144,7 +1182,10 @@ class PeerMeshNodeImpl implements PeerMeshNode { signal: AbortSignal, ): Promise { const stream = await this.#peer.connectMeshControl( - { ...dialTarget(target.reachability), directDeadlineMs: CONNECT_DEADLINE_MS }, + { + ...dialTarget(target.reachability), + directDeadlineMs: CONNECT_DEADLINE_MS, + }, signal, ); try { @@ -1158,7 +1199,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { throw new Error('Peer Mesh authority rejected the leave request'); } await this.#applySync(meshId, response.roster, [], []); - this.#recentlyReached.add(target.peerId); + this.#recentlyReached.add(target.reachability.lease.peerId); } finally { await stream.close().catch(() => undefined); } @@ -1180,11 +1221,12 @@ class PeerMeshNodeImpl implements PeerMeshNode { if (!reachability || !advertisement) { throw new Error('Peer Mesh local evidence is unavailable'); } - const discovered = this.resolveRoutes(target.peerId); + const targetPeerId = target.reachability.lease.peerId; + const discovered = this.resolveRoutes(targetPeerId); const targetRoutes = dialTarget(target.reachability); const stream = await this.#peer.connectMeshControl( { - peerId: target.peerId, + peerId: targetPeerId, routeHints: mergeAddresses(discovered?.routeHints ?? [], targetRoutes.routeHints), coordinationRelays: mergeAddresses( discovered?.coordinationRelays ?? [], @@ -1223,7 +1265,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { response.reachability, response.advertisements, ); - this.#recentlyReached.add(target.peerId); + this.#recentlyReached.add(targetPeerId); if (!response.more) return; } finally { await stream.close().catch(() => undefined); @@ -1241,6 +1283,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { async #refreshLocalEvidenceOnce(): Promise { const reachability = await this.#reachability.refresh(); + this.#recordReachabilityReceipt(reachability); const identity = this.#peer.identity(); const now = this.#now(); return this.#store.mutate(async (current) => { @@ -1255,7 +1298,10 @@ class PeerMeshNodeImpl implements PeerMeshNode { reachability: mergeReachability(current.reachability, [reachability], now), advertisements: mergeAdvertisements(current.advertisements, advertisements), }, - result: Object.freeze({ reachability, advertisements: Object.freeze(advertisements) }), + result: Object.freeze({ + reachability, + advertisements: Object.freeze(advertisements), + }), }; }); } @@ -1336,10 +1382,13 @@ class PeerMeshNodeImpl implements PeerMeshNode { expectedPeerId: string, allowExpired = false, ): SignedPeerReachabilityLeaseV1 { - const signed = this.#reachability.verify(value, expectedPeerId, { allowExpired }); + const signed = this.#reachability.verify(value, expectedPeerId, { + allowExpired, + }); if (allowExpired && signed.lease.expiresAt + REACHABILITY_HISTORY_MS <= this.#now()) { throw new Error('Peer Mesh reachability is outside the recovery horizon'); } + this.#recordReachabilityReceipt(signed); return signed; } @@ -1394,6 +1443,35 @@ class PeerMeshNodeImpl implements PeerMeshNode { if (!valid) throw new Error('Peer Mesh member advertisement signature is invalid'); } + #recordReachabilityReceipt(signed: SignedPeerReachabilityLeaseV1): void { + const previous = this.#reachabilityReceipts.get(signed.lease.peerId); + if (previous && previous.revision > signed.lease.revision) return; + if ( + previous && + previous.revision === signed.lease.revision && + previous.signature !== signed.signature + ) { + return; + } + this.#reachabilityReceipts.set( + signed.lease.peerId, + peerReachabilityLeaseReceipt({ + signed, + wallNow: this.#now(), + monotonicNow: this.#monotonicNow(), + ...(previous ? { previous } : {}), + }), + ); + } + + #isReachabilityCurrent(signed: SignedPeerReachabilityLeaseV1): boolean { + return isPeerReachabilityLeaseCurrent( + signed, + this.#reachabilityReceipts.get(signed.lease.peerId), + this.#monotonicNow(), + ); + } + async #applySync( meshId: string, rosterValue: SignedPeerMeshRosterV1, @@ -1510,6 +1588,8 @@ class PeerMeshNodeImpl implements PeerMeshNode { stored.advertisements, this.#now(), this.#recentlyReached, + (peerId) => this.resolveRoutes(peerId), + (signed) => this.#isReachabilityCurrent(signed), ); } @@ -1522,7 +1602,10 @@ class PeerMeshNodeImpl implements PeerMeshNode { void Promise.allSettled( targets.map(async (target) => { const stream = await this.#peer.connectMeshControl( - { ...dialTarget(target.reachability), directDeadlineMs: CONNECT_DEADLINE_MS }, + { + ...dialTarget(target.reachability), + directDeadlineMs: CONNECT_DEADLINE_MS, + }, signal, ); try { @@ -1539,7 +1622,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { if (response.kind === 'roster-rejected') { throw new Error('Peer Mesh roster announcement was rejected'); } - this.#recentlyReached.add(target.peerId); + this.#recentlyReached.add(target.reachability.lease.peerId); } finally { await stream.close().catch(() => undefined); } @@ -1791,7 +1874,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { stored.reachability, this.#peer.identity().peerId, this.#now(), - ).filter(({ peerId }) => peerId !== remotePeerId), + ).filter(({ reachability }) => reachability.lease.peerId !== remotePeerId), ); } this.#recentlyReached.add(remotePeerId); @@ -1956,7 +2039,9 @@ class PeerMeshNodeImpl implements PeerMeshNode { (mesh) => mesh.roster.roster.meshId === stored.transitMeshId && isActiveMembership(mesh, localPeerId), ); - const eligibleRelays = eligibleTransitEvidence(stored, localPeerId, now); + const eligibleRelays = eligibleTransitEvidence(stored, localPeerId, now, (signed) => + this.#isReachabilityCurrent(signed), + ); const relayCandidates = transitRelayCandidates(eligibleRelays); const approvedRelayPeerIds = [ ...new Set( @@ -1985,6 +2070,7 @@ function eligibleTransitEvidence( stored: PeerMeshStoredStateV1, localPeerId: string, now: number, + isCurrent: (signed: SignedPeerReachabilityLeaseV1) => boolean, ): readonly PeerMeshTransitEvidence[] { const evidence = new Map(); for (const { advertisement } of stored.advertisements) { @@ -1995,8 +2081,12 @@ function eligibleTransitEvidence( ) { continue; } - const lease = latestReachability(stored.reachability, advertisement.peerId, now, false); - if (!lease || lease.lease.directRoutes.length + lease.lease.coordinationRoutes.length === 0) { + const lease = latestReachability(stored.reachability, advertisement.peerId, now, true); + if ( + !lease || + !isCurrent(lease) || + lease.lease.directRoutes.length + lease.lease.coordinationRoutes.length === 0 + ) { continue; } evidence.set(advertisement.peerId, { @@ -2072,16 +2162,19 @@ function peerMeshStatus( state: PeerMeshStateV1, identity: ReturnType, endpointKind: 'client' | 'host' | undefined, - reachability: readonly SignedPeerReachabilityLeaseV1[] = [], - advertisements: readonly SignedPeerMeshMemberAdvertisementV1[] = [], - now = Date.now(), - recentlyReached: ReadonlySet = new Set(), + reachability: readonly SignedPeerReachabilityLeaseV1[], + advertisements: readonly SignedPeerMeshMemberAdvertisementV1[], + now: number, + recentlyReached: ReadonlySet, + resolveRoutes: (peerId: string) => RuntimeHostPeerRouteResolution, + isCurrent: (signed: SignedPeerReachabilityLeaseV1) => boolean, ): PeerMeshStatus { const meshId = state.roster.roster.meshId; const localAdvertisement = findAdvertisement(advertisements, meshId, identity.peerId); return Object.freeze({ role: state.role === 'authority' ? 'authority' : 'member', - authorityPeerId: state.role === 'authority' ? identity.peerId : state.authority.peerId, + authorityPeerId: + state.role === 'authority' ? identity.peerId : state.authority.reachability.lease.peerId, roster: state.roster, pendingInvitationCount: state.role === 'authority' @@ -2104,21 +2197,25 @@ function peerMeshStatus( const advertisement = findAdvertisement(advertisements, meshId, peerId)?.advertisement; const signed = latestReachability(reachability, peerId, now, true); const lease = signed?.lease; - const hasLocator = Boolean( - lease && lease.directRoutes.length + lease.coordinationRoutes.length > 0, - ); - const state = !hasLocator - ? ('needs_repair' as const) - : lease!.expiresAt <= now - ? ('reconnecting' as const) - : recentlyReached.has(peerId) - ? ('reachable' as const) - : ('connecting' as const); + const resolution = resolveRoutes(peerId); + const current = Boolean(signed && isCurrent(signed)); + const memberState = + resolution.state === 'exhausted' + ? ('needs_repair' as const) + : resolution.state === 'recovering' + ? signed + ? ('reconnecting' as const) + : ('connecting' as const) + : current && recentlyReached.has(peerId) + ? ('reachable' as const) + : current + ? ('connecting' as const) + : ('reconnecting' as const); return Object.freeze({ peerId, ...(advertisement?.endpointKind ? { endpointKind: advertisement.endpointKind } : {}), ...(advertisement?.displayName ? { displayName: advertisement.displayName } : {}), - state, + state: memberState, ...(lease ? { expiresAt: lease.expiresAt } : {}), }); }), @@ -2131,14 +2228,13 @@ function currentAuthorityTarget( reachability: readonly SignedPeerReachabilityLeaseV1[], now: number, ): PeerMeshAuthorityTarget | undefined { - const learned = latestReachability(reachability, state.authority.peerId, now, true); + const authorityPeerId = state.authority.reachability.lease.peerId; + const learned = latestReachability(reachability, authorityPeerId, now, true); const fallback = usableHistoricalReachability(state.authority.reachability, now) ? state.authority.reachability : undefined; const selected = selectReachability(learned, fallback); - return selected - ? Object.freeze({ peerId: state.authority.peerId, reachability: selected }) - : undefined; + return selected ? Object.freeze({ reachability: selected }) : undefined; } function rosterAnnouncementTargets( @@ -2156,12 +2252,7 @@ function rosterAnnouncementTargets( members.has(signed.lease.peerId) && usableHistoricalReachability(signed, now), ) - .map((signed) => - Object.freeze({ - peerId: signed.lease.peerId, - reachability: signed, - }), - ), + .map((signed) => Object.freeze({ reachability: signed })), ); } @@ -2390,7 +2481,11 @@ function responseEvidence( state.roster.roster.members.includes(signed.lease.peerId) && signed.lease.revision > (knownLeases.get(signed.lease.peerId) ?? 0), ) - .map((value) => ({ kind: 'reachability' as const, peerId: value.lease.peerId, value })), + .map((value) => ({ + kind: 'reachability' as const, + peerId: value.lease.peerId, + value, + })), ...advertisements .filter( ({ advertisement }) => @@ -2468,6 +2563,47 @@ function latestReachability( return includeHistorical && usableHistoricalReachability(signed, now) ? signed : undefined; } +function emptyRouteResolution(state: 'recovering' | 'exhausted'): RuntimeHostPeerRouteResolution { + return Object.freeze({ + state, + routeHints: Object.freeze([]), + coordinationRelays: Object.freeze([]), + transitRelayPeerIds: Object.freeze([]), + }); +} + +function hasPeerRecoverySource( + stored: PeerMeshStoredStateV1, + sharedMeshIds: readonly string[], + targetPeerId: string, + localPeerId: string, + now: number, +): boolean { + const shared = new Set(sharedMeshIds); + const sourcePeerIds = new Set( + stored.meshes + .filter(({ roster }) => shared.has(roster.roster.meshId)) + .flatMap(({ roster }) => roster.roster.members) + .filter((peerId) => peerId !== localPeerId && peerId !== targetPeerId), + ); + for (const sourcePeerId of sourcePeerIds) { + const signed = latestReachability(stored.reachability, sourcePeerId, now, true); + if (signed && hasReachabilityRoutes(signed)) return true; + } + return stored.meshes.some( + (state) => + state.role === 'replica' && + shared.has(state.roster.roster.meshId) && + sourcePeerIds.has(state.authority.reachability.lease.peerId) && + usableHistoricalReachability(state.authority.reachability, now) && + hasReachabilityRoutes(state.authority.reachability), + ); +} + +function hasReachabilityRoutes(signed: SignedPeerReachabilityLeaseV1): boolean { + return signed.lease.directRoutes.length + signed.lease.coordinationRoutes.length > 0; +} + function usableHistoricalReachability(signed: SignedPeerReachabilityLeaseV1, now: number): boolean { return signed.lease.expiresAt + REACHABILITY_HISTORY_MS > now; } @@ -2487,7 +2623,7 @@ function peerTarget( now: number, ): PeerMeshAuthorityTarget | undefined { const signed = latestReachability(reachability, peerId, now, true); - return signed ? Object.freeze({ peerId, reachability: signed }) : undefined; + return signed ? Object.freeze({ reachability: signed }) : undefined; } function dialTarget(reachability: SignedPeerReachabilityLeaseV1): { diff --git a/packages/runtime-host/src/peer-mesh/store.ts b/packages/runtime-host/src/peer-mesh/store.ts index 13681f8564..3ee65b12c8 100644 --- a/packages/runtime-host/src/peer-mesh/store.ts +++ b/packages/runtime-host/src/peer-mesh/store.ts @@ -287,7 +287,7 @@ export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMe }); } const authority = decodeAuthorityTarget(record.authority); - if (!roster.roster.members.includes(authority.peerId)) { + if (!roster.roster.members.includes(authority.reachability.lease.peerId)) { throw new Error('Peer Mesh authority is not present in its roster'); } return Object.freeze({ diff --git a/packages/runtime-host/src/peer-reachability/index.ts b/packages/runtime-host/src/peer-reachability/index.ts index 82444ad154..cb297f37e2 100644 --- a/packages/runtime-host/src/peer-reachability/index.ts +++ b/packages/runtime-host/src/peer-reachability/index.ts @@ -26,10 +26,13 @@ export { PEER_REACHABILITY_MAX_RECORD_BYTES, PEER_REACHABILITY_MAX_ROUTES_PER_CLASS, PEER_REACHABILITY_REFRESH_LEAD_MS, + isPeerReachabilityLeaseCurrent, + peerReachabilityLeaseReceipt, peerReachabilityLeaseSigningBytes, samePeerReachabilityRoutes, verifySignedPeerReachabilityLease, type PeerReachabilityIdentity, + type PeerReachabilityLeaseReceipt, type PeerReachabilityLeaseV1, type SignedPeerReachabilityLeaseV1, } from './model.js'; diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index 3efbf9819a..deff39f1ce 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -100,7 +100,10 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const; export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const; // Increment when the same protocol version no longer guarantees safe Client-Host // interoperability. Mismatches are rejected before domain commands are admitted. -export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 97 as const; +export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 98 as const; +// 98: Peer Mesh invitations carry signed reachability leases and member route +// projections use the convergent recovery state machine. Older peers decode a +// different strict wire shape. // 97: Host status replaces unsigned route arrays with a self-signed, bounded // reachability lease. Older peers cannot validate the locator revision or its // target identity before retaining it for reconnect. From 75bff962592961f027985775e952f34b5f80d3db Mon Sep 17 00:00:00 2001 From: Wang Date: Thu, 3 Sep 2026 02:01:31 +0800 Subject: [PATCH 03/13] fix(peer): bind mesh authority identity Generated-by: Codex (gpt-5.6-sol) --- .../src/__tests__/peer-mesh.test.ts | 88 +++++++- packages/runtime-host/src/peer-mesh/model.ts | 24 +-- packages/runtime-host/src/peer-mesh/node.ts | 200 ++++++++++-------- packages/runtime-host/src/peer-mesh/store.ts | 18 +- .../runtime-host/src/protocol/peer-mesh.ts | 10 +- 5 files changed, 220 insertions(+), 120 deletions(-) diff --git a/packages/runtime-host/src/__tests__/peer-mesh.test.ts b/packages/runtime-host/src/__tests__/peer-mesh.test.ts index bc195c081f..5bdf363e9b 100644 --- a/packages/runtime-host/src/__tests__/peer-mesh.test.ts +++ b/packages/runtime-host/src/__tests__/peer-mesh.test.ts @@ -19,7 +19,7 @@ import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { setImmediate as waitForImmediate, setTimeout as delay } from 'node:timers/promises'; @@ -267,6 +267,7 @@ test('rejects a modified authority-signed roster', () => { { version: 1, meshId: peerMeshId(keys.publicKey), + authorityPeerId: 'peer-a', revision: 1, members: ['peer-a'], closed: false, @@ -432,6 +433,91 @@ test('repairs an existing membership with a fresh invitation after every locator } }); +test('does not let a member replace the signed Mesh authority locator', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-peer-mesh-authority-binding-')); + const network = new MemoryPeerNetwork(); + const authorityPeer = network.create('peer-a'); + const memberBPeer = network.create('peer-b'); + const memberCPeer = network.create('peer-c'); + const authority = await openPeerMeshNode({ + dataRoot: join(root, 'authority'), + peer: authorityPeer, + }); + const memberB = await openPeerMeshNode({ + dataRoot: join(root, 'member-b'), + peer: memberBPeer, + }); + const memberC = await openPeerMeshNode({ + dataRoot: join(root, 'member-c'), + peer: memberCPeer, + }); + const serving = [authority.serve(), memberB.serve(), memberC.serve()]; + try { + const meshId = (await authority.create()).roster.roster.meshId; + await memberB.join(await authority.invite(meshId)); + await memberC.join(await authority.invite(meshId)); + const invitation = await authority.invite(meshId); + + await assert.rejects( + memberB.join({ ...invitation, reachability: memberCPeer.current() }), + /wrong authority identity/u, + ); + assert.equal(memberB.status()[0]?.authorityPeerId, 'peer-a'); + } finally { + await Promise.allSettled([authority.close(), memberB.close(), memberC.close()]); + await Promise.allSettled([authorityPeer.close(), memberBPeer.close(), memberCPeer.close()]); + await Promise.allSettled(serving); + await rm(root, { recursive: true, force: true }); + } +}); + +test('keeps Mesh membership while pruning reachability beyond its recovery horizon', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-peer-mesh-expired-reachability-')); + const network = new MemoryPeerNetwork(); + const authorityPeer = network.create('peer-a'); + const memberPeer = network.create('peer-b'); + let now = Date.now(); + const authority = await openPeerMeshNode({ + dataRoot: join(root, 'authority'), + peer: authorityPeer, + now: () => now, + }); + const memberRoot = join(root, 'member'); + let member = await openPeerMeshNode({ + dataRoot: memberRoot, + peer: memberPeer, + now: () => now, + }); + const serving = authority.serve(); + try { + const meshId = (await authority.create()).roster.roster.meshId; + await member.join(await authority.invite(meshId)); + await member.close(); + + now += PEER_REACHABILITY_LEASE_TTL_MS + 24 * 60 * 60 * 1_000 + 1; + member = await openPeerMeshNode({ + dataRoot: memberRoot, + peer: memberPeer, + now: () => now, + }); + + assert.equal(member.status()[0]?.roster.roster.meshId, meshId); + const persisted = JSON.parse(await readFile(join(memberRoot, 'peer-mesh.json'), 'utf8')) as { + readonly reachability: readonly { + readonly lease: { readonly peerId: string }; + }[]; + }; + assert.deepEqual( + persisted.reachability.map(({ lease }) => lease.peerId), + ['peer-b'], + ); + } finally { + await Promise.allSettled([authority.close(), member.close()]); + await Promise.allSettled([authorityPeer.close(), memberPeer.close(), serving]); + await rm(root, { recursive: true, force: true }); + } +}); + test('publishes a changed local route promptly and refreshes a live cached peer route', { timeout: 10_000, }, async () => { diff --git a/packages/runtime-host/src/peer-mesh/model.ts b/packages/runtime-host/src/peer-mesh/model.ts index 97f11ac89b..07327e9edb 100644 --- a/packages/runtime-host/src/peer-mesh/model.ts +++ b/packages/runtime-host/src/peer-mesh/model.ts @@ -31,10 +31,6 @@ import { decodePeerMeshInvitation as decodePeerMeshInvitationWire, type PeerMeshInvitationV1, } from '../protocol/peer-mesh.js'; -import { - decodeSignedPeerReachabilityLease, - type SignedPeerReachabilityLeaseV1, -} from '../peer-reachability/index.js'; import { PEER_MESH_MEMBER_ADVERTISEMENT_MAX_BYTES, PEER_MESH_MAX_MEMBERS } from './limits.js'; import { canonicalPeerMeshDisplayName } from './display-name.js'; @@ -52,6 +48,7 @@ export { export interface PeerMeshRosterV1 { readonly version: 1; readonly meshId: string; + readonly authorityPeerId: string; readonly revision: number; readonly members: readonly string[]; readonly closed: boolean; @@ -64,10 +61,6 @@ export interface SignedPeerMeshRosterV1 { readonly signature: string; } -export interface PeerMeshAuthorityTarget { - readonly reachability: SignedPeerReachabilityLeaseV1; -} - export interface PeerMeshAuthorityKeyPair { readonly publicKey: string; readonly privateKey: string; @@ -202,7 +195,7 @@ export function validatePeerMeshInvitation(value: unknown): PeerMeshInvitationV1 } export function canonicalPeerMeshRoster(value: unknown): PeerMeshRosterV1 { - const keys = ['version', 'meshId', 'revision', 'members', 'closed']; + const keys = ['version', 'meshId', 'authorityPeerId', 'revision', 'members', 'closed']; if (value && typeof value === 'object' && !Array.isArray(value)) { if (Object.hasOwn(value, 'displayName')) keys.push('displayName'); } @@ -214,10 +207,15 @@ export function canonicalPeerMeshRoster(value: unknown): PeerMeshRosterV1 { if (members.length === 0 || new Set(members).size !== members.length) { throw new Error('Peer Mesh roster members must be unique and non-empty'); } + const authorityPeerId = token(record.authorityPeerId, 'authorityPeerId', 256); + if (!members.includes(authorityPeerId)) { + throw new Error('Peer Mesh authority must be present in its roster'); + } if (typeof record.closed !== 'boolean') throw new Error('Invalid Peer Mesh roster closed'); return Object.freeze({ version: 1, meshId: string(record.meshId, 'meshId', 128), + authorityPeerId, revision: integer(record.revision, 'revision', 1), members: Object.freeze(members), closed: record.closed, @@ -227,13 +225,6 @@ export function canonicalPeerMeshRoster(value: unknown): PeerMeshRosterV1 { }); } -export function decodeAuthorityTarget(value: unknown): PeerMeshAuthorityTarget { - const record = exactObject(value, 'Peer Mesh authority target', ['reachability']); - return Object.freeze({ - reachability: decodeSignedPeerReachabilityLease(record.reachability), - }); -} - export function canonicalPeerMeshMemberAdvertisement( value: unknown, ): PeerMeshMemberAdvertisementV1 { @@ -312,6 +303,7 @@ export function peerMeshMemberAdvertisementSigningBytes( function encodeRoster(roster: PeerMeshRosterV1): Buffer { return Buffer.from( `maka.peer-mesh.roster.v1\n${JSON.stringify({ + authorityPeerId: roster.authorityPeerId, closed: roster.closed, ...(roster.displayName ? { displayName: roster.displayName } : {}), members: roster.members, diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index 86548f268c..250270a2ad 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -45,7 +45,6 @@ import { peerMeshId, peerMeshInvitationSecretDigest, signPeerMeshRoster, - type PeerMeshAuthorityTarget, type SignedPeerMeshMemberAdvertisementV1, type SignedPeerMeshRosterV1, } from './model.js'; @@ -111,10 +110,6 @@ interface PeerMeshEvidenceRevision { readonly revision: number; } -interface PeerMeshAdvertisementRevision extends PeerMeshEvidenceRevision { - readonly meshId: string; -} - interface SyncPeerMeshRequest { readonly kind: 'sync'; readonly meshId: string; @@ -122,7 +117,7 @@ interface SyncPeerMeshRequest { readonly reachability: SignedPeerReachabilityLeaseV1; readonly advertisement: SignedPeerMeshMemberAdvertisementV1; readonly knownReachability: readonly PeerMeshEvidenceRevision[]; - readonly knownAdvertisements: readonly PeerMeshAdvertisementRevision[]; + readonly knownAdvertisements: readonly PeerMeshEvidenceRevision[]; } type SyncPeerMeshResponse = @@ -300,11 +295,38 @@ class PeerMeshNodeImpl implements PeerMeshNode { async initialize(): Promise { const stored = this.#store.read(); for (const lease of stored.reachability) { - this.#validateReachability(lease, lease.lease.peerId, true); + const signed = this.#reachability.verify(lease, lease.lease.peerId, { + allowExpired: true, + }); + if (usableHistoricalReachability(signed, this.#now())) { + this.#recordReachabilityReceipt(signed); + } + } + for (const state of stored.meshes) { + if (state.role !== 'replica') continue; + const signed = this.#reachability.verify( + state.authorityReachability, + state.roster.roster.authorityPeerId, + { allowExpired: true }, + ); + if (usableHistoricalReachability(signed, this.#now())) { + this.#recordReachabilityReceipt(signed); + } } for (const advertisement of stored.advertisements) { this.#assertAdvertisementSignature(advertisement); } + if (stored.reachability.some((signed) => !usableHistoricalReachability(signed, this.#now()))) { + await this.#store.mutate((current) => ({ + state: { + ...current, + reachability: current.reachability.filter((signed) => + usableHistoricalReachability(signed, this.#now()), + ), + }, + result: undefined, + })); + } await this.#refreshLocalEvidence(); this.#unsubscribeReachability = this.#reachability.subscribe(() => { this.#triggerReconciliation(); @@ -422,6 +444,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { canonicalPeerMeshRoster({ version: 1, meshId: peerMeshId(keys.publicKey), + authorityPeerId: identity.peerId, revision: 1, members: [identity.peerId], closed: false, @@ -541,6 +564,12 @@ class PeerMeshNodeImpl implements PeerMeshNode { if (existing?.role === 'authority') { throw new Error('This peer already belongs to that Peer Mesh'); } + if ( + existing?.role === 'replica' && + authorityReachability.lease.peerId !== existing.roster.roster.authorityPeerId + ) { + throw new Error('Peer Mesh repair invitation has the wrong authority identity'); + } assertRejoinSettled(existing, localPeerId); if (pending && pending.invitation.secret !== invitation.secret) { throw new Error('This Peer Mesh already has an unresolved join attempt'); @@ -564,6 +593,9 @@ class PeerMeshNodeImpl implements PeerMeshNode { }, operationSignal, ); + if (stream.peerId !== authorityReachability.lease.peerId) { + throw new Error('Peer Mesh control stream has the wrong peer identity'); + } const localReachability = await this.#reachability.refresh(); const localAdvertisement = await this.#signLocalAdvertisement(invitation.meshId); await this.#store.mutate((current) => { @@ -628,6 +660,11 @@ class PeerMeshNodeImpl implements PeerMeshNode { signal: AbortSignal, ): Promise { signal.throwIfAborted(); + const authorityReachability = this.#validateReachability( + invitation.reachability, + invitation.reachability.lease.peerId, + true, + ); const dispatch = await this.#store.mutate((current) => { const pending = current.pendingJoins.find( ({ invitation: candidate }) => @@ -672,8 +709,11 @@ class PeerMeshNodeImpl implements PeerMeshNode { if ( roster.roster.meshId !== invitation.meshId || roster.authorityPublicKey !== invitation.authorityPublicKey || + roster.roster.authorityPeerId !== stream.peerId || + invitation.reachability.lease.peerId !== stream.peerId || !roster.roster.members.includes(identity.peerId) ) { + await this.#discardPendingJoin(invitation); throw new Error('Peer Mesh authority returned an unrelated roster'); } const reachability = this.#validateReachabilityPage(response.reachability, roster, true); @@ -697,9 +737,13 @@ class PeerMeshNodeImpl implements PeerMeshNode { } const state: PeerMeshReplicaStateV1 = { role: 'replica', - authority: { - reachability: authorityReachabilityFor(invitation), - }, + authorityReachability: + selectReachability( + reachability.find( + ({ lease }) => lease.peerId === selectedRoster.roster.authorityPeerId, + ), + authorityReachability, + ) ?? authorityReachability, roster: selectedRoster, desiredMembership: pending.phase === 'leave_pending' ? 'left' : 'active', }; @@ -781,9 +825,21 @@ class PeerMeshNodeImpl implements PeerMeshNode { } async #resumePendingJoin(pending: PendingPeerMeshJoin, signal: AbortSignal): Promise { + const authorityReachability = this.#validateReachability( + pending.invitation.reachability, + pending.invitation.reachability.lease.peerId, + true, + ); + const existing = findMesh(this.#store.read().meshes, pending.invitation.meshId); + if ( + existing?.role === 'replica' && + authorityReachability.lease.peerId !== existing.roster.roster.authorityPeerId + ) { + throw new Error('Peer Mesh repair invitation has the wrong authority identity'); + } const stream = await this.#peer.connectMeshControl( { - ...dialTarget(pending.invitation.reachability), + ...dialTarget(authorityReachability), directDeadlineMs: CONNECT_DEADLINE_MS, }, signal, @@ -1080,7 +1136,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { | { readonly kind: 'membership'; readonly meshId: string; - readonly target: PeerMeshAuthorityTarget; + readonly target: SignedPeerReachabilityLeaseV1; readonly desiredMembership: 'active' | 'left'; readonly roster: SignedPeerMeshRosterV1; } @@ -1109,7 +1165,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { .filter( (peerId) => peerId !== identity.peerId && - (state.role === 'authority' || peerId !== state.authority.reachability.lease.peerId), + (state.role === 'authority' || peerId !== state.roster.roster.authorityPeerId), ) .flatMap((peerId) => { const target = peerTarget(peerId, stored.reachability, now); @@ -1152,7 +1208,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { } catch (error) { if (lifetimeSignal.aborted) lifetimeSignal.throwIfAborted(); if (operation.kind === 'membership') { - this.#recentlyReached.delete(operation.target.reachability.lease.peerId); + this.#recentlyReached.delete(operation.target.lease.peerId); } if (operation.kind === 'join' || operation.desiredMembership === 'left') { failures.push(error); @@ -1177,13 +1233,13 @@ class PeerMeshNodeImpl implements PeerMeshNode { async #notifyLeave( meshId: string, - target: PeerMeshAuthorityTarget, + target: SignedPeerReachabilityLeaseV1, roster: SignedPeerMeshRosterV1, signal: AbortSignal, ): Promise { const stream = await this.#peer.connectMeshControl( { - ...dialTarget(target.reachability), + ...dialTarget(target), directDeadlineMs: CONNECT_DEADLINE_MS, }, signal, @@ -1199,7 +1255,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { throw new Error('Peer Mesh authority rejected the leave request'); } await this.#applySync(meshId, response.roster, [], []); - this.#recentlyReached.add(target.reachability.lease.peerId); + this.#recentlyReached.add(target.lease.peerId); } finally { await stream.close().catch(() => undefined); } @@ -1207,7 +1263,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { async #syncPeer( meshId: string, - target: PeerMeshAuthorityTarget, + target: SignedPeerReachabilityLeaseV1, signal: AbortSignal, ): Promise { for (let page = 0; page <= PEER_MESH_MAX_MEMBERS; page += 1) { @@ -1221,9 +1277,9 @@ class PeerMeshNodeImpl implements PeerMeshNode { if (!reachability || !advertisement) { throw new Error('Peer Mesh local evidence is unavailable'); } - const targetPeerId = target.reachability.lease.peerId; + const targetPeerId = target.lease.peerId; const discovered = this.resolveRoutes(targetPeerId); - const targetRoutes = dialTarget(target.reachability); + const targetRoutes = dialTarget(target); const stream = await this.#peer.connectMeshControl( { peerId: targetPeerId, @@ -1539,6 +1595,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { { version: 1, meshId: state.roster.roster.meshId, + authorityPeerId: state.roster.roster.authorityPeerId, revision: state.roster.roster.revision + 1, members: next.members, closed: next.closed, @@ -1595,7 +1652,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { #scheduleRosterAnnouncement( roster: SignedPeerMeshRosterV1, - targets: readonly PeerMeshAuthorityTarget[], + targets: readonly SignedPeerReachabilityLeaseV1[], ): void { if (targets.length === 0 || this.#lifetime.signal.aborted) return; const signal = this.#lifetime.signal; @@ -1603,7 +1660,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { targets.map(async (target) => { const stream = await this.#peer.connectMeshControl( { - ...dialTarget(target.reachability), + ...dialTarget(target), directDeadlineMs: CONNECT_DEADLINE_MS, }, signal, @@ -1622,7 +1679,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { if (response.kind === 'roster-rejected') { throw new Error('Peer Mesh roster announcement was rejected'); } - this.#recentlyReached.add(target.reachability.lease.peerId); + this.#recentlyReached.add(target.lease.peerId); } finally { await stream.close().catch(() => undefined); } @@ -1874,7 +1931,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { stored.reachability, this.#peer.identity().peerId, this.#now(), - ).filter(({ reachability }) => reachability.lease.peerId !== remotePeerId), + ).filter(({ lease }) => lease.peerId !== remotePeerId), ); } this.#recentlyReached.add(remotePeerId); @@ -2173,8 +2230,7 @@ function peerMeshStatus( const localAdvertisement = findAdvertisement(advertisements, meshId, identity.peerId); return Object.freeze({ role: state.role === 'authority' ? 'authority' : 'member', - authorityPeerId: - state.role === 'authority' ? identity.peerId : state.authority.reachability.lease.peerId, + authorityPeerId: state.roster.roster.authorityPeerId, roster: state.roster, pendingInvitationCount: state.role === 'authority' @@ -2227,14 +2283,13 @@ function currentAuthorityTarget( state: PeerMeshReplicaStateV1, reachability: readonly SignedPeerReachabilityLeaseV1[], now: number, -): PeerMeshAuthorityTarget | undefined { - const authorityPeerId = state.authority.reachability.lease.peerId; +): SignedPeerReachabilityLeaseV1 | undefined { + const authorityPeerId = state.roster.roster.authorityPeerId; const learned = latestReachability(reachability, authorityPeerId, now, true); - const fallback = usableHistoricalReachability(state.authority.reachability, now) - ? state.authority.reachability + const fallback = usableHistoricalReachability(state.authorityReachability, now) + ? state.authorityReachability : undefined; - const selected = selectReachability(learned, fallback); - return selected ? Object.freeze({ reachability: selected }) : undefined; + return selectReachability(learned, fallback); } function rosterAnnouncementTargets( @@ -2242,17 +2297,15 @@ function rosterAnnouncementTargets( reachability: readonly SignedPeerReachabilityLeaseV1[], localPeerId: string, now: number, -): readonly PeerMeshAuthorityTarget[] { +): readonly SignedPeerReachabilityLeaseV1[] { const members = new Set(memberPeerIds); return Object.freeze( - reachability - .filter( - (signed) => - signed.lease.peerId !== localPeerId && - members.has(signed.lease.peerId) && - usableHistoricalReachability(signed, now), - ) - .map((signed) => Object.freeze({ reachability: signed })), + reachability.filter( + (signed) => + signed.lease.peerId !== localPeerId && + members.has(signed.lease.peerId) && + usableHistoricalReachability(signed, now), + ), ); } @@ -2331,7 +2384,8 @@ function selectRoster( ): SignedPeerMeshRosterV1 { if ( current.roster.meshId !== candidate.roster.meshId || - current.authorityPublicKey !== candidate.authorityPublicKey + current.authorityPublicKey !== candidate.authorityPublicKey || + current.roster.authorityPeerId !== candidate.roster.authorityPeerId ) { throw new Error('Peer Mesh roster has the wrong authority'); } @@ -2438,7 +2492,7 @@ function reachabilityRevisions( function advertisementRevisions( advertisements: readonly SignedPeerMeshMemberAdvertisementV1[], roster: SignedPeerMeshRosterV1, -): readonly PeerMeshAdvertisementRevision[] { +): readonly PeerMeshEvidenceRevision[] { return Object.freeze( advertisements .filter( @@ -2448,7 +2502,6 @@ function advertisementRevisions( ) .map(({ advertisement }) => Object.freeze({ - meshId: advertisement.meshId, peerId: advertisement.peerId, revision: advertisement.revision, }), @@ -2462,7 +2515,7 @@ function responseEvidence( reachability: readonly SignedPeerReachabilityLeaseV1[], advertisements: readonly SignedPeerMeshMemberAdvertisementV1[], knownReachability: readonly PeerMeshEvidenceRevision[], - knownAdvertisements: readonly PeerMeshAdvertisementRevision[], + knownAdvertisements: readonly PeerMeshEvidenceRevision[], now: number, ): { readonly reachability: readonly SignedPeerReachabilityLeaseV1[]; @@ -2470,9 +2523,7 @@ function responseEvidence( readonly more: boolean; } { const knownLeases = new Map(knownReachability.map(({ peerId, revision }) => [peerId, revision])); - const knownAds = new Map( - knownAdvertisements.map(({ meshId, peerId, revision }) => [`${meshId}\n${peerId}`, revision]), - ); + const knownAds = new Map(knownAdvertisements.map(({ peerId, revision }) => [peerId, revision])); const missing = [ ...reachability .filter( @@ -2491,7 +2542,7 @@ function responseEvidence( ({ advertisement }) => advertisement.meshId === state.roster.roster.meshId && state.roster.roster.members.includes(advertisement.peerId) && - advertisement.revision > (knownAds.get(advertisementKey(advertisement)) ?? 0), + advertisement.revision > (knownAds.get(advertisement.peerId) ?? 0), ) .map((value) => ({ kind: 'advertisement' as const, @@ -2594,9 +2645,9 @@ function hasPeerRecoverySource( (state) => state.role === 'replica' && shared.has(state.roster.roster.meshId) && - sourcePeerIds.has(state.authority.reachability.lease.peerId) && - usableHistoricalReachability(state.authority.reachability, now) && - hasReachabilityRoutes(state.authority.reachability), + sourcePeerIds.has(state.roster.roster.authorityPeerId) && + usableHistoricalReachability(state.authorityReachability, now) && + hasReachabilityRoutes(state.authorityReachability), ); } @@ -2621,9 +2672,8 @@ function peerTarget( peerId: string, reachability: readonly SignedPeerReachabilityLeaseV1[], now: number, -): PeerMeshAuthorityTarget | undefined { - const signed = latestReachability(reachability, peerId, now, true); - return signed ? Object.freeze({ reachability: signed }) : undefined; +): SignedPeerReachabilityLeaseV1 | undefined { + return latestReachability(reachability, peerId, now, true); } function dialTarget(reachability: SignedPeerReachabilityLeaseV1): { @@ -2638,10 +2688,6 @@ function dialTarget(reachability: SignedPeerReachabilityLeaseV1): { }); } -function authorityReachabilityFor(invitation: PeerMeshInvitationV1): SignedPeerReachabilityLeaseV1 { - return invitation.reachability; -} - function findAdvertisement( advertisements: readonly SignedPeerMeshMemberAdvertisementV1[], meshId: string, @@ -2737,7 +2783,7 @@ function decodeControlRequest(value: unknown): PeerMeshControlRequest { reachability: decodeSignedPeerReachabilityLease(record.reachability), advertisement: decodeSignedPeerMeshMemberAdvertisement(record.advertisement), knownReachability: decodeEvidenceRevisions(record.knownReachability), - knownAdvertisements: decodeAdvertisementRevisions(record.knownAdvertisements), + knownAdvertisements: decodeEvidenceRevisions(record.knownAdvertisements), }; } if (record.kind === 'leave' && hasExactKeys(record, ['kind', 'meshId', 'roster'])) { @@ -2869,16 +2915,16 @@ function assertEvidencePageSize( function decodeEvidenceRevisions(value: unknown): readonly PeerMeshEvidenceRevision[] { if (!Array.isArray(value) || value.length > PEER_MESH_MAX_MEMBERS) { - throw new Error('Invalid Peer Mesh reachability revisions'); + throw new Error('Invalid Peer Mesh evidence revisions'); } const revisions = value.map((entry) => { const record = recordValue(entry); if (!hasExactKeys(record, ['peerId', 'revision'])) { - throw new Error('Invalid Peer Mesh reachability revision'); + throw new Error('Invalid Peer Mesh evidence revision'); } const revision = record.revision; if (!Number.isSafeInteger(revision) || (revision as number) < 1) { - throw new Error('Invalid Peer Mesh reachability revision'); + throw new Error('Invalid Peer Mesh evidence revision'); } return Object.freeze({ peerId: requiredString(record.peerId, 256), @@ -2886,33 +2932,7 @@ function decodeEvidenceRevisions(value: unknown): readonly PeerMeshEvidenceRevis }); }); if (new Set(revisions.map(({ peerId }) => peerId)).size !== revisions.length) { - throw new Error('Duplicate Peer Mesh reachability revision'); - } - return Object.freeze(revisions); -} - -function decodeAdvertisementRevisions(value: unknown): readonly PeerMeshAdvertisementRevision[] { - if (!Array.isArray(value) || value.length > PEER_MESH_MAX_MEMBERS) { - throw new Error('Invalid Peer Mesh advertisement revisions'); - } - const revisions = value.map((entry) => { - const record = recordValue(entry); - if (!hasExactKeys(record, ['meshId', 'peerId', 'revision'])) { - throw new Error('Invalid Peer Mesh advertisement revision'); - } - const revision = record.revision; - if (!Number.isSafeInteger(revision) || (revision as number) < 1) { - throw new Error('Invalid Peer Mesh advertisement revision'); - } - return Object.freeze({ - meshId: requiredString(record.meshId, 128), - peerId: requiredString(record.peerId, 256), - revision: revision as number, - }); - }); - const keys = revisions.map(({ meshId, peerId }) => `${meshId}\n${peerId}`); - if (new Set(keys).size !== keys.length) { - throw new Error('Duplicate Peer Mesh advertisement revision'); + throw new Error('Duplicate Peer Mesh evidence revision'); } return Object.freeze(revisions); } diff --git a/packages/runtime-host/src/peer-mesh/store.ts b/packages/runtime-host/src/peer-mesh/store.ts index 3ee65b12c8..55f36836fb 100644 --- a/packages/runtime-host/src/peer-mesh/store.ts +++ b/packages/runtime-host/src/peer-mesh/store.ts @@ -24,7 +24,6 @@ import { type FileLifetimeOwner, } from '@maka/storage/file-lifetime-owner'; import { - decodeAuthorityTarget, decodeSignedPeerMeshMemberAdvertisement, decodeSignedPeerMeshRoster, PEER_MESH_MAX_INVITATION_RECORDS, @@ -32,7 +31,6 @@ import { PEER_MESH_MAX_MESHES, PEER_MESH_MAX_PENDING_INVITATIONS, type PeerMeshAuthorityKeyPair, - type PeerMeshAuthorityTarget, type SignedPeerMeshMemberAdvertisementV1, type SignedPeerMeshRosterV1, validatePeerMeshInvitation, @@ -75,7 +73,7 @@ export interface PeerMeshAuthorityStateV1 extends PeerMeshStateBase { export interface PeerMeshReplicaStateV1 extends PeerMeshStateBase { readonly role: 'replica'; - readonly authority: PeerMeshAuthorityTarget; + readonly authorityReachability: SignedPeerReachabilityLeaseV1; readonly desiredMembership: 'active' | 'left'; } @@ -259,7 +257,7 @@ export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMe const expectedKeys = record.role === 'authority' ? ['role', 'roster', 'authorityPrivateKey', 'invitations'] - : ['role', 'roster', 'authority', 'desiredMembership']; + : ['role', 'roster', 'authorityReachability', 'desiredMembership']; if ( Object.keys(record).length !== expectedKeys.length || expectedKeys.some((key) => !Object.hasOwn(record, key)) @@ -271,8 +269,8 @@ export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMe } const roster = decodeSignedPeerMeshRoster(record.roster); if (record.role === 'authority') { - if (!roster.roster.members.includes(localPeerId)) { - throw new Error('Peer Mesh authority is not present in its roster'); + if (roster.roster.authorityPeerId !== localPeerId) { + throw new Error('Peer Mesh authority identity does not match its roster'); } const privateKey = boundedString(record.authorityPrivateKey, 'authorityPrivateKey', 256); validatePeerMeshAuthorityKeyPair({ @@ -286,13 +284,13 @@ export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMe invitations: Object.freeze(decodeInvitations(record.invitations)), }); } - const authority = decodeAuthorityTarget(record.authority); - if (!roster.roster.members.includes(authority.reachability.lease.peerId)) { - throw new Error('Peer Mesh authority is not present in its roster'); + const authorityReachability = decodeSignedPeerReachabilityLease(record.authorityReachability); + if (authorityReachability.lease.peerId !== roster.roster.authorityPeerId) { + throw new Error('Peer Mesh authority reachability has the wrong identity'); } return Object.freeze({ role: 'replica', - authority, + authorityReachability, roster, desiredMembership: decodeDesiredMembership(record.desiredMembership), }); diff --git a/packages/runtime-host/src/protocol/peer-mesh.ts b/packages/runtime-host/src/protocol/peer-mesh.ts index 523c147044..f5623f8d8a 100644 --- a/packages/runtime-host/src/protocol/peer-mesh.ts +++ b/packages/runtime-host/src/protocol/peer-mesh.ts @@ -30,7 +30,7 @@ import { PEER_MESH_MAX_MEMBERS, PEER_MESH_MAX_MESHES } from '../peer-mesh/limits import { decodeSignedPeerReachabilityLease, type SignedPeerReachabilityLeaseV1, -} from '../peer-reachability/index.js'; +} from '../peer-reachability/model.js'; const PEER_ID_MAX_BYTES = 256; const MESH_ID_MAX_BYTES = 128; @@ -316,7 +316,9 @@ export function decodePeerMeshQueryResult(value: unknown): PeerMeshQueryResult { localPeerId: requireString(localPeerId, 'Peer Mesh localPeerId', PEER_ID_MAX_BYTES), ...(record.localDisplayName === undefined ? {} - : { localDisplayName: canonicalPeerMeshDisplayName(record.localDisplayName) }), + : { + localDisplayName: canonicalPeerMeshDisplayName(record.localDisplayName), + }), transit: decodePeerMeshTransitProjection(record.transit), }), meshes: Object.freeze(record.meshes.map(decodePeerMeshProjection)), @@ -433,7 +435,9 @@ function decodePeerMeshMemberProjection(value: unknown): PeerMeshMemberProjectio state: record.state, ...(record.expiresAt === undefined ? {} - : { expiresAt: requireCount(record.expiresAt, 'Peer Mesh member route expiry') }), + : { + expiresAt: requireCount(record.expiresAt, 'Peer Mesh member route expiry'), + }), }; } From 56f6eb5b83bc529fd08972b64b540ed938107b30 Mon Sep 17 00:00:00 2001 From: Wang Date: Thu, 3 Sep 2026 02:33:49 +0800 Subject: [PATCH 04/13] refactor(peer): keep one reachability authority Generated-by: Codex (gpt-5.6-sol) --- packages/runtime-host/src/peer-mesh/node.ts | 45 ++------------------ packages/runtime-host/src/peer-mesh/store.ts | 8 +--- 2 files changed, 4 insertions(+), 49 deletions(-) diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index 250270a2ad..daed808618 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -302,17 +302,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { this.#recordReachabilityReceipt(signed); } } - for (const state of stored.meshes) { - if (state.role !== 'replica') continue; - const signed = this.#reachability.verify( - state.authorityReachability, - state.roster.roster.authorityPeerId, - { allowExpired: true }, - ); - if (usableHistoricalReachability(signed, this.#now())) { - this.#recordReachabilityReceipt(signed); - } - } for (const advertisement of stored.advertisements) { this.#assertAdvertisementSignature(advertisement); } @@ -737,13 +726,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { } const state: PeerMeshReplicaStateV1 = { role: 'replica', - authorityReachability: - selectReachability( - reachability.find( - ({ lease }) => lease.peerId === selectedRoster.roster.authorityPeerId, - ), - authorityReachability, - ) ?? authorityReachability, roster: selectedRoster, desiredMembership: pending.phase === 'leave_pending' ? 'left' : 'active', }; @@ -756,7 +738,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { pendingJoins: current.pendingJoins.filter((candidate) => candidate !== pending), reachability: mergeReachability( current.reachability, - [...reachability, localReachability], + [...reachability, authorityReachability, localReachability], this.#now(), ), advertisements: mergeAdvertisements(current.advertisements, [ @@ -2284,12 +2266,7 @@ function currentAuthorityTarget( reachability: readonly SignedPeerReachabilityLeaseV1[], now: number, ): SignedPeerReachabilityLeaseV1 | undefined { - const authorityPeerId = state.roster.roster.authorityPeerId; - const learned = latestReachability(reachability, authorityPeerId, now, true); - const fallback = usableHistoricalReachability(state.authorityReachability, now) - ? state.authorityReachability - : undefined; - return selectReachability(learned, fallback); + return latestReachability(reachability, state.roster.roster.authorityPeerId, now, true); } function rosterAnnouncementTargets( @@ -2641,14 +2618,7 @@ function hasPeerRecoverySource( const signed = latestReachability(stored.reachability, sourcePeerId, now, true); if (signed && hasReachabilityRoutes(signed)) return true; } - return stored.meshes.some( - (state) => - state.role === 'replica' && - shared.has(state.roster.roster.meshId) && - sourcePeerIds.has(state.roster.roster.authorityPeerId) && - usableHistoricalReachability(state.authorityReachability, now) && - hasReachabilityRoutes(state.authorityReachability), - ); + return false; } function hasReachabilityRoutes(signed: SignedPeerReachabilityLeaseV1): boolean { @@ -2659,15 +2629,6 @@ function usableHistoricalReachability(signed: SignedPeerReachabilityLeaseV1, now return signed.lease.expiresAt + REACHABILITY_HISTORY_MS > now; } -function selectReachability( - first: SignedPeerReachabilityLeaseV1 | undefined, - second: SignedPeerReachabilityLeaseV1 | undefined, -): SignedPeerReachabilityLeaseV1 | undefined { - if (!first) return second; - if (!second || first.lease.revision >= second.lease.revision) return first; - return second; -} - function peerTarget( peerId: string, reachability: readonly SignedPeerReachabilityLeaseV1[], diff --git a/packages/runtime-host/src/peer-mesh/store.ts b/packages/runtime-host/src/peer-mesh/store.ts index 55f36836fb..8c3a1a1141 100644 --- a/packages/runtime-host/src/peer-mesh/store.ts +++ b/packages/runtime-host/src/peer-mesh/store.ts @@ -73,7 +73,6 @@ export interface PeerMeshAuthorityStateV1 extends PeerMeshStateBase { export interface PeerMeshReplicaStateV1 extends PeerMeshStateBase { readonly role: 'replica'; - readonly authorityReachability: SignedPeerReachabilityLeaseV1; readonly desiredMembership: 'active' | 'left'; } @@ -257,7 +256,7 @@ export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMe const expectedKeys = record.role === 'authority' ? ['role', 'roster', 'authorityPrivateKey', 'invitations'] - : ['role', 'roster', 'authorityReachability', 'desiredMembership']; + : ['role', 'roster', 'desiredMembership']; if ( Object.keys(record).length !== expectedKeys.length || expectedKeys.some((key) => !Object.hasOwn(record, key)) @@ -284,13 +283,8 @@ export function decodePeerMeshState(value: unknown, localPeerId: string): PeerMe invitations: Object.freeze(decodeInvitations(record.invitations)), }); } - const authorityReachability = decodeSignedPeerReachabilityLease(record.authorityReachability); - if (authorityReachability.lease.peerId !== roster.roster.authorityPeerId) { - throw new Error('Peer Mesh authority reachability has the wrong identity'); - } return Object.freeze({ role: 'replica', - authorityReachability, roster, desiredMembership: decodeDesiredMembership(record.desiredMembership), }); From 5f4b1237aaf22895ab975d64f927057dc53c6e3e Mon Sep 17 00:00:00 2001 From: Wang Date: Thu, 3 Sep 2026 03:33:23 +0800 Subject: [PATCH 05/13] fix(peer): preserve converged mesh exports Generated-by: Codex (gpt-5.6-sol) --- packages/runtime-host/src/peer-mesh/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/runtime-host/src/peer-mesh/index.ts b/packages/runtime-host/src/peer-mesh/index.ts index 8e1105a618..403502ec76 100644 --- a/packages/runtime-host/src/peer-mesh/index.ts +++ b/packages/runtime-host/src/peer-mesh/index.ts @@ -18,7 +18,6 @@ */ export { - type PeerMeshAuthorityTarget, type PeerMeshRosterV1, type SignedPeerMeshRosterV1, } from './model.js'; From c37206b77992b19dac848e6115573cc61710deb0 Mon Sep 17 00:00:00 2001 From: Wang Date: Thu, 3 Sep 2026 04:38:01 +0800 Subject: [PATCH 06/13] fix(peer): adopt reachability after Mesh authorization Generated-by: Codex (gpt-5.6-sol) --- packages/runtime-host/src/peer-mesh/node.ts | 98 +++++++++++++++------ 1 file changed, 70 insertions(+), 28 deletions(-) diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index daed808618..6f5bcdfd5e 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -533,7 +533,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { join(invitationValue: PeerMeshInvitationV1, signal?: AbortSignal): Promise { return this.#admitMesh(async () => { const invitation = validatePeerMeshInvitation(invitationValue); - const authorityReachability = this.#validateReachability( + const authorityReachability = this.#authenticateReachability( invitation.reachability, invitation.reachability.lease.peerId, true, @@ -649,7 +649,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { signal: AbortSignal, ): Promise { signal.throwIfAborted(); - const authorityReachability = this.#validateReachability( + const authorityReachability = this.#authenticateReachability( invitation.reachability, invitation.reachability.lease.peerId, true, @@ -749,6 +749,11 @@ class PeerMeshNodeImpl implements PeerMeshNode { result: undefined, }; }); + this.#recordReachabilityReceipts([ + ...reachability, + authorityReachability, + localReachability, + ]); signal.throwIfAborted(); await this.#refreshLocalEvidence(); signal.throwIfAborted(); @@ -807,7 +812,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { } async #resumePendingJoin(pending: PendingPeerMeshJoin, signal: AbortSignal): Promise { - const authorityReachability = this.#validateReachability( + const authorityReachability = this.#authenticateReachability( pending.invitation.reachability, pending.invitation.reachability.lease.peerId, true, @@ -1407,7 +1412,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { if (!roster.roster.members.includes(signed.lease.peerId)) { throw new Error('Peer Mesh reachability is outside the active roster'); } - return this.#validateReachability(signed, signed.lease.peerId, allowExpired); + return this.#authenticateReachability(signed, signed.lease.peerId, allowExpired); }); if (new Set(reachability.map(({ lease }) => lease.peerId)).size !== reachability.length) { throw new Error('Duplicate Peer Mesh reachability leases'); @@ -1415,7 +1420,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { return Object.freeze(reachability); } - #validateReachability( + #authenticateReachability( value: SignedPeerReachabilityLeaseV1, expectedPeerId: string, allowExpired = false, @@ -1426,7 +1431,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { if (allowExpired && signed.lease.expiresAt + REACHABILITY_HISTORY_MS <= this.#now()) { throw new Error('Peer Mesh reachability is outside the recovery horizon'); } - this.#recordReachabilityReceipt(signed); return signed; } @@ -1482,24 +1486,49 @@ class PeerMeshNodeImpl implements PeerMeshNode { } #recordReachabilityReceipt(signed: SignedPeerReachabilityLeaseV1): void { - const previous = this.#reachabilityReceipts.get(signed.lease.peerId); - if (previous && previous.revision > signed.lease.revision) return; - if ( - previous && - previous.revision === signed.lease.revision && - previous.signature !== signed.signature - ) { - return; + this.#recordReachabilityReceipts([signed]); + } + + #recordReachabilityReceipts(values: readonly SignedPeerReachabilityLeaseV1[]): void { + const retained = this.#retainedReachabilityReceiptPeerIds(); + for (const peerId of this.#reachabilityReceipts.keys()) { + if (!retained.has(peerId)) this.#reachabilityReceipts.delete(peerId); + } + for (const signed of values) { + if (!retained.has(signed.lease.peerId)) continue; + const previous = this.#reachabilityReceipts.get(signed.lease.peerId); + if (previous && previous.revision > signed.lease.revision) continue; + if ( + previous && + previous.revision === signed.lease.revision && + previous.signature !== signed.signature + ) { + continue; + } + this.#reachabilityReceipts.set( + signed.lease.peerId, + peerReachabilityLeaseReceipt({ + signed, + wallNow: this.#now(), + monotonicNow: this.#monotonicNow(), + ...(previous ? { previous } : {}), + }), + ); } - this.#reachabilityReceipts.set( - signed.lease.peerId, - peerReachabilityLeaseReceipt({ - signed, - wallNow: this.#now(), - monotonicNow: this.#monotonicNow(), - ...(previous ? { previous } : {}), - }), - ); + } + + #retainedReachabilityReceiptPeerIds(): ReadonlySet { + const stored = this.#store.read(); + const localPeerId = this.#peer.identity().peerId; + const retained = new Set([localPeerId]); + for (const state of stored.meshes) { + if (!isActiveMembership(state, localPeerId)) continue; + for (const peerId of state.roster.roster.members) retained.add(peerId); + } + for (const pending of stored.pendingJoins) { + retained.add(pending.invitation.reachability.lease.peerId); + } + return retained; } #isReachabilityCurrent(signed: SignedPeerReachabilityLeaseV1): boolean { @@ -1521,7 +1550,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { const reachability = this.#validateReachabilityPage(reachabilityValues, roster, true); const advertisements = this.#validateAdvertisementPage(advertisementValues, roster); const localPeerId = this.#peer.identity().peerId; - await this.#store.mutate((current) => { + const accepted = await this.#store.mutate((current) => { const state = findMesh(current.meshes, meshId); if (!state || state.roster.authorityPublicKey !== roster.authorityPublicKey) { throw new Error('Peer Mesh synchronization has the wrong authority'); @@ -1542,9 +1571,10 @@ class PeerMeshNodeImpl implements PeerMeshNode { ? current.advertisements : mergeAdvertisements(current.advertisements, advertisements), }, - result: undefined, + result: isActiveMembership(next, localPeerId), }; }); + if (accepted) this.#recordReachabilityReceipts(reachability); await this.#refreshLocalEvidence(); await this.#reconcileTransit(); } @@ -1721,7 +1751,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { response = await this.#redeem( request, stream.peerId, - this.#validateReachability(request.reachability, stream.peerId), + this.#authenticateReachability(request.reachability, stream.peerId), this.#validateAdvertisementForPeer(request.advertisement, request.meshId, stream.peerId), ); } else if (request.kind === 'sync') { @@ -1903,6 +1933,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { }; }); if (response.kind === 'invitation-redeemed') { + this.#recordReachabilityReceipt(remoteReachability); const stored = this.#store.read(); const state = findMesh(stored.meshes, request.meshId); if (state) { @@ -1983,7 +2014,10 @@ class PeerMeshNodeImpl implements PeerMeshNode { } async #sync(request: SyncPeerMeshRequest, remotePeerId: string): Promise { - const remoteReachability = this.#validateReachability(request.reachability, remotePeerId); + const remoteReachability = this.#authenticateReachability( + request.reachability, + remotePeerId, + ); const remoteAdvertisement = this.#validateAdvertisementForPeer( request.advertisement, request.meshId, @@ -2055,7 +2089,15 @@ class PeerMeshNodeImpl implements PeerMeshNode { }; }); if (response.kind === 'sync-result') { - this.#recentlyReached.add(remotePeerId); + const localPeerId = this.#peer.identity().peerId; + if ( + !response.roster.roster.closed && + response.roster.roster.members.includes(localPeerId) && + response.roster.roster.members.includes(remotePeerId) + ) { + this.#recordReachabilityReceipt(remoteReachability); + this.#recentlyReached.add(remotePeerId); + } await this.#reconcileTransit(); } return response; From bc56bb28430cc704140afbb9a9380a138fc91b6b Mon Sep 17 00:00:00 2001 From: Wang Date: Thu, 3 Sep 2026 04:46:34 +0800 Subject: [PATCH 07/13] style(peer): format Mesh reachability flow Generated-by: Codex (gpt-5.6-sol) --- packages/runtime-host/src/peer-mesh/node.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index 6f5bcdfd5e..9c9208e1b4 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -749,11 +749,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { result: undefined, }; }); - this.#recordReachabilityReceipts([ - ...reachability, - authorityReachability, - localReachability, - ]); + this.#recordReachabilityReceipts([...reachability, authorityReachability, localReachability]); signal.throwIfAborted(); await this.#refreshLocalEvidence(); signal.throwIfAborted(); @@ -2014,10 +2010,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { } async #sync(request: SyncPeerMeshRequest, remotePeerId: string): Promise { - const remoteReachability = this.#authenticateReachability( - request.reachability, - remotePeerId, - ); + const remoteReachability = this.#authenticateReachability(request.reachability, remotePeerId); const remoteAdvertisement = this.#validateAdvertisementForPeer( request.advertisement, request.meshId, From d8ab2c11b9638b12aabc3a634e720820c8cfe758 Mon Sep 17 00:00:00 2001 From: Wang Date: Thu, 3 Sep 2026 05:00:47 +0800 Subject: [PATCH 08/13] refactor(peer): narrow Mesh reachability receipts Generated-by: Codex (gpt-5.6-sol) --- packages/runtime-host/src/peer-mesh/node.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index 9c9208e1b4..9870ebf882 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -1322,7 +1322,6 @@ class PeerMeshNodeImpl implements PeerMeshNode { async #refreshLocalEvidenceOnce(): Promise { const reachability = await this.#reachability.refresh(); - this.#recordReachabilityReceipt(reachability); const identity = this.#peer.identity(); const now = this.#now(); return this.#store.mutate(async (current) => { @@ -1516,13 +1515,12 @@ class PeerMeshNodeImpl implements PeerMeshNode { #retainedReachabilityReceiptPeerIds(): ReadonlySet { const stored = this.#store.read(); const localPeerId = this.#peer.identity().peerId; - const retained = new Set([localPeerId]); + const retained = new Set(); for (const state of stored.meshes) { if (!isActiveMembership(state, localPeerId)) continue; - for (const peerId of state.roster.roster.members) retained.add(peerId); - } - for (const pending of stored.pendingJoins) { - retained.add(pending.invitation.reachability.lease.peerId); + for (const peerId of state.roster.roster.members) { + if (peerId !== localPeerId) retained.add(peerId); + } } return retained; } From 876c69d6325b8b1434ca7eedbb2581dd95124550 Mon Sep 17 00:00:00 2001 From: Wang Date: Thu, 3 Sep 2026 05:06:07 +0800 Subject: [PATCH 09/13] fix(peer): bound Mesh recovery sweep state Generated-by: Codex (gpt-5.6-sol) --- packages/runtime-host/src/peer-mesh/node.ts | 32 +++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index 9870ebf882..6de4c4df3c 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -941,6 +941,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { const now = this.#now(); const stored = this.#store.read(); const localPeerId = this.#peer.identity().peerId; + this.#pruneCompletedRecoverySweeps(stored, localPeerId); const sharedMeshIds = stored.meshes .filter( (state) => @@ -982,21 +983,34 @@ class PeerMeshNodeImpl implements PeerMeshNode { this.#completedRecoverySweeps.delete(peerId); const localPeerId = this.#peer.identity().peerId; const stored = this.#store.read(); - const visible = stored.meshes.some( - (mesh) => - isActiveMembership(mesh, localPeerId) && mesh.roster.roster.members.includes(peerId), - ); - if (!visible) { - this.#completedRecoverySweeps.add(peerId); - return; - } + const visible = this.#pruneCompletedRecoverySweeps(stored, localPeerId).has(peerId); + if (!visible) return; // A signed route can remain within its TTL after a peer restarted or // rotated Relay reservations. Every connection establishment therefore // asks the Mesh control plane for its newest record. Callers with a // self-contained invitation run this reconciliation in parallel with the // first dial; callers without usable routes wait for it. await this.reconcile(signal); - this.#completedRecoverySweeps.add(peerId); + if (this.#pruneCompletedRecoverySweeps().has(peerId)) { + this.#completedRecoverySweeps.add(peerId); + } + } + + #pruneCompletedRecoverySweeps( + stored = this.#store.read(), + localPeerId = this.#peer.identity().peerId, + ): ReadonlySet { + const visiblePeerIds = new Set(); + for (const state of stored.meshes) { + if (!isActiveMembership(state, localPeerId)) continue; + for (const memberPeerId of state.roster.roster.members) { + if (memberPeerId !== localPeerId) visiblePeerIds.add(memberPeerId); + } + } + for (const peerId of this.#completedRecoverySweeps) { + if (!visiblePeerIds.has(peerId)) this.#completedRecoverySweeps.delete(peerId); + } + return visiblePeerIds; } reconcile(signal?: AbortSignal): Promise { From c6956bb0436ee0daf0b7e8dd1cae8eaae577e1dd Mon Sep 17 00:00:00 2001 From: Wang Date: Thu, 3 Sep 2026 05:37:21 +0800 Subject: [PATCH 10/13] fix(peer-mesh): revalidate synchronization targets Generated-by: Codex (gpt-5.6-sol) --- .../src/__tests__/peer-mesh.test.ts | 64 +++++++++++++++++-- packages/runtime-host/src/peer-mesh/node.ts | 28 ++++---- 2 files changed, 77 insertions(+), 15 deletions(-) diff --git a/packages/runtime-host/src/__tests__/peer-mesh.test.ts b/packages/runtime-host/src/__tests__/peer-mesh.test.ts index 5bdf363e9b..e7b4b4493f 100644 --- a/packages/runtime-host/src/__tests__/peer-mesh.test.ts +++ b/packages/runtime-host/src/__tests__/peer-mesh.test.ts @@ -174,6 +174,42 @@ test('authenticates three peers, consumes invitations once, and keeps authority } }); +test('does not synchronize reachability to a peer removed after reconciliation selected it', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-peer-mesh-revoked-sync-')); + const network = new MemoryPeerNetwork(); + const authorityPeer = network.create('peer-a'); + const memberPeer = network.create('peer-b'); + const authority = await openPeerMeshNode({ + dataRoot: join(root, 'authority'), + peer: authorityPeer, + }); + const member = await openPeerMeshNode({ + dataRoot: join(root, 'member'), + peer: memberPeer, + }); + const serving = [authority.serve(), member.serve()]; + try { + const mesh = await authority.create(); + await member.join(await authority.invite(mesh.roster.roster.meshId)); + await authority.reconcile(); + const synchronizedBeforeRemoval = memberPeer.receivedControlCount('sync'); + + const connection = authorityPeer.stallNextConnection(); + const reconciliation = authority.reconcile(); + await connection.started; + await authority.remove(mesh.roster.roster.meshId, 'peer-b'); + connection.release(); + await reconciliation; + + assert.equal(memberPeer.receivedControlCount('sync'), synchronizedBeforeRemoval); + } finally { + await Promise.allSettled([authority.close(), member.close()]); + await Promise.allSettled(serving); + await Promise.allSettled([authorityPeer.close(), memberPeer.close()]); + await rm(root, { recursive: true, force: true }); + } +}); + test('commits an offline leave locally and reconciles it after restart', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-peer-mesh-leave-')); const network = new MemoryPeerNetwork(); @@ -1187,6 +1223,7 @@ class MemoryPeerClient implements PeerMeshTransport, PeerReachabilityPublisher { } | undefined; #failNextSignature = false; + readonly #receivedControlKinds: string[] = []; constructor( private readonly peerId: string, @@ -1318,6 +1355,10 @@ class MemoryPeerClient implements PeerMeshTransport, PeerReachabilityPublisher { return { started, release }; } + receivedControlCount(kind: string): number { + return this.#receivedControlKinds.filter((candidate) => candidate === kind).length; + } + failNextSignature(): void { this.#failNextSignature = true; } @@ -1401,7 +1442,14 @@ class MemoryPeerClient implements PeerMeshTransport, PeerReachabilityPublisher { if (!remote || !remote.#reachable) { throw new Error('Peer is unavailable'); } - const [localStream, remoteStream] = memoryStreamPair(this.peerId, input.peerId); + const [localStream, remoteStream] = memoryStreamPair(this.peerId, input.peerId, (bytes) => { + const newline = bytes.indexOf(0x0a); + if (newline < 0) return; + const value = JSON.parse(bytes.subarray(0, newline).toString('utf8')) as { + readonly kind?: unknown; + }; + if (typeof value.kind === 'string') remote.#receivedControlKinds.push(value.kind); + }); if (remote.#failNextResponse) { remote.#failNextResponse = false; remoteStream.failNextWrite(); @@ -1500,9 +1548,13 @@ function memorySignature(peerId: string, payload: Buffer): Buffer { return createHash('sha256').update(peerId).update(payload).digest(); } -function memoryStreamPair(localPeerId: string, remotePeerId: string): [MemoryStream, MemoryStream] { +function memoryStreamPair( + localPeerId: string, + remotePeerId: string, + observeRemote?: (bytes: Buffer) => void, +): [MemoryStream, MemoryStream] { const local = new MemoryStream(remotePeerId); - const remote = new MemoryStream(localPeerId); + const remote = new MemoryStream(localPeerId, observeRemote); local.connect(remote); remote.connect(local); return [local, remote]; @@ -1515,7 +1567,10 @@ class MemoryStream implements RuntimeHostPeerNativeStream { #closed = false; #failNextWrite = false; - constructor(readonly peerId: string) {} + constructor( + readonly peerId: string, + private readonly observe?: (bytes: Buffer) => void, + ) {} connect(remote: MemoryStream): void { this.#remote = remote; @@ -1554,6 +1609,7 @@ class MemoryStream implements RuntimeHostPeerNativeStream { } push(chunk: Buffer | null): void { + if (chunk) this.observe?.(chunk); const waiter = this.#waiters.shift(); if (waiter) waiter(chunk); else this.#incoming.push(chunk); diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index 6de4c4df3c..a495220f83 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -1263,18 +1263,10 @@ class PeerMeshNodeImpl implements PeerMeshNode { target: SignedPeerReachabilityLeaseV1, signal: AbortSignal, ): Promise { + const localPeerId = this.#peer.identity().peerId; + const targetPeerId = target.lease.peerId; for (let page = 0; page <= PEER_MESH_MAX_MEMBERS; page += 1) { - await this.#refreshLocalEvidence(); - const stored = this.#store.read(); - const state = findMesh(stored.meshes, meshId); - const localPeerId = this.#peer.identity().peerId; - if (!state || !isActiveMembership(state, localPeerId)) return; - const reachability = latestReachability(stored.reachability, localPeerId, this.#now(), true); - const advertisement = findAdvertisement(stored.advertisements, meshId, localPeerId); - if (!reachability || !advertisement) { - throw new Error('Peer Mesh local evidence is unavailable'); - } - const targetPeerId = target.lease.peerId; + if (!isActiveMeshMember(this.#store.read().meshes, meshId, localPeerId, targetPeerId)) return; const discovered = this.resolveRoutes(targetPeerId); const targetRoutes = dialTarget(target); const stream = await this.#peer.connectMeshControl( @@ -1291,6 +1283,20 @@ class PeerMeshNodeImpl implements PeerMeshNode { signal, ); try { + await this.#refreshLocalEvidence(); + const stored = this.#store.read(); + if (!isActiveMeshMember(stored.meshes, meshId, localPeerId, targetPeerId)) return; + const state = findMesh(stored.meshes, meshId)!; + const reachability = latestReachability( + stored.reachability, + localPeerId, + this.#now(), + true, + ); + const advertisement = findAdvertisement(stored.advertisements, meshId, localPeerId); + if (!reachability || !advertisement) { + throw new Error('Peer Mesh local evidence is unavailable'); + } const response = await exchangeControl( stream, { From c9403057c4206d9ceaabfd16f5ab9cc1378b78d5 Mon Sep 17 00:00:00 2001 From: Wang Date: Thu, 3 Sep 2026 06:13:09 +0800 Subject: [PATCH 11/13] fix(peer-mesh): preserve authenticated recovery evidence Generated-by: Codex (gpt-5.6-sol) --- .../src/__tests__/peer-mesh.test.ts | 106 ++++++++++++++++++ packages/runtime-host/src/peer-mesh/node.ts | 48 +++----- .../src/peer-reachability/index.ts | 1 + 3 files changed, 123 insertions(+), 32 deletions(-) diff --git a/packages/runtime-host/src/__tests__/peer-mesh.test.ts b/packages/runtime-host/src/__tests__/peer-mesh.test.ts index e7b4b4493f..c3f982ea75 100644 --- a/packages/runtime-host/src/__tests__/peer-mesh.test.ts +++ b/packages/runtime-host/src/__tests__/peer-mesh.test.ts @@ -30,6 +30,7 @@ import { generatePeerMeshAuthorityKeyPair, peerMeshId, signPeerMeshRoster, + type SignedPeerMeshMemberAdvertisementV1, } from '../peer-mesh/model.js'; import { openPeerMeshNode as openPeerMeshNodeImpl, @@ -40,6 +41,7 @@ import { canonicalPeerReachabilityLease, decodeSignedPeerReachabilityLease, PEER_REACHABILITY_LEASE_TTL_MS, + PEER_REACHABILITY_MAX_CLOCK_SKEW_MS, PEER_REACHABILITY_REFRESH_LEAD_MS, peerReachabilityLeaseSigningBytes, samePeerReachabilityRoutes, @@ -554,6 +556,109 @@ test('keeps Mesh membership while pruning reachability beyond its recovery horiz } }); +test('recovers persisted Mesh reachability after the wall clock moves backward', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-peer-mesh-clock-rollback-')); + const network = new MemoryPeerNetwork(); + const authorityPeer = network.create('peer-a'); + const memberPeer = network.create('peer-b'); + let now = Date.now(); + const authority = await openPeerMeshNode({ + dataRoot: join(root, 'authority'), + peer: authorityPeer, + now: () => now, + }); + const memberRoot = join(root, 'member'); + let member = await openPeerMeshNode({ + dataRoot: memberRoot, + peer: memberPeer, + now: () => now, + }); + const serving = authority.serve(); + try { + const meshId = (await authority.create()).roster.roster.meshId; + await member.join(await authority.invite(meshId)); + const previous = memberPeer.current(); + await member.close(); + + now -= PEER_REACHABILITY_MAX_CLOCK_SKEW_MS + 1; + member = await openPeerMeshNode({ + dataRoot: memberRoot, + peer: memberPeer, + now: () => now, + }); + + assert.equal(member.status()[0]?.roster.roster.meshId, meshId); + assert.equal(memberPeer.current().lease.revision, previous.lease.revision + 1); + assert.equal(memberPeer.current().lease.issuedAt, now); + } finally { + await Promise.allSettled([authority.close(), member.close()]); + await Promise.allSettled([authorityPeer.close(), memberPeer.close(), serving]); + await rm(root, { recursive: true, force: true }); + } +}); + +test('rejects conflicting reachability facts at the same revision during Mesh sync', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-peer-mesh-reachability-equivocation-')); + const network = new MemoryPeerNetwork(); + const authorityPeer = network.create('peer-a'); + const memberPeer = network.create('peer-b'); + const authority = await openPeerMeshNode({ + dataRoot: join(root, 'authority'), + peer: authorityPeer, + }); + const memberRoot = join(root, 'member'); + const member = await openPeerMeshNode({ dataRoot: memberRoot, peer: memberPeer }); + const serving = authority.serve(); + try { + const meshId = (await authority.create()).roster.roster.meshId; + await member.join(await authority.invite(meshId)); + const originalRoutes = authority.resolveRoutes('peer-b'); + const original = memberPeer.current(); + const conflictingLease = canonicalPeerReachabilityLease({ + ...original.lease, + directRoutes: ['/memory/conflicting/p2p/peer-b'], + }); + const proof = await memberPeer.signIdentity( + peerReachabilityLeaseSigningBytes(conflictingLease), + ); + const conflicting = decodeSignedPeerReachabilityLease({ + lease: conflictingLease, + publicKey: proof.publicKey.toString('base64url'), + signature: proof.signature.toString('base64url'), + }); + const persisted = JSON.parse(await readFile(join(memberRoot, 'peer-mesh.json'), 'utf8')) as { + readonly advertisements: readonly SignedPeerMeshMemberAdvertisementV1[]; + }; + const advertisement = persisted.advertisements.find( + ({ advertisement: candidate }) => + candidate.meshId === meshId && candidate.peerId === 'peer-b', + ); + assert.ok(advertisement); + + const stream = await memberPeer.connectMeshControl({ peerId: 'peer-a' }); + await stream.write( + Buffer.from( + `${JSON.stringify({ + kind: 'sync', + meshId, + roster: authority.status()[0]?.roster, + reachability: conflicting, + advertisement, + knownReachability: [], + knownAdvertisements: [], + })}\n`, + ), + ); + + assert.equal(await stream.read(), null); + assert.deepEqual(authority.resolveRoutes('peer-b'), originalRoutes); + } finally { + await Promise.allSettled([authority.close(), member.close()]); + await Promise.allSettled([authorityPeer.close(), memberPeer.close(), serving]); + await rm(root, { recursive: true, force: true }); + } +}); + test('publishes a changed local route promptly and refreshes a live cached peer route', { timeout: 10_000, }, async () => { @@ -1255,6 +1360,7 @@ class MemoryPeerClient implements PeerMeshTransport, PeerReachabilityPublisher { const now = this.#now(); if ( this.#reachability && + this.#reachability.lease.issuedAt <= now && this.#reachability.lease.expiresAt > now + PEER_REACHABILITY_REFRESH_LEAD_MS && samePeerReachabilityRoutes(this.#reachability.lease, identity) ) { diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index a495220f83..0d483432f7 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -51,8 +51,10 @@ import { import { canonicalPeerMeshDisplayName } from './display-name.js'; import type { PeerMeshInvitationV1 } from '../protocol/peer-mesh.js'; import { + authenticateSignedPeerReachabilityLease, decodeSignedPeerReachabilityLease, isPeerReachabilityLeaseCurrent, + PEER_REACHABILITY_MAX_CLOCK_SKEW_MS, peerReachabilityLeaseReceipt, type PeerReachabilityPublisher, type PeerReachabilityLeaseReceipt, @@ -294,23 +296,29 @@ class PeerMeshNodeImpl implements PeerMeshNode { async initialize(): Promise { const stored = this.#store.read(); + const now = this.#now(); for (const lease of stored.reachability) { - const signed = this.#reachability.verify(lease, lease.lease.peerId, { - allowExpired: true, + const signed = authenticateSignedPeerReachabilityLease({ + value: lease, + expectedPeerId: lease.lease.peerId, + verifyIdentity: this.#peer.verifyIdentity.bind(this.#peer), }); - if (usableHistoricalReachability(signed, this.#now())) { + if ( + usableHistoricalReachability(signed, now) && + signed.lease.issuedAt <= now + PEER_REACHABILITY_MAX_CLOCK_SKEW_MS + ) { this.#recordReachabilityReceipt(signed); } } for (const advertisement of stored.advertisements) { this.#assertAdvertisementSignature(advertisement); } - if (stored.reachability.some((signed) => !usableHistoricalReachability(signed, this.#now()))) { + if (stored.reachability.some((signed) => !usableHistoricalReachability(signed, now))) { await this.#store.mutate((current) => ({ state: { ...current, reachability: current.reachability.filter((signed) => - usableHistoricalReachability(signed, this.#now()), + usableHistoricalReachability(signed, now), ), }, result: undefined, @@ -1833,11 +1841,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { ) { return { state: current, result: rejected('invalid') }; } - const reachability = mergeAuthenticatedReachability( - current.reachability, - remoteReachability, - now, - ); + const reachability = mergeReachability(current.reachability, [remoteReachability], now); const advertisements = mergeAdvertisements(current.advertisements, [remoteAdvertisement]); const evidence = initialEvidence( state, @@ -1919,11 +1923,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { redeemedInvitation(invitation, remotePeerId), ], }; - const reachability = mergeAuthenticatedReachability( - current.reachability, - remoteReachability, - now, - ); + const reachability = mergeReachability(current.reachability, [remoteReachability], now); const advertisements = mergeAdvertisements(current.advertisements, [remoteAdvertisement]); const evidence = initialEvidence( updated, @@ -2054,7 +2054,7 @@ class PeerMeshNodeImpl implements PeerMeshNode { const remoteMember = !roster.roster.closed && roster.roster.members.includes(remotePeerId); const reachability = localMember && remoteMember - ? mergeAuthenticatedReachability(current.reachability, remoteReachability, this.#now()) + ? mergeReachability(current.reachability, [remoteReachability], this.#now()) : current.reachability; const advertisements = localMember && remoteMember @@ -2458,22 +2458,6 @@ function mergeReachability( ); } -function mergeAuthenticatedReachability( - current: readonly SignedPeerReachabilityLeaseV1[], - candidate: SignedPeerReachabilityLeaseV1, - now: number, -): readonly SignedPeerReachabilityLeaseV1[] { - const existing = current.find(({ lease }) => lease.peerId === candidate.lease.peerId); - if (existing && existing.lease.revision > candidate.lease.revision) { - return current; - } - return mergeReachability( - current.filter(({ lease }) => lease.peerId !== candidate.lease.peerId), - [candidate], - now, - ); -} - function mergeAdvertisements( current: readonly SignedPeerMeshMemberAdvertisementV1[], candidates: readonly SignedPeerMeshMemberAdvertisementV1[], diff --git a/packages/runtime-host/src/peer-reachability/index.ts b/packages/runtime-host/src/peer-reachability/index.ts index cb297f37e2..20b7fb98c0 100644 --- a/packages/runtime-host/src/peer-reachability/index.ts +++ b/packages/runtime-host/src/peer-reachability/index.ts @@ -18,6 +18,7 @@ */ export { + authenticateSignedPeerReachabilityLease, canonicalPeerReachabilityLease, decodeSignedPeerReachabilityLease, PEER_REACHABILITY_LEASE_TTL_MS, From 134fdd75176db55fc2108f4af9921e7312442639 Mon Sep 17 00:00:00 2001 From: Wang Date: Thu, 3 Sep 2026 06:38:09 +0800 Subject: [PATCH 12/13] fix(peer-mesh): bind anti-entropy summaries to facts Generated-by: Codex (gpt-5.6-sol) --- .../src/__tests__/peer-mesh.test.ts | 82 +++++++++++++ packages/runtime-host/src/peer-mesh/node.ts | 111 +++++++++++++----- 2 files changed, 165 insertions(+), 28 deletions(-) diff --git a/packages/runtime-host/src/__tests__/peer-mesh.test.ts b/packages/runtime-host/src/__tests__/peer-mesh.test.ts index c3f982ea75..0366b3b4b4 100644 --- a/packages/runtime-host/src/__tests__/peer-mesh.test.ts +++ b/packages/runtime-host/src/__tests__/peer-mesh.test.ts @@ -659,6 +659,88 @@ test('rejects conflicting reachability facts at the same revision during Mesh sy } }); +test('rejects equal-revision conflicts exposed by Mesh anti-entropy summaries', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-peer-mesh-summary-equivocation-')); + const network = new MemoryPeerNetwork(); + const authorityPeer = network.create('peer-a'); + const memberBPeer = network.create('peer-b'); + const memberCPeer = network.create('peer-c'); + const authorityRoot = join(root, 'authority'); + const authority = await openPeerMeshNode({ dataRoot: authorityRoot, peer: authorityPeer }); + const memberB = await openPeerMeshNode({ dataRoot: join(root, 'member-b'), peer: memberBPeer }); + const memberC = await openPeerMeshNode({ dataRoot: join(root, 'member-c'), peer: memberCPeer }); + const serving = [authority.serve(), memberB.serve(), memberC.serve()]; + try { + const meshId = (await authority.create()).roster.roster.meshId; + await memberB.join(await authority.invite(meshId)); + await memberC.join(await authority.invite(meshId)); + await memberC.reconcile(); + const originalRoutes = memberC.resolveRoutes('peer-b'); + assert.deepEqual(originalRoutes?.routeHints, ['/memory/peer-b/p2p/peer-b']); + + const original = memberBPeer.current(); + const conflictingLease = canonicalPeerReachabilityLease({ + ...original.lease, + directRoutes: ['/memory/conflicting/p2p/peer-b'], + }); + const conflictingProof = await memberBPeer.signIdentity( + peerReachabilityLeaseSigningBytes(conflictingLease), + ); + const conflicting = memberBPeer.verify( + decodeSignedPeerReachabilityLease({ + lease: conflictingLease, + publicKey: conflictingProof.publicKey.toString('base64url'), + signature: conflictingProof.signature.toString('base64url'), + }), + 'peer-b', + ); + const conflictingDigest = createHash('sha256') + .update(peerReachabilityLeaseSigningBytes(conflicting.lease)) + .digest('hex'); + const persisted = JSON.parse(await readFile(join(authorityRoot, 'peer-mesh.json'), 'utf8')) as { + readonly advertisements: readonly SignedPeerMeshMemberAdvertisementV1[]; + }; + const advertisement = persisted.advertisements.find( + ({ advertisement: candidate }) => + candidate.meshId === meshId && candidate.peerId === 'peer-a', + ); + assert.ok(advertisement); + + const stream = await authorityPeer.connectMeshControl({ peerId: 'peer-c' }); + await stream.write( + Buffer.from( + `${JSON.stringify({ + kind: 'sync', + meshId, + roster: authority.status()[0]?.roster, + reachability: authorityPeer.current(), + advertisement, + knownReachability: [ + { + peerId: 'peer-b', + revision: conflictingLease.revision, + digest: conflictingDigest, + }, + ], + knownAdvertisements: [], + })}\n`, + ), + ); + + assert.equal(await stream.read(), null); + assert.deepEqual(memberC.resolveRoutes('peer-b'), originalRoutes); + } finally { + await Promise.allSettled([authority.close(), memberB.close(), memberC.close()]); + await Promise.allSettled([ + authorityPeer.close(), + memberBPeer.close(), + memberCPeer.close(), + ...serving, + ]); + await rm(root, { recursive: true, force: true }); + } +}); + test('publishes a changed local route promptly and refreshes a live cached peer route', { timeout: 10_000, }, async () => { diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index 0d483432f7..bc45841835 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -24,6 +24,7 @@ import type { RuntimeHostPeerTransitRelayCandidate, RuntimeHostPeerTransitSnapshot, } from '../transport/peer-native.js'; +import { createHash } from 'node:crypto'; import { setTimeout as delay } from 'node:timers/promises'; import { performance } from 'node:perf_hooks'; import { @@ -55,6 +56,7 @@ import { decodeSignedPeerReachabilityLease, isPeerReachabilityLeaseCurrent, PEER_REACHABILITY_MAX_CLOCK_SKEW_MS, + peerReachabilityLeaseSigningBytes, peerReachabilityLeaseReceipt, type PeerReachabilityPublisher, type PeerReachabilityLeaseReceipt, @@ -107,9 +109,10 @@ type RedeemInvitationResponse = type RedeemInvitationRejectionReason = 'invalid' | 'expired' | 'closed' | 'full'; -interface PeerMeshEvidenceRevision { +interface PeerMeshEvidenceSummary { readonly peerId: string; readonly revision: number; + readonly digest: string; } interface SyncPeerMeshRequest { @@ -118,8 +121,8 @@ interface SyncPeerMeshRequest { readonly roster: SignedPeerMeshRosterV1; readonly reachability: SignedPeerReachabilityLeaseV1; readonly advertisement: SignedPeerMeshMemberAdvertisementV1; - readonly knownReachability: readonly PeerMeshEvidenceRevision[]; - readonly knownAdvertisements: readonly PeerMeshEvidenceRevision[]; + readonly knownReachability: readonly PeerMeshEvidenceSummary[]; + readonly knownAdvertisements: readonly PeerMeshEvidenceSummary[]; } type SyncPeerMeshResponse = @@ -1313,12 +1316,12 @@ class PeerMeshNodeImpl implements PeerMeshNode { roster: state.roster, reachability, advertisement, - knownReachability: reachabilityRevisions( + knownReachability: reachabilitySummaries( stored.reachability, state.roster, this.#now(), ), - knownAdvertisements: advertisementRevisions(stored.advertisements, state.roster), + knownAdvertisements: advertisementSummaries(stored.advertisements, state.roster), }, decodeSyncResponse, signal, @@ -2448,7 +2451,7 @@ function mergeReachability( } if ( candidate.lease.revision === existing.lease.revision && - JSON.stringify(candidate) !== JSON.stringify(existing) + reachabilityFactDigest(candidate) !== reachabilityFactDigest(existing) ) { throw new Error('Peer reachability revision identifies conflicting facts'); } @@ -2474,7 +2477,7 @@ function mergeAdvertisements( } if ( candidate.advertisement.revision === existing.advertisement.revision && - JSON.stringify(candidate) !== JSON.stringify(existing) + advertisementFactDigest(candidate) !== advertisementFactDigest(existing) ) { throw new Error('Peer Mesh advertisement revision identifies conflicting facts'); } @@ -2486,11 +2489,11 @@ function mergeAdvertisements( ); } -function reachabilityRevisions( +function reachabilitySummaries( reachability: readonly SignedPeerReachabilityLeaseV1[], roster: SignedPeerMeshRosterV1, now: number, -): readonly PeerMeshEvidenceRevision[] { +): readonly PeerMeshEvidenceSummary[] { return Object.freeze( reachability .filter( @@ -2498,15 +2501,21 @@ function reachabilityRevisions( usableHistoricalReachability(signed, now) && roster.roster.members.includes(signed.lease.peerId), ) - .map(({ lease }) => Object.freeze({ peerId: lease.peerId, revision: lease.revision })) + .map((signed) => + Object.freeze({ + peerId: signed.lease.peerId, + revision: signed.lease.revision, + digest: reachabilityFactDigest(signed), + }), + ) .sort((left, right) => left.peerId.localeCompare(right.peerId)), ); } -function advertisementRevisions( +function advertisementSummaries( advertisements: readonly SignedPeerMeshMemberAdvertisementV1[], roster: SignedPeerMeshRosterV1, -): readonly PeerMeshEvidenceRevision[] { +): readonly PeerMeshEvidenceSummary[] { return Object.freeze( advertisements .filter( @@ -2514,10 +2523,11 @@ function advertisementRevisions( advertisement.meshId === roster.roster.meshId && roster.roster.members.includes(advertisement.peerId), ) - .map(({ advertisement }) => + .map((signed) => Object.freeze({ - peerId: advertisement.peerId, - revision: advertisement.revision, + peerId: signed.advertisement.peerId, + revision: signed.advertisement.revision, + digest: advertisementFactDigest(signed), }), ) .sort((left, right) => left.peerId.localeCompare(right.peerId)), @@ -2528,23 +2538,31 @@ function responseEvidence( state: PeerMeshStateV1, reachability: readonly SignedPeerReachabilityLeaseV1[], advertisements: readonly SignedPeerMeshMemberAdvertisementV1[], - knownReachability: readonly PeerMeshEvidenceRevision[], - knownAdvertisements: readonly PeerMeshEvidenceRevision[], + knownReachability: readonly PeerMeshEvidenceSummary[], + knownAdvertisements: readonly PeerMeshEvidenceSummary[], now: number, ): { readonly reachability: readonly SignedPeerReachabilityLeaseV1[]; readonly advertisements: readonly SignedPeerMeshMemberAdvertisementV1[]; readonly more: boolean; } { - const knownLeases = new Map(knownReachability.map(({ peerId, revision }) => [peerId, revision])); - const knownAds = new Map(knownAdvertisements.map(({ peerId, revision }) => [peerId, revision])); + const knownLeases = new Map(knownReachability.map((summary) => [summary.peerId, summary])); + const knownAds = new Map(knownAdvertisements.map((summary) => [summary.peerId, summary])); const missing = [ ...reachability .filter( (signed) => usableHistoricalReachability(signed, now) && state.roster.roster.members.includes(signed.lease.peerId) && - signed.lease.revision > (knownLeases.get(signed.lease.peerId) ?? 0), + evidenceRequiresTransfer( + { + peerId: signed.lease.peerId, + revision: signed.lease.revision, + digest: reachabilityFactDigest(signed), + }, + knownLeases.get(signed.lease.peerId), + 'Peer reachability', + ), ) .map((value) => ({ kind: 'reachability' as const, @@ -2553,10 +2571,18 @@ function responseEvidence( })), ...advertisements .filter( - ({ advertisement }) => - advertisement.meshId === state.roster.roster.meshId && - state.roster.roster.members.includes(advertisement.peerId) && - advertisement.revision > (knownAds.get(advertisement.peerId) ?? 0), + (signed) => + signed.advertisement.meshId === state.roster.roster.meshId && + state.roster.roster.members.includes(signed.advertisement.peerId) && + evidenceRequiresTransfer( + { + peerId: signed.advertisement.peerId, + revision: signed.advertisement.revision, + digest: advertisementFactDigest(signed), + }, + knownAds.get(signed.advertisement.peerId), + 'Peer Mesh advertisement', + ), ) .map((value) => ({ kind: 'advertisement' as const, @@ -2580,6 +2606,31 @@ function responseEvidence( }); } +function evidenceRequiresTransfer( + local: PeerMeshEvidenceSummary, + remote: PeerMeshEvidenceSummary | undefined, + label: string, +): boolean { + if (!remote || local.revision > remote.revision) return true; + if (local.revision < remote.revision) return false; + if (local.digest !== remote.digest) { + throw new Error(`${label} revision identifies conflicting facts`); + } + return false; +} + +function reachabilityFactDigest(signed: SignedPeerReachabilityLeaseV1): string { + return evidenceDigest(peerReachabilityLeaseSigningBytes(signed.lease)); +} + +function advertisementFactDigest(signed: SignedPeerMeshMemberAdvertisementV1): string { + return evidenceDigest(peerMeshMemberAdvertisementSigningBytes(signed.advertisement)); +} + +function evidenceDigest(bytes: Uint8Array): string { + return createHash('sha256').update(bytes).digest('hex'); +} + function initialEvidence( state: PeerMeshStateV1, reachability: readonly SignedPeerReachabilityLeaseV1[], @@ -2780,8 +2831,8 @@ function decodeControlRequest(value: unknown): PeerMeshControlRequest { roster: decodeSignedPeerMeshRoster(record.roster), reachability: decodeSignedPeerReachabilityLease(record.reachability), advertisement: decodeSignedPeerMeshMemberAdvertisement(record.advertisement), - knownReachability: decodeEvidenceRevisions(record.knownReachability), - knownAdvertisements: decodeEvidenceRevisions(record.knownAdvertisements), + knownReachability: decodeEvidenceSummaries(record.knownReachability), + knownAdvertisements: decodeEvidenceSummaries(record.knownAdvertisements), }; } if (record.kind === 'leave' && hasExactKeys(record, ['kind', 'meshId', 'roster'])) { @@ -2911,22 +2962,26 @@ function assertEvidencePageSize( } } -function decodeEvidenceRevisions(value: unknown): readonly PeerMeshEvidenceRevision[] { +function decodeEvidenceSummaries(value: unknown): readonly PeerMeshEvidenceSummary[] { if (!Array.isArray(value) || value.length > PEER_MESH_MAX_MEMBERS) { throw new Error('Invalid Peer Mesh evidence revisions'); } const revisions = value.map((entry) => { const record = recordValue(entry); - if (!hasExactKeys(record, ['peerId', 'revision'])) { + if (!hasExactKeys(record, ['peerId', 'revision', 'digest'])) { throw new Error('Invalid Peer Mesh evidence revision'); } const revision = record.revision; if (!Number.isSafeInteger(revision) || (revision as number) < 1) { throw new Error('Invalid Peer Mesh evidence revision'); } + if (typeof record.digest !== 'string' || !/^[0-9a-f]{64}$/u.test(record.digest)) { + throw new Error('Invalid Peer Mesh evidence digest'); + } return Object.freeze({ peerId: requiredString(record.peerId, 256), revision: revision as number, + digest: record.digest, }); }); if (new Set(revisions.map(({ peerId }) => peerId)).size !== revisions.length) { From a8f119a9ceb293763163d6a70f044db80f6f69a9 Mon Sep 17 00:00:00 2001 From: Wang Date: Thu, 3 Sep 2026 07:50:35 +0800 Subject: [PATCH 13/13] fix(peer-mesh): verify reachability at trust boundary Generated-by: Codex (gpt-5.6-sol) --- packages/runtime-host/src/peer-mesh/node.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/runtime-host/src/peer-mesh/node.ts b/packages/runtime-host/src/peer-mesh/node.ts index bc45841835..89ce4eae96 100644 --- a/packages/runtime-host/src/peer-mesh/node.ts +++ b/packages/runtime-host/src/peer-mesh/node.ts @@ -58,6 +58,7 @@ import { PEER_REACHABILITY_MAX_CLOCK_SKEW_MS, peerReachabilityLeaseSigningBytes, peerReachabilityLeaseReceipt, + verifySignedPeerReachabilityLease, type PeerReachabilityPublisher, type PeerReachabilityLeaseReceipt, type SignedPeerReachabilityLeaseV1, @@ -1451,8 +1452,12 @@ class PeerMeshNodeImpl implements PeerMeshNode { expectedPeerId: string, allowExpired = false, ): SignedPeerReachabilityLeaseV1 { - const signed = this.#reachability.verify(value, expectedPeerId, { - allowExpired, + const signed = verifySignedPeerReachabilityLease({ + value, + expectedPeerId, + now: this.#now(), + verifyIdentity: this.#peer.verifyIdentity.bind(this.#peer), + ...(allowExpired ? { allowExpired: true } : {}), }); if (allowExpired && signed.lease.expiresAt + REACHABILITY_HISTORY_MS <= this.#now()) { throw new Error('Peer Mesh reachability is outside the recovery horizon');