fix(web): wire API reachability, lenient string search params, sidebar state, chart and cache memoization

apps/local-ui/src/data/query-cache.test.ts

@@ -1,7 +1,8 @@
-import { Effect, Layer, Option } from "effect"
+import { Array as Arr, Effect, Layer, Option } from "effect"
 import { TestClock } from "effect/testing"
 import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"
-import { LocalQueryCache } from "./query-cache"
+import { GC_MS, hashKey } from "./cache-store"
+import { type CacheState, collectUnused, LocalQueryCache } from "./query-cache"
 
 // The cache as the runtime builds it, on a test clock: `gcTime` drops what nothing used for an hour.
 
@@ -10,12 +11,21 @@ const OTHER = ["local", "trace", "t2"]
 
 beforeEach(() => {
 	vi.stubGlobal("window", { location: new URL("http://127.0.0.1:4318/") })
-	vi.stubGlobal("document", { visibilityState: "visible", addEventListener: () => undefined, removeEventListener: () => undefined })
+	vi.stubGlobal("document", {
+		visibilityState: "visible",
+		addEventListener: () => undefined,
+		removeEventListener: () => undefined,
+	})
 })
 afterEach(() => vi.unstubAllGlobals())
 
 const run = <A>(program: Effect.Effect<A, never, LocalQueryCache>) =>
-	Effect.runPromise(program.pipe(Effect.provide(LocalQueryCache.layer.pipe(Layer.provideMerge(TestClock.layer()))), Effect.scoped))
+	Effect.runPromise(
+		program.pipe(
+			Effect.provide(LocalQueryCache.layer.pipe(Layer.provideMerge(TestClock.layer()))),
+			Effect.scoped,
+		),
+	)
 
 const isCached = (key: ReadonlyArray<unknown>) =>
 	Effect.gen(function* () {
@@ -53,3 +63,45 @@ describe("LocalQueryCache gcTime", () => {
 		expect(gone).toBe(true)
 	})
 })
+
+describe("collectUnused", () => {
+	const initial: CacheState = { entries: new Map(), lastUsed: new Map() }
+
+	/** A tab left open on a relative range: every minute is a new key, used once. */
+	const afterMinutes = (minutes: number): CacheState =>
+		Arr.reduce(
+			Arr.makeBy(minutes, (minute) => minute),
+			initial,
+			(state, minute) => {
+				const nowMs = minute * 60_000
+				const key = ["local", "logs", `window-${minute}`]
+				return collectUnused(
+					{
+						entries: new Map([
+							...state.entries,
+							[hashKey(key), { key, data: minute, updatedAtMs: nowMs }],
+						]),
+						lastUsed: new Map(state.lastUsed).set(hashKey(key), nowMs),
+					},
+					nowMs,
+				)
+			},
+		)
+
+	test("use times are kept only for the results still cached", () => {
+		const state = afterMinutes(300)
+
+		expect(state.entries.size).toBe(GC_MS / 60_000)
+		expect([...state.lastUsed.keys()]).toEqual([...state.entries.keys()])
+	})
+
+	test("a result used within the hour keeps its use time", () => {
+		const key = ["local", "trace", "t1"]
+		const state: CacheState = {
+			entries: new Map([[hashKey(key), { key, data: 1, updatedAtMs: 0 }]]),
+			lastUsed: new Map([[hashKey(key), 1_000]]),
+		}
+
+		expect(collectUnused(state, GC_MS).lastUsed.get(hashKey(key))).toBe(1_000)
+	})
+})

apps/local-ui/src/data/query-cache.ts

@@ -23,6 +23,21 @@ const SAVE_DELAY_MS = 2_000
 /** A cached result: exact for the key, or an older window standing in for it. */
 export type CacheHit = Readonly<{ data: CacheValue; updatedAtMs: number; isStandIn: boolean }>
 
+/** The results and when each was last read or written (`gcTime`'s clock), replaced together. */
+export type CacheState = Readonly<{ entries: CacheEntries; lastUsed: ReadonlyMap<string, number> }>
+
+/**
+ * `gcTime` applied: drops what nothing used for an hour, and the use times of what it dropped.
+ * Relative ranges mint a new key per window, so use times kept past their entry grow without bound.
+ */
+export const collectUnused = (state: CacheState, nowMs: number): CacheState => {
+	const entries = dropUnused(state.entries, state.lastUsed, nowMs)
+	return {
+		entries,
+		lastUsed: new Map(Arr.filter(Arr.fromIterable(state.lastUsed), ([hash]) => entries.has(hash))),
+	}
+}
+
 /** The saved results this page was opened with (Flags: `hasCache` before anything loads). */
 export const readSavedEntries: Effect.Effect<CacheEntries> = Effect.gen(function* () {
 	const nowMs = yield* Clock.currentTimeMillis
@@ -47,19 +62,19 @@ export class LocalQueryCache extends Context.Service<
 		Effect.gen(function* () {
 			const storageKey = storageKeyFor(localServerPortForLocation(window.location))
 			const restored = yield* readSavedEntries
-			const entries = yield* Ref.make<CacheEntries>(restored)
 			// `gcTime`'s clock per entry: a restored result starts unobserved at load.
 			const startedAtMs = yield* Clock.currentTimeMillis
-			const lastUsed = yield* Ref.make<ReadonlyMap<string, number>>(
-				new Map(Arr.map([...restored.keys()], (hash) => [hash, startedAtMs])),
-			)
+			const state = yield* Ref.make<CacheState>({
+				entries: restored,
+				lastUsed: new Map(Arr.map([...restored.keys()], (hash) => [hash, startedAtMs])),
+			})
 			const isSaveScheduled = yield* Ref.make(false)
 			const scope = yield* Effect.scope
 			/** Drops what nothing used for an hour; returns what is left. */
 			const collect = Effect.gen(function* () {
 				const nowMs = yield* Clock.currentTimeMillis
-				const used = yield* Ref.get(lastUsed)
-				return yield* Ref.updateAndGet(entries, (current) => dropUnused(current, used, nowMs))
+				const current = yield* Ref.updateAndGet(state, (before) => collectUnused(before, nowMs))
+				return current.entries
 			})
 			const save = Effect.flatMap(collect, (current) =>
 				Effect.sync(() => writeLocalStorage(storageKey, serialize(current))),
@@ -92,7 +107,10 @@ export class LocalQueryCache extends Context.Service<
 						),
 			)
 			const touch = (hash: string, nowMs: number) =>
-				Ref.update(lastUsed, (used) => new Map(used).set(hash, nowMs))
+				Ref.update(state, (before) => ({
+					entries: before.entries,
+					lastUsed: new Map(before.lastUsed).set(hash, nowMs),
+				}))
 			return {
 				lookup: (key) =>
 					Effect.gen(function* () {
@@ -120,15 +138,14 @@ export class LocalQueryCache extends Context.Service<
 						onSome: (value) =>
 							Effect.gen(function* () {
 								const now = yield* Clock.currentTimeMillis
-								yield* touch(hashKey(key), now)
-								yield* Ref.update(
-									entries,
-									(current) =>
-										new Map([
-											...current,
-											[hashKey(key), { key, data: value, updatedAtMs }],
-										]),
-								)
+								const hash = hashKey(key)
+								yield* Ref.update(state, (before) => ({
+									entries: new Map([
+										...before.entries,
+										[hash, { key, data: value, updatedAtMs }],
+									]),
+									lastUsed: new Map(before.lastUsed).set(hash, now),
+								}))
 								yield* collect
 								yield* scheduleSave
 							}),

apps/web/src/data/warehouse.ts

@@ -1,10 +1,13 @@
 import { QueryEngineExecuteBatchRequest } from "@maple/query-engine"
+import { isBlipping, originOf } from "@maple/web-data/lib/peer-reachability"
 import {
+	ApiReachability,
 	MapleApiV2Http,
 	MapleInternalHttp,
 	WarehouseExecutor,
 } from "@maple/web-data/warehouse/warehouse-services"
 import { Effect, Layer } from "effect"
+import { apiBaseUrl } from "../lib/services/common/api-base-url"
 import { MapleApiV2Client, MapleInternalApiClient } from "../platform/api-clients"
 import { withResourceTiming } from "../lib/resource-timing"
 import { makeExecuteBatcher } from "./execute-batcher"
@@ -32,8 +35,14 @@ const WarehouseExecutorLive = Layer.effect(
 	}),
 )
 
+/** The API origin's reachability clock, which the HTTP client keeps (`withRequestTimeout`). */
+export const ApiReachabilityLive = Layer.succeed(ApiReachability, {
+	isBlipping: (nowMs) => isBlipping(originOf(apiBaseUrl), nowMs),
+})
+
 export const WarehouseLive = Layer.mergeAll(
 	WarehouseExecutorLive,
+	ApiReachabilityLive,
 	Layer.effect(MapleInternalHttp, Effect.service(MapleInternalApiClient)),
 	Layer.effect(MapleApiV2Http, Effect.service(MapleApiV2Client)),
 ).pipe(Layer.provide([MapleInternalApiClient.layer, MapleApiV2Client.layer]))

apps/web/src/lib/services/common/http-client.test.ts

@@ -1,17 +1,28 @@
 import { assert, describe, it } from "@effect/vitest"
-import { Effect, Fiber, Result } from "effect"
+import { MapleInternalApi } from "@maple/domain/http"
+import { Effect, Fiber, Layer, Result } from "effect"
+import { HttpApiClient } from "effect/http-api"
 import { TestClock } from "effect/testing"
 import { FetchHttpClient, HttpClient, HttpClientError } from "effect/http"
 
+import { PEER_OUTAGE_GRACE_MS } from "@maple/web-data/lib/peer-reachability"
+import {
+	MapleInternalHttp,
+	runWarehouseQuery,
+	WarehouseQueryError,
+	WarehouseUnreachableError,
+} from "@maple/web-data/warehouse/effect-utils"
+import { ApiReachabilityLive } from "../../../data/warehouse"
+import { apiBaseUrl } from "./api-base-url"
 import { withRequestTimeout } from "./http-client"
 import { isRetryableTransportError } from "./retry-policy"
 
 const NEVER_RESPONDS: typeof globalThis.fetch = () => new Promise<Response>(() => {})
 
-const get = (fetchStub: typeof globalThis.fetch) =>
+const get = (fetchStub: typeof globalThis.fetch, url = "https://api.test/anything") =>
 	Effect.gen(function* () {
 		const client = withRequestTimeout(yield* HttpClient.HttpClient)
-		return yield* client.get("https://api.test/anything")
+		return yield* client.get(url)
 	}).pipe(Effect.provide(FetchHttpClient.layer), Effect.provideService(FetchHttpClient.Fetch, fetchStub))
 
 /**
@@ -56,3 +67,45 @@ describe("withRequestTimeout", () => {
 		}),
 	)
 })
+
+const DROPS: typeof globalThis.fetch = () => Promise.reject(new TypeError("Failed to fetch"))
+const ANSWERS_500: typeof globalThis.fetch = async () => new Response("{}", { status: 500 })
+
+// `runWarehouseQuery` runs on the internal client; these reads go through `get` instead.
+const InternalHttp = Layer.effect(MapleInternalHttp, HttpApiClient.make(MapleInternalApi)).pipe(
+	Layer.provide(FetchHttpClient.layer),
+)
+
+/** A warehouse read over the app's client, failing as the dashboard would report it. */
+const queryFailure = (fetchStub: typeof globalThis.fetch) =>
+	runWarehouseQuery("listThings", () => get(fetchStub, `${apiBaseUrl}/internal/things`)).pipe(
+		Effect.provide([ApiReachabilityLive, InternalHttp]),
+		Effect.flip,
+	)
+
+describe("API reachability", () => {
+	it.effect("reports a dropped connection as unreachable, then as an outage once it lasts", () =>
+		Effect.gen(function* () {
+			// Any response ends a run left over from another test.
+			yield* get(ANSWERS_500, `${apiBaseUrl}/internal/things`)
+
+			assert.instanceOf(yield* queryFailure(DROPS), WarehouseUnreachableError)
+
+			yield* TestClock.adjust(PEER_OUTAGE_GRACE_MS)
+			assert.instanceOf(yield* queryFailure(DROPS), WarehouseQueryError)
+		}),
+	)
+
+	it.effect("starts a new grace window after the API answers, even with a 500", () =>
+		Effect.gen(function* () {
+			yield* get(ANSWERS_500, `${apiBaseUrl}/internal/things`)
+			assert.instanceOf(yield* queryFailure(DROPS), WarehouseUnreachableError)
+
+			yield* TestClock.adjust(PEER_OUTAGE_GRACE_MS - 1)
+			yield* get(ANSWERS_500, `${apiBaseUrl}/internal/things`)
+			yield* TestClock.adjust(PEER_OUTAGE_GRACE_MS - 1)
+
+			assert.instanceOf(yield* queryFailure(DROPS), WarehouseUnreachableError)
+		}),
+	)
+})

apps/web/src/lib/services/common/http-client.ts

@@ -1,32 +1,45 @@
 import { HttpClient, HttpClientError } from "effect/http"
 import { Clock, Duration, Effect } from "effect"
-import { noteUnreachable, originOf } from "@maple/web-data/lib/peer-reachability"
+import { noteReachable, noteUnreachable, originOf } from "@maple/web-data/lib/peer-reachability"
 
 const CLIENT_TIMEOUT = Duration.seconds(45)
 
+const isTransportError = (error: unknown): boolean =>
+	HttpClientError.isHttpClientError(error) && error.reason._tag === "TransportError"
+
 /**
  * The client-wide request deadline. Timing out the fiber interrupts it, which
- * aborts the in-flight fetch, and the origin is marked unreachable here.
+ * aborts the in-flight fetch.
+ *
+ * It also keeps the origin's reachability clock (`peer-reachability.ts`): a
+ * request that got no response (the deadline included) starts or extends the
+ * run of failures, and any response at all, a 500 included, ends it.
  */
 export const withRequestTimeout = (client: HttpClient.HttpClient): HttpClient.HttpClient =>
-	HttpClient.transform(client, (effect, request) =>
-		effect.pipe(
+	HttpClient.transform(client, (effect, request) => {
+		const origin = originOf(request.url)
+		return effect.pipe(
 			Effect.timeoutOrElse({
 				duration: CLIENT_TIMEOUT,
-				orElse: () =>
-					Effect.gen(function* () {
-						const now = yield* Clock.currentTimeMillis
-						noteUnreachable(originOf(request.url), now)
-						const description = `No response within ${Duration.toMillis(CLIENT_TIMEOUT)}ms`
-						// A `TimeoutError` cause marks it non-retryable (`isRetryableTransportError`).
-						return yield* new HttpClientError.HttpClientError({
+				orElse: () => {
+					const description = `No response within ${Duration.toMillis(CLIENT_TIMEOUT)}ms`
+					// A `TimeoutError` cause marks it non-retryable (`isRetryableTransportError`).
+					return Effect.fail(
+						new HttpClientError.HttpClientError({
 							reason: new HttpClientError.TransportError({
 								request,
 								description,
 								cause: new DOMException(description, "TimeoutError"),
 							}),
-						})
-					}),
+						}),
+					)
+				},
 			}),
-		),
-	)
+			Effect.tap(() => Effect.sync(() => noteReachable(origin))),
+			Effect.tapError((error) =>
+				isTransportError(error)
+					? Effect.map(Clock.currentTimeMillis, (now) => noteUnreachable(origin, now))
+					: Effect.void,
+			),
+		)
+	})

apps/web/src/main.test.ts

@@ -0,0 +1,27 @@
+import { Schema } from "effect"
+import { describe, expect, test, vi } from "vitest"
+
+// The shell's favorites listener targets `window` at import.
+vi.hoisted(() => Object.assign(globalThis, { window: new EventTarget() }))
+import { Flags } from "./main"
+import * as Shell from "./shell"
+
+describe("Flags", () => {
+	test("flags that went through the codec still carry every shell flag the init reads", () => {
+		const decoded = Schema.decodeSync(Flags)({
+			hasStoredSession: false,
+			timeZone: "UTC",
+			knownToRenderApp: {},
+			shell: {
+				isSidebarOpen: false,
+				isMobile: false,
+				theme: "light",
+				pinnedNamespaces: {},
+				dashboardFavorites: [],
+				platform: "windows",
+			},
+		})
+
+		expect(Shell.init(decoded.shell)).toMatchObject({ platform: "windows", isSidebarOpen: false })
+	})
+})

apps/web/src/main.ts

@@ -31,6 +31,7 @@ export const Flags = Schema.Struct({
 	timeZone: Schema.String,
 	knownToRenderApp: Schema.Record(Schema.String, Schema.Boolean),
 	shell: Schema.Struct({
+		isSidebarOpen: Schema.optionalKey(Schema.Boolean),
 		isMobile: Schema.Boolean,
 		isFilterCollapsed: Schema.optionalKey(Schema.Boolean),
 		isBelowSm: Schema.optionalKey(Schema.Boolean),
@@ -39,6 +40,7 @@ export const Flags = Schema.Struct({
 		dashboardFavorites: Schema.Array(Schema.String),
 		rememberedSurfaces: Schema.optionalKey(Schema.Record(Schema.String, Schema.Array(Shell.NavSurface))),
 		isQuotaApproachingDismissed: Schema.optionalKey(Schema.Boolean),
+		platform: Schema.optionalKey(Shell.Platform),
 	}),
 })
 export type Flags = typeof Flags.Type