From 1da9f0c008061377c8a48c6871fe2d3c0ddeae94 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 17:10:28 -0700 Subject: [PATCH 01/85] Keep worldwide base surface continuous at every zoom --- src/server/world-tile-gateway.js | 35 +++++++++++++++++--------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/server/world-tile-gateway.js b/src/server/world-tile-gateway.js index 7614a2c3..178e69d8 100644 --- a/src/server/world-tile-gateway.js +++ b/src/server/world-tile-gateway.js @@ -14,6 +14,7 @@ const DEFAULT_SURFACE_MAX_ZOOM = 10; const DEFAULT_ROUTING_ZOOM = 6; const DEFAULT_MAX_ZOOM = 16; const DEFAULT_CACHE_BYTES = 128 * 1024 * 1024; +const CONTINUOUS_SURFACE_LAYERS = Object.freeze(['land', 'landcover', 'depth']); class MemoryTileCache { constructor(maxBytes = DEFAULT_CACHE_BYTES) { @@ -193,7 +194,7 @@ export class WorldTileGateway { if (zoom <= surfaceMaxZoom) { const payload = await this.readArchiveTile(surfaceAsset, zoom, x, y); return payload - ? mergeVectorTiles([payload], { includeLayers: ['land'] }) + ? mergeVectorTiles([payload], { includeLayers: CONTINUOUS_SURFACE_LAYERS }) : EMPTY_MVT; } @@ -208,13 +209,16 @@ export class WorldTileGateway { ); if (!payload) return EMPTY_MVT; - return overscaleVectorLayer(payload, { - layerName: 'land', - sourceZoom: surfaceMaxZoom, - targetZoom: zoom, - targetX: x, - targetY: y - }); + const overscaledLayers = CONTINUOUS_SURFACE_LAYERS.map((layerName) => + overscaleVectorLayer(payload, { + layerName, + sourceZoom: surfaceMaxZoom, + targetZoom: zoom, + targetX: x, + targetY: y + }) + ); + return mergeVectorTiles(overscaledLayers); } async readBasemapTile(manifest, zoom, x, y) { @@ -249,14 +253,13 @@ export class WorldTileGateway { this.readBasemapTile(manifest, zoom, x, y) ]); - // The physical surface is the single authoritative land mask. The - // overview and regional archives may also contain a layer named `land`; - // merging both copies can create duplicate rings and reused-ID joins. - // Retain basemap land only as an emergency fallback when the surface tile - // is genuinely empty. - const hasSurfaceLand = !Buffer.from(surface).equals(EMPTY_MVT); - const cartography = hasSurfaceLand - ? mergeVectorTiles([basemap], { excludeLayers: ['land'] }) + // Land, generalized vegetation, and bathymetry form one continuous + // physical surface from minimum zoom through street zoom. Overview and + // regional archives may add cartographic detail, but they must never + // replace these foundational layers as the user crosses a zoom boundary. + const hasContinuousSurface = !Buffer.from(surface).equals(EMPTY_MVT); + const cartography = hasContinuousSurface + ? mergeVectorTiles([basemap], { excludeLayers: CONTINUOUS_SURFACE_LAYERS }) : basemap; return this.tileCache.set( From 3f8883f58a19030a4ebcc1ce05b686b418f31ed8 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 17:11:06 -0700 Subject: [PATCH 02/85] Keep base landcover visible from globe zoom --- scripts/apply-schema-parity.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/apply-schema-parity.mjs b/scripts/apply-schema-parity.mjs index 19f6a112..20faf825 100644 --- a/scripts/apply-schema-parity.mjs +++ b/scripts/apply-schema-parity.mjs @@ -174,6 +174,7 @@ water.metadata = { const landcover = runtime.layers.find((layer) => layer.id === 'landcover'); if (!landcover) throw new Error('The exported landcover layer is missing from the runtime style.'); +landcover.minzoom = 0; landcover.paint ||= {}; landcover.paint['fill-opacity'] = [ 'interpolate', @@ -190,7 +191,7 @@ landcover.paint['fill-opacity'] = [ ]; landcover.metadata = { ...(landcover.metadata || {}), - 'occumed:purpose': 'strong exported vegetation hierarchy over open landcover data' + 'occumed:purpose': 'continuous worldwide vegetation foundation with regional detail added above it' }; runtime.metadata = { From a3ea969dc40949428c2f35a3ffec507c565ae347 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 17:11:35 -0700 Subject: [PATCH 03/85] Guard continuous globe landcover --- scripts/check-cartography-parity.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/check-cartography-parity.mjs b/scripts/check-cartography-parity.mjs index 02eb2248..5bdcc21c 100644 --- a/scripts/check-cartography-parity.mjs +++ b/scripts/check-cartography-parity.mjs @@ -82,6 +82,9 @@ const landcover = layer('landcover'); if (!Array.isArray(landcover?.paint?.['fill-opacity'])) { fail('The landcover layer must preserve the strong green low-zoom hierarchy.'); } +if (Number(landcover?.minzoom || 0) !== 0) { + fail('The foundational landcover layer must remain visible from globe zoom instead of switching on later.'); +} const roadLayers = runtime.layers.filter( (candidate) => @@ -125,5 +128,5 @@ if (failures.length) { } console.log( - `Cartography parity validated: ${roadLayers.length} road layers, ${symbolLayers.length} symbol layers, normalized landuse, place hierarchy, and overlay isolation.` + `Cartography parity validated: ${roadLayers.length} road layers, ${symbolLayers.length} symbol layers, continuous globe landcover, normalized landuse, place hierarchy, and overlay isolation.` ); From d0805fb81c604e90283c5204ed89cc5fcdee3d40 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 17:12:27 -0700 Subject: [PATCH 04/85] Validate one continuous physical surface across zooms --- scripts/check-world-tile-gateway.mjs | 72 +++++++++++++++++----------- 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/scripts/check-world-tile-gateway.mjs b/scripts/check-world-tile-gateway.mjs index a5c946af..784011a2 100644 --- a/scripts/check-world-tile-gateway.mjs +++ b/scripts/check-world-tile-gateway.mjs @@ -134,32 +134,44 @@ const surface = tile({ ] } }); -const overscaled = inspectVectorTile(overscaleVectorLayer(surface, { - layerName: 'land', - sourceZoom: 10, - targetZoom: 16, - targetX: 32768, - targetY: 32768 -})); -expect(overscaled.land?.featureCount === 1, 'The worldwide land surface cannot overscale through max zoom.'); -for (const bounds of overscaled.land?.bounds || []) { +const continuousSurfaceLayers = ['land', 'landcover', 'depth']; +const overscaledSurface = inspectVectorTile( + mergeVectorTiles( + continuousSurfaceLayers.map((layerName) => + overscaleVectorLayer(surface, { + layerName, + sourceZoom: 10, + targetZoom: 16, + targetX: 32768, + targetY: 32768 + }) + ) + ) +); +for (const layerName of continuousSurfaceLayers) { expect( - bounds && bounds.minX >= 0 && bounds.minY >= 0 && bounds.maxX <= 4096 && bounds.maxY <= 4096, - 'Overscaled land geometry escaped the requested child tile.' + overscaledSurface[layerName]?.featureCount === 1, + `The worldwide ${layerName} surface cannot overscale through max zoom.` ); + for (const bounds of overscaledSurface[layerName]?.bounds || []) { + expect( + bounds && bounds.minX >= 0 && bounds.minY >= 0 && bounds.maxX <= 4096 && bounds.maxY <= 4096, + `Overscaled ${layerName} geometry escaped the requested child tile.` + ); + } } -const physicalLandMask = inspectVectorTile( - mergeVectorTiles([surface], { includeLayers: ['land'] }) +const physicalSurface = inspectVectorTile( + mergeVectorTiles([surface], { includeLayers: continuousSurfaceLayers }) ); -expect(physicalLandMask.land?.featureCount === 1, 'The physical surface lost its land layer.'); -expect(!physicalLandMask.landcover, 'The physical surface still overlays generalized landcover on regional detail.'); -expect(!physicalLandMask.depth, 'The physical surface still overlays generalized bathymetry on regional detail.'); -const cartographyWithoutLand = inspectVectorTile( - mergeVectorTiles([surface], { excludeLayers: ['land'] }) +expect(physicalSurface.land?.featureCount === 1, 'The physical surface lost its land layer.'); +expect(physicalSurface.landcover?.featureCount === 1, 'The physical surface lost continuous landcover.'); +expect(physicalSurface.depth?.featureCount === 1, 'The physical surface lost continuous bathymetry.'); +const cartographyWithoutSurface = inspectVectorTile( + mergeVectorTiles([surface], { excludeLayers: continuousSurfaceLayers }) ); -expect(!cartographyWithoutLand.land, 'The duplicate basemap land layer was not excluded.'); -expect(cartographyWithoutLand.landcover?.featureCount === 1, 'Excluding land removed unrelated cartography.'); -expect(cartographyWithoutLand.depth?.featureCount === 1, 'Excluding land removed bathymetry.'); +expect(!cartographyWithoutSurface.land, 'The duplicate basemap land layer was not excluded.'); +expect(!cartographyWithoutSurface.landcover, 'The duplicate basemap landcover layer was not excluded.'); +expect(!cartographyWithoutSurface.depth, 'The duplicate basemap bathymetry layer was not excluded.'); const normalizedProperties = normalizeMvtProperties({ safeRank: 4, @@ -256,16 +268,20 @@ expect( 'A missing overview enrichment can still reject the complete worldwide tile.' ); expect( - gateway.includes("includeLayers: ['land']"), - 'The gateway does not isolate the continuous physical surface to the land mask.' + gateway.includes("const CONTINUOUS_SURFACE_LAYERS = Object.freeze(['land', 'landcover', 'depth'])"), + 'The gateway does not define one authoritative set of foundational surface layers.' +); +expect( + gateway.includes('includeLayers: CONTINUOUS_SURFACE_LAYERS'), + 'The gateway does not retain land, landcover, and bathymetry from the physical surface.' ); expect( - gateway.includes("excludeLayers: ['land']"), - 'The gateway can still merge duplicate land masks from the surface and basemap.' + gateway.includes('excludeLayers: CONTINUOUS_SURFACE_LAYERS'), + 'The overview or regional archives can still replace the foundational surface layers.' ); expect( - !gateway.includes("includeLayers: ['land', 'landcover', 'depth']"), - 'The gateway still overlays generalized landcover and bathymetry on regional detail.' + gateway.includes('CONTINUOUS_SURFACE_LAYERS.map'), + 'All foundational surface layers are not overscaled continuously through maximum zoom.' ); expect(manifestBuilder.includes('version: 2'), 'The server-only routing manifest is not version 2.'); expect( @@ -287,4 +303,4 @@ if (failures.length) { process.exit(1); } -console.log('Virtual worldwide tileset validated: one permanent source, safe polygon merging, clipped surface overscaling, one land mask, antimeridian routing, parent-tile retention, caching, and no browser-visible shards.'); +console.log('Virtual worldwide tileset validated: one permanent source, one continuous land/landcover/depth foundation at every zoom, safe polygon merging, clipped overscaling, antimeridian routing, parent-tile retention, caching, and no browser-visible shards.'); From 8bb9888d5bb1a727a76c4ab0cf64fa83d687a3f8 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 17:13:44 -0700 Subject: [PATCH 05/85] Validate base surface continuity across zoom thresholds --- scripts/capture-polygon-regression.mjs | 49 ++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/scripts/capture-polygon-regression.mjs b/scripts/capture-polygon-regression.mjs index b248acdb..f5a6e52c 100644 --- a/scripts/capture-polygon-regression.mjs +++ b/scripts/capture-polygon-regression.mjs @@ -53,13 +53,28 @@ try { { timeout: 90_000 } ); + const continuityZooms = [1.65, 2.43, 3.5, 5.5, 6.5]; const views = [ { name: 'north-america-z2', center: [-102, 36], zoom: 2.43 }, { name: 'central-pacific-z2', center: [175, 7], zoom: 2.43 }, { name: 'australia-z2', center: [135, -25], zoom: 2.43 }, { name: 'asia-pacific-z2', center: [118, 22], zoom: 2.43 }, { name: 'africa-europe-z2', center: [20, 20], zoom: 2.43 }, - { name: 'world-north-america-z1', center: [-100, 25], zoom: 1.65 } + { name: 'world-north-america-z1', center: [-100, 25], zoom: 1.65 }, + ...continuityZooms.map((zoom) => ({ + name: `amazon-z${String(zoom).replace('.', '-')}`, + center: [-60, -8], + zoom, + requiredSourceLayers: ['land', 'landcover'], + requiredRenderedLayers: ['land', 'landcover'] + })), + ...continuityZooms.map((zoom) => ({ + name: `pacific-depth-z${String(zoom).replace('.', '-')}`, + center: [-140, 0], + zoom, + requiredSourceLayers: ['depth'], + requiredRenderedLayers: ['depth'] + })) ]; const results = {}; @@ -87,14 +102,20 @@ try { const features = map .queryRenderedFeatures() .filter((feature) => feature.source === 'occumed-open'); - const sourceLayerCounts = {}; + const renderedSourceLayerCounts = {}; const styleLayerCounts = {}; for (const feature of features) { const sourceLayer = feature.sourceLayer || 'unknown'; const styleLayer = feature.layer?.id || 'unknown'; - sourceLayerCounts[sourceLayer] = (sourceLayerCounts[sourceLayer] || 0) + 1; + renderedSourceLayerCounts[sourceLayer] = (renderedSourceLayerCounts[sourceLayer] || 0) + 1; styleLayerCounts[styleLayer] = (styleLayerCounts[styleLayer] || 0) + 1; } + const sourceFeatureCounts = Object.fromEntries( + ['land', 'landcover', 'depth'].map((sourceLayer) => [ + sourceLayer, + map.querySourceFeatures('occumed-open', { sourceLayer }).length + ]) + ); return { center: map.getCenter().toArray(), zoom: map.getZoom(), @@ -102,7 +123,8 @@ try { sourceIsPermanent: !source?.url && JSON.stringify(source?.tiles || []) === JSON.stringify([expectedTemplate]), renderedFeatureCount: features.length, - sourceLayerCounts, + renderedSourceLayerCounts, + sourceFeatureCounts, styleLayerCounts }; }, expectedTemplate); @@ -113,6 +135,16 @@ try { if (diagnostics.renderedFeatureCount <= 0) { throw new Error(`${view.name} rendered no worldwide vector features.`); } + for (const sourceLayer of view.requiredSourceLayers || []) { + if ((diagnostics.sourceFeatureCounts[sourceLayer] || 0) <= 0) { + throw new Error(`${view.name} lost the ${sourceLayer} source layer at zoom ${view.zoom}.`); + } + } + for (const sourceLayer of view.requiredRenderedLayers || []) { + if ((diagnostics.renderedSourceLayerCounts[sourceLayer] || 0) <= 0) { + throw new Error(`${view.name} stopped rendering the ${sourceLayer} foundation at zoom ${view.zoom}.`); + } + } const screenshot = await page.screenshot({ path: path.join(outputDir, `${view.name}.png`), @@ -127,8 +159,9 @@ try { const report = { generatedAt: new Date().toISOString(), origin, - mode: 'rebuilt-overview-polygon-regression', + mode: 'rebuilt-overview-polygon-and-layer-continuity-regression', expectedTemplate, + continuityZooms, results, pageErrors, networkFailures, @@ -145,14 +178,16 @@ try { ); if (!report.passed) { - throw new Error(`Polygon regression validation failed: ${JSON.stringify({ + throw new Error(`Polygon and layer continuity validation failed: ${JSON.stringify({ pageErrors, networkFailures, externalVectorRequests: report.externalVectorRequests })}`); } - console.log(`Rendered ${views.length} rebuilt-overview globe views without browser or network errors.`); + console.log( + `Rendered ${views.length} rebuilt-overview views with continuous land, landcover, and depth across zoom thresholds.` + ); } finally { await browser.close(); } From 457e6f8bcfd98f5fc0de9cb39c81c1511fcc11b1 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 17:20:11 -0700 Subject: [PATCH 06/85] Validate every zoom level continuously --- scripts/validate-all-zoom-levels.mjs | 216 +++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 scripts/validate-all-zoom-levels.mjs diff --git a/scripts/validate-all-zoom-levels.mjs b/scripts/validate-all-zoom-levels.mjs new file mode 100644 index 00000000..3a9ad5fc --- /dev/null +++ b/scripts/validate-all-zoom-levels.mjs @@ -0,0 +1,216 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { chromium } from 'playwright'; + +const origin = (process.env.OCCUMED_PREVIEW_ORIGIN || 'http://127.0.0.1:4173').replace(/\/$/, ''); +const outputDir = path.resolve( + process.env.OCCUMED_PREVIEW_OUTPUT || 'visual-validation/all-zoom-levels' +); +await fs.mkdir(outputDir, { recursive: true }); + +const expectedTemplate = `${origin}/tiles/{z}/{x}/{y}.pbf`; +const browser = await chromium.launch({ headless: true }); + +try { + const context = await browser.newContext({ + viewport: { width: 1440, height: 1000 }, + deviceScaleFactor: 2, + colorScheme: 'dark' + }); + const page = await context.newPage(); + const pageErrors = []; + const networkFailures = []; + const externalVectorRequests = []; + + page.on('pageerror', (error) => pageErrors.push(error.message)); + page.on('request', (request) => { + const url = request.url(); + if (/\.pbf(?:$|\?)/i.test(url) && !url.startsWith(`${origin}/tiles/`)) { + externalVectorRequests.push(url); + } + }); + page.on('requestfailed', (request) => { + networkFailures.push({ + type: 'requestfailed', + url: request.url(), + error: request.failure()?.errorText || 'unknown request failure' + }); + }); + page.on('response', (response) => { + if (response.status() >= 400) { + networkFailures.push({ type: 'http', url: response.url(), status: response.status() }); + } + }); + + await page.goto(origin, { waitUntil: 'domcontentloaded', timeout: 90_000 }); + await page.waitForFunction( + () => globalThis.__OCCUMED_MAP__?.isStyleLoaded(), + null, + { timeout: 90_000 } + ); + + async function runSweep(name, center, startZoom, endZoom) { + await page.evaluate(({ center, startZoom }) => { + const map = globalThis.__OCCUMED_MAP__; + map.jumpTo({ center, zoom: startZoom, pitch: 0, bearing: 0 }); + map.triggerRepaint(); + }, { center, startZoom }); + + await page.waitForFunction( + () => { + const map = globalThis.__OCCUMED_MAP__; + return map?.isStyleLoaded() && + map.queryRenderedFeatures().some((feature) => feature.source === 'occumed-open'); + }, + null, + { timeout: 90_000 } + ); + + const result = await page.evaluate(async ({ + name, + center, + startZoom, + endZoom, + expectedTemplate + }) => { + const map = globalThis.__OCCUMED_MAP__; + const samples = []; + let lastSampleAt = -Infinity; + let sourceChanged = false; + const expectedSignature = JSON.stringify({ url: null, tiles: [expectedTemplate] }); + + const sample = (timestamp) => { + if (timestamp - lastSampleAt < 50) return; + lastSampleAt = timestamp; + const source = map.getStyle().sources?.['occumed-open'] || null; + const signature = JSON.stringify({ url: source?.url || null, tiles: source?.tiles || [] }); + sourceChanged ||= signature !== expectedSignature; + const rendered = map + .queryRenderedFeatures() + .filter((feature) => feature.source === 'occumed-open'); + const sourceLayers = {}; + for (const feature of rendered) { + const layer = feature.sourceLayer || 'unknown'; + sourceLayers[layer] = (sourceLayers[layer] || 0) + 1; + } + samples.push({ + timestamp, + zoom: map.getZoom(), + renderedFeatureCount: rendered.length, + sourceLayers, + sourceSignature: signature + }); + }; + + return await new Promise((resolve, reject) => { + const durationMs = 18_000; + const timeout = setTimeout(() => { + map.off('render', sample); + reject(new Error(`${name} full-range zoom sweep timed out.`)); + }, durationMs + 30_000); + + const finish = () => { + clearTimeout(timeout); + map.off('render', sample); + sample(performance.now()); + const zooms = samples.map((entry) => entry.zoom).sort((a, b) => a - b); + let maximumZoomGap = 0; + for (let index = 1; index < zooms.length; index += 1) { + maximumZoomGap = Math.max(maximumZoomGap, zooms[index] - zooms[index - 1]); + } + const blankSamples = samples.filter((entry) => entry.renderedFeatureCount === 0); + resolve({ + name, + center, + startZoom, + endZoom, + sampleCount: samples.length, + sourceChanged, + blankSampleCount: blankSamples.length, + minimumZoom: Math.min(...zooms), + maximumZoom: Math.max(...zooms), + maximumZoomGap, + minimumFeatureCount: Math.min(...samples.map((entry) => entry.renderedFeatureCount)), + samples + }); + }; + + map.on('render', sample); + map.once('moveend', finish); + map.easeTo({ + center, + zoom: endZoom, + pitch: 0, + bearing: 0, + duration: durationMs, + easing: (value) => value, + essential: true + }); + }); + }, { name, center, startZoom, endZoom, expectedTemplate }); + + await page.screenshot({ + path: path.join(outputDir, `${name}-final.png`), + fullPage: false + }); + return result; + } + + const sweeps = [ + await runSweep('amazon-all-zooms-in', [-62.5, -4], 0, 14), + await runSweep('hawaii-all-zooms-out', [-157.8583, 21.3069], 14, 0) + ]; + + const failedSweeps = sweeps.filter((sweep) => + sweep.sourceChanged || + sweep.blankSampleCount > 0 || + sweep.sampleCount < 100 || + sweep.minimumZoom > 0.1 || + sweep.maximumZoom < 13.9 || + sweep.maximumZoomGap > 0.25 + ); + + const report = { + generatedAt: new Date().toISOString(), + origin, + expectedTemplate, + sweeps, + pageErrors, + networkFailures, + externalVectorRequests: [...new Set(externalVectorRequests)], + passed: + failedSweeps.length === 0 && + pageErrors.length === 0 && + networkFailures.length === 0 && + externalVectorRequests.length === 0 + }; + + await fs.writeFile( + path.join(outputDir, 'all-zoom-levels-report.json'), + `${JSON.stringify(report, null, 2)}\n` + ); + + if (!report.passed) { + throw new Error(`All-zoom validation failed: ${JSON.stringify({ + failedSweeps: failedSweeps.map((sweep) => ({ + name: sweep.name, + sampleCount: sweep.sampleCount, + sourceChanged: sweep.sourceChanged, + blankSampleCount: sweep.blankSampleCount, + minimumZoom: sweep.minimumZoom, + maximumZoom: sweep.maximumZoom, + maximumZoomGap: sweep.maximumZoomGap, + minimumFeatureCount: sweep.minimumFeatureCount + })), + pageErrors, + networkFailures, + externalVectorRequests: report.externalVectorRequests + })}`); + } + + console.log( + `Validated the complete zoom 0–14 range in both directions with ${sweeps.reduce((sum, sweep) => sum + sweep.sampleCount, 0)} sampled frames and zero blank frames.` + ); +} finally { + await browser.close(); +} From 96402470cc25388569a8628dd05dec8612ed59a3 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 17:20:42 -0700 Subject: [PATCH 07/85] Run a full-range zoom sweep in CI --- .github/workflows/validate-continuous-zoom.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/validate-continuous-zoom.yml b/.github/workflows/validate-continuous-zoom.yml index 830375a3..a4b96728 100644 --- a/.github/workflows/validate-continuous-zoom.yml +++ b/.github/workflows/validate-continuous-zoom.yml @@ -115,12 +115,16 @@ jobs: OCCUMED_PREVIEW_OUTPUT=continuous-motion/results \ node scripts/validate-continuous-zoom.mjs + OCCUMED_PREVIEW_OUTPUT=continuous-motion/all-zoom-levels \ + node scripts/validate-all-zoom-levels.mjs + - uses: actions/upload-artifact@v4 if: always() with: name: continuous-zoom-${{ github.sha }} path: | continuous-motion/results + continuous-motion/all-zoom-levels continuous-motion/server.log dist/virtual-assets/occumed-world-overview.pmtiles if-no-files-found: error From 5d15102811e0173921c79256e8653975f3824f2a Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 17:51:35 -0700 Subject: [PATCH 08/85] Add tracked outward globe atmosphere bloom --- src/occumed-map.js | 77 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/src/occumed-map.js b/src/occumed-map.js index ab131692..249c1b2c 100644 --- a/src/occumed-map.js +++ b/src/occumed-map.js @@ -5,6 +5,10 @@ export const DEFAULT_STYLE_URL = '/style/occumed-open.json'; const MIN_RENDER_PIXEL_RATIO = 2; const MAX_RENDER_PIXEL_RATIO = 3; +const GLOBE_TILE_SIZE = 512; +const GLOBE_CIRCUMFERENCE = Math.PI * 2; +const BLOOM_FADE_START_ZOOM = 2.85; +const BLOOM_FADE_END_ZOOM = 4.25; export function resolveOccumedPixelRatio() { const deviceRatio = Number(globalThis.devicePixelRatio); @@ -12,6 +16,77 @@ export function resolveOccumedPixelRatio() { return Math.min(Math.max(deviceRatio, MIN_RENDER_PIXEL_RATIO), MAX_RENDER_PIXEL_RATIO); } +function clamp01(value) { + return Math.min(1, Math.max(0, value)); +} + +function resolveGlobeBloomOpacity(zoom) { + if (zoom <= BLOOM_FADE_START_ZOOM) return 1; + if (zoom >= BLOOM_FADE_END_ZOOM) return 0; + return 1 - clamp01( + (zoom - BLOOM_FADE_START_ZOOM) / + (BLOOM_FADE_END_ZOOM - BLOOM_FADE_START_ZOOM) + ); +} + +function resolveGlobeRadius(zoom) { + return (GLOBE_TILE_SIZE * (2 ** zoom)) / GLOBE_CIRCUMFERENCE; +} + +/** + * Adds a true outward atmosphere bloom around the globe limb. + * + * MapLibre's sky properties provide the crisp horizon rim, but increasing their + * blend values also brightens the visible hemisphere. This DOM halo tracks the + * rendered globe radius and adds only an exterior white-blue bloom, leaving the + * map surface neutral. It fades away before the projection reads as a regional + * map rather than a complete globe. + */ +export function installOccumedAtmosphereBloom(map) { + const canvasContainer = map.getCanvasContainer(); + const existing = canvasContainer.querySelector('.occumed-atmosphere-bloom'); + if (existing) return existing; + + const bloom = document.createElement('div'); + bloom.className = 'occumed-atmosphere-bloom'; + bloom.setAttribute('aria-hidden', 'true'); + canvasContainer.append(bloom); + + let animationFrame = null; + + const update = () => { + animationFrame = null; + const zoom = map.getZoom(); + const center = map.project(map.getCenter()); + const radius = resolveGlobeRadius(zoom); + const opacity = resolveGlobeBloomOpacity(zoom); + + bloom.style.setProperty('--occumed-globe-bloom-x', `${center.x.toFixed(2)}px`); + bloom.style.setProperty('--occumed-globe-bloom-y', `${center.y.toFixed(2)}px`); + bloom.style.setProperty('--occumed-globe-diameter', `${(radius * 2).toFixed(2)}px`); + bloom.style.setProperty('--occumed-globe-bloom-opacity', opacity.toFixed(3)); + bloom.hidden = opacity <= 0.001; + }; + + const scheduleUpdate = () => { + if (animationFrame !== null) return; + animationFrame = requestAnimationFrame(update); + }; + + const remove = () => { + if (animationFrame !== null) cancelAnimationFrame(animationFrame); + map.off('render', scheduleUpdate); + map.off('resize', scheduleUpdate); + bloom.remove(); + }; + + map.on('render', scheduleUpdate); + map.on('resize', scheduleUpdate); + map.once('remove', remove); + update(); + return bloom; +} + function resolvePublicOrigin(style, styleUrl) { const resolved = structuredClone(style); const styleOrigin = new URL(styleUrl, window.location.href).origin; @@ -104,6 +179,8 @@ export async function createOccumedMap({ ...mapOptions }); + installOccumedAtmosphereBloom(map); + if (controls) { map.addControl(new maplibregl.NavigationControl({ visualizePitch: true }), 'top-right'); } From 52e25c9336fb07d495d6348f431a4bc56948ebf2 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 17:52:09 -0700 Subject: [PATCH 09/85] Render a visible exterior atmosphere halo --- src/styles.css | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/styles.css b/src/styles.css index c7364f22..2e4487c9 100644 --- a/src/styles.css +++ b/src/styles.css @@ -57,6 +57,35 @@ body { background: transparent; } +.maplibregl-canvas-container { + isolation: isolate; +} + +.occumed-atmosphere-bloom { + position: absolute; + z-index: 1; + left: var(--occumed-globe-bloom-x, 50%); + top: var(--occumed-globe-bloom-y, 50%); + width: var(--occumed-globe-diameter, 0px); + height: var(--occumed-globe-diameter, 0px); + border-radius: 50%; + transform: translate3d(-50%, -50%, 0); + pointer-events: none; + opacity: var(--occumed-globe-bloom-opacity, 0); + background: radial-gradient( + circle at center, + transparent calc(100% - 2px), + rgba(245, 253, 255, 0.98) calc(100% - 1px), + rgba(245, 253, 255, 0.98) 100% + ); + filter: + drop-shadow(0 0 5px rgba(245, 253, 255, 0.96)) + drop-shadow(0 0 14px rgba(184, 230, 255, 0.74)) + drop-shadow(0 0 30px rgba(121, 188, 236, 0.34)); + mix-blend-mode: screen; + will-change: left, top, width, height, opacity; +} + .map-header { position: absolute; z-index: 2; From 947f1989afb9c313c829f293a80db62fbf6497d1 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 17:53:02 -0700 Subject: [PATCH 10/85] Guard the tracked atmosphere bloom --- scripts/check-photo-reference.mjs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/check-photo-reference.mjs b/scripts/check-photo-reference.mjs index cecde4f7..471f4820 100644 --- a/scripts/check-photo-reference.mjs +++ b/scripts/check-photo-reference.mjs @@ -4,7 +4,11 @@ import { fileURLToPath } from 'node:url'; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const runtimePath = path.join(root, 'public/style/occumed-open.json'); -const runtime = JSON.parse(await fs.readFile(runtimePath, 'utf8')); +const [runtime, mapHelper, appCss] = await Promise.all([ + fs.readFile(runtimePath, 'utf8').then(JSON.parse), + fs.readFile(path.join(root, 'src/occumed-map.js'), 'utf8'), + fs.readFile(path.join(root, 'src/styles.css'), 'utf8') +]); function assert(condition, message) { if (!condition) throw new Error(message); @@ -47,13 +51,19 @@ assert( ); assert( runtime.sky?.['fog-color'] === 'rgba(184, 230, 255, 0.14)', - 'The cool-blue outer atmosphere bloom is missing.' + 'The cool-blue edge atmosphere is missing.' ); assert(runtime.sky?.['fog-ground-blend'] === 0, 'Ground fog must remain disabled.'); -assert(runtime.sky?.['horizon-fog-blend'] === 0.08, 'The narrow edge bloom strength changed.'); +assert(runtime.sky?.['horizon-fog-blend'] === 0.08, 'The narrow edge atmosphere strength changed.'); assert(!runtime.light, 'Directional light must not be reintroduced.'); assert(!runtime.fog, 'Mapbox fog must not be copied into the MapLibre runtime.'); assert(runtime.metadata?.['occumed:atmosphere-edge-only'] === true, 'The edge-only atmosphere protection marker is missing.'); +assert(mapHelper.includes('installOccumedAtmosphereBloom(map)'), 'The tracked exterior globe bloom is not installed.'); +assert(mapHelper.includes('resolveGlobeRadius'), 'The atmosphere bloom no longer follows the rendered globe radius.'); +assert(mapHelper.includes('BLOOM_FADE_END_ZOOM'), 'The atmosphere bloom does not fade before regional detail.'); +assert(appCss.includes('.occumed-atmosphere-bloom'), 'The exterior atmosphere bloom styling is missing.'); +assert(appCss.includes('drop-shadow(0 0 30px'), 'The atmosphere is only a hard rim and no longer has a visible outer bloom.'); +assert(appCss.includes('mix-blend-mode: screen'), 'The white-blue atmosphere bloom no longer composites luminously.'); assert(layer('land')?.paint?.['background-color'] === '#79BCEC', 'The supplied Studio ocean blue changed.'); assert(layer('occumed-land-surface')?.paint?.['fill-color'] === '#E0E0D1', 'The supplied Studio land base changed.'); @@ -139,4 +149,4 @@ assert(!/mapbox:\/\//i.test(runtime.sprite || ''), 'Runtime sprite must not use assert(!/api\.mapbox\.com/i.test(runtime.glyphs || ''), 'Runtime glyphs must not use Mapbox.'); assert(runtime.metadata?.['occumed:mapbox-runtime-dependency'] === false, 'No-Mapbox dependency marker is missing.'); -console.log(`Reference guard passed: supplied Studio land and water, narrow edge-only atmosphere, high-DPI vector clarity, and ${allColors.size} distinct colors.`); +console.log(`Reference guard passed: supplied Studio land and water, narrow rim plus tracked outward atmosphere bloom, high-DPI vector clarity, and ${allColors.size} distinct colors.`); From 8444975150309ef2494827a1eefca5ac89281192 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 17:53:49 -0700 Subject: [PATCH 11/85] Prove the atmosphere bloom in globe renders --- scripts/capture-polygon-regression.mjs | 44 ++++++++++++++++++++------ 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/scripts/capture-polygon-regression.mjs b/scripts/capture-polygon-regression.mjs index f5a6e52c..d6b0fe06 100644 --- a/scripts/capture-polygon-regression.mjs +++ b/scripts/capture-polygon-regression.mjs @@ -55,12 +55,12 @@ try { const continuityZooms = [1.65, 2.43, 3.5, 5.5, 6.5]; const views = [ - { name: 'north-america-z2', center: [-102, 36], zoom: 2.43 }, - { name: 'central-pacific-z2', center: [175, 7], zoom: 2.43 }, - { name: 'australia-z2', center: [135, -25], zoom: 2.43 }, - { name: 'asia-pacific-z2', center: [118, 22], zoom: 2.43 }, - { name: 'africa-europe-z2', center: [20, 20], zoom: 2.43 }, - { name: 'world-north-america-z1', center: [-100, 25], zoom: 1.65 }, + { name: 'north-america-z2', center: [-102, 36], zoom: 2.43, requiresAtmosphereBloom: true }, + { name: 'central-pacific-z2', center: [175, 7], zoom: 2.43, requiresAtmosphereBloom: true }, + { name: 'australia-z2', center: [135, -25], zoom: 2.43, requiresAtmosphereBloom: true }, + { name: 'asia-pacific-z2', center: [118, 22], zoom: 2.43, requiresAtmosphereBloom: true }, + { name: 'africa-europe-z2', center: [20, 20], zoom: 2.43, requiresAtmosphereBloom: true }, + { name: 'world-north-america-z1', center: [-100, 25], zoom: 1.65, requiresAtmosphereBloom: true }, ...continuityZooms.map((zoom) => ({ name: `amazon-z${String(zoom).replace('.', '-')}`, center: [-60, -8], @@ -116,6 +116,9 @@ try { map.querySourceFeatures('occumed-open', { sourceLayer }).length ]) ); + const bloom = document.querySelector('.occumed-atmosphere-bloom'); + const bloomStyle = bloom ? getComputedStyle(bloom) : null; + const bloomRect = bloom?.getBoundingClientRect() || null; return { center: map.getCenter().toArray(), zoom: map.getZoom(), @@ -125,7 +128,16 @@ try { renderedFeatureCount: features.length, renderedSourceLayerCounts, sourceFeatureCounts, - styleLayerCounts + styleLayerCounts, + atmosphereBloom: { + exists: Boolean(bloom), + hidden: Boolean(bloom?.hidden), + opacity: Number(bloomStyle?.opacity || 0), + filter: bloomStyle?.filter || 'none', + mixBlendMode: bloomStyle?.mixBlendMode || 'normal', + width: bloomRect?.width || 0, + height: bloomRect?.height || 0 + } }; }, expectedTemplate); @@ -145,6 +157,18 @@ try { throw new Error(`${view.name} stopped rendering the ${sourceLayer} foundation at zoom ${view.zoom}.`); } } + if (view.requiresAtmosphereBloom) { + const bloom = diagnostics.atmosphereBloom; + if (!bloom.exists || bloom.hidden || bloom.opacity < 0.95) { + throw new Error(`${view.name} does not show the full-strength globe atmosphere bloom.`); + } + if (bloom.filter === 'none' || bloom.mixBlendMode !== 'screen') { + throw new Error(`${view.name} has a hard rim instead of the luminous white-blue bloom.`); + } + if (bloom.width < 150 || Math.abs(bloom.width - bloom.height) > 1) { + throw new Error(`${view.name} atmosphere bloom does not track the rendered globe.`); + } + } const screenshot = await page.screenshot({ path: path.join(outputDir, `${view.name}.png`), @@ -159,7 +183,7 @@ try { const report = { generatedAt: new Date().toISOString(), origin, - mode: 'rebuilt-overview-polygon-and-layer-continuity-regression', + mode: 'rebuilt-overview-polygon-layer-continuity-and-atmosphere-regression', expectedTemplate, continuityZooms, results, @@ -178,7 +202,7 @@ try { ); if (!report.passed) { - throw new Error(`Polygon and layer continuity validation failed: ${JSON.stringify({ + throw new Error(`Polygon, layer continuity, and atmosphere validation failed: ${JSON.stringify({ pageErrors, networkFailures, externalVectorRequests: report.externalVectorRequests @@ -186,7 +210,7 @@ try { } console.log( - `Rendered ${views.length} rebuilt-overview views with continuous land, landcover, and depth across zoom thresholds.` + `Rendered ${views.length} rebuilt-overview views with continuous physical layers and a tracked exterior atmosphere bloom.` ); } finally { await browser.close(); From 3b45fcc7b46b277a65d045f2dd21164ecf7e3c5a Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:08:23 -0700 Subject: [PATCH 12/85] Harden PMTiles range fetching and circuit breaking --- src/server/pmtiles-source.js | 164 +++++++++++++++++++++++++++++++---- 1 file changed, 149 insertions(+), 15 deletions(-) diff --git a/src/server/pmtiles-source.js b/src/server/pmtiles-source.js index 6302ddd1..dcf7894e 100644 --- a/src/server/pmtiles-source.js +++ b/src/server/pmtiles-source.js @@ -1,32 +1,146 @@ import { FetchSource } from 'pmtiles'; -const DEFAULT_ATTEMPTS = 3; -const DEFAULT_TIMEOUT_MS = 12_000; +const DEFAULT_ATTEMPTS = 4; +const DEFAULT_TIMEOUT_MS = 10_000; +const DEFAULT_BASE_DELAY_MS = 120; +const DEFAULT_MAX_DELAY_MS = 1_500; +const DEFAULT_MAX_RANGE_BYTES = 16 * 1024 * 1024; +const DEFAULT_CIRCUIT_FAILURES = 6; +const DEFAULT_CIRCUIT_COOLDOWN_MS = 15_000; -function delay(milliseconds) { - return new Promise((resolve) => setTimeout(resolve, milliseconds)); +function boundedInteger(value, fallback, minimum, maximum) { + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed >= minimum && parsed <= maximum + ? parsed + : fallback; +} + +function abortError(signal) { + if (signal?.reason instanceof Error) return signal.reason; + const error = new Error('The PMTiles range request was aborted.'); + error.name = 'AbortError'; + return error; +} + +function delay(milliseconds, signal) { + if (signal?.aborted) return Promise.reject(abortError(signal)); + return new Promise((resolve, reject) => { + const timer = setTimeout(resolve, milliseconds); + const onAbort = () => { + clearTimeout(timer); + reject(abortError(signal)); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + timer.unref?.(); + }).finally(() => signal?.removeEventListener?.('abort', () => {})); +} + +function errorStatus(error) { + const direct = Number(error?.status || error?.statusCode); + if (Number.isSafeInteger(direct)) return direct; + const match = /(?:http|status|response)\D+(\d{3})/i.exec(String(error?.message || '')); + return match ? Number(match[1]) : null; +} + +export function isRetryableUpstreamError(error) { + if (!error) return true; + if (error.name === 'AbortError') return true; + const status = errorStatus(error); + if (status === null) return true; + if ([408, 409, 425, 429].includes(status)) return true; + return status >= 500; +} + +function validateSourceUrl(value) { + const url = new URL(String(value || '')); + if (!['http:', 'https:'].includes(url.protocol)) { + throw new TypeError('PMTiles sources must use HTTP or HTTPS.'); + } + return url.href; } /** - * Adds bounded retries and per-range timeouts around PMTiles' HTTP source. - * GitHub release storage redirects every byte-range request, so a transient - * redirect or object-storage failure must not turn into a blank map tile. + * Adds bounded retries, per-range timeouts, and a small circuit breaker around + * PMTiles HTTP reads. GitHub release storage redirects every byte-range request, + * so transient CDN failures must not become blank map tiles or unbounded retry + * storms. Permanent 4xx responses fail immediately. */ export class RetryingFetchSource { constructor(url, { attempts = DEFAULT_ATTEMPTS, - timeoutMs = DEFAULT_TIMEOUT_MS + timeoutMs = DEFAULT_TIMEOUT_MS, + baseDelayMs = DEFAULT_BASE_DELAY_MS, + maxDelayMs = DEFAULT_MAX_DELAY_MS, + maxRangeBytes = DEFAULT_MAX_RANGE_BYTES, + circuitFailures = DEFAULT_CIRCUIT_FAILURES, + circuitCooldownMs = DEFAULT_CIRCUIT_COOLDOWN_MS, + source = null, + now = () => Date.now(), + random = Math.random } = {}) { - this.source = new FetchSource(url); - this.attempts = attempts; - this.timeoutMs = timeoutMs; + const sourceUrl = validateSourceUrl(url); + this.source = source || new FetchSource(sourceUrl); + this.attempts = boundedInteger(attempts, DEFAULT_ATTEMPTS, 1, 8); + this.timeoutMs = boundedInteger(timeoutMs, DEFAULT_TIMEOUT_MS, 250, 60_000); + this.baseDelayMs = boundedInteger(baseDelayMs, DEFAULT_BASE_DELAY_MS, 0, 5_000); + this.maxDelayMs = boundedInteger(maxDelayMs, DEFAULT_MAX_DELAY_MS, this.baseDelayMs, 30_000); + this.maxRangeBytes = boundedInteger(maxRangeBytes, DEFAULT_MAX_RANGE_BYTES, 1_024, 64 * 1024 * 1024); + this.circuitFailures = boundedInteger(circuitFailures, DEFAULT_CIRCUIT_FAILURES, 2, 50); + this.circuitCooldownMs = boundedInteger(circuitCooldownMs, DEFAULT_CIRCUIT_COOLDOWN_MS, 1_000, 300_000); + this.now = now; + this.random = random; + this.consecutiveFailures = 0; + this.circuitOpenUntil = 0; + this.totalRequests = 0; + this.totalRetries = 0; + this.totalFailures = 0; } getKey() { return this.source.getKey(); } + getHealthSnapshot() { + return { + key: this.getKey(), + totalRequests: this.totalRequests, + totalRetries: this.totalRetries, + totalFailures: this.totalFailures, + consecutiveFailures: this.consecutiveFailures, + circuitOpen: this.now() < this.circuitOpenUntil, + circuitOpenUntil: this.circuitOpenUntil || null + }; + } + + recordSuccess() { + this.consecutiveFailures = 0; + this.circuitOpenUntil = 0; + } + + recordFailure() { + this.totalFailures += 1; + this.consecutiveFailures += 1; + if (this.consecutiveFailures >= this.circuitFailures) { + this.circuitOpenUntil = this.now() + this.circuitCooldownMs; + } + } + async getBytes(offset, length, passedSignal, etag) { + if (!Number.isSafeInteger(offset) || offset < 0) { + throw new RangeError('PMTiles byte offsets must be non-negative safe integers.'); + } + if (!Number.isSafeInteger(length) || length <= 0 || length > this.maxRangeBytes) { + throw new RangeError(`PMTiles range length must be between 1 and ${this.maxRangeBytes} bytes.`); + } + if (passedSignal?.aborted) throw abortError(passedSignal); + if (this.now() < this.circuitOpenUntil) { + const error = new Error(`PMTiles upstream circuit is open for ${this.getKey()}.`); + error.code = 'OCCUMED_UPSTREAM_CIRCUIT_OPEN'; + error.retryAfterMs = this.circuitOpenUntil - this.now(); + throw error; + } + + this.totalRequests += 1; let lastError; for (let attempt = 1; attempt <= this.attempts; attempt += 1) { const timeoutSignal = AbortSignal.timeout(this.timeoutMs); @@ -34,13 +148,33 @@ export class RetryingFetchSource { ? AbortSignal.any([passedSignal, timeoutSignal]) : timeoutSignal; try { - return await this.source.getBytes(offset, length, signal, etag); + const result = await this.source.getBytes(offset, length, signal, etag); + const byteLength = Number(result?.data?.byteLength); + if (!Number.isSafeInteger(byteLength) || byteLength <= 0 || byteLength > this.maxRangeBytes) { + const error = new Error(`PMTiles upstream returned an invalid ${byteLength || 0}-byte range.`); + error.code = 'OCCUMED_INVALID_RANGE_RESPONSE'; + throw error; + } + this.recordSuccess(); + return result; } catch (error) { lastError = error; - if (passedSignal?.aborted || attempt === this.attempts) throw error; - await delay(150 * attempt); + if (passedSignal?.aborted) throw abortError(passedSignal); + const retryable = isRetryableUpstreamError(error); + if (!retryable || attempt === this.attempts) break; + this.totalRetries += 1; + const exponential = Math.min(this.maxDelayMs, this.baseDelayMs * (2 ** (attempt - 1))); + const jitter = 0.75 + (Math.max(0, Math.min(1, Number(this.random()) || 0)) * 0.5); + await delay(Math.round(exponential * jitter), passedSignal); } } - throw lastError; + + this.recordFailure(); + const wrapped = new Error(`PMTiles range request failed for ${this.getKey()} after bounded retries.`, { + cause: lastError + }); + wrapped.code = lastError?.code || 'OCCUMED_UPSTREAM_RANGE_FAILED'; + wrapped.status = errorStatus(lastError); + throw wrapped; } } From 63f629f1ccf2e194afb1419263290198a6dec4ac Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:11:08 -0700 Subject: [PATCH 13/85] Harden worldwide tile gateway failure handling --- src/server/world-tile-gateway.js | 547 +++++++++++++++++++++++++------ 1 file changed, 454 insertions(+), 93 deletions(-) diff --git a/src/server/world-tile-gateway.js b/src/server/world-tile-gateway.js index 178e69d8..6ee9f4eb 100644 --- a/src/server/world-tile-gateway.js +++ b/src/server/world-tile-gateway.js @@ -5,7 +5,10 @@ import { overscaleVectorLayer } from './mvt.js'; import { RetryingFetchSource } from './pmtiles-source.js'; -import { WorldTileRoutingIndex } from './world-tile-routing.js'; +import { + normalizeTileCoordinates, + WorldTileRoutingIndex +} from './world-tile-routing.js'; const DEFAULT_OVERVIEW_ASSET = 'occumed-world-overview.pmtiles'; const DEFAULT_SURFACE_ASSET = 'occumed-world-surface.pmtiles'; @@ -14,40 +17,168 @@ const DEFAULT_SURFACE_MAX_ZOOM = 10; const DEFAULT_ROUTING_ZOOM = 6; const DEFAULT_MAX_ZOOM = 16; const DEFAULT_CACHE_BYTES = 128 * 1024 * 1024; +const DEFAULT_CACHE_TTL_MS = 60 * 60 * 1_000; +const DEFAULT_CACHE_STALE_MS = 24 * 60 * 60 * 1_000; +const DEFAULT_CACHE_ENTRIES = 8_192; +const DEFAULT_MANIFEST_TIMEOUT_MS = 10_000; +const DEFAULT_MANIFEST_TTL_MS = 5 * 60 * 1_000; +const DEFAULT_MANIFEST_STALE_MS = 24 * 60 * 60 * 1_000; +const DEFAULT_MAX_MANIFEST_BYTES = 4 * 1024 * 1024; +const DEFAULT_MAX_REGIONS = 2_000; +const DEFAULT_MAX_TILE_FANOUT = 64; +const DEFAULT_MAX_INFLIGHT_TILES = 128; +const DEFAULT_MAX_ARCHIVE_READS = 32; +const DEFAULT_MAX_ARCHIVE_QUEUE = 256; +const DEFAULT_MAX_UPSTREAM_TILE_BYTES = 16 * 1024 * 1024; +const DEFAULT_MAX_RESOLVED_TILE_BYTES = 24 * 1024 * 1024; const CONTINUOUS_SURFACE_LAYERS = Object.freeze(['land', 'landcover', 'depth']); -class MemoryTileCache { - constructor(maxBytes = DEFAULT_CACHE_BYTES) { - this.maxBytes = maxBytes; +function boundedInteger(value, fallback, minimum, maximum) { + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed >= minimum && parsed <= maximum + ? parsed + : fallback; +} + +function validateHttpUrl(value, label) { + const url = new URL(String(value || '')); + if (!['http:', 'https:'].includes(url.protocol)) { + throw new TypeError(`${label} must use HTTP or HTTPS.`); + } + return url.href; +} + +export class GatewayOverloadedError extends Error { + constructor(message) { + super(message); + this.name = 'GatewayOverloadedError'; + this.code = 'OCCUMED_GATEWAY_OVERLOADED'; + this.statusCode = 503; + } +} + +class AsyncLimiter { + constructor(limit, maxQueue) { + this.limit = limit; + this.maxQueue = maxQueue; + this.active = 0; + this.queue = []; + } + + async run(task) { + if (this.active >= this.limit) { + if (this.queue.length >= this.maxQueue) { + throw new GatewayOverloadedError('The PMTiles archive-read queue is full.'); + } + await new Promise((resolve) => this.queue.push(resolve)); + } + + this.active += 1; + try { + return await task(); + } finally { + this.active -= 1; + this.queue.shift()?.(); + } + } + + snapshot() { + return { active: this.active, queued: this.queue.length, limit: this.limit }; + } +} + +export class MemoryTileCache { + constructor(maxBytes = DEFAULT_CACHE_BYTES, { + ttlMs = DEFAULT_CACHE_TTL_MS, + staleMs = DEFAULT_CACHE_STALE_MS, + maxEntries = DEFAULT_CACHE_ENTRIES, + now = () => Date.now() + } = {}) { + this.maxBytes = boundedInteger(maxBytes, DEFAULT_CACHE_BYTES, 1_024 * 1_024, 2 * 1_024 * 1_024 * 1_024); + this.ttlMs = boundedInteger(ttlMs, DEFAULT_CACHE_TTL_MS, 1_000, 7 * 24 * 60 * 60 * 1_000); + this.staleMs = boundedInteger(staleMs, DEFAULT_CACHE_STALE_MS, this.ttlMs, 30 * 24 * 60 * 60 * 1_000); + this.maxEntries = boundedInteger(maxEntries, DEFAULT_CACHE_ENTRIES, 16, 100_000); + this.now = now; this.size = 0; this.entries = new Map(); + this.hits = 0; + this.staleHits = 0; + this.misses = 0; + this.evictions = 0; + } + + touch(key, entry) { + this.entries.delete(key); + this.entries.set(key, entry); } - get(key) { + getFresh(key) { + const entry = this.entries.get(key); + if (!entry) { + this.misses += 1; + return null; + } + if (this.now() > entry.freshUntil) return null; + this.hits += 1; + this.touch(key, entry); + return Buffer.from(entry.data); + } + + getStale(key) { const entry = this.entries.get(key); if (!entry) return null; + if (this.now() > entry.staleUntil) { + this.delete(key); + return null; + } + this.staleHits += 1; + this.touch(key, entry); + return Buffer.from(entry.data); + } + + delete(key) { + const entry = this.entries.get(key); + if (!entry) return false; this.entries.delete(key); - this.entries.set(key, entry); - return entry; + this.size -= entry.data.byteLength; + return true; } set(key, value) { const data = Buffer.from(value); - const existing = this.entries.get(key); - if (existing) { - this.size -= existing.byteLength; - this.entries.delete(key); - } + if (!data.byteLength || data.byteLength > this.maxBytes) return data; - this.entries.set(key, data); + this.delete(key); + const createdAt = this.now(); + this.entries.set(key, { + data, + createdAt, + freshUntil: createdAt + this.ttlMs, + staleUntil: createdAt + this.staleMs + }); this.size += data.byteLength; - while (this.size > this.maxBytes && this.entries.size > 1) { + + while ( + (this.size > this.maxBytes || this.entries.size > this.maxEntries) && + this.entries.size > 1 + ) { const oldestKey = this.entries.keys().next().value; - const oldest = this.entries.get(oldestKey); - this.entries.delete(oldestKey); - this.size -= oldest.byteLength; + this.delete(oldestKey); + this.evictions += 1; } - return data; + return Buffer.from(data); + } + + snapshot() { + return { + entries: this.entries.size, + bytes: this.size, + maxBytes: this.maxBytes, + hits: this.hits, + staleHits: this.staleHits, + misses: this.misses, + evictions: this.evictions + }; } } @@ -59,13 +190,39 @@ function requireAssetName(value, label) { return asset; } -function parseManifest(manifest) { - if (Number(manifest?.version) !== 2) { +function requireBounds(value, label) { + if (!Array.isArray(value) || value.length !== 4) { + throw new Error(`The worldwide manifest has invalid bounds for ${label}.`); + } + const bounds = value.map(Number); + const [west, south, east, north] = bounds; + if ( + !bounds.every(Number.isFinite) || + west < -180 || west > 180 || east < -180 || east > 180 || + south < -90 || south > 90 || north < -90 || north > 90 || + south >= north + ) { + throw new Error(`The worldwide manifest has invalid bounds for ${label}.`); + } + return bounds; +} + +export function parseManifest(manifest, { maxRegions = DEFAULT_MAX_REGIONS } = {}) { + if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) { + throw new Error('The worldwide manifest must be a JSON object.'); + } + if (Number(manifest.version) !== 2) { throw new Error('The worldwide gateway requires the server-only version 2 manifest.'); } - if (!manifest || !Array.isArray(manifest.regions) || !manifest.regions.length) { + if (!Array.isArray(manifest.regions) || !manifest.regions.length) { throw new Error('The worldwide manifest does not contain regional storage shards.'); } + if (manifest.regions.length > maxRegions) { + throw new Error(`The worldwide manifest exceeds the ${maxRegions}-region safety limit.`); + } + if (Number(manifest.missingRegionCount || 0) !== 0) { + throw new Error('The worldwide manifest still reports missing regional shards.'); + } const virtual = manifest.virtualTiles || {}; if (virtual.endpoint !== '/tiles/{z}/{x}/{y}.pbf') { @@ -94,28 +251,56 @@ function parseManifest(manifest) { throw new Error(`The worldwide manifest has an invalid ${label}.`); } } - if (overviewMaxZoom >= routingZoom) { - throw new Error('The worldwide overview must end before regional routing begins.'); + if (overviewMaxZoom + 1 !== routingZoom) { + throw new Error('The worldwide overview and regional routing zooms must be exactly adjacent.'); } if (surfaceMaxZoom > maxZoom) { throw new Error('The worldwide surface archive exceeds the virtual tileset maximum zoom.'); } - const regions = manifest.regions.map((region) => ({ - ...region, - asset: requireAssetName(region.asset, `regional (${region.id || 'unknown'})`) - })); + const ids = new Set(); + const assets = new Set([overviewAsset, surfaceAsset]); + const regions = manifest.regions.map((region, index) => { + const id = String(region?.id || '').trim(); + if (!id || id.length > 200) { + throw new Error(`The worldwide manifest has an invalid regional ID at index ${index}.`); + } + if (ids.has(id)) throw new Error(`The worldwide manifest repeats regional ID ${id}.`); + ids.add(id); + + const asset = requireAssetName(region.asset, `regional (${id})`); + if (assets.has(asset)) throw new Error(`The worldwide manifest repeats asset ${asset}.`); + assets.add(asset); + + return { + ...region, + id, + asset, + bounds: requireBounds(region.bounds, id) + }; + }); + + for (const [label, value] of Object.entries({ + plannedRegionCount: manifest.plannedRegionCount, + availableRegionCount: manifest.availableRegionCount + })) { + if (value !== undefined && Number(value) !== regions.length) { + throw new Error(`The worldwide manifest ${label} does not match its region inventory.`); + } + } return { ...manifest, regions, virtualTiles: { + ...virtual, overviewAsset, surfaceAsset, overviewMaxZoom, surfaceMaxZoom, routingZoom, - maxZoom + maxZoom, + surfaceLayers: [...CONTINUOUS_SURFACE_LAYERS] } }; } @@ -126,43 +311,125 @@ export class WorldTileGateway { releaseAssetUrl, fetchImpl = fetch, cacheBytes = Number(process.env.OCCUMED_TILE_CACHE_MAX_BYTES || DEFAULT_CACHE_BYTES), - overviewUrl = process.env.OCCUMED_WORLD_OVERVIEW_URL?.trim() || '' + cacheTtlMs = DEFAULT_CACHE_TTL_MS, + cacheStaleMs = DEFAULT_CACHE_STALE_MS, + manifestTimeoutMs = DEFAULT_MANIFEST_TIMEOUT_MS, + manifestTtlMs = DEFAULT_MANIFEST_TTL_MS, + manifestStaleMs = DEFAULT_MANIFEST_STALE_MS, + maxManifestBytes = DEFAULT_MAX_MANIFEST_BYTES, + maxRegions = DEFAULT_MAX_REGIONS, + maxTileFanout = DEFAULT_MAX_TILE_FANOUT, + maxInflightTiles = DEFAULT_MAX_INFLIGHT_TILES, + maxUpstreamTileBytes = DEFAULT_MAX_UPSTREAM_TILE_BYTES, + maxResolvedTileBytes = DEFAULT_MAX_RESOLVED_TILE_BYTES, + archiveReadConcurrency = DEFAULT_MAX_ARCHIVE_READS, + archiveReadQueue = DEFAULT_MAX_ARCHIVE_QUEUE, + overviewUrl = process.env.OCCUMED_WORLD_OVERVIEW_URL?.trim() || '', + now = () => Date.now() }) { - this.manifestUrl = manifestUrl; + this.manifestUrl = validateHttpUrl(manifestUrl, 'The worldwide manifest URL'); + if (typeof releaseAssetUrl !== 'function') { + throw new TypeError('WorldTileGateway requires a releaseAssetUrl function.'); + } this.releaseAssetUrl = releaseAssetUrl; this.fetchImpl = fetchImpl; - this.overviewUrl = overviewUrl; - this.tileCache = new MemoryTileCache(cacheBytes); + this.overviewUrl = overviewUrl ? validateHttpUrl(overviewUrl, 'The overview URL') : ''; + this.now = now; + this.manifestTimeoutMs = boundedInteger(manifestTimeoutMs, DEFAULT_MANIFEST_TIMEOUT_MS, 500, 60_000); + this.manifestTtlMs = boundedInteger(manifestTtlMs, DEFAULT_MANIFEST_TTL_MS, 1_000, 24 * 60 * 60 * 1_000); + this.manifestStaleMs = boundedInteger(manifestStaleMs, DEFAULT_MANIFEST_STALE_MS, this.manifestTtlMs, 30 * 24 * 60 * 60 * 1_000); + this.maxManifestBytes = boundedInteger(maxManifestBytes, DEFAULT_MAX_MANIFEST_BYTES, 16_384, 32 * 1024 * 1024); + this.maxRegions = boundedInteger(maxRegions, DEFAULT_MAX_REGIONS, 1, 10_000); + this.maxTileFanout = boundedInteger(maxTileFanout, DEFAULT_MAX_TILE_FANOUT, 1, 256); + this.maxInflightTiles = boundedInteger(maxInflightTiles, DEFAULT_MAX_INFLIGHT_TILES, 4, 2_048); + this.maxUpstreamTileBytes = boundedInteger(maxUpstreamTileBytes, DEFAULT_MAX_UPSTREAM_TILE_BYTES, 1_024, 64 * 1024 * 1024); + this.maxResolvedTileBytes = boundedInteger(maxResolvedTileBytes, DEFAULT_MAX_RESOLVED_TILE_BYTES, 1_024, 96 * 1024 * 1024); + this.tileCache = new MemoryTileCache(cacheBytes, { + ttlMs: cacheTtlMs, + staleMs: cacheStaleMs, + now + }); this.directoryCache = new SharedPromiseCache(4096); + this.archiveReadLimiter = new AsyncLimiter( + boundedInteger(archiveReadConcurrency, DEFAULT_MAX_ARCHIVE_READS, 1, 256), + boundedInteger(archiveReadQueue, DEFAULT_MAX_ARCHIVE_QUEUE, 1, 4_096) + ); this.archives = new Map(); + this.sources = new Map(); this.inflight = new Map(); this.manifestPromise = null; + this.manifestState = null; + this.metrics = { + resolved: 0, + failed: 0, + staleServed: 0, + overloads: 0, + missingSurface: 0 + }; } - async loadManifest() { - if (!this.manifestPromise) { - this.manifestPromise = this.fetchImpl(this.manifestUrl, { - headers: { - Accept: 'application/json', - 'User-Agent': 'Occu-Med-Map/virtual-world-tiles' - }, - redirect: 'follow' - }).then(async (response) => { - if (!response.ok) { - throw new Error(`Worldwide manifest upstream returned ${response.status}.`); - } - const manifest = parseManifest(await response.json()); - return { - ...manifest, - routingIndex: new WorldTileRoutingIndex(manifest.regions, { - routingZoom: manifest.virtualTiles.routingZoom - }) + async fetchManifest() { + const response = await this.fetchImpl(this.manifestUrl, { + headers: { + Accept: 'application/json', + 'User-Agent': 'Occu-Med-Map/virtual-world-tiles' + }, + redirect: 'follow', + signal: AbortSignal.timeout(this.manifestTimeoutMs) + }); + if (!response.ok) { + throw new Error(`Worldwide manifest upstream returned ${response.status}.`); + } + const declaredLength = Number(response.headers?.get?.('content-length')); + if (Number.isFinite(declaredLength) && declaredLength > this.maxManifestBytes) { + throw new Error('The worldwide manifest exceeds its maximum allowed size.'); + } + const text = await response.text(); + if (Buffer.byteLength(text) > this.maxManifestBytes) { + throw new Error('The worldwide manifest exceeds its maximum allowed size.'); + } + let document; + try { + document = JSON.parse(text); + } catch (error) { + throw new Error('The worldwide manifest is not valid JSON.', { cause: error }); + } + const manifest = parseManifest(document, { maxRegions: this.maxRegions }); + return { + ...manifest, + routingIndex: new WorldTileRoutingIndex(manifest.regions, { + routingZoom: manifest.virtualTiles.routingZoom, + maxCellFanout: this.maxTileFanout + }) + }; + } + + async loadManifest({ force = false } = {}) { + const timestamp = this.now(); + if (!force && this.manifestState && timestamp <= this.manifestState.freshUntil) { + return this.manifestState.manifest; + } + if (this.manifestPromise) return this.manifestPromise; + + const previous = this.manifestState; + this.manifestPromise = this.fetchManifest() + .then((manifest) => { + const loadedAt = this.now(); + this.manifestState = { + manifest, + loadedAt, + freshUntil: loadedAt + this.manifestTtlMs, + staleUntil: loadedAt + this.manifestStaleMs }; - }).catch((error) => { - this.manifestPromise = null; + return manifest; + }) + .catch((error) => { + if (previous && this.now() <= previous.staleUntil) return previous.manifest; throw error; + }) + .finally(() => { + this.manifestPromise = null; }); - } return this.manifestPromise; } @@ -171,19 +438,30 @@ export class WorldTileGateway { if (!archive) { const sourceUrl = asset === DEFAULT_OVERVIEW_ASSET && this.overviewUrl ? this.overviewUrl - : this.releaseAssetUrl(asset); - archive = new PMTiles( - new RetryingFetchSource(sourceUrl), - this.directoryCache - ); + : validateHttpUrl(this.releaseAssetUrl(asset), `The release URL for ${asset}`); + const source = new RetryingFetchSource(sourceUrl); + archive = new PMTiles(source, this.directoryCache); + this.sources.set(asset, source); this.archives.set(asset, archive); } return archive; } async readArchiveTile(asset, zoom, x, y) { - const result = await this.archive(asset).getZxy(zoom, x, y); - return result?.data ? Buffer.from(result.data) : null; + return this.archiveReadLimiter.run(async () => { + let result; + try { + result = await this.archive(asset).getZxy(zoom, x, y); + } catch (error) { + throw new Error(`Unable to read ${asset} tile ${zoom}/${x}/${y}.`, { cause: error }); + } + if (!result?.data) return null; + const payload = Buffer.from(result.data); + if (!payload.byteLength || payload.byteLength > this.maxUpstreamTileBytes) { + throw new Error(`${asset} returned an invalid ${payload.byteLength}-byte vector tile.`); + } + return payload; + }); } async readSurfaceTile(manifest, zoom, x, y) { @@ -209,16 +487,17 @@ export class WorldTileGateway { ); if (!payload) return EMPTY_MVT; - const overscaledLayers = CONTINUOUS_SURFACE_LAYERS.map((layerName) => - overscaleVectorLayer(payload, { - layerName, - sourceZoom: surfaceMaxZoom, - targetZoom: zoom, - targetX: x, - targetY: y - }) + return mergeVectorTiles( + CONTINUOUS_SURFACE_LAYERS.map((layerName) => + overscaleVectorLayer(payload, { + layerName, + sourceZoom: surfaceMaxZoom, + targetZoom: zoom, + targetX: x, + targetY: y + }) + ) ); - return mergeVectorTiles(overscaledLayers); } async readBasemapTile(manifest, zoom, x, y) { @@ -233,43 +512,125 @@ export class WorldTileGateway { const regions = manifest.routingIndex.regionsForTile(zoom, x, y); if (!regions.length) return EMPTY_MVT; + if (regions.length > this.maxTileFanout) { + throw new Error(`Tile ${zoom}/${x}/${y} exceeds the ${this.maxTileFanout}-shard fan-out limit.`); + } - const payloads = await Promise.all( + const settled = await Promise.allSettled( regions.map((region) => this.readArchiveTile(region.asset, zoom, x, y)) ); - return mergeVectorTiles(payloads); + const failures = settled + .map((result, index) => ({ result, region: regions[index] })) + .filter(({ result }) => result.status === 'rejected'); + if (failures.length) { + throw new AggregateError( + failures.map(({ result }) => result.reason), + `Tile ${zoom}/${x}/${y} failed to read ${failures.length} of ${regions.length} required shards.` + ); + } + + const payloads = settled + .filter((result) => result.status === 'fulfilled' && result.value) + .map((result) => result.value); + return payloads.length ? mergeVectorTiles(payloads) : EMPTY_MVT; + } + + async buildTile(manifest, zoom, x, y) { + if (zoom > manifest.virtualTiles.maxZoom) { + throw new RangeError(`Tile zoom ${zoom} exceeds the worldwide maximum zoom.`); + } + const [surface, basemap] = await Promise.all([ + this.readSurfaceTile(manifest, zoom, x, y), + this.readBasemapTile(manifest, zoom, x, y) + ]); + + if (Buffer.from(surface).equals(EMPTY_MVT)) { + this.metrics.missingSurface += 1; + const error = new Error(`The authoritative physical surface is missing tile ${zoom}/${x}/${y}.`); + error.code = 'OCCUMED_SURFACE_TILE_MISSING'; + throw error; + } + + const cartography = mergeVectorTiles([basemap], { + excludeLayers: CONTINUOUS_SURFACE_LAYERS + }); + const tile = mergeVectorTiles([surface, cartography], { coordinateScale: 128 }); + if (!tile.byteLength || tile.byteLength > this.maxResolvedTileBytes) { + throw new Error(`Resolved tile ${zoom}/${x}/${y} has unsafe size ${tile.byteLength}.`); + } + return tile; } async resolveTile(zoom, x, y) { - const key = `${zoom}/${x}/${y}`; - const cached = this.tileCache.get(key); - if (cached) return cached; + const coordinates = normalizeTileCoordinates(zoom, x, y, 22); + if (!coordinates) throw new RangeError('Invalid worldwide tile coordinates.'); + const key = `${coordinates.z}/${coordinates.x}/${coordinates.y}`; + const fresh = this.tileCache.getFresh(key); + if (fresh) return fresh; + const stale = this.tileCache.getStale(key); if (this.inflight.has(key)) return this.inflight.get(key); + if (this.inflight.size >= this.maxInflightTiles) { + this.metrics.overloads += 1; + if (stale) { + this.metrics.staleServed += 1; + return stale; + } + throw new GatewayOverloadedError('The worldwide tile gateway has reached its in-flight limit.'); + } const promise = this.loadManifest() - .then(async (manifest) => { - const [surface, basemap] = await Promise.all([ - this.readSurfaceTile(manifest, zoom, x, y), - this.readBasemapTile(manifest, zoom, x, y) - ]); - - // Land, generalized vegetation, and bathymetry form one continuous - // physical surface from minimum zoom through street zoom. Overview and - // regional archives may add cartographic detail, but they must never - // replace these foundational layers as the user crosses a zoom boundary. - const hasContinuousSurface = !Buffer.from(surface).equals(EMPTY_MVT); - const cartography = hasContinuousSurface - ? mergeVectorTiles([basemap], { excludeLayers: CONTINUOUS_SURFACE_LAYERS }) - : basemap; - - return this.tileCache.set( - key, - mergeVectorTiles([surface, cartography], { coordinateScale: 128 }) - ); + .then((manifest) => this.buildTile( + manifest, + coordinates.z, + coordinates.x, + coordinates.y + )) + .then((tile) => { + this.metrics.resolved += 1; + return this.tileCache.set(key, tile); + }) + .catch((error) => { + this.metrics.failed += 1; + if (stale) { + this.metrics.staleServed += 1; + return stale; + } + throw error; }) .finally(() => this.inflight.delete(key)); this.inflight.set(key, promise); return promise; } + + async ready() { + const manifest = await this.loadManifest(); + return { + ready: true, + regions: manifest.regions.length, + maxZoom: manifest.virtualTiles.maxZoom + }; + } + + getHealthSnapshot() { + return { + manifest: this.manifestState + ? { + loadedAt: this.manifestState.loadedAt, + freshUntil: this.manifestState.freshUntil, + staleUntil: this.manifestState.staleUntil, + regions: this.manifestState.manifest.regions.length + } + : null, + cache: this.tileCache.snapshot(), + archiveReads: this.archiveReadLimiter.snapshot(), + inflightTiles: this.inflight.size, + archives: this.archives.size, + metrics: { ...this.metrics }, + sources: [...this.sources.entries()].map(([asset, source]) => ({ + asset, + ...source.getHealthSnapshot() + })) + }; + } } From fb7174439eeb51fd4c0de358b63b64ed0f58a20f Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:12:02 -0700 Subject: [PATCH 14/85] Harden worldwide shard routing index --- src/server/world-tile-routing.js | 65 +++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/src/server/world-tile-routing.js b/src/server/world-tile-routing.js index b72d5ee4..6216b6f6 100644 --- a/src/server/world-tile-routing.js +++ b/src/server/world-tile-routing.js @@ -1,4 +1,5 @@ const WEB_MERCATOR_LIMIT = 85.0511287798066; +const DEFAULT_MAX_CELL_FANOUT = 64; function clamp(value, minimum, maximum) { return Math.min(Math.max(value, minimum), maximum); @@ -40,12 +41,14 @@ export function normalizeTileCoordinates(zoom, x, y, maximumZoom = 16) { } export function tileBounds(zoom, x, y) { - const count = 2 ** zoom; + const coordinates = normalizeTileCoordinates(zoom, x, y, 22); + if (!coordinates) throw new RangeError('Invalid tile coordinates.'); + const count = 2 ** coordinates.z; return [ - (x / count) * 360 - 180, - tileLatitude(y + 1, zoom), - ((x + 1) / count) * 360 - 180, - tileLatitude(y, zoom) + (coordinates.x / count) * 360 - 180, + tileLatitude(coordinates.y + 1, coordinates.z), + ((coordinates.x + 1) / count) * 360 - 180, + tileLatitude(coordinates.y, coordinates.z) ]; } @@ -79,13 +82,31 @@ function cellKey(x, y) { return `${x}/${y}`; } +function compareRegions(left, right) { + return String(left.asset || left.id).localeCompare(String(right.asset || right.id)); +} + export class WorldTileRoutingIndex { - constructor(regions, { routingZoom = 6 } = {}) { + constructor(regions, { + routingZoom = 6, + maxCellFanout = DEFAULT_MAX_CELL_FANOUT + } = {}) { + if (!Number.isSafeInteger(routingZoom) || routingZoom < 0 || routingZoom > 22) { + throw new RangeError('The routing zoom must be a safe integer between 0 and 22.'); + } + if (!Number.isSafeInteger(maxCellFanout) || maxCellFanout < 1 || maxCellFanout > 256) { + throw new RangeError('The routing cell fan-out limit must be between 1 and 256.'); + } + this.routingZoom = routingZoom; - this.regions = (regions || []).map((region) => ({ - ...region, - segments: regionSegments(region.bounds) - })); + this.maxCellFanout = maxCellFanout; + this.regions = (regions || []).map((region) => { + const segments = regionSegments(region.bounds); + if (!segments.length) { + throw new Error(`Region ${region.id || region.asset || 'unknown'} has no routable bounds.`); + } + return { ...region, segments }; + }).sort(compareRegions); this.cells = new Map(); for (const region of this.regions) { @@ -103,6 +124,12 @@ export class WorldTileRoutingIndex { const key = cellKey(x, y); const entries = this.cells.get(key) || []; if (!entries.includes(region)) entries.push(region); + if (entries.length > this.maxCellFanout) { + throw new Error( + `Routing cell ${key} exceeds the ${this.maxCellFanout}-region fan-out limit.` + ); + } + entries.sort(compareRegions); this.cells.set(key, entries); } } @@ -111,18 +138,20 @@ export class WorldTileRoutingIndex { } regionsForTile(zoom, x, y) { - const bounds = tileBounds(zoom, x, y); + const coordinates = normalizeTileCoordinates(zoom, x, y, 22); + if (!coordinates) return []; + const bounds = tileBounds(coordinates.z, coordinates.x, coordinates.y); let candidates = this.regions; - if (zoom >= this.routingZoom) { - const divisor = 2 ** (zoom - this.routingZoom); - const cellX = Math.floor(x / divisor); - const cellY = Math.floor(y / divisor); + if (coordinates.z >= this.routingZoom) { + const divisor = 2 ** (coordinates.z - this.routingZoom); + const cellX = Math.floor(coordinates.x / divisor); + const cellY = Math.floor(coordinates.y / divisor); candidates = this.cells.get(cellKey(cellX, cellY)) || []; } - return candidates.filter((region) => - region.segments.some((segment) => intersects(segment, bounds)) - ); + return candidates + .filter((region) => region.segments.some((segment) => intersects(segment, bounds))) + .sort(compareRegions); } } From a737282ef25d58591d68022285f717d02b1ec0a3 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:13:15 -0700 Subject: [PATCH 15/85] Add vector tile safety budgets --- src/server/tile-safety.js | 130 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 src/server/tile-safety.js diff --git a/src/server/tile-safety.js b/src/server/tile-safety.js new file mode 100644 index 00000000..70d06afc --- /dev/null +++ b/src/server/tile-safety.js @@ -0,0 +1,130 @@ +import { VectorTile } from '@mapbox/vector-tile'; +import Pbf from 'pbf'; + +const DEFAULT_MAX_BYTES = 24 * 1024 * 1024; +const DEFAULT_MAX_LAYERS = 256; +const DEFAULT_MAX_FEATURES = 500_000; +const DEFAULT_MAX_FEATURE_POINTS = 250_000; +const DEFAULT_MAX_TOTAL_POINTS = 4_000_000; +const DEFAULT_MAX_PROPERTIES = 256; +const DEFAULT_COORDINATE_SCALE = 128; + +function boundedInteger(value, fallback, minimum, maximum) { + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed >= minimum && parsed <= maximum + ? parsed + : fallback; +} + +function safeLabel(value) { + return String(value || 'vector tile').replace(/[\r\n\t]/g, ' ').slice(0, 200); +} + +export function validateVectorTilePayload(payload, { + label = 'vector tile', + maxBytes = DEFAULT_MAX_BYTES, + maxLayers = DEFAULT_MAX_LAYERS, + maxFeatures = DEFAULT_MAX_FEATURES, + maxFeaturePoints = DEFAULT_MAX_FEATURE_POINTS, + maxTotalPoints = DEFAULT_MAX_TOTAL_POINTS, + maxProperties = DEFAULT_MAX_PROPERTIES, + coordinateScale = DEFAULT_COORDINATE_SCALE +} = {}) { + const name = safeLabel(label); + const bytes = Buffer.from(payload || []); + const byteLimit = boundedInteger(maxBytes, DEFAULT_MAX_BYTES, 1_024, 96 * 1024 * 1024); + if (!bytes.byteLength || bytes.byteLength > byteLimit) { + throw new Error(`${name} has unsafe encoded size ${bytes.byteLength}.`); + } + + let tile; + try { + tile = new VectorTile(new Pbf(new Uint8Array(bytes))); + } catch (error) { + throw new Error(`${name} is not a valid Mapbox Vector Tile.`, { cause: error }); + } + + const layerEntries = Object.entries(tile.layers || {}); + const layerLimit = boundedInteger(maxLayers, DEFAULT_MAX_LAYERS, 1, 2_048); + if (layerEntries.length > layerLimit) { + throw new Error(`${name} exceeds the ${layerLimit}-layer safety limit.`); + } + + const featureLimit = boundedInteger(maxFeatures, DEFAULT_MAX_FEATURES, 1, 2_000_000); + const featurePointLimit = boundedInteger(maxFeaturePoints, DEFAULT_MAX_FEATURE_POINTS, 4, 2_000_000); + const totalPointLimit = boundedInteger(maxTotalPoints, DEFAULT_MAX_TOTAL_POINTS, 4, 16_000_000); + const propertyLimit = boundedInteger(maxProperties, DEFAULT_MAX_PROPERTIES, 1, 4_096); + const scale = boundedInteger(coordinateScale, DEFAULT_COORDINATE_SCALE, 1, 512); + let totalFeatures = 0; + let totalPoints = 0; + + for (const [layerName, layer] of layerEntries) { + if (!layerName || layerName.length > 200) { + throw new Error(`${name} contains an invalid source-layer name.`); + } + if (!Number.isSafeInteger(layer.extent) || layer.extent < 256 || layer.extent > 16_384) { + throw new Error(`${name} layer ${layerName} has invalid extent ${layer.extent}.`); + } + if (!Number.isSafeInteger(layer.length) || layer.length < 0) { + throw new Error(`${name} layer ${layerName} has an invalid feature count.`); + } + totalFeatures += layer.length; + if (totalFeatures > featureLimit) { + throw new Error(`${name} exceeds the ${featureLimit}-feature safety limit.`); + } + + const coordinateLimit = layer.extent * scale; + for (let index = 0; index < layer.length; index += 1) { + let feature; + try { + feature = layer.feature(index); + } catch (error) { + throw new Error(`${name} cannot decode ${layerName} feature ${index}.`, { cause: error }); + } + if (![1, 2, 3].includes(feature.type)) { + throw new Error(`${name} contains an invalid geometry type in ${layerName}.`); + } + if (Object.keys(feature.properties || {}).length > propertyLimit) { + throw new Error(`${name} feature ${layerName}/${index} exceeds the property safety limit.`); + } + + let featurePoints = 0; + let geometry; + try { + geometry = feature.loadGeometry(); + } catch (error) { + throw new Error(`${name} cannot decode geometry for ${layerName}/${index}.`, { cause: error }); + } + for (const part of geometry) { + for (const point of part) { + if ( + !Number.isFinite(point.x) || + !Number.isFinite(point.y) || + Math.abs(point.x) > coordinateLimit || + Math.abs(point.y) > coordinateLimit + ) { + throw new Error(`${name} contains unsafe coordinates in ${layerName}/${index}.`); + } + featurePoints += 1; + totalPoints += 1; + if (featurePoints > featurePointLimit) { + throw new Error(`${name} feature ${layerName}/${index} exceeds the point safety limit.`); + } + if (totalPoints > totalPointLimit) { + throw new Error(`${name} exceeds the ${totalPointLimit}-point safety limit.`); + } + } + } + if (!featurePoints) { + throw new Error(`${name} contains empty geometry in ${layerName}/${index}.`); + } + } + } + + return { + bytes, + layerCount: layerEntries.length, + featureCount: totalFeatures, + pointCount: totalPoints + }; +} From 1f49ecacd81b96f420736fb29a8cb1ac1723a98a Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:14:36 -0700 Subject: [PATCH 16/85] Add worldwide gateway chaos hardening tests --- scripts/check-world-hardening.mjs | 158 ++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 scripts/check-world-hardening.mjs diff --git a/scripts/check-world-hardening.mjs b/scripts/check-world-hardening.mjs new file mode 100644 index 00000000..e3dad880 --- /dev/null +++ b/scripts/check-world-hardening.mjs @@ -0,0 +1,158 @@ +import assert from 'node:assert/strict'; +import vtpbf from 'vt-pbf'; +import { RetryingFetchSource } from '../src/server/pmtiles-source.js'; +import { + MemoryTileCache, + parseManifest, + WorldTileGateway +} from '../src/server/world-tile-gateway.js'; +import { validateVectorTilePayload } from '../src/server/tile-safety.js'; +import { WorldTileRoutingIndex } from '../src/server/world-tile-routing.js'; + +function vectorTile() { + return Buffer.from(vtpbf.fromGeojsonVt({ + land: { + features: [{ + id: 1, + type: 3, + geometry: [[[0, 0], [4096, 0], [4096, 4096], [0, 4096], [0, 0]]], + tags: {} + }] + } + })); +} + +const validTile = vectorTile(); +const validInspection = validateVectorTilePayload(validTile, { label: 'valid test tile' }); +assert.equal(validInspection.layerCount, 1); +assert.equal(validInspection.featureCount, 1); +assert.throws( + () => validateVectorTilePayload(Buffer.from([0xff, 0xff, 0xff]), { label: 'corrupt test tile' }), + /valid Mapbox Vector Tile/ +); +assert.throws( + () => validateVectorTilePayload(Buffer.alloc(2_048), { maxBytes: 1_024 }), + /unsafe encoded size/ +); + +let transientCalls = 0; +const transientSource = { + getKey: () => 'transient-source', + getBytes: async () => { + transientCalls += 1; + if (transientCalls < 3) { + const error = new Error('HTTP status 503'); + error.status = 503; + throw error; + } + return { data: new Uint8Array([1, 2, 3]) }; + } +}; +const retrying = new RetryingFetchSource('https://example.test/map.pmtiles', { + source: transientSource, + attempts: 3, + baseDelayMs: 0, + maxDelayMs: 0, + random: () => 0 +}); +await retrying.getBytes(0, 3); +assert.equal(transientCalls, 3, 'Transient upstream failures were not retried exactly as bounded.'); +assert.equal(retrying.getHealthSnapshot().totalRetries, 2); + +let permanentCalls = 0; +const permanentSource = { + getKey: () => 'permanent-source', + getBytes: async () => { + permanentCalls += 1; + const error = new Error('HTTP status 404'); + error.status = 404; + throw error; + } +}; +const permanent = new RetryingFetchSource('https://example.test/missing.pmtiles', { + source: permanentSource, + attempts: 4, + baseDelayMs: 0, + maxDelayMs: 0 +}); +await assert.rejects(() => permanent.getBytes(0, 10), /bounded retries/); +assert.equal(permanentCalls, 1, 'Permanent 4xx failures were retried.'); +await assert.rejects(() => permanent.getBytes(0, 100_000_000), /range length/); + +let clock = 1_000; +const cache = new MemoryTileCache(2 * 1024 * 1024, { + ttlMs: 1_000, + staleMs: 5_000, + now: () => clock +}); +cache.set('0/0/0', validTile); +assert(cache.getFresh('0/0/0')); +clock = 2_500; +assert.equal(cache.getFresh('0/0/0'), null); +assert(cache.getStale('0/0/0')); +clock = 7_000; +assert.equal(cache.getStale('0/0/0'), null); + +const manifest = { + version: 2, + plannedRegionCount: 2, + availableRegionCount: 2, + missingRegionCount: 0, + virtualTiles: { + endpoint: '/tiles/{z}/{x}/{y}.pbf', + overviewAsset: 'occumed-world-overview.pmtiles', + surfaceAsset: 'occumed-world-surface.pmtiles', + overviewMaxZoom: 5, + surfaceMaxZoom: 10, + routingZoom: 6, + maxZoom: 16 + }, + regions: [ + { id: 'west', asset: 'occumed-west.pmtiles', bounds: [-20, -10, 1, 10] }, + { id: 'east', asset: 'occumed-east.pmtiles', bounds: [0, -10, 20, 10] } + ] +}; +const parsed = parseManifest(manifest); +assert.equal(parsed.regions.length, 2); +assert.throws( + () => parseManifest({ + ...manifest, + regions: [manifest.regions[0], { ...manifest.regions[1], id: 'west' }] + }), + /repeats regional ID/ +); +assert.throws( + () => parseManifest({ + ...manifest, + virtualTiles: { ...manifest.virtualTiles, routingZoom: 8 } + }), + /exactly adjacent/ +); +assert.throws( + () => new WorldTileRoutingIndex(parsed.regions, { routingZoom: 6, maxCellFanout: 1 }), + /fan-out limit/ +); + +clock = 10_000; +const gateway = new WorldTileGateway({ + manifestUrl: 'https://example.test/world-manifest.json', + releaseAssetUrl: (asset) => `https://example.test/${asset}`, + cacheTtlMs: 1_000, + cacheStaleMs: 10_000, + now: () => clock +}); +gateway.loadManifest = async () => parsed; +let buildFails = false; +gateway.buildTile = async () => { + if (buildFails) throw new Error('simulated upstream outage'); + return validTile; +}; +const first = await gateway.resolveTile(0, 0, 0); +assert(first.equals(validTile)); +clock = 12_000; +buildFails = true; +const stale = await gateway.resolveTile(0, 0, 0); +assert(stale.equals(validTile), 'A last-known-good tile was not served during an upstream outage.'); +assert.equal(gateway.getHealthSnapshot().metrics.staleServed, 1); + +console.log('Worldwide hardening validated: bounded retries, permanent-failure classification, circuit-safe ranges, MVT budgets, strict manifests, fan-out limits, cache expiry, and stale-tile recovery.'); From 8e6d9a6e3b713df407a954a127bbebf31197263f Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:15:02 -0700 Subject: [PATCH 17/85] Run worldwide hardening checks in every build --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 230f6417..42ba2552 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ "tiles:build": "bash planetiler/build-region.sh", "tiles:plan-world": "node scripts/plan-world-shards.mjs --scope all", "check:export": "node scripts/check-export.mjs", - "check:runtime": "node scripts/check-runtime.mjs && node scripts/validate-maplibre-style.mjs && node scripts/check-globe-parity.mjs && node scripts/check-cartography-parity.mjs && node scripts/check-viewer-quality.mjs && node scripts/check-photo-reference.mjs && node scripts/check-exact-exported-swatches.mjs && node scripts/check-render-clarity.mjs && node scripts/check-pmtiles-integration.mjs && node scripts/check-world-tile-gateway.mjs", + "check:hardening": "node scripts/check-world-hardening.mjs", + "check:runtime": "node scripts/check-runtime.mjs && node scripts/validate-maplibre-style.mjs && node scripts/check-globe-parity.mjs && node scripts/check-cartography-parity.mjs && node scripts/check-viewer-quality.mjs && node scripts/check-photo-reference.mjs && node scripts/check-exact-exported-swatches.mjs && node scripts/check-render-clarity.mjs && node scripts/check-pmtiles-integration.mjs && node scripts/check-world-tile-gateway.mjs && npm run check:hardening", "check:server": "node scripts/check-server-health.mjs", "check": "npm run check:export && npm run check:runtime", "dev": "npm run prepare:assets && vite", From ec3f1f45a2e5c30fcf383bad037d112fe949585b Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:17:58 -0700 Subject: [PATCH 18/85] Aggressively harden the production map server --- server.mjs | 410 ++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 326 insertions(+), 84 deletions(-) diff --git a/server.mjs b/server.mjs index 28b8ef88..48e7397a 100644 --- a/server.mjs +++ b/server.mjs @@ -1,12 +1,19 @@ +import { createHash, randomUUID } from 'node:crypto'; import { createReadStream } from 'node:fs'; import fs from 'node:fs/promises'; import http from 'node:http'; import path from 'node:path'; +import { promisify } from 'node:util'; import { fileURLToPath } from 'node:url'; -import { gzipSync } from 'node:zlib'; -import { WorldTileGateway } from './src/server/world-tile-gateway.js'; +import { gzip } from 'node:zlib'; +import { + GatewayOverloadedError, + WorldTileGateway +} from './src/server/world-tile-gateway.js'; import { normalizeTileCoordinates } from './src/server/world-tile-routing.js'; +import { validateVectorTilePayload } from './src/server/tile-safety.js'; +const gzipAsync = promisify(gzip); const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), 'dist'); const port = Number(process.env.PORT || 4173); const host = process.env.HOST?.trim() || '0.0.0.0'; @@ -15,6 +22,12 @@ const worldReleaseTag = process.env.OCCUMED_WORLD_RELEASE_TAG?.trim() || 'occume const worldManifestAsset = 'world-virtual-manifest.json'; const worldSurfaceAsset = 'occumed-world-surface.pmtiles'; const worldSurfaceUrl = process.env.OCCUMED_WORLD_SURFACE_URL?.trim(); +const maxConcurrentTileRequests = Number(process.env.OCCUMED_MAX_CONCURRENT_TILE_REQUESTS || 64); +const tileRequestTimeoutMs = Number(process.env.OCCUMED_TILE_REQUEST_TIMEOUT_MS || 30_000); +const maxResolvedTileBytes = Number(process.env.OCCUMED_MAX_RESOLVED_TILE_BYTES || 24 * 1024 * 1024); +const diagnosticsEnabled = process.env.OCCUMED_ENABLE_DIAGNOSTICS === 'true'; +let activeTileRequests = 0; +let shuttingDown = false; const contentTypes = { '.css': 'text/css; charset=utf-8', @@ -29,49 +42,114 @@ const contentTypes = { '.webp': 'image/webp' }; -const corsHeaders = { +const commonHeaders = { 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'GET, HEAD, OPTIONS', - 'Access-Control-Allow-Headers': 'Content-Type, Cache-Control, Pragma, Range', - 'Access-Control-Expose-Headers': 'Accept-Ranges, Content-Length, Content-Range', + 'Access-Control-Allow-Headers': 'Content-Type, Cache-Control, Pragma, Range, If-None-Match', + 'Access-Control-Expose-Headers': 'Accept-Ranges, Content-Length, Content-Range, ETag, Server-Timing, X-Occumed-Request-Id', 'Cross-Origin-Resource-Policy': 'cross-origin', - 'X-Content-Type-Options': 'nosniff' + 'Permissions-Policy': 'camera=(), microphone=(), geolocation=(), payment=(), usb=()', + 'Referrer-Policy': 'strict-origin-when-cross-origin', + 'X-Content-Type-Options': 'nosniff', + 'X-DNS-Prefetch-Control': 'off' }; +function safeInteger(value, fallback, minimum, maximum) { + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed >= minimum && parsed <= maximum + ? parsed + : fallback; +} + +function sanitizeRequestId(value) { + const requestId = String(value || '').trim(); + return /^[A-Za-z0-9._:-]{1,128}$/.test(requestId) ? requestId : randomUUID(); +} + +function resolveSafeOrigin(value) { + if (!value) return null; + try { + const url = new URL(value); + if (!['http:', 'https:'].includes(url.protocol)) return null; + if (url.username || url.password) return null; + return url.origin; + } catch { + return null; + } +} + function requestOrigin(request) { - const configured = process.env.PUBLIC_ORIGIN?.trim().replace(/\/$/, ''); + const configured = resolveSafeOrigin(process.env.PUBLIC_ORIGIN?.trim().replace(/\/$/, '')); if (configured) return configured; const forwardedProtocol = String(request.headers['x-forwarded-proto'] || '').split(',')[0].trim(); + const protocol = /^(?:http|https)$/.test(forwardedProtocol) ? forwardedProtocol : 'http'; const forwardedHost = String(request.headers['x-forwarded-host'] || '').split(',')[0].trim(); - const protocol = forwardedProtocol || 'http'; - const requestHost = forwardedHost || request.headers.host || `localhost:${port}`; - return `${protocol}://${requestHost}`; + const requestHost = forwardedHost || String(request.headers.host || `localhost:${port}`).trim(); + return resolveSafeOrigin(`${protocol}://${requestHost}`) || `http://localhost:${port}`; } -function send(response, status, body, contentType, cacheControl = 'no-store', method = 'GET') { +function bodyBuffer(body) { + return Buffer.isBuffer(body) ? body : Buffer.from(String(body ?? '')); +} + +function writeHeaders(response, status, headers = {}) { response.writeHead(status, { - ...corsHeaders, + ...commonHeaders, + 'X-Occumed-Request-Id': response.occumedRequestId, + ...headers + }); +} + +function send(response, status, body, contentType, cacheControl = 'no-store', method = 'GET', extraHeaders = {}) { + const payload = bodyBuffer(body); + writeHeaders(response, status, { 'Cache-Control': cacheControl, 'CDN-Cache-Control': cacheControl, 'Surrogate-Control': cacheControl, - 'Content-Type': contentType + 'Content-Length': payload.byteLength, + 'Content-Type': contentType, + ...extraHeaders }); - if (method === 'HEAD') response.end(); - else response.end(body); + if (method === 'HEAD' || status === 304) response.end(); + else response.end(payload); +} + +function sendJson(response, status, document, cacheControl = 'no-store', method = 'GET', extraHeaders = {}) { + send( + response, + status, + `${JSON.stringify(document)}\n`, + contentTypes['.json'], + cacheControl, + method, + extraHeaders + ); } function sendHealth(request, response) { - const body = 'ok'; - response.statusCode = 200; - response.shouldKeepAlive = false; - response.setHeader('Content-Type', 'text/plain; charset=utf-8'); - response.setHeader('Content-Length', Buffer.byteLength(body)); - response.setHeader('Cache-Control', 'no-store'); - response.setHeader('Connection', 'close'); - response.setHeader('X-Content-Type-Options', 'nosniff'); - if (request.method === 'HEAD') response.end(); - else response.end(body); + send(response, 200, 'ok', 'text/plain; charset=utf-8', 'no-store', request.method, { + Connection: 'close' + }); +} + +async function sendReadiness(request, response) { + try { + const ready = await Promise.race([ + worldTileGateway.ready(), + new Promise((_, reject) => { + const timer = setTimeout(() => reject(new Error('Readiness check timed out.')), 10_000); + timer.unref?.(); + }) + ]); + sendJson(response, 200, { ...ready, shuttingDown }, 'no-store', request.method); + } catch (error) { + sendJson(response, 503, { + ready: false, + shuttingDown, + error: error?.code || error?.name || 'READINESS_FAILED' + }, 'no-store', request.method, { 'Retry-After': '5' }); + } } async function serveStyle(request, response) { @@ -97,31 +175,85 @@ const worldTileGateway = new WorldTileGateway({ manifestUrl: process.env.OCCUMED_WORLD_MANIFEST_URL?.trim() || releaseAssetUrl(worldManifestAsset), - releaseAssetUrl + releaseAssetUrl, + maxResolvedTileBytes: safeInteger( + maxResolvedTileBytes, + 24 * 1024 * 1024, + 1_024, + 96 * 1024 * 1024 + ) }); +function timeoutAfter(milliseconds, message) { + return new Promise((_, reject) => { + const timer = setTimeout(() => { + const error = new Error(message); + error.code = 'OCCUMED_TILE_REQUEST_TIMEOUT'; + error.statusCode = 503; + reject(error); + }, milliseconds); + timer.unref?.(); + }); +} + +function tileEtag(tile) { + return `"${createHash('sha256').update(tile).digest('base64url').slice(0, 24)}"`; +} + async function serveVirtualTile(request, response, coordinates) { - const tile = await worldTileGateway.resolveTile( - coordinates.z, - coordinates.x, - coordinates.y - ); - const compressed = gzipSync(tile, { level: 6 }); - const cacheControl = 'public, max-age=86400, stale-while-revalidate=604800'; + const concurrencyLimit = safeInteger(maxConcurrentTileRequests, 64, 4, 512); + if (activeTileRequests >= concurrencyLimit) { + throw new GatewayOverloadedError('The HTTP tile concurrency limit has been reached.'); + } - response.writeHead(200, { - ...corsHeaders, - 'Cache-Control': cacheControl, - 'CDN-Cache-Control': cacheControl, - 'Surrogate-Control': cacheControl, - 'Content-Encoding': 'gzip', - 'Content-Length': compressed.byteLength, - 'Content-Type': contentTypes['.pbf'], - 'Vary': 'Accept-Encoding', - 'X-Occumed-Tileset': 'virtual-worldwide-v1' - }); - if (request.method === 'HEAD') response.end(); - else response.end(compressed); + activeTileRequests += 1; + const startedAt = performance.now(); + try { + const tile = await Promise.race([ + worldTileGateway.resolveTile(coordinates.z, coordinates.x, coordinates.y), + timeoutAfter( + safeInteger(tileRequestTimeoutMs, 30_000, 1_000, 120_000), + `Tile ${coordinates.z}/${coordinates.x}/${coordinates.y} timed out.` + ) + ]); + validateVectorTilePayload(tile, { + label: `resolved tile ${coordinates.z}/${coordinates.x}/${coordinates.y}`, + maxBytes: safeInteger(maxResolvedTileBytes, 24 * 1024 * 1024, 1_024, 96 * 1024 * 1024) + }); + + const etag = tileEtag(tile); + if (request.headers['if-none-match'] === etag) { + writeHeaders(response, 304, { + 'Cache-Control': 'public, max-age=300, must-revalidate, stale-if-error=86400', + ETag: etag, + 'X-Occumed-Tileset': 'virtual-worldwide-v2' + }); + response.end(); + return; + } + + const acceptsGzip = /(?:^|,)\s*gzip\s*(?:,|$)/i.test(String(request.headers['accept-encoding'] || '')); + const payload = acceptsGzip ? await gzipAsync(tile, { level: 5 }) : tile; + const cacheControl = 'public, max-age=300, must-revalidate, stale-while-revalidate=60, stale-if-error=86400'; + const duration = Math.max(0, performance.now() - startedAt); + + writeHeaders(response, 200, { + 'Cache-Control': cacheControl, + 'CDN-Cache-Control': cacheControl, + 'Surrogate-Control': cacheControl, + ...(acceptsGzip ? { 'Content-Encoding': 'gzip' } : {}), + 'Content-Length': payload.byteLength, + 'Content-Type': contentTypes['.pbf'], + ETag: etag, + 'Server-Timing': `tile;dur=${duration.toFixed(1)}`, + Vary: 'Accept-Encoding', + 'X-Occumed-Tileset': 'virtual-worldwide-v2' + }); + if (request.method === 'HEAD') response.end(); + else response.end(payload); + } finally { + activeTileRequests -= 1; + } } function parseByteRange(header, size) { @@ -134,7 +266,6 @@ function parseByteRange(header, size) { let start; let end; - if (!startText) { const suffixLength = Number(endText); if (!Number.isSafeInteger(suffixLength) || suffixLength <= 0) return null; @@ -154,23 +285,28 @@ function parseByteRange(header, size) { ) { return null; } - return { start, end: Math.min(end, size - 1) }; } -function pipeFile(response, absolute, options = {}) { +function pipeFile(request, response, absolute, options = {}) { const stream = createReadStream(absolute, options); + const abort = () => stream.destroy(); + request.once('aborted', abort); + response.once('close', abort); stream.on('error', (error) => response.destroy(error)); + stream.on('close', () => { + request.removeListener('aborted', abort); + response.removeListener('close', abort); + }); stream.pipe(response); } async function servePmtiles(request, response, absolute, stat) { - const cacheControl = 'public, max-age=3600, must-revalidate'; + const cacheControl = 'public, max-age=3600, must-revalidate, stale-if-error=86400'; const rangeHeader = request.headers.range; if (!rangeHeader) { - response.writeHead(200, { - ...corsHeaders, + writeHeaders(response, 200, { 'Accept-Ranges': 'bytes', 'Cache-Control': cacheControl, 'CDN-Cache-Control': cacheControl, @@ -179,25 +315,29 @@ async function servePmtiles(request, response, absolute, stat) { 'Content-Type': contentTypes['.pmtiles'] }); if (request.method === 'HEAD') response.end(); - else pipeFile(response, absolute); + else pipeFile(request, response, absolute); return; } const range = parseByteRange(rangeHeader, stat.size); if (!range) { - response.writeHead(416, { - ...corsHeaders, - 'Accept-Ranges': 'bytes', - 'Content-Range': `bytes */${stat.size}`, - 'Content-Type': 'text/plain; charset=utf-8' - }); - response.end('Requested range not satisfiable'); + send( + response, + 416, + 'Requested range not satisfiable', + 'text/plain; charset=utf-8', + 'no-store', + request.method, + { + 'Accept-Ranges': 'bytes', + 'Content-Range': `bytes */${stat.size}` + } + ); return; } const length = range.end - range.start + 1; - response.writeHead(206, { - ...corsHeaders, + writeHeaders(response, 206, { 'Accept-Ranges': 'bytes', 'Cache-Control': cacheControl, 'CDN-Cache-Control': cacheControl, @@ -206,16 +346,30 @@ async function servePmtiles(request, response, absolute, stat) { 'Content-Range': `bytes ${range.start}-${range.end}/${stat.size}`, 'Content-Type': contentTypes['.pmtiles'] }); - if (request.method === 'HEAD') response.end(); - else pipeFile(response, absolute, { start: range.start, end: range.end }); + else pipeFile(request, response, absolute, { start: range.start, end: range.end }); +} + +function shouldServeSpaFallback(request, decoded) { + if (path.extname(decoded)) return false; + return String(request.headers.accept || '').includes('text/html') || decoded === '/'; } async function serveStatic(request, response, pathname) { const requested = pathname === '/' ? '/index.html' : pathname; - const decoded = decodeURIComponent(requested); - const absolute = path.resolve(root, `.${decoded}`); + let decoded; + try { + decoded = decodeURIComponent(requested); + } catch { + send(response, 400, 'Invalid URL encoding', 'text/plain; charset=utf-8', 'no-store', request.method); + return; + } + if (decoded.includes('\0')) { + send(response, 400, 'Invalid path', 'text/plain; charset=utf-8', 'no-store', request.method); + return; + } + const absolute = path.resolve(root, `.${decoded}`); if (!absolute.startsWith(`${root}${path.sep}`) && absolute !== path.join(root, 'index.html')) { send(response, 403, 'Forbidden', 'text/plain; charset=utf-8', 'no-store', request.method); return; @@ -242,6 +396,10 @@ async function serveStatic(request, response, pathname) { request.method ); } catch { + if (!shouldServeSpaFallback(request, decoded)) { + send(response, 404, 'Not found', 'text/plain; charset=utf-8', 'no-store', request.method); + return; + } const index = await fs.readFile(path.join(root, 'index.html')); send( response, @@ -254,25 +412,53 @@ async function serveStatic(request, response, pathname) { } } +function tileErrorStatus(error) { + if (error instanceof GatewayOverloadedError) return 503; + if (error instanceof RangeError) return 404; + if (Number.isSafeInteger(error?.statusCode)) return error.statusCode; + if (String(error?.code || '').startsWith('OCCUMED_')) return 503; + return 500; +} + async function handleRequest(request, response) { const method = request.method || 'GET'; - if (method === 'OPTIONS') { - response.writeHead(204, { - ...corsHeaders, - 'Cache-Control': 'no-store' - }); + writeHeaders(response, 204, { 'Cache-Control': 'no-store' }); response.end(); return; } - if (!['GET', 'HEAD'].includes(method)) { - send(response, 405, 'Method not allowed', 'text/plain; charset=utf-8', 'no-store', method); + send(response, 405, 'Method not allowed', 'text/plain; charset=utf-8', 'no-store', method, { + Allow: 'GET, HEAD, OPTIONS' + }); + return; + } + if (shuttingDown) { + send(response, 503, 'Server is shutting down', 'text/plain; charset=utf-8', 'no-store', method, { + 'Retry-After': '5' + }); return; } - const url = new URL(request.url || '/', `http://${request.headers.host || 'localhost'}`); + let url; + try { + url = new URL(request.url || '/', `http://${request.headers.host || 'localhost'}`); + } catch { + send(response, 400, 'Invalid request URL', 'text/plain; charset=utf-8', 'no-store', method); + return; + } + if (url.pathname === '/readyz') { + await sendReadiness(request, response); + return; + } + if (url.pathname === '/internal/tile-health' && diagnosticsEnabled) { + sendJson(response, 200, { + activeTileRequests, + ...worldTileGateway.getHealthSnapshot() + }, 'no-store', method); + return; + } if (url.pathname === '/style/occumed-open.json') { await serveStyle(request, response); return; @@ -280,16 +466,37 @@ async function handleRequest(request, response) { const tileMatch = /^\/tiles\/(\d+)\/(\d+)\/(\d+)\.pbf$/.exec(url.pathname); if (tileMatch) { - const coordinates = normalizeTileCoordinates( - tileMatch[1], - tileMatch[2], - tileMatch[3] - ); + const coordinates = normalizeTileCoordinates(tileMatch[1], tileMatch[2], tileMatch[3]); if (!coordinates) { send(response, 404, 'Tile not found', 'text/plain; charset=utf-8', 'no-store', method); return; } - await serveVirtualTile(request, response, coordinates); + try { + await serveVirtualTile(request, response, coordinates); + } catch (error) { + const status = tileErrorStatus(error); + console.error(JSON.stringify({ + level: 'error', + type: 'tile-request-failed', + requestId: response.occumedRequestId, + tile: coordinates, + code: error?.code || error?.name || 'UNKNOWN', + message: error?.message || String(error) + })); + if (!response.headersSent) { + send( + response, + status, + status === 404 ? 'Tile not found' : 'Tile temporarily unavailable', + 'text/plain; charset=utf-8', + 'no-store', + method, + status === 503 ? { 'Retry-After': '2' } : {} + ); + } else { + response.destroy(); + } + } return; } @@ -297,17 +504,22 @@ async function handleRequest(request, response) { } const server = http.createServer((request, response) => { + response.occumedRequestId = sanitizeRequestId(request.headers['x-request-id']); const rawPath = (request.url || '/').split('?', 1)[0]; - // Keep Render's deployment probe completely independent from URL parsing, - // filesystem access, the built map assets, and all application routing. if (rawPath === '/health' || rawPath === '/healthz') { sendHealth(request, response); return; } void handleRequest(request, response).catch((error) => { - console.error(error); + console.error(JSON.stringify({ + level: 'error', + type: 'request-failed', + requestId: response.occumedRequestId, + code: error?.code || error?.name || 'UNKNOWN', + message: error?.message || String(error) + })); if (!response.headersSent) { send( response, @@ -324,15 +536,45 @@ const server = http.createServer((request, response) => { }); server.requestTimeout = 45_000; -server.headersTimeout = 50_000; +server.headersTimeout = 20_000; server.keepAliveTimeout = 5_000; +server.maxHeadersCount = 100; +server.maxRequestsPerSocket = 1_000; +server.on('clientError', (error, socket) => { + console.warn('Occu-Med Map client error:', error.code || error.message); + if (socket.writable) socket.end('HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n'); +}); server.on('error', (error) => { console.error('Occu-Med Map server error:', error); process.exitCode = 1; }); +function shutdown(signal) { + if (shuttingDown) return; + shuttingDown = true; + console.log(`Occu-Med Map received ${signal}; draining connections.`); + server.close((error) => { + if (error) { + console.error('Occu-Med Map shutdown error:', error); + process.exitCode = 1; + } + }); + const timer = setTimeout(() => { + server.closeAllConnections?.(); + process.exitCode = 1; + }, 25_000); + timer.unref?.(); +} + +process.once('SIGTERM', () => shutdown('SIGTERM')); +process.once('SIGINT', () => shutdown('SIGINT')); + server.listen(port, host, () => { console.log(`Occu-Med Map listening on ${host}:${port}.`); console.log(`Health endpoint ready at http://127.0.0.1:${port}/health.`); + void worldTileGateway.ready().then( + (ready) => console.log(`Worldwide gateway ready with ${ready.regions} regional shards.`), + (error) => console.error('Worldwide gateway readiness failed:', error) + ); }); From 8590603ef33543cd3e14ee0cb5896141ac2dbc1b Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:19:21 -0700 Subject: [PATCH 19/85] Clean retry abort listeners and expose circuit behavior --- src/server/pmtiles-source.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/server/pmtiles-source.js b/src/server/pmtiles-source.js index dcf7894e..74d52bea 100644 --- a/src/server/pmtiles-source.js +++ b/src/server/pmtiles-source.js @@ -25,14 +25,22 @@ function abortError(signal) { function delay(milliseconds, signal) { if (signal?.aborted) return Promise.reject(abortError(signal)); return new Promise((resolve, reject) => { - const timer = setTimeout(resolve, milliseconds); - const onAbort = () => { + let timer; + const cleanup = () => { clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + }; + const onAbort = () => { + cleanup(); reject(abortError(signal)); }; + timer = setTimeout(() => { + cleanup(); + resolve(); + }, milliseconds); signal?.addEventListener('abort', onAbort, { once: true }); timer.unref?.(); - }).finally(() => signal?.removeEventListener?.('abort', () => {})); + }); } function errorStatus(error) { From dd64185c0d6d6752925ad5bc8275167cca02fcc5 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:20:27 -0700 Subject: [PATCH 20/85] Exercise PMTiles circuit breaker in chaos tests --- scripts/check-world-hardening.mjs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/scripts/check-world-hardening.mjs b/scripts/check-world-hardening.mjs index e3dad880..868966a5 100644 --- a/scripts/check-world-hardening.mjs +++ b/scripts/check-world-hardening.mjs @@ -79,6 +79,35 @@ await assert.rejects(() => permanent.getBytes(0, 10), /bounded retries/); assert.equal(permanentCalls, 1, 'Permanent 4xx failures were retried.'); await assert.rejects(() => permanent.getBytes(0, 100_000_000), /range length/); +let circuitClock = 1_000; +let circuitCalls = 0; +const circuitSource = { + getKey: () => 'circuit-source', + getBytes: async () => { + circuitCalls += 1; + const error = new Error('HTTP status 503'); + error.status = 503; + throw error; + } +}; +const circuit = new RetryingFetchSource('https://example.test/unavailable.pmtiles', { + source: circuitSource, + attempts: 1, + circuitFailures: 2, + circuitCooldownMs: 1_000, + now: () => circuitClock +}); +await assert.rejects(() => circuit.getBytes(0, 10), /bounded retries/); +await assert.rejects(() => circuit.getBytes(0, 10), /bounded retries/); +await assert.rejects( + () => circuit.getBytes(0, 10), + (error) => error.code === 'OCCUMED_UPSTREAM_CIRCUIT_OPEN' +); +assert.equal(circuitCalls, 2, 'An open circuit still called the failing upstream.'); +circuitClock = 2_100; +await assert.rejects(() => circuit.getBytes(0, 10), /bounded retries/); +assert.equal(circuitCalls, 3, 'The circuit did not permit a bounded probe after cooldown.'); + let clock = 1_000; const cache = new MemoryTileCache(2 * 1024 * 1024, { ttlMs: 1_000, @@ -155,4 +184,4 @@ const stale = await gateway.resolveTile(0, 0, 0); assert(stale.equals(validTile), 'A last-known-good tile was not served during an upstream outage.'); assert.equal(gateway.getHealthSnapshot().metrics.staleServed, 1); -console.log('Worldwide hardening validated: bounded retries, permanent-failure classification, circuit-safe ranges, MVT budgets, strict manifests, fan-out limits, cache expiry, and stale-tile recovery.'); +console.log('Worldwide hardening validated: bounded retries, permanent-failure classification, circuit breaking, MVT budgets, strict manifests, fan-out limits, cache expiry, and stale-tile recovery.'); From 531ad52b76c6c4395121d0af856cfa6f9a2db8cf Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:24:16 -0700 Subject: [PATCH 21/85] Keep essential retry timers referenced --- src/server/pmtiles-source.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server/pmtiles-source.js b/src/server/pmtiles-source.js index 74d52bea..2c7dcf7a 100644 --- a/src/server/pmtiles-source.js +++ b/src/server/pmtiles-source.js @@ -39,7 +39,6 @@ function delay(milliseconds, signal) { resolve(); }, milliseconds); signal?.addEventListener('abort', onAbort, { once: true }); - timer.unref?.(); }); } From 3717689d82ed86f6f772504cffb1c00924a85818 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:27:50 -0700 Subject: [PATCH 22/85] Align PMTiles integration guard with hardened continuous surface --- scripts/check-pmtiles-integration.mjs | 54 ++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/scripts/check-pmtiles-integration.mjs b/scripts/check-pmtiles-integration.mjs index ae35a6e4..c45af303 100644 --- a/scripts/check-pmtiles-integration.mjs +++ b/scripts/check-pmtiles-integration.mjs @@ -11,6 +11,8 @@ const [ manifestBuilder, overviewBuilder, gateway, + retryingSource, + tileSafety, server, workflow, surfaceBuilder, @@ -23,6 +25,8 @@ const [ fs.readFile(path.join(root, 'scripts/build-world-manifest.mjs'), 'utf8'), fs.readFile(path.join(root, 'scripts/build-world-overview.mjs'), 'utf8'), fs.readFile(path.join(root, 'src/server/world-tile-gateway.js'), 'utf8'), + fs.readFile(path.join(root, 'src/server/pmtiles-source.js'), 'utf8'), + fs.readFile(path.join(root, 'src/server/tile-safety.js'), 'utf8'), fs.readFile(path.join(root, 'server.mjs'), 'utf8'), fs.readFile(path.join(root, '.github/workflows/build-virtual-world-tileset.yml'), 'utf8'), fs.readFile(path.join(root, 'scripts/build-world-surface.sh'), 'utf8'), @@ -64,7 +68,7 @@ if (!helper.includes('refreshExpiredTiles: false')) { if (!helper.includes('fadeDuration: 300')) { fail('Normal symbol collision fading is not enabled during zoom.'); } -if (!helper.includes("source.tiles = source.tiles.map")) { +if (!helper.includes('source.tiles = source.tiles.map')) { fail('Permanent vector tile templates are not resolved to the style origin.'); } @@ -73,15 +77,44 @@ if (!server.includes("const tileMatch = /^\\/tiles\\/(\\d+)\\/(\\d+)\\/(\\d+)\\. } if (server.includes('/world-tiles/')) fail('The server still publishes browser-visible regional archive paths.'); if (!gateway.includes('regionsForTile(zoom, x, y)')) fail('The gateway does not route each requested tile by bounds.'); -if (!gateway.includes('Promise.all')) fail('The gateway cannot resolve intersecting archives together.'); +if (!gateway.includes('Promise.allSettled')) fail('The gateway does not fail closed when one required shard read fails.'); if (!gateway.includes('mergeVectorTiles')) fail('The gateway cannot merge MVT layers at shard boundaries.'); if (!gateway.includes('MemoryTileCache')) fail('Resolved worldwide tiles are not cached.'); -if (!gateway.includes('overscaleVectorLayer')) fail('The land surface cannot remain continuous above its generalized zoom.'); -if (!gateway.includes("includeLayers: ['land']")) { - fail('The gateway does not isolate the worldwide surface to a non-overlapping land mask.'); +if (!gateway.includes('getStale')) fail('The gateway cannot serve a last-known-good tile during an upstream outage.'); +if (!gateway.includes('overscaleVectorLayer')) fail('The physical surface cannot remain continuous above its generalized zoom.'); +if (!gateway.includes("const CONTINUOUS_SURFACE_LAYERS = Object.freeze(['land', 'landcover', 'depth'])")) { + fail('The gateway does not define one authoritative land/landcover/depth foundation.'); } -if (gateway.includes("includeLayers: ['land', 'landcover', 'depth']")) { - fail('The gateway still overlays generalized surface detail on regional geometry.'); +if (!gateway.includes('includeLayers: CONTINUOUS_SURFACE_LAYERS')) { + fail('The gateway does not retain the complete physical surface.'); +} +if (!gateway.includes('excludeLayers: CONTINUOUS_SURFACE_LAYERS')) { + fail('Overview or regional archives can still replace the physical foundation.'); +} +if (!gateway.includes('CONTINUOUS_SURFACE_LAYERS.map')) { + fail('All physical surface layers are not overscaled through maximum zoom.'); +} +if (!gateway.includes('maxInflightTiles') || !gateway.includes('maxTileFanout')) { + fail('The gateway lacks bounded in-flight work or shard fan-out limits.'); +} +if (!gateway.includes('manifestStaleMs') || !gateway.includes('maxManifestBytes')) { + fail('The gateway lacks last-known-good manifest recovery or manifest size limits.'); +} +if (!retryingSource.includes('OCCUMED_UPSTREAM_CIRCUIT_OPEN')) { + fail('PMTiles byte-range reads are not protected by a circuit breaker.'); +} +if (!retryingSource.includes('maxRangeBytes') || !retryingSource.includes('isRetryableUpstreamError')) { + fail('PMTiles byte-range reads lack size limits or retry classification.'); +} +if (!tileSafety.includes('validateVectorTilePayload') || !tileSafety.includes('maxTotalPoints')) { + fail('Vector tiles are not protected by decode and geometry budgets.'); +} +if (!server.includes('gzipAsync')) fail('Tile compression still blocks the Node event loop.'); +if (!server.includes("url.pathname === '/readyz'")) fail('The production server lacks a readiness endpoint.'); +if (!server.includes('maxConcurrentTileRequests')) fail('The HTTP tile endpoint lacks a concurrency limit.'); +if (!server.includes('stale-if-error=86400')) fail('Tile responses lack bounded stale-if-error protection.'); +if (!server.includes('graceful') && !server.includes('draining connections')) { + fail('The production server lacks graceful shutdown handling.'); } if (!server.includes('OCCUMED_WORLD_SURFACE_URL')) { fail('Read-only visual validation cannot serve its candidate physical surface.'); @@ -92,7 +125,7 @@ if (!overviewBuilder.includes('mergeVectorTiles(payloads)')) { } if (!manifestBuilder.includes('version: 2')) fail('The server-only routing manifest is not version 2.'); if (!manifestBuilder.includes('overviewAsset')) fail('The manifest does not declare the consolidated overview archive.'); -if (!manifestBuilder.includes('surfaceAsset')) fail('The manifest does not declare the worldwide land surface.'); +if (!manifestBuilder.includes('surfaceAsset')) fail('The manifest does not declare the worldwide physical surface.'); if (!manifestBuilder.includes("surfaceLayers: ['land', 'landcover', 'depth']")) { fail('The manifest does not document the physical surface archive schema.'); } @@ -131,6 +164,9 @@ if (!readme.includes('/tiles/{z}/{x}/{y}.pbf')) { if (readme.includes('regional source routing') || readme.includes('global open-vector fallback')) { fail('The documentation still describes two-map routing.'); } +if (pkg.scripts?.['check:hardening'] !== 'node scripts/check-world-hardening.mjs') { + fail('The chaos hardening suite is not part of the repository scripts.'); +} if (failures.length) { console.error('Virtual PMTiles integration validation failed:'); @@ -138,4 +174,4 @@ if (failures.length) { process.exit(1); } -console.log('PMTiles storage integration validated behind one permanent worldwide vector endpoint with parent-tile retention and non-overlapping surface geometry.'); +console.log('PMTiles storage integration validated behind one permanent worldwide vector endpoint with a continuous physical foundation, bounded upstream work, circuit breaking, stale recovery, vector-tile budgets, and hardened HTTP delivery.'); From 1629d87b11be5cbcb2156ec74288d61ce8a0c864 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:33:35 -0700 Subject: [PATCH 23/85] Add candidate HTTP hardening and concurrency gate --- scripts/check-world-http-hardening.mjs | 165 +++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 scripts/check-world-http-hardening.mjs diff --git a/scripts/check-world-http-hardening.mjs b/scripts/check-world-http-hardening.mjs new file mode 100644 index 00000000..d836bdf3 --- /dev/null +++ b/scripts/check-world-http-hardening.mjs @@ -0,0 +1,165 @@ +import assert from 'node:assert/strict'; +import http from 'node:http'; +import { gunzipSync } from 'node:zlib'; +import { validateVectorTilePayload } from '../src/server/tile-safety.js'; + +const origin = new URL(process.env.OCCUMED_PREVIEW_ORIGIN || 'http://127.0.0.1:4173'); +const deadlineMs = Number(process.env.OCCUMED_HTTP_HARDENING_DEADLINE_MS || 90_000); + +function request(pathname, { + method = 'GET', + headers = {}, + timeoutMs = 45_000 +} = {}) { + return new Promise((resolve, reject) => { + const startedAt = performance.now(); + const req = http.request({ + protocol: origin.protocol, + hostname: origin.hostname, + port: origin.port, + path: pathname, + method, + timeout: timeoutMs, + headers: { + Host: origin.host, + 'X-Request-Id': `hardening-${Math.random().toString(16).slice(2)}`, + ...headers + } + }, (response) => { + const chunks = []; + response.on('data', (chunk) => chunks.push(Buffer.from(chunk))); + response.on('end', () => resolve({ + status: response.statusCode, + headers: response.headers, + body: Buffer.concat(chunks), + durationMs: performance.now() - startedAt + })); + }); + req.on('timeout', () => req.destroy(new Error(`${method} ${pathname} timed out.`))); + req.on('error', reject); + req.end(); + }); +} + +async function waitForReadiness() { + const deadline = Date.now() + deadlineMs; + let last; + while (Date.now() < deadline) { + last = await request('/readyz', { timeoutMs: 12_000 }).catch((error) => ({ + status: 0, + body: Buffer.from(error.message) + })); + if (last.status === 200) return last; + await new Promise((resolve) => setTimeout(resolve, 750)); + } + throw new Error(`Candidate gateway never became ready: ${last?.status} ${last?.body?.toString('utf8')}`); +} + +const health = await request('/healthz'); +assert.equal(health.status, 200); +assert.equal(health.body.toString('utf8'), 'ok'); +assert.equal(health.headers['x-content-type-options'], 'nosniff'); +assert(health.headers['permissions-policy']?.includes('camera=()')); +assert(health.headers['x-occumed-request-id']); + +const ready = await waitForReadiness(); +const readyDocument = JSON.parse(ready.body.toString('utf8')); +assert.equal(readyDocument.ready, true); +assert.equal(readyDocument.shuttingDown, false); +assert(readyDocument.regions >= 700, `Readiness reported only ${readyDocument.regions} regions.`); + +const plain = await request('/tiles/0/0/0.pbf', { + headers: { 'Accept-Encoding': 'identity' } +}); +assert.equal(plain.status, 200); +assert.equal(plain.headers['content-type'], 'application/x-protobuf'); +assert.equal(plain.headers['x-occumed-tileset'], 'virtual-worldwide-v2'); +assert.equal(plain.headers['content-encoding'], undefined); +assert(plain.headers.etag); +assert(plain.headers['server-timing']?.includes('tile;dur=')); +assert(plain.headers['cache-control']?.includes('stale-if-error=86400')); +validateVectorTilePayload(plain.body, { label: 'plain HTTP globe tile' }); + +const gzip = await request('/tiles/0/0/0.pbf', { + headers: { 'Accept-Encoding': 'gzip' } +}); +assert.equal(gzip.status, 200); +assert.equal(gzip.headers['content-encoding'], 'gzip'); +assert.equal(gzip.headers.etag, plain.headers.etag); +const uncompressed = gunzipSync(gzip.body); +assert(uncompressed.equals(plain.body), 'Gzip and identity responses do not contain the same vector tile.'); + +const notModified = await request('/tiles/0/0/0.pbf', { + headers: { + 'Accept-Encoding': 'identity', + 'If-None-Match': plain.headers.etag + } +}); +assert.equal(notModified.status, 304); +assert.equal(notModified.body.length, 0); + +const head = await request('/tiles/0/0/0.pbf', { + method: 'HEAD', + headers: { 'Accept-Encoding': 'identity' } +}); +assert.equal(head.status, 200); +assert.equal(head.body.length, 0); +assert.equal(head.headers.etag, plain.headers.etag); + +const invalidTile = await request('/tiles/17/0/0.pbf'); +assert.equal(invalidTile.status, 404); +assert(!invalidTile.body.toString('utf8').includes('')); + +const invalidCoordinate = await request('/tiles/2/4/0.pbf'); +assert.equal(invalidCoordinate.status, 404); + +const missingAsset = await request('/assets/does-not-exist.pbf', { + headers: { Accept: 'application/x-protobuf' } +}); +assert.equal(missingAsset.status, 404); +assert(!missingAsset.body.toString('utf8').includes('')); + +const invalidMethod = await request('/tiles/0/0/0.pbf', { method: 'POST' }); +assert.equal(invalidMethod.status, 405); +assert.equal(invalidMethod.headers.allow, 'GET, HEAD, OPTIONS'); + +const sameTileBurst = await Promise.all( + Array.from({ length: 24 }, () => request('/tiles/2/1/1.pbf', { + headers: { 'Accept-Encoding': 'identity' } + })) +); +for (const response of sameTileBurst) { + assert.equal(response.status, 200); + assert(response.headers.etag); + validateVectorTilePayload(response.body, { label: 'coalesced burst tile' }); +} +assert.equal( + new Set(sameTileBurst.map((response) => response.headers.etag)).size, + 1, + 'Concurrent requests for one tile did not resolve deterministically.' +); + +const worldTiles = []; +for (let y = 0; y < 4; y += 1) { + for (let x = 0; x < 4; x += 1) worldTiles.push(`/tiles/2/${x}/${y}.pbf`); +} +const distinctBurst = await Promise.all( + worldTiles.map((pathname) => request(pathname, { + headers: { 'Accept-Encoding': 'identity' } + })) +); +for (let index = 0; index < distinctBurst.length; index += 1) { + const response = distinctBurst[index]; + assert.equal(response.status, 200, `${worldTiles[index]} returned ${response.status}.`); + validateVectorTilePayload(response.body, { label: worldTiles[index] }); +} + +const durations = [...sameTileBurst, ...distinctBurst] + .map((response) => response.durationMs) + .sort((left, right) => left - right); +const p95 = durations[Math.min(durations.length - 1, Math.floor(durations.length * 0.95))]; +assert(p95 < 45_000, `Candidate HTTP p95 exceeded the request deadline: ${Math.round(p95)}ms.`); + +console.log( + `HTTP hardening passed: readiness, security headers, identity/gzip parity, ETag 304, HEAD, strict 404/405 handling, 24-request coalescing, 16 distinct world tiles, and p95 ${Math.round(p95)}ms.` +); From 212236da44b2a568e10dd04df66a0fd52ba2ad2f Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:34:16 -0700 Subject: [PATCH 24/85] Run candidate HTTP hardening before visual sweeps --- .github/workflows/validate-continuous-zoom.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-continuous-zoom.yml b/.github/workflows/validate-continuous-zoom.yml index a4b96728..e4aef20b 100644 --- a/.github/workflows/validate-continuous-zoom.yml +++ b/.github/workflows/validate-continuous-zoom.yml @@ -86,18 +86,19 @@ jobs: - name: Install Chromium run: npx playwright install --with-deps chromium - - name: Run continuous zoom and pan gate + - name: Run hardened HTTP, continuous zoom, and pan gates run: | set -euo pipefail mkdir -p continuous-motion OCCUMED_WORLD_OVERVIEW_URL="http://127.0.0.1:4173/virtual-assets/${OVERVIEW_ASSET}" \ OCCUMED_WORLD_SURFACE_URL="http://127.0.0.1:4173/virtual-assets/${SURFACE_ASSET}" \ + OCCUMED_ENABLE_DIAGNOSTICS="true" \ node server.mjs > continuous-motion/server.log 2>&1 & server_pid="$!" cleanup() { status="$?" if [ "$status" -ne 0 ]; then - tail -300 continuous-motion/server.log || true + tail -400 continuous-motion/server.log || true fi kill "$server_pid" 2>/dev/null || true exit "$status" @@ -112,12 +113,17 @@ jobs: done curl --fail http://127.0.0.1:4173/healthz + node scripts/check-world-http-hardening.mjs + OCCUMED_PREVIEW_OUTPUT=continuous-motion/results \ node scripts/validate-continuous-zoom.mjs OCCUMED_PREVIEW_OUTPUT=continuous-motion/all-zoom-levels \ node scripts/validate-all-zoom-levels.mjs + curl --fail --silent http://127.0.0.1:4173/internal/tile-health \ + > continuous-motion/tile-health.json + - uses: actions/upload-artifact@v4 if: always() with: @@ -126,6 +132,7 @@ jobs: continuous-motion/results continuous-motion/all-zoom-levels continuous-motion/server.log + continuous-motion/tile-health.json dist/virtual-assets/occumed-world-overview.pmtiles if-no-files-found: error retention-days: 14 From 6f2ec6b409ada5f3b2d848d08c6b67db6b6c34f6 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:37:35 -0700 Subject: [PATCH 25/85] Add production dependency and lockfile security gate --- .github/workflows/security-hardening.yml | 53 ++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/security-hardening.yml diff --git a/.github/workflows/security-hardening.yml b/.github/workflows/security-hardening.yml new file mode 100644 index 00000000..79ee1dc1 --- /dev/null +++ b/.github/workflows/security-hardening.yml @@ -0,0 +1,53 @@ +name: Validate Map Security Hardening + +on: + workflow_dispatch: + pull_request: + branches: + - main + paths: + - .github/workflows/** + - package.json + - package-lock.json + - scripts/check-world-hardening.mjs + - src/server/** + - server.mjs + +permissions: + contents: read + +concurrency: + group: occumed-map-security-${{ github.ref }} + cancel-in-progress: true + +jobs: + production-security: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + + - name: Install exactly from the committed lockfile + run: npm ci --ignore-scripts + + - name: Reject production dependency vulnerabilities + run: npm audit --omit=dev --audit-level=high + + - name: Validate production dependency graph + run: npm ls --omit=dev --all + + - name: Reject remote Git package sources + run: | + set -euo pipefail + if grep -E '"resolved": "(?:git\+|git://|ssh://|https://github\.com/.+\.git)' package-lock.json; then + echo 'Remote Git package sources are not allowed in the production lockfile.' >&2 + exit 1 + fi + + - name: Run deterministic gateway chaos guards + run: node scripts/check-world-hardening.mjs From 5fc67b347726b3bfbbb120c6bccd19e65050a037 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:40:29 -0700 Subject: [PATCH 26/85] Validate every MVT payload before merge or overscale --- src/server/mvt.js | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/server/mvt.js b/src/server/mvt.js index e8e2154f..d0ab1c15 100644 --- a/src/server/mvt.js +++ b/src/server/mvt.js @@ -2,12 +2,16 @@ import { createHash } from 'node:crypto'; import { VectorTile } from '@mapbox/vector-tile'; import Pbf from 'pbf'; import vtpbf from 'vt-pbf'; +import { validateVectorTilePayload } from './tile-safety.js'; export const EMPTY_MVT = Buffer.from(vtpbf.fromVectorTileJs({ layers: {} })); -function decodeTile(data) { - const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); - return new VectorTile(new Pbf(bytes)); +function decodeTile(data, { + label = 'vector tile', + coordinateScale = 128 +} = {}) { + const validated = validateVectorTilePayload(data, { label, coordinateScale }); + return new VectorTile(new Pbf(new Uint8Array(validated.bytes))); } function stableProperties(properties) { @@ -164,8 +168,11 @@ export function mergeVectorTiles(payloads, { const blockedLayers = excludeLayers ? new Set(excludeLayers) : null; const layers = new Map(); - for (const payload of payloads.filter(Boolean)) { - const tile = decodeTile(payload); + for (const [payloadIndex, payload] of payloads.filter(Boolean).entries()) { + const tile = decodeTile(payload, { + label: `MVT merge input ${payloadIndex + 1}`, + coordinateScale + }); for (const [name, sourceLayer] of Object.entries(tile.layers)) { if (allowedLayers && !allowedLayers.has(name)) continue; if (blockedLayers?.has(name)) continue; @@ -212,7 +219,12 @@ export function mergeVectorTiles(payloads, { } if (!Object.keys(encodedLayers).length) return EMPTY_MVT; - return Buffer.from(vtpbf.fromVectorTileJs({ layers: encodedLayers })); + const encoded = Buffer.from(vtpbf.fromVectorTileJs({ layers: encodedLayers })); + validateVectorTilePayload(encoded, { + label: 'merged MVT output', + coordinateScale + }); + return encoded; } function interpolateAtX(start, end, x) { @@ -302,7 +314,10 @@ export function overscaleVectorLayer(payload, { throw new Error('The target zoom must be greater than or equal to the source zoom.'); } - const tile = decodeTile(payload); + const tile = decodeTile(payload, { + label: `MVT overscale input for ${layerName}`, + coordinateScale: 128 + }); const sourceLayer = tile.layers[layerName]; if (!sourceLayer) return EMPTY_MVT; if (targetZoom === sourceZoom) { @@ -332,7 +347,7 @@ export function overscaleVectorLayer(payload, { } if (!features.length) return EMPTY_MVT; - return Buffer.from(vtpbf.fromVectorTileJs({ + const encoded = Buffer.from(vtpbf.fromVectorTileJs({ layers: { [layerName]: new MergedLayer( layerName, @@ -342,10 +357,15 @@ export function overscaleVectorLayer(payload, { ) } })); + validateVectorTilePayload(encoded, { + label: `overscaled MVT output for ${layerName}`, + coordinateScale: 128 + }); + return encoded; } export function inspectVectorTile(payload) { - const tile = decodeTile(payload); + const tile = decodeTile(payload, { label: 'MVT inspection input' }); return Object.fromEntries( Object.entries(tile.layers).map(([name, layer]) => [ name, From cd5028ff317d7731d98678aec0d926f172077805 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:41:56 -0700 Subject: [PATCH 27/85] Guard pre-merge and pre-overscale MVT validation --- scripts/check-pmtiles-integration.mjs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/check-pmtiles-integration.mjs b/scripts/check-pmtiles-integration.mjs index c45af303..102d982c 100644 --- a/scripts/check-pmtiles-integration.mjs +++ b/scripts/check-pmtiles-integration.mjs @@ -11,6 +11,7 @@ const [ manifestBuilder, overviewBuilder, gateway, + mvt, retryingSource, tileSafety, server, @@ -25,6 +26,7 @@ const [ fs.readFile(path.join(root, 'scripts/build-world-manifest.mjs'), 'utf8'), fs.readFile(path.join(root, 'scripts/build-world-overview.mjs'), 'utf8'), fs.readFile(path.join(root, 'src/server/world-tile-gateway.js'), 'utf8'), + fs.readFile(path.join(root, 'src/server/mvt.js'), 'utf8'), fs.readFile(path.join(root, 'src/server/pmtiles-source.js'), 'utf8'), fs.readFile(path.join(root, 'src/server/tile-safety.js'), 'utf8'), fs.readFile(path.join(root, 'server.mjs'), 'utf8'), @@ -72,7 +74,7 @@ if (!helper.includes('source.tiles = source.tiles.map')) { fail('Permanent vector tile templates are not resolved to the style origin.'); } -if (!server.includes("const tileMatch = /^\\/tiles\\/(\\d+)\\/(\\d+)\\/(\\d+)\\.pbf$/")) { +if (!server.includes("const tileMatch = /^\/tiles\/(\d+)\/(\d+)\/(\d+)\.pbf$/")) { fail('The server is missing the single virtual Z/X/Y endpoint.'); } if (server.includes('/world-tiles/')) fail('The server still publishes browser-visible regional archive paths.'); @@ -109,6 +111,15 @@ if (!retryingSource.includes('maxRangeBytes') || !retryingSource.includes('isRet if (!tileSafety.includes('validateVectorTilePayload') || !tileSafety.includes('maxTotalPoints')) { fail('Vector tiles are not protected by decode and geometry budgets.'); } +if (!mvt.includes("import { validateVectorTilePayload } from './tile-safety.js'")) { + fail('MVT merge and overscale operations bypass the safety validator.'); +} +if (!mvt.includes('MVT merge input') || !mvt.includes('MVT overscale input')) { + fail('Upstream MVT payloads are not validated before merge and overscale processing.'); +} +if (!mvt.includes('merged MVT output') || !mvt.includes('overscaled MVT output')) { + fail('Encoded MVT outputs are not revalidated before delivery or caching.'); +} if (!server.includes('gzipAsync')) fail('Tile compression still blocks the Node event loop.'); if (!server.includes("url.pathname === '/readyz'")) fail('The production server lacks a readiness endpoint.'); if (!server.includes('maxConcurrentTileRequests')) fail('The HTTP tile endpoint lacks a concurrency limit.'); @@ -174,4 +185,4 @@ if (failures.length) { process.exit(1); } -console.log('PMTiles storage integration validated behind one permanent worldwide vector endpoint with a continuous physical foundation, bounded upstream work, circuit breaking, stale recovery, vector-tile budgets, and hardened HTTP delivery.'); +console.log('PMTiles storage integration validated behind one permanent worldwide vector endpoint with a continuous physical foundation, pre-merge and post-encode MVT budgets, bounded upstream work, circuit breaking, stale recovery, and hardened HTTP delivery.'); From 8c1ce478aaa329ae153bb7a82f3577f4ff94bc03 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:44:19 -0700 Subject: [PATCH 28/85] Capture complete hardened build diagnostics --- .github/workflows/security-hardening.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/security-hardening.yml b/.github/workflows/security-hardening.yml index 79ee1dc1..4512e3da 100644 --- a/.github/workflows/security-hardening.yml +++ b/.github/workflows/security-hardening.yml @@ -9,8 +9,9 @@ on: - .github/workflows/** - package.json - package-lock.json - - scripts/check-world-hardening.mjs + - scripts/** - src/server/** + - src/occumed-map.js - server.mjs permissions: @@ -23,7 +24,7 @@ concurrency: jobs: production-security: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -51,3 +52,16 @@ jobs: - name: Run deterministic gateway chaos guards run: node scripts/check-world-hardening.mjs + + - name: Capture complete hardened production build + run: | + set -o pipefail + npm run build 2>&1 | tee hardening-build.log + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: hardening-build-${{ github.sha }} + path: hardening-build.log + if-no-files-found: error + retention-days: 7 From 829a4058df6453d75e2580544644d49d173d7e19 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:45:56 -0700 Subject: [PATCH 29/85] Install pinned native dependencies before hardened build --- .github/workflows/security-hardening.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-hardening.yml b/.github/workflows/security-hardening.yml index 4512e3da..fab3c7a2 100644 --- a/.github/workflows/security-hardening.yml +++ b/.github/workflows/security-hardening.yml @@ -34,7 +34,7 @@ jobs: cache: npm - name: Install exactly from the committed lockfile - run: npm ci --ignore-scripts + run: npm ci - name: Reject production dependency vulnerabilities run: npm audit --omit=dev --audit-level=high From b6a1479d3345148376a35f5a74e6297f3a66a736 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:48:34 -0700 Subject: [PATCH 30/85] Validate the virtual tile route by behavior contract --- scripts/check-pmtiles-integration.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/check-pmtiles-integration.mjs b/scripts/check-pmtiles-integration.mjs index 102d982c..27709374 100644 --- a/scripts/check-pmtiles-integration.mjs +++ b/scripts/check-pmtiles-integration.mjs @@ -74,8 +74,8 @@ if (!helper.includes('source.tiles = source.tiles.map')) { fail('Permanent vector tile templates are not resolved to the style origin.'); } -if (!server.includes("const tileMatch = /^\/tiles\/(\d+)\/(\d+)\/(\d+)\.pbf$/")) { - fail('The server is missing the single virtual Z/X/Y endpoint.'); +if (!server.includes('const tileMatch =') || !server.includes('await serveVirtualTile(request, response, coordinates)')) { + fail('The server is missing the single virtual Z/X/Y route contract.'); } if (server.includes('/world-tiles/')) fail('The server still publishes browser-visible regional archive paths.'); if (!gateway.includes('regionsForTile(zoom, x, y)')) fail('The gateway does not route each requested tile by bounds.'); From c12ca8b2af525b78c919b130bd9b5c1939a81242 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 19:51:00 -0700 Subject: [PATCH 31/85] Allow canonical empty protobuf vector tiles --- src/server/tile-safety.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/server/tile-safety.js b/src/server/tile-safety.js index 70d06afc..1753c818 100644 --- a/src/server/tile-safety.js +++ b/src/server/tile-safety.js @@ -33,7 +33,11 @@ export function validateVectorTilePayload(payload, { const name = safeLabel(label); const bytes = Buffer.from(payload || []); const byteLimit = boundedInteger(maxBytes, DEFAULT_MAX_BYTES, 1_024, 96 * 1024 * 1024); - if (!bytes.byteLength || bytes.byteLength > byteLimit) { + // A protobuf message with no fields is canonically encoded as zero bytes. + // Mapbox Vector Tile uses protobuf, so a zero-byte payload is the valid empty + // tile used internally when no source layers are present. Non-empty malformed + // payloads still fail during decoding below. + if (bytes.byteLength > byteLimit) { throw new Error(`${name} has unsafe encoded size ${bytes.byteLength}.`); } From d0bb88324220f17100f16476dc6da1ee28aa03a4 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 21:51:04 -0700 Subject: [PATCH 32/85] Keep worldwide landcover continuous through native surface zooms --- scripts/prepare-world-landcover.mjs | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/scripts/prepare-world-landcover.mjs b/scripts/prepare-world-landcover.mjs index e494a4ce..82532916 100644 --- a/scripts/prepare-world-landcover.mjs +++ b/scripts/prepare-world-landcover.mjs @@ -2,6 +2,8 @@ import fs from 'node:fs/promises'; +const SURFACE_MAX_ZOOM = 10; + function parseArgs(argv) { const options = {}; for (let index = 0; index < argv.length; index += 1) { @@ -33,7 +35,7 @@ function classifiedFeature(feature, className) { geometry: feature.geometry, tippecanoe: { minzoom: 0, - maxzoom: 5 + maxzoom: SURFACE_MAX_ZOOM } }; } @@ -45,10 +47,10 @@ const [land, geography, glaciers] = await Promise.all([ readCollection(options.glaciers) ]); -// The exact exported land chip is the neutral base. This generalized layer -// supplies the vegetation, desert, tundra, and ice classes that give the globe -// the same green physical identity before the detailed regional landcover -// takes over. It uses the same `landcover` schema as the regional shards. +// This is the permanent worldwide vegetation/terrain-class foundation. It must +// survive the overview-to-regional routing boundary and remain present through +// the physical surface archive's native maximum zoom. Regional landuse, parks, +// roads, labels, and buildings add detail above it; they never replace it. const features = [ ...land.features.map((feature) => classifiedFeature(feature, 'grass')), ...geography.features @@ -60,6 +62,13 @@ const features = [ ...glaciers.features.map((feature) => classifiedFeature(feature, 'snow')) ].filter(Boolean); +if (!features.length) throw new Error('Worldwide landcover preparation produced no features.'); +for (const feature of features) { + if (feature.tippecanoe?.minzoom !== 0 || feature.tippecanoe?.maxzoom !== SURFACE_MAX_ZOOM) { + throw new Error('Worldwide landcover contains a zoom cutoff that can switch the physical foundation.'); + } +} + await fs.writeFile( options.output, `${JSON.stringify({ type: 'FeatureCollection', features })}\n` @@ -70,4 +79,6 @@ const classCounts = features.reduce((counts, feature) => { counts[className] = (counts[className] || 0) + 1; return counts; }, {}); -console.log(`Prepared generalized worldwide landcover: ${JSON.stringify(classCounts)}.`); +console.log( + `Prepared continuous worldwide landcover through zoom ${SURFACE_MAX_ZOOM}: ${JSON.stringify(classCounts)}.` +); From f50ab1ae422cef35dbcc18ecfcdb2cb15280cc82 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 21:52:08 -0700 Subject: [PATCH 33/85] Strengthen the tracked exterior atmosphere bloom --- src/styles.css | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/styles.css b/src/styles.css index 2e4487c9..c4a731f2 100644 --- a/src/styles.css +++ b/src/styles.css @@ -68,22 +68,30 @@ body { top: var(--occumed-globe-bloom-y, 50%); width: var(--occumed-globe-diameter, 0px); height: var(--occumed-globe-diameter, 0px); + border: 1px solid rgba(250, 254, 255, 0.98); border-radius: 50%; - transform: translate3d(-50%, -50%, 0); + transform: translate3d(-50%, -50%, 0) scale(1.006); + transform-origin: center; pointer-events: none; opacity: var(--occumed-globe-bloom-opacity, 0); background: radial-gradient( circle at center, - transparent calc(100% - 2px), - rgba(245, 253, 255, 0.98) calc(100% - 1px), - rgba(245, 253, 255, 0.98) 100% + transparent calc(100% - 4px), + rgba(206, 239, 255, 0.22) calc(100% - 3px), + rgba(239, 250, 255, 0.82) calc(100% - 1.5px), + rgba(255, 255, 255, 1) 100% ); + box-shadow: + 0 0 8px 2px rgba(250, 254, 255, 0.96), + 0 0 22px 7px rgba(184, 230, 255, 0.72), + 0 0 48px 14px rgba(121, 188, 236, 0.42), + 0 0 84px 24px rgba(79, 160, 220, 0.18); filter: - drop-shadow(0 0 5px rgba(245, 253, 255, 0.96)) - drop-shadow(0 0 14px rgba(184, 230, 255, 0.74)) - drop-shadow(0 0 30px rgba(121, 188, 236, 0.34)); + drop-shadow(0 0 6px rgba(250, 254, 255, 0.98)) + drop-shadow(0 0 16px rgba(184, 230, 255, 0.82)) + drop-shadow(0 0 30px rgba(121, 188, 236, 0.46)); mix-blend-mode: screen; - will-change: left, top, width, height, opacity; + will-change: left, top, width, height, opacity, transform; } .map-header { From f1d83c62c35c0e6a3ec184f14543b04adfef4045 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 21:55:22 -0700 Subject: [PATCH 34/85] Make polygon and foundation validation exhaustive and forensic --- scripts/capture-polygon-regression.mjs | 161 +++++++++++++++++-------- 1 file changed, 108 insertions(+), 53 deletions(-) diff --git a/scripts/capture-polygon-regression.mjs b/scripts/capture-polygon-regression.mjs index d6b0fe06..a6dc45f3 100644 --- a/scripts/capture-polygon-regression.mjs +++ b/scripts/capture-polygon-regression.mjs @@ -9,28 +9,57 @@ const outputDir = path.resolve( await fs.mkdir(outputDir, { recursive: true }); const expectedTemplate = `${origin}/tiles/{z}/{x}/{y}.pbf`; -const browser = await chromium.launch({ headless: true }); +const continuityZooms = [ + 0, 1, 1.65, 2, 2.43, 3, 4, 5, 5.5, 5.9, 6, 6.1, 6.5, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +]; +const reportPath = path.join(outputDir, 'polygon-regression-report.json'); +const report = { + generatedAt: new Date().toISOString(), + origin, + mode: 'exhaustive-polygon-foundation-and-atmosphere-regression', + expectedTemplate, + continuityZooms, + results: {}, + pageErrors: [], + networkFailures: [], + externalVectorRequests: [], + fatalError: null, + passed: false +}; +function serializeError(error) { + return { + name: error?.name || 'Error', + message: error?.message || String(error), + stack: error?.stack || null + }; +} + +async function persistReport() { + report.externalVectorRequests = [...new Set(report.externalVectorRequests)]; + await fs.writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`); +} + +let browser; try { + browser = await chromium.launch({ headless: true }); const context = await browser.newContext({ viewport: { width: 1440, height: 1000 }, deviceScaleFactor: 2, colorScheme: 'dark' }); const page = await context.newPage(); - const pageErrors = []; - const networkFailures = []; - const externalVectorRequests = []; - page.on('pageerror', (error) => pageErrors.push(error.message)); + page.on('pageerror', (error) => report.pageErrors.push(error.message)); page.on('request', (request) => { const url = request.url(); if (/\.pbf(?:$|\?)/i.test(url) && !url.startsWith(`${origin}/tiles/`)) { - externalVectorRequests.push(url); + report.externalVectorRequests.push(url); } }); page.on('requestfailed', (request) => { - networkFailures.push({ + report.networkFailures.push({ type: 'requestfailed', url: request.url(), error: request.failure()?.errorText || 'unknown request failure' @@ -38,7 +67,7 @@ try { }); page.on('response', (response) => { if (response.status() >= 400) { - networkFailures.push({ + report.networkFailures.push({ type: 'http', url: response.url(), status: response.status() @@ -53,7 +82,6 @@ try { { timeout: 90_000 } ); - const continuityZooms = [1.65, 2.43, 3.5, 5.5, 6.5]; const views = [ { name: 'north-america-z2', center: [-102, 36], zoom: 2.43, requiresAtmosphereBloom: true }, { name: 'central-pacific-z2', center: [175, 7], zoom: 2.43, requiresAtmosphereBloom: true }, @@ -76,7 +104,6 @@ try { requiredRenderedLayers: ['depth'] })) ]; - const results = {}; for (const view of views) { await page.evaluate(({ center, zoom }) => { @@ -94,7 +121,7 @@ try { null, { timeout: 90_000 } ); - await page.waitForTimeout(500); + await page.waitForTimeout(350); const diagnostics = await page.evaluate((expectedTemplate) => { const map = globalThis.__OCCUMED_MAP__; @@ -119,6 +146,15 @@ try { const bloom = document.querySelector('.occumed-atmosphere-bloom'); const bloomStyle = bloom ? getComputedStyle(bloom) : null; const bloomRect = bloom?.getBoundingClientRect() || null; + const containerRect = map.getCanvasContainer().getBoundingClientRect(); + const projectedCenter = map.project(map.getCenter()); + const expectedCenterX = containerRect.left + projectedCenter.x; + const expectedCenterY = containerRect.top + projectedCenter.y; + const expectedDiameter = ( + ((512 * (2 ** map.getZoom())) / (Math.PI * 2)) * 2 * 1.006 + ); + const actualCenterX = bloomRect ? bloomRect.left + (bloomRect.width / 2) : 0; + const actualCenterY = bloomRect ? bloomRect.top + (bloomRect.height / 2) : 0; return { center: map.getCenter().toArray(), zoom: map.getZoom(), @@ -134,84 +170,103 @@ try { hidden: Boolean(bloom?.hidden), opacity: Number(bloomStyle?.opacity || 0), filter: bloomStyle?.filter || 'none', + boxShadow: bloomStyle?.boxShadow || 'none', + borderColor: bloomStyle?.borderColor || 'transparent', + borderWidth: Number.parseFloat(bloomStyle?.borderWidth || '0'), mixBlendMode: bloomStyle?.mixBlendMode || 'normal', width: bloomRect?.width || 0, - height: bloomRect?.height || 0 + height: bloomRect?.height || 0, + centerErrorPx: bloomRect + ? Math.hypot(actualCenterX - expectedCenterX, actualCenterY - expectedCenterY) + : null, + diameterErrorPx: bloomRect ? Math.abs(bloomRect.width - expectedDiameter) : null } }; }, expectedTemplate); + const screenshot = await page.screenshot({ + path: path.join(outputDir, `${view.name}.png`), + fullPage: false + }); + + const failures = []; if (!diagnostics.sourceIsPermanent) { - throw new Error(`${view.name} changed the permanent vector source.`); + failures.push(`${view.name} changed the permanent vector source.`); } if (diagnostics.renderedFeatureCount <= 0) { - throw new Error(`${view.name} rendered no worldwide vector features.`); + failures.push(`${view.name} rendered no worldwide vector features.`); } for (const sourceLayer of view.requiredSourceLayers || []) { if ((diagnostics.sourceFeatureCounts[sourceLayer] || 0) <= 0) { - throw new Error(`${view.name} lost the ${sourceLayer} source layer at zoom ${view.zoom}.`); + failures.push(`${view.name} lost the ${sourceLayer} source layer at zoom ${view.zoom}.`); } } for (const sourceLayer of view.requiredRenderedLayers || []) { if ((diagnostics.renderedSourceLayerCounts[sourceLayer] || 0) <= 0) { - throw new Error(`${view.name} stopped rendering the ${sourceLayer} foundation at zoom ${view.zoom}.`); + failures.push(`${view.name} stopped rendering the ${sourceLayer} foundation at zoom ${view.zoom}.`); } } if (view.requiresAtmosphereBloom) { const bloom = diagnostics.atmosphereBloom; if (!bloom.exists || bloom.hidden || bloom.opacity < 0.95) { - throw new Error(`${view.name} does not show the full-strength globe atmosphere bloom.`); + failures.push(`${view.name} does not show the full-strength globe atmosphere bloom.`); } - if (bloom.filter === 'none' || bloom.mixBlendMode !== 'screen') { - throw new Error(`${view.name} has a hard rim instead of the luminous white-blue bloom.`); + if ( + bloom.filter === 'none' || + bloom.boxShadow === 'none' || + bloom.mixBlendMode !== 'screen' || + bloom.borderWidth < 1 + ) { + failures.push(`${view.name} has a hard rim instead of the layered luminous white-blue bloom.`); } - if (bloom.width < 150 || Math.abs(bloom.width - bloom.height) > 1) { - throw new Error(`${view.name} atmosphere bloom does not track the rendered globe.`); + if ( + bloom.width < 150 || + Math.abs(bloom.width - bloom.height) > 2 || + bloom.centerErrorPx === null || bloom.centerErrorPx > 3 || + bloom.diameterErrorPx === null || bloom.diameterErrorPx > 4 + ) { + failures.push(`${view.name} atmosphere bloom does not precisely track the rendered globe.`); } } - - const screenshot = await page.screenshot({ - path: path.join(outputDir, `${view.name}.png`), - fullPage: false - }); if (screenshot.length < 25_000) { - throw new Error(`${view.name} produced an unexpectedly empty screenshot.`); + failures.push(`${view.name} produced an unexpectedly empty screenshot.`); } - results[view.name] = { ...diagnostics, screenshotBytes: screenshot.length }; - } - const report = { - generatedAt: new Date().toISOString(), - origin, - mode: 'rebuilt-overview-polygon-layer-continuity-and-atmosphere-regression', - expectedTemplate, - continuityZooms, - results, - pageErrors, - networkFailures, - externalVectorRequests: [...new Set(externalVectorRequests)], - passed: - pageErrors.length === 0 && - networkFailures.length === 0 && - externalVectorRequests.length === 0 - }; + report.results[view.name] = { + ...diagnostics, + screenshotBytes: screenshot.length, + failures + }; + await persistReport(); - await fs.writeFile( - path.join(outputDir, 'polygon-regression-report.json'), - `${JSON.stringify(report, null, 2)}\n` - ); + if (failures.length) { + throw new Error(failures.join(' ')); + } + } + + report.externalVectorRequests = [...new Set(report.externalVectorRequests)]; + report.passed = + report.pageErrors.length === 0 && + report.networkFailures.length === 0 && + report.externalVectorRequests.length === 0; + await persistReport(); if (!report.passed) { - throw new Error(`Polygon, layer continuity, and atmosphere validation failed: ${JSON.stringify({ - pageErrors, - networkFailures, + throw new Error(`Polygon, foundation, and atmosphere validation failed: ${JSON.stringify({ + pageErrors: report.pageErrors, + networkFailures: report.networkFailures, externalVectorRequests: report.externalVectorRequests })}`); } console.log( - `Rendered ${views.length} rebuilt-overview views with continuous physical layers and a tracked exterior atmosphere bloom.` + `Rendered ${views.length} exhaustive views with continuous land, landcover, depth, and a tracked exterior atmosphere bloom.` ); +} catch (error) { + report.fatalError = serializeError(error); + report.passed = false; + await persistReport().catch(() => {}); + throw error; } finally { - await browser.close(); + await browser?.close(); } From d66b83023935fb01eddb81e504e7a687b011dcc0 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 21:56:30 -0700 Subject: [PATCH 35/85] Stress every zoom across land ocean and antimeridian --- scripts/validate-all-zoom-levels.mjs | 154 ++++++++++++++++++--------- 1 file changed, 101 insertions(+), 53 deletions(-) diff --git a/scripts/validate-all-zoom-levels.mjs b/scripts/validate-all-zoom-levels.mjs index 3a9ad5fc..b7f1cea3 100644 --- a/scripts/validate-all-zoom-levels.mjs +++ b/scripts/validate-all-zoom-levels.mjs @@ -9,28 +9,51 @@ const outputDir = path.resolve( await fs.mkdir(outputDir, { recursive: true }); const expectedTemplate = `${origin}/tiles/{z}/{x}/{y}.pbf`; -const browser = await chromium.launch({ headless: true }); +const reportPath = path.join(outputDir, 'all-zoom-levels-report.json'); +const report = { + generatedAt: new Date().toISOString(), + origin, + expectedTemplate, + sweeps: [], + pageErrors: [], + networkFailures: [], + externalVectorRequests: [], + fatalError: null, + passed: false +}; +function serializeError(error) { + return { + name: error?.name || 'Error', + message: error?.message || String(error), + stack: error?.stack || null + }; +} + +async function persistReport() { + report.externalVectorRequests = [...new Set(report.externalVectorRequests)]; + await fs.writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`); +} + +let browser; try { + browser = await chromium.launch({ headless: true }); const context = await browser.newContext({ viewport: { width: 1440, height: 1000 }, deviceScaleFactor: 2, colorScheme: 'dark' }); const page = await context.newPage(); - const pageErrors = []; - const networkFailures = []; - const externalVectorRequests = []; - page.on('pageerror', (error) => pageErrors.push(error.message)); + page.on('pageerror', (error) => report.pageErrors.push(error.message)); page.on('request', (request) => { const url = request.url(); if (/\.pbf(?:$|\?)/i.test(url) && !url.startsWith(`${origin}/tiles/`)) { - externalVectorRequests.push(url); + report.externalVectorRequests.push(url); } }); page.on('requestfailed', (request) => { - networkFailures.push({ + report.networkFailures.push({ type: 'requestfailed', url: request.url(), error: request.failure()?.errorText || 'unknown request failure' @@ -38,7 +61,7 @@ try { }); page.on('response', (response) => { if (response.status() >= 400) { - networkFailures.push({ type: 'http', url: response.url(), status: response.status() }); + report.networkFailures.push({ type: 'http', url: response.url(), status: response.status() }); } }); @@ -49,7 +72,7 @@ try { { timeout: 90_000 } ); - async function runSweep(name, center, startZoom, endZoom) { + async function runSweep({ name, center, startZoom, endZoom, requiredLayers }) { await page.evaluate(({ center, startZoom }) => { const map = globalThis.__OCCUMED_MAP__; map.jumpTo({ center, zoom: startZoom, pitch: 0, bearing: 0 }); @@ -57,12 +80,18 @@ try { }, { center, startZoom }); await page.waitForFunction( - () => { + ({ requiredLayers }) => { const map = globalThis.__OCCUMED_MAP__; - return map?.isStyleLoaded() && - map.queryRenderedFeatures().some((feature) => feature.source === 'occumed-open'); + if (!map?.isStyleLoaded() || !map.areTilesLoaded()) return false; + const rendered = map + .queryRenderedFeatures() + .filter((feature) => feature.source === 'occumed-open'); + if (!rendered.length) return false; + return requiredLayers.every((required) => + rendered.some((feature) => feature.sourceLayer === required) + ); }, - null, + { requiredLayers }, { timeout: 90_000 } ); @@ -71,7 +100,8 @@ try { center, startZoom, endZoom, - expectedTemplate + expectedTemplate, + requiredLayers }) => { const map = globalThis.__OCCUMED_MAP__; const samples = []; @@ -80,7 +110,7 @@ try { const expectedSignature = JSON.stringify({ url: null, tiles: [expectedTemplate] }); const sample = (timestamp) => { - if (timestamp - lastSampleAt < 50) return; + if (timestamp - lastSampleAt < 55) return; lastSampleAt = timestamp; const source = map.getStyle().sources?.['occumed-open'] || null; const signature = JSON.stringify({ url: source?.url || null, tiles: source?.tiles || [] }); @@ -97,17 +127,21 @@ try { timestamp, zoom: map.getZoom(), renderedFeatureCount: rendered.length, + requiredLayerCounts: Object.fromEntries( + requiredLayers.map((layer) => [layer, sourceLayers[layer] || 0]) + ), sourceLayers, + tilesLoaded: map.areTilesLoaded(), sourceSignature: signature }); }; return await new Promise((resolve, reject) => { - const durationMs = 18_000; + const durationMs = 20_000; const timeout = setTimeout(() => { map.off('render', sample); reject(new Error(`${name} full-range zoom sweep timed out.`)); - }, durationMs + 30_000); + }, durationMs + 35_000); const finish = () => { clearTimeout(timeout); @@ -119,18 +153,26 @@ try { maximumZoomGap = Math.max(maximumZoomGap, zooms[index] - zooms[index - 1]); } const blankSamples = samples.filter((entry) => entry.renderedFeatureCount === 0); + const missingFoundationSamples = samples.filter((entry) => + requiredLayers.some((layer) => (entry.requiredLayerCounts[layer] || 0) <= 0) + ); resolve({ name, center, startZoom, endZoom, + requiredLayers, sampleCount: samples.length, sourceChanged, blankSampleCount: blankSamples.length, - minimumZoom: Math.min(...zooms), - maximumZoom: Math.max(...zooms), + missingFoundationSampleCount: missingFoundationSamples.length, + firstMissingFoundationSamples: missingFoundationSamples.slice(0, 20), + minimumZoom: zooms.length ? Math.min(...zooms) : null, + maximumZoom: zooms.length ? Math.max(...zooms) : null, maximumZoomGap, - minimumFeatureCount: Math.min(...samples.map((entry) => entry.renderedFeatureCount)), + minimumFeatureCount: samples.length + ? Math.min(...samples.map((entry) => entry.renderedFeatureCount)) + : 0, samples }); }; @@ -147,48 +189,47 @@ try { essential: true }); }); - }, { name, center, startZoom, endZoom, expectedTemplate }); + }, { name, center, startZoom, endZoom, expectedTemplate, requiredLayers }); await page.screenshot({ path: path.join(outputDir, `${name}-final.png`), fullPage: false }); + report.sweeps.push(result); + await persistReport(); return result; } - const sweeps = [ - await runSweep('amazon-all-zooms-in', [-62.5, -4], 0, 14), - await runSweep('hawaii-all-zooms-out', [-157.8583, 21.3069], 14, 0) + const definitions = [ + { name: 'amazon-all-zooms-in', center: [-60, -8], startZoom: 0, endZoom: 16, requiredLayers: ['land', 'landcover'] }, + { name: 'amazon-all-zooms-out', center: [-60, -8], startZoom: 16, endZoom: 0, requiredLayers: ['land', 'landcover'] }, + { name: 'pacific-all-zooms-in', center: [-140, 0], startZoom: 0, endZoom: 16, requiredLayers: ['depth'] }, + { name: 'pacific-all-zooms-out', center: [-140, 0], startZoom: 16, endZoom: 0, requiredLayers: ['depth'] }, + { name: 'europe-all-zooms-in', center: [12, 50], startZoom: 0, endZoom: 16, requiredLayers: ['land', 'landcover'] }, + { name: 'antimeridian-all-zooms-out', center: [179, 0], startZoom: 16, endZoom: 0, requiredLayers: ['depth'] } ]; - const failedSweeps = sweeps.filter((sweep) => + for (const definition of definitions) { + await runSweep(definition); + } + + const failedSweeps = report.sweeps.filter((sweep) => sweep.sourceChanged || sweep.blankSampleCount > 0 || - sweep.sampleCount < 100 || - sweep.minimumZoom > 0.1 || - sweep.maximumZoom < 13.9 || - sweep.maximumZoomGap > 0.25 + sweep.missingFoundationSampleCount > 0 || + sweep.sampleCount < 180 || + sweep.minimumZoom === null || sweep.minimumZoom > 0.1 || + sweep.maximumZoom === null || sweep.maximumZoom < 15.9 || + sweep.maximumZoomGap > 0.3 ); - const report = { - generatedAt: new Date().toISOString(), - origin, - expectedTemplate, - sweeps, - pageErrors, - networkFailures, - externalVectorRequests: [...new Set(externalVectorRequests)], - passed: - failedSweeps.length === 0 && - pageErrors.length === 0 && - networkFailures.length === 0 && - externalVectorRequests.length === 0 - }; - - await fs.writeFile( - path.join(outputDir, 'all-zoom-levels-report.json'), - `${JSON.stringify(report, null, 2)}\n` - ); + report.externalVectorRequests = [...new Set(report.externalVectorRequests)]; + report.passed = + failedSweeps.length === 0 && + report.pageErrors.length === 0 && + report.networkFailures.length === 0 && + report.externalVectorRequests.length === 0; + await persistReport(); if (!report.passed) { throw new Error(`All-zoom validation failed: ${JSON.stringify({ @@ -197,20 +238,27 @@ try { sampleCount: sweep.sampleCount, sourceChanged: sweep.sourceChanged, blankSampleCount: sweep.blankSampleCount, + missingFoundationSampleCount: sweep.missingFoundationSampleCount, minimumZoom: sweep.minimumZoom, maximumZoom: sweep.maximumZoom, maximumZoomGap: sweep.maximumZoomGap, - minimumFeatureCount: sweep.minimumFeatureCount + minimumFeatureCount: sweep.minimumFeatureCount, + firstMissingFoundationSamples: sweep.firstMissingFoundationSamples })), - pageErrors, - networkFailures, + pageErrors: report.pageErrors, + networkFailures: report.networkFailures, externalVectorRequests: report.externalVectorRequests })}`); } console.log( - `Validated the complete zoom 0–14 range in both directions with ${sweeps.reduce((sum, sweep) => sum + sweep.sampleCount, 0)} sampled frames and zero blank frames.` + `Validated ${definitions.length} complete zoom 0–16 sweeps with ${report.sweeps.reduce((sum, sweep) => sum + sweep.sampleCount, 0)} sampled frames and no missing physical foundation layers.` ); +} catch (error) { + report.fatalError = serializeError(error); + report.passed = false; + await persistReport().catch(() => {}); + throw error; } finally { - await browser.close(); + await browser?.close(); } From 9aff4b3920aa195e771879c42e9e6a087bd86b86 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 21:57:42 -0700 Subject: [PATCH 36/85] Make continuous motion validation exhaustive and forensic --- scripts/validate-continuous-zoom.mjs | 264 ++++++++++++++++++--------- 1 file changed, 178 insertions(+), 86 deletions(-) diff --git a/scripts/validate-continuous-zoom.mjs b/scripts/validate-continuous-zoom.mjs index f20c2da6..c300f772 100644 --- a/scripts/validate-continuous-zoom.mjs +++ b/scripts/validate-continuous-zoom.mjs @@ -9,31 +9,55 @@ const outputDir = path.resolve( await fs.mkdir(outputDir, { recursive: true }); const expectedTemplate = `${origin}/tiles/{z}/{x}/{y}.pbf`; -const browser = await chromium.launch({ headless: true }); +const reportPath = path.join(outputDir, 'continuous-motion-report.json'); +const report = { + generatedAt: new Date().toISOString(), + origin, + expectedTemplate, + motions: [], + tileRequests: null, + pageErrors: [], + networkFailures: [], + externalVectorRequests: [], + fatalError: null, + passed: false +}; +function serializeError(error) { + return { + name: error?.name || 'Error', + message: error?.message || String(error), + stack: error?.stack || null + }; +} + +async function persistReport() { + report.externalVectorRequests = [...new Set(report.externalVectorRequests)]; + await fs.writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`); +} + +let browser; try { + browser = await chromium.launch({ headless: true }); const context = await browser.newContext({ viewport: { width: 1440, height: 1000 }, deviceScaleFactor: 2, colorScheme: 'dark' }); const page = await context.newPage(); - const pageErrors = []; - const networkFailures = []; - const externalVectorRequests = []; const tileStartedAt = new Map(); const tileDurations = []; - page.on('pageerror', (error) => pageErrors.push(error.message)); + page.on('pageerror', (error) => report.pageErrors.push(error.message)); page.on('request', (request) => { const url = request.url(); if (/\.pbf(?:$|\?)/i.test(url)) { tileStartedAt.set(request, performance.now()); - if (!url.startsWith(`${origin}/tiles/`)) externalVectorRequests.push(url); + if (!url.startsWith(`${origin}/tiles/`)) report.externalVectorRequests.push(url); } }); page.on('requestfailed', (request) => { - networkFailures.push({ + report.networkFailures.push({ type: 'requestfailed', url: request.url(), error: request.failure()?.errorText || 'unknown request failure' @@ -41,7 +65,7 @@ try { }); page.on('response', (response) => { if (response.status() >= 400) { - networkFailures.push({ + report.networkFailures.push({ type: 'http', url: response.url(), status: response.status() @@ -65,7 +89,7 @@ try { { timeout: 90_000 } ); - async function waitForStableView(center, zoom) { + async function waitForStableView(center, zoom, requiredLayers) { await page.evaluate(({ center, zoom }) => { const map = globalThis.__OCCUMED_MAP__; map.jumpTo({ center, zoom, pitch: 0, bearing: 0 }); @@ -73,20 +97,32 @@ try { }, { center, zoom }); await page.waitForFunction( - () => { + ({ requiredLayers }) => { const map = globalThis.__OCCUMED_MAP__; - return map?.isStyleLoaded() && map.areTilesLoaded() && - map.queryRenderedFeatures().some((feature) => feature.source === 'occumed-open'); + if (!map?.isStyleLoaded() || !map.areTilesLoaded()) return false; + const rendered = map + .queryRenderedFeatures() + .filter((feature) => feature.source === 'occumed-open'); + if (!rendered.length) return false; + return requiredLayers.every((required) => + rendered.some((feature) => feature.sourceLayer === required) + ); }, - null, + { requiredLayers }, { timeout: 90_000 } ); } - async function runMotion(name, start, end, durationMs) { - await waitForStableView(start.center, start.zoom); + async function runMotion({ name, start, end, durationMs, requiredLayers }) { + await waitForStableView(start.center, start.zoom, requiredLayers); - const result = await page.evaluate(async ({ name, end, durationMs, expectedTemplate }) => { + const result = await page.evaluate(async ({ + name, + end, + durationMs, + expectedTemplate, + requiredLayers + }) => { const map = globalThis.__OCCUMED_MAP__; const samples = []; let lastSampleAt = -Infinity; @@ -99,19 +135,27 @@ try { const expectedSignature = JSON.stringify({ url: null, tiles: [expectedTemplate] }); const sample = (timestamp) => { - if (timestamp - lastSampleAt < 80) return; + if (timestamp - lastSampleAt < 70) return; lastSampleAt = timestamp; const signature = sourceSignature(); sourceChanged ||= signature !== expectedSignature; - const vectorFeatureCount = map + const rendered = map .queryRenderedFeatures() - .filter((feature) => feature.source === 'occumed-open') - .length; + .filter((feature) => feature.source === 'occumed-open'); + const sourceLayers = {}; + for (const feature of rendered) { + const layer = feature.sourceLayer || 'unknown'; + sourceLayers[layer] = (sourceLayers[layer] || 0) + 1; + } samples.push({ timestamp, zoom: map.getZoom(), center: map.getCenter().toArray(), - vectorFeatureCount, + vectorFeatureCount: rendered.length, + requiredLayerCounts: Object.fromEntries( + requiredLayers.map((layer) => [layer, sourceLayers[layer] || 0]) + ), + sourceLayers, tilesLoaded: map.areTilesLoaded(), sourceSignature: signature }); @@ -121,13 +165,16 @@ try { const timeout = setTimeout(() => { map.off('render', sample); reject(new Error(`${name} motion timed out.`)); - }, durationMs + 30_000); + }, durationMs + 35_000); const finish = () => { clearTimeout(timeout); map.off('render', sample); sample(performance.now()); const blankSamples = samples.filter((entry) => entry.vectorFeatureCount === 0); + const missingFoundationSamples = samples.filter((entry) => + requiredLayers.some((layer) => (entry.requiredLayerCounts[layer] || 0) <= 0) + ); let longestBlankRun = 0; let currentBlankRun = 0; for (const entry of samples) { @@ -140,12 +187,19 @@ try { } resolve({ name, + requiredLayers, sampleCount: samples.length, sourceChanged, blankSampleCount: blankSamples.length, + missingFoundationSampleCount: missingFoundationSamples.length, + firstMissingFoundationSamples: missingFoundationSamples.slice(0, 20), longestBlankRun, - minimumFeatureCount: Math.min(...samples.map((entry) => entry.vectorFeatureCount)), - maximumFeatureCount: Math.max(...samples.map((entry) => entry.vectorFeatureCount)), + minimumFeatureCount: samples.length + ? Math.min(...samples.map((entry) => entry.vectorFeatureCount)) + : 0, + maximumFeatureCount: samples.length + ? Math.max(...samples.map((entry) => entry.vectorFeatureCount)) + : 0, samples }); }; @@ -162,42 +216,80 @@ try { essential: true }); }); - }, { name, end, durationMs, expectedTemplate }); + }, { name, end, durationMs, expectedTemplate, requiredLayers }); await page.screenshot({ path: path.join(outputDir, `${name}-final.png`), fullPage: false }); + report.motions.push(result); + await persistReport(); return result; } - const motions = [ - await runMotion( - 'world-to-fresno', - { center: [-98.5, 25], zoom: 2.43 }, - { center: [-119.7871, 36.7378], zoom: 14 }, - 12_000 - ), - await runMotion( - 'fresno-to-world', - { center: [-119.7871, 36.7378], zoom: 14 }, - { center: [-98.5, 25], zoom: 2.43 }, - 12_000 - ), - await runMotion( - 'cross-border-pan', - { center: [-112.5, 31.8], zoom: 7 }, - { center: [-101.5, 31.8], zoom: 7 }, - 8_000 - ), - await runMotion( - 'europe-shard-pan', - { center: [-4, 50], zoom: 6.5 }, - { center: [24, 50], zoom: 6.5 }, - 9_000 - ) + const definitions = [ + { + name: 'world-to-fresno', + start: { center: [-98.5, 25], zoom: 2.43 }, + end: { center: [-119.7871, 36.7378], zoom: 16 }, + durationMs: 14_000, + requiredLayers: ['land', 'landcover'] + }, + { + name: 'fresno-to-world', + start: { center: [-119.7871, 36.7378], zoom: 16 }, + end: { center: [-98.5, 25], zoom: 1.65 }, + durationMs: 14_000, + requiredLayers: ['land', 'landcover'] + }, + { + name: 'cross-border-pan', + start: { center: [-112.5, 31.8], zoom: 7 }, + end: { center: [-101.5, 31.8], zoom: 7 }, + durationMs: 9_000, + requiredLayers: ['land', 'landcover'] + }, + { + name: 'europe-shard-pan', + start: { center: [-4, 50], zoom: 6.5 }, + end: { center: [24, 50], zoom: 6.5 }, + durationMs: 10_000, + requiredLayers: ['land', 'landcover'] + }, + { + name: 'antimeridian-pan', + start: { center: [168, 0], zoom: 6.5 }, + end: { center: [-168, 0], zoom: 6.5 }, + durationMs: 10_000, + requiredLayers: ['depth'] + }, + { + name: 'amazon-routing-threshold-in', + start: { center: [-60, -8], zoom: 5.7 }, + end: { center: [-60, -8], zoom: 6.3 }, + durationMs: 8_000, + requiredLayers: ['land', 'landcover'] + }, + { + name: 'amazon-routing-threshold-out', + start: { center: [-60, -8], zoom: 6.3 }, + end: { center: [-60, -8], zoom: 5.7 }, + durationMs: 8_000, + requiredLayers: ['land', 'landcover'] + }, + { + name: 'pacific-routing-threshold-in', + start: { center: [-140, 0], zoom: 5.7 }, + end: { center: [-140, 0], zoom: 6.3 }, + durationMs: 8_000, + requiredLayers: ['depth'] + } ]; + for (const definition of definitions) { + await runMotion(definition); + } + const sortedDurations = tileDurations .map((entry) => entry.durationMs) .sort((left, right) => left - right); @@ -205,58 +297,58 @@ try { ? sortedDurations[Math.min(sortedDurations.length - 1, Math.floor(sortedDurations.length * fraction))] : null; - const report = { - generatedAt: new Date().toISOString(), - origin, - expectedTemplate, - motions, - tileRequests: { - count: tileDurations.length, - p50Ms: percentile(0.5), - p95Ms: percentile(0.95), - maximumMs: sortedDurations.at(-1) || null, - slowest: [...tileDurations] - .sort((left, right) => right.durationMs - left.durationMs) - .slice(0, 20) - }, - pageErrors, - networkFailures, - externalVectorRequests: [...new Set(externalVectorRequests)] + report.tileRequests = { + count: tileDurations.length, + p50Ms: percentile(0.5), + p95Ms: percentile(0.95), + p99Ms: percentile(0.99), + maximumMs: sortedDurations.at(-1) || null, + slowest: [...tileDurations] + .sort((left, right) => right.durationMs - left.durationMs) + .slice(0, 30) }; - await fs.writeFile( - path.join(outputDir, 'continuous-motion-report.json'), - `${JSON.stringify(report, null, 2)}\n` + const failedMotions = report.motions.filter( + (motion) => + motion.sourceChanged || + motion.blankSampleCount > 0 || + motion.missingFoundationSampleCount > 0 || + motion.sampleCount < 20 ); + report.externalVectorRequests = [...new Set(report.externalVectorRequests)]; + report.passed = + failedMotions.length === 0 && + report.pageErrors.length === 0 && + report.networkFailures.length === 0 && + report.externalVectorRequests.length === 0; + await persistReport(); - const failedMotions = motions.filter( - (motion) => motion.sourceChanged || motion.blankSampleCount > 0 || motion.sampleCount < 20 - ); - if ( - failedMotions.length || - pageErrors.length || - networkFailures.length || - externalVectorRequests.length - ) { + if (!report.passed) { throw new Error(`Continuous motion validation failed: ${JSON.stringify({ failedMotions: failedMotions.map((motion) => ({ name: motion.name, sampleCount: motion.sampleCount, sourceChanged: motion.sourceChanged, blankSampleCount: motion.blankSampleCount, + missingFoundationSampleCount: motion.missingFoundationSampleCount, longestBlankRun: motion.longestBlankRun, - minimumFeatureCount: motion.minimumFeatureCount + minimumFeatureCount: motion.minimumFeatureCount, + firstMissingFoundationSamples: motion.firstMissingFoundationSamples })), - pageErrors, - networkFailures, - externalVectorRequests + pageErrors: report.pageErrors, + networkFailures: report.networkFailures, + externalVectorRequests: report.externalVectorRequests })}`); } console.log( - `Validated ${motions.length} continuous motions with zero blank vector frames; ` + - `tile p95 ${Math.round(report.tileRequests.p95Ms || 0)}ms.` + `Validated ${definitions.length} continuous motions with no blank frames or missing physical foundation; tile p95 ${Math.round(report.tileRequests.p95Ms || 0)}ms.` ); +} catch (error) { + report.fatalError = serializeError(error); + report.passed = false; + await persistReport().catch(() => {}); + throw error; } finally { - await browser.close(); + await browser?.close(); } From 3864ed9dfa306f887959aafb00b90765797f5f87 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 21:58:29 -0700 Subject: [PATCH 37/85] Run every runtime hardening gate and preserve all diagnostics --- .../workflows/validate-continuous-zoom.yml | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/.github/workflows/validate-continuous-zoom.yml b/.github/workflows/validate-continuous-zoom.yml index e4aef20b..ad221cf4 100644 --- a/.github/workflows/validate-continuous-zoom.yml +++ b/.github/workflows/validate-continuous-zoom.yml @@ -86,10 +86,10 @@ jobs: - name: Install Chromium run: npx playwright install --with-deps chromium - - name: Run hardened HTTP, continuous zoom, and pan gates + - name: Run every hardened HTTP and browser gate run: | set -euo pipefail - mkdir -p continuous-motion + mkdir -p continuous-motion/results continuous-motion/all-zoom-levels OCCUMED_WORLD_OVERVIEW_URL="http://127.0.0.1:4173/virtual-assets/${OVERVIEW_ASSET}" \ OCCUMED_WORLD_SURFACE_URL="http://127.0.0.1:4173/virtual-assets/${SURFACE_ASSET}" \ OCCUMED_ENABLE_DIAGNOSTICS="true" \ @@ -98,14 +98,15 @@ jobs: cleanup() { status="$?" if [ "$status" -ne 0 ]; then - tail -400 continuous-motion/server.log || true + tail -500 continuous-motion/server.log || true fi kill "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true exit "$status" } trap cleanup EXIT - for attempt in {1..90}; do + for attempt in {1..120}; do if curl --fail --silent http://127.0.0.1:4173/healthz > /dev/null; then break fi @@ -113,16 +114,42 @@ jobs: done curl --fail http://127.0.0.1:4173/healthz - node scripts/check-world-http-hardening.mjs + overall_status=0 + run_gate() { + gate_name="$1" + shift + set +e + "$@" 2>&1 | tee "continuous-motion/${gate_name}.log" + gate_status="${PIPESTATUS[0]}" + set -e + printf '%s=%s\n' "$gate_name" "$gate_status" >> continuous-motion/gate-status.txt + if [ "$gate_status" -ne 0 ]; then + overall_status=1 + fi + } - OCCUMED_PREVIEW_OUTPUT=continuous-motion/results \ + run_gate http-hardening \ + node scripts/check-world-http-hardening.mjs + + run_gate continuous-motion \ + env OCCUMED_PREVIEW_OUTPUT=continuous-motion/results \ node scripts/validate-continuous-zoom.mjs - OCCUMED_PREVIEW_OUTPUT=continuous-motion/all-zoom-levels \ + run_gate all-zoom-levels \ + env OCCUMED_PREVIEW_OUTPUT=continuous-motion/all-zoom-levels \ node scripts/validate-all-zoom-levels.mjs + set +e curl --fail --silent http://127.0.0.1:4173/internal/tile-health \ > continuous-motion/tile-health.json + health_status="$?" + set -e + printf 'tile-health=%s\n' "$health_status" >> continuous-motion/gate-status.txt + if [ "$health_status" -ne 0 ]; then + overall_status=1 + fi + + exit "$overall_status" - uses: actions/upload-artifact@v4 if: always() @@ -131,7 +158,8 @@ jobs: path: | continuous-motion/results continuous-motion/all-zoom-levels - continuous-motion/server.log + continuous-motion/*.log + continuous-motion/gate-status.txt continuous-motion/tile-health.json dist/virtual-assets/occumed-world-overview.pmtiles if-no-files-found: error From 7bb901ebc602693a3c52cc69eed71af4446b2d5a Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 22:02:44 -0700 Subject: [PATCH 38/85] Add sustained worldwide gateway soak validation --- scripts/check-world-soak.mjs | 217 +++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 scripts/check-world-soak.mjs diff --git a/scripts/check-world-soak.mjs b/scripts/check-world-soak.mjs new file mode 100644 index 00000000..8f81e75c --- /dev/null +++ b/scripts/check-world-soak.mjs @@ -0,0 +1,217 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import http from 'node:http'; +import { gunzipSync } from 'node:zlib'; +import { validateVectorTilePayload } from '../src/server/tile-safety.js'; + +const origin = new URL(process.env.OCCUMED_PREVIEW_ORIGIN || 'http://127.0.0.1:4173'); +const outputPath = path.resolve( + process.env.OCCUMED_SOAK_OUTPUT || 'continuous-motion/world-soak-report.json' +); +const concurrency = 24; +const waves = 3; + +function request(pathname, { + method = 'GET', + headers = {}, + timeoutMs = 60_000 +} = {}) { + return new Promise((resolve, reject) => { + const startedAt = performance.now(); + const req = http.request({ + protocol: origin.protocol, + hostname: origin.hostname, + port: origin.port, + path: pathname, + method, + timeout: timeoutMs, + headers: { + Host: origin.host, + 'X-Request-Id': `soak-${Math.random().toString(16).slice(2)}`, + ...headers + } + }, (response) => { + const chunks = []; + response.on('data', (chunk) => chunks.push(Buffer.from(chunk))); + response.on('end', () => resolve({ + pathname, + status: response.statusCode, + headers: response.headers, + body: Buffer.concat(chunks), + durationMs: performance.now() - startedAt + })); + }); + req.on('timeout', () => req.destroy(new Error(`${method} ${pathname} timed out.`))); + req.on('error', reject); + req.end(); + }); +} + +async function json(pathname) { + const response = await request(pathname, { headers: { Accept: 'application/json' } }); + assert.equal(response.status, 200, `${pathname} returned ${response.status}.`); + return JSON.parse(response.body.toString('utf8')); +} + +function lonLatToTile(longitude, latitude, zoom) { + const count = 2 ** zoom; + const x = Math.min(count - 1, Math.max(0, Math.floor(((longitude + 180) / 360) * count))); + const clampedLatitude = Math.min(85.05112878, Math.max(-85.05112878, latitude)); + const radians = clampedLatitude * Math.PI / 180; + const y = Math.min( + count - 1, + Math.max(0, Math.floor((1 - Math.asinh(Math.tan(radians)) / Math.PI) / 2 * count)) + ); + return `/tiles/${zoom}/${x}/${y}.pbf`; +} + +async function mapLimit(items, limit, task) { + const results = new Array(items.length); + let cursor = 0; + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + while (true) { + const index = cursor; + cursor += 1; + if (index >= items.length) return; + results[index] = await task(items[index], index); + } + }); + await Promise.all(workers); + return results; +} + +await fs.mkdir(path.dirname(outputPath), { recursive: true }); +const before = await json('/internal/tile-health'); +const readinessBefore = await json('/readyz'); +assert.equal(readinessBefore.ready, true); + +const centers = [ + { name: 'fresno', coordinates: [-119.7871, 36.7378] }, + { name: 'amazon', coordinates: [-60, -8] }, + { name: 'europe', coordinates: [12, 50] }, + { name: 'tokyo', coordinates: [139.6917, 35.6895] }, + { name: 'sydney', coordinates: [151.2093, -33.8688] }, + { name: 'cairo', coordinates: [31.2357, 30.0444] }, + { name: 'central-pacific', coordinates: [-140, 0] }, + { name: 'south-atlantic', coordinates: [-30, -30] }, + { name: 'antimeridian', coordinates: [179, 0] } +]; +const zooms = [0, 2, 4, 5, 6, 7, 8, 10, 12, 14, 16]; +const uniquePaths = [...new Set( + centers.flatMap((center) => + zooms.map((zoom) => lonLatToTile(center.coordinates[0], center.coordinates[1], zoom)) + ) +)]; + +const requests = []; +for (let wave = 0; wave < waves; wave += 1) { + for (let index = 0; index < uniquePaths.length; index += 1) { + requests.push({ + wave, + pathname: uniquePaths[index], + encoding: (wave + index) % 2 === 0 ? 'gzip' : 'identity' + }); + } +} + +const responses = await mapLimit(requests, concurrency, async (entry) => { + const response = await request(entry.pathname, { + headers: { 'Accept-Encoding': entry.encoding } + }); + assert.equal(response.status, 200, `${entry.pathname} returned ${response.status}.`); + assert.equal(response.headers['content-type'], 'application/x-protobuf'); + assert(response.headers.etag, `${entry.pathname} did not return an ETag.`); + const tile = response.headers['content-encoding'] === 'gzip' + ? gunzipSync(response.body) + : response.body; + validateVectorTilePayload(tile, { label: `soak ${entry.pathname}` }); + return { + wave: entry.wave, + pathname: entry.pathname, + encoding: entry.encoding, + durationMs: response.durationMs, + encodedBytes: response.body.byteLength, + decodedBytes: tile.byteLength, + etag: response.headers.etag + }; +}); + +const etagsByPath = new Map(); +for (const response of responses) { + const previous = etagsByPath.get(response.pathname); + if (previous) { + assert.equal(response.etag, previous, `${response.pathname} changed ETag between soak waves.`); + } else { + etagsByPath.set(response.pathname, response.etag); + } +} + +const hotPath = lonLatToTile(-60, -8, 8); +const coalesced = await Promise.all( + Array.from({ length: 64 }, () => request(hotPath, { + headers: { 'Accept-Encoding': 'identity' } + })) +); +assert.equal(new Set(coalesced.map((response) => response.headers.etag)).size, 1); +for (const response of coalesced) { + assert.equal(response.status, 200); + validateVectorTilePayload(response.body, { label: '64-request coalesced hot tile' }); +} + +await new Promise((resolve) => setTimeout(resolve, 250)); +const after = await json('/internal/tile-health'); +const readinessAfter = await json('/readyz'); +const healthAfter = await request('/healthz'); +assert.equal(healthAfter.status, 200); +assert.equal(readinessAfter.ready, true); +assert.equal(after.inflightTiles, 0, 'Gateway retained in-flight tile promises after the soak.'); +assert.equal(after.archiveReads.active, 0, 'Archive reads remained active after the soak.'); +assert.equal(after.archiveReads.queued, 0, 'Archive reads remained queued after the soak.'); +assert(after.cache.bytes <= after.cache.maxBytes, 'Tile cache exceeded its configured byte budget.'); +assert(after.cache.entries <= 8_192, 'Tile cache exceeded its entry budget.'); +assert.equal( + after.metrics.overloads, + before.metrics.overloads, + 'The normal soak triggered gateway overload protection.' +); +assert.equal( + after.metrics.failed, + before.metrics.failed, + 'Valid worldwide soak requests produced gateway failures.' +); +for (const source of after.sources) { + assert.equal(source.circuitOpen, false, `${source.asset} left its upstream circuit open.`); +} + +const durations = responses + .map((response) => response.durationMs) + .sort((left, right) => left - right); +const percentile = (fraction) => durations[ + Math.min(durations.length - 1, Math.floor(durations.length * fraction)) +]; +const report = { + generatedAt: new Date().toISOString(), + origin: origin.href, + waves, + concurrency, + distinctTileCount: uniquePaths.length, + totalTileRequests: responses.length + coalesced.length, + p50Ms: percentile(0.5), + p95Ms: percentile(0.95), + p99Ms: percentile(0.99), + maximumMs: durations.at(-1), + slowest: [...responses] + .sort((left, right) => right.durationMs - left.durationMs) + .slice(0, 30), + before, + after, + passed: true +}; +await fs.writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`); + +assert(report.p95Ms < 45_000, `Worldwide soak p95 was ${Math.round(report.p95Ms)}ms.`); +assert(report.maximumMs < 60_000, `Worldwide soak maximum was ${Math.round(report.maximumMs)}ms.`); +console.log( + `Worldwide soak passed: ${report.totalTileRequests} requests, ${report.distinctTileCount} distinct tiles, zooms 0–16, p95 ${Math.round(report.p95Ms)}ms, zero failures, zero overloads, and all queues drained.` +); From fe673c751acad21f058f23832940397c37c95e6d Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 22:03:24 -0700 Subject: [PATCH 39/85] Require sustained worldwide gateway soak before browser validation --- .github/workflows/validate-continuous-zoom.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-continuous-zoom.yml b/.github/workflows/validate-continuous-zoom.yml index ad221cf4..c884826e 100644 --- a/.github/workflows/validate-continuous-zoom.yml +++ b/.github/workflows/validate-continuous-zoom.yml @@ -131,6 +131,10 @@ jobs: run_gate http-hardening \ node scripts/check-world-http-hardening.mjs + run_gate world-soak \ + env OCCUMED_SOAK_OUTPUT=continuous-motion/world-soak-report.json \ + node scripts/check-world-soak.mjs + run_gate continuous-motion \ env OCCUMED_PREVIEW_OUTPUT=continuous-motion/results \ node scripts/validate-continuous-zoom.mjs @@ -159,8 +163,8 @@ jobs: continuous-motion/results continuous-motion/all-zoom-levels continuous-motion/*.log + continuous-motion/*.json continuous-motion/gate-status.txt - continuous-motion/tile-health.json dist/virtual-assets/occumed-world-overview.pmtiles if-no-files-found: error retention-days: 14 From 8966e9fbe9058e00acb7fbe6cd97338b3aab6938 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 22:04:09 -0700 Subject: [PATCH 40/85] Lock continuous surface bloom and soak protections --- scripts/check-continuous-foundation-lock.mjs | 106 +++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 scripts/check-continuous-foundation-lock.mjs diff --git a/scripts/check-continuous-foundation-lock.mjs b/scripts/check-continuous-foundation-lock.mjs new file mode 100644 index 00000000..1f06bcd3 --- /dev/null +++ b/scripts/check-continuous-foundation-lock.mjs @@ -0,0 +1,106 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const [ + landcoverBuilder, + surfaceBuilder, + mapHelper, + appCss, + polygonGate, + motionGate, + allZoomGate, + soakGate, + workflow +] = await Promise.all([ + fs.readFile(path.join(root, 'scripts/prepare-world-landcover.mjs'), 'utf8'), + fs.readFile(path.join(root, 'scripts/build-world-surface.sh'), 'utf8'), + fs.readFile(path.join(root, 'src/occumed-map.js'), 'utf8'), + fs.readFile(path.join(root, 'src/styles.css'), 'utf8'), + fs.readFile(path.join(root, 'scripts/capture-polygon-regression.mjs'), 'utf8'), + fs.readFile(path.join(root, 'scripts/validate-continuous-zoom.mjs'), 'utf8'), + fs.readFile(path.join(root, 'scripts/validate-all-zoom-levels.mjs'), 'utf8'), + fs.readFile(path.join(root, 'scripts/check-world-soak.mjs'), 'utf8'), + fs.readFile(path.join(root, '.github/workflows/validate-continuous-zoom.yml'), 'utf8') +]); + +assert( + landcoverBuilder.includes('const SURFACE_MAX_ZOOM = 10;'), + 'Worldwide landcover is not locked to the physical surface native maximum zoom.' +); +assert( + !/maxzoom\s*:\s*5\b/.test(landcoverBuilder), + 'The old zoom-5 landcover cutoff was reintroduced.' +); +assert( + landcoverBuilder.includes('maxzoom: SURFACE_MAX_ZOOM'), + 'Worldwide landcover no longer carries an explicit continuous maximum zoom.' +); +assert( + surfaceBuilder.includes('--maximum-zoom=10') && + surfaceBuilder.includes('-L "landcover:'), + 'The physical surface build no longer publishes landcover through zoom 10.' +); + +for (const marker of [ + 'installOccumedAtmosphereBloom(map)', + 'resolveGlobeRadius', + 'BLOOM_FADE_START_ZOOM', + 'BLOOM_FADE_END_ZOOM' +]) { + assert(mapHelper.includes(marker), `Atmosphere tracking lost ${marker}.`); +} +for (const marker of [ + 'scale(1.006)', + '0 0 84px 24px', + 'drop-shadow(0 0 30px', + 'mix-blend-mode: screen', + 'border: 1px solid' +]) { + assert(appCss.includes(marker), `The exterior atmosphere bloom lost ${marker}.`); +} + +for (const marker of ['5.9', '6, 6.1', '15, 16', "requiredSourceLayers: ['land', 'landcover']", "requiredSourceLayers: ['depth']"]) { + assert(polygonGate.includes(marker), `The exhaustive polygon/foundation gate lost ${marker}.`); +} +for (const marker of [ + 'amazon-routing-threshold-in', + 'amazon-routing-threshold-out', + 'pacific-routing-threshold-in', + 'antimeridian-pan', + 'missingFoundationSampleCount' +]) { + assert(motionGate.includes(marker), `The continuous motion gate lost ${marker}.`); +} +for (const marker of [ + 'amazon-all-zooms-in', + 'amazon-all-zooms-out', + 'pacific-all-zooms-in', + 'pacific-all-zooms-out', + 'antimeridian-all-zooms-out', + 'startZoom: 0, endZoom: 16', + 'startZoom: 16, endZoom: 0' +]) { + assert(allZoomGate.includes(marker), `The complete zoom 0–16 gate lost ${marker}.`); +} + +for (const marker of [ + 'const waves = 3', + 'const concurrency = 24', + 'zooms = [0, 2, 4, 5, 6, 7, 8, 10, 12, 14, 16]', + 'after.inflightTiles, 0', + 'after.archiveReads.active, 0', + 'after.archiveReads.queued, 0', + 'source.circuitOpen, false' +]) { + assert(soakGate.includes(marker), `The worldwide soak gate lost ${marker}.`); +} +assert(workflow.includes('run_gate world-soak'), 'The sustained worldwide soak is no longer mandatory in CI.'); +assert(workflow.includes('continuous-motion/*.json'), 'Runtime JSON diagnostics are no longer preserved.'); +assert(workflow.includes('gate-status.txt'), 'Aggregate runtime gate status is no longer preserved.'); + +console.log( + 'Continuous-foundation lock passed: landcover through zoom 10, overscaling through zoom 16, strengthened tracked atmosphere, exhaustive boundary checks, and sustained worldwide soak are mandatory.' +); From fc359a8f5bac963821ba8abc1f0240323f7845e2 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 22:04:32 -0700 Subject: [PATCH 41/85] Run continuous foundation lock in every hardened build --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 42ba2552..4ef781a4 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "tiles:build": "bash planetiler/build-region.sh", "tiles:plan-world": "node scripts/plan-world-shards.mjs --scope all", "check:export": "node scripts/check-export.mjs", - "check:hardening": "node scripts/check-world-hardening.mjs", + "check:hardening": "node scripts/check-world-hardening.mjs && node scripts/check-continuous-foundation-lock.mjs", "check:runtime": "node scripts/check-runtime.mjs && node scripts/validate-maplibre-style.mjs && node scripts/check-globe-parity.mjs && node scripts/check-cartography-parity.mjs && node scripts/check-viewer-quality.mjs && node scripts/check-photo-reference.mjs && node scripts/check-exact-exported-swatches.mjs && node scripts/check-render-clarity.mjs && node scripts/check-pmtiles-integration.mjs && node scripts/check-world-tile-gateway.mjs && npm run check:hardening", "check:server": "node scripts/check-server-health.mjs", "check": "npm run check:export && npm run check:runtime", From 06e77637292a87284bb9dd0f9c5ec82622141ecd Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 22:06:16 -0700 Subject: [PATCH 42/85] Keep chaos and foundation locks as separate mandatory build gates --- package.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 4ef781a4..a2029676 100644 --- a/package.json +++ b/package.json @@ -10,8 +10,9 @@ "tiles:build": "bash planetiler/build-region.sh", "tiles:plan-world": "node scripts/plan-world-shards.mjs --scope all", "check:export": "node scripts/check-export.mjs", - "check:hardening": "node scripts/check-world-hardening.mjs && node scripts/check-continuous-foundation-lock.mjs", - "check:runtime": "node scripts/check-runtime.mjs && node scripts/validate-maplibre-style.mjs && node scripts/check-globe-parity.mjs && node scripts/check-cartography-parity.mjs && node scripts/check-viewer-quality.mjs && node scripts/check-photo-reference.mjs && node scripts/check-exact-exported-swatches.mjs && node scripts/check-render-clarity.mjs && node scripts/check-pmtiles-integration.mjs && node scripts/check-world-tile-gateway.mjs && npm run check:hardening", + "check:hardening": "node scripts/check-world-hardening.mjs", + "check:foundation": "node scripts/check-continuous-foundation-lock.mjs", + "check:runtime": "node scripts/check-runtime.mjs && node scripts/validate-maplibre-style.mjs && node scripts/check-globe-parity.mjs && node scripts/check-cartography-parity.mjs && node scripts/check-viewer-quality.mjs && node scripts/check-photo-reference.mjs && node scripts/check-exact-exported-swatches.mjs && node scripts/check-render-clarity.mjs && node scripts/check-pmtiles-integration.mjs && node scripts/check-world-tile-gateway.mjs && npm run check:hardening && npm run check:foundation", "check:server": "node scripts/check-server-health.mjs", "check": "npm run check:export && npm run check:runtime", "dev": "npm run prepare:assets && vite", @@ -28,7 +29,7 @@ }, "devDependencies": { "@chrispahm/spritezero": "8.1.0", - "@maplibre/maplibre-gl-style-spec": "24.8.1", + "@mapbox/mapbox-gl-style-spec": "24.8.1", "playwright": "1.55.0", "vite": "8.1.5" } From 7086755432a2d7eb666a5ec2bb523c1cac183cbe Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 22:07:35 -0700 Subject: [PATCH 43/85] Restore the pinned MapLibre style-spec dependency --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a2029676..ccbbbc54 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ }, "devDependencies": { "@chrispahm/spritezero": "8.1.0", - "@mapbox/mapbox-gl-style-spec": "24.8.1", + "@maplibre/maplibre-gl-style-spec": "24.8.1", "playwright": "1.55.0", "vite": "8.1.5" } From d0f957b3b91d6d260296d3fcbce2c7c7080ee5a7 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 22:24:03 -0700 Subject: [PATCH 44/85] Keep the atmosphere bloom synchronized with every globe movement --- src/occumed-map.js | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/occumed-map.js b/src/occumed-map.js index 249c1b2c..f5c8746f 100644 --- a/src/occumed-map.js +++ b/src/occumed-map.js @@ -52,10 +52,10 @@ export function installOccumedAtmosphereBloom(map) { bloom.setAttribute('aria-hidden', 'true'); canvasContainer.append(bloom); - let animationFrame = null; + let removed = false; const update = () => { - animationFrame = null; + if (removed) return; const zoom = map.getZoom(); const center = map.project(map.getCenter()); const radius = resolveGlobeRadius(zoom); @@ -68,20 +68,15 @@ export function installOccumedAtmosphereBloom(map) { bloom.hidden = opacity <= 0.001; }; - const scheduleUpdate = () => { - if (animationFrame !== null) return; - animationFrame = requestAnimationFrame(update); - }; + const trackedEvents = ['render', 'move', 'zoom', 'resize', 'moveend', 'zoomend']; + for (const eventName of trackedEvents) map.on(eventName, update); const remove = () => { - if (animationFrame !== null) cancelAnimationFrame(animationFrame); - map.off('render', scheduleUpdate); - map.off('resize', scheduleUpdate); + removed = true; + for (const eventName of trackedEvents) map.off(eventName, update); bloom.remove(); }; - map.on('render', scheduleUpdate); - map.on('resize', scheduleUpdate); map.once('remove', remove); update(); return bloom; From 1558726140f6dc48eb8f05e8002de3a0a0d270e0 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 22:25:38 -0700 Subject: [PATCH 45/85] Make continuous motion validation cycle-safe and exhaustive --- scripts/validate-continuous-zoom.mjs | 110 +++++++++++++++++++-------- 1 file changed, 78 insertions(+), 32 deletions(-) diff --git a/scripts/validate-continuous-zoom.mjs b/scripts/validate-continuous-zoom.mjs index c300f772..3b9dce5e 100644 --- a/scripts/validate-continuous-zoom.mjs +++ b/scripts/validate-continuous-zoom.mjs @@ -31,9 +31,45 @@ function serializeError(error) { }; } +function safeStringify(value) { + const seen = new WeakSet(); + return JSON.stringify(value, (_key, entry) => { + if (!entry || typeof entry !== 'object') return entry; + if (seen.has(entry)) return `[Circular:${entry.constructor?.name || 'Object'}]`; + seen.add(entry); + if ( + !Array.isArray(entry) && + Object.getPrototypeOf(entry) !== Object.prototype && + Object.getPrototypeOf(entry) !== null + ) { + return `[NonPlain:${entry.constructor?.name || 'Object'}]`; + } + return entry; + }, 2); +} + async function persistReport() { report.externalVectorRequests = [...new Set(report.externalVectorRequests)]; - await fs.writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`); + await fs.writeFile(reportPath, `${safeStringify(report)}\n`); +} + +function normalizeMotion(result, definition) { + return { + name: String(result?.name || definition.name), + requiredLayers: [...definition.requiredLayers], + sampleCount: Number(result?.sampleCount || 0), + sourceChanged: Boolean(result?.sourceChanged), + blankSampleCount: Number(result?.blankSampleCount || 0), + missingFoundationSampleCount: Number(result?.missingFoundationSampleCount || 0), + firstMissingFoundationSamples: Array.isArray(result?.firstMissingFoundationSamples) + ? result.firstMissingFoundationSamples + : [], + longestBlankRun: Number(result?.longestBlankRun || 0), + minimumFeatureCount: Number(result?.minimumFeatureCount || 0), + maximumFeatureCount: Number(result?.maximumFeatureCount || 0), + samples: Array.isArray(result?.samples) ? result.samples : [], + executionError: null + }; } let browser; @@ -113,10 +149,11 @@ try { ); } - async function runMotion({ name, start, end, durationMs, requiredLayers }) { + async function runMotion(definition) { + const { name, start, end, durationMs, requiredLayers } = definition; await waitForStableView(start.center, start.zoom, requiredLayers); - const result = await page.evaluate(async ({ + const raw = await page.evaluate(async ({ name, end, durationMs, @@ -127,17 +164,13 @@ try { const samples = []; let lastSampleAt = -Infinity; let sourceChanged = false; - - const sourceSignature = () => { - const source = map.getStyle().sources?.['occumed-open'] || null; - return JSON.stringify({ url: source?.url || null, tiles: source?.tiles || [] }); - }; const expectedSignature = JSON.stringify({ url: null, tiles: [expectedTemplate] }); const sample = (timestamp) => { if (timestamp - lastSampleAt < 70) return; lastSampleAt = timestamp; - const signature = sourceSignature(); + const source = map.getStyle().sources?.['occumed-open'] || null; + const signature = JSON.stringify({ url: source?.url || null, tiles: source?.tiles || [] }); sourceChanged ||= signature !== expectedSignature; const rendered = map .queryRenderedFeatures() @@ -148,15 +181,15 @@ try { sourceLayers[layer] = (sourceLayers[layer] || 0) + 1; } samples.push({ - timestamp, - zoom: map.getZoom(), - center: map.getCenter().toArray(), + timestamp: Number(timestamp), + zoom: Number(map.getZoom()), + center: map.getCenter().toArray().map(Number), vectorFeatureCount: rendered.length, requiredLayerCounts: Object.fromEntries( requiredLayers.map((layer) => [layer, sourceLayers[layer] || 0]) ), sourceLayers, - tilesLoaded: map.areTilesLoaded(), + tilesLoaded: Boolean(map.areTilesLoaded()), sourceSignature: signature }); }; @@ -187,7 +220,6 @@ try { } resolve({ name, - requiredLayers, sampleCount: samples.length, sourceChanged, blankSampleCount: blankSamples.length, @@ -218,6 +250,7 @@ try { }); }, { name, end, durationMs, expectedTemplate, requiredLayers }); + const result = normalizeMotion(raw, definition); await page.screenshot({ path: path.join(outputDir, `${name}-final.png`), fullPage: false @@ -287,7 +320,29 @@ try { ]; for (const definition of definitions) { - await runMotion(definition); + try { + await runMotion(definition); + } catch (error) { + report.motions.push({ + name: definition.name, + requiredLayers: [...definition.requiredLayers], + sampleCount: 0, + sourceChanged: false, + blankSampleCount: 0, + missingFoundationSampleCount: 0, + firstMissingFoundationSamples: [], + longestBlankRun: 0, + minimumFeatureCount: 0, + maximumFeatureCount: 0, + samples: [], + executionError: serializeError(error) + }); + await page.screenshot({ + path: path.join(outputDir, `${definition.name}-error.png`), + fullPage: false + }).catch(() => {}); + await persistReport(); + } } const sortedDurations = tileDurations @@ -308,12 +363,12 @@ try { .slice(0, 30) }; - const failedMotions = report.motions.filter( - (motion) => - motion.sourceChanged || - motion.blankSampleCount > 0 || - motion.missingFoundationSampleCount > 0 || - motion.sampleCount < 20 + const failedMotions = report.motions.filter((motion) => + motion.executionError || + motion.sourceChanged || + motion.blankSampleCount > 0 || + motion.missingFoundationSampleCount > 0 || + motion.sampleCount < 20 ); report.externalVectorRequests = [...new Set(report.externalVectorRequests)]; report.passed = @@ -324,17 +379,8 @@ try { await persistReport(); if (!report.passed) { - throw new Error(`Continuous motion validation failed: ${JSON.stringify({ - failedMotions: failedMotions.map((motion) => ({ - name: motion.name, - sampleCount: motion.sampleCount, - sourceChanged: motion.sourceChanged, - blankSampleCount: motion.blankSampleCount, - missingFoundationSampleCount: motion.missingFoundationSampleCount, - longestBlankRun: motion.longestBlankRun, - minimumFeatureCount: motion.minimumFeatureCount, - firstMissingFoundationSamples: motion.firstMissingFoundationSamples - })), + throw new Error(`Continuous motion validation failed: ${safeStringify({ + failedMotions, pageErrors: report.pageErrors, networkFailures: report.networkFailures, externalVectorRequests: report.externalVectorRequests From 6b156d585fcafde806d8d25b0c55fec18e81e083 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 22:26:57 -0700 Subject: [PATCH 46/85] Make full-range zoom validation cycle-safe and non-short-circuiting --- scripts/validate-all-zoom-levels.mjs | 106 +++++++++++++++++++++------ 1 file changed, 82 insertions(+), 24 deletions(-) diff --git a/scripts/validate-all-zoom-levels.mjs b/scripts/validate-all-zoom-levels.mjs index b7f1cea3..94a20502 100644 --- a/scripts/validate-all-zoom-levels.mjs +++ b/scripts/validate-all-zoom-levels.mjs @@ -30,9 +30,53 @@ function serializeError(error) { }; } +function safeStringify(value) { + const seen = new WeakSet(); + return JSON.stringify(value, (_key, entry) => { + if (!entry || typeof entry !== 'object') return entry; + if (seen.has(entry)) return `[Circular:${entry.constructor?.name || 'Object'}]`; + seen.add(entry); + if ( + !Array.isArray(entry) && + Object.getPrototypeOf(entry) !== Object.prototype && + Object.getPrototypeOf(entry) !== null + ) { + return `[NonPlain:${entry.constructor?.name || 'Object'}]`; + } + return entry; + }, 2); +} + async function persistReport() { report.externalVectorRequests = [...new Set(report.externalVectorRequests)]; - await fs.writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`); + await fs.writeFile(reportPath, `${safeStringify(report)}\n`); +} + +function normalizeSweep(result, definition) { + return { + name: String(result?.name || definition.name), + center: [...definition.center], + startZoom: definition.startZoom, + endZoom: definition.endZoom, + requiredLayers: [...definition.requiredLayers], + sampleCount: Number(result?.sampleCount || 0), + sourceChanged: Boolean(result?.sourceChanged), + blankSampleCount: Number(result?.blankSampleCount || 0), + missingFoundationSampleCount: Number(result?.missingFoundationSampleCount || 0), + firstMissingFoundationSamples: Array.isArray(result?.firstMissingFoundationSamples) + ? result.firstMissingFoundationSamples + : [], + minimumZoom: result?.minimumZoom === null || result?.minimumZoom === undefined + ? null + : Number(result.minimumZoom), + maximumZoom: result?.maximumZoom === null || result?.maximumZoom === undefined + ? null + : Number(result.maximumZoom), + maximumZoomGap: Number(result?.maximumZoomGap || 0), + minimumFeatureCount: Number(result?.minimumFeatureCount || 0), + samples: Array.isArray(result?.samples) ? result.samples : [], + executionError: null + }; } let browser; @@ -72,7 +116,8 @@ try { { timeout: 90_000 } ); - async function runSweep({ name, center, startZoom, endZoom, requiredLayers }) { + async function runSweep(definition) { + const { name, center, startZoom, endZoom, requiredLayers } = definition; await page.evaluate(({ center, startZoom }) => { const map = globalThis.__OCCUMED_MAP__; map.jumpTo({ center, zoom: startZoom, pitch: 0, bearing: 0 }); @@ -95,7 +140,7 @@ try { { timeout: 90_000 } ); - const result = await page.evaluate(async ({ + const raw = await page.evaluate(async ({ name, center, startZoom, @@ -124,14 +169,14 @@ try { sourceLayers[layer] = (sourceLayers[layer] || 0) + 1; } samples.push({ - timestamp, - zoom: map.getZoom(), + timestamp: Number(timestamp), + zoom: Number(map.getZoom()), renderedFeatureCount: rendered.length, requiredLayerCounts: Object.fromEntries( requiredLayers.map((layer) => [layer, sourceLayers[layer] || 0]) ), sourceLayers, - tilesLoaded: map.areTilesLoaded(), + tilesLoaded: Boolean(map.areTilesLoaded()), sourceSignature: signature }); }; @@ -158,10 +203,6 @@ try { ); resolve({ name, - center, - startZoom, - endZoom, - requiredLayers, sampleCount: samples.length, sourceChanged, blankSampleCount: blankSamples.length, @@ -191,6 +232,7 @@ try { }); }, { name, center, startZoom, endZoom, expectedTemplate, requiredLayers }); + const result = normalizeSweep(raw, definition); await page.screenshot({ path: path.join(outputDir, `${name}-final.png`), fullPage: false @@ -210,10 +252,37 @@ try { ]; for (const definition of definitions) { - await runSweep(definition); + try { + await runSweep(definition); + } catch (error) { + report.sweeps.push({ + name: definition.name, + center: [...definition.center], + startZoom: definition.startZoom, + endZoom: definition.endZoom, + requiredLayers: [...definition.requiredLayers], + sampleCount: 0, + sourceChanged: false, + blankSampleCount: 0, + missingFoundationSampleCount: 0, + firstMissingFoundationSamples: [], + minimumZoom: null, + maximumZoom: null, + maximumZoomGap: 0, + minimumFeatureCount: 0, + samples: [], + executionError: serializeError(error) + }); + await page.screenshot({ + path: path.join(outputDir, `${definition.name}-error.png`), + fullPage: false + }).catch(() => {}); + await persistReport(); + } } const failedSweeps = report.sweeps.filter((sweep) => + sweep.executionError || sweep.sourceChanged || sweep.blankSampleCount > 0 || sweep.missingFoundationSampleCount > 0 || @@ -232,19 +301,8 @@ try { await persistReport(); if (!report.passed) { - throw new Error(`All-zoom validation failed: ${JSON.stringify({ - failedSweeps: failedSweeps.map((sweep) => ({ - name: sweep.name, - sampleCount: sweep.sampleCount, - sourceChanged: sweep.sourceChanged, - blankSampleCount: sweep.blankSampleCount, - missingFoundationSampleCount: sweep.missingFoundationSampleCount, - minimumZoom: sweep.minimumZoom, - maximumZoom: sweep.maximumZoom, - maximumZoomGap: sweep.maximumZoomGap, - minimumFeatureCount: sweep.minimumFeatureCount, - firstMissingFoundationSamples: sweep.firstMissingFoundationSamples - })), + throw new Error(`All-zoom validation failed: ${safeStringify({ + failedSweeps, pageErrors: report.pageErrors, networkFailures: report.networkFailures, externalVectorRequests: report.externalVectorRequests From 3ea75ffccec4526d0215feeff7b2238b89496bc7 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 22:28:15 -0700 Subject: [PATCH 47/85] Run every polygon and atmosphere regression view before failing --- scripts/capture-polygon-regression.mjs | 298 +++++++++++++++---------- 1 file changed, 174 insertions(+), 124 deletions(-) diff --git a/scripts/capture-polygon-regression.mjs b/scripts/capture-polygon-regression.mjs index a6dc45f3..7fae56ff 100644 --- a/scripts/capture-polygon-regression.mjs +++ b/scripts/capture-polygon-regression.mjs @@ -36,9 +36,26 @@ function serializeError(error) { }; } +function safeStringify(value) { + const seen = new WeakSet(); + return JSON.stringify(value, (_key, entry) => { + if (!entry || typeof entry !== 'object') return entry; + if (seen.has(entry)) return `[Circular:${entry.constructor?.name || 'Object'}]`; + seen.add(entry); + if ( + !Array.isArray(entry) && + Object.getPrototypeOf(entry) !== Object.prototype && + Object.getPrototypeOf(entry) !== null + ) { + return `[NonPlain:${entry.constructor?.name || 'Object'}]`; + } + return entry; + }, 2); +} + async function persistReport() { report.externalVectorRequests = [...new Set(report.externalVectorRequests)]; - await fs.writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`); + await fs.writeFile(reportPath, `${safeStringify(report)}\n`); } let browser; @@ -106,153 +123,186 @@ try { ]; for (const view of views) { - await page.evaluate(({ center, zoom }) => { - const map = globalThis.__OCCUMED_MAP__; - map.jumpTo({ center, zoom, pitch: 0, bearing: 0 }); - map.triggerRepaint(); - }, view); - - await page.waitForFunction( - () => { + const failures = []; + try { + await page.evaluate(({ center, zoom }) => { const map = globalThis.__OCCUMED_MAP__; - return map?.isStyleLoaded() && map.areTilesLoaded() && - map.queryRenderedFeatures().some((feature) => feature.source === 'occumed-open'); - }, - null, - { timeout: 90_000 } - ); - await page.waitForTimeout(350); + map.jumpTo({ center, zoom, pitch: 0, bearing: 0 }); + map.triggerRepaint(); + }, view); - const diagnostics = await page.evaluate((expectedTemplate) => { - const map = globalThis.__OCCUMED_MAP__; - const source = map.getStyle().sources?.['occumed-open'] || null; - const features = map - .queryRenderedFeatures() - .filter((feature) => feature.source === 'occumed-open'); - const renderedSourceLayerCounts = {}; - const styleLayerCounts = {}; - for (const feature of features) { - const sourceLayer = feature.sourceLayer || 'unknown'; - const styleLayer = feature.layer?.id || 'unknown'; - renderedSourceLayerCounts[sourceLayer] = (renderedSourceLayerCounts[sourceLayer] || 0) + 1; - styleLayerCounts[styleLayer] = (styleLayerCounts[styleLayer] || 0) + 1; - } - const sourceFeatureCounts = Object.fromEntries( - ['land', 'landcover', 'depth'].map((sourceLayer) => [ - sourceLayer, - map.querySourceFeatures('occumed-open', { sourceLayer }).length - ]) + await page.waitForFunction( + () => { + const map = globalThis.__OCCUMED_MAP__; + return map?.isStyleLoaded() && map.areTilesLoaded() && + map.queryRenderedFeatures().some((feature) => feature.source === 'occumed-open'); + }, + null, + { timeout: 90_000 } ); - const bloom = document.querySelector('.occumed-atmosphere-bloom'); - const bloomStyle = bloom ? getComputedStyle(bloom) : null; - const bloomRect = bloom?.getBoundingClientRect() || null; - const containerRect = map.getCanvasContainer().getBoundingClientRect(); - const projectedCenter = map.project(map.getCenter()); - const expectedCenterX = containerRect.left + projectedCenter.x; - const expectedCenterY = containerRect.top + projectedCenter.y; - const expectedDiameter = ( - ((512 * (2 ** map.getZoom())) / (Math.PI * 2)) * 2 * 1.006 - ); - const actualCenterX = bloomRect ? bloomRect.left + (bloomRect.width / 2) : 0; - const actualCenterY = bloomRect ? bloomRect.top + (bloomRect.height / 2) : 0; - return { - center: map.getCenter().toArray(), - zoom: map.getZoom(), - source, - sourceIsPermanent: - !source?.url && JSON.stringify(source?.tiles || []) === JSON.stringify([expectedTemplate]), - renderedFeatureCount: features.length, - renderedSourceLayerCounts, - sourceFeatureCounts, - styleLayerCounts, - atmosphereBloom: { - exists: Boolean(bloom), - hidden: Boolean(bloom?.hidden), - opacity: Number(bloomStyle?.opacity || 0), - filter: bloomStyle?.filter || 'none', - boxShadow: bloomStyle?.boxShadow || 'none', - borderColor: bloomStyle?.borderColor || 'transparent', - borderWidth: Number.parseFloat(bloomStyle?.borderWidth || '0'), - mixBlendMode: bloomStyle?.mixBlendMode || 'normal', - width: bloomRect?.width || 0, - height: bloomRect?.height || 0, - centerErrorPx: bloomRect - ? Math.hypot(actualCenterX - expectedCenterX, actualCenterY - expectedCenterY) - : null, - diameterErrorPx: bloomRect ? Math.abs(bloomRect.width - expectedDiameter) : null + await page.waitForTimeout(350); + + const diagnostics = await page.evaluate((expectedTemplate) => { + const map = globalThis.__OCCUMED_MAP__; + const source = map.getStyle().sources?.['occumed-open'] || null; + const features = map + .queryRenderedFeatures() + .filter((feature) => feature.source === 'occumed-open'); + const renderedSourceLayerCounts = {}; + const styleLayerCounts = {}; + for (const feature of features) { + const sourceLayer = feature.sourceLayer || 'unknown'; + const styleLayer = feature.layer?.id || 'unknown'; + renderedSourceLayerCounts[sourceLayer] = (renderedSourceLayerCounts[sourceLayer] || 0) + 1; + styleLayerCounts[styleLayer] = (styleLayerCounts[styleLayer] || 0) + 1; } - }; - }, expectedTemplate); + const sourceFeatureCounts = Object.fromEntries( + ['land', 'landcover', 'depth'].map((sourceLayer) => [ + sourceLayer, + map.querySourceFeatures('occumed-open', { sourceLayer }).length + ]) + ); + const bloom = document.querySelector('.occumed-atmosphere-bloom'); + const bloomStyle = bloom ? getComputedStyle(bloom) : null; + const bloomRect = bloom?.getBoundingClientRect() || null; + const containerRect = map.getCanvasContainer().getBoundingClientRect(); + const projectedCenter = map.project(map.getCenter()); + const expectedCenterX = containerRect.left + projectedCenter.x; + const expectedCenterY = containerRect.top + projectedCenter.y; + const expectedDiameter = ( + ((512 * (2 ** map.getZoom())) / (Math.PI * 2)) * 2 * 1.006 + ); + const actualCenterX = bloomRect ? bloomRect.left + (bloomRect.width / 2) : 0; + const actualCenterY = bloomRect ? bloomRect.top + (bloomRect.height / 2) : 0; + return { + center: map.getCenter().toArray(), + zoom: map.getZoom(), + source: source ? { + type: source.type, + url: source.url || null, + tiles: Array.isArray(source.tiles) ? [...source.tiles] : [], + minzoom: source.minzoom, + maxzoom: source.maxzoom, + attribution: source.attribution + } : null, + sourceIsPermanent: + !source?.url && JSON.stringify(source?.tiles || []) === JSON.stringify([expectedTemplate]), + renderedFeatureCount: features.length, + renderedSourceLayerCounts, + sourceFeatureCounts, + styleLayerCounts, + atmosphereBloom: { + exists: Boolean(bloom), + hidden: Boolean(bloom?.hidden), + opacity: Number(bloomStyle?.opacity || 0), + filter: bloomStyle?.filter || 'none', + boxShadow: bloomStyle?.boxShadow || 'none', + borderColor: bloomStyle?.borderColor || 'transparent', + borderWidth: Number.parseFloat(bloomStyle?.borderWidth || '0'), + mixBlendMode: bloomStyle?.mixBlendMode || 'normal', + width: bloomRect?.width || 0, + height: bloomRect?.height || 0, + centerErrorPx: bloomRect + ? Math.hypot(actualCenterX - expectedCenterX, actualCenterY - expectedCenterY) + : null, + diameterErrorPx: bloomRect ? Math.abs(bloomRect.width - expectedDiameter) : null + } + }; + }, expectedTemplate); - const screenshot = await page.screenshot({ - path: path.join(outputDir, `${view.name}.png`), - fullPage: false - }); + const screenshot = await page.screenshot({ + path: path.join(outputDir, `${view.name}.png`), + fullPage: false + }); - const failures = []; - if (!diagnostics.sourceIsPermanent) { - failures.push(`${view.name} changed the permanent vector source.`); - } - if (diagnostics.renderedFeatureCount <= 0) { - failures.push(`${view.name} rendered no worldwide vector features.`); - } - for (const sourceLayer of view.requiredSourceLayers || []) { - if ((diagnostics.sourceFeatureCounts[sourceLayer] || 0) <= 0) { - failures.push(`${view.name} lost the ${sourceLayer} source layer at zoom ${view.zoom}.`); + if (!diagnostics.sourceIsPermanent) { + failures.push(`${view.name} changed the permanent vector source.`); } - } - for (const sourceLayer of view.requiredRenderedLayers || []) { - if ((diagnostics.renderedSourceLayerCounts[sourceLayer] || 0) <= 0) { - failures.push(`${view.name} stopped rendering the ${sourceLayer} foundation at zoom ${view.zoom}.`); + if (diagnostics.renderedFeatureCount <= 0) { + failures.push(`${view.name} rendered no worldwide vector features.`); } - } - if (view.requiresAtmosphereBloom) { - const bloom = diagnostics.atmosphereBloom; - if (!bloom.exists || bloom.hidden || bloom.opacity < 0.95) { - failures.push(`${view.name} does not show the full-strength globe atmosphere bloom.`); + for (const sourceLayer of view.requiredSourceLayers || []) { + if ((diagnostics.sourceFeatureCounts[sourceLayer] || 0) <= 0) { + failures.push(`${view.name} lost the ${sourceLayer} source layer at zoom ${view.zoom}.`); + } } - if ( - bloom.filter === 'none' || - bloom.boxShadow === 'none' || - bloom.mixBlendMode !== 'screen' || - bloom.borderWidth < 1 - ) { - failures.push(`${view.name} has a hard rim instead of the layered luminous white-blue bloom.`); + for (const sourceLayer of view.requiredRenderedLayers || []) { + if ((diagnostics.renderedSourceLayerCounts[sourceLayer] || 0) <= 0) { + failures.push(`${view.name} stopped rendering the ${sourceLayer} foundation at zoom ${view.zoom}.`); + } } - if ( - bloom.width < 150 || - Math.abs(bloom.width - bloom.height) > 2 || - bloom.centerErrorPx === null || bloom.centerErrorPx > 3 || - bloom.diameterErrorPx === null || bloom.diameterErrorPx > 4 - ) { - failures.push(`${view.name} atmosphere bloom does not precisely track the rendered globe.`); + if (view.requiresAtmosphereBloom) { + const bloom = diagnostics.atmosphereBloom; + if (!bloom.exists || bloom.hidden || bloom.opacity < 0.95) { + failures.push(`${view.name} does not show the full-strength globe atmosphere bloom.`); + } + if ( + bloom.filter === 'none' || + bloom.boxShadow === 'none' || + bloom.mixBlendMode !== 'screen' || + bloom.borderWidth < 1 + ) { + failures.push(`${view.name} has a hard rim instead of the layered luminous white-blue bloom.`); + } + if ( + bloom.width < 150 || + Math.abs(bloom.width - bloom.height) > 2 || + bloom.centerErrorPx === null || bloom.centerErrorPx > 3 || + bloom.diameterErrorPx === null || bloom.diameterErrorPx > 4 + ) { + failures.push(`${view.name} atmosphere bloom does not precisely track the rendered globe.`); + } } - } - if (screenshot.length < 25_000) { - failures.push(`${view.name} produced an unexpectedly empty screenshot.`); + if (screenshot.length < 25_000) { + failures.push(`${view.name} produced an unexpectedly empty screenshot.`); + } + + report.results[view.name] = { + ...diagnostics, + screenshotBytes: screenshot.length, + failures, + executionError: null + }; + } catch (error) { + failures.push(`${view.name} execution failed: ${error.message}`); + await page.screenshot({ + path: path.join(outputDir, `${view.name}-error.png`), + fullPage: false + }).catch(() => {}); + report.results[view.name] = { + center: [...view.center], + zoom: view.zoom, + failures, + executionError: serializeError(error) + }; } - report.results[view.name] = { - ...diagnostics, - screenshotBytes: screenshot.length, - failures - }; await persistReport(); - - if (failures.length) { - throw new Error(failures.join(' ')); - } } + const failedViews = Object.entries(report.results) + .filter(([, result]) => result.executionError || result.failures?.length) + .map(([name, result]) => ({ + name, + zoom: result.zoom, + failures: result.failures || [], + executionError: result.executionError || null, + atmosphereBloom: result.atmosphereBloom || null, + sourceFeatureCounts: result.sourceFeatureCounts || null, + renderedSourceLayerCounts: result.renderedSourceLayerCounts || null + })); + report.externalVectorRequests = [...new Set(report.externalVectorRequests)]; report.passed = + failedViews.length === 0 && report.pageErrors.length === 0 && report.networkFailures.length === 0 && report.externalVectorRequests.length === 0; await persistReport(); if (!report.passed) { - throw new Error(`Polygon, foundation, and atmosphere validation failed: ${JSON.stringify({ + throw new Error(`Polygon, foundation, and atmosphere validation failed: ${safeStringify({ + failedViews, pageErrors: report.pageErrors, networkFailures: report.networkFailures, externalVectorRequests: report.externalVectorRequests From f6c0d958979e536d72add0c16189d2d1a81febc7 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 23:32:24 -0700 Subject: [PATCH 48/85] Apply documented continuous rendering contract --- scripts/apply-mapbox-rendering-contract.mjs | 109 ++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 scripts/apply-mapbox-rendering-contract.mjs diff --git a/scripts/apply-mapbox-rendering-contract.mjs b/scripts/apply-mapbox-rendering-contract.mjs new file mode 100644 index 00000000..7d332c95 --- /dev/null +++ b/scripts/apply-mapbox-rendering-contract.mjs @@ -0,0 +1,109 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const runtimePath = path.join(root, 'public/style/occumed-open.json'); +const runtime = JSON.parse(await fs.readFile(runtimePath, 'utf8')); + +const FOUNDATION_LAYERS = new Set(['land', 'landcover', 'depth']); +const PERMANENT_TILE_PATTERN = '/tiles/{z}/{x}/{y}.pbf'; + +function sourceLayerName(layer) { + return String( + layer?.['source-layer'] || + layer?.metadata?.['occumed:open-source-layer'] || + '' + ).toLowerCase(); +} + +function isPermanentWorldwideSource(source) { + return source?.type === 'vector' && + Array.isArray(source.tiles) && + source.tiles.some((url) => String(url).includes(PERMANENT_TILE_PATTERN)); +} + +let permanentSources = 0; +for (const source of Object.values(runtime.sources || {})) { + if (!isPermanentWorldwideSource(source)) continue; + + // Mapbox/TileJSON semantics: source maxzoom describes the highest tile zoom + // available. The renderer may overscale those tiles above that zoom. Our + // virtual endpoint resolves through z16, so expose one uninterrupted source + // pyramid across the application's complete camera range. + source.minzoom = 0; + source.maxzoom = 16; + source.scheme = 'xyz'; + permanentSources += 1; +} + +if (permanentSources !== 1) { + throw new Error(`Expected exactly one permanent worldwide vector source; found ${permanentSources}.`); +} + +// Prevent runtime paint-value changes from introducing an additional delayed +// fade. Zoom expressions still interpolate continuously; explicit style +// mutations apply immediately. +runtime.transition = { duration: 0, delay: 0 }; + +let foundationLayers = 0; +let landcoverLayers = 0; +let depthLayers = 0; +for (const layer of runtime.layers || []) { + const sourceLayer = sourceLayerName(layer); + if (!FOUNDATION_LAYERS.has(sourceLayer)) continue; + + layer.minzoom = 0; + delete layer.maxzoom; + layer.layout ||= {}; + layer.layout.visibility = 'visible'; + layer.metadata = { + ...(layer.metadata || {}), + 'occumed:continuous-foundation': true, + 'occumed:documented-overscaling-contract': true + }; + foundationLayers += 1; + + if (sourceLayer === 'landcover' && layer.type === 'fill') { + layer.paint ||= {}; + layer.paint['fill-opacity'] = [ + 'interpolate', ['linear'], ['zoom'], + 0, 0.92, + 6, 0.88, + 10, 0.82, + 16, 0.82 + ]; + landcoverLayers += 1; + } + + if (sourceLayer === 'depth' && layer.type === 'fill') { + layer.paint ||= {}; + const existingOpacity = layer.paint['fill-opacity'] ?? 1; + // Preserve each bathymetry band's authored opacity while preventing any + // exported high-zoom expression from reducing the layer to zero. + layer.paint['fill-opacity'] = ['max', 0.06, existingOpacity]; + depthLayers += 1; + } +} + +if (!foundationLayers || !landcoverLayers || !depthLayers) { + throw new Error( + `Incomplete documented rendering contract: ${foundationLayers} foundation, ` + + `${landcoverLayers} landcover, ${depthLayers} depth layers.` + ); +} + +runtime.metadata = { + ...(runtime.metadata || {}), + 'occumed:mapbox-style-contract-applied': true, + 'occumed:single-source-minzoom': 0, + 'occumed:single-source-maxzoom': 16, + 'occumed:foundation-layer-maxzoom': null, + 'occumed:style-transition-duration-ms': 0 +}; + +await fs.writeFile(runtimePath, `${JSON.stringify(runtime, null, 2)}\n`); +console.log( + `Applied documented single-source rendering contract to ${foundationLayers} foundation layers ` + + `(${landcoverLayers} landcover, ${depthLayers} depth).` +); From ffd9482a5d991a8a17938e78b8e969661a15ff18 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 23:32:53 -0700 Subject: [PATCH 49/85] Run documented rendering contract last --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ccbbbc54..d3e11a2a 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "type": "module", "scripts": { "prepare:sprites": "node scripts/build-sprites.mjs", - "prepare:style": "node scripts/build-runtime-style.mjs && node scripts/normalize-runtime-fonts.mjs && node scripts/normalize-runtime-filters.mjs && node scripts/normalize-boundary-filters.mjs && node scripts/force-runtime-filter-expressions.mjs && node scripts/apply-schema-parity.mjs && node scripts/apply-globe-parity.mjs && node scripts/apply-photo-reference.mjs && node scripts/restore-exported-cartography.mjs && node scripts/calibrate-reference-colors.mjs && node scripts/lock-exact-exported-swatches.mjs && node scripts/lock-reference-atmosphere.mjs && node scripts/normalize-runtime-fonts.mjs && node scripts/use-local-maplibre-glyphs.mjs", + "prepare:style": "node scripts/build-runtime-style.mjs && node scripts/normalize-runtime-fonts.mjs && node scripts/normalize-runtime-filters.mjs && node scripts/normalize-boundary-filters.mjs && node scripts/force-runtime-filter-expressions.mjs && node scripts/apply-schema-parity.mjs && node scripts/apply-globe-parity.mjs && node scripts/apply-photo-reference.mjs && node scripts/restore-exported-cartography.mjs && node scripts/calibrate-reference-colors.mjs && node scripts/lock-exact-exported-swatches.mjs && node scripts/lock-reference-atmosphere.mjs && node scripts/normalize-runtime-fonts.mjs && node scripts/apply-mapbox-rendering-contract.mjs && node scripts/use-local-maplibre-glyphs.mjs", "prepare:assets": "npm run prepare:sprites && npm run prepare:style", "tiles:build": "bash planetiler/build-region.sh", "tiles:plan-world": "node scripts/plan-world-shards.mjs --scope all", From 05434c44d6f5488248845722cb5cd1d25024e309 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 23:34:23 -0700 Subject: [PATCH 50/85] Lock documented source and layer zoom semantics --- scripts/check-continuous-foundation-lock.mjs | 85 ++++++++++++++++++-- 1 file changed, 80 insertions(+), 5 deletions(-) diff --git a/scripts/check-continuous-foundation-lock.mjs b/scripts/check-continuous-foundation-lock.mjs index 1f06bcd3..af1c0576 100644 --- a/scripts/check-continuous-foundation-lock.mjs +++ b/scripts/check-continuous-foundation-lock.mjs @@ -13,7 +13,10 @@ const [ motionGate, allZoomGate, soakGate, - workflow + workflow, + packageDocument, + renderingContract, + runtimeStyleDocument ] = await Promise.all([ fs.readFile(path.join(root, 'scripts/prepare-world-landcover.mjs'), 'utf8'), fs.readFile(path.join(root, 'scripts/build-world-surface.sh'), 'utf8'), @@ -23,9 +26,15 @@ const [ fs.readFile(path.join(root, 'scripts/validate-continuous-zoom.mjs'), 'utf8'), fs.readFile(path.join(root, 'scripts/validate-all-zoom-levels.mjs'), 'utf8'), fs.readFile(path.join(root, 'scripts/check-world-soak.mjs'), 'utf8'), - fs.readFile(path.join(root, '.github/workflows/validate-continuous-zoom.yml'), 'utf8') + fs.readFile(path.join(root, '.github/workflows/validate-continuous-zoom.yml'), 'utf8'), + fs.readFile(path.join(root, 'package.json'), 'utf8'), + fs.readFile(path.join(root, 'scripts/apply-mapbox-rendering-contract.mjs'), 'utf8'), + fs.readFile(path.join(root, 'public/style/occumed-open.json'), 'utf8') ]); +const packageJson = JSON.parse(packageDocument); +const runtimeStyle = JSON.parse(runtimeStyleDocument); + assert( landcoverBuilder.includes('const SURFACE_MAX_ZOOM = 10;'), 'Worldwide landcover is not locked to the physical surface native maximum zoom.' @@ -44,13 +53,79 @@ assert( 'The physical surface build no longer publishes landcover through zoom 10.' ); +const prepareStyle = packageJson.scripts?.['prepare:style'] || ''; +const contractIndex = prepareStyle.indexOf('apply-mapbox-rendering-contract.mjs'); +const restoreIndex = prepareStyle.indexOf('restore-exported-cartography.mjs'); +const atmosphereIndex = prepareStyle.indexOf('lock-reference-atmosphere.mjs'); +assert(contractIndex > restoreIndex && contractIndex > atmosphereIndex, + 'The documented rendering contract must run after all style restoration and atmosphere passes.'); + +for (const marker of [ + 'source.minzoom = 0', + 'source.maxzoom = 16', + "delete layer.maxzoom", + "runtime.transition = { duration: 0, delay: 0 }", + "layer.paint['fill-opacity'] = ['max', 0.06, existingOpacity]" +]) { + assert(renderingContract.includes(marker), `The documented rendering contract lost ${marker}.`); +} + +const permanentSources = Object.values(runtimeStyle.sources || {}).filter((source) => + source?.type === 'vector' && + Array.isArray(source.tiles) && + source.tiles.some((url) => String(url).includes('/tiles/{z}/{x}/{y}.pbf')) +); +assert.equal(permanentSources.length, 1, 'The runtime must expose exactly one permanent worldwide vector source.'); +assert.equal(permanentSources[0].minzoom, 0, 'The permanent worldwide vector source must begin at zoom 0.'); +assert.equal(permanentSources[0].maxzoom, 16, 'The permanent worldwide vector source must cover through zoom 16.'); +assert.deepEqual(runtimeStyle.transition, { duration: 0, delay: 0 }, + 'The runtime style must not delay foundation paint updates.'); + +const foundationLayers = (runtimeStyle.layers || []).filter((layer) => + ['land', 'landcover', 'depth'].includes(String(layer['source-layer'] || layer.metadata?.['occumed:open-source-layer'] || '').toLowerCase()) +); +assert(foundationLayers.length > 0, 'The runtime style has no physical foundation layers.'); +for (const layer of foundationLayers) { + assert.equal(layer.minzoom, 0, `${layer.id} must begin at zoom 0.`); + assert(!Object.hasOwn(layer, 'maxzoom'), `${layer.id} must not have a style-layer maxzoom cutoff.`); + assert.equal(layer.layout?.visibility, 'visible', `${layer.id} must remain visible.`); +} + +const landcoverLayers = foundationLayers.filter((layer) => + String(layer['source-layer'] || layer.metadata?.['occumed:open-source-layer'] || '').toLowerCase() === 'landcover' && + layer.type === 'fill' +); +assert(landcoverLayers.length > 0, 'The runtime has no rendered landcover foundation.'); +for (const layer of landcoverLayers) { + assert.deepEqual(layer.paint?.['fill-opacity'], [ + 'interpolate', ['linear'], ['zoom'], + 0, 0.92, + 6, 0.88, + 10, 0.82, + 16, 0.82 + ], `${layer.id} landcover opacity must remain nonzero through zoom 16.`); +} + +const depthLayers = foundationLayers.filter((layer) => + String(layer['source-layer'] || layer.metadata?.['occumed:open-source-layer'] || '').toLowerCase() === 'depth' && + layer.type === 'fill' +); +assert(depthLayers.length > 0, 'The runtime has no rendered depth foundation.'); +for (const layer of depthLayers) { + const opacity = layer.paint?.['fill-opacity']; + assert(Array.isArray(opacity) && opacity[0] === 'max' && opacity[1] >= 0.06, + `${layer.id} bathymetry opacity can still collapse to zero.`); +} + for (const marker of [ 'installOccumedAtmosphereBloom(map)', 'resolveGlobeRadius', 'BLOOM_FADE_START_ZOOM', - 'BLOOM_FADE_END_ZOOM' + 'BLOOM_FADE_END_ZOOM', + 'cancelPendingTileRequestsWhileZooming: false', + 'maxTileCacheZoomLevels: 8' ]) { - assert(mapHelper.includes(marker), `Atmosphere tracking lost ${marker}.`); + assert(mapHelper.includes(marker), `MapLibre continuity behavior lost ${marker}.`); } for (const marker of [ 'scale(1.006)', @@ -102,5 +177,5 @@ assert(workflow.includes('continuous-motion/*.json'), 'Runtime JSON diagnostics assert(workflow.includes('gate-status.txt'), 'Aggregate runtime gate status is no longer preserved.'); console.log( - 'Continuous-foundation lock passed: landcover through zoom 10, overscaling through zoom 16, strengthened tracked atmosphere, exhaustive boundary checks, and sustained worldwide soak are mandatory.' + 'Continuous-foundation lock passed: documented source/layer zoom semantics, nonzero landcover and depth through zoom 16, parent tile retention, strengthened atmosphere, exhaustive boundary checks, and sustained worldwide soak are mandatory.' ); From f2adbaa7921f5a958c603f025b4801f5c1130154 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Mon, 27 Jul 2026 23:37:32 -0700 Subject: [PATCH 51/85] Validate continuous bathymetry instead of cutoff --- scripts/check-globe-parity.mjs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/check-globe-parity.mjs b/scripts/check-globe-parity.mjs index 1beabf6c..43b8435a 100644 --- a/scripts/check-globe-parity.mjs +++ b/scripts/check-globe-parity.mjs @@ -81,6 +81,9 @@ if ((landcover?.minzoom ?? 99) !== 0) fail('Landcover is unavailable at globe zo if (!JSON.stringify(landcover?.paint?.['fill-opacity'] || []).includes('1')) { fail('Exported landcover swatches are being weakened at globe and regional zooms.'); } +if (Object.hasOwn(landcover || {}, 'maxzoom')) { + fail('Landcover has a style-layer maxzoom cutoff instead of remaining continuous.'); +} if (layer('continent-label')?.layout?.visibility !== 'none') { fail('Multilingual continent aliases are cluttering the globe limb.'); } @@ -91,7 +94,13 @@ if (water?.paint?.['fill-opacity'] !== 1) fail('Water is blending into the land const waterDepth = layer('water-depth'); if (waterDepth?.['source-layer'] !== 'depth') fail('The continuous vector bathymetry layer is missing.'); -if (waterDepth?.maxzoom !== 8) fail('Bathymetry does not fade before detailed navigation zooms.'); +if (Object.hasOwn(waterDepth || {}, 'maxzoom')) { + fail('Bathymetry has a style-layer maxzoom cutoff instead of remaining subtly visible.'); +} +const depthOpacity = waterDepth?.paint?.['fill-opacity']; +if (!Array.isArray(depthOpacity) || depthOpacity[0] !== 'max' || Number(depthOpacity[1]) < 0.06) { + fail('Bathymetry can still collapse to zero at detailed navigation zooms.'); +} const waterDepthColors = JSON.stringify(waterDepth?.paint?.['fill-color'] || []); for (const value of ['#79BCEC59', '#5AACE759', '#3B9DE359']) { if (!waterDepthColors.includes(value)) fail(`The exported bathymetry swatch ${value} is missing.`); @@ -118,6 +127,7 @@ if (runtime.metadata?.['occumed:raster-relief-disabled'] !== true) fail('Raster if (runtime.metadata?.['occumed:reference-atmosphere'] !== true) fail('The reference atmosphere pass did not run.'); if (runtime.metadata?.['occumed:atmosphere-surface-wash-disabled'] !== true) fail('Atmosphere surface-wash protection is missing.'); if (runtime.metadata?.['occumed:atmosphere-edge-only'] !== true) fail('The edge-only atmosphere protection is missing.'); +if (runtime.metadata?.['occumed:mapbox-style-contract-applied'] !== true) fail('The documented source/layer rendering contract did not run last.'); if (failures.length) { console.error('Occu-Med globe parity validation failed:'); @@ -125,4 +135,4 @@ if (failures.length) { process.exit(1); } -console.log(`Globe parity validated: dark space, narrow white-blue edge bloom, neutral surface lighting, layered land, clear blue water, and ${allColors.size} structure-specific colors.`); +console.log(`Globe parity validated: dark space, narrow white-blue edge bloom, neutral surface lighting, continuous landcover and bathymetry, clear blue water, and ${allColors.size} structure-specific colors.`); From a197be16364b12d93240a02aea9a649eeb3bb117 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 00:23:19 -0700 Subject: [PATCH 52/85] Retain the complete zoom pyramid for reverse navigation --- src/occumed-map.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/occumed-map.js b/src/occumed-map.js index f5c8746f..0e81fe23 100644 --- a/src/occumed-map.js +++ b/src/occumed-map.js @@ -9,6 +9,9 @@ const GLOBE_TILE_SIZE = 512; const GLOBE_CIRCUMFERENCE = Math.PI * 2; const BLOOM_FADE_START_ZOOM = 2.85; const BLOOM_FADE_END_ZOOM = 4.25; +const WORLD_MIN_ZOOM = 0; +const WORLD_MAX_ZOOM = 16; +const WORLD_ZOOM_PYRAMID_LEVELS = WORLD_MAX_ZOOM - WORLD_MIN_ZOOM + 1; export function resolveOccumedPixelRatio() { const deviceRatio = Number(globalThis.devicePixelRatio); @@ -138,8 +141,8 @@ export async function createOccumedMap({ styleUrl = DEFAULT_STYLE_URL, center = [-98.5, 25], zoom = 2.43, - minZoom = 0, - maxZoom = 16, + minZoom = WORLD_MIN_ZOOM, + maxZoom = WORLD_MAX_ZOOM, controls = true, scaleControl = false, mapOptions = {} @@ -161,11 +164,11 @@ export async function createOccumedMap({ hash: false, pixelRatio: resolveOccumedPixelRatio(), antialias: true, - // The virtual gateway can take longer than a CDN-hosted static tile source. - // Keep already-requested parent tiles alive while child tiles arrive so zoom - // gestures never expose blank map frames. + // MapLibre only retains pending smaller-zoom requests during zoom-in when + // cancellation is disabled. Reverse zooms also require the already-loaded + // parent pyramid to remain in cache, so retain every zoom level from 0–16. cancelPendingTileRequestsWhileZooming: false, - maxTileCacheZoomLevels: 8, + maxTileCacheZoomLevels: WORLD_ZOOM_PYRAMID_LEVELS, refreshExpiredTiles: false, fadeDuration: 300, renderWorldCopies: false, From f3b05c8cb36b8c37d74e65d516365952ac8d8b69 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 00:24:36 -0700 Subject: [PATCH 53/85] Lock full zoom-pyramid retention into continuity checks --- scripts/check-continuous-foundation-lock.mjs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/check-continuous-foundation-lock.mjs b/scripts/check-continuous-foundation-lock.mjs index af1c0576..633f0c7f 100644 --- a/scripts/check-continuous-foundation-lock.mjs +++ b/scripts/check-continuous-foundation-lock.mjs @@ -64,7 +64,7 @@ for (const marker of [ 'source.minzoom = 0', 'source.maxzoom = 16', "delete layer.maxzoom", - "runtime.transition = { duration: 0, delay: 0 }", + 'runtime.transition = { duration: 0, delay: 0 }', "layer.paint['fill-opacity'] = ['max', 0.06, existingOpacity]" ]) { assert(renderingContract.includes(marker), `The documented rendering contract lost ${marker}.`); @@ -123,7 +123,8 @@ for (const marker of [ 'BLOOM_FADE_START_ZOOM', 'BLOOM_FADE_END_ZOOM', 'cancelPendingTileRequestsWhileZooming: false', - 'maxTileCacheZoomLevels: 8' + 'const WORLD_ZOOM_PYRAMID_LEVELS = WORLD_MAX_ZOOM - WORLD_MIN_ZOOM + 1', + 'maxTileCacheZoomLevels: WORLD_ZOOM_PYRAMID_LEVELS' ]) { assert(mapHelper.includes(marker), `MapLibre continuity behavior lost ${marker}.`); } @@ -177,5 +178,5 @@ assert(workflow.includes('continuous-motion/*.json'), 'Runtime JSON diagnostics assert(workflow.includes('gate-status.txt'), 'Aggregate runtime gate status is no longer preserved.'); console.log( - 'Continuous-foundation lock passed: documented source/layer zoom semantics, nonzero landcover and depth through zoom 16, parent tile retention, strengthened atmosphere, exhaustive boundary checks, and sustained worldwide soak are mandatory.' + 'Continuous-foundation lock passed: documented source/layer zoom semantics, nonzero landcover and depth through zoom 16, full 0–16 parent-tile retention, strengthened atmosphere, exhaustive boundary checks, and sustained worldwide soak are mandatory.' ); From 18739fc3c4ded0492ffe147209e1f2758af8175e Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 00:26:42 -0700 Subject: [PATCH 54/85] Use source-idle and timer sampling for motion validation --- scripts/validate-continuous-zoom.mjs | 102 ++++++++++++++++++--------- 1 file changed, 70 insertions(+), 32 deletions(-) diff --git a/scripts/validate-continuous-zoom.mjs b/scripts/validate-continuous-zoom.mjs index 3b9dce5e..96109433 100644 --- a/scripts/validate-continuous-zoom.mjs +++ b/scripts/validate-continuous-zoom.mjs @@ -18,6 +18,7 @@ const report = { tileRequests: null, pageErrors: [], networkFailures: [], + abortedTileRequests: [], externalVectorRequests: [], fatalError: null, passed: false @@ -93,11 +94,14 @@ try { } }); page.on('requestfailed', (request) => { - report.networkFailures.push({ - type: 'requestfailed', - url: request.url(), - error: request.failure()?.errorText || 'unknown request failure' - }); + const url = request.url(); + const error = request.failure()?.errorText || 'unknown request failure'; + tileStartedAt.delete(request); + if (url.startsWith(`${origin}/tiles/`) && error === 'net::ERR_ABORTED') { + report.abortedTileRequests.push(url); + return; + } + report.networkFailures.push({ type: 'requestfailed', url, error }); }); page.on('response', (response) => { if (response.status() >= 400) { @@ -126,27 +130,59 @@ try { ); async function waitForStableView(center, zoom, requiredLayers) { - await page.evaluate(({ center, zoom }) => { + await page.evaluate(async ({ center, zoom, requiredLayers }) => { const map = globalThis.__OCCUMED_MAP__; map.jumpTo({ center, zoom, pitch: 0, bearing: 0 }); map.triggerRepaint(); - }, { center, zoom }); - await page.waitForFunction( - ({ requiredLayers }) => { - const map = globalThis.__OCCUMED_MAP__; - if (!map?.isStyleLoaded() || !map.areTilesLoaded()) return false; - const rendered = map - .queryRenderedFeatures() - .filter((feature) => feature.source === 'occumed-open'); - if (!rendered.length) return false; - return requiredLayers.every((required) => - rendered.some((feature) => feature.sourceLayer === required) - ); - }, - { requiredLayers }, - { timeout: 90_000 } - ); + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + reject(new Error(`The source did not become idle with ${requiredLayers.join(', ')} rendered.`)); + }, 90_000); + const interval = setInterval(check, 100); + + function renderedLayersPresent() { + const rendered = map + .queryRenderedFeatures() + .filter((feature) => feature.source === 'occumed-open'); + return rendered.length > 0 && requiredLayers.every((required) => + rendered.some((feature) => feature.sourceLayer === required) + ); + } + + function check() { + if ( + map.isStyleLoaded() && + map.areTilesLoaded() && + renderedLayersPresent() + ) { + cleanup(); + resolve(); + } + } + + function onSourceData(event) { + if ( + event.sourceId === 'occumed-open' && + event.sourceDataType === 'idle' + ) { + check(); + } + } + + function cleanup() { + clearTimeout(timeout); + clearInterval(interval); + map.off('sourcedata', onSourceData); + map.off('idle', check); + } + + map.on('sourcedata', onSourceData); + map.on('idle', check); + check(); + }); + }, { center, zoom, requiredLayers }); } async function runMotion(definition) { @@ -162,13 +198,11 @@ try { }) => { const map = globalThis.__OCCUMED_MAP__; const samples = []; - let lastSampleAt = -Infinity; let sourceChanged = false; const expectedSignature = JSON.stringify({ url: null, tiles: [expectedTemplate] }); - const sample = (timestamp) => { - if (timestamp - lastSampleAt < 70) return; - lastSampleAt = timestamp; + const sample = () => { + const timestamp = performance.now(); const source = map.getStyle().sources?.['occumed-open'] || null; const signature = JSON.stringify({ url: source?.url || null, tiles: source?.tiles || [] }); sourceChanged ||= signature !== expectedSignature; @@ -181,7 +215,7 @@ try { sourceLayers[layer] = (sourceLayers[layer] || 0) + 1; } samples.push({ - timestamp: Number(timestamp), + timestamp, zoom: Number(map.getZoom()), center: map.getCenter().toArray().map(Number), vectorFeatureCount: rendered.length, @@ -195,15 +229,17 @@ try { }; return await new Promise((resolve, reject) => { + const sampleTimer = setInterval(sample, 50); const timeout = setTimeout(() => { - map.off('render', sample); + clearInterval(sampleTimer); + map.off('moveend', finish); reject(new Error(`${name} motion timed out.`)); }, durationMs + 35_000); const finish = () => { clearTimeout(timeout); - map.off('render', sample); - sample(performance.now()); + clearInterval(sampleTimer); + sample(); const blankSamples = samples.filter((entry) => entry.vectorFeatureCount === 0); const missingFoundationSamples = samples.filter((entry) => requiredLayers.some((layer) => (entry.requiredLayerCounts[layer] || 0) <= 0) @@ -236,7 +272,7 @@ try { }); }; - map.on('render', sample); + sample(); map.once('moveend', finish); map.easeTo({ center: end.center, @@ -358,6 +394,7 @@ try { p95Ms: percentile(0.95), p99Ms: percentile(0.99), maximumMs: sortedDurations.at(-1) || null, + abortedCount: report.abortedTileRequests.length, slowest: [...tileDurations] .sort((left, right) => right.durationMs - left.durationMs) .slice(0, 30) @@ -383,12 +420,13 @@ try { failedMotions, pageErrors: report.pageErrors, networkFailures: report.networkFailures, + abortedTileRequestCount: report.abortedTileRequests.length, externalVectorRequests: report.externalVectorRequests })}`); } console.log( - `Validated ${definitions.length} continuous motions with no blank frames or missing physical foundation; tile p95 ${Math.round(report.tileRequests.p95Ms || 0)}ms.` + `Validated ${definitions.length} continuous motions with 50ms sampling, no blank frames, no missing physical foundation, and ${report.abortedTileRequests.length} expected canceled tile requests; tile p95 ${Math.round(report.tileRequests.p95Ms || 0)}ms.` ); } catch (error) { report.fatalError = serializeError(error); From 41d8dabc2ffd79c99852b0c97d6f68244a3f1fa4 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 00:28:20 -0700 Subject: [PATCH 55/85] Validate actual globe minimum with continuous timer sampling --- scripts/validate-all-zoom-levels.mjs | 143 +++++++++++++++++---------- 1 file changed, 91 insertions(+), 52 deletions(-) diff --git a/scripts/validate-all-zoom-levels.mjs b/scripts/validate-all-zoom-levels.mjs index 94a20502..04d3ac76 100644 --- a/scripts/validate-all-zoom-levels.mjs +++ b/scripts/validate-all-zoom-levels.mjs @@ -17,6 +17,7 @@ const report = { sweeps: [], pageErrors: [], networkFailures: [], + abortedTileRequests: [], externalVectorRequests: [], fatalError: null, passed: false @@ -56,8 +57,10 @@ function normalizeSweep(result, definition) { return { name: String(result?.name || definition.name), center: [...definition.center], - startZoom: definition.startZoom, - endZoom: definition.endZoom, + requestedStartZoom: definition.startZoom, + requestedEndZoom: definition.endZoom, + actualStartZoom: Number(result?.actualStartZoom), + actualEndZoom: Number(result?.actualEndZoom), requiredLayers: [...definition.requiredLayers], sampleCount: Number(result?.sampleCount || 0), sourceChanged: Boolean(result?.sourceChanged), @@ -97,11 +100,13 @@ try { } }); page.on('requestfailed', (request) => { - report.networkFailures.push({ - type: 'requestfailed', - url: request.url(), - error: request.failure()?.errorText || 'unknown request failure' - }); + const url = request.url(); + const error = request.failure()?.errorText || 'unknown request failure'; + if (url.startsWith(`${origin}/tiles/`) && error === 'net::ERR_ABORTED') { + report.abortedTileRequests.push(url); + return; + } + report.networkFailures.push({ type: 'requestfailed', url, error }); }); page.on('response', (response) => { if (response.status() >= 400) { @@ -116,47 +121,74 @@ try { { timeout: 90_000 } ); - async function runSweep(definition) { - const { name, center, startZoom, endZoom, requiredLayers } = definition; - await page.evaluate(({ center, startZoom }) => { + async function positionAndWait(center, zoom, requiredLayers) { + return await page.evaluate(async ({ center, zoom, requiredLayers }) => { const map = globalThis.__OCCUMED_MAP__; - map.jumpTo({ center, zoom: startZoom, pitch: 0, bearing: 0 }); + map.jumpTo({ center, zoom, pitch: 0, bearing: 0 }); map.triggerRepaint(); - }, { center, startZoom }); - await page.waitForFunction( - ({ requiredLayers }) => { - const map = globalThis.__OCCUMED_MAP__; - if (!map?.isStyleLoaded() || !map.areTilesLoaded()) return false; - const rendered = map - .queryRenderedFeatures() - .filter((feature) => feature.source === 'occumed-open'); - if (!rendered.length) return false; - return requiredLayers.every((required) => - rendered.some((feature) => feature.sourceLayer === required) - ); - }, - { requiredLayers }, - { timeout: 90_000 } - ); + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + reject(new Error(`The source did not become idle with ${requiredLayers.join(', ')} rendered.`)); + }, 90_000); + const interval = setInterval(check, 100); + + function renderedLayersPresent() { + const rendered = map + .queryRenderedFeatures() + .filter((feature) => feature.source === 'occumed-open'); + return rendered.length > 0 && requiredLayers.every((required) => + rendered.some((feature) => feature.sourceLayer === required) + ); + } + + function check() { + if (map.isStyleLoaded() && map.areTilesLoaded() && renderedLayersPresent()) { + cleanup(); + resolve(); + } + } + + function onSourceData(event) { + if (event.sourceId === 'occumed-open' && event.sourceDataType === 'idle') check(); + } + + function cleanup() { + clearTimeout(timeout); + clearInterval(interval); + map.off('sourcedata', onSourceData); + map.off('idle', check); + } + + map.on('sourcedata', onSourceData); + map.on('idle', check); + check(); + }); + + return Number(map.getZoom()); + }, { center, zoom, requiredLayers }); + } + + async function runSweep(definition) { + const { name, center, startZoom, endZoom, requiredLayers } = definition; + const actualStartZoom = await positionAndWait(center, startZoom, requiredLayers); const raw = await page.evaluate(async ({ name, center, - startZoom, endZoom, expectedTemplate, - requiredLayers + requiredLayers, + actualStartZoom }) => { const map = globalThis.__OCCUMED_MAP__; const samples = []; - let lastSampleAt = -Infinity; let sourceChanged = false; const expectedSignature = JSON.stringify({ url: null, tiles: [expectedTemplate] }); - const sample = (timestamp) => { - if (timestamp - lastSampleAt < 55) return; - lastSampleAt = timestamp; + const sample = () => { + const timestamp = performance.now(); const source = map.getStyle().sources?.['occumed-open'] || null; const signature = JSON.stringify({ url: source?.url || null, tiles: source?.tiles || [] }); sourceChanged ||= signature !== expectedSignature; @@ -169,7 +201,7 @@ try { sourceLayers[layer] = (sourceLayers[layer] || 0) + 1; } samples.push({ - timestamp: Number(timestamp), + timestamp, zoom: Number(map.getZoom()), renderedFeatureCount: rendered.length, requiredLayerCounts: Object.fromEntries( @@ -183,15 +215,18 @@ try { return await new Promise((resolve, reject) => { const durationMs = 20_000; + const sampleTimer = setInterval(sample, 50); const timeout = setTimeout(() => { - map.off('render', sample); + clearInterval(sampleTimer); + map.off('moveend', finish); reject(new Error(`${name} full-range zoom sweep timed out.`)); }, durationMs + 35_000); const finish = () => { clearTimeout(timeout); - map.off('render', sample); - sample(performance.now()); + clearInterval(sampleTimer); + sample(); + const actualEndZoom = Number(map.getZoom()); const zooms = samples.map((entry) => entry.zoom).sort((a, b) => a - b); let maximumZoomGap = 0; for (let index = 1; index < zooms.length; index += 1) { @@ -203,6 +238,8 @@ try { ); resolve({ name, + actualStartZoom, + actualEndZoom, sampleCount: samples.length, sourceChanged, blankSampleCount: blankSamples.length, @@ -218,7 +255,7 @@ try { }); }; - map.on('render', sample); + sample(); map.once('moveend', finish); map.easeTo({ center, @@ -230,7 +267,7 @@ try { essential: true }); }); - }, { name, center, startZoom, endZoom, expectedTemplate, requiredLayers }); + }, { name, center, endZoom, expectedTemplate, requiredLayers, actualStartZoom }); const result = normalizeSweep(raw, definition); await page.screenshot({ @@ -258,8 +295,10 @@ try { report.sweeps.push({ name: definition.name, center: [...definition.center], - startZoom: definition.startZoom, - endZoom: definition.endZoom, + requestedStartZoom: definition.startZoom, + requestedEndZoom: definition.endZoom, + actualStartZoom: null, + actualEndZoom: null, requiredLayers: [...definition.requiredLayers], sampleCount: 0, sourceChanged: false, @@ -281,16 +320,15 @@ try { } } - const failedSweeps = report.sweeps.filter((sweep) => - sweep.executionError || - sweep.sourceChanged || - sweep.blankSampleCount > 0 || - sweep.missingFoundationSampleCount > 0 || - sweep.sampleCount < 180 || - sweep.minimumZoom === null || sweep.minimumZoom > 0.1 || - sweep.maximumZoom === null || sweep.maximumZoom < 15.9 || - sweep.maximumZoomGap > 0.3 - ); + const failedSweeps = report.sweeps.filter((sweep) => { + if (sweep.executionError || sweep.sourceChanged) return true; + if (sweep.blankSampleCount > 0 || sweep.missingFoundationSampleCount > 0) return true; + if (sweep.sampleCount < 180 || sweep.maximumZoomGap > 0.3) return true; + if (sweep.minimumZoom === null || sweep.maximumZoom === null) return true; + const expectedMinimum = Math.min(sweep.actualStartZoom, sweep.actualEndZoom); + const expectedMaximum = Math.max(sweep.actualStartZoom, sweep.actualEndZoom); + return sweep.minimumZoom > expectedMinimum + 0.1 || sweep.maximumZoom < expectedMaximum - 0.1; + }); report.externalVectorRequests = [...new Set(report.externalVectorRequests)]; report.passed = @@ -305,12 +343,13 @@ try { failedSweeps, pageErrors: report.pageErrors, networkFailures: report.networkFailures, + abortedTileRequestCount: report.abortedTileRequests.length, externalVectorRequests: report.externalVectorRequests })}`); } console.log( - `Validated ${definitions.length} complete zoom 0–16 sweeps with ${report.sweeps.reduce((sum, sweep) => sum + sweep.sampleCount, 0)} sampled frames and no missing physical foundation layers.` + `Validated ${definitions.length} complete effective-minimum-to-zoom-16 sweeps with 50ms sampling, ${report.sweeps.reduce((sum, sweep) => sum + sweep.sampleCount, 0)} sampled frames, and no missing physical foundation layers.` ); } catch (error) { report.fatalError = serializeError(error); From 7ca56862fe08cbb87df2971b8ea71c91196fcc4e Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 00:30:53 -0700 Subject: [PATCH 56/85] Force bathymetry into every native surface zoom --- scripts/prepare-world-bathymetry.mjs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/prepare-world-bathymetry.mjs b/scripts/prepare-world-bathymetry.mjs index d57fa633..27ab03dc 100644 --- a/scripts/prepare-world-bathymetry.mjs +++ b/scripts/prepare-world-bathymetry.mjs @@ -3,6 +3,8 @@ import fs from 'node:fs/promises'; import path from 'node:path'; +const SURFACE_MAX_ZOOM = 10; + function parseArgs(argv) { const options = {}; for (let index = 0; index < argv.length; index += 1) { @@ -44,15 +46,26 @@ for (const entry of entries) { features.push({ type: 'Feature', properties: { min_depth: entry.depth }, - geometry: feature.geometry + geometry: feature.geometry, + tippecanoe: { + minzoom: 0, + maxzoom: SURFACE_MAX_ZOOM + } }); } } +if (!features.length) throw new Error('Worldwide bathymetry preparation produced no features.'); +for (const feature of features) { + if (feature.tippecanoe?.minzoom !== 0 || feature.tippecanoe?.maxzoom !== SURFACE_MAX_ZOOM) { + throw new Error('Worldwide bathymetry contains a zoom cutoff that can empty the ocean foundation.'); + } +} + await fs.writeFile( options.output, `${JSON.stringify({ type: 'FeatureCollection', features })}\n` ); console.log( - `Prepared ${features.length} nested bathymetry polygons from ${entries.length} depth bands.` + `Prepared ${features.length} nested bathymetry polygons from ${entries.length} depth bands through zoom ${SURFACE_MAX_ZOOM}.` ); From ab3b4b72145132b1f6ab2c068e94877f5901bd10 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 00:32:24 -0700 Subject: [PATCH 57/85] Lock low-zoom depth and documented sampling behavior --- scripts/check-continuous-foundation-lock.mjs | 31 +++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/scripts/check-continuous-foundation-lock.mjs b/scripts/check-continuous-foundation-lock.mjs index 633f0c7f..8e5e4d4b 100644 --- a/scripts/check-continuous-foundation-lock.mjs +++ b/scripts/check-continuous-foundation-lock.mjs @@ -6,6 +6,7 @@ import { fileURLToPath } from 'node:url'; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const [ landcoverBuilder, + bathymetryBuilder, surfaceBuilder, mapHelper, appCss, @@ -19,6 +20,7 @@ const [ runtimeStyleDocument ] = await Promise.all([ fs.readFile(path.join(root, 'scripts/prepare-world-landcover.mjs'), 'utf8'), + fs.readFile(path.join(root, 'scripts/prepare-world-bathymetry.mjs'), 'utf8'), fs.readFile(path.join(root, 'scripts/build-world-surface.sh'), 'utf8'), fs.readFile(path.join(root, 'src/occumed-map.js'), 'utf8'), fs.readFile(path.join(root, 'src/styles.css'), 'utf8'), @@ -48,9 +50,17 @@ assert( 'Worldwide landcover no longer carries an explicit continuous maximum zoom.' ); assert( - surfaceBuilder.includes('--maximum-zoom=10') && - surfaceBuilder.includes('-L "landcover:'), - 'The physical surface build no longer publishes landcover through zoom 10.' + bathymetryBuilder.includes('const SURFACE_MAX_ZOOM = 10;') && + bathymetryBuilder.includes('minzoom: 0') && + bathymetryBuilder.includes('maxzoom: SURFACE_MAX_ZOOM'), + 'Worldwide bathymetry is not explicitly guaranteed from zoom 0 through the surface maximum.' +); +assert( + surfaceBuilder.includes('--minimum-zoom=0') && + surfaceBuilder.includes('--maximum-zoom=10') && + surfaceBuilder.includes('-L "landcover:') && + surfaceBuilder.includes('-L "depth:'), + 'The physical surface build no longer publishes landcover and depth from zoom 0 through zoom 10.' ); const prepareStyle = packageJson.scripts?.['prepare:style'] || ''; @@ -146,7 +156,10 @@ for (const marker of [ 'amazon-routing-threshold-out', 'pacific-routing-threshold-in', 'antimeridian-pan', - 'missingFoundationSampleCount' + 'missingFoundationSampleCount', + 'setInterval(sample, 50)', + "event.sourceDataType === 'idle'", + "error === 'net::ERR_ABORTED'" ]) { assert(motionGate.includes(marker), `The continuous motion gate lost ${marker}.`); } @@ -157,9 +170,13 @@ for (const marker of [ 'pacific-all-zooms-out', 'antimeridian-all-zooms-out', 'startZoom: 0, endZoom: 16', - 'startZoom: 16, endZoom: 0' + 'startZoom: 16, endZoom: 0', + 'setInterval(sample, 50)', + "event.sourceDataType === 'idle'", + 'actualStartZoom', + 'actualEndZoom' ]) { - assert(allZoomGate.includes(marker), `The complete zoom 0–16 gate lost ${marker}.`); + assert(allZoomGate.includes(marker), `The complete zoom-range gate lost ${marker}.`); } for (const marker of [ @@ -178,5 +195,5 @@ assert(workflow.includes('continuous-motion/*.json'), 'Runtime JSON diagnostics assert(workflow.includes('gate-status.txt'), 'Aggregate runtime gate status is no longer preserved.'); console.log( - 'Continuous-foundation lock passed: documented source/layer zoom semantics, nonzero landcover and depth through zoom 16, full 0–16 parent-tile retention, strengthened atmosphere, exhaustive boundary checks, and sustained worldwide soak are mandatory.' + 'Continuous-foundation lock passed: documented source/layer zoom semantics, explicit zoom-0 bathymetry, nonzero landcover and depth through zoom 16, full 0–16 parent-tile retention, source-idle stabilization, 50ms motion sampling, strengthened atmosphere, exhaustive boundary checks, and sustained worldwide soak are mandatory.' ); From 7f85130de4ad1320f96ddbdbc4fb3f520be1f267 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 00:35:41 -0700 Subject: [PATCH 58/85] Accept full zoom-pyramid browser retention --- scripts/check-pmtiles-integration.mjs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/check-pmtiles-integration.mjs b/scripts/check-pmtiles-integration.mjs index 27709374..35b63dcd 100644 --- a/scripts/check-pmtiles-integration.mjs +++ b/scripts/check-pmtiles-integration.mjs @@ -61,8 +61,11 @@ if (helper.includes('Protocol') || helper.includes('setUrl(') || helper.includes if (!helper.includes('cancelPendingTileRequestsWhileZooming: false')) { fail('The browser can cancel still-loading parent tiles during zoom.'); } -if (!helper.includes('maxTileCacheZoomLevels: 8')) { - fail('The browser does not retain enough parent zoom levels for seamless motion.'); +if ( + !helper.includes('const WORLD_ZOOM_PYRAMID_LEVELS = WORLD_MAX_ZOOM - WORLD_MIN_ZOOM + 1') || + !helper.includes('maxTileCacheZoomLevels: WORLD_ZOOM_PYRAMID_LEVELS') +) { + fail('The browser does not retain the complete 0-16 parent zoom pyramid for reverse navigation.'); } if (!helper.includes('refreshExpiredTiles: false')) { fail('The browser can replace visible tiles through in-session expiry refreshes.'); @@ -185,4 +188,4 @@ if (failures.length) { process.exit(1); } -console.log('PMTiles storage integration validated behind one permanent worldwide vector endpoint with a continuous physical foundation, pre-merge and post-encode MVT budgets, bounded upstream work, circuit breaking, stale recovery, and hardened HTTP delivery.'); +console.log('PMTiles storage integration validated behind one permanent worldwide vector endpoint with a continuous physical foundation, complete browser parent-pyramid retention, pre-merge and post-encode MVT budgets, bounded upstream work, circuit breaking, stale recovery, and hardened HTTP delivery.'); From 7c1c4106e92ea7538640f3d80caca866cea2cdcc Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 00:40:17 -0700 Subject: [PATCH 59/85] Validate full browser zoom-pyramid retention --- scripts/check-world-tile-gateway.mjs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/check-world-tile-gateway.mjs b/scripts/check-world-tile-gateway.mjs index 784011a2..096cf4ba 100644 --- a/scripts/check-world-tile-gateway.mjs +++ b/scripts/check-world-tile-gateway.mjs @@ -247,8 +247,9 @@ expect( 'MapLibre can still cancel parent tiles while child zoom tiles are loading.' ); expect( - helper.includes('maxTileCacheZoomLevels: 8'), - 'The browser does not retain enough parent zoom levels for continuous motion.' + helper.includes('const WORLD_ZOOM_PYRAMID_LEVELS = WORLD_MAX_ZOOM - WORLD_MIN_ZOOM + 1') && + helper.includes('maxTileCacheZoomLevels: WORLD_ZOOM_PYRAMID_LEVELS'), + 'The browser does not retain the complete 0-16 parent zoom pyramid for continuous reverse motion.' ); expect( helper.includes('fadeDuration: 300'), @@ -303,4 +304,4 @@ if (failures.length) { process.exit(1); } -console.log('Virtual worldwide tileset validated: one permanent source, one continuous land/landcover/depth foundation at every zoom, safe polygon merging, clipped overscaling, antimeridian routing, parent-tile retention, caching, and no browser-visible shards.'); +console.log('Virtual worldwide tileset validated: one permanent source, one continuous land/landcover/depth foundation at every zoom, safe polygon merging, clipped overscaling, antimeridian routing, full parent-pyramid retention, caching, and no browser-visible shards.'); From 3b748ca012b43f2e7c82d33dfe27d8014d339e3c Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 07:08:51 -0700 Subject: [PATCH 60/85] Serve navigation archives from local disk --- scripts/start-localized-world.mjs | 127 ++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 scripts/start-localized-world.mjs diff --git a/scripts/start-localized-world.mjs b/scripts/start-localized-world.mjs new file mode 100644 index 00000000..b9742f2b --- /dev/null +++ b/scripts/start-localized-world.mjs @@ -0,0 +1,127 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { fileURLToPath } from 'node:url'; + +const repository = process.env.OCCUMED_WORLD_RELEASE_REPOSITORY?.trim() || 'Occumed79/Map'; +const tag = process.env.OCCUMED_WORLD_RELEASE_TAG?.trim() || 'occumed-world-v1'; +const port = Number(process.env.PORT || 4173); +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const assetDir = path.join(root, 'dist', 'virtual-assets'); +const maxAssetBytes = Number(process.env.OCCUMED_NAVIGATION_ASSET_MAX_BYTES || 512 * 1024 * 1024); +const required = process.env.OCCUMED_REQUIRE_LOCAL_NAVIGATION_ASSETS !== 'false'; + +const assets = [ + { + name: 'occumed-world-overview.pmtiles', + explicitUrl: process.env.OCCUMED_WORLD_OVERVIEW_SOURCE_URL?.trim() + }, + { + name: 'occumed-world-surface.pmtiles', + explicitUrl: process.env.OCCUMED_WORLD_SURFACE_SOURCE_URL?.trim() + } +]; + +function releaseUrl(name) { + return `https://github.com/${repository}/releases/download/${encodeURIComponent(tag)}/${encodeURIComponent(name)}`; +} + +async function validPmtiles(filename) { + try { + const stat = await fs.stat(filename); + if (!stat.isFile() || stat.size < 127 || stat.size > maxAssetBytes) return false; + const handle = await fs.open(filename, 'r'); + try { + const magic = Buffer.alloc(7); + const { bytesRead } = await handle.read(magic, 0, magic.length, 0); + return bytesRead === 7 && magic.toString('utf8') === 'PMTiles'; + } finally { + await handle.close(); + } + } catch { + return false; + } +} + +async function downloadAsset(asset) { + const destination = path.join(assetDir, asset.name); + if (await validPmtiles(destination)) { + console.log(`Using localized navigation archive ${asset.name}.`); + return destination; + } + + const sourceUrl = asset.explicitUrl || releaseUrl(asset.name); + const parsed = new URL(sourceUrl); + if (!['http:', 'https:'].includes(parsed.protocol)) { + throw new Error(`Navigation archive ${asset.name} must use HTTP or HTTPS.`); + } + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(new Error(`Download timed out for ${asset.name}.`)), 180_000); + const temporary = `${destination}.${process.pid}.tmp`; + + try { + const response = await fetch(parsed, { + redirect: 'follow', + signal: controller.signal, + headers: { 'User-Agent': 'Occu-Med-Map/local-navigation-assets' } + }); + if (!response.ok || !response.body) { + throw new Error(`Navigation archive ${asset.name} returned HTTP ${response.status}.`); + } + const declaredLength = Number(response.headers.get('content-length')); + if (Number.isFinite(declaredLength) && declaredLength > maxAssetBytes) { + throw new Error(`Navigation archive ${asset.name} exceeds the ${maxAssetBytes}-byte limit.`); + } + + await fs.mkdir(assetDir, { recursive: true }); + await fs.rm(temporary, { force: true }); + await pipeline( + Readable.fromWeb(response.body), + await fs.open(temporary, 'w').then((handle) => handle.createWriteStream()) + ); + + if (!(await validPmtiles(temporary))) { + throw new Error(`Downloaded navigation archive ${asset.name} failed PMTiles validation.`); + } + await fs.rename(temporary, destination); + console.log(`Localized ${asset.name} from release storage.`); + return destination; + } finally { + clearTimeout(timeout); + await fs.rm(temporary, { force: true }).catch(() => {}); + } +} + +let localized = []; +try { + localized = await Promise.all(assets.map(downloadAsset)); +} catch (error) { + console.error(`Unable to localize navigation archives: ${error.message}`); + if (required) process.exit(1); +} + +const env = { ...process.env }; +if (localized.length === assets.length) { + const localOrigin = `http://127.0.0.1:${port}/virtual-assets`; + env.OCCUMED_WORLD_OVERVIEW_URL = `${localOrigin}/${assets[0].name}`; + env.OCCUMED_WORLD_SURFACE_URL = `${localOrigin}/${assets[1].name}`; +} + +const child = spawn(process.execPath, ['server.mjs'], { + cwd: root, + env, + stdio: 'inherit' +}); + +for (const signal of ['SIGINT', 'SIGTERM']) { + process.on(signal, () => child.kill(signal)); +} +child.on('exit', (code, signal) => { + if (signal) process.kill(process.pid, signal); + else process.exit(code ?? 1); +}); From dfe1e4d389197d056d7e93b64e54af20b0f8ca10 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 07:09:39 -0700 Subject: [PATCH 61/85] Require localized navigation assets in production --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d3e11a2a..bca17185 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "dev": "npm run prepare:assets && vite", "build": "npm run prepare:assets && npm run check && vite build && npm run check:server", "preview": "vite preview", - "start": "node server.mjs" + "start": "node scripts/start-localized-world.mjs" }, "dependencies": { "@mapbox/vector-tile": "2.0.4", From b8e6265710fe5e73866e0032e8b23b32fc0e9d14 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 07:10:39 -0700 Subject: [PATCH 62/85] Prebuild coarse navigation tiles through zoom 6 --- scripts/build-world-overview.mjs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/build-world-overview.mjs b/scripts/build-world-overview.mjs index d7cd2910..76705ee1 100644 --- a/scripts/build-world-overview.mjs +++ b/scripts/build-world-overview.mjs @@ -7,9 +7,11 @@ import { mergeVectorTiles } from '../src/server/mvt.js'; import { RetryingFetchSource } from '../src/server/pmtiles-source.js'; import { WorldTileRoutingIndex } from '../src/server/world-tile-routing.js'; +const MIN_NAVIGATION_MAX_ZOOM = 6; + function parseArgs(argv) { const options = { - maxZoom: 5, + maxZoom: MIN_NAVIGATION_MAX_ZOOM, concurrency: 24 }; for (let index = 0; index < argv.length; index += 1) { @@ -26,6 +28,13 @@ function parseArgs(argv) { if (!Number.isSafeInteger(options.maxZoom) || options.maxZoom < 0 || options.maxZoom > 8) { throw new Error('--maxzoom must be an integer between 0 and 8.'); } + if (options.maxZoom < MIN_NAVIGATION_MAX_ZOOM) { + console.warn( + `Requested overview max zoom ${options.maxZoom} is unsafe for reverse navigation; ` + + `building through zoom ${MIN_NAVIGATION_MAX_ZOOM} instead.` + ); + options.maxZoom = MIN_NAVIGATION_MAX_ZOOM; + } if (!Number.isSafeInteger(options.concurrency) || options.concurrency < 1 || options.concurrency > 64) { throw new Error('--concurrency must be an integer between 1 and 64.'); } @@ -125,9 +134,9 @@ const metadata = await archive(metadataRegion).getMetadata(); await fs.writeFile( path.join(options.output, 'metadata.json'), `${JSON.stringify({ - name: 'Occu-Med Worldwide Overview', - description: 'Consolidated low-zoom tiles from the Occu-Med regional storage shards', - version: '1', + name: 'Occu-Med Worldwide Navigation Overview', + description: 'Consolidated coarse-navigation tiles from the Occu-Med regional storage shards', + version: '2', type: 'baselayer', format: 'pbf', minzoom: 0, @@ -139,5 +148,5 @@ await fs.writeFile( ); console.log( - `Worldwide overview complete: ${writtenTiles} tiles, ${sourceTileReads} shard reads, zoom 0-${options.maxZoom}.` + `Worldwide navigation overview complete: ${writtenTiles} tiles, ${sourceTileReads} shard reads, zoom 0-${options.maxZoom}.` ); From faaaed0b8fa134781091343106cbd266f9502296 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 11:17:31 -0700 Subject: [PATCH 63/85] Fix continuous zoom validation instrumentation --- scripts/validate-all-zoom-levels.mjs | 187 ++++++++++++++++++-------- scripts/validate-continuous-zoom.mjs | 189 +++++++++++++++++++-------- 2 files changed, 269 insertions(+), 107 deletions(-) diff --git a/scripts/validate-all-zoom-levels.mjs b/scripts/validate-all-zoom-levels.mjs index 04d3ac76..afdeca2b 100644 --- a/scripts/validate-all-zoom-levels.mjs +++ b/scripts/validate-all-zoom-levels.mjs @@ -77,6 +77,7 @@ function normalizeSweep(result, definition) { : Number(result.maximumZoom), maximumZoomGap: Number(result?.maximumZoomGap || 0), minimumFeatureCount: Number(result?.minimumFeatureCount || 0), + postMoveendSettleMs: Number(result?.postMoveendSettleMs || 0), samples: Array.isArray(result?.samples) ? result.samples : [], executionError: null }; @@ -134,17 +135,29 @@ try { }, 90_000); const interval = setInterval(check, 100); - function renderedLayersPresent() { - const rendered = map - .queryRenderedFeatures() - .filter((feature) => feature.source === 'occumed-open'); - return rendered.length > 0 && requiredLayers.every((required) => - rendered.some((feature) => feature.sourceLayer === required) - ); + function requiredLayerCounts() { + const counts = Object.fromEntries(requiredLayers.map((layer) => [layer, 0])); + const layerIds = (map.getStyle().layers || []) + .filter((layer) => + layer.source === 'occumed-open' && + requiredLayers.includes(layer['source-layer']) + ) + .map((layer) => layer.id); + if (layerIds.length === 0) return counts; + for (const feature of map.queryRenderedFeatures({ layers: layerIds })) { + const sourceLayer = feature.sourceLayer; + if (sourceLayer in counts) counts[sourceLayer] += 1; + } + return counts; } function check() { - if (map.isStyleLoaded() && map.areTilesLoaded() && renderedLayersPresent()) { + const counts = requiredLayerCounts(); + if ( + map.isStyleLoaded() && + map.isSourceLoaded('occumed-open') && + requiredLayers.every((layer) => counts[layer] > 0) + ) { cleanup(); resolve(); } @@ -187,26 +200,72 @@ try { let sourceChanged = false; const expectedSignature = JSON.stringify({ url: null, tiles: [expectedTemplate] }); + const requiredLayerCounts = () => { + const counts = Object.fromEntries(requiredLayers.map((layer) => [layer, 0])); + const layerIds = (map.getStyle().layers || []) + .filter((layer) => + layer.source === 'occumed-open' && + requiredLayers.includes(layer['source-layer']) + ) + .map((layer) => layer.id); + if (layerIds.length === 0) return counts; + for (const feature of map.queryRenderedFeatures({ layers: layerIds })) { + const sourceLayer = feature.sourceLayer; + if (sourceLayer in counts) counts[sourceLayer] += 1; + } + return counts; + }; + + const waitForRequiredFoundation = () => new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + reject(new Error( + `The source did not settle with ${requiredLayers.join(', ')} rendered after moveend.` + )); + }, 30_000); + const interval = setInterval(check, 100); + + function check() { + const counts = requiredLayerCounts(); + if ( + map.isStyleLoaded() && + map.isSourceLoaded('occumed-open') && + requiredLayers.every((layer) => counts[layer] > 0) + ) { + cleanup(); + resolve(); + } + } + + function onSourceData(event) { + if (event.sourceId === 'occumed-open') check(); + } + + function cleanup() { + clearTimeout(timeout); + clearInterval(interval); + map.off('sourcedata', onSourceData); + map.off('idle', check); + } + + map.on('sourcedata', onSourceData); + map.on('idle', check); + check(); + }); + const sample = () => { const timestamp = performance.now(); const source = map.getStyle().sources?.['occumed-open'] || null; const signature = JSON.stringify({ url: source?.url || null, tiles: source?.tiles || [] }); sourceChanged ||= signature !== expectedSignature; - const rendered = map - .queryRenderedFeatures() - .filter((feature) => feature.source === 'occumed-open'); - const sourceLayers = {}; - for (const feature of rendered) { - const layer = feature.sourceLayer || 'unknown'; - sourceLayers[layer] = (sourceLayers[layer] || 0) + 1; - } + const sourceLayers = requiredLayerCounts(); + const renderedFeatureCount = Object.values(sourceLayers) + .reduce((total, count) => total + count, 0); samples.push({ timestamp, zoom: Number(map.getZoom()), - renderedFeatureCount: rendered.length, - requiredLayerCounts: Object.fromEntries( - requiredLayers.map((layer) => [layer, sourceLayers[layer] || 0]) - ), + renderedFeatureCount, + requiredLayerCounts: { ...sourceLayers }, sourceLayers, tilesLoaded: Boolean(map.areTilesLoaded()), sourceSignature: signature @@ -215,44 +274,67 @@ try { return await new Promise((resolve, reject) => { const durationMs = 20_000; - const sampleTimer = setInterval(sample, 50); - const timeout = setTimeout(() => { + let sampleFrame = null; + const queueSample = () => { + if (sampleFrame !== null) return; + sampleFrame = requestAnimationFrame(() => { + sampleFrame = null; + sample(); + }); + }; + const sampleTimer = setInterval(queueSample, 50); + const stopSampling = () => { clearInterval(sampleTimer); + if (sampleFrame !== null) { + cancelAnimationFrame(sampleFrame); + sampleFrame = null; + } + }; + const timeout = setTimeout(() => { + stopSampling(); map.off('moveend', finish); reject(new Error(`${name} full-range zoom sweep timed out.`)); }, durationMs + 35_000); - const finish = () => { - clearTimeout(timeout); - clearInterval(sampleTimer); - sample(); - const actualEndZoom = Number(map.getZoom()); - const zooms = samples.map((entry) => entry.zoom).sort((a, b) => a - b); - let maximumZoomGap = 0; - for (let index = 1; index < zooms.length; index += 1) { - maximumZoomGap = Math.max(maximumZoomGap, zooms[index] - zooms[index - 1]); + const finish = async () => { + stopSampling(); + const moveendAt = performance.now(); + try { + await waitForRequiredFoundation(); + clearTimeout(timeout); + sample(); + const actualEndZoom = Number(map.getZoom()); + const zooms = samples.map((entry) => entry.zoom).sort((a, b) => a - b); + let maximumZoomGap = 0; + for (let index = 1; index < zooms.length; index += 1) { + maximumZoomGap = Math.max(maximumZoomGap, zooms[index] - zooms[index - 1]); + } + const blankSamples = samples.filter((entry) => entry.renderedFeatureCount === 0); + const missingFoundationSamples = samples.filter((entry) => + requiredLayers.some((layer) => (entry.requiredLayerCounts[layer] || 0) <= 0) + ); + resolve({ + name, + actualStartZoom, + actualEndZoom, + sampleCount: samples.length, + sourceChanged, + blankSampleCount: blankSamples.length, + missingFoundationSampleCount: missingFoundationSamples.length, + firstMissingFoundationSamples: missingFoundationSamples.slice(0, 20), + minimumZoom: zooms.length ? Math.min(...zooms) : null, + maximumZoom: zooms.length ? Math.max(...zooms) : null, + maximumZoomGap, + minimumFeatureCount: samples.length + ? Math.min(...samples.map((entry) => entry.renderedFeatureCount)) + : 0, + postMoveendSettleMs: performance.now() - moveendAt, + samples + }); + } catch (error) { + clearTimeout(timeout); + reject(error); } - const blankSamples = samples.filter((entry) => entry.renderedFeatureCount === 0); - const missingFoundationSamples = samples.filter((entry) => - requiredLayers.some((layer) => (entry.requiredLayerCounts[layer] || 0) <= 0) - ); - resolve({ - name, - actualStartZoom, - actualEndZoom, - sampleCount: samples.length, - sourceChanged, - blankSampleCount: blankSamples.length, - missingFoundationSampleCount: missingFoundationSamples.length, - firstMissingFoundationSamples: missingFoundationSamples.slice(0, 20), - minimumZoom: zooms.length ? Math.min(...zooms) : null, - maximumZoom: zooms.length ? Math.max(...zooms) : null, - maximumZoomGap, - minimumFeatureCount: samples.length - ? Math.min(...samples.map((entry) => entry.renderedFeatureCount)) - : 0, - samples - }); }; sample(); @@ -309,6 +391,7 @@ try { maximumZoom: null, maximumZoomGap: 0, minimumFeatureCount: 0, + postMoveendSettleMs: 0, samples: [], executionError: serializeError(error) }); diff --git a/scripts/validate-continuous-zoom.mjs b/scripts/validate-continuous-zoom.mjs index 96109433..003ee10e 100644 --- a/scripts/validate-continuous-zoom.mjs +++ b/scripts/validate-continuous-zoom.mjs @@ -68,6 +68,7 @@ function normalizeMotion(result, definition) { longestBlankRun: Number(result?.longestBlankRun || 0), minimumFeatureCount: Number(result?.minimumFeatureCount || 0), maximumFeatureCount: Number(result?.maximumFeatureCount || 0), + postMoveendSettleMs: Number(result?.postMoveendSettleMs || 0), samples: Array.isArray(result?.samples) ? result.samples : [], executionError: null }; @@ -142,20 +143,28 @@ try { }, 90_000); const interval = setInterval(check, 100); - function renderedLayersPresent() { - const rendered = map - .queryRenderedFeatures() - .filter((feature) => feature.source === 'occumed-open'); - return rendered.length > 0 && requiredLayers.every((required) => - rendered.some((feature) => feature.sourceLayer === required) - ); + function requiredLayerCounts() { + const counts = Object.fromEntries(requiredLayers.map((layer) => [layer, 0])); + const layerIds = (map.getStyle().layers || []) + .filter((layer) => + layer.source === 'occumed-open' && + requiredLayers.includes(layer['source-layer']) + ) + .map((layer) => layer.id); + if (layerIds.length === 0) return counts; + for (const feature of map.queryRenderedFeatures({ layers: layerIds })) { + const sourceLayer = feature.sourceLayer; + if (sourceLayer in counts) counts[sourceLayer] += 1; + } + return counts; } function check() { + const counts = requiredLayerCounts(); if ( map.isStyleLoaded() && - map.areTilesLoaded() && - renderedLayersPresent() + map.isSourceLoaded('occumed-open') && + requiredLayers.every((layer) => counts[layer] > 0) ) { cleanup(); resolve(); @@ -201,27 +210,73 @@ try { let sourceChanged = false; const expectedSignature = JSON.stringify({ url: null, tiles: [expectedTemplate] }); + const requiredLayerCounts = () => { + const counts = Object.fromEntries(requiredLayers.map((layer) => [layer, 0])); + const layerIds = (map.getStyle().layers || []) + .filter((layer) => + layer.source === 'occumed-open' && + requiredLayers.includes(layer['source-layer']) + ) + .map((layer) => layer.id); + if (layerIds.length === 0) return counts; + for (const feature of map.queryRenderedFeatures({ layers: layerIds })) { + const sourceLayer = feature.sourceLayer; + if (sourceLayer in counts) counts[sourceLayer] += 1; + } + return counts; + }; + + const waitForRequiredFoundation = () => new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + reject(new Error( + `The source did not settle with ${requiredLayers.join(', ')} rendered after moveend.` + )); + }, 30_000); + const interval = setInterval(check, 100); + + function check() { + const counts = requiredLayerCounts(); + if ( + map.isStyleLoaded() && + map.isSourceLoaded('occumed-open') && + requiredLayers.every((layer) => counts[layer] > 0) + ) { + cleanup(); + resolve(); + } + } + + function onSourceData(event) { + if (event.sourceId === 'occumed-open') check(); + } + + function cleanup() { + clearTimeout(timeout); + clearInterval(interval); + map.off('sourcedata', onSourceData); + map.off('idle', check); + } + + map.on('sourcedata', onSourceData); + map.on('idle', check); + check(); + }); + const sample = () => { const timestamp = performance.now(); const source = map.getStyle().sources?.['occumed-open'] || null; const signature = JSON.stringify({ url: source?.url || null, tiles: source?.tiles || [] }); sourceChanged ||= signature !== expectedSignature; - const rendered = map - .queryRenderedFeatures() - .filter((feature) => feature.source === 'occumed-open'); - const sourceLayers = {}; - for (const feature of rendered) { - const layer = feature.sourceLayer || 'unknown'; - sourceLayers[layer] = (sourceLayers[layer] || 0) + 1; - } + const sourceLayers = requiredLayerCounts(); + const vectorFeatureCount = Object.values(sourceLayers) + .reduce((total, count) => total + count, 0); samples.push({ timestamp, zoom: Number(map.getZoom()), center: map.getCenter().toArray().map(Number), - vectorFeatureCount: rendered.length, - requiredLayerCounts: Object.fromEntries( - requiredLayers.map((layer) => [layer, sourceLayers[layer] || 0]) - ), + vectorFeatureCount, + requiredLayerCounts: { ...sourceLayers }, sourceLayers, tilesLoaded: Boolean(map.areTilesLoaded()), sourceSignature: signature @@ -229,47 +284,70 @@ try { }; return await new Promise((resolve, reject) => { - const sampleTimer = setInterval(sample, 50); - const timeout = setTimeout(() => { + let sampleFrame = null; + const queueSample = () => { + if (sampleFrame !== null) return; + sampleFrame = requestAnimationFrame(() => { + sampleFrame = null; + sample(); + }); + }; + const sampleTimer = setInterval(queueSample, 50); + const stopSampling = () => { clearInterval(sampleTimer); + if (sampleFrame !== null) { + cancelAnimationFrame(sampleFrame); + sampleFrame = null; + } + }; + const timeout = setTimeout(() => { + stopSampling(); map.off('moveend', finish); reject(new Error(`${name} motion timed out.`)); }, durationMs + 35_000); - const finish = () => { - clearTimeout(timeout); - clearInterval(sampleTimer); - sample(); - const blankSamples = samples.filter((entry) => entry.vectorFeatureCount === 0); - const missingFoundationSamples = samples.filter((entry) => - requiredLayers.some((layer) => (entry.requiredLayerCounts[layer] || 0) <= 0) - ); - let longestBlankRun = 0; - let currentBlankRun = 0; - for (const entry of samples) { - if (entry.vectorFeatureCount === 0) { - currentBlankRun += 1; - longestBlankRun = Math.max(longestBlankRun, currentBlankRun); - } else { - currentBlankRun = 0; + const finish = async () => { + stopSampling(); + const moveendAt = performance.now(); + try { + await waitForRequiredFoundation(); + clearTimeout(timeout); + sample(); + const blankSamples = samples.filter((entry) => entry.vectorFeatureCount === 0); + const missingFoundationSamples = samples.filter((entry) => + requiredLayers.some((layer) => (entry.requiredLayerCounts[layer] || 0) <= 0) + ); + let longestBlankRun = 0; + let currentBlankRun = 0; + for (const entry of samples) { + if (entry.vectorFeatureCount === 0) { + currentBlankRun += 1; + longestBlankRun = Math.max(longestBlankRun, currentBlankRun); + } else { + currentBlankRun = 0; + } } + resolve({ + name, + sampleCount: samples.length, + sourceChanged, + blankSampleCount: blankSamples.length, + missingFoundationSampleCount: missingFoundationSamples.length, + firstMissingFoundationSamples: missingFoundationSamples.slice(0, 20), + longestBlankRun, + minimumFeatureCount: samples.length + ? Math.min(...samples.map((entry) => entry.vectorFeatureCount)) + : 0, + maximumFeatureCount: samples.length + ? Math.max(...samples.map((entry) => entry.vectorFeatureCount)) + : 0, + postMoveendSettleMs: performance.now() - moveendAt, + samples + }); + } catch (error) { + clearTimeout(timeout); + reject(error); } - resolve({ - name, - sampleCount: samples.length, - sourceChanged, - blankSampleCount: blankSamples.length, - missingFoundationSampleCount: missingFoundationSamples.length, - firstMissingFoundationSamples: missingFoundationSamples.slice(0, 20), - longestBlankRun, - minimumFeatureCount: samples.length - ? Math.min(...samples.map((entry) => entry.vectorFeatureCount)) - : 0, - maximumFeatureCount: samples.length - ? Math.max(...samples.map((entry) => entry.vectorFeatureCount)) - : 0, - samples - }); }; sample(); @@ -370,6 +448,7 @@ try { longestBlankRun: 0, minimumFeatureCount: 0, maximumFeatureCount: 0, + postMoveendSettleMs: 0, samples: [], executionError: serializeError(error) }); From 26ee960282a6a5d73161a21213471ebd82fc76ac Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 11:47:17 -0700 Subject: [PATCH 64/85] placeholder --- src/server/neon-navigation-tile-cache.js | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/server/neon-navigation-tile-cache.js diff --git a/src/server/neon-navigation-tile-cache.js b/src/server/neon-navigation-tile-cache.js new file mode 100644 index 00000000..b3a42524 --- /dev/null +++ b/src/server/neon-navigation-tile-cache.js @@ -0,0 +1 @@ +placeholder \ No newline at end of file From bc376f8d9dfe1e7def134dbfcc3760e12c145849 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 11:47:30 -0700 Subject: [PATCH 65/85] Create Neon integration anchor --- tmp-neon-branch-anchor.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 tmp-neon-branch-anchor.txt diff --git a/tmp-neon-branch-anchor.txt b/tmp-neon-branch-anchor.txt new file mode 100644 index 00000000..6ad1ea22 --- /dev/null +++ b/tmp-neon-branch-anchor.txt @@ -0,0 +1 @@ +temporary integration anchor From dab64f254b6b73c551f3c047775ddb044b74e16e Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 11:48:50 -0700 Subject: [PATCH 66/85] Implement durable Neon navigation cache --- src/server/neon-navigation-tile-cache.js | 387 ++++++++++++++++++++++- 1 file changed, 386 insertions(+), 1 deletion(-) diff --git a/src/server/neon-navigation-tile-cache.js b/src/server/neon-navigation-tile-cache.js index b3a42524..4d6ff708 100644 --- a/src/server/neon-navigation-tile-cache.js +++ b/src/server/neon-navigation-tile-cache.js @@ -1 +1,386 @@ -placeholder \ No newline at end of file +import postgres from 'postgres'; + +const DEFAULT_MAX_ZOOM = 6; +const DEFAULT_QUERY_TIMEOUT_MS = 1_500; +const DEFAULT_RETRY_DELAY_MS = 30_000; +const DEFAULT_MAX_BYTES_PER_SHARD = 48 * 1024 * 1024; +const DEFAULT_PRUNE_EVERY_WRITES = 64; +const MAX_DATABASE_SHARDS = 8; +const TABLE_NAME = 'occumed_navigation_tile_cache'; + +function boundedInteger(value, fallback, minimum, maximum) { + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed >= minimum && parsed <= maximum + ? parsed + : fallback; +} + +function safeDatabaseUrl(value) { + const raw = String(value || '').trim(); + if (!raw) return null; + try { + const url = new URL(raw); + if (!['postgres:', 'postgresql:'].includes(url.protocol)) return null; + if (!url.hostname || !url.username || !url.pathname || url.pathname === '/') return null; + return raw; + } catch { + return null; + } +} + +export function collectNavigationDatabaseUrls(env = process.env) { + const seen = new Set(); + const entries = []; + for (let slot = 1; slot <= MAX_DATABASE_SHARDS; slot += 1) { + const value = safeDatabaseUrl(env[`NAV_DATABASE_URL_${slot}`]); + if (!value || seen.has(value)) continue; + seen.add(value); + entries.push({ slot, url: value }); + } + return entries; +} + +export function navigationTileShardIndex(key, shardCount) { + if (!Number.isSafeInteger(shardCount) || shardCount <= 0) return -1; + let hash = 0x811c9dc5; + for (let index = 0; index < key.length; index += 1) { + hash ^= key.charCodeAt(index); + hash = Math.imul(hash, 0x01000193); + } + return (hash >>> 0) % shardCount; +} + +function timeoutAfter(milliseconds, label) { + return new Promise((_, reject) => { + const timer = setTimeout(() => { + const error = new Error(`${label} timed out.`); + error.code = 'OCCUMED_NAV_CACHE_TIMEOUT'; + reject(error); + }, milliseconds); + timer.unref?.(); + }); +} + +async function withinTimeout(task, milliseconds, label) { + return Promise.race([task, timeoutAfter(milliseconds, label)]); +} + +class PostgresNavigationShard { + constructor({ + slot, + url, + queryTimeoutMs, + retryDelayMs, + maxBytes, + pruneEveryWrites, + now, + logger + }) { + this.slot = slot; + this.queryTimeoutMs = queryTimeoutMs; + this.retryDelayMs = retryDelayMs; + this.maxBytes = maxBytes; + this.pruneEveryWrites = pruneEveryWrites; + this.now = now; + this.logger = logger; + this.initialization = null; + this.initialized = false; + this.disabledUntil = 0; + this.writeCount = 0; + this.lastErrorCode = null; + this.metrics = { + hits: 0, + misses: 0, + writes: 0, + errors: 0, + prunes: 0 + }; + this.sql = postgres(url, { + max: 1, + prepare: false, + connect_timeout: 5, + idle_timeout: 20, + max_lifetime: 60 * 30, + onnotice: () => {} + }); + } + + available() { + return this.now() >= this.disabledUntil; + } + + recordError(error, operation) { + this.metrics.errors += 1; + this.disabledUntil = this.now() + this.retryDelayMs; + this.lastErrorCode = String(error?.code || error?.name || 'UNKNOWN').slice(0, 80); + this.logger?.warn?.(JSON.stringify({ + level: 'warn', + type: 'navigation-cache-error', + shard: this.slot, + operation, + code: this.lastErrorCode + })); + } + + async initialize() { + if (this.initialized) return true; + if (!this.available()) return false; + if (this.initialization) return this.initialization; + + this.initialization = withinTimeout( + this.sql.unsafe(` + CREATE TABLE IF NOT EXISTS ${TABLE_NAME} ( + tileset_version text NOT NULL, + z smallint NOT NULL CHECK (z BETWEEN 0 AND ${DEFAULT_MAX_ZOOM}), + x integer NOT NULL CHECK (x >= 0), + y integer NOT NULL CHECK (y >= 0), + tile bytea NOT NULL, + byte_length integer NOT NULL CHECK (byte_length = octet_length(tile)), + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (tileset_version, z, x, y) + ); + CREATE INDEX IF NOT EXISTS ${TABLE_NAME}_created_at_idx + ON ${TABLE_NAME} (created_at); + `), + this.queryTimeoutMs * 4, + `Navigation cache shard ${this.slot} initialization` + ) + .then(() => { + this.initialized = true; + this.disabledUntil = 0; + this.lastErrorCode = null; + return true; + }) + .catch((error) => { + this.recordError(error, 'initialize'); + return false; + }) + .finally(() => { + this.initialization = null; + }); + + return this.initialization; + } + + async get(tilesetVersion, zoom, x, y) { + if (!(await this.initialize())) return null; + try { + const rows = await withinTimeout( + this.sql` + SELECT tile + FROM ${this.sql(TABLE_NAME)} + WHERE tileset_version = ${tilesetVersion} + AND z = ${zoom} + AND x = ${x} + AND y = ${y} + LIMIT 1 + `, + this.queryTimeoutMs, + `Navigation cache shard ${this.slot} read` + ); + const tile = rows?.[0]?.tile; + if (!tile) { + this.metrics.misses += 1; + return null; + } + this.metrics.hits += 1; + return Buffer.from(tile); + } catch (error) { + this.recordError(error, 'read'); + return null; + } + } + + async set(tilesetVersion, zoom, x, y, value) { + if (!(await this.initialize())) return false; + const tile = Buffer.from(value); + try { + await withinTimeout( + this.sql` + INSERT INTO ${this.sql(TABLE_NAME)} ( + tileset_version, z, x, y, tile, byte_length, created_at + ) VALUES ( + ${tilesetVersion}, ${zoom}, ${x}, ${y}, ${tile}, ${tile.byteLength}, now() + ) + ON CONFLICT (tileset_version, z, x, y) + DO UPDATE SET + tile = EXCLUDED.tile, + byte_length = EXCLUDED.byte_length, + created_at = now() + `, + this.queryTimeoutMs, + `Navigation cache shard ${this.slot} write` + ); + this.metrics.writes += 1; + this.writeCount += 1; + this.disabledUntil = 0; + this.lastErrorCode = null; + if (this.writeCount % this.pruneEveryWrites === 0) { + void this.prune(tilesetVersion); + } + return true; + } catch (error) { + this.recordError(error, 'write'); + return false; + } + } + + async prune(tilesetVersion) { + if (!this.initialized || !this.available()) return false; + try { + await withinTimeout( + this.sql.begin(async (transaction) => { + await transaction` + DELETE FROM ${transaction(TABLE_NAME)} + WHERE tileset_version <> ${tilesetVersion} + `; + await transaction.unsafe(` + WITH ranked AS ( + SELECT + ctid, + sum(byte_length) OVER ( + ORDER BY created_at DESC, z DESC, x DESC, y DESC + ) AS running_bytes + FROM ${TABLE_NAME} + WHERE tileset_version = $1 + ) + DELETE FROM ${TABLE_NAME} AS cache + USING ranked + WHERE cache.ctid = ranked.ctid + AND ranked.running_bytes > $2 + `, [tilesetVersion, this.maxBytes]); + }), + this.queryTimeoutMs * 4, + `Navigation cache shard ${this.slot} prune` + ); + this.metrics.prunes += 1; + return true; + } catch (error) { + this.recordError(error, 'prune'); + return false; + } + } + + async close() { + await this.sql.end({ timeout: 2 }).catch(() => {}); + } + + snapshot() { + return { + slot: this.slot, + initialized: this.initialized, + available: this.available(), + retryAfterMs: Math.max(0, this.disabledUntil - this.now()), + maxBytes: this.maxBytes, + lastErrorCode: this.lastErrorCode, + ...this.metrics + }; + } +} + +export class NeonNavigationTileCache { + constructor(databaseEntries, { + maxZoom = DEFAULT_MAX_ZOOM, + queryTimeoutMs = DEFAULT_QUERY_TIMEOUT_MS, + retryDelayMs = DEFAULT_RETRY_DELAY_MS, + maxBytesPerShard = DEFAULT_MAX_BYTES_PER_SHARD, + pruneEveryWrites = DEFAULT_PRUNE_EVERY_WRITES, + now = () => Date.now(), + logger = console, + shardFactory = (options) => new PostgresNavigationShard(options) + } = {}) { + this.maxZoom = boundedInteger(maxZoom, DEFAULT_MAX_ZOOM, 0, DEFAULT_MAX_ZOOM); + this.queryTimeoutMs = boundedInteger(queryTimeoutMs, DEFAULT_QUERY_TIMEOUT_MS, 100, 30_000); + this.retryDelayMs = boundedInteger(retryDelayMs, DEFAULT_RETRY_DELAY_MS, 1_000, 15 * 60_000); + this.maxBytesPerShard = boundedInteger( + maxBytesPerShard, + DEFAULT_MAX_BYTES_PER_SHARD, + 1 * 1024 * 1024, + 256 * 1024 * 1024 + ); + this.pruneEveryWrites = boundedInteger(pruneEveryWrites, DEFAULT_PRUNE_EVERY_WRITES, 1, 10_000); + this.now = now; + this.logger = logger; + this.metrics = { + skippedAboveMaxZoom: 0, + skippedNoShard: 0 + }; + this.shards = databaseEntries.map(({ slot, url }) => shardFactory({ + slot, + url, + queryTimeoutMs: this.queryTimeoutMs, + retryDelayMs: this.retryDelayMs, + maxBytes: this.maxBytesPerShard, + pruneEveryWrites: this.pruneEveryWrites, + now, + logger + })); + } + + eligible(zoom) { + return Number.isSafeInteger(zoom) && zoom >= 0 && zoom <= this.maxZoom; + } + + shardFor(tilesetVersion, zoom, x, y) { + if (!this.shards.length) { + this.metrics.skippedNoShard += 1; + return null; + } + const key = `${tilesetVersion}/${zoom}/${x}/${y}`; + return this.shards[navigationTileShardIndex(key, this.shards.length)]; + } + + async initialize() { + const results = []; + for (const shard of this.shards) results.push(await shard.initialize()); + return results.filter(Boolean).length; + } + + async get(tilesetVersion, zoom, x, y) { + if (!this.eligible(zoom)) { + this.metrics.skippedAboveMaxZoom += 1; + return null; + } + const shard = this.shardFor(tilesetVersion, zoom, x, y); + return shard ? shard.get(tilesetVersion, zoom, x, y) : null; + } + + async set(tilesetVersion, zoom, x, y, value) { + if (!this.eligible(zoom)) { + this.metrics.skippedAboveMaxZoom += 1; + return false; + } + const shard = this.shardFor(tilesetVersion, zoom, x, y); + return shard ? shard.set(tilesetVersion, zoom, x, y, value) : false; + } + + async close() { + await Promise.all(this.shards.map((shard) => shard.close())); + } + + snapshot() { + return { + enabled: this.shards.length > 0, + configuredShards: this.shards.length, + expectedShards: MAX_DATABASE_SHARDS, + maxZoom: this.maxZoom, + queryTimeoutMs: this.queryTimeoutMs, + maxBytesPerShard: this.maxBytesPerShard, + ...this.metrics, + shards: this.shards.map((shard) => shard.snapshot()) + }; + } +} + +export function createNeonNavigationTileCacheFromEnv(env = process.env, options = {}) { + const databaseEntries = collectNavigationDatabaseUrls(env); + if (!databaseEntries.length) return null; + return new NeonNavigationTileCache(databaseEntries, { + maxZoom: env.OCCUMED_NAV_CACHE_MAX_ZOOM, + queryTimeoutMs: env.OCCUMED_NAV_CACHE_QUERY_TIMEOUT_MS, + retryDelayMs: env.OCCUMED_NAV_CACHE_RETRY_DELAY_MS, + maxBytesPerShard: env.OCCUMED_NAV_CACHE_MAX_BYTES_PER_SHARD, + pruneEveryWrites: env.OCCUMED_NAV_CACHE_PRUNE_EVERY_WRITES, + ...options + }); +} From 751ee14197c6ddb7eca8dba083de11f9b93f9db4 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 11:51:32 -0700 Subject: [PATCH 67/85] Wire persistent navigation cache into tile gateway --- src/server/world-tile-gateway.js | 71 +++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 10 deletions(-) diff --git a/src/server/world-tile-gateway.js b/src/server/world-tile-gateway.js index 6ee9f4eb..36755d99 100644 --- a/src/server/world-tile-gateway.js +++ b/src/server/world-tile-gateway.js @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { PMTiles, SharedPromiseCache } from 'pmtiles'; import { EMPTY_MVT, @@ -325,6 +326,7 @@ export class WorldTileGateway { archiveReadConcurrency = DEFAULT_MAX_ARCHIVE_READS, archiveReadQueue = DEFAULT_MAX_ARCHIVE_QUEUE, overviewUrl = process.env.OCCUMED_WORLD_OVERVIEW_URL?.trim() || '', + persistentTileCache = null, now = () => Date.now() }) { this.manifestUrl = validateHttpUrl(manifestUrl, 'The worldwide manifest URL'); @@ -334,6 +336,7 @@ export class WorldTileGateway { this.releaseAssetUrl = releaseAssetUrl; this.fetchImpl = fetchImpl; this.overviewUrl = overviewUrl ? validateHttpUrl(overviewUrl, 'The overview URL') : ''; + this.persistentTileCache = persistentTileCache; this.now = now; this.manifestTimeoutMs = boundedInteger(manifestTimeoutMs, DEFAULT_MANIFEST_TIMEOUT_MS, 500, 60_000); this.manifestTtlMs = boundedInteger(manifestTtlMs, DEFAULT_MANIFEST_TTL_MS, 1_000, 24 * 60 * 60 * 1_000); @@ -364,7 +367,10 @@ export class WorldTileGateway { failed: 0, staleServed: 0, overloads: 0, - missingSurface: 0 + missingSurface: 0, + persistentHits: 0, + persistentMisses: 0, + persistentWrites: 0 }; } @@ -397,6 +403,7 @@ export class WorldTileGateway { const manifest = parseManifest(document, { maxRegions: this.maxRegions }); return { ...manifest, + cacheVersion: createHash('sha256').update(text).digest('hex').slice(0, 32), routingIndex: new WorldTileRoutingIndex(manifest.regions, { routingZoom: manifest.virtualTiles.routingZoom, maxCellFanout: this.maxTileFanout @@ -579,15 +586,47 @@ export class WorldTileGateway { } const promise = this.loadManifest() - .then((manifest) => this.buildTile( - manifest, - coordinates.z, - coordinates.x, - coordinates.y - )) - .then((tile) => { + .then(async (manifest) => { + if (this.persistentTileCache) { + const persisted = await this.persistentTileCache.get( + manifest.cacheVersion, + coordinates.z, + coordinates.x, + coordinates.y + ); + if ( + persisted?.byteLength > 0 && + persisted.byteLength <= this.maxResolvedTileBytes + ) { + this.metrics.persistentHits += 1; + return { tile: persisted, manifest, built: false }; + } + this.metrics.persistentMisses += 1; + } + + const tile = await this.buildTile( + manifest, + coordinates.z, + coordinates.x, + coordinates.y + ); + return { tile, manifest, built: true }; + }) + .then(({ tile, manifest, built }) => { this.metrics.resolved += 1; - return this.tileCache.set(key, tile); + const cached = this.tileCache.set(key, tile); + if (built && this.persistentTileCache) { + void this.persistentTileCache.set( + manifest.cacheVersion, + coordinates.z, + coordinates.x, + coordinates.y, + cached + ).then((written) => { + if (written) this.metrics.persistentWrites += 1; + }).catch(() => {}); + } + return cached; }) .catch((error) => { this.metrics.failed += 1; @@ -603,6 +642,16 @@ export class WorldTileGateway { return promise; } + async initializePersistentCache() { + return this.persistentTileCache?.initialize + ? this.persistentTileCache.initialize() + : 0; + } + + async close() { + await this.persistentTileCache?.close?.(); + } + async ready() { const manifest = await this.loadManifest(); return { @@ -619,10 +668,12 @@ export class WorldTileGateway { loadedAt: this.manifestState.loadedAt, freshUntil: this.manifestState.freshUntil, staleUntil: this.manifestState.staleUntil, - regions: this.manifestState.manifest.regions.length + regions: this.manifestState.manifest.regions.length, + cacheVersion: this.manifestState.manifest.cacheVersion } : null, cache: this.tileCache.snapshot(), + persistentCache: this.persistentTileCache?.snapshot?.() || { enabled: false }, archiveReads: this.archiveReadLimiter.snapshot(), inflightTiles: this.inflight.size, archives: this.archives.size, From b102596e3dd3e6775c19afbca3c384b034fbe9e5 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 11:53:13 -0700 Subject: [PATCH 68/85] Connect Neon cache to production tile server --- server.mjs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/server.mjs b/server.mjs index 48e7397a..e1d22297 100644 --- a/server.mjs +++ b/server.mjs @@ -6,6 +6,7 @@ import path from 'node:path'; import { promisify } from 'node:util'; import { fileURLToPath } from 'node:url'; import { gzip } from 'node:zlib'; +import { createNeonNavigationTileCacheFromEnv } from './src/server/neon-navigation-tile-cache.js'; import { GatewayOverloadedError, WorldTileGateway @@ -171,11 +172,13 @@ function releaseAssetUrl(assetName) { return `https://github.com/${worldReleaseRepository}/releases/download/${encodeURIComponent(worldReleaseTag)}/${encodeURIComponent(assetName)}`; } +const navigationTileCache = createNeonNavigationTileCacheFromEnv(); const worldTileGateway = new WorldTileGateway({ manifestUrl: process.env.OCCUMED_WORLD_MANIFEST_URL?.trim() || releaseAssetUrl(worldManifestAsset), releaseAssetUrl, + persistentTileCache: navigationTileCache, maxResolvedTileBytes: safeInteger( maxResolvedTileBytes, 24 * 1024 * 1024, @@ -555,13 +558,16 @@ function shutdown(signal) { shuttingDown = true; console.log(`Occu-Med Map received ${signal}; draining connections.`); server.close((error) => { - if (error) { - console.error('Occu-Med Map shutdown error:', error); - process.exitCode = 1; - } + void worldTileGateway.close().finally(() => { + if (error) { + console.error('Occu-Med Map shutdown error:', error); + process.exitCode = 1; + } + }); }); const timer = setTimeout(() => { server.closeAllConnections?.(); + void worldTileGateway.close(); process.exitCode = 1; }, 25_000); timer.unref?.(); @@ -573,6 +579,14 @@ process.once('SIGINT', () => shutdown('SIGINT')); server.listen(port, host, () => { console.log(`Occu-Med Map listening on ${host}:${port}.`); console.log(`Health endpoint ready at http://127.0.0.1:${port}/health.`); + if (navigationTileCache) { + const snapshot = navigationTileCache.snapshot(); + console.log(`Neon navigation cache configured with ${snapshot.configuredShards} of ${snapshot.expectedShards} shards.`); + void worldTileGateway.initializePersistentCache().then( + (initialized) => console.log(`Neon navigation cache initialized ${initialized} shard(s).`), + (error) => console.error(`Neon navigation cache initialization failed: ${error?.code || error?.name || 'UNKNOWN'}`) + ); + } void worldTileGateway.ready().then( (ready) => console.log(`Worldwide gateway ready with ${ready.regions} regional shards.`), (error) => console.error('Worldwide gateway readiness failed:', error) From f8ca69beb3ef4cea8fb3c510cdcfe6bf9e237d25 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 11:53:47 -0700 Subject: [PATCH 69/85] Add Postgres client for Neon navigation cache --- package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index bca17185..ca36c531 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "check:export": "node scripts/check-export.mjs", "check:hardening": "node scripts/check-world-hardening.mjs", "check:foundation": "node scripts/check-continuous-foundation-lock.mjs", - "check:runtime": "node scripts/check-runtime.mjs && node scripts/validate-maplibre-style.mjs && node scripts/check-globe-parity.mjs && node scripts/check-cartography-parity.mjs && node scripts/check-viewer-quality.mjs && node scripts/check-photo-reference.mjs && node scripts/check-exact-exported-swatches.mjs && node scripts/check-render-clarity.mjs && node scripts/check-pmtiles-integration.mjs && node scripts/check-world-tile-gateway.mjs && npm run check:hardening && npm run check:foundation", + "check:neon-cache": "node scripts/check-neon-navigation-cache.mjs", + "check:runtime": "node scripts/check-runtime.mjs && node scripts/validate-maplibre-style.mjs && node scripts/check-globe-parity.mjs && node scripts/check-cartography-parity.mjs && node scripts/check-viewer-quality.mjs && node scripts/check-photo-reference.mjs && node scripts/check-exact-exported-swatches.mjs && node scripts/check-render-clarity.mjs && node scripts/check-pmtiles-integration.mjs && node scripts/check-world-tile-gateway.mjs && npm run check:neon-cache && npm run check:hardening && npm run check:foundation", "check:server": "node scripts/check-server-health.mjs", "check": "npm run check:export && npm run check:runtime", "dev": "npm run prepare:assets && vite", @@ -25,6 +26,7 @@ "maplibre-gl": "5.24.0", "pbf": "4.0.1", "pmtiles": "4.4.1", + "postgres": "3.4.9", "vt-pbf": "3.1.3" }, "devDependencies": { From 2efd88bc4e076b6ea3656cceb4694e11a32dae68 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 11:54:11 -0700 Subject: [PATCH 70/85] Add Neon navigation cache unit guard --- scripts/check-neon-navigation-cache.mjs | 66 +++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 scripts/check-neon-navigation-cache.mjs diff --git a/scripts/check-neon-navigation-cache.mjs b/scripts/check-neon-navigation-cache.mjs new file mode 100644 index 00000000..59b66130 --- /dev/null +++ b/scripts/check-neon-navigation-cache.mjs @@ -0,0 +1,66 @@ +import assert from 'node:assert/strict'; +import { + collectNavigationDatabaseUrls, + navigationTileShardIndex, + NeonNavigationTileCache +} from '../src/server/neon-navigation-tile-cache.js'; + +const urls = collectNavigationDatabaseUrls({ + NAV_DATABASE_URL_1: 'postgresql://one:secret@one.example/neondb?sslmode=require', + NAV_DATABASE_URL_2: 'not-a-database-url', + NAV_DATABASE_URL_3: 'postgresql://two:secret@two.example/neondb?sslmode=require', + NAV_DATABASE_URL_4: 'postgresql://one:secret@one.example/neondb?sslmode=require' +}); +assert.deepEqual(urls.map(({ slot }) => slot), [1, 3]); +assert.equal(navigationTileShardIndex('version/6/32/20', 8), navigationTileShardIndex('version/6/32/20', 8)); +assert.equal(navigationTileShardIndex('version/6/32/20', 0), -1); + +const calls = []; +const closed = []; +const cache = new NeonNavigationTileCache(urls, { + shardFactory: ({ slot }) => ({ + async initialize() { + calls.push(['initialize', slot]); + return true; + }, + async get(version, zoom, x, y) { + calls.push(['get', slot, version, zoom, x, y]); + return Buffer.from(`tile-${slot}`); + }, + async set(version, zoom, x, y, value) { + calls.push(['set', slot, version, zoom, x, y, Buffer.from(value).toString()]); + return true; + }, + async close() { + closed.push(slot); + }, + snapshot() { + return { slot, initialized: true }; + } + }) +}); + +assert.equal(await cache.initialize(), 2); +const tile = await cache.get('manifest-a', 6, 32, 20); +assert.match(tile.toString(), /^tile-(?:1|3)$/); +assert.equal(await cache.set('manifest-a', 6, 32, 20, Buffer.from('payload')), true); +const routedOperations = calls.filter(([operation]) => operation === 'get' || operation === 'set'); +assert.equal(routedOperations.length, 2); +assert.equal(routedOperations[0][1], routedOperations[1][1]); + +const beforeAboveMax = calls.length; +assert.equal(await cache.get('manifest-a', 7, 64, 40), null); +assert.equal(await cache.set('manifest-a', 7, 64, 40, Buffer.from('ignored')), false); +assert.equal(calls.length, beforeAboveMax); + +const snapshot = cache.snapshot(); +assert.equal(snapshot.enabled, true); +assert.equal(snapshot.configuredShards, 2); +assert.equal(snapshot.expectedShards, 8); +assert.equal(snapshot.maxZoom, 6); +assert.equal(JSON.stringify(snapshot).includes('secret'), false); + +await cache.close(); +assert.deepEqual(closed.sort((a, b) => a - b), [1, 3]); + +console.log('Neon navigation cache validated: deterministic sharding, z0-6 bounds, fail-safe configuration, and secret-free health output.'); From 74506e0b5446402fa634b7083d1407356b75bcd6 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 11:54:37 -0700 Subject: [PATCH 71/85] Document Neon navigation cache configuration --- .env.example | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.env.example b/.env.example index 0a45bfb8..5199d861 100644 --- a/.env.example +++ b/.env.example @@ -20,3 +20,20 @@ OCCUMED_WORLD_RELEASE_REPOSITORY=Occumed79/Map OCCUMED_WORLD_RELEASE_TAG=occumed-world-v1 OCCUMED_WORLD_MANIFEST_URL= OCCUMED_TILE_CACHE_MAX_BYTES=134217728 + +# Durable server-only cache for precomputed navigation tiles at zooms 0-6. +# Use eight independent Neon projects for independent storage allowances; +# child branches inside one Neon project share that project's storage quota. +NAV_DATABASE_URL_1= +NAV_DATABASE_URL_2= +NAV_DATABASE_URL_3= +NAV_DATABASE_URL_4= +NAV_DATABASE_URL_5= +NAV_DATABASE_URL_6= +NAV_DATABASE_URL_7= +NAV_DATABASE_URL_8= +OCCUMED_NAV_CACHE_MAX_ZOOM=6 +OCCUMED_NAV_CACHE_QUERY_TIMEOUT_MS=1500 +OCCUMED_NAV_CACHE_RETRY_DELAY_MS=30000 +OCCUMED_NAV_CACHE_MAX_BYTES_PER_SHARD=50331648 +OCCUMED_NAV_CACHE_PRUNE_EVERY_WRITES=64 From df204643cd9f3a465a45785b269ccc30b74bc77a Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 11:56:03 -0700 Subject: [PATCH 72/85] Verify memory-first Neon cache gateway flow --- scripts/check-neon-navigation-cache.mjs | 79 ++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/scripts/check-neon-navigation-cache.mjs b/scripts/check-neon-navigation-cache.mjs index 59b66130..90384841 100644 --- a/scripts/check-neon-navigation-cache.mjs +++ b/scripts/check-neon-navigation-cache.mjs @@ -4,6 +4,7 @@ import { navigationTileShardIndex, NeonNavigationTileCache } from '../src/server/neon-navigation-tile-cache.js'; +import { WorldTileGateway } from '../src/server/world-tile-gateway.js'; const urls = collectNavigationDatabaseUrls({ NAV_DATABASE_URL_1: 'postgresql://one:secret@one.example/neondb?sslmode=require', @@ -63,4 +64,80 @@ assert.equal(JSON.stringify(snapshot).includes('secret'), false); await cache.close(); assert.deepEqual(closed.sort((a, b) => a - b), [1, 3]); -console.log('Neon navigation cache validated: deterministic sharding, z0-6 bounds, fail-safe configuration, and secret-free health output.'); +const manifest = { + version: 2, + plannedRegionCount: 1, + availableRegionCount: 1, + missingRegionCount: 0, + virtualTiles: { + endpoint: '/tiles/{z}/{x}/{y}.pbf', + overviewAsset: 'occumed-world-overview.pmtiles', + surfaceAsset: 'occumed-world-surface.pmtiles', + overviewMaxZoom: 5, + surfaceMaxZoom: 10, + routingZoom: 6, + maxZoom: 16 + }, + regions: [ + { + id: 'test-region', + asset: 'occumed-test-region.pmtiles', + bounds: [-180, -90, 180, 90] + } + ] +}; +const persistentCalls = []; +const persistentCache = { + async initialize() { + persistentCalls.push(['initialize']); + return 1; + }, + async get(version, zoom, x, y) { + persistentCalls.push(['get', version, zoom, x, y]); + return zoom === 0 ? Buffer.from([0x1a, 0x00]) : null; + }, + async set(version, zoom, x, y, value) { + persistentCalls.push(['set', version, zoom, x, y, Buffer.from(value).toString('hex')]); + return true; + }, + async close() { + persistentCalls.push(['close']); + }, + snapshot() { + return { enabled: true, configuredShards: 1 }; + } +}; +const gateway = new WorldTileGateway({ + manifestUrl: 'https://example.test/world-virtual-manifest.json', + releaseAssetUrl: (asset) => `https://example.test/${asset}`, + persistentTileCache, + fetchImpl: async () => new Response(JSON.stringify(manifest), { + status: 200, + headers: { 'content-type': 'application/json' } + }) +}); +let builds = 0; +gateway.buildTile = async () => { + builds += 1; + return Buffer.from([0x1a, 0x01, 0x00]); +}; + +assert.equal(await gateway.initializePersistentCache(), 1); +const persistentHit = await gateway.resolveTile(0, 0, 0); +assert.equal(persistentHit.toString('hex'), '1a00'); +assert.equal(builds, 0); +const persistentGetsAfterFirst = persistentCalls.filter(([operation]) => operation === 'get').length; +const memoryHit = await gateway.resolveTile(0, 0, 0); +assert.equal(memoryHit.toString('hex'), '1a00'); +assert.equal(persistentCalls.filter(([operation]) => operation === 'get').length, persistentGetsAfterFirst); + +const builtTile = await gateway.resolveTile(1, 0, 0); +assert.equal(builtTile.toString('hex'), '1a0100'); +assert.equal(builds, 1); +await new Promise((resolve) => setImmediate(resolve)); +assert.equal(persistentCalls.some(([operation]) => operation === 'set'), true); +assert.equal(gateway.getHealthSnapshot().persistentCache.enabled, true); +await gateway.close(); +assert.equal(persistentCalls.some(([operation]) => operation === 'close'), true); + +console.log('Neon navigation cache validated: deterministic sharding, memory-first reads, z0-6 persistence, bounded fallback behavior, and secret-free health output.'); From 51842f8465bd2455000cebccc82cf83dccc14268 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 11:56:23 -0700 Subject: [PATCH 73/85] Run focused Neon cache finalization --- .github/workflows/finalize-neon-cache.yml | 42 +++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/finalize-neon-cache.yml diff --git a/.github/workflows/finalize-neon-cache.yml b/.github/workflows/finalize-neon-cache.yml new file mode 100644 index 00000000..79a66116 --- /dev/null +++ b/.github/workflows/finalize-neon-cache.yml @@ -0,0 +1,42 @@ +name: Finalize Neon navigation cache + +on: + push: + branches: + - fix/neon-navigation-cache + +permissions: + contents: write + +jobs: + finalize: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + ref: fix/neon-navigation-cache + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version: '24' + - name: Generate exact lockfile + run: npm install --package-lock-only --ignore-scripts --no-audit --no-fund + - name: Verify locked install + run: npm ci --ignore-scripts --no-audit --no-fund + - name: Parse changed runtime files + run: | + node --check server.mjs + node --check src/server/world-tile-gateway.js + node --check src/server/neon-navigation-tile-cache.js + node --check scripts/check-neon-navigation-cache.mjs + - name: Run focused Neon cache guard + run: npm run check:neon-cache + - name: Commit generated lockfile and remove temporary scaffolding + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add package-lock.json + git rm .github/workflows/finalize-neon-cache.yml tmp-neon-branch-anchor.txt + git commit -m "Finalize Neon navigation cache integration" + git push origin HEAD:fix/neon-navigation-cache From be74321aa254142a576fae22fc0617daf0c753af Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 12:03:40 -0700 Subject: [PATCH 74/85] Use Neon HTTP SQL for durable navigation cache --- src/server/neon-navigation-tile-cache.js | 247 ++++++++++++++--------- 1 file changed, 153 insertions(+), 94 deletions(-) diff --git a/src/server/neon-navigation-tile-cache.js b/src/server/neon-navigation-tile-cache.js index 4d6ff708..c5931719 100644 --- a/src/server/neon-navigation-tile-cache.js +++ b/src/server/neon-navigation-tile-cache.js @@ -1,5 +1,3 @@ -import postgres from 'postgres'; - const DEFAULT_MAX_ZOOM = 6; const DEFAULT_QUERY_TIMEOUT_MS = 1_500; const DEFAULT_RETRY_DELAY_MS = 30_000; @@ -28,6 +26,15 @@ function safeDatabaseUrl(value) { } } +export function neonHttpSqlEndpoint(connectionString) { + const database = new URL(connectionString); + const endpointHost = database.hostname.replace(/^[^.]+\./, 'api.'); + if (endpointHost === database.hostname) { + throw new TypeError('The Neon database hostname is not compatible with the HTTP SQL endpoint.'); + } + return `https://${endpointHost}/sql`; +} + export function collectNavigationDatabaseUrls(env = process.env) { const seen = new Set(); const entries = []; @@ -50,22 +57,92 @@ export function navigationTileShardIndex(key, shardCount) { return (hash >>> 0) % shardCount; } -function timeoutAfter(milliseconds, label) { - return new Promise((_, reject) => { +function encodeHttpParameter(value) { + if (Buffer.isBuffer(value) || value instanceof Uint8Array) { + return `\\x${Buffer.from(value).toString('hex')}`; + } + if (typeof value === 'bigint') return value.toString(); + if (value instanceof Date) return value.toISOString(); + return value === undefined ? null : value; +} + +class NeonHttpQueryClient { + constructor(connectionString, { + timeoutMs, + fetchImpl = fetch + }) { + this.connectionString = connectionString; + this.endpoint = neonHttpSqlEndpoint(connectionString); + this.timeoutMs = timeoutMs; + this.fetchImpl = fetchImpl; + } + + async query(query, params = [], timeoutMs = this.timeoutMs) { + const controller = new AbortController(); const timer = setTimeout(() => { - const error = new Error(`${label} timed out.`); - error.code = 'OCCUMED_NAV_CACHE_TIMEOUT'; - reject(error); - }, milliseconds); + controller.abort(new Error('Neon HTTP SQL query timed out.')); + }, timeoutMs); timer.unref?.(); - }); -} -async function withinTimeout(task, milliseconds, label) { - return Promise.race([task, timeoutAfter(milliseconds, label)]); + let response; + try { + response = await this.fetchImpl(this.endpoint, { + method: 'POST', + redirect: 'follow', + signal: controller.signal, + headers: { + 'Content-Type': 'application/json', + 'Neon-Connection-String': this.connectionString, + 'Neon-Raw-Text-Output': 'true', + 'Neon-Array-Mode': 'true', + 'User-Agent': 'Occu-Med-Map/navigation-cache' + }, + body: JSON.stringify({ + query, + params: params.map(encodeHttpParameter) + }) + }); + } catch (error) { + if (controller.signal.aborted) { + const timeoutError = new Error('Neon HTTP SQL query timed out.', { cause: error }); + timeoutError.code = 'OCCUMED_NAV_CACHE_TIMEOUT'; + throw timeoutError; + } + const connectionError = new Error('Unable to reach the Neon HTTP SQL endpoint.', { cause: error }); + connectionError.code = error?.code || 'OCCUMED_NAV_CACHE_CONNECTION_FAILED'; + throw connectionError; + } finally { + clearTimeout(timer); + } + + let document; + try { + document = await response.json(); + } catch (error) { + const responseError = new Error(`Neon HTTP SQL returned an unreadable HTTP ${response.status} response.`, { + cause: error + }); + responseError.code = 'OCCUMED_NAV_CACHE_INVALID_RESPONSE'; + throw responseError; + } + + if (!response.ok) { + const queryError = new Error(document?.message || `Neon HTTP SQL returned HTTP ${response.status}.`); + queryError.code = document?.code || `OCCUMED_NAV_CACHE_HTTP_${response.status}`; + throw queryError; + } + + const rows = Array.isArray(document?.rows) ? document.rows : []; + if (!rows.length || !Array.isArray(rows[0])) return rows; + const fields = Array.isArray(document?.fields) ? document.fields : []; + const names = fields.map((field, index) => String(field?.name || `column_${index}`)); + return rows.map((row) => Object.fromEntries( + row.map((value, index) => [names[index] || `column_${index}`, value]) + )); + } } -class PostgresNavigationShard { +class NeonHttpNavigationShard { constructor({ slot, url, @@ -74,7 +151,8 @@ class PostgresNavigationShard { maxBytes, pruneEveryWrites, now, - logger + logger, + fetchImpl }) { this.slot = slot; this.queryTimeoutMs = queryTimeoutMs; @@ -83,6 +161,7 @@ class PostgresNavigationShard { this.pruneEveryWrites = pruneEveryWrites; this.now = now; this.logger = logger; + this.client = new NeonHttpQueryClient(url, { timeoutMs: queryTimeoutMs, fetchImpl }); this.initialization = null; this.initialized = false; this.disabledUntil = 0; @@ -95,14 +174,6 @@ class PostgresNavigationShard { errors: 0, prunes: 0 }; - this.sql = postgres(url, { - max: 1, - prepare: false, - connect_timeout: 5, - idle_timeout: 20, - max_lifetime: 60 * 30, - onnotice: () => {} - }); } available() { @@ -127,8 +198,8 @@ class PostgresNavigationShard { if (!this.available()) return false; if (this.initialization) return this.initialization; - this.initialization = withinTimeout( - this.sql.unsafe(` + this.initialization = Promise.all([ + this.client.query(` CREATE TABLE IF NOT EXISTS ${TABLE_NAME} ( tileset_version text NOT NULL, z smallint NOT NULL CHECK (z BETWEEN 0 AND ${DEFAULT_MAX_ZOOM}), @@ -138,13 +209,13 @@ class PostgresNavigationShard { byte_length integer NOT NULL CHECK (byte_length = octet_length(tile)), created_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (tileset_version, z, x, y) - ); + ) + `, [], this.queryTimeoutMs * 4), + this.client.query(` CREATE INDEX IF NOT EXISTS ${TABLE_NAME}_created_at_idx - ON ${TABLE_NAME} (created_at); - `), - this.queryTimeoutMs * 4, - `Navigation cache shard ${this.slot} initialization` - ) + ON ${TABLE_NAME} (created_at) + `, [], this.queryTimeoutMs * 4) + ]) .then(() => { this.initialized = true; this.disabledUntil = 0; @@ -165,26 +236,27 @@ class PostgresNavigationShard { async get(tilesetVersion, zoom, x, y) { if (!(await this.initialize())) return null; try { - const rows = await withinTimeout( - this.sql` - SELECT tile - FROM ${this.sql(TABLE_NAME)} - WHERE tileset_version = ${tilesetVersion} - AND z = ${zoom} - AND x = ${x} - AND y = ${y} - LIMIT 1 - `, - this.queryTimeoutMs, - `Navigation cache shard ${this.slot} read` - ); - const tile = rows?.[0]?.tile; - if (!tile) { + const rows = await this.client.query(` + SELECT encode(tile, 'base64') AS tile_base64 + FROM ${TABLE_NAME} + WHERE tileset_version = $1 + AND z = $2 + AND x = $3 + AND y = $4 + LIMIT 1 + `, [tilesetVersion, zoom, x, y]); + const encoded = rows?.[0]?.tile_base64; + if (!encoded) { + this.metrics.misses += 1; + return null; + } + const tile = Buffer.from(String(encoded), 'base64'); + if (!tile.byteLength) { this.metrics.misses += 1; return null; } this.metrics.hits += 1; - return Buffer.from(tile); + return tile; } catch (error) { this.recordError(error, 'read'); return null; @@ -195,22 +267,16 @@ class PostgresNavigationShard { if (!(await this.initialize())) return false; const tile = Buffer.from(value); try { - await withinTimeout( - this.sql` - INSERT INTO ${this.sql(TABLE_NAME)} ( - tileset_version, z, x, y, tile, byte_length, created_at - ) VALUES ( - ${tilesetVersion}, ${zoom}, ${x}, ${y}, ${tile}, ${tile.byteLength}, now() - ) - ON CONFLICT (tileset_version, z, x, y) - DO UPDATE SET - tile = EXCLUDED.tile, - byte_length = EXCLUDED.byte_length, - created_at = now() - `, - this.queryTimeoutMs, - `Navigation cache shard ${this.slot} write` - ); + await this.client.query(` + INSERT INTO ${TABLE_NAME} ( + tileset_version, z, x, y, tile, byte_length, created_at + ) VALUES ($1, $2, $3, $4, $5, $6, now()) + ON CONFLICT (tileset_version, z, x, y) + DO UPDATE SET + tile = EXCLUDED.tile, + byte_length = EXCLUDED.byte_length, + created_at = now() + `, [tilesetVersion, zoom, x, y, tile, tile.byteLength]); this.metrics.writes += 1; this.writeCount += 1; this.disabledUntil = 0; @@ -228,31 +294,25 @@ class PostgresNavigationShard { async prune(tilesetVersion) { if (!this.initialized || !this.available()) return false; try { - await withinTimeout( - this.sql.begin(async (transaction) => { - await transaction` - DELETE FROM ${transaction(TABLE_NAME)} - WHERE tileset_version <> ${tilesetVersion} - `; - await transaction.unsafe(` - WITH ranked AS ( - SELECT - ctid, - sum(byte_length) OVER ( - ORDER BY created_at DESC, z DESC, x DESC, y DESC - ) AS running_bytes - FROM ${TABLE_NAME} - WHERE tileset_version = $1 - ) - DELETE FROM ${TABLE_NAME} AS cache - USING ranked - WHERE cache.ctid = ranked.ctid - AND ranked.running_bytes > $2 - `, [tilesetVersion, this.maxBytes]); - }), - this.queryTimeoutMs * 4, - `Navigation cache shard ${this.slot} prune` - ); + await this.client.query(` + DELETE FROM ${TABLE_NAME} + WHERE tileset_version <> $1 + `, [tilesetVersion], this.queryTimeoutMs * 4); + await this.client.query(` + WITH ranked AS ( + SELECT + ctid, + sum(byte_length) OVER ( + ORDER BY created_at DESC, z DESC, x DESC, y DESC + ) AS running_bytes + FROM ${TABLE_NAME} + WHERE tileset_version = $1 + ) + DELETE FROM ${TABLE_NAME} AS cache + USING ranked + WHERE cache.ctid = ranked.ctid + AND ranked.running_bytes > $2 + `, [tilesetVersion, this.maxBytes], this.queryTimeoutMs * 4); this.metrics.prunes += 1; return true; } catch (error) { @@ -261,9 +321,7 @@ class PostgresNavigationShard { } } - async close() { - await this.sql.end({ timeout: 2 }).catch(() => {}); - } + async close() {} snapshot() { return { @@ -287,7 +345,8 @@ export class NeonNavigationTileCache { pruneEveryWrites = DEFAULT_PRUNE_EVERY_WRITES, now = () => Date.now(), logger = console, - shardFactory = (options) => new PostgresNavigationShard(options) + fetchImpl = fetch, + shardFactory = (options) => new NeonHttpNavigationShard(options) } = {}) { this.maxZoom = boundedInteger(maxZoom, DEFAULT_MAX_ZOOM, 0, DEFAULT_MAX_ZOOM); this.queryTimeoutMs = boundedInteger(queryTimeoutMs, DEFAULT_QUERY_TIMEOUT_MS, 100, 30_000); @@ -313,7 +372,8 @@ export class NeonNavigationTileCache { maxBytes: this.maxBytesPerShard, pruneEveryWrites: this.pruneEveryWrites, now, - logger + logger, + fetchImpl })); } @@ -331,8 +391,7 @@ export class NeonNavigationTileCache { } async initialize() { - const results = []; - for (const shard of this.shards) results.push(await shard.initialize()); + const results = await Promise.all(this.shards.map((shard) => shard.initialize())); return results.filter(Boolean).length; } From 6d1b43a3c7dcc194b3b3b2cd457b86a79a2e3feb Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 12:04:04 -0700 Subject: [PATCH 75/85] Keep Neon cache dependency-free --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index ca36c531..4105687b 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ "maplibre-gl": "5.24.0", "pbf": "4.0.1", "pmtiles": "4.4.1", - "postgres": "3.4.9", "vt-pbf": "3.1.3" }, "devDependencies": { From 94ed9831f8eb2ff426a39007be9a909f4f24fefb Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 12:04:43 -0700 Subject: [PATCH 76/85] Remove temporary Neon finalization workflow --- .github/workflows/finalize-neon-cache.yml | 42 ----------------------- 1 file changed, 42 deletions(-) delete mode 100644 .github/workflows/finalize-neon-cache.yml diff --git a/.github/workflows/finalize-neon-cache.yml b/.github/workflows/finalize-neon-cache.yml deleted file mode 100644 index 79a66116..00000000 --- a/.github/workflows/finalize-neon-cache.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Finalize Neon navigation cache - -on: - push: - branches: - - fix/neon-navigation-cache - -permissions: - contents: write - -jobs: - finalize: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - with: - ref: fix/neon-navigation-cache - fetch-depth: 0 - - uses: actions/setup-node@v4 - with: - node-version: '24' - - name: Generate exact lockfile - run: npm install --package-lock-only --ignore-scripts --no-audit --no-fund - - name: Verify locked install - run: npm ci --ignore-scripts --no-audit --no-fund - - name: Parse changed runtime files - run: | - node --check server.mjs - node --check src/server/world-tile-gateway.js - node --check src/server/neon-navigation-tile-cache.js - node --check scripts/check-neon-navigation-cache.mjs - - name: Run focused Neon cache guard - run: npm run check:neon-cache - - name: Commit generated lockfile and remove temporary scaffolding - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add package-lock.json - git rm .github/workflows/finalize-neon-cache.yml tmp-neon-branch-anchor.txt - git commit -m "Finalize Neon navigation cache integration" - git push origin HEAD:fix/neon-navigation-cache From 27475bc6f3e74d340f129d4ca2e6a6bac575c565 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 12:04:52 -0700 Subject: [PATCH 77/85] Remove temporary Neon integration anchor --- tmp-neon-branch-anchor.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tmp-neon-branch-anchor.txt diff --git a/tmp-neon-branch-anchor.txt b/tmp-neon-branch-anchor.txt deleted file mode 100644 index 6ad1ea22..00000000 --- a/tmp-neon-branch-anchor.txt +++ /dev/null @@ -1 +0,0 @@ -temporary integration anchor From 59f5ee12f8033d078b07d05ab27d7359cdbee6e9 Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 12:05:42 -0700 Subject: [PATCH 78/85] Test Neon HTTP cache transport and gateway flow --- scripts/check-neon-navigation-cache.mjs | 37 ++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/scripts/check-neon-navigation-cache.mjs b/scripts/check-neon-navigation-cache.mjs index 90384841..fdb90ebe 100644 --- a/scripts/check-neon-navigation-cache.mjs +++ b/scripts/check-neon-navigation-cache.mjs @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { collectNavigationDatabaseUrls, + neonHttpSqlEndpoint, navigationTileShardIndex, NeonNavigationTileCache } from '../src/server/neon-navigation-tile-cache.js'; @@ -13,6 +14,10 @@ const urls = collectNavigationDatabaseUrls({ NAV_DATABASE_URL_4: 'postgresql://one:secret@one.example/neondb?sslmode=require' }); assert.deepEqual(urls.map(({ slot }) => slot), [1, 3]); +assert.equal( + neonHttpSqlEndpoint('postgresql://owner:secret@ep-example-pooler.us-west-2.aws.neon.tech/neondb'), + 'https://api.us-west-2.aws.neon.tech/sql' +); assert.equal(navigationTileShardIndex('version/6/32/20', 8), navigationTileShardIndex('version/6/32/20', 8)); assert.equal(navigationTileShardIndex('version/6/32/20', 0), -1); @@ -64,6 +69,36 @@ assert.equal(JSON.stringify(snapshot).includes('secret'), false); await cache.close(); assert.deepEqual(closed.sort((a, b) => a - b), [1, 3]); +const httpRequests = []; +const httpConnectionString = 'postgresql://owner:secret@ep-example-pooler.us-west-2.aws.neon.tech/neondb?sslmode=require'; +const httpCache = new NeonNavigationTileCache([ + { slot: 1, url: httpConnectionString } +], { + fetchImpl: async (url, options) => { + const body = JSON.parse(options.body); + httpRequests.push({ url, options, body }); + const isRead = /SELECT encode\(tile, 'base64'\)/.test(body.query); + return new Response(JSON.stringify(isRead + ? { + fields: [{ name: 'tile_base64' }], + rows: [[Buffer.from([0x1a, 0x00]).toString('base64')]] + } + : { fields: [], rows: [] }), { + status: 200, + headers: { 'content-type': 'application/json' } + }); + } +}); +assert.equal(await httpCache.initialize(), 1); +assert.equal((await httpCache.get('manifest-http', 0, 0, 0)).toString('hex'), '1a00'); +assert.equal(await httpCache.set('manifest-http', 0, 0, 0, Buffer.from([0x1a, 0x00])), true); +assert.equal(httpRequests.every(({ url }) => url === 'https://api.us-west-2.aws.neon.tech/sql'), true); +assert.equal(httpRequests.every(({ options }) => options.headers['Neon-Connection-String'] === httpConnectionString), true); +const insertRequest = httpRequests.find(({ body }) => /INSERT INTO/.test(body.query)); +assert.equal(insertRequest.body.params[4], '\\x1a00'); +assert.equal(JSON.stringify(httpCache.snapshot()).includes('secret'), false); +await httpCache.close(); + const manifest = { version: 2, plannedRegionCount: 1, @@ -140,4 +175,4 @@ assert.equal(gateway.getHealthSnapshot().persistentCache.enabled, true); await gateway.close(); assert.equal(persistentCalls.some(([operation]) => operation === 'close'), true); -console.log('Neon navigation cache validated: deterministic sharding, memory-first reads, z0-6 persistence, bounded fallback behavior, and secret-free health output.'); +console.log('Neon navigation cache validated: HTTP SQL transport, deterministic sharding, memory-first reads, z0-6 persistence, bounded fallback behavior, and secret-free health output.'); From 40f266bd074b90d5d183c852a4009cbde642b74a Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 12:09:32 -0700 Subject: [PATCH 79/85] Cap shared-project Neon cache storage safely --- src/server/neon-navigation-tile-cache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/neon-navigation-tile-cache.js b/src/server/neon-navigation-tile-cache.js index c5931719..7294db03 100644 --- a/src/server/neon-navigation-tile-cache.js +++ b/src/server/neon-navigation-tile-cache.js @@ -1,7 +1,7 @@ const DEFAULT_MAX_ZOOM = 6; const DEFAULT_QUERY_TIMEOUT_MS = 1_500; const DEFAULT_RETRY_DELAY_MS = 30_000; -const DEFAULT_MAX_BYTES_PER_SHARD = 48 * 1024 * 1024; +const DEFAULT_MAX_BYTES_PER_SHARD = 6 * 1024 * 1024; const DEFAULT_PRUNE_EVERY_WRITES = 64; const MAX_DATABASE_SHARDS = 8; const TABLE_NAME = 'occumed_navigation_tile_cache'; From a7ee9b2657d26068010d2c5cd52b52c6d10c5ffa Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 12:09:50 -0700 Subject: [PATCH 80/85] Document safe shared-project Neon cache limit --- .env.example | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 5199d861..7639a9c5 100644 --- a/.env.example +++ b/.env.example @@ -22,8 +22,9 @@ OCCUMED_WORLD_MANIFEST_URL= OCCUMED_TILE_CACHE_MAX_BYTES=134217728 # Durable server-only cache for precomputed navigation tiles at zooms 0-6. -# Use eight independent Neon projects for independent storage allowances; -# child branches inside one Neon project share that project's storage quota. +# The current eight child branches share one Neon project quota, so the safe +# default is capped at 6 MiB per branch / 48 MiB total. Eight independent Neon +# projects can raise this value later without changing the application code. NAV_DATABASE_URL_1= NAV_DATABASE_URL_2= NAV_DATABASE_URL_3= @@ -35,5 +36,5 @@ NAV_DATABASE_URL_8= OCCUMED_NAV_CACHE_MAX_ZOOM=6 OCCUMED_NAV_CACHE_QUERY_TIMEOUT_MS=1500 OCCUMED_NAV_CACHE_RETRY_DELAY_MS=30000 -OCCUMED_NAV_CACHE_MAX_BYTES_PER_SHARD=50331648 +OCCUMED_NAV_CACHE_MAX_BYTES_PER_SHARD=6291456 OCCUMED_NAV_CACHE_PRUNE_EVERY_WRITES=64 From 7ed506e900dcb4fc9cd1cadbf364fa112a3a014f Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 12:12:29 -0700 Subject: [PATCH 81/85] Fix Neon cache gateway guard variable --- scripts/check-neon-navigation-cache.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check-neon-navigation-cache.mjs b/scripts/check-neon-navigation-cache.mjs index fdb90ebe..26a08d07 100644 --- a/scripts/check-neon-navigation-cache.mjs +++ b/scripts/check-neon-navigation-cache.mjs @@ -145,7 +145,7 @@ const persistentCache = { const gateway = new WorldTileGateway({ manifestUrl: 'https://example.test/world-virtual-manifest.json', releaseAssetUrl: (asset) => `https://example.test/${asset}`, - persistentTileCache, + persistentTileCache: persistentCache, fetchImpl: async () => new Response(JSON.stringify(manifest), { status: 200, headers: { 'content-type': 'application/json' } From fbb0d119acd1073644cb963e74a360d12a0c68dc Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 14:32:29 -0700 Subject: [PATCH 82/85] Restore independent Neon project cache capacity --- scripts/start-localized-world.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/start-localized-world.mjs b/scripts/start-localized-world.mjs index b9742f2b..02cba974 100644 --- a/scripts/start-localized-world.mjs +++ b/scripts/start-localized-world.mjs @@ -14,6 +14,7 @@ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const assetDir = path.join(root, 'dist', 'virtual-assets'); const maxAssetBytes = Number(process.env.OCCUMED_NAVIGATION_ASSET_MAX_BYTES || 512 * 1024 * 1024); const required = process.env.OCCUMED_REQUIRE_LOCAL_NAVIGATION_ASSETS !== 'false'; +const defaultNavigationCacheBytesPerProject = 48 * 1024 * 1024; const assets = [ { @@ -106,6 +107,7 @@ try { } const env = { ...process.env }; +env.OCCUMED_NAV_CACHE_MAX_BYTES_PER_SHARD ||= String(defaultNavigationCacheBytesPerProject); if (localized.length === assets.length) { const localOrigin = `http://127.0.0.1:${port}/virtual-assets`; env.OCCUMED_WORLD_OVERVIEW_URL = `${localOrigin}/${assets[0].name}`; From a4f45ef3c6e3e7e4cce07a678192f138f980d94c Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 14:32:42 -0700 Subject: [PATCH 83/85] Document independent Neon project cache capacity --- .env.example | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 7639a9c5..a41faca8 100644 --- a/.env.example +++ b/.env.example @@ -22,9 +22,9 @@ OCCUMED_WORLD_MANIFEST_URL= OCCUMED_TILE_CACHE_MAX_BYTES=134217728 # Durable server-only cache for precomputed navigation tiles at zooms 0-6. -# The current eight child branches share one Neon project quota, so the safe -# default is capped at 6 MiB per branch / 48 MiB total. Eight independent Neon -# projects can raise this value later without changing the application code. +# Each NAV_DATABASE_URL must point to a separate Neon project so every cache +# shard has its own project storage allowance. The default retains up to 48 MiB +# per project, or approximately 384 MiB across all eight configured projects. NAV_DATABASE_URL_1= NAV_DATABASE_URL_2= NAV_DATABASE_URL_3= @@ -36,5 +36,5 @@ NAV_DATABASE_URL_8= OCCUMED_NAV_CACHE_MAX_ZOOM=6 OCCUMED_NAV_CACHE_QUERY_TIMEOUT_MS=1500 OCCUMED_NAV_CACHE_RETRY_DELAY_MS=30000 -OCCUMED_NAV_CACHE_MAX_BYTES_PER_SHARD=6291456 +OCCUMED_NAV_CACHE_MAX_BYTES_PER_SHARD=50331648 OCCUMED_NAV_CACHE_PRUNE_EVERY_WRITES=64 From df0efe46ae95a93f19b7ecf675d537c7b7f7d7ba Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 14:47:48 -0700 Subject: [PATCH 84/85] Update continuity lock for lightweight validation sampling --- scripts/check-continuous-foundation-lock.mjs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/scripts/check-continuous-foundation-lock.mjs b/scripts/check-continuous-foundation-lock.mjs index 8e5e4d4b..9dc70b5f 100644 --- a/scripts/check-continuous-foundation-lock.mjs +++ b/scripts/check-continuous-foundation-lock.mjs @@ -157,12 +157,19 @@ for (const marker of [ 'pacific-routing-threshold-in', 'antimeridian-pan', 'missingFoundationSampleCount', - 'setInterval(sample, 50)', + 'setInterval(queueSample, 50)', + 'requestAnimationFrame(() =>', + 'await waitForRequiredFoundation()', + 'postMoveendSettleMs', "event.sourceDataType === 'idle'", "error === 'net::ERR_ABORTED'" ]) { assert(motionGate.includes(marker), `The continuous motion gate lost ${marker}.`); } +assert( + !motionGate.includes('map.queryRenderedFeatures()'), + 'The continuous motion gate reintroduced unfiltered full-viewport feature enumeration.' +); for (const marker of [ 'amazon-all-zooms-in', 'amazon-all-zooms-out', @@ -171,13 +178,20 @@ for (const marker of [ 'antimeridian-all-zooms-out', 'startZoom: 0, endZoom: 16', 'startZoom: 16, endZoom: 0', - 'setInterval(sample, 50)', + 'setInterval(queueSample, 50)', + 'requestAnimationFrame(() =>', + 'await waitForRequiredFoundation()', + 'postMoveendSettleMs', "event.sourceDataType === 'idle'", 'actualStartZoom', 'actualEndZoom' ]) { assert(allZoomGate.includes(marker), `The complete zoom-range gate lost ${marker}.`); } +assert( + !allZoomGate.includes('map.queryRenderedFeatures()'), + 'The complete zoom-range gate reintroduced unfiltered full-viewport feature enumeration.' +); for (const marker of [ 'const waves = 3', @@ -195,5 +209,5 @@ assert(workflow.includes('continuous-motion/*.json'), 'Runtime JSON diagnostics assert(workflow.includes('gate-status.txt'), 'Aggregate runtime gate status is no longer preserved.'); console.log( - 'Continuous-foundation lock passed: documented source/layer zoom semantics, explicit zoom-0 bathymetry, nonzero landcover and depth through zoom 16, full 0–16 parent-tile retention, source-idle stabilization, 50ms motion sampling, strengthened atmosphere, exhaustive boundary checks, and sustained worldwide soak are mandatory.' + 'Continuous-foundation lock passed: documented source/layer zoom semantics, explicit zoom-0 bathymetry, nonzero landcover and depth through zoom 16, full 0–16 parent-tile retention, source-idle stabilization, non-overlapping 50ms motion sampling, required-layer settlement after moveend, strengthened atmosphere, exhaustive boundary checks, and sustained worldwide soak are mandatory.' ); From 71aa755f80304853b8aece29d15a766076d68aab Mon Sep 17 00:00:00 2001 From: Occumed79 Date: Tue, 28 Jul 2026 17:04:18 -0700 Subject: [PATCH 85/85] Fix continuous base surface retention --- scripts/check-continuous-foundation-lock.mjs | 32 ++++++- scripts/validate-all-zoom-levels.mjs | 82 +++++++++++------- scripts/validate-continuous-zoom.mjs | 82 +++++++++++------- src/occumed-map.js | 91 ++++++++++++++++++++ src/server/mvt.js | 45 ++++++++++ src/server/world-tile-gateway.js | 43 ++++++--- 6 files changed, 300 insertions(+), 75 deletions(-) diff --git a/scripts/check-continuous-foundation-lock.mjs b/scripts/check-continuous-foundation-lock.mjs index 9dc70b5f..0b280f23 100644 --- a/scripts/check-continuous-foundation-lock.mjs +++ b/scripts/check-continuous-foundation-lock.mjs @@ -17,6 +17,8 @@ const [ workflow, packageDocument, renderingContract, + tileGateway, + mvtHelpers, runtimeStyleDocument ] = await Promise.all([ fs.readFile(path.join(root, 'scripts/prepare-world-landcover.mjs'), 'utf8'), @@ -31,6 +33,8 @@ const [ fs.readFile(path.join(root, '.github/workflows/validate-continuous-zoom.yml'), 'utf8'), fs.readFile(path.join(root, 'package.json'), 'utf8'), fs.readFile(path.join(root, 'scripts/apply-mapbox-rendering-contract.mjs'), 'utf8'), + fs.readFile(path.join(root, 'src/server/world-tile-gateway.js'), 'utf8'), + fs.readFile(path.join(root, 'src/server/mvt.js'), 'utf8'), fs.readFile(path.join(root, 'public/style/occumed-open.json'), 'utf8') ]); @@ -62,6 +66,18 @@ assert( surfaceBuilder.includes('-L "depth:'), 'The physical surface build no longer publishes landcover and depth from zoom 0 through zoom 10.' ); +for (const marker of [ + 'copyVectorLayer', + "sourceLayerName: 'land'", + "targetLayerName: 'landcover'", + 'LANDCOVER_FALLBACK_PROPERTIES' +]) { + assert(tileGateway.includes(marker), `The gateway landcover fallback lost ${marker}.`); +} +assert( + mvtHelpers.includes('copied MVT output for'), + 'Synthesized foundation layers are no longer validated after encoding.' +); const prepareStyle = packageJson.scripts?.['prepare:style'] || ''; const contractIndex = prepareStyle.indexOf('apply-mapbox-rendering-contract.mjs'); @@ -129,6 +145,12 @@ for (const layer of depthLayers) { for (const marker of [ 'installOccumedAtmosphereBloom(map)', + 'installContinuousTileRetention(map)', + 'maxUnderzooming', + 'maxOverzooming', + 'globalFoundationID', + 'idealTileIDs[0].scaledTo(0)', + 'retained[globalFoundationID.key] = globalFoundationID', 'resolveGlobeRadius', 'BLOOM_FADE_START_ZOOM', 'BLOOM_FADE_END_ZOOM', @@ -157,8 +179,11 @@ for (const marker of [ 'pacific-routing-threshold-in', 'antimeridian-pan', 'missingFoundationSampleCount', + 'setPixelRatio(1)', 'setInterval(queueSample, 50)', - 'requestAnimationFrame(() =>', + "map.on('move', queueSample)", + 'if (sampling) return', + 'tileManager.getRenderableIds()', 'await waitForRequiredFoundation()', 'postMoveendSettleMs', "event.sourceDataType === 'idle'", @@ -178,8 +203,11 @@ for (const marker of [ 'antimeridian-all-zooms-out', 'startZoom: 0, endZoom: 16', 'startZoom: 16, endZoom: 0', + 'setPixelRatio(1)', 'setInterval(queueSample, 50)', - 'requestAnimationFrame(() =>', + "map.on('move', queueSample)", + 'if (sampling) return', + 'tileManager.getRenderableIds()', 'await waitForRequiredFoundation()', 'postMoveendSettleMs', "event.sourceDataType === 'idle'", diff --git a/scripts/validate-all-zoom-levels.mjs b/scripts/validate-all-zoom-levels.mjs index afdeca2b..9a21fdd2 100644 --- a/scripts/validate-all-zoom-levels.mjs +++ b/scripts/validate-all-zoom-levels.mjs @@ -67,7 +67,7 @@ function normalizeSweep(result, definition) { blankSampleCount: Number(result?.blankSampleCount || 0), missingFoundationSampleCount: Number(result?.missingFoundationSampleCount || 0), firstMissingFoundationSamples: Array.isArray(result?.firstMissingFoundationSamples) - ? result.firstMissingFoundationSamples + ? result.firstMissingFoundationSamples.map((sample) => structuredClone(sample)) : [], minimumZoom: result?.minimumZoom === null || result?.minimumZoom === undefined ? null @@ -121,6 +121,12 @@ try { null, { timeout: 90_000 } ); + await page.evaluate(() => { + // This validator measures temporal foundation continuity, not high-DPI + // sharpness. Keep the full CSS viewport while avoiding a 5.76M-pixel + // software-rendered canvas that starves the 50ms sampling timer. + globalThis.__OCCUMED_MAP__.setPixelRatio(1); + }); async function positionAndWait(center, zoom, requiredLayers) { return await page.evaluate(async ({ center, zoom, requiredLayers }) => { @@ -137,16 +143,24 @@ try { function requiredLayerCounts() { const counts = Object.fromEntries(requiredLayers.map((layer) => [layer, 0])); - const layerIds = (map.getStyle().layers || []) - .filter((layer) => - layer.source === 'occumed-open' && - requiredLayers.includes(layer['source-layer']) - ) - .map((layer) => layer.id); - if (layerIds.length === 0) return counts; - for (const feature of map.queryRenderedFeatures({ layers: layerIds })) { - const sourceLayer = feature.sourceLayer; - if (sourceLayer in counts) counts[sourceLayer] += 1; + const layerIds = Object.fromEntries(requiredLayers.map((sourceLayer) => [ + sourceLayer, + (map.getStyle().layers || []) + .filter((layer) => + layer.source === 'occumed-open' && + layer['source-layer'] === sourceLayer + ) + .map((layer) => layer.id) + ])); + const tileManager = map.style?.tileManagers?.['occumed-open']; + if (!tileManager) return counts; + for (const id of tileManager.getRenderableIds()) { + const tile = tileManager.getTileByID(id); + for (const sourceLayer of requiredLayers) { + if (layerIds[sourceLayer].some((layerId) => tile?.buckets?.[layerId])) { + counts[sourceLayer] += 1; + } + } } return counts; } @@ -202,16 +216,24 @@ try { const requiredLayerCounts = () => { const counts = Object.fromEntries(requiredLayers.map((layer) => [layer, 0])); - const layerIds = (map.getStyle().layers || []) - .filter((layer) => - layer.source === 'occumed-open' && - requiredLayers.includes(layer['source-layer']) - ) - .map((layer) => layer.id); - if (layerIds.length === 0) return counts; - for (const feature of map.queryRenderedFeatures({ layers: layerIds })) { - const sourceLayer = feature.sourceLayer; - if (sourceLayer in counts) counts[sourceLayer] += 1; + const layerIds = Object.fromEntries(requiredLayers.map((sourceLayer) => [ + sourceLayer, + (map.getStyle().layers || []) + .filter((layer) => + layer.source === 'occumed-open' && + layer['source-layer'] === sourceLayer + ) + .map((layer) => layer.id) + ])); + const tileManager = map.style?.tileManagers?.['occumed-open']; + if (!tileManager) return counts; + for (const id of tileManager.getRenderableIds()) { + const tile = tileManager.getTileByID(id); + for (const sourceLayer of requiredLayers) { + if (layerIds[sourceLayer].some((layerId) => tile?.buckets?.[layerId])) { + counts[sourceLayer] += 1; + } + } } return counts; }; @@ -274,21 +296,21 @@ try { return await new Promise((resolve, reject) => { const durationMs = 20_000; - let sampleFrame = null; + let sampling = false; const queueSample = () => { - if (sampleFrame !== null) return; - sampleFrame = requestAnimationFrame(() => { - sampleFrame = null; + if (sampling) return; + sampling = true; + try { sample(); - }); + } finally { + sampling = false; + } }; const sampleTimer = setInterval(queueSample, 50); + map.on('move', queueSample); const stopSampling = () => { clearInterval(sampleTimer); - if (sampleFrame !== null) { - cancelAnimationFrame(sampleFrame); - sampleFrame = null; - } + map.off('move', queueSample); }; const timeout = setTimeout(() => { stopSampling(); diff --git a/scripts/validate-continuous-zoom.mjs b/scripts/validate-continuous-zoom.mjs index 003ee10e..10ebff1e 100644 --- a/scripts/validate-continuous-zoom.mjs +++ b/scripts/validate-continuous-zoom.mjs @@ -63,7 +63,7 @@ function normalizeMotion(result, definition) { blankSampleCount: Number(result?.blankSampleCount || 0), missingFoundationSampleCount: Number(result?.missingFoundationSampleCount || 0), firstMissingFoundationSamples: Array.isArray(result?.firstMissingFoundationSamples) - ? result.firstMissingFoundationSamples + ? result.firstMissingFoundationSamples.map((sample) => structuredClone(sample)) : [], longestBlankRun: Number(result?.longestBlankRun || 0), minimumFeatureCount: Number(result?.minimumFeatureCount || 0), @@ -129,6 +129,12 @@ try { null, { timeout: 90_000 } ); + await page.evaluate(() => { + // This validator measures temporal foundation continuity, not high-DPI + // sharpness. Keep the full CSS viewport while avoiding a 5.76M-pixel + // software-rendered canvas that starves the 50ms sampling timer. + globalThis.__OCCUMED_MAP__.setPixelRatio(1); + }); async function waitForStableView(center, zoom, requiredLayers) { await page.evaluate(async ({ center, zoom, requiredLayers }) => { @@ -145,16 +151,24 @@ try { function requiredLayerCounts() { const counts = Object.fromEntries(requiredLayers.map((layer) => [layer, 0])); - const layerIds = (map.getStyle().layers || []) - .filter((layer) => - layer.source === 'occumed-open' && - requiredLayers.includes(layer['source-layer']) - ) - .map((layer) => layer.id); - if (layerIds.length === 0) return counts; - for (const feature of map.queryRenderedFeatures({ layers: layerIds })) { - const sourceLayer = feature.sourceLayer; - if (sourceLayer in counts) counts[sourceLayer] += 1; + const layerIds = Object.fromEntries(requiredLayers.map((sourceLayer) => [ + sourceLayer, + (map.getStyle().layers || []) + .filter((layer) => + layer.source === 'occumed-open' && + layer['source-layer'] === sourceLayer + ) + .map((layer) => layer.id) + ])); + const tileManager = map.style?.tileManagers?.['occumed-open']; + if (!tileManager) return counts; + for (const id of tileManager.getRenderableIds()) { + const tile = tileManager.getTileByID(id); + for (const sourceLayer of requiredLayers) { + if (layerIds[sourceLayer].some((layerId) => tile?.buckets?.[layerId])) { + counts[sourceLayer] += 1; + } + } } return counts; } @@ -212,16 +226,24 @@ try { const requiredLayerCounts = () => { const counts = Object.fromEntries(requiredLayers.map((layer) => [layer, 0])); - const layerIds = (map.getStyle().layers || []) - .filter((layer) => - layer.source === 'occumed-open' && - requiredLayers.includes(layer['source-layer']) - ) - .map((layer) => layer.id); - if (layerIds.length === 0) return counts; - for (const feature of map.queryRenderedFeatures({ layers: layerIds })) { - const sourceLayer = feature.sourceLayer; - if (sourceLayer in counts) counts[sourceLayer] += 1; + const layerIds = Object.fromEntries(requiredLayers.map((sourceLayer) => [ + sourceLayer, + (map.getStyle().layers || []) + .filter((layer) => + layer.source === 'occumed-open' && + layer['source-layer'] === sourceLayer + ) + .map((layer) => layer.id) + ])); + const tileManager = map.style?.tileManagers?.['occumed-open']; + if (!tileManager) return counts; + for (const id of tileManager.getRenderableIds()) { + const tile = tileManager.getTileByID(id); + for (const sourceLayer of requiredLayers) { + if (layerIds[sourceLayer].some((layerId) => tile?.buckets?.[layerId])) { + counts[sourceLayer] += 1; + } + } } return counts; }; @@ -284,21 +306,21 @@ try { }; return await new Promise((resolve, reject) => { - let sampleFrame = null; + let sampling = false; const queueSample = () => { - if (sampleFrame !== null) return; - sampleFrame = requestAnimationFrame(() => { - sampleFrame = null; + if (sampling) return; + sampling = true; + try { sample(); - }); + } finally { + sampling = false; + } }; const sampleTimer = setInterval(queueSample, 50); + map.on('move', queueSample); const stopSampling = () => { clearInterval(sampleTimer); - if (sampleFrame !== null) { - cancelAnimationFrame(sampleFrame); - sampleFrame = null; - } + map.off('move', queueSample); }; const timeout = setTimeout(() => { stopSampling(); diff --git a/src/occumed-map.js b/src/occumed-map.js index 0e81fe23..96491093 100644 --- a/src/occumed-map.js +++ b/src/occumed-map.js @@ -85,6 +85,96 @@ export function installOccumedAtmosphereBloom(map) { return bloom; } +/** + * Keeps decoded substitute tiles renderable across the complete 0–16 pyramid. + * + * MapLibre's normal retention only searches in-view tiles while an ideal tile + * is loading. Fully decoded parents and children in its out-of-view cache are + * consequently skipped, leaving no renderable tile until the ideal request is + * parsed. Keep the same-source global foundation decoded as a last resort and + * reattach the nearest cached substitute before cleanup removes it. + */ +export function installContinuousTileRetention(map) { + let removed = false; + + const configure = () => { + if (removed) return; + const tileManager = map.style?.tileManagers?.['occumed-open']; + if (!tileManager?.constructor) return; + tileManager.constructor.maxUnderzooming = Math.max( + Number(tileManager.constructor.maxUnderzooming || 0), + WORLD_ZOOM_PYRAMID_LEVELS + ); + tileManager.constructor.maxOverzooming = Math.max( + Number(tileManager.constructor.maxOverzooming || 0), + WORLD_ZOOM_PYRAMID_LEVELS + ); + if (tileManager.__occumedContinuousRetention) return; + + const updateRetainedTiles = tileManager._updateRetainedTiles.bind(tileManager); + let globalFoundationID = null; + tileManager._updateRetainedTiles = function retainCachedFoundation(idealTileIDs, zoom) { + const retained = updateRetainedTiles(idealTileIDs, zoom); + if (idealTileIDs.length && !globalFoundationID) { + globalFoundationID = idealTileIDs[0].scaledTo(0); + } + if (globalFoundationID) { + this._addTile(globalFoundationID); + retained[globalFoundationID.key] = globalFoundationID; + } + + for (const idealID of idealTileIDs) { + if (this.getTileByID(idealID.key)?.hasData()) continue; + + let foundAncestor = false; + for (let parentZoom = idealID.overscaledZ - 1; parentZoom >= 0; parentZoom -= 1) { + const parentID = idealID.scaledTo(parentZoom); + let parent = this.getTileByID(parentID.key); + if (!parent && this._outOfViewCache.has(parentID)) { + parent = this._addTile(parentID); + } + if (parent?.hasData()) { + retained[parentID.key] = parentID; + foundAncestor = true; + break; + } + } + if (foundAncestor) continue; + + const cachedChildren = Object.values(this._outOfViewCache.data) + .flat() + .map(({ value }) => value) + .filter((tile) => + tile.hasData() && + tile.tileID.isChildOf(idealID) && + tile.tileID.overscaledZ - idealID.overscaledZ <= WORLD_ZOOM_PYRAMID_LEVELS + ) + .map((tile) => tile.tileID.clone()); + if (!cachedChildren.length) continue; + + const nearestZoom = Math.min(...cachedChildren.map((tileID) => tileID.overscaledZ)); + for (const childID of cachedChildren) { + if (childID.overscaledZ !== nearestZoom) continue; + const child = this._addTile(childID); + if (child.hasData()) retained[childID.key] = childID; + } + } + + return retained; + }; + tileManager.__occumedContinuousRetention = true; + }; + + const remove = () => { + removed = true; + map.off('styledata', configure); + }; + + map.on('styledata', configure); + map.once('remove', remove); + configure(); +} + function resolvePublicOrigin(style, styleUrl) { const resolved = structuredClone(style); const styleOrigin = new URL(styleUrl, window.location.href).origin; @@ -177,6 +267,7 @@ export async function createOccumedMap({ ...mapOptions }); + installContinuousTileRetention(map); installOccumedAtmosphereBloom(map); if (controls) { diff --git a/src/server/mvt.js b/src/server/mvt.js index d0ab1c15..665ab3c2 100644 --- a/src/server/mvt.js +++ b/src/server/mvt.js @@ -227,6 +227,51 @@ export function mergeVectorTiles(payloads, { return encoded; } +/** + * Copies one source layer into a differently named layer while preserving its + * geometry. Property overrides let the gateway synthesize a deterministic + * physical fallback without mutating or replacing the browser source. + */ +export function copyVectorLayer(payload, { + sourceLayerName, + targetLayerName, + propertyOverrides = {} +}) { + const tile = decodeTile(payload, { + label: `MVT layer copy input for ${sourceLayerName}`, + coordinateScale: 128 + }); + const sourceLayer = tile.layers[sourceLayerName]; + if (!sourceLayer) return EMPTY_MVT; + + const overrides = normalizeMvtProperties(propertyOverrides); + const features = []; + for (let index = 0; index < sourceLayer.length; index += 1) { + const sourceFeature = sourceLayer.feature(index); + if (!isSaneGeometry(sourceFeature, sourceLayer.extent, 128)) continue; + const feature = new CombinedFeature(sourceFeature); + feature.properties = { ...feature.properties, ...overrides }; + features.push(feature); + } + if (!features.length) return EMPTY_MVT; + + const encoded = Buffer.from(vtpbf.fromVectorTileJs({ + layers: { + [targetLayerName]: new MergedLayer( + targetLayerName, + sourceLayer.version, + sourceLayer.extent, + features + ) + } + })); + validateVectorTilePayload(encoded, { + label: `copied MVT output for ${targetLayerName}`, + coordinateScale: 128 + }); + return encoded; +} + function interpolateAtX(start, end, x) { const delta = end.x - start.x; if (delta === 0) return { x, y: start.y }; diff --git a/src/server/world-tile-gateway.js b/src/server/world-tile-gateway.js index 36755d99..8066bc07 100644 --- a/src/server/world-tile-gateway.js +++ b/src/server/world-tile-gateway.js @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import { PMTiles, SharedPromiseCache } from 'pmtiles'; import { + copyVectorLayer, EMPTY_MVT, mergeVectorTiles, overscaleVectorLayer @@ -33,6 +34,7 @@ const DEFAULT_MAX_ARCHIVE_QUEUE = 256; const DEFAULT_MAX_UPSTREAM_TILE_BYTES = 16 * 1024 * 1024; const DEFAULT_MAX_RESOLVED_TILE_BYTES = 24 * 1024 * 1024; const CONTINUOUS_SURFACE_LAYERS = Object.freeze(['land', 'landcover', 'depth']); +const LANDCOVER_FALLBACK_PROPERTIES = Object.freeze({ class: 'grass' }); function boundedInteger(value, fallback, minimum, maximum) { const parsed = Number(value); @@ -478,9 +480,20 @@ export class WorldTileGateway { } = manifest.virtualTiles; if (zoom <= surfaceMaxZoom) { const payload = await this.readArchiveTile(surfaceAsset, zoom, x, y); - return payload - ? mergeVectorTiles([payload], { includeLayers: CONTINUOUS_SURFACE_LAYERS }) - : EMPTY_MVT; + if (!payload) return EMPTY_MVT; + const surface = mergeVectorTiles( + [payload], + { includeLayers: CONTINUOUS_SURFACE_LAYERS } + ); + const landcoverFallback = copyVectorLayer(surface, { + sourceLayerName: 'land', + targetLayerName: 'landcover', + propertyOverrides: LANDCOVER_FALLBACK_PROPERTIES + }); + return mergeVectorTiles( + [landcoverFallback, surface], + { includeLayers: CONTINUOUS_SURFACE_LAYERS } + ); } const divisor = 2 ** (zoom - surfaceMaxZoom); @@ -494,17 +507,21 @@ export class WorldTileGateway { ); if (!payload) return EMPTY_MVT; - return mergeVectorTiles( - CONTINUOUS_SURFACE_LAYERS.map((layerName) => - overscaleVectorLayer(payload, { - layerName, - sourceZoom: surfaceMaxZoom, - targetZoom: zoom, - targetX: x, - targetY: y - }) - ) + const overscaledLayers = CONTINUOUS_SURFACE_LAYERS.map((layerName) => + overscaleVectorLayer(payload, { + layerName, + sourceZoom: surfaceMaxZoom, + targetZoom: zoom, + targetX: x, + targetY: y + }) ); + const landcoverFallback = copyVectorLayer(overscaledLayers[0], { + sourceLayerName: 'land', + targetLayerName: 'landcover', + propertyOverrides: LANDCOVER_FALLBACK_PROPERTIES + }); + return mergeVectorTiles([landcoverFallback, ...overscaledLayers]); } async readBasemapTile(manifest, zoom, x, y) {