diff --git a/.gitignore b/.gitignore index ebc14aed..653bff2e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules/ +node_modules dist/ .generated/ public/style/occumed-open.json @@ -11,3 +12,8 @@ public/sprites/occumed@2x.png .env.* !.env.example .DS_Store +build/offline-global/ +config/immutable-owner-plan.json +immutable-inputs/ +.build-tools/ +visual-validation/immutable-*/ diff --git a/README.md b/README.md index 596ec160..b32a786c 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,7 @@ The goal is a close visual replica without a Mapbox token or Mapbox-hosted runti - MapLibre GL JS - one permanent MapLibre vector source at `/tiles/{z}/{x}/{y}.pbf` -- a server-side virtual worldwide tileset backed by the 754 PMTiles storage archives -- one consolidated zoom 0–5 overview and one generalized worldwide physical surface -- open elevation data used only for hillshade, not as a second basemap +- one immutable worldwide foundation plus deterministic non-overlapping PMTiles owners - locally compiled sprites - browser-local glyph rendering - generated runtime style at `/style/occumed-open.json` @@ -32,17 +30,19 @@ No `VITE_MAPBOX_ACCESS_TOKEN`, `mapbox-gl`, `mapbox://` URL, Mapbox API endpoint MapLibre sees only `occumed-open`, whose URL is permanent from zoom 0 through 16. The browser does not load the world manifest, select an archive, register a PMTiles protocol, or replace a source while the map moves. -The Node tile gateway resolves each Z/X/Y request on the server: +The existing worldwide and regional PMTiles archives are offline inputs only. +The offline builder assigns one authority per layer family, clips and normalizes +geometry, removes duplicates and contained overlaps, rejects malformed, +oversized, and tile-shaped surface polygons, and writes each final Z/X/Y once. -- zoom 0–5 comes from a consolidated overview built from the same regional schema; -- zoom 6–16 is resolved against every storage shard intersecting the requested tile; -- boundary tiles are decoded, deduplicated by stable feature ID, and re-encoded as one MVT; -- the worldwide `land` layer is merged into the same response at every zoom; -- nested Natural Earth bathymetry bands are served as the `depth` layer at globe - and regional zooms, then fade before detailed navigation zooms; -- completed virtual tiles are held in a bounded in-memory cache and exposed with CDN cache headers. +Production loads a versioned ownership manifest, performs one deterministic +owner lookup, and returns the selected archive's stored MVT bytes unchanged. +It does not connect to Neon, merge shards, synthesize landcover, create +geometry, or stretch parent/child tiles. The browser never sees the owner +inventory and never switches its single source. -The PMTiles archives and routing manifest are storage implementation details. Their URLs never appear in the MapLibre style. +See [the immutable tileset build guide](docs/offline-global-tileset.md) for +the complete offline build and mandatory visual validation workflow. ## Source of truth @@ -73,6 +73,12 @@ Optional Render variable: PUBLIC_ORIGIN=https://map-yxjb.onrender.com ``` +Required tileset location (unless deployed at `dist/immutable-world/manifest.json`): + +```text +OCCUMED_IMMUTABLE_TILESET_MANIFEST=/absolute/path/to/immutable-world/manifest.json +``` + ## Reuse Install the repository in the consuming application: @@ -105,10 +111,11 @@ The build verifies: - the original export remains intact; - the generated style passes the MapLibre style specification; - no active source, sprite, or glyph URL points to Mapbox; -- globe, terrain, landcover, water, labels, and viewer-quality settings remain calibrated to the screenshot reference set; +- globe, landcover, water, labels, and viewer-quality settings remain calibrated to the screenshot reference set; - only one permanent vector source and one same-origin Z/X/Y template exist in the style; - browser-side PMTiles routing, `source.setUrl()`, fallback URLs, and OpenFreeMap are absent; -- the routing index includes every intersecting shard, including antimeridian segments; -- duplicate features are removed while boundary geometry is preserved; -- the overview, physical surface, regional merge, in-memory cache, and virtual release workflow remain wired; +- the immutable manifest is complete, non-overlapping, and fail-closed; +- every split-prefix ancestor and descendant tile has one deterministic prebuilt owner; +- production contains no Neon tile cache, runtime merge, geometry creation, landcover synthesis, or parent/child stretch path; +- mandatory static and exact-camera motion captures reject seams, tile footprints, stretched polygons, inconsistent neighbors, blank frames, and source switching; - no application-specific overlay data is included. diff --git a/docs/offline-global-tileset-recovery.md b/docs/offline-global-tileset-recovery.md new file mode 100644 index 00000000..3c0c968f --- /dev/null +++ b/docs/offline-global-tileset-recovery.md @@ -0,0 +1,118 @@ +# Offline Global Tileset Recovery Handoff + +Status captured after Codex usage credits were exhausted on 2026-07-28/29. + +## Critical state + +- Target branch: `rebuild/offline-global-tileset` +- Remote branch was still identical to `main` at `ba835de833b7c62d26f7767a0cf06ed8dec287ee` when this handoff was written. +- The implementation described below had not yet been committed or pushed by Codex. +- The Codex workspace is therefore the only known location of the uncommitted source changes and generated artifacts. +- Do not delete or reset that Codex workspace before exporting/committing its changes. + +## Intended architecture + +- One browser vector source. +- Immutable worldwide PMTiles output. +- Deterministic non-overlapping owner partitions when a single physical file is too large. +- Every `z/x/y` tile belongs to exactly one owner. +- No Neon tile cache in the active production path. +- No runtime shard merging. +- No runtime landcover synthesis. +- No runtime parent/child stretching or overscaling. +- No production geometry creation. +- Offline-only clipping, schema normalization, deduplication, ancestor materialization, and malformed-feature rejection. + +## Input inventory discovered + +- 754 regional PMTiles archives plus worldwide overview and surface archives. +- Published regional input total reported by Codex: approximately 258.0 GB. +- Inputs were resolved from the existing GitHub Release and SHA-256 locked. +- Local `dist/virtual-assets` PMTiles files were only 325-byte fixtures and were correctly rejected as production inputs. + +## Representative artifact results + +Codex completed a representative validation build covering 17,901 exact `z/x/y` tiles plus three deterministic owner partitions for Fresno and both sides of the antimeridian. + +### Foundation + +- Final stable size: `250,938,363` bytes. +- Reported SHA-256 prefix: `b76dfa...`. +- Independent PMTiles verification passed after staged write, fsync, pending-file verification, and atomic rename. + +### Representative manifest + +- Total size across foundation and three non-overlapping owners: `272,198,460` bytes. +- A later manifest/artifact version after Fresno landcover correction was reported as `729e166...`. + +## Builder defects already found and corrected locally + +1. Node `Buffer` was returned where the PMTiles reader required an exact-range `ArrayBuffer`. +2. Polygon bounds used `Math.min(...points)` / `Math.max(...points)` and exceeded the JavaScript argument limit on detailed polygons; replaced with bounded linear scans. +3. Same-property polygon containment/deduplication was quadratic on dense tiles; replaced with deterministic spatial bucket indexing. +4. Legitimate buffered overview polygons exceeded the strict tile box; offline normalization was changed to clip to the exact tile boundary and revalidate encoded output. +5. The inherited runtime overscale helper validated intermediate MVT too early; ancestor materialization was moved into the offline builder and changed to feature-granular transform, clip, reject, then encode. +6. PMTiles owner metadata inherited an invalid center zoom; metadata was changed to derive center and zoom bounds from the owner’s actual addressed tiles. +7. Initial foundation output changed after the success report; writer was hardened to staging-path generation, verification, fsync, pending-copy verification, descriptor close, and atomic promotion. +8. Fresno z16 tiles lacked landcover because the overview z6 parent had no Fresno landcover. Authority was corrected so overview owns landcover at z0-6 and exactly one deterministic regional archive owns landcover above z6 for each partition. +9. Legacy AWS terrain/hillshade remained as a second browser source and caused readiness/network failures; Codex removed that source and dependent hillshade layer from the generated runtime style to enforce one browser source. + +## Validation state reached before credit exhaustion + +### Static views + +Codex reported all nine static views passing after the Fresno landcover rebuild: + +- Global +- North America +- South America +- Europe +- Pacific +- Antimeridian +- Fresno regional +- Fresno city +- Fresno street + +Reported results: + +- Required land rendered. +- Regional landcover rendered at Fresno street level. +- Transportation rendered. +- Exactly one style source. +- Zero network failures. +- Zero page errors. +- Zero detected seams. +- Zero rectangular tile footprints. +- Zero stretching detections. +- Zero blank-frame detections. +- Zero source changes. +- Zero neighboring-tile inconsistency detections. + +### Motion validation + +Motion validation was not finished. + +The last reported change corrected only the first motion setup predicate: global starting views should require immutable-source readiness at the target camera, not visible landcover, because the global contract is land plus ocean depth. Motion frames were still intended to undergo every pixel/source gate. + +## Immediate recovery instruction for the next Codex turn + +Do not rebuild first. Preserve the workspace immediately: + +1. Confirm the workspace still contains the uncommitted changes and generated reports. +2. Confirm the current branch is `rebuild/offline-global-tileset`. +3. Review `git status --short` and `git diff --stat`. +4. Exclude `node_modules`, downloaded PMTiles inputs, browser binaries, temporary build directories, and generated PMTiles artifacts from Git. +5. Commit all source, configuration, workflow, manifest, validator, report, and tracked screenshot changes. +6. Push the branch. +7. Open a draft PR. +8. Report the commit SHA and PR number before resuming any build or validation. + +## Work still required after preservation + +- Finish all continuous motion validations. +- Inspect saved screenshots manually, not only numeric gates. +- Prove complete deterministic worldwide ownership for every intended tile. +- Execute the production-scale immutable partition build, not only the 17,901-tile representative artifact. +- Keep every final PMTiles release asset below GitHub’s per-asset limit. +- Upload only final immutable partitions and the versioned ownership manifest. +- Keep the PR draft until the actual worldwide artifact set exists and is visually verified. diff --git a/docs/offline-global-tileset.md b/docs/offline-global-tileset.md new file mode 100644 index 00000000..e8c15da8 --- /dev/null +++ b/docs/offline-global-tileset.md @@ -0,0 +1,137 @@ +# Immutable worldwide PMTiles build + +The production map uses one browser source and an immutable PMTiles ownership +manifest. Existing worldwide and regional archives are build inputs only. +Production never decodes MVT geometry, merges shards, queries Neon, synthesizes +landcover, or substitutes parent/child tiles. + +## Authority and ownership + +| Output layers | Sole offline authority | +|---|---| +| `land`, `depth` | `occumed-world-surface.pmtiles` | +| `landcover` | `occumed-world-overview.pmtiles` | +| roads, buildings, boundaries, labels, other cartography | one regional owner | + +The deterministic plan starts with z6 logical cells. A cell whose locked +candidate input set is too large is split through z7 or z8. Leaf prefixes never +overlap. When a prefix is split, its own ancestor tile is assigned explicitly +to the first descendant owner in stable child order. Consequently every +addressed z0–z16 tile resolves to exactly one foundation, exact-tile, or prefix +owner. + +Missing entries are immutable empty results from that owner; production never +tries a second archive. + +## Full offline build + +The complete published input inventory is about 270 GB. Use a build host with +enough space for the locked input cache, work files, and final owners. All +outputs are created at staging paths, verified as PMTiles, fsynced, copied to a +pending file, verified again, and atomically promoted. + +Create the SHA-locked owner plan and worldwide foundation target list: + +```bash +npm run tiles:plan-world -- \ + --output config/immutable-owner-plan.json + +npm run tiles:targets-foundation -- \ + --plan config/immutable-owner-plan.json \ + --output build/offline-global/foundation-targets.json +``` + +Localize the two worldwide inputs and build the z0–z6 foundation: + +```bash +npm run tiles:localize -- \ + --plan config/immutable-owner-plan.json \ + --targets build/offline-global/foundation-targets.json \ + --output-dir immutable-inputs \ + --report build/offline-global/foundation-inputs.json + +npm run tiles:build-foundation -- \ + --plan config/immutable-owner-plan.json \ + --targets build/offline-global/foundation-targets.json \ + --input-report build/offline-global/foundation-inputs.json \ + --output-dir build/offline-global/artifact +``` + +For every `owners[].id` in the plan, localize its locked candidates, enumerate +the exact addresses in their PMTiles directories, and build that owner. The +input directory is a shared digest-verified cache, so repeated candidates are +reused. + +```bash +OWNER_ID=z6-10-24 + +npm run tiles:localize -- \ + --plan config/immutable-owner-plan.json \ + --owner-id "$OWNER_ID" \ + --output-dir immutable-inputs \ + --report "build/offline-global/$OWNER_ID-inputs.json" + +npm run tiles:targets-owner -- \ + --plan config/immutable-owner-plan.json \ + --owner-id "$OWNER_ID" \ + --input-report "build/offline-global/$OWNER_ID-inputs.json" \ + --output "build/offline-global/$OWNER_ID-targets.json" + +npm run tiles:build-owner -- \ + --plan config/immutable-owner-plan.json \ + --owner-id "$OWNER_ID" \ + --targets "build/offline-global/$OWNER_ID-targets.json" \ + --input-report "build/offline-global/$OWNER_ID-inputs.json" \ + --output-dir build/offline-global/artifact +``` + +Finalize only after every planned owner report exists: + +```bash +npm run tiles:finalize -- \ + --plan config/immutable-owner-plan.json \ + --foundation build/offline-global/artifact/reports/foundation.json \ + --owner-dir build/offline-global/artifact/reports/owners \ + --output build/offline-global/artifact/manifest.json +``` + +The finalizer marks an incomplete inventory as `validationFixture: true`. +Production rejects it. `OCCUMED_ALLOW_PARTIAL_TILESET_FIXTURE=true` exists only +for the bounded local visual fixture and must not be set in deployment. + +## Production + +Set `OCCUMED_IMMUTABLE_TILESET_MANIFEST` to the local manifest path or an HTTPS +manifest URL. Local assets are resolved beneath the manifest directory. Remote +assets use the manifest's optional `assetBaseUrl`. + +The server validates completeness, artifact identity, SHA metadata, safe asset +paths, non-overlapping prefixes, and unique exact-tile assignments before +serving tiles. Each request performs a bounded index lookup and a single +PMTiles byte-range read. Stored gzip MVT bytes are returned unchanged. + +## Mandatory validation + +Run the structural and production checks: + +```bash +npm run build +``` + +Run the browser gate against the finalized manifest and validation target +document: + +```bash +PLAYWRIGHT_BROWSERS_PATH=build/offline-global/playwright \ +npm run check:immutable-visuals -- \ + --manifest build/offline-global/artifact/manifest.json \ + --targets build/offline-global/representative-targets.json \ + --output-dir visual-validation/immutable-final +``` + +The target document must cover global, North America, South America, Europe, +Pacific, antimeridian, regional, city, and street views, plus 30 exact-camera +motion checkpoints. Validation fails for rectangular footprints, vertical or +horizontal seams, stretched polygons, inconsistent neighbors, blank frames, +source switching, page errors, or tile delivery failures. Saved screenshots +must also be inspected at full size before an artifact is accepted. diff --git a/package-lock.json b/package-lock.json index 065d3364..c987e504 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@chrispahm/spritezero": "8.1.0", "@maplibre/maplibre-gl-style-spec": "24.8.1", "playwright": "1.55.0", + "pngjs": "7.0.0", "vite": "8.1.5" } }, @@ -1414,6 +1415,16 @@ "fflate": "^0.8.2" } }, + "node_modules/pngjs": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", + "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.19.0" + } + }, "node_modules/postcss": { "version": "8.5.23", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz", diff --git a/package.json b/package.json index 4105687b..53534f10 100644 --- a/package.json +++ b/package.json @@ -8,18 +8,26 @@ "prepare:style": "node scripts/build-runtime-style.mjs && node scripts/normalize-runtime-fonts.mjs && node scripts/normalize-runtime-filters.mjs && node scripts/normalize-boundary-filters.mjs && node scripts/force-runtime-filter-expressions.mjs && node scripts/apply-schema-parity.mjs && node scripts/apply-globe-parity.mjs && node scripts/apply-photo-reference.mjs && node scripts/restore-exported-cartography.mjs && node scripts/calibrate-reference-colors.mjs && node scripts/lock-exact-exported-swatches.mjs && node scripts/lock-reference-atmosphere.mjs && node scripts/normalize-runtime-fonts.mjs && node scripts/apply-mapbox-rendering-contract.mjs && node scripts/use-local-maplibre-glyphs.mjs", "prepare:assets": "npm run prepare:sprites && npm run prepare:style", "tiles:build": "bash planetiler/build-region.sh", - "tiles:plan-world": "node scripts/plan-world-shards.mjs --scope all", + "tiles:plan-world": "node scripts/offline-tileset/plan-owners.mjs", + "tiles:targets-foundation": "node scripts/offline-tileset/generate-foundation-targets.mjs", + "tiles:targets-owner": "node scripts/offline-tileset/generate-owner-targets.mjs", + "tiles:localize": "node scripts/offline-tileset/localize-representative-inputs.mjs", + "tiles:build-foundation": "node scripts/offline-tileset/build-foundation.mjs", + "tiles:build-owner": "node scripts/offline-tileset/build-owner.mjs", + "tiles:finalize": "node scripts/offline-tileset/finalize-manifest.mjs", + "tiles:targets-fixture": "node scripts/offline-tileset/generate-representative-targets.mjs", + "tiles:localize-fixture": "node scripts/offline-tileset/localize-representative-inputs.mjs", + "tiles:build-fixture": "node scripts/offline-tileset/build-representative.mjs", "check:export": "node scripts/check-export.mjs", - "check:hardening": "node scripts/check-world-hardening.mjs", - "check:foundation": "node scripts/check-continuous-foundation-lock.mjs", - "check:neon-cache": "node scripts/check-neon-navigation-cache.mjs", - "check:runtime": "node scripts/check-runtime.mjs && node scripts/validate-maplibre-style.mjs && node scripts/check-globe-parity.mjs && node scripts/check-cartography-parity.mjs && node scripts/check-viewer-quality.mjs && node scripts/check-photo-reference.mjs && node scripts/check-exact-exported-swatches.mjs && node scripts/check-render-clarity.mjs && node scripts/check-pmtiles-integration.mjs && node scripts/check-world-tile-gateway.mjs && npm run check:neon-cache && npm run check:hardening && npm run check:foundation", + "check:architecture": "node scripts/check-immutable-architecture.mjs", + "check:runtime": "node scripts/check-immutable-architecture.mjs && node scripts/check-runtime.mjs && node scripts/validate-maplibre-style.mjs && node scripts/check-globe-parity.mjs && node scripts/check-cartography-parity.mjs && node scripts/check-viewer-quality.mjs && node scripts/check-photo-reference.mjs && node scripts/check-exact-exported-swatches.mjs && node scripts/check-render-clarity.mjs", + "check:immutable-visuals": "node scripts/validate-immutable-visuals.mjs", "check:server": "node scripts/check-server-health.mjs", "check": "npm run check:export && npm run check:runtime", "dev": "npm run prepare:assets && vite", "build": "npm run prepare:assets && npm run check && vite build && npm run check:server", "preview": "vite preview", - "start": "node scripts/start-localized-world.mjs" + "start": "node server.mjs" }, "dependencies": { "@mapbox/vector-tile": "2.0.4", @@ -31,6 +39,7 @@ "devDependencies": { "@chrispahm/spritezero": "8.1.0", "@maplibre/maplibre-gl-style-spec": "24.8.1", + "pngjs": "7.0.0", "playwright": "1.55.0", "vite": "8.1.5" } diff --git a/scripts/apply-globe-parity.mjs b/scripts/apply-globe-parity.mjs index bfba9293..a0f50f6a 100644 --- a/scripts/apply-globe-parity.mjs +++ b/scripts/apply-globe-parity.mjs @@ -49,35 +49,14 @@ runtime.sky = { delete runtime.fog; delete runtime.light; -const hillshade = runtime.layers.find((layer) => layer.id === 'occumed-hillshade'); -if (!hillshade) throw new Error('The generated open hillshade layer is missing.'); - -hillshade.paint = { - 'hillshade-exaggeration': [ - 'interpolate', - ['linear'], - ['zoom'], - 2, - 0.04, - 8, - 0.16, - 15, - 0.11 - ], - 'hillshade-shadow-color': 'hsla(215, 22%, 28%, 0.42)', - 'hillshade-highlight-color': 'hsla(48, 38%, 96%, 0.34)', - 'hillshade-accent-color': 'hsla(95, 26%, 46%, 0.32)', - 'hillshade-illumination-direction': 335, - 'hillshade-illumination-anchor': 'map' -}; - runtime.metadata = { ...(runtime.metadata || {}), 'occumed:mapbox-fog-translated-to-maplibre-sky': true, 'occumed:globe-parity-pass': 3, 'occumed:stable-orientation-neutral-atmosphere': true, + 'occumed:external-terrain-disabled': true, 'occumed:visual-quality-pass': true }; await fs.writeFile(runtimePath, `${JSON.stringify(runtime, null, 2)}\n`); -console.log('Applied the refined Occu-Med globe rim and stable hillshade.'); +console.log('Applied the refined Occu-Med globe rim without an external terrain source.'); diff --git a/scripts/build-runtime-style.mjs b/scripts/build-runtime-style.mjs index 8ef6001e..06a9d57d 100644 --- a/scripts/build-runtime-style.mjs +++ b/scripts/build-runtime-style.mjs @@ -14,10 +14,6 @@ const vectorTilesUrl = const glyphsUrl = process.env.OCCUMED_GLYPHS_URL || 'https://demotiles.maplibre.org/font/{fontstack}/{range}.pbf'; -const terrainUrl = - process.env.OCCUMED_TERRAIN_URL || - 'https://s3.amazonaws.com/elevation-tiles-prod/terrarium/{z}/{x}/{y}.png'; - const original = JSON.parse(await fs.readFile(sourcePath, 'utf8')); const fontMap = new Map([ @@ -296,8 +292,6 @@ function rewriteLayer(layer, targetSourceLayer) { const convertedLayers = []; const skippedLayers = []; const sourceLayerMappings = {}; -let hillshadeInserted = false; - for (const layer of original.layers || []) { if (!layer.source) { convertedLayers.push(clone(layer)); @@ -324,26 +318,10 @@ for (const layer of original.layers || []) { const sourceLayer = layer['source-layer']; if (sourceLayer === 'hillshade') { - if (!hillshadeInserted) { - convertedLayers.push({ - id: 'occumed-hillshade', - type: 'hillshade', - source: 'occumed-terrain', - minzoom: 2, - maxzoom: 16, - paint: { - 'hillshade-exaggeration': ['interpolate', ['linear'], ['zoom'], 2, 0.16, 8, 0.34, 15, 0.24], - 'hillshade-shadow-color': 'hsl(215, 18%, 30%)', - 'hillshade-highlight-color': 'hsl(48, 40%, 94%)', - 'hillshade-accent-color': 'hsl(95, 18%, 55%)', - 'hillshade-illumination-direction': 335, - 'hillshade-illumination-anchor': 'viewport' - }, - metadata: { 'occumed:original-source-layer': 'hillshade' } - }); - hillshadeInserted = true; - } - skippedLayers.push({ id: layer.id, reason: 'replaced by open raster DEM hillshade' }); + skippedLayers.push({ + id: layer.id, + reason: 'one-source immutable architecture has no external terrain source' + }); continue; } @@ -381,15 +359,6 @@ const runtimeStyle = { maxzoom: 16, attribution: '© OpenStreetMap contributors' - }, - 'occumed-terrain': { - type: 'raster-dem', - tiles: [terrainUrl], - encoding: 'terrarium', - tileSize: 256, - minzoom: 0, - maxzoom: 15, - attribution: 'Elevation data via the AWS Terrain Tiles public dataset' } }, sprite: '__OCCUMED_PUBLIC_ORIGIN__/sprites/occumed', @@ -411,7 +380,7 @@ const report = { endpoints: { vectorTiles: vectorTilesUrl, glyphs: glyphsUrl, - terrain: terrainUrl, + terrain: null, relief: null, sprite: '__OCCUMED_PUBLIC_ORIGIN__/sprites/occumed' } diff --git a/scripts/calibrate-reference-colors.mjs b/scripts/calibrate-reference-colors.mjs index 165c7a53..d5174ca4 100644 --- a/scripts/calibrate-reference-colors.mjs +++ b/scripts/calibrate-reference-colors.mjs @@ -190,30 +190,6 @@ if (!water) throw new Error('The exported water layer is missing.'); water.paint ||= {}; water.paint['fill-opacity'] = 1; -// Strengthen physical form without changing any vector structure color. -const hillshade = layer('occumed-hillshade'); -if (!hillshade) throw new Error('The open hillshade layer is missing.'); -hillshade.minzoom = 1.5; -hillshade.maxzoom = 16; -hillshade.paint = { - 'hillshade-exaggeration': [ - 'interpolate', - ['linear'], - ['zoom'], - 1.5, 0.07, - 4.5, 0.13, - 7.5, 0.2, - 10, 0.27, - 13, 0.31, - 16, 0.2 - ], - 'hillshade-shadow-color': '#52685B', - 'hillshade-highlight-color': '#FFF8E8', - 'hillshade-accent-color': '#6E8D69', - 'hillshade-illumination-direction': 335, - 'hillshade-illumination-anchor': 'map' -}; - // Bring the exported road and boundary hierarchy into view at the same scales as // the supplied Studio references. Their individual line colors remain untouched. const entryZooms = new Map([ diff --git a/scripts/check-exact-exported-swatches.mjs b/scripts/check-exact-exported-swatches.mjs index 32f0574e..705da777 100644 --- a/scripts/check-exact-exported-swatches.mjs +++ b/scripts/check-exact-exported-swatches.mjs @@ -33,9 +33,9 @@ const EXACT = { wetland: '#A5CAD6', water: '#79BCEC', waterShadow: '#7293EE', - depthShallow: '#79BCEC59', - depthMid: '#5AACE759', - depthDeep: '#3B9DE359' + depthShallow: '#79BCEC', + depthMid: '#6EB6EA', + depthDeep: '#63B1E9' }; expect(color('land', 'background-color') === EXACT.ocean, 'The permanent ocean background changed.'); @@ -117,10 +117,8 @@ expect(color('landuse', 'fill-opacity') === 1, 'Detailed landuse colors are weak expect(color('water', 'fill-opacity') === 1, 'Water is not fully opaque.'); expect(!runtime.layers.some((candidate) => candidate.type === 'raster'), 'A raster basemap was reintroduced.'); -const hillshade = layer('occumed-hillshade'); -expect(hillshade?.paint?.['hillshade-shadow-color'] === '#0000004D', 'Hillshade shadows are tinting terrain.'); -expect(hillshade?.paint?.['hillshade-highlight-color'] === '#FFFFFF4D', 'Hillshade highlights are tinting terrain.'); -expect(hillshade?.paint?.['hillshade-accent-color'] === '#00000026', 'Hillshade accents are tinting terrain.'); +expect(!runtime.layers.some((candidate) => candidate.type === 'hillshade'), 'An external hillshade layer was reintroduced.'); +expect(Object.keys(runtime.sources || {}).length === 1, 'The palette style no longer has exactly one browser source.'); expect(runtime.metadata?.['occumed:reference-color-system'] === 'continuous-world-v10', 'The continuous-world color pass did not run.'); expect(runtime.metadata?.['occumed:raster-relief-disabled'] === true, 'Raster relief protection is missing.'); diff --git a/scripts/check-globe-parity.mjs b/scripts/check-globe-parity.mjs index 43b8435a..37600eb8 100644 --- a/scripts/check-globe-parity.mjs +++ b/scripts/check-globe-parity.mjs @@ -102,16 +102,13 @@ if (!Array.isArray(depthOpacity) || depthOpacity[0] !== 'max' || Number(depthOpa fail('Bathymetry can still collapse to zero at detailed navigation zooms.'); } const waterDepthColors = JSON.stringify(waterDepth?.paint?.['fill-color'] || []); -for (const value of ['#79BCEC59', '#5AACE759', '#3B9DE359']) { +for (const value of ['#79BCEC', '#6EB6EA', '#63B1E9']) { if (!waterDepthColors.includes(value)) fail(`The exported bathymetry swatch ${value} is missing.`); } - -const hillshade = layer('occumed-hillshade'); -if (!hillshade) fail('The open hillshade layer is missing.'); -if (hillshade?.minzoom !== 1.5) fail('Hillshade begins too late to shape the globe.'); -if (hillshade?.paint?.['hillshade-illumination-anchor'] !== 'map') fail('Hillshade must remain geographically anchored.'); -if (hillshade?.paint?.['hillshade-shadow-color'] !== '#0000004D') fail('Hillshade shadows are tinting exported colors.'); -if (hillshade?.paint?.['hillshade-highlight-color'] !== '#FFFFFF4D') fail('Hillshade highlights are tinting exported colors.'); +if (runtime.layers.some((candidate) => candidate.type === 'hillshade')) { + fail('The one-source style contains an external hillshade layer.'); +} +if (Object.keys(runtime.sources || {}).length !== 1) fail('The globe style does not use exactly one browser source.'); if ((layer('road-motorway-trunk')?.minzoom ?? 99) > 2) fail('Major highways enter too late for the supplied regional hierarchy.'); if ((layer('admin-0-boundary')?.minzoom ?? 99) > 0) fail('Country boundaries are unavailable at globe zoom.'); diff --git a/scripts/check-immutable-architecture.mjs b/scripts/check-immutable-architecture.mjs new file mode 100644 index 00000000..a69c8eea --- /dev/null +++ b/scripts/check-immutable-architecture.mjs @@ -0,0 +1,233 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + computeImmutableArtifactVersion, + validateImmutableManifest +} from '../src/server/immutable-world-tileset.js'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const read = (filename) => fs.readFile(path.join(root, filename), 'utf8'); +const [ + server, + browser, + styleBuilder, + normalizer, + packageText +] = await Promise.all([ + read('server.mjs'), + read('src/occumed-map.js'), + read('scripts/build-runtime-style.mjs'), + read('scripts/offline-tileset/mvt-normalizer.mjs'), + read('package.json') +]); + +for (const forbidden of [ + 'neon-navigation-cache', + 'world-tile-gateway', + 'mergeVectorTiles', + 'world-routing', + 'overscale-vector-tile' +]) { + assert( + !server.includes(forbidden), + `Production server still references forbidden runtime module or operation: ${forbidden}` + ); +} +assert( + server.includes("from './src/server/immutable-world-tileset.js'"), + 'Production server does not use the immutable PMTiles store.' +); +assert( + !server.includes('validateVectorTilePayload'), + 'Production server still decodes or validates geometry instead of returning prebuilt bytes.' +); +assert( + server.includes("'Content-Encoding': resolved.contentEncoding"), + 'Production server does not preserve the stored PMTiles tile compression.' +); + +for (const forbidden of [ + 'installContinuousTileRetention', + '_updateRetainedTiles', + 'maxTileCacheZoomLevels' +]) { + assert(!browser.includes(forbidden), `Browser still stretches or pins parent/child tiles: ${forbidden}`); +} +assert( + browser.includes('tileManager.constructor.maxUnderzooming = 0') && + browser.includes('tileManager.constructor.maxOverzooming = 0'), + 'Browser does not explicitly disable MapLibre parent/child fallback depths.' +); +assert( + browser.includes('cancelPendingTileRequestsWhileZooming: true'), + 'Browser must use MapLibre normal request cancellation without a retained-tile patch.' +); + +assert(!styleBuilder.includes('occumed-terrain'), 'Style builder still creates a second terrain source.'); +assert(!styleBuilder.includes('raster-dem'), 'Style builder still creates a raster DEM source.'); +assert( + styleBuilder.includes("reason: 'one-source immutable architecture has no external terrain source'"), + 'Style builder does not document why exported hillshade layers are excluded.' +); + +for (const authority of [ + "land: 'world-surface'", + "depth: 'world-surface'", + "landcover: 'world-overview'", + "cartography: 'regional-owner'" +]) { + assert(normalizer.includes(authority), `Offline authority is missing: ${authority}`); +} +assert( + normalizer.includes('isLargeAxisAlignedRectangle'), + 'Offline normalizer does not reject large tile-shaped surface polygons.' +); +assert( + normalizer.includes('rejectedMalformed') && normalizer.includes('rejectedOversized'), + 'Offline normalizer does not count malformed and oversized feature rejection.' +); + +const packageJson = JSON.parse(packageText); +assert.equal(packageJson.scripts.start, 'node server.mjs', 'Production start command bypasses the immutable server.'); +assert( + packageJson.scripts['check:runtime'].includes('check-immutable-architecture.mjs'), + 'Runtime validation does not include the immutable architecture lock.' +); + +function fixtureManifest(overrides = {}) { + const manifest = { + schemaVersion: 1, + generatedAt: '2000-01-01T00:00:00.000Z', + planVersion: 'a'.repeat(64), + browserSourceId: 'occumed-open', + minZoom: 0, + maxZoom: 16, + complete: false, + validationFixture: true, + logicalOwnerCount: 1, + plannedOwnerCount: 2, + builtOwnerCount: 1, + defaultOwner: 'foundation', + totalBytes: 508, + authorities: { + land: 'world-surface', + depth: 'world-surface', + landcover: 'world-overview', + cartography: 'regional-owner' + }, + runtimePolicy: { + neonTileCache: false, + runtimeShardMerge: false, + runtimeLandcoverSynthesis: false, + runtimeGeometry: false, + parentChildStretching: false + }, + foundation: { + id: 'foundation', + file: 'foundation.pmtiles', + bytes: 254, + sha256: 'b'.repeat(64), + maxZoom: 6 + }, + owners: [{ + id: 'z6-10-24', + prefix: { z: 6, x: 10, y: 24 }, + file: 'owners/z6-10-24.pmtiles', + bytes: 254, + sha256: 'c'.repeat(64) + }], + ...overrides + }; + manifest.artifactVersion = computeImmutableArtifactVersion(manifest); + return manifest; +} + +const partial = fixtureManifest(); +assert.throws( + () => validateImmutableManifest(partial), + /incomplete/, + 'Production accepted a partial validation fixture without an explicit override.' +); +assert.doesNotThrow( + () => validateImmutableManifest(partial, { allowPartial: true }), + 'The explicit validation-fixture override did not work.' +); + +const overlapping = fixtureManifest({ + plannedOwnerCount: 2, + builtOwnerCount: 2, + complete: true, + validationFixture: false, + owners: [ + partial.owners[0], + { + id: 'z7-20-48', + prefix: { z: 7, x: 20, y: 48 }, + file: 'owners/z7-20-48.pmtiles', + bytes: 254, + sha256: 'd'.repeat(64) + } + ] +}); +overlapping.artifactVersion = computeImmutableArtifactVersion(overlapping); +assert.throws( + () => validateImmutableManifest(overlapping), + /Overlapping immutable owner prefixes/, + 'Manifest accepted overlapping prebuilt owners.' +); + +const splitAncestor = fixtureManifest({ + plannedOwnerCount: 1, + owners: [{ + id: 'z8-40-96', + prefix: { z: 8, x: 40, y: 96 }, + exactTiles: [{ z: 7, x: 20, y: 48 }], + file: 'owners/z8-40-96.pmtiles', + bytes: 254, + sha256: 'd'.repeat(64) + }] +}); +splitAncestor.artifactVersion = computeImmutableArtifactVersion(splitAncestor); +assert.doesNotThrow( + () => validateImmutableManifest(splitAncestor, { allowPartial: true }), + 'A deterministic split-prefix ancestor assignment was rejected.' +); + +const duplicateExact = fixtureManifest({ + plannedOwnerCount: 2, + builtOwnerCount: 2, + complete: true, + validationFixture: false, + owners: [ + splitAncestor.owners[0], + { + id: 'z8-41-96', + prefix: { z: 8, x: 41, y: 96 }, + exactTiles: [{ z: 7, x: 20, y: 48 }], + file: 'owners/z8-41-96.pmtiles', + bytes: 254, + sha256: 'e'.repeat(64) + } + ] +}); +duplicateExact.artifactVersion = computeImmutableArtifactVersion(duplicateExact); +assert.throws( + () => validateImmutableManifest(duplicateExact), + /assigned to both/, + 'Manifest accepted two owners for one exact split-prefix ancestor tile.' +); + +const regenerated = fixtureManifest({ generatedAt: '2030-01-01T00:00:00.000Z' }); +assert.equal( + partial.artifactVersion, + regenerated.artifactVersion, + 'Artifact identity changes with a non-semantic generation timestamp.' +); + +console.log( + 'Immutable architecture locked: one source, prebuilt PMTiles reads only, deterministic identity, ' + + 'fail-closed completeness, exact split-ancestor ownership, non-overlapping owners, and no ' + + 'runtime merge/synthesis/stretching.' +); diff --git a/scripts/check-photo-reference.mjs b/scripts/check-photo-reference.mjs index 471f4820..0dae6a68 100644 --- a/scripts/check-photo-reference.mjs +++ b/scripts/check-photo-reference.mjs @@ -83,7 +83,7 @@ assert(JSON.stringify(landcover?.paint?.['fill-opacity'] || []).includes('1'), ' assert(layer('landuse')?.paint?.['fill-opacity'] === 1, 'Detailed landuse swatches are being weakened by extra opacity.'); assert(layer('water')?.paint?.['fill-opacity'] === 1, 'Water must remain fully opaque.'); assert(layer('water-depth')?.['source-layer'] === 'depth', 'The reference bathymetry layer is missing.'); -for (const required of ['#79BCEC59', '#5AACE759', '#3B9DE359']) { +for (const required of ['#79BCEC', '#6EB6EA', '#63B1E9']) { assert( JSON.stringify(layer('water-depth')?.paint?.['fill-color'] || []).includes(required), `The exported bathymetry swatch ${required} is missing.` @@ -92,11 +92,8 @@ for (const required of ['#79BCEC59', '#5AACE759', '#3B9DE359']) { assert(!runtime.layers.some((candidate) => candidate.type === 'raster'), 'A raster fallback basemap was reintroduced.'); -const hillshade = layer('occumed-hillshade'); -assert(hillshade?.minzoom === 1.5, 'Hillshade must begin early enough to shape the globe.'); -assert(hillshade?.paint?.['hillshade-illumination-anchor'] === 'map', 'Hillshade must stay fixed to geography.'); -assert(hillshade?.paint?.['hillshade-shadow-color'] === '#0000004D', 'Hillshade shadows are tinting the exported palette.'); -assert(hillshade?.paint?.['hillshade-highlight-color'] === '#FFFFFF4D', 'Hillshade highlights are tinting the exported palette.'); +assert(!runtime.layers.some((candidate) => candidate.type === 'hillshade'), 'An external hillshade layer was reintroduced.'); +assert(Object.keys(runtime.sources || {}).length === 1, 'The reference style no longer has exactly one browser source.'); const paintedLayers = runtime.layers.filter((candidate) => candidate.paint); const allColors = collectHex(paintedLayers.map((candidate) => candidate.paint)); diff --git a/scripts/check-runtime.mjs b/scripts/check-runtime.mjs index 11ac673b..6f8421c5 100644 --- a/scripts/check-runtime.mjs +++ b/scripts/check-runtime.mjs @@ -31,9 +31,9 @@ if (runtime.metadata?.['occumed:glyph-rendering'] !== 'local-maplibre') { } if (!String(runtime.sprite).includes('/sprites/occumed')) fail('Local Occu-Med sprite endpoint is missing.'); -const allowedSources = new Set(['occumed-open', 'occumed-terrain']); -for (const sourceName of Object.keys(runtime.sources || {})) { - if (!allowedSources.has(sourceName)) fail(`Unexpected shared source: ${sourceName}`); +const sourceNames = Object.keys(runtime.sources || {}); +if (sourceNames.length !== 1 || sourceNames[0] !== 'occumed-open') { + fail(`Runtime must expose exactly one browser source (occumed-open); found ${sourceNames.join(', ') || 'none'}.`); } const requiredSourceLayers = new Set([ @@ -61,7 +61,7 @@ const minimumLayerCount = Math.max(100, Math.floor(original.layers.length * 0.7) if (runtime.layers.length < minimumLayerCount) { fail(`Runtime style is too incomplete: ${runtime.layers.length} layers; expected at least ${minimumLayerCount}.`); } -if (!runtime.layers.some((layer) => layer.id === 'occumed-hillshade')) fail('Open hillshade layer is missing.'); +if (runtime.layers.some((layer) => layer.type === 'hillshade')) fail('External terrain hillshade remains active.'); if (!runtime.layers.some((layer) => layer.id === 'occumed-land-surface')) fail('Worldwide land surface layer is missing.'); if (runtime.layers.some((layer) => layer.type === 'raster')) fail('The runtime contains a second raster basemap.'); if (runtime.layers.filter((layer) => layer.type === 'symbol').length < 20) { diff --git a/scripts/check-viewer-quality.mjs b/scripts/check-viewer-quality.mjs index cde6003a..697cb215 100644 --- a/scripts/check-viewer-quality.mjs +++ b/scripts/check-viewer-quality.mjs @@ -52,11 +52,8 @@ if (layer('water')?.paint?.['fill-opacity'] !== 1) fail('Water is translucent an if (runtime.layers.some((candidate) => candidate.type === 'raster')) fail('A raster fallback basemap was reintroduced.'); -const hillshade = layer('occumed-hillshade'); -if (hillshade?.minzoom !== 1.5) fail('Hillshade begins too late to give the globe physical form.'); -if (hillshade?.paint?.['hillshade-shadow-color'] !== '#0000004D') fail('Hillshade shadows are tinting exported colors.'); -if (hillshade?.paint?.['hillshade-highlight-color'] !== '#FFFFFF4D') fail('Hillshade highlights are tinting exported colors.'); -if (!JSON.stringify(hillshade?.paint?.['hillshade-exaggeration'] || []).includes('0.2')) fail('Regional terrain definition is too weak.'); +if (runtime.layers.some((candidate) => candidate.type === 'hillshade')) fail('An external hillshade layer was reintroduced.'); +if (Object.keys(runtime.sources || {}).length !== 1) fail('Viewer no longer uses exactly one browser source.'); const allColors = collectHex(runtime.layers.map((candidate) => candidate.paint || {})); if (allColors.size < 25) fail(`The exported per-structure palette was flattened to only ${allColors.size} colors.`); diff --git a/scripts/lock-exact-exported-swatches.mjs b/scripts/lock-exact-exported-swatches.mjs index 5018992b..f14ab0a1 100644 --- a/scripts/lock-exact-exported-swatches.mjs +++ b/scripts/lock-exact-exported-swatches.mjs @@ -30,9 +30,12 @@ const EXACT = Object.freeze({ wetland: '#A5CAD6', // supplied Studio chip, converted from Display-P3 water: '#79BCEC', // hsl(205, 75%, 70%) waterShadow: '#7293EE', // hsl(224, 79%, 69%) - depthShallow: '#79BCEC59', // hsla(205, 75%, 70%, 0.35) - depthMid: '#5AACE759', // hsla(205, 75%, 63%, 0.35) - depthDeep: '#3B9DE359' // hsla(205, 75%, 56%, 0.35) + // Preblended, opaque equivalents of the exported 35%-alpha bathymetry + // swatches over the permanent ocean background. Opaque depth is idempotent: + // nested bands cannot accumulate different alpha at tile boundaries. + depthShallow: '#79BCEC', + depthMid: '#6EB6EA', + depthDeep: '#63B1E9' }); function requireLayer(id) { @@ -151,31 +154,17 @@ water.paint['fill-color'] = EXACT.water; water.paint['fill-opacity'] = 1; const waterDepth = requireLayer('water-depth'); -waterDepth.maxzoom = 8; +delete waterDepth.maxzoom; waterDepth.paint['fill-antialias'] = false; waterDepth.paint['fill-color'] = [ 'interpolate', ['linear'], - ['zoom'], - 6, - [ - 'interpolate', - ['linear'], - ['get', 'min_depth'], - 0, EXACT.depthShallow, - 200, EXACT.depthMid, - 7000, EXACT.depthDeep - ], - 8, - [ - 'interpolate', - ['linear'], - ['get', 'min_depth'], - 0, '#79BCEC00', - 200, '#5AACE700', - 7000, '#2D96E100' - ] + ['get', 'min_depth'], + 0, EXACT.depthShallow, + 200, EXACT.depthMid, + 7000, EXACT.depthDeep ]; +waterDepth.paint['fill-opacity'] = 1; const waterway = requireLayer('waterway'); waterway.paint['line-color'] = EXACT.water; @@ -209,28 +198,9 @@ for (const group of REFERENCE_STUDIO_EXPRESSION_SWATCHES) { } } -// Use neutral light and shadow only. Terrain may change lightness, but never hue. -const hillshade = requireLayer('occumed-hillshade'); -hillshade.minzoom = 1.5; -hillshade.maxzoom = 16; -hillshade.paint = { - 'hillshade-exaggeration': [ - 'interpolate', - ['linear'], - ['zoom'], - 1.5, 0.04, - 5, 0.09, - 8, 0.15, - 11, 0.2, - 14, 0.24, - 16, 0.16 - ], - 'hillshade-shadow-color': '#0000004D', - 'hillshade-highlight-color': '#FFFFFF4D', - 'hillshade-accent-color': '#00000026', - 'hillshade-illumination-direction': 335, - 'hillshade-illumination-anchor': 'map' -}; +if (runtime.layers.some((candidate) => candidate.type === 'hillshade')) { + throw new Error('One-source runtime must not contain a hillshade layer.'); +} runtime.metadata = { ...(runtime.metadata || {}), @@ -241,6 +211,8 @@ runtime.metadata = { 'occumed:palette-source': 'supplied-mapbox-studio-screenshots-display-p3-to-srgb', 'occumed:layer-specific-palette': true, 'occumed:raster-relief-disabled': true, + 'occumed:external-terrain-disabled': true, + 'occumed:opaque-depth-palette': true, 'occumed:high-dpi-vector-clarity': true, 'occumed:exact-swatches': EXACT, 'occumed:unavailable-reference-swatches': REFERENCE_STUDIO_UNAVAILABLE_SWATCHES diff --git a/scripts/offline-tileset/build-archive.mjs b/scripts/offline-tileset/build-archive.mjs new file mode 100644 index 00000000..9c714979 --- /dev/null +++ b/scripts/offline-tileset/build-archive.mjs @@ -0,0 +1,190 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { zxyToTileId } from 'pmtiles'; +import { compileImmutableTile } from './mvt-normalizer.mjs'; +import { openLocalPmtiles } from './local-pmtiles.mjs'; +import { DeterministicPmtilesWriter } from './pmtiles-writer.mjs'; + +function parseArguments(argv) { + const result = { regional: [] }; + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (!token.startsWith('--')) throw new Error(`Unexpected argument: ${token}`); + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${token}.`); + const key = token.slice(2); + if (key === 'regional') result.regional.push(value); + else result[key] = value; + index += 1; + } + for (const required of ['targets', 'overview', 'surface', 'output', 'report', 'workdir']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +function parseRegional(value) { + const separator = value.indexOf('='); + if (separator <= 0) throw new Error(`Regional input must be asset=path: ${value}`); + return { + assetName: value.slice(0, separator), + filename: path.resolve(value.slice(separator + 1)) + }; +} + +function mergeCounts(target, report) { + for (const key of [ + 'rejectedMalformed', + 'rejectedOversized', + 'rejectedTileFootprints', + 'removedExactDuplicates', + 'removedContainedOverlaps', + 'clippedEmpty' + ]) { + target[key] = (target[key] || 0) + Number(report[key] || 0); + } + for (const [layer, counts] of Object.entries(report.layers || {})) { + target.layers[layer] ||= { featureCount: 0, pointCount: 0, tileCount: 0 }; + target.layers[layer].featureCount += Number(counts.featureCount || 0); + target.layers[layer].pointCount += Number(counts.pointCount || 0); + target.layers[layer].tileCount += 1; + } +} + +const options = parseArguments(process.argv.slice(2)); +const targetDocument = JSON.parse(await fs.readFile(path.resolve(options.targets), 'utf8')); +const targetName = options['target-name'] || 'foundation'; +const targets = targetDocument.targets?.[targetName] || targetDocument[targetName]; +if (!Array.isArray(targets) || !targets.length) { + throw new Error(`Target list is empty: ${targetName}`); +} +const ordered = [...targets] + .map(({ z, x, y }) => ({ z: Number(z), x: Number(x), y: Number(y) })) + .sort((left, right) => + zxyToTileId(left.z, left.x, left.y) - zxyToTileId(right.z, right.x, right.y) + ); +for (let index = 1; index < ordered.length; index += 1) { + if (zxyToTileId(ordered[index - 1].z, ordered[index - 1].x, ordered[index - 1].y) === + zxyToTileId(ordered[index].z, ordered[index].x, ordered[index].y)) { + throw new Error(`Duplicate target tile: ${ordered[index].z}/${ordered[index].x}/${ordered[index].y}`); + } +} + +const startedAt = performance.now(); +const overview = await openLocalPmtiles(path.resolve(options.overview)); +const surface = await openLocalPmtiles(path.resolve(options.surface)); +const regional = []; +for (const input of options.regional.map(parseRegional)) { + const opened = await openLocalPmtiles(input.filename); + opened.assetName = input.assetName; + regional.push(opened); +} +const writer = await new DeterministicPmtilesWriter({ + output: path.resolve(options.output), + workDirectory: path.resolve(options.workdir), + metadata: { + name: `Occu-Med immutable owner ${targetName}`, + type: 'baselayer', + format: 'pbf', + compression: 'gzip', + authority_land: 'world-surface', + authority_depth: 'world-surface', + authority_landcover: 'world-overview', + authority_cartography: 'regional-owner', + runtime_merge: false, + runtime_geometry: false + } +}).initialize(); + +const aggregate = { + rejectedMalformed: 0, + rejectedOversized: 0, + rejectedTileFootprints: 0, + removedExactDuplicates: 0, + removedContainedOverlaps: 0, + clippedEmpty: 0, + layers: {} +}; +let completed = 0; +const concurrency = Math.max(1, Math.min(8, Number(options.concurrency || 4))); + +try { + for (let start = 0; start < ordered.length; start += concurrency) { + const batch = ordered.slice(start, start + concurrency); + const compiled = await Promise.all( + batch.map((tile) => compileImmutableTile({ + ...tile, + overview, + surface, + regional + })) + ); + for (let index = 0; index < batch.length; index += 1) { + await writer.addTile({ ...batch[index], data: compiled[index].data }); + mergeCounts(aggregate, compiled[index].report); + completed += 1; + } + if (completed % 250 === 0 || completed === ordered.length) { + console.log(`[${targetName}] ${completed}/${ordered.length} tiles normalized.`); + } + } + const finalized = await writer.finalize(); + const durationSeconds = (performance.now() - startedAt) / 1_000; + const report = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + targetName, + file: options['manifest-file'] || path.basename(options.output), + bytes: finalized.bytes, + sha256: finalized.sha256, + addressedTiles: finalized.addressedTiles, + tileEntries: finalized.tileEntries, + tileContents: finalized.tileContents, + minZoom: finalized.minZoom, + maxZoom: Number(options['foundation-max-zoom'] || finalized.maxZoom), + durationSeconds, + authorities: { + land: 'world-surface', + depth: 'world-surface', + landcover: 'world-overview', + cartography: 'regional-owner' + }, + inputs: { + overview: path.basename(options.overview), + surface: path.basename(options.surface), + regional: regional.map((opened) => opened.assetName) + }, + normalization: aggregate + }; + if (options['owner-id']) { + const [z, x, y] = String(options['owner-prefix']).split('/').map(Number); + const selectedOwner = targetDocument.selectedOwners?.find( + (owner) => owner.id === options['owner-id'] + ); + report.owner = { + id: options['owner-id'], + prefix: { z, x, y }, + exactTiles: selectedOwner?.exactTiles || [] + }; + } + const reportPath = path.resolve(options.report); + await fs.mkdir(path.dirname(reportPath), { recursive: true }); + const pending = `${reportPath}.pending-${process.pid}`; + await fs.writeFile(pending, `${JSON.stringify(report, null, 2)}\n`, { flag: 'wx' }); + await fs.rename(pending, reportPath); + console.log( + `[${targetName}] built ${finalized.bytes} bytes in ${durationSeconds.toFixed(1)} seconds; ` + + `SHA-256 ${finalized.sha256}.` + ); +} catch (error) { + await writer.abort(); + throw error; +} finally { + await Promise.allSettled([ + overview.close(), + surface.close(), + ...regional.map((opened) => opened.close()) + ]); +} diff --git a/scripts/offline-tileset/build-foundation.mjs b/scripts/offline-tileset/build-foundation.mjs new file mode 100644 index 00000000..4a44be3c --- /dev/null +++ b/scripts/offline-tileset/build-foundation.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`Invalid argument near ${key || ''}.`); + } + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'targets', 'input-report', 'output-dir']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +async function runNode(script, args) { + await new Promise((resolve, reject) => { + const child = spawn(process.execPath, [script, ...args], { stdio: 'inherit' }); + child.on('error', reject); + child.on('exit', (code, signal) => { + if (signal) reject(new Error(`${path.basename(script)} terminated by ${signal}.`)); + else if (code !== 0) reject(new Error(`${path.basename(script)} exited with ${code}.`)); + else resolve(); + }); + }); +} + +const options = parseArguments(process.argv.slice(2)); +const [plan, targets, inputReport] = await Promise.all([ + fs.readFile(path.resolve(options.plan), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options.targets), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options['input-report']), 'utf8').then(JSON.parse) +]); +if (plan.planVersion !== targets.planVersion || plan.planVersion !== inputReport.planVersion) { + throw new Error('Plan, targets, and localized inputs do not share one immutable plan version.'); +} +const inputByAsset = new Map(inputReport.inputs.map((input) => [input.asset, input.path])); +const overview = inputByAsset.get(plan.inputs.overview.asset); +const surface = inputByAsset.get(plan.inputs.surface.asset); +if (!overview || !surface) throw new Error('Localized overview or surface input is missing.'); + +const outputDir = path.resolve(options['output-dir']); +await fs.mkdir(path.join(outputDir, 'reports'), { recursive: true }); +await fs.mkdir(path.join(outputDir, 'work'), { recursive: true }); +const output = path.join(outputDir, 'foundation.pmtiles'); +const report = path.join(outputDir, 'reports/foundation.json'); +if (await fs.stat(output).catch(() => null) || await fs.stat(report).catch(() => null)) { + throw new Error('Immutable foundation output already exists.'); +} +await runNode(path.join(scriptDirectory, 'build-archive.mjs'), [ + '--targets', path.resolve(options.targets), + '--target-name', 'foundation', + '--overview', overview, + '--surface', surface, + '--output', output, + '--report', report, + '--workdir', path.join(outputDir, 'work/foundation'), + '--manifest-file', 'foundation.pmtiles', + '--foundation-max-zoom', String(plan.routingZoom), + '--concurrency', String(options.concurrency || 4) +]); diff --git a/scripts/offline-tileset/build-owner.mjs b/scripts/offline-tileset/build-owner.mjs new file mode 100644 index 00000000..762716ba --- /dev/null +++ b/scripts/offline-tileset/build-owner.mjs @@ -0,0 +1,83 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`Invalid argument near ${key || ''}.`); + } + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'owner-id', 'targets', 'input-report', 'output-dir']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +async function runNode(script, args) { + await new Promise((resolve, reject) => { + const child = spawn(process.execPath, [script, ...args], { stdio: 'inherit' }); + child.on('error', reject); + child.on('exit', (code, signal) => { + if (signal) reject(new Error(`${path.basename(script)} terminated by ${signal}.`)); + else if (code !== 0) reject(new Error(`${path.basename(script)} exited with ${code}.`)); + else resolve(); + }); + }); +} + +const options = parseArguments(process.argv.slice(2)); +const [plan, targets, inputReport] = await Promise.all([ + fs.readFile(path.resolve(options.plan), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options.targets), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options['input-report']), 'utf8').then(JSON.parse) +]); +if (plan.planVersion !== targets.planVersion || plan.planVersion !== inputReport.planVersion) { + throw new Error('Plan, targets, and localized inputs do not share one immutable plan version.'); +} +const owner = plan.owners.find((candidate) => candidate.id === options['owner-id']); +if (!owner) throw new Error(`Owner is not present in the plan: ${options['owner-id']}`); +if (!targets.targets?.[owner.id]?.length) throw new Error(`Owner targets are empty: ${owner.id}`); +const inputByAsset = new Map(inputReport.inputs.map((input) => [input.asset, input.path])); +const overview = inputByAsset.get(plan.inputs.overview.asset); +const surface = inputByAsset.get(plan.inputs.surface.asset); +if (!overview || !surface) throw new Error('Localized overview or surface input is missing.'); +const regional = owner.candidates.flatMap((candidate) => { + const filename = inputByAsset.get(candidate.asset); + if (!filename) throw new Error(`Localized owner input is missing: ${candidate.asset}`); + return ['--regional', `${candidate.asset}=${filename}`]; +}); + +const outputDir = path.resolve(options['output-dir']); +await fs.mkdir(path.join(outputDir, 'owners'), { recursive: true }); +await fs.mkdir(path.join(outputDir, 'reports/owners'), { recursive: true }); +await fs.mkdir(path.join(outputDir, 'work'), { recursive: true }); +const output = path.join(outputDir, `owners/${owner.id}.pmtiles`); +const report = path.join(outputDir, `reports/owners/${owner.id}.json`); +if (await fs.stat(output).catch(() => null) || await fs.stat(report).catch(() => null)) { + throw new Error(`Immutable owner output already exists: ${owner.id}`); +} +await runNode(path.join(scriptDirectory, 'build-archive.mjs'), [ + '--targets', path.resolve(options.targets), + '--target-name', owner.id, + '--overview', overview, + '--surface', surface, + ...regional, + '--output', output, + '--report', report, + '--workdir', path.join(outputDir, `work/${owner.id}`), + '--manifest-file', `owners/${owner.id}.pmtiles`, + '--owner-id', owner.id, + '--owner-prefix', `${owner.prefix.z}/${owner.prefix.x}/${owner.prefix.y}`, + '--concurrency', String(options.concurrency || 4) +]); diff --git a/scripts/offline-tileset/build-representative.mjs b/scripts/offline-tileset/build-representative.mjs new file mode 100644 index 00000000..2a57334e --- /dev/null +++ b/scripts/offline-tileset/build-representative.mjs @@ -0,0 +1,140 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); +const FOUNDATION_WORLD_MAX_ZOOM = 6; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) throw new Error(`Unexpected argument: ${key}`); + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${key}.`); + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'targets', 'input-report', 'output-dir']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +async function runNode(script, args) { + await new Promise((resolve, reject) => { + const child = spawn(process.execPath, [script, ...args], { + stdio: 'inherit' + }); + child.on('error', reject); + child.on('exit', (code, signal) => { + if (signal) reject(new Error(`${path.basename(script)} terminated by ${signal}.`)); + else if (code !== 0) reject(new Error(`${path.basename(script)} exited with ${code}.`)); + else resolve(); + }); + }); +} + +const options = parseArguments(process.argv.slice(2)); +const planPath = path.resolve(options.plan); +const targetsPath = path.resolve(options.targets); +const inputReportPath = path.resolve(options['input-report']); +const outputDir = path.resolve(options['output-dir']); +const stat = await fs.stat(outputDir).catch(() => null); +if (stat) throw new Error(`Representative output directory already exists: ${outputDir}`); +await fs.mkdir(path.join(outputDir, 'owners'), { recursive: true }); +await fs.mkdir(path.join(outputDir, 'reports'), { recursive: true }); +await fs.mkdir(path.join(outputDir, 'work'), { recursive: true }); + +const [plan, targets, inputReport] = await Promise.all([ + fs.readFile(planPath, 'utf8').then(JSON.parse), + fs.readFile(targetsPath, 'utf8').then(JSON.parse), + fs.readFile(inputReportPath, 'utf8').then(JSON.parse) +]); +if (plan.planVersion !== inputReport.planVersion) { + throw new Error('Localized inputs do not match the owner plan.'); +} +const inputByAsset = new Map(inputReport.inputs.map((input) => [input.asset, input.path])); +const overview = inputByAsset.get(plan.inputs.overview.asset); +const surface = inputByAsset.get(plan.inputs.surface.asset); +if (!overview || !surface) throw new Error('Localized overview or surface input is missing.'); + +const buildScript = path.join(scriptDirectory, 'build-archive.mjs'); +const finalizeScript = path.join(scriptDirectory, 'finalize-manifest.mjs'); +const startedAt = performance.now(); +const foundationOutput = path.join(outputDir, 'foundation.pmtiles'); +const foundationReport = path.join(outputDir, 'reports/foundation.json'); +await runNode(buildScript, [ + '--targets', targetsPath, + '--target-name', 'foundation', + '--overview', overview, + '--surface', surface, + '--output', foundationOutput, + '--report', foundationReport, + '--workdir', path.join(outputDir, 'work/foundation'), + '--manifest-file', 'foundation.pmtiles', + '--foundation-max-zoom', String(FOUNDATION_WORLD_MAX_ZOOM) +]); + +const ownerReports = []; +for (const selected of targets.selectedOwners) { + const planOwner = plan.owners.find((owner) => owner.id === selected.id); + if (!planOwner) throw new Error(`Selected owner is not in the locked plan: ${selected.id}`); + const ownerOutput = path.join(outputDir, `owners/${selected.id}.pmtiles`); + const ownerReport = path.join(outputDir, `reports/${selected.id}.json`); + const regionalArgs = []; + const requiredNames = new Set( + (selected.requiredCandidates || planOwner.candidates).map((candidate) => candidate.asset) + ); + for (const candidate of planOwner.candidates.filter((item) => requiredNames.has(item.asset))) { + const filename = inputByAsset.get(candidate.asset); + if (!filename) throw new Error(`Localized owner input is missing: ${candidate.asset}`); + regionalArgs.push('--regional', `${candidate.asset}=${filename}`); + } + await runNode(buildScript, [ + '--targets', targetsPath, + '--target-name', selected.id, + '--overview', overview, + '--surface', surface, + ...regionalArgs, + '--output', ownerOutput, + '--report', ownerReport, + '--workdir', path.join(outputDir, `work/${selected.id}`), + '--manifest-file', `owners/${selected.id}.pmtiles`, + '--owner-id', selected.id, + '--owner-prefix', `${selected.prefix.z}/${selected.prefix.x}/${selected.prefix.y}` + ]); + ownerReports.push(ownerReport); +} + +const manifestPath = path.join(outputDir, 'manifest.json'); +await runNode(finalizeScript, [ + '--plan', planPath, + '--foundation', foundationReport, + ...ownerReports.flatMap((report) => ['--owner', report]), + '--output', manifestPath +]); +const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf8')); +const durationSeconds = (performance.now() - startedAt) / 1_000; +const buildReport = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + artifactVersion: manifest.artifactVersion, + durationSeconds, + totalBytes: manifest.totalBytes, + addressedTiles: targets.totalAddressedTiles, + foundationReport: path.relative(outputDir, foundationReport), + ownerReports: ownerReports.map((report) => path.relative(outputDir, report)), + manifest: path.relative(outputDir, manifestPath) +}; +await fs.writeFile( + path.join(outputDir, 'representative-build-report.json'), + `${JSON.stringify(buildReport, null, 2)}\n` +); +console.log( + `Representative immutable tileset ${manifest.artifactVersion} built in ` + + `${durationSeconds.toFixed(1)} seconds (${manifest.totalBytes} bytes).` +); diff --git a/scripts/offline-tileset/finalize-manifest.mjs b/scripts/offline-tileset/finalize-manifest.mjs new file mode 100644 index 00000000..a5bd1051 --- /dev/null +++ b/scripts/offline-tileset/finalize-manifest.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { computeImmutableArtifactVersion, validateImmutableManifest } from '../../src/server/immutable-world-tileset.js'; + +function parseArguments(argv) { + const result = { owners: [] }; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) throw new Error(`Unexpected argument: ${key}`); + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${key}.`); + const name = key.slice(2); + if (name === 'owner') result.owners.push(value); + else result[name] = value; + index += 1; + } + for (const required of ['plan', 'foundation', 'output']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +const options = parseArguments(process.argv.slice(2)); +const ownerReportFiles = [...options.owners]; +if (options['owner-dir']) { + const ownerDirectory = path.resolve(options['owner-dir']); + const entries = await fs.readdir(ownerDirectory, { withFileTypes: true }); + ownerReportFiles.push( + ...entries + .filter((entry) => entry.isFile() && entry.name.endsWith('.json')) + .map((entry) => path.join(ownerDirectory, entry.name)) + ); +} +const [plan, foundationReport, ...ownerReports] = await Promise.all([ + fs.readFile(path.resolve(options.plan), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options.foundation), 'utf8').then(JSON.parse), + ...[...new Set(ownerReportFiles)].map((filename) => + fs.readFile(path.resolve(filename), 'utf8').then(JSON.parse) + ) +]); +if (plan.schemaVersion !== 1 || !plan.planVersion) { + throw new Error('Immutable owner plan is invalid.'); +} + +const foundation = { + id: 'foundation', + file: foundationReport.file, + bytes: foundationReport.bytes, + sha256: foundationReport.sha256, + maxZoom: foundationReport.maxZoom +}; +const owners = ownerReports + .map((report) => ({ + id: report.owner.id, + prefix: report.owner.prefix, + exactTiles: report.owner.exactTiles || [], + file: report.file, + bytes: report.bytes, + sha256: report.sha256 + })) + .sort((left, right) => + left.prefix.z - right.prefix.z || + left.prefix.x - right.prefix.x || + left.prefix.y - right.prefix.y + ); +const complete = owners.length === plan.plannedOwnerCount; +const manifest = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + planVersion: plan.planVersion, + browserSourceId: 'occumed-open', + minZoom: 0, + maxZoom: 16, + complete, + validationFixture: !complete, + logicalOwnerCount: plan.logicalOwnerCount, + plannedOwnerCount: plan.plannedOwnerCount, + builtOwnerCount: owners.length, + defaultOwner: foundation.id, + totalBytes: foundation.bytes + owners.reduce((sum, owner) => sum + owner.bytes, 0), + authorities: { + land: 'world-surface', + depth: 'world-surface', + landcover: 'world-overview', + cartography: 'regional-owner' + }, + runtimePolicy: { + neonTileCache: false, + runtimeShardMerge: false, + runtimeLandcoverSynthesis: false, + runtimeGeometry: false, + parentChildStretching: false + }, + foundation, + owners +}; +if (options['asset-base-url']) manifest.assetBaseUrl = options['asset-base-url']; +manifest.artifactVersion = computeImmutableArtifactVersion(manifest); +validateImmutableManifest(manifest, { allowPartial: !complete }); + +const outputPath = path.resolve(options.output); +await fs.mkdir(path.dirname(outputPath), { recursive: true }); +const pending = `${outputPath}.pending-${process.pid}`; +await fs.writeFile(pending, `${JSON.stringify(manifest, null, 2)}\n`, { flag: 'wx' }); +await fs.rename(pending, outputPath); +console.log( + `Finalized immutable manifest ${manifest.artifactVersion}: ${owners.length} of ` + + `${plan.plannedOwnerCount} owners, ${manifest.totalBytes} bytes.` +); diff --git a/scripts/offline-tileset/generate-foundation-targets.mjs b/scripts/offline-tileset/generate-foundation-targets.mjs new file mode 100644 index 00000000..99caef25 --- /dev/null +++ b/scripts/offline-tileset/generate-foundation-targets.mjs @@ -0,0 +1,46 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`Invalid argument near ${key || ''}.`); + } + result[key.slice(2)] = value; + index += 1; + } + if (!result.plan || !result.output) throw new Error('--plan and --output are required.'); + return result; +} + +const options = parseArguments(process.argv.slice(2)); +const plan = JSON.parse(await fs.readFile(path.resolve(options.plan), 'utf8')); +const maxZoom = Number(options['max-zoom'] || plan.routingZoom); +if (!Number.isSafeInteger(maxZoom) || maxZoom < 0 || maxZoom > plan.routingZoom) { + throw new Error(`Foundation max zoom must be between 0 and ${plan.routingZoom}.`); +} + +const foundation = []; +for (let z = 0; z <= maxZoom; z += 1) { + const width = 2 ** z; + for (let y = 0; y < width; y += 1) { + for (let x = 0; x < width; x += 1) foundation.push({ z, x, y }); + } +} +const document = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + planVersion: plan.planVersion, + selectedOwners: [], + targets: { foundation }, + totalAddressedTiles: foundation.length +}; +const output = path.resolve(options.output); +await fs.mkdir(path.dirname(output), { recursive: true }); +await fs.writeFile(output, `${JSON.stringify(document, null, 2)}\n`, { flag: 'wx' }); +console.log(`Generated ${foundation.length} foundation targets through z${maxZoom}.`); diff --git a/scripts/offline-tileset/generate-owner-targets.mjs b/scripts/offline-tileset/generate-owner-targets.mjs new file mode 100644 index 00000000..1ccfc6a0 --- /dev/null +++ b/scripts/offline-tileset/generate-owner-targets.mjs @@ -0,0 +1,113 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { tileIdToZxy, zxyToTileId } from 'pmtiles'; +import { + openLocalPmtiles, + visitPmtilesTileAddresses +} from './local-pmtiles.mjs'; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`Invalid argument near ${key || ''}.`); + } + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'owner-id', 'input-report', 'output']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +function prefixContains(prefix, tile) { + if (tile.z < prefix.z) return false; + const divisor = 2 ** (tile.z - prefix.z); + return ( + Math.floor(tile.x / divisor) === prefix.x && + Math.floor(tile.y / divisor) === prefix.y + ); +} + +const options = parseArguments(process.argv.slice(2)); +const [plan, inputReport] = await Promise.all([ + fs.readFile(path.resolve(options.plan), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options['input-report']), 'utf8').then(JSON.parse) +]); +if (plan.planVersion !== inputReport.planVersion) { + throw new Error('Localized inputs do not match the immutable owner plan.'); +} +const owner = plan.owners.find((candidate) => candidate.id === options['owner-id']); +if (!owner) throw new Error(`Owner is not present in the plan: ${options['owner-id']}`); +if (!inputReport.ownerIds?.includes(owner.id)) { + throw new Error(`Input report was not localized for ${owner.id}.`); +} +const inputByAsset = new Map(inputReport.inputs.map((input) => [input.asset, input.path])); +const minZoom = Number(options['min-zoom'] || plan.routingZoom + 1); +const maxZoom = Number(options['max-zoom'] || 16); +const targetIds = new Set( + (owner.exactTiles || []).map(({ z, x, y }) => zxyToTileId(z, x, y)) +); +let scannedTiles = 0; +let scannedDirectories = 0; + +for (const candidate of owner.candidates) { + const filename = inputByAsset.get(candidate.asset); + if (!filename) throw new Error(`Localized owner input is missing: ${candidate.asset}`); + const opened = await openLocalPmtiles(filename, { cacheEntries: 2_048 }); + try { + const scanned = await visitPmtilesTileAddresses(opened, ({ z, x, y, tileId }) => { + if ( + z >= minZoom && + z <= maxZoom && + prefixContains(owner.prefix, { z, x, y }) + ) { + targetIds.add(tileId); + } + }); + scannedTiles += scanned.addressedTiles; + scannedDirectories += scanned.directoryCount; + } finally { + await opened.close(); + } +} + +const targets = [...targetIds] + .sort((left, right) => left - right) + .map((tileId) => { + const [z, x, y] = tileIdToZxy(tileId); + return { z, x, y }; + }); +if (!targets.length) throw new Error(`Owner ${owner.id} has no addressed output tiles.`); +const document = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + planVersion: plan.planVersion, + selectedOwners: [{ + id: owner.id, + prefix: owner.prefix, + exactTiles: owner.exactTiles || [], + requiredCandidates: owner.candidates + }], + targets: { + [owner.id]: targets + }, + totalAddressedTiles: targets.length, + inventory: { + candidateCount: owner.candidates.length, + scannedTiles, + scannedDirectories + } +}; +const output = path.resolve(options.output); +await fs.mkdir(path.dirname(output), { recursive: true }); +await fs.writeFile(output, `${JSON.stringify(document, null, 2)}\n`, { flag: 'wx' }); +console.log( + `Generated ${targets.length} exact targets for ${owner.id} from ` + + `${scannedTiles} addressed input tiles.` +); diff --git a/scripts/offline-tileset/generate-representative-targets.mjs b/scripts/offline-tileset/generate-representative-targets.mjs new file mode 100644 index 00000000..f75073a9 --- /dev/null +++ b/scripts/offline-tileset/generate-representative-targets.mjs @@ -0,0 +1,246 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { zxyToTileId } from 'pmtiles'; + +const VIEWPORT = { width: 1440, height: 1000 }; +const TILE_SIZE = 512; +const OVERSCAN_TILES = 3; +const FOUNDATION_WORLD_MAX_ZOOM = 6; +const MOTION_FRAMES = 10; +const BUILD_MOTION_SAMPLES = 120; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) throw new Error(`Unexpected argument: ${key}`); + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${key}.`); + result[key.slice(2)] = value; + index += 1; + } + if (!result.plan || !result.output) throw new Error('Required: --plan and --output.'); + return result; +} + +function lonToTileX(lon, zoom) { + return ((lon + 180) / 360) * 2 ** zoom; +} + +function latToTileY(lat, zoom) { + const clipped = Math.min(85.05112878, Math.max(-85.05112878, lat)); + const radians = clipped * Math.PI / 180; + return ((1 - Math.asinh(Math.tan(radians)) / Math.PI) / 2) * 2 ** zoom; +} + +function tileKey(tile) { + return `${tile.z}/${tile.x}/${tile.y}`; +} + +function cameraTiles({ center, zoom }) { + const z = Math.max(0, Math.min(16, Math.floor(zoom))); + const width = 2 ** z; + const centerX = Math.floor(lonToTileX(center[0], z)); + const centerY = Math.floor(latToTileY(center[1], z)); + const radiusX = Math.ceil(VIEWPORT.width / TILE_SIZE / 2) + OVERSCAN_TILES; + const radiusY = Math.ceil(VIEWPORT.height / TILE_SIZE / 2) + OVERSCAN_TILES; + const tiles = []; + for (let deltaY = -radiusY; deltaY <= radiusY; deltaY += 1) { + const y = centerY + deltaY; + if (y < 0 || y >= width) continue; + for (let deltaX = -radiusX; deltaX <= radiusX; deltaX += 1) { + const x = ((centerX + deltaX) % width + width) % width; + tiles.push({ z, x, y }); + } + } + return tiles; +} + +function ease(value) { + return value < 0.5 + ? 4 * value ** 3 + : 1 - ((-2 * value + 2) ** 3) / 2; +} + +function interpolateLongitude(start, end, amount) { + let delta = end - start; + if (delta > 180) delta -= 360; + if (delta < -180) delta += 360; + let value = start + delta * amount; + while (value > 180) value -= 360; + while (value < -180) value += 360; + return value; +} + +function motionCameras(start, end, count) { + return Array.from({ length: count }, (_, index) => { + const amount = count === 1 ? 1 : ease(index / (count - 1)); + return { + center: [ + interpolateLongitude(start.center[0], end.center[0], amount), + start.center[1] + (end.center[1] - start.center[1]) * amount + ], + zoom: start.zoom + (end.zoom - start.zoom) * amount + }; + }); +} + +function prefixContains(prefix, tile) { + if (prefix.z > tile.z) return false; + const divisor = 2 ** (tile.z - prefix.z); + return ( + Math.floor(tile.x / divisor) === prefix.x && + Math.floor(tile.y / divisor) === prefix.y + ); +} + +function tileBounds(tile) { + const width = 2 ** tile.z; + const west = (tile.x / width) * 360 - 180; + const east = ((tile.x + 1) / width) * 360 - 180; + const north = Math.atan(Math.sinh(Math.PI * (1 - (2 * tile.y) / width))) * 180 / Math.PI; + const south = Math.atan(Math.sinh(Math.PI * (1 - (2 * (tile.y + 1)) / width))) * 180 / Math.PI; + return [west, south, east, north]; +} + +function splitAntimeridianBounds(bounds) { + const [west, south, east, north] = bounds; + return west <= east + ? [[west, south, east, north]] + : [[west, south, 180, north], [-180, south, east, north]]; +} + +function boundsIntersect(left, right) { + return !( + left[2] <= right[0] || + left[0] >= right[2] || + left[3] <= right[1] || + left[1] >= right[3] + ); +} + +function candidateIntersectsTile(candidate, tile) { + const target = tileBounds(tile); + return splitAntimeridianBounds(candidate.bounds).some((part) => + boundsIntersect(part, target) + ); +} + +function ownerForPoint(plan, [lon, lat], zoom = 16) { + const tile = { + z: zoom, + x: Math.floor(lonToTileX(lon, zoom)), + y: Math.floor(latToTileY(lat, zoom)) + }; + const matches = plan.owners.filter((owner) => prefixContains(owner.prefix, tile)); + if (matches.length !== 1) { + throw new Error(`Expected one planned owner for ${lon},${lat}; found ${matches.length}.`); + } + return matches[0]; +} + +function sortedTiles(map) { + return [...map.values()].sort((left, right) => + zxyToTileId(left.z, left.x, left.y) - zxyToTileId(right.z, right.x, right.y) + ); +} + +const options = parseArguments(process.argv.slice(2)); +const plan = JSON.parse(await fs.readFile(path.resolve(options.plan), 'utf8')); +const global = { center: [-20, 18], zoom: 2.2 }; +const fresnoStreet = { center: [-119.7871, 36.7378], zoom: 16 }; +const antimeridianEast = { center: [179.65, 8], zoom: 8 }; +const antimeridianWest = { center: [-179.65, 8], zoom: 8 }; +const staticViews = [ + { name: 'global', ...global }, + { name: 'north-america', center: [-100, 40], zoom: 4 }, + { name: 'south-america', center: [-60, -15], zoom: 4 }, + { name: 'europe', center: [12, 50], zoom: 5 }, + { name: 'pacific', center: [155, 0], zoom: 5 }, + { name: 'antimeridian', center: [179.5, 8], zoom: 6 }, + { name: 'fresno-regional', center: fresnoStreet.center, zoom: 8 }, + { name: 'fresno-city', center: fresnoStreet.center, zoom: 12 }, + { name: 'fresno-street', ...fresnoStreet } +]; +const motions = [ + { name: 'global-to-street', start: global, end: fresnoStreet }, + { name: 'street-to-global', start: fresnoStreet, end: global }, + { name: 'antimeridian-crossing', start: antimeridianEast, end: antimeridianWest } +]; +const validationFrames = motions.flatMap((motion) => + motionCameras(motion.start, motion.end, MOTION_FRAMES) + .map((camera, index) => ({ motion: motion.name, index, ...camera })) +); +const buildCameras = [ + ...staticViews, + ...validationFrames, + ...motions.flatMap((motion) => + motionCameras(motion.start, motion.end, BUILD_MOTION_SAMPLES) + ) +]; + +const selectedOwners = [ + ownerForPoint(plan, fresnoStreet.center), + ownerForPoint(plan, antimeridianEast.center), + ownerForPoint(plan, antimeridianWest.center) +]; +const ownerById = new Map(selectedOwners.map((owner) => [owner.id, owner])); +if (ownerById.size !== 3) throw new Error('Representative target requires three distinct owners.'); +const targetMaps = Object.fromEntries([ + ['foundation', new Map()], + ...[...ownerById].map(([id]) => [id, new Map()]) +]); + +for (let z = 0; z <= FOUNDATION_WORLD_MAX_ZOOM; z += 1) { + const width = 2 ** z; + for (let y = 0; y < width; y += 1) { + for (let x = 0; x < width; x += 1) { + const tile = { z, x, y }; + targetMaps.foundation.set(tileKey(tile), tile); + } + } +} +for (const camera of buildCameras) { + for (const tile of cameraTiles(camera)) { + if (tile.z <= FOUNDATION_WORLD_MAX_ZOOM) { + targetMaps.foundation.set(tileKey(tile), tile); + continue; + } + const owner = [...ownerById.values()].find((candidate) => + prefixContains(candidate.prefix, tile) + ); + const target = owner ? targetMaps[owner.id] : targetMaps.foundation; + target.set(tileKey(tile), tile); + } +} + +const document = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + viewport: VIEWPORT, + staticViews, + validationFrames, + selectedOwners: [...ownerById.values()].map((owner) => ({ + ...owner, + requiredCandidates: owner.candidates.filter((candidate) => + [...targetMaps[owner.id].values()].some((tile) => + candidateIntersectsTile(candidate, tile) + ) + ) + })), + targets: Object.fromEntries( + Object.entries(targetMaps).map(([name, tiles]) => [name, sortedTiles(tiles)]) + ) +}; +document.totalAddressedTiles = Object.values(document.targets) + .reduce((sum, targets) => sum + targets.length, 0); + +const outputPath = path.resolve(options.output); +await fs.mkdir(path.dirname(outputPath), { recursive: true }); +await fs.writeFile(outputPath, `${JSON.stringify(document, null, 2)}\n`); +console.log( + `Generated ${document.totalAddressedTiles} exact representative tile targets across ` + + `${Object.keys(document.targets).length} non-overlapping owners.` +); diff --git a/scripts/offline-tileset/local-pmtiles.mjs b/scripts/offline-tileset/local-pmtiles.mjs new file mode 100644 index 00000000..fa7aabed --- /dev/null +++ b/scripts/offline-tileset/local-pmtiles.mjs @@ -0,0 +1,127 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { PMTiles, SharedPromiseCache, tileIdToZxy } from 'pmtiles'; + +/** + * Exact-range local source for the PMTiles JavaScript reader. + * + * Node Buffers may expose a larger backing ArrayBuffer than the requested + * range. PMTiles requires the returned ArrayBuffer to contain exactly the + * requested bytes, so every read is sliced to its own byteOffset/byteLength. + */ +export class LocalPmtilesSource { + constructor(filename) { + this.filename = path.resolve(filename); + this.handlePromise = fs.open(this.filename, 'r'); + this.sizePromise = this.handlePromise.then((handle) => handle.stat()).then((stat) => stat.size); + this.closed = false; + } + + getKey() { + return this.filename; + } + + async getBytes(offset, length) { + if (this.closed) throw new Error(`PMTiles source is closed: ${this.filename}`); + const start = Number(offset); + const size = Number(length); + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(size) || + start < 0 || + size <= 0 + ) { + throw new RangeError(`Invalid PMTiles range ${offset}+${length}.`); + } + + const [handle, fileSize] = await Promise.all([this.handlePromise, this.sizePromise]); + if (start >= fileSize) { + throw new RangeError(`PMTiles range starts beyond EOF: ${start} >= ${fileSize}.`); + } + const actualSize = Math.min(size, fileSize - start); + const buffer = Buffer.allocUnsafe(actualSize); + const { bytesRead } = await handle.read(buffer, 0, actualSize, start); + if (bytesRead !== actualSize) { + throw new Error( + `Short PMTiles read from ${this.filename}: expected ${actualSize}, received ${bytesRead}.` + ); + } + + return { + data: buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) + }; + } + + async close() { + if (this.closed) return; + this.closed = true; + await (await this.handlePromise).close(); + } +} + +export async function openLocalPmtiles(filename, { cacheEntries = 256 } = {}) { + const source = new LocalPmtilesSource(filename); + const archive = new PMTiles(source, new SharedPromiseCache(cacheEntries)); + const header = await archive.getHeader(); + return { + archive, + header, + source, + async close() { + await source.close(); + } + }; +} + +export function tilePayload(result) { + if (!result?.data) return null; + return Buffer.from( + result.data, + result.data.byteOffset || 0, + result.data.byteLength + ); +} + +/** + * Visit every addressed z/x/y in a local PMTiles archive without reading tile + * payloads. Run-length entries are expanded because every exact address must + * be assigned to one immutable output owner. + */ +export async function visitPmtilesTileAddresses(opened, visit) { + const { archive, header, source } = opened; + const visitedDirectories = new Set(); + let addressedTiles = 0; + + async function walk(offset, length, depth) { + if (depth > 4) throw new Error('PMTiles directory depth exceeds the v3 safety limit.'); + const key = `${offset}:${length}`; + if (visitedDirectories.has(key)) return; + visitedDirectories.add(key); + const entries = await archive.cache.getDirectory(source, offset, length, header); + for (const entry of entries) { + if (entry.runLength > 0) { + for (let delta = 0; delta < entry.runLength; delta += 1) { + const tileId = entry.tileId + delta; + const [z, x, y] = tileIdToZxy(tileId); + await visit({ z, x, y, tileId }); + addressedTiles += 1; + } + } else { + await walk( + header.leafDirectoryOffset + entry.offset, + entry.length, + depth + 1 + ); + } + } + } + + await walk(header.rootDirectoryOffset, header.rootDirectoryLength, 0); + return { + addressedTiles, + directoryCount: visitedDirectories.size + }; +} diff --git a/scripts/offline-tileset/localize-representative-inputs.mjs b/scripts/offline-tileset/localize-representative-inputs.mjs new file mode 100644 index 00000000..dabfbb26 --- /dev/null +++ b/scripts/offline-tileset/localize-representative-inputs.mjs @@ -0,0 +1,172 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) throw new Error(`Unexpected argument: ${key}`); + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${key}.`); + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'output-dir', 'report']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + if (!result.targets && !result['owner-id']) { + throw new Error('Either --targets or --owner-id is required.'); + } + return result; +} + +async function hashFile(filename) { + const hash = createHash('sha256'); + const handle = await fs.open(filename, 'r'); + try { + for await (const chunk of handle.createReadStream()) hash.update(chunk); + } finally { + await handle.close().catch(() => {}); + } + return hash.digest('hex'); +} + +function releaseUrl(plan, asset) { + return `https://github.com/${plan.repository}/releases/download/` + + `${encodeURIComponent(plan.releaseTag)}/${encodeURIComponent(asset)}`; +} + +async function validLockedFile(filename, lock) { + try { + const stat = await fs.stat(filename); + return ( + stat.isFile() && + stat.size === lock.bytes && + await hashFile(filename) === lock.sha256 + ); + } catch { + return false; + } +} + +async function downloadLocked(plan, lock, outputDir) { + const destination = path.join(outputDir, lock.asset); + if (await validLockedFile(destination, lock)) { + console.log(`Using SHA-locked input ${lock.asset}.`); + return { + asset: lock.asset, + path: destination, + bytes: lock.bytes, + sha256: lock.sha256, + reused: true + }; + } + + const temporary = `${destination}.pending-${process.pid}-${Date.now()}`; + const response = await fetch(lock.url || releaseUrl(plan, lock.asset), { + redirect: 'follow', + headers: { 'User-Agent': 'Occu-Med-Map/offline-input-localizer' } + }); + if (!response.ok || !response.body) { + throw new Error(`Unable to download ${lock.asset}: HTTP ${response.status}.`); + } + const declared = Number(response.headers.get('content-length')); + if (Number.isFinite(declared) && declared !== lock.bytes) { + throw new Error( + `Published byte count changed for ${lock.asset}: expected ${lock.bytes}, received ${declared}.` + ); + } + + const handle = await fs.open(temporary, 'wx'); + const hash = createHash('sha256'); + let bytes = 0; + let nextProgress = 256 * 1024 * 1024; + try { + for await (const chunk of response.body) { + const buffer = Buffer.from(chunk); + await handle.write(buffer); + hash.update(buffer); + bytes += buffer.byteLength; + if (bytes >= nextProgress) { + console.log(`[${lock.asset}] ${bytes}/${lock.bytes} bytes downloaded.`); + nextProgress += 256 * 1024 * 1024; + } + } + await handle.sync(); + } finally { + await handle.close(); + } + const digest = hash.digest('hex'); + if (bytes !== lock.bytes || digest !== lock.sha256) { + throw new Error( + `Input lock failed for ${lock.asset}: ${bytes}/${lock.bytes} bytes, ${digest}/${lock.sha256}.` + ); + } + await fs.rename(temporary, destination); + const verified = await validLockedFile(destination, lock); + if (!verified) throw new Error(`Promoted input failed verification: ${lock.asset}.`); + console.log(`Localized ${lock.asset}: ${bytes} bytes, SHA-256 ${digest}.`); + return { + asset: lock.asset, + path: destination, + bytes, + sha256: digest, + reused: false + }; +} + +const options = parseArguments(process.argv.slice(2)); +const plan = JSON.parse(await fs.readFile(path.resolve(options.plan), 'utf8')); +const targets = options.targets + ? JSON.parse(await fs.readFile(path.resolve(options.targets), 'utf8')) + : null; +const selectedIds = new Set( + options['owner-id'] + ? [options['owner-id']] + : (targets.selectedOwners || []).map((owner) => owner.id) +); +const selected = plan.owners.filter((owner) => selectedIds.has(owner.id)); +if (selected.length !== selectedIds.size) throw new Error('Representative owner selection is not in the locked plan.'); + +const locks = new Map(); +for (const input of [plan.inputs.overview, plan.inputs.surface]) locks.set(input.asset, input); +for (const selectedOwner of selected) { + const owner = selected.find((candidate) => candidate.id === selectedOwner.id); + if (!owner) throw new Error(`Selected owner is missing from plan: ${selectedOwner.id}`); + const requiredNames = new Set( + ( + targets?.selectedOwners?.find((candidate) => candidate.id === selectedOwner.id) + ?.requiredCandidates || + owner.candidates + ).map((candidate) => candidate.asset) + ); + for (const candidate of owner.candidates.filter((item) => requiredNames.has(item.asset))) { + locks.set(candidate.asset, { + ...candidate, + url: releaseUrl(plan, candidate.asset) + }); + } +} + +const outputDir = path.resolve(options['output-dir']); +await fs.mkdir(outputDir, { recursive: true }); +const results = []; +for (const lock of [...locks.values()].sort((left, right) => left.asset.localeCompare(right.asset))) { + results.push(await downloadLocked(plan, lock, outputDir)); +} +const report = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + planVersion: plan.planVersion, + ownerIds: [...selectedIds].sort(), + inputCount: results.length, + totalBytes: results.reduce((sum, result) => sum + result.bytes, 0), + inputs: results +}; +const reportPath = path.resolve(options.report); +await fs.mkdir(path.dirname(reportPath), { recursive: true }); +await fs.writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`); +console.log(`Localized ${results.length} locked inputs totaling ${report.totalBytes} bytes.`); diff --git a/scripts/offline-tileset/mvt-normalizer.mjs b/scripts/offline-tileset/mvt-normalizer.mjs new file mode 100644 index 00000000..82a85b73 --- /dev/null +++ b/scripts/offline-tileset/mvt-normalizer.mjs @@ -0,0 +1,765 @@ +import { createHash } from 'node:crypto'; +import { VectorTile } from '@mapbox/vector-tile'; +import Pbf from 'pbf'; +import vtpbf from 'vt-pbf'; +import { validateVectorTilePayload } from '../../src/server/tile-safety.js'; + +const DEFAULT_EXTENT = 4096; +const INPUT_COORDINATE_SCALE = 128; +const MAX_FEATURE_POINTS = 250_000; +const MAX_TILE_POINTS = 2_000_000; +const SURFACE_RECTANGLE_LAYERS = new Set([ + 'landcover', + 'landuse', + 'park', + 'protected_area', + 'water' +]); + +const LAYER_ALIASES = new Map([ + ['aerodrome', 'aerodrome_label'], + ['aerodrome_label', 'aerodrome_label'], + ['aeroway', 'aeroway'], + ['admin', 'boundary'], + ['boundary', 'boundary'], + ['building', 'building'], + ['buildings', 'building'], + ['depth', 'depth'], + ['housenum_label', 'housenumber'], + ['housenumber', 'housenumber'], + ['land', 'land'], + ['land_cover', 'landcover'], + ['landcover', 'landcover'], + ['landuse', 'landuse'], + ['landuse_overlay', 'landuse'], + ['mountain_peak', 'mountain_peak'], + ['park', 'park'], + ['place', 'place'], + ['place_label', 'place'], + ['poi', 'poi'], + ['poi_label', 'poi'], + ['protected_area', 'park'], + ['road', 'transportation'], + ['road_label', 'transportation_name'], + ['transportation', 'transportation'], + ['transportation_name', 'transportation_name'], + ['water', 'water'], + ['water_name', 'water_name'], + ['waterway', 'waterway'] +]); + +function canonicalLayerName(name) { + const normalized = String(name || '').trim().toLowerCase().replaceAll('-', '_'); + return LAYER_ALIASES.get(normalized) || normalized; +} + +function normalizePropertyValue(value) { + if (typeof value === 'bigint') return value.toString(); + if ( + typeof value === 'number' && + (!Number.isFinite(value) || (Number.isInteger(value) && !Number.isSafeInteger(value))) + ) { + return String(value); + } + return value; +} + +function normalizeProperties(layerName, properties = {}) { + const output = Object.fromEntries( + Object.entries(properties) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => [key, normalizePropertyValue(value)]) + ); + if (output['name_en'] && !output['name:en']) output['name:en'] = output['name_en']; + if (output.name && !output['name:latin']) output['name:latin'] = output.name; + if (layerName === 'landcover') { + if (output.class === 'farmland') output.class = 'crop'; + if (output.class === 'ice') output.class = 'snow'; + output.class ||= 'grass'; + } + if ( + ['transportation', 'transportation_name'].includes(layerName) && + !output.class && + output.type + ) { + output.class = output.type; + } + if (layerName === 'building') { + if (output.height !== undefined && output.render_height === undefined) { + output.render_height = output.height; + } + if (output.min_height !== undefined && output.render_min_height === undefined) { + output.render_min_height = output.min_height; + } + } + return output; +} + +function pointEqual(left, right) { + return left.x === right.x && left.y === right.y; +} + +function roundPoint(point) { + return { x: Math.round(point.x), y: Math.round(point.y) }; +} + +function dedupePoints(points, { close = false } = {}) { + const result = []; + for (const raw of points) { + const point = roundPoint(raw); + if (!result.length || !pointEqual(result.at(-1), point)) result.push(point); + } + while (result.length > 1 && pointEqual(result[0], result.at(-1))) result.pop(); + if (close && result.length >= 3) result.push({ ...result[0] }); + return result; +} + +function interpolateAtX(start, end, x) { + const delta = end.x - start.x; + if (delta === 0) return { x, y: start.y }; + const ratio = (x - start.x) / delta; + return { x, y: start.y + (end.y - start.y) * ratio }; +} + +function interpolateAtY(start, end, y) { + const delta = end.y - start.y; + if (delta === 0) return { x: start.x, y }; + const ratio = (y - start.y) / delta; + return { x: start.x + (end.x - start.x) * ratio, y }; +} + +function clipPolygonBoundary(points, inside, intersect) { + if (!points.length) return []; + const result = []; + let previous = points.at(-1); + let previousInside = inside(previous); + for (const current of points) { + const currentInside = inside(current); + if (currentInside) { + if (!previousInside) result.push(intersect(previous, current)); + result.push(current); + } else if (previousInside) { + result.push(intersect(previous, current)); + } + previous = current; + previousInside = currentInside; + } + return result; +} + +function clipPolygonRing(points, extent) { + if (points.length < 3) return []; + let ring = pointEqual(points[0], points.at(-1)) ? points.slice(0, -1) : [...points]; + ring = clipPolygonBoundary(ring, (point) => point.x >= 0, (a, b) => interpolateAtX(a, b, 0)); + ring = clipPolygonBoundary(ring, (point) => point.x <= extent, (a, b) => interpolateAtX(a, b, extent)); + ring = clipPolygonBoundary(ring, (point) => point.y >= 0, (a, b) => interpolateAtY(a, b, 0)); + ring = clipPolygonBoundary(ring, (point) => point.y <= extent, (a, b) => interpolateAtY(a, b, extent)); + ring = dedupePoints(ring, { close: true }); + return ring.length >= 4 ? ring : []; +} + +function regionCode(point, extent) { + let code = 0; + if (point.x < 0) code |= 1; + else if (point.x > extent) code |= 2; + if (point.y < 0) code |= 4; + else if (point.y > extent) code |= 8; + return code; +} + +function clipLineSegment(start, end, extent) { + let left = { ...start }; + let right = { ...end }; + let leftCode = regionCode(left, extent); + let rightCode = regionCode(right, extent); + while (true) { + if (!(leftCode | rightCode)) return [left, right]; + if (leftCode & rightCode) return null; + const code = leftCode || rightCode; + let point; + if (code & 8) point = interpolateAtY(left, right, extent); + else if (code & 4) point = interpolateAtY(left, right, 0); + else if (code & 2) point = interpolateAtX(left, right, extent); + else point = interpolateAtX(left, right, 0); + if (code === leftCode) { + left = point; + leftCode = regionCode(left, extent); + } else { + right = point; + rightCode = regionCode(right, extent); + } + } +} + +function clipLineString(points, extent) { + const fragments = []; + let current = []; + for (let index = 1; index < points.length; index += 1) { + const clipped = clipLineSegment(points[index - 1], points[index], extent); + if (!clipped) { + if (current.length >= 2) fragments.push(dedupePoints(current)); + current = []; + continue; + } + const [start, end] = clipped; + if (!current.length) current.push(start, end); + else if (pointEqual(roundPoint(current.at(-1)), roundPoint(start))) current.push(end); + else { + if (current.length >= 2) fragments.push(dedupePoints(current)); + current = [start, end]; + } + } + if (current.length >= 2) fragments.push(dedupePoints(current)); + return fragments.filter((fragment) => fragment.length >= 2); +} + +function transformAndClipGeometry(feature, { + scale, + offsetX, + offsetY, + extent +}) { + const transformed = []; + let pointCount = 0; + for (const part of feature.loadGeometry()) { + const points = []; + for (const point of part) { + if ( + !Number.isFinite(point.x) || + !Number.isFinite(point.y) || + Math.abs(point.x) > feature.extent * INPUT_COORDINATE_SCALE || + Math.abs(point.y) > feature.extent * INPUT_COORDINATE_SCALE + ) { + return { geometry: [], pointCount, malformed: true }; + } + points.push({ + x: point.x * scale - offsetX, + y: point.y * scale - offsetY + }); + pointCount += 1; + if (pointCount > MAX_FEATURE_POINTS) { + return { geometry: [], pointCount, oversized: true }; + } + } + if (feature.type === 1) { + transformed.push( + ...points + .filter((point) => point.x >= 0 && point.x <= extent && point.y >= 0 && point.y <= extent) + .map((point) => [roundPoint(point)]) + ); + } else if (feature.type === 2) { + transformed.push(...clipLineString(points, extent)); + } else if (feature.type === 3) { + const ring = clipPolygonRing(points, extent); + if (ring.length) transformed.push(ring); + } + } + return { geometry: transformed, pointCount }; +} + +function geometryBounds(geometry) { + let minX = Infinity; + let minY = Infinity; + let maxX = -Infinity; + let maxY = -Infinity; + let points = 0; + for (const part of geometry) { + for (const point of part) { + minX = Math.min(minX, point.x); + minY = Math.min(minY, point.y); + maxX = Math.max(maxX, point.x); + maxY = Math.max(maxY, point.y); + points += 1; + } + } + return points ? { minX, minY, maxX, maxY, points } : null; +} + +function isLargeAxisAlignedRectangle(feature, extent) { + if (feature.type !== 3 || !SURFACE_RECTANGLE_LAYERS.has(feature.layerName)) return false; + const bounds = feature.bounds; + if (!bounds) return false; + const width = bounds.maxX - bounds.minX; + const height = bounds.maxY - bounds.minY; + if (width <= 0 || height <= 0 || (width * height) / (extent * extent) < 0.06) { + return false; + } + + const points = feature.geometry.flat(); + if (points.length < 4) return false; + const epsilon = 1; + const onBoundary = (point) => + Math.abs(point.x - bounds.minX) <= epsilon || + Math.abs(point.x - bounds.maxX) <= epsilon || + Math.abs(point.y - bounds.minY) <= epsilon || + Math.abs(point.y - bounds.maxY) <= epsilon; + if (!points.every(onBoundary)) return false; + for (let index = 1; index < points.length; index += 1) { + const previous = points[index - 1]; + const current = points[index]; + if ( + Math.abs(previous.x - current.x) > epsilon && + Math.abs(previous.y - current.y) > epsilon + ) { + return false; + } + } + return true; +} + +function stableProperties(properties) { + return JSON.stringify( + Object.entries(properties) + .sort(([left], [right]) => left.localeCompare(right)) + ); +} + +function geometrySignature(feature) { + const hash = createHash('sha1'); + hash.update(`${feature.type}|${stableProperties(feature.properties)}|`); + for (const part of feature.geometry) { + hash.update('['); + for (const point of part) hash.update(`${point.x},${point.y};`); + hash.update(']'); + } + return hash.digest('base64url'); +} + +function boxesIntersect(left, right) { + return !( + left.maxX < right.minX || + left.minX > right.maxX || + left.maxY < right.minY || + left.minY > right.maxY + ); +} + +function orientation(a, b, c) { + return Math.sign((b.y - a.y) * (c.x - b.x) - (b.x - a.x) * (c.y - b.y)); +} + +function pointOnSegment(a, b, point) { + return ( + point.x >= Math.min(a.x, b.x) && + point.x <= Math.max(a.x, b.x) && + point.y >= Math.min(a.y, b.y) && + point.y <= Math.max(a.y, b.y) && + orientation(a, b, point) === 0 + ); +} + +function segmentsIntersect(a, b, c, d) { + const o1 = orientation(a, b, c); + const o2 = orientation(a, b, d); + const o3 = orientation(c, d, a); + const o4 = orientation(c, d, b); + if (o1 !== o2 && o3 !== o4) return true; + return ( + (o1 === 0 && pointOnSegment(a, b, c)) || + (o2 === 0 && pointOnSegment(a, b, d)) || + (o3 === 0 && pointOnSegment(c, d, a)) || + (o4 === 0 && pointOnSegment(c, d, b)) + ); +} + +function pointInRing(point, ring) { + let inside = false; + for (let index = 0, previous = ring.length - 1; index < ring.length; previous = index++) { + const a = ring[index]; + const b = ring[previous]; + if (pointOnSegment(a, b, point)) return true; + const intersects = + (a.y > point.y) !== (b.y > point.y) && + point.x < ((b.x - a.x) * (point.y - a.y)) / (b.y - a.y || 1) + a.x; + if (intersects) inside = !inside; + } + return inside; +} + +function ringsCross(left, right) { + for (let leftIndex = 1; leftIndex < left.length; leftIndex += 1) { + for (let rightIndex = 1; rightIndex < right.length; rightIndex += 1) { + if ( + segmentsIntersect( + left[leftIndex - 1], + left[leftIndex], + right[rightIndex - 1], + right[rightIndex] + ) + ) { + return true; + } + } + } + return false; +} + +function polygonContains(container, candidate) { + if (!boxesIntersect(container.bounds, candidate.bounds)) return false; + if ( + container.bounds.minX > candidate.bounds.minX || + container.bounds.minY > candidate.bounds.minY || + container.bounds.maxX < candidate.bounds.maxX || + container.bounds.maxY < candidate.bounds.maxY + ) { + return false; + } + const containerRing = container.geometry[0]; + const candidateRing = candidate.geometry[0]; + if (!containerRing?.length || !candidateRing?.length) return false; + if (ringsCross(containerRing, candidateRing)) return false; + return candidateRing.slice(0, -1).every((point) => pointInRing(point, containerRing)); +} + +function bucketKeys(bounds, extent, bucketCount = 16) { + const size = extent / bucketCount; + const minX = Math.max(0, Math.min(bucketCount - 1, Math.floor(bounds.minX / size))); + const minY = Math.max(0, Math.min(bucketCount - 1, Math.floor(bounds.minY / size))); + const maxX = Math.max(0, Math.min(bucketCount - 1, Math.floor(bounds.maxX / size))); + const maxY = Math.max(0, Math.min(bucketCount - 1, Math.floor(bounds.maxY / size))); + const keys = []; + for (let x = minX; x <= maxX; x += 1) { + for (let y = minY; y <= maxY; y += 1) keys.push(`${x}/${y}`); + } + return keys; +} + +function dedupeAndRejectLayer(features, extent, report) { + const ordered = [...features].sort((left, right) => { + const properties = stableProperties(left.properties).localeCompare(stableProperties(right.properties)); + return properties || geometrySignature(left).localeCompare(geometrySignature(right)); + }); + const accepted = []; + const exact = new Set(); + const buckets = new Map(); + + for (const feature of ordered) { + if (isLargeAxisAlignedRectangle(feature, extent)) { + report.rejectedTileFootprints += 1; + continue; + } + const signature = geometrySignature(feature); + if (exact.has(signature)) { + report.removedExactDuplicates += 1; + continue; + } + + if (feature.type === 3) { + const properties = stableProperties(feature.properties); + const candidates = new Set(); + for (const key of bucketKeys(feature.bounds, extent)) { + for (const index of buckets.get(key) || []) candidates.add(index); + } + let contained = false; + for (const index of candidates) { + const existing = accepted[index]; + if (!existing || existing.removed || stableProperties(existing.properties) !== properties) continue; + if (polygonContains(existing, feature)) { + contained = true; + report.removedContainedOverlaps += 1; + break; + } + if (polygonContains(feature, existing)) { + existing.removed = true; + report.removedContainedOverlaps += 1; + } + } + if (contained) continue; + } + + exact.add(signature); + const index = accepted.length; + accepted.push(feature); + if (feature.type === 3) { + for (const key of bucketKeys(feature.bounds, extent)) { + const list = buckets.get(key) || []; + list.push(index); + buckets.set(key, list); + } + } + } + + const result = accepted.filter((feature) => !feature.removed); + if (result[0]?.layerName === 'depth') { + result.sort((left, right) => { + const leftDepth = Number(left.properties.min_depth || 0); + const rightDepth = Number(right.properties.min_depth || 0); + return leftDepth - rightDepth || geometrySignature(left).localeCompare(geometrySignature(right)); + }); + } + return result; +} + +class EncodedFeature { + constructor(feature) { + this.id = feature.id; + this.type = feature.type; + this.properties = feature.properties; + this.extent = feature.extent; + this.geometry = feature.geometry; + } + + loadGeometry() { + return this.geometry; + } +} + +class EncodedLayer { + constructor(name, features, extent = DEFAULT_EXTENT) { + this.name = name; + this.version = 2; + this.extent = extent; + this.features = features.map((feature) => new EncodedFeature(feature)); + this.length = this.features.length; + } + + feature(index) { + return this.features[index]; + } +} + +function decodeTile(payload, label) { + const bytes = validateVectorTilePayload(payload, { + label, + coordinateScale: INPUT_COORDINATE_SCALE, + maxBytes: 32 * 1024 * 1024 + }).bytes; + return new VectorTile(new Pbf(new Uint8Array(bytes))); +} + +/** + * Normalize selected source layers from one MVT into an exact target z/x/y. + */ +export function normalizeTileContribution(payload, { + sourceZoom, + targetZoom, + targetX, + targetY, + includeLayers, + excludeLayers, + authority, + report +}) { + if (!payload) return []; + const tile = decodeTile(payload, `offline ${authority} input`); + const include = includeLayers ? new Set(includeLayers.map(canonicalLayerName)) : null; + const exclude = excludeLayers ? new Set(excludeLayers.map(canonicalLayerName)) : null; + const scale = 2 ** (targetZoom - sourceZoom); + if (scale < 1) throw new Error('Offline materialization cannot downscale from a child tile.'); + const childX = ((targetX % scale) + scale) % scale; + const childY = ((targetY % scale) + scale) % scale; + const output = []; + let tilePoints = 0; + + for (const [inputName, layer] of Object.entries(tile.layers)) { + const layerName = canonicalLayerName(inputName); + if (include && !include.has(layerName)) continue; + if (exclude?.has(layerName)) continue; + for (let index = 0; index < layer.length; index += 1) { + const sourceFeature = layer.feature(index); + const normalized = transformAndClipGeometry(sourceFeature, { + scale, + offsetX: childX * layer.extent, + offsetY: childY * layer.extent, + extent: layer.extent + }); + if (normalized.malformed) { + report.rejectedMalformed += 1; + continue; + } + if (normalized.oversized) { + report.rejectedOversized += 1; + continue; + } + if (!normalized.geometry.length) { + report.clippedEmpty += 1; + continue; + } + tilePoints += normalized.geometry.reduce( + (sum, part) => sum + part.length, + 0 + ); + if (tilePoints > MAX_TILE_POINTS) { + throw new Error(`Offline tile exceeded ${MAX_TILE_POINTS} normalized geometry points.`); + } + const feature = { + id: sourceFeature.id, + type: sourceFeature.type, + properties: normalizeProperties(layerName, sourceFeature.properties), + extent: layer.extent, + geometry: normalized.geometry, + bounds: geometryBounds(normalized.geometry), + layerName, + authority + }; + output.push(feature); + } + } + return output; +} + +export function encodeNormalizedTile(contributions, { + label = 'offline normalized tile', + report = {} +} = {}) { + Object.assign(report, { + rejectedMalformed: report.rejectedMalformed || 0, + rejectedOversized: report.rejectedOversized || 0, + rejectedTileFootprints: report.rejectedTileFootprints || 0, + removedExactDuplicates: report.removedExactDuplicates || 0, + removedContainedOverlaps: report.removedContainedOverlaps || 0, + clippedEmpty: report.clippedEmpty || 0 + }); + const grouped = new Map(); + for (const feature of contributions) { + const list = grouped.get(feature.layerName) || []; + list.push(feature); + grouped.set(feature.layerName, list); + } + + const layers = {}; + for (const [layerName, features] of [...grouped.entries()].sort(([left], [right]) => left.localeCompare(right))) { + const extent = features[0]?.extent || DEFAULT_EXTENT; + const normalized = dedupeAndRejectLayer(features, extent, report); + if (normalized.length) layers[layerName] = new EncodedLayer(layerName, normalized, extent); + } + const encoded = Buffer.from(vtpbf.fromVectorTileJs({ layers })); + validateVectorTilePayload(encoded, { + label, + coordinateScale: 1, + maxBytes: 32 * 1024 * 1024, + maxTotalPoints: MAX_TILE_POINTS + }); + report.layers = Object.fromEntries( + Object.entries(layers).map(([name, layer]) => [ + name, + { + featureCount: layer.length, + pointCount: layer.features.reduce( + (sum, feature) => sum + feature.geometry.reduce((partSum, part) => partSum + part.length, 0), + 0 + ) + } + ]) + ); + report.bytes = encoded.byteLength; + return encoded; +} + +export async function readTileOrAncestor(openedArchive, z, x, y) { + const maxZoom = Number(openedArchive.header.maxZoom); + const sourceZoom = Math.min(z, maxZoom); + const divisor = 2 ** (z - sourceZoom); + const sourceX = Math.floor(x / divisor); + const sourceY = Math.floor(y / divisor); + const result = await openedArchive.archive.getZxy(sourceZoom, sourceX, sourceY); + return { + payload: result?.data ? Buffer.from(result.data) : null, + sourceZoom, + sourceX, + sourceY + }; +} + +/** + * Compile one final immutable tile with a strict offline authority matrix: + * surface -> land/depth, overview -> landcover, regional -> cartography. + */ +export async function compileImmutableTile({ + z, + x, + y, + surface, + overview, + regional = [] +}) { + const report = { + tile: { z, x, y }, + authorities: { + land: 'world-surface', + depth: 'world-surface', + landcover: 'world-overview', + cartography: 'regional-owner' + }, + policy: { + syntheticLandcover: false, + runtimeMerge: false, + runtimeGeometry: false, + parentChildStretching: false + }, + rejectedMalformed: 0, + rejectedOversized: 0, + rejectedTileFootprints: 0, + removedExactDuplicates: 0, + removedContainedOverlaps: 0, + clippedEmpty: 0 + }; + const [surfaceTile, overviewTile] = await Promise.all([ + readTileOrAncestor(surface, z, x, y), + readTileOrAncestor(overview, z, x, y) + ]); + const contributions = []; + if (surfaceTile.payload) { + contributions.push( + ...normalizeTileContribution(surfaceTile.payload, { + sourceZoom: surfaceTile.sourceZoom, + targetZoom: z, + targetX: x, + targetY: y, + includeLayers: ['land', 'depth'], + authority: 'world-surface', + report + }) + ); + } + if (overviewTile.payload) { + contributions.push( + ...normalizeTileContribution(overviewTile.payload, { + sourceZoom: overviewTile.sourceZoom, + targetZoom: z, + targetX: x, + targetY: y, + includeLayers: ['landcover'], + authority: 'world-overview', + report + }) + ); + if (z <= Number(overview.header.maxZoom)) { + contributions.push( + ...normalizeTileContribution(overviewTile.payload, { + sourceZoom: overviewTile.sourceZoom, + targetZoom: z, + targetX: x, + targetY: y, + excludeLayers: ['land', 'landcover', 'depth'], + authority: 'world-overview-cartography', + report + }) + ); + } + } + if (z > Number(overview.header.maxZoom)) { + for (const opened of regional) { + const result = await opened.archive.getZxy(z, x, y); + if (!result?.data) continue; + contributions.push( + ...normalizeTileContribution(Buffer.from(result.data), { + sourceZoom: z, + targetZoom: z, + targetX: x, + targetY: y, + excludeLayers: ['land', 'landcover', 'depth'], + authority: opened.assetName || opened.source?.filename || 'regional-cartography', + report + }) + ); + } + } + return { + data: encodeNormalizedTile(contributions, { + label: `immutable offline tile ${z}/${x}/${y}`, + report + }), + report + }; +} diff --git a/scripts/offline-tileset/plan-owners.mjs b/scripts/offline-tileset/plan-owners.mjs new file mode 100644 index 00000000..42744612 --- /dev/null +++ b/scripts/offline-tileset/plan-owners.mjs @@ -0,0 +1,246 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const DEFAULT_RELEASE = 'occumed-world-v1'; +const DEFAULT_ROUTING_ZOOM = 6; +const DEFAULT_MAX_PREFIX_ZOOM = 8; +const DEFAULT_MAX_CANDIDATE_BYTES = 2 * 1024 * 1024 * 1024; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) throw new Error(`Unexpected argument: ${key}`); + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${key}.`); + result[key.slice(2)] = value; + index += 1; + } + return result; +} + +function lonToTileX(lon, zoom) { + return Math.floor(((lon + 180) / 360) * 2 ** zoom); +} + +function latToTileY(lat, zoom) { + const clipped = Math.min(85.05112878, Math.max(-85.05112878, lat)); + const radians = clipped * Math.PI / 180; + return Math.floor( + ((1 - Math.asinh(Math.tan(radians)) / Math.PI) / 2) * 2 ** zoom + ); +} + +function tileBounds(z, x, y) { + const width = 2 ** z; + const west = (x / width) * 360 - 180; + const east = ((x + 1) / width) * 360 - 180; + const north = Math.atan(Math.sinh(Math.PI * (1 - (2 * y) / width))) * 180 / Math.PI; + const south = Math.atan(Math.sinh(Math.PI * (1 - (2 * (y + 1)) / width))) * 180 / Math.PI; + return [west, south, east, north]; +} + +function splitAntimeridianBounds(bounds) { + const [west, south, east, north] = bounds; + return west <= east + ? [[west, south, east, north]] + : [[west, south, 180, north], [-180, south, east, north]]; +} + +function boundsIntersect(left, right) { + return !( + left[2] <= right[0] || + left[0] >= right[2] || + left[3] <= right[1] || + left[1] >= right[3] + ); +} + +function regionIntersectsTile(region, tile) { + const bounds = tileBounds(tile.z, tile.x, tile.y); + return splitAntimeridianBounds(region.bounds).some((part) => boundsIntersect(part, bounds)); +} + +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries( + Object.keys(value) + .sort() + .filter((key) => key !== 'generatedAt') + .map((key) => [key, canonical(value[key])]) + ); +} + +function planHash(plan) { + return createHash('sha256') + .update(`${JSON.stringify(canonical(plan))}\n`) + .digest('hex'); +} + +async function fetchJson(url) { + const response = await fetch(url, { + redirect: 'follow', + headers: { 'User-Agent': 'Occu-Med-Map/offline-owner-plan' } + }); + if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}.`); + return response.json(); +} + +function splitOwner(prefix, candidates, options, output) { + const candidateBytes = candidates.reduce((sum, candidate) => sum + candidate.bytes, 0); + if ( + candidateBytes <= options.maxCandidateBytes || + prefix.z >= options.maxPrefixZoom + ) { + const owner = { + id: `z${prefix.z}-${prefix.x}-${prefix.y}`, + prefix, + exactTiles: [], + candidateBytes, + candidates: candidates + .map((candidate) => ({ + id: candidate.id, + asset: candidate.asset, + bounds: candidate.bounds, + bytes: candidate.bytes, + sha256: candidate.sha256, + url: candidate.url + })) + .sort((left, right) => left.asset.localeCompare(right.asset)) + }; + output.push(owner); + return owner; + } + + let deterministicAncestorOwner = null; + for (let deltaY = 0; deltaY < 2; deltaY += 1) { + for (let deltaX = 0; deltaX < 2; deltaX += 1) { + const child = { + z: prefix.z + 1, + x: prefix.x * 2 + deltaX, + y: prefix.y * 2 + deltaY + }; + const childCandidates = candidates.filter((candidate) => + regionIntersectsTile(candidate, child) + ); + if (childCandidates.length) { + const childOwner = splitOwner(child, childCandidates, options, output); + deterministicAncestorOwner ||= childOwner; + } + } + } + if (!deterministicAncestorOwner) { + throw new Error(`Split owner ${prefix.z}/${prefix.x}/${prefix.y} has no descendants.`); + } + if (prefix.z > options.routingZoom) { + deterministicAncestorOwner.exactTiles.push(prefix); + } + return deterministicAncestorOwner; +} + +const options = parseArguments(process.argv.slice(2)); +const repository = options.repository || 'Occumed79/Map'; +const tag = options.tag || DEFAULT_RELEASE; +const outputPath = path.resolve( + options.output || path.join(root, 'config/immutable-owner-plan.json') +); +const routingZoom = Number(options['routing-zoom'] || DEFAULT_ROUTING_ZOOM); +const maxPrefixZoom = Number(options['max-prefix-zoom'] || DEFAULT_MAX_PREFIX_ZOOM); +const maxCandidateBytes = Number( + options['max-candidate-bytes'] || DEFAULT_MAX_CANDIDATE_BYTES +); + +const releaseApi = `https://api.github.com/repos/${repository}/releases/tags/${tag}`; +const release = await fetchJson(releaseApi); +const byName = new Map( + (release.assets || []).map((asset) => [ + asset.name, + { + asset: asset.name, + bytes: Number(asset.size), + sha256: String(asset.digest || '').replace(/^sha256:/, ''), + url: asset.browser_download_url + } + ]) +); +const manifestAsset = byName.get('world-virtual-manifest.json'); +if (!manifestAsset) throw new Error('Published world-virtual-manifest.json is missing.'); +const published = await fetchJson(manifestAsset.url); +const overview = byName.get(published.virtualTiles?.overviewAsset); +const surface = byName.get(published.virtualTiles?.surfaceAsset); +if (!overview || !surface) throw new Error('Published overview or surface archive is missing.'); + +const regions = (published.regions || []).map((region) => { + const asset = byName.get(region.asset); + if (!asset || !asset.sha256 || asset.sha256.length !== 64) { + throw new Error(`Release asset lock is incomplete for ${region.asset}.`); + } + return { + id: region.id, + bounds: region.bounds.map(Number), + ...asset + }; +}); + +const owners = []; +let logicalOwnerCount = 0; +const width = 2 ** routingZoom; +for (let y = 0; y < width; y += 1) { + for (let x = 0; x < width; x += 1) { + const prefix = { z: routingZoom, x, y }; + const candidates = regions.filter((region) => regionIntersectsTile(region, prefix)); + if (candidates.length) { + logicalOwnerCount += 1; + splitOwner(prefix, candidates, { + maxCandidateBytes, + maxPrefixZoom, + routingZoom + }, owners); + } + } +} +owners.sort((left, right) => + left.prefix.z - right.prefix.z || + left.prefix.x - right.prefix.x || + left.prefix.y - right.prefix.y +); + +const plan = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + repository, + releaseTag: tag, + routingZoom, + maxPrefixZoom, + maxCandidateBytes, + authorities: { + land: 'world-surface', + depth: 'world-surface', + landcover: 'world-overview', + cartography: 'regional-owner' + }, + inputs: { + overview, + surface, + regionCount: regions.length, + regionalBytes: regions.reduce((sum, region) => sum + region.bytes, 0) + }, + owners, + logicalOwnerCount, + plannedOwnerCount: owners.length +}; +plan.planVersion = planHash(plan); + +await fs.mkdir(path.dirname(outputPath), { recursive: true }); +await fs.writeFile(outputPath, `${JSON.stringify(plan, null, 2)}\n`); +console.log( + `Planned ${owners.length} deterministic physical owners across ${logicalOwnerCount} logical z${routingZoom} cells ` + + `from ${regions.length} SHA-locked regional inputs ` + + `(${plan.inputs.regionalBytes} bytes); plan ${plan.planVersion}.` +); diff --git a/scripts/offline-tileset/pmtiles-writer.mjs b/scripts/offline-tileset/pmtiles-writer.mjs new file mode 100644 index 00000000..c74092e8 --- /dev/null +++ b/scripts/offline-tileset/pmtiles-writer.mjs @@ -0,0 +1,378 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import path from 'node:path'; +import { pipeline } from 'node:stream/promises'; +import { gzipSync } from 'node:zlib'; +import { PMTiles, zxyToTileId } from 'pmtiles'; +import { LocalPmtilesSource, tilePayload } from './local-pmtiles.mjs'; + +const HEADER_BYTES = 127; +const ROOT_DIRECTORY_BUDGET = 16_384 - HEADER_BYTES; +const GZIP = 2; +const MVT = 1; + +function putUint64(view, offset, value) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`Unsafe PMTiles uint64 value: ${value}`); + } + view.setBigUint64(offset, BigInt(value), true); +} + +function encodeVarint(value) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`Unsafe PMTiles varint value: ${value}`); + } + const bytes = []; + let remaining = BigInt(value); + while (remaining >= 0x80n) { + bytes.push(Number((remaining & 0x7fn) | 0x80n)); + remaining >>= 7n; + } + bytes.push(Number(remaining)); + return Buffer.from(bytes); +} + +function serializeEntries(entries) { + const chunks = [encodeVarint(entries.length)]; + let previousId = 0; + for (const entry of entries) { + chunks.push(encodeVarint(entry.tileId - previousId)); + previousId = entry.tileId; + } + for (const entry of entries) chunks.push(encodeVarint(entry.runLength)); + for (const entry of entries) chunks.push(encodeVarint(entry.length)); + for (let index = 0; index < entries.length; index += 1) { + const entry = entries[index]; + const contiguous = + index > 0 && + entry.offset === entries[index - 1].offset + entries[index - 1].length; + chunks.push(encodeVarint(contiguous ? 0 : entry.offset + 1)); + } + return gzipSync(Buffer.concat(chunks), { level: 9 }); +} + +function buildRootAndLeaves(entries) { + if (entries.length < 16_384) { + const root = serializeEntries(entries); + if (root.byteLength <= ROOT_DIRECTORY_BUDGET) { + return { root, leaves: Buffer.alloc(0), leafCount: 0 }; + } + } + + let leafSize = Math.max(4_096, Math.ceil(entries.length / 3_500)); + while (true) { + const rootEntries = []; + const leafChunks = []; + let leafOffset = 0; + for (let index = 0; index < entries.length; index += leafSize) { + const leaf = serializeEntries(entries.slice(index, index + leafSize)); + rootEntries.push({ + tileId: entries[index].tileId, + offset: leafOffset, + length: leaf.byteLength, + runLength: 0 + }); + leafChunks.push(leaf); + leafOffset += leaf.byteLength; + } + const root = serializeEntries(rootEntries); + if (root.byteLength <= ROOT_DIRECTORY_BUDGET) { + return { + root, + leaves: Buffer.concat(leafChunks), + leafCount: leafChunks.length + }; + } + leafSize = Math.ceil(leafSize * 1.2); + } +} + +function clampE7(value, minimum, maximum) { + return Math.round(Math.min(maximum, Math.max(minimum, value)) * 10_000_000); +} + +function serializeHeader(header) { + const buffer = Buffer.alloc(HEADER_BYTES); + buffer.write('PMTiles', 0, 7, 'utf8'); + buffer[7] = 3; + const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength); + putUint64(view, 8, header.rootOffset); + putUint64(view, 16, header.rootLength); + putUint64(view, 24, header.metadataOffset); + putUint64(view, 32, header.metadataLength); + putUint64(view, 40, header.leafOffset); + putUint64(view, 48, header.leafLength); + putUint64(view, 56, header.tileDataOffset); + putUint64(view, 64, header.tileDataLength); + putUint64(view, 72, header.addressedTiles); + putUint64(view, 80, header.tileEntries); + putUint64(view, 88, header.tileContents); + buffer[96] = 1; + buffer[97] = GZIP; + buffer[98] = GZIP; + buffer[99] = MVT; + buffer[100] = header.minZoom; + buffer[101] = header.maxZoom; + view.setInt32(102, clampE7(header.bounds[0], -180, 180), true); + view.setInt32(106, clampE7(header.bounds[1], -85.0511288, 85.0511288), true); + view.setInt32(110, clampE7(header.bounds[2], -180, 180), true); + view.setInt32(114, clampE7(header.bounds[3], -85.0511288, 85.0511288), true); + buffer[118] = header.center[2]; + view.setInt32(119, clampE7(header.center[0], -180, 180), true); + view.setInt32(123, clampE7(header.center[1], -85.0511288, 85.0511288), true); + return buffer; +} + +async function sha256File(filename) { + const hash = createHash('sha256'); + await pipeline(fs.createReadStream(filename), hash); + return hash.digest('hex'); +} + +async function syncFile(filename) { + const handle = await fsp.open(filename, 'r'); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +export async function verifyPmtiles(filename, expected = {}) { + const stat = await fsp.stat(filename); + if (!stat.isFile() || stat.size < HEADER_BYTES) { + throw new Error(`Invalid PMTiles output: ${filename}`); + } + const source = new LocalPmtilesSource(filename); + try { + const archive = new PMTiles(source); + const header = await archive.getHeader(); + if (header.specVersion !== 3 || header.tileType !== MVT) { + throw new Error(`Unexpected PMTiles header for ${filename}.`); + } + if (expected.addressedTiles !== undefined && + header.numAddressedTiles !== expected.addressedTiles) { + throw new Error( + `PMTiles addressed-tile mismatch: expected ${expected.addressedTiles}, ` + + `received ${header.numAddressedTiles}.` + ); + } + for (const tile of expected.sampleTiles || []) { + const payload = tilePayload(await archive.getZxy(tile.z, tile.x, tile.y)); + if (!payload?.byteLength) { + throw new Error(`PMTiles verification sample is missing ${tile.z}/${tile.x}/${tile.y}.`); + } + } + return { + bytes: stat.size, + sha256: await sha256File(filename), + header + }; + } finally { + await source.close(); + } +} + +/** + * Deterministic PMTiles v3 writer for prebuilt MVT tiles. + * + * Tiles must be added in increasing Hilbert tile-id order. Payloads are + * compressed and content-deduplicated before the final directory is emitted. + */ +export class DeterministicPmtilesWriter { + constructor({ + output, + workDirectory, + metadata = {}, + bounds = [-180, -85.0511288, 180, 85.0511288] + }) { + this.output = path.resolve(output); + this.workDirectory = path.resolve(workDirectory); + this.metadata = metadata; + this.bounds = bounds; + this.entries = []; + this.content = new Map(); + this.dataBytes = 0; + this.addressedTiles = 0; + this.previousTileId = -1; + this.minZoom = 30; + this.maxZoom = 0; + this.coordinates = []; + this.dataPath = path.join( + this.workDirectory, + `tile-data-${process.pid}-${Date.now()}.bin` + ); + this.dataHandlePromise = null; + } + + async initialize() { + await fsp.mkdir(this.workDirectory, { recursive: true }); + await fsp.mkdir(path.dirname(this.output), { recursive: true }); + this.dataHandlePromise = fsp.open(this.dataPath, 'wx'); + return this; + } + + async addTile({ z, x, y, data }) { + if (!this.dataHandlePromise) throw new Error('PMTiles writer is not initialized.'); + const tileId = zxyToTileId(z, x, y); + if (tileId <= this.previousTileId) { + throw new Error(`PMTiles tiles are not strictly ordered at ${z}/${x}/${y}.`); + } + this.previousTileId = tileId; + this.minZoom = Math.min(this.minZoom, z); + this.maxZoom = Math.max(this.maxZoom, z); + this.coordinates.push({ z, x, y }); + + const raw = Buffer.from(data); + if (!raw.byteLength) throw new Error(`Cannot add an empty tile payload at ${z}/${x}/${y}.`); + const contentKey = createHash('sha256').update(raw).digest('hex'); + let stored = this.content.get(contentKey); + if (!stored) { + const compressed = + raw[0] === 0x1f && raw[1] === 0x8b + ? raw + : gzipSync(raw, { level: 9 }); + const handle = await this.dataHandlePromise; + await handle.write(compressed, 0, compressed.byteLength, this.dataBytes); + stored = { offset: this.dataBytes, length: compressed.byteLength }; + this.content.set(contentKey, stored); + this.dataBytes += compressed.byteLength; + } + + const previous = this.entries.at(-1); + if ( + previous && + tileId === previous.tileId + previous.runLength && + previous.offset === stored.offset && + previous.runLength < 0xffff_ffff + ) { + previous.runLength += 1; + } else { + this.entries.push({ + tileId, + offset: stored.offset, + length: stored.length, + runLength: 1 + }); + } + this.addressedTiles += 1; + } + + async finalize() { + if (!this.entries.length) throw new Error('Cannot finalize an empty PMTiles archive.'); + const dataHandle = await this.dataHandlePromise; + await dataHandle.sync(); + await dataHandle.close(); + this.dataHandlePromise = null; + + const { root, leaves, leafCount } = buildRootAndLeaves(this.entries); + const metadata = gzipSync( + Buffer.from(`${JSON.stringify(this.metadata)}\n`), + { level: 9 } + ); + const rootOffset = HEADER_BYTES; + const metadataOffset = rootOffset + root.byteLength; + const leafOffset = metadataOffset + metadata.byteLength; + const tileDataOffset = leafOffset + leaves.byteLength; + const center = [ + (this.bounds[0] + this.bounds[2]) / 2, + (this.bounds[1] + this.bounds[3]) / 2, + this.minZoom + ]; + const header = serializeHeader({ + rootOffset, + rootLength: root.byteLength, + metadataOffset, + metadataLength: metadata.byteLength, + leafOffset, + leafLength: leaves.byteLength, + tileDataOffset, + tileDataLength: this.dataBytes, + addressedTiles: this.addressedTiles, + tileEntries: this.entries.length, + tileContents: this.content.size, + minZoom: this.minZoom, + maxZoom: this.maxZoom, + bounds: this.bounds, + center + }); + + const unique = `${process.pid}-${Date.now()}`; + const staged = path.join(this.workDirectory, `archive-${unique}.staged.pmtiles`); + const pending = `${this.output}.pending-${unique}`; + const stagedHandle = await fsp.open(staged, 'wx'); + try { + await stagedHandle.write(header); + await stagedHandle.write(root); + await stagedHandle.write(metadata); + await stagedHandle.write(leaves); + let position = tileDataOffset; + for await (const chunk of fs.createReadStream(this.dataPath)) { + await stagedHandle.write(chunk, 0, chunk.byteLength, position); + position += chunk.byteLength; + } + if (position !== tileDataOffset + this.dataBytes) { + throw new Error('PMTiles tile-data copy ended at an unexpected offset.'); + } + await stagedHandle.sync(); + } finally { + await stagedHandle.close().catch(() => {}); + } + + const samples = [ + this.coordinates[0], + this.coordinates[Math.floor(this.coordinates.length / 2)], + this.coordinates.at(-1) + ]; + const stagedVerification = await verifyPmtiles(staged, { + addressedTiles: this.addressedTiles, + sampleTiles: samples + }); + await fsp.copyFile(staged, pending, fs.constants.COPYFILE_EXCL); + await syncFile(pending); + const pendingVerification = await verifyPmtiles(pending, { + addressedTiles: this.addressedTiles, + sampleTiles: samples + }); + if ( + stagedVerification.bytes !== pendingVerification.bytes || + stagedVerification.sha256 !== pendingVerification.sha256 + ) { + throw new Error('PMTiles staged and pending files are not byte-identical.'); + } + + await fsp.rename(pending, this.output); + await syncFile(this.output); + const finalVerification = await verifyPmtiles(this.output, { + addressedTiles: this.addressedTiles, + sampleTiles: samples + }); + if (finalVerification.sha256 !== stagedVerification.sha256) { + throw new Error('PMTiles output changed during atomic promotion.'); + } + + await Promise.allSettled([ + fsp.unlink(staged), + fsp.unlink(this.dataPath) + ]); + return { + ...finalVerification, + addressedTiles: this.addressedTiles, + tileEntries: this.entries.length, + tileContents: this.content.size, + leafCount, + minZoom: this.minZoom, + maxZoom: this.maxZoom + }; + } + + async abort() { + if (this.dataHandlePromise) { + const handle = await this.dataHandlePromise.catch(() => null); + await handle?.close().catch(() => {}); + this.dataHandlePromise = null; + } + await fsp.unlink(this.dataPath).catch(() => {}); + } +} diff --git a/scripts/restore-exported-cartography.mjs b/scripts/restore-exported-cartography.mjs index 660c5056..6d96a1ae 100644 --- a/scripts/restore-exported-cartography.mjs +++ b/scripts/restore-exported-cartography.mjs @@ -375,26 +375,6 @@ water.paint['fill-opacity'] = [ 1 ]; -const hillshade = runtime.layers.find((layer) => layer.id === 'occumed-hillshade'); -if (!hillshade) throw new Error('The open hillshade layer is missing after cartography restoration.'); -hillshade.paint['hillshade-exaggeration'] = [ - 'interpolate', - ['linear'], - ['zoom'], - 3, - 0.06, - 6, - 0.16, - 8, - 0.24, - 10, - 0.28, - 14, - 0.24, - 16, - 0.16 -]; - for (const layer of runtime.layers || []) { if (layer.type !== 'symbol') continue; const haloWidth = layer.paint?.['text-halo-width']; diff --git a/scripts/validate-immutable-visuals.mjs b/scripts/validate-immutable-visuals.mjs new file mode 100644 index 00000000..20544c85 --- /dev/null +++ b/scripts/validate-immutable-visuals.mjs @@ -0,0 +1,826 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { chromium } from 'playwright'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const require = createRequire(import.meta.url); +const { PNG } = require('pngjs'); +const VIEWPORT = { width: 1440, height: 1000 }; +const MOTION_DURATION_MS = 12_600; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) throw new Error(`Unexpected argument: ${key}`); + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${key}.`); + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['manifest', 'targets', 'output-dir']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +function safeName(value) { + return String(value).replace(/[^a-z0-9-]+/gi, '-').replace(/^-|-$/g, '').toLowerCase(); +} + +async function waitForReadiness(origin, expectedStatus, timeoutMs = 30_000) { + const started = Date.now(); + let lastStatus = null; + while (Date.now() - started < timeoutMs) { + try { + const response = await fetch(`${origin}/readyz`, { cache: 'no-store' }); + lastStatus = response.status; + if (response.status === expectedStatus) { + return { + status: response.status, + body: await response.json().catch(() => null) + }; + } + } catch { + // The child may still be binding its port. + } + await new Promise((resolve) => setTimeout(resolve, 150)); + } + throw new Error(`Readiness did not return ${expectedStatus}; last status was ${lastStatus}.`); +} + +async function startServer({ + manifest, + port, + allowPartial, + logPath +}) { + const handle = await fs.open(logPath, 'wx'); + const child = spawn(process.execPath, ['server.mjs'], { + cwd: root, + env: { + ...process.env, + HOST: '127.0.0.1', + PORT: String(port), + OCCUMED_IMMUTABLE_TILESET_MANIFEST: manifest, + OCCUMED_ALLOW_PARTIAL_TILESET_FIXTURE: allowPartial ? 'true' : 'false' + }, + stdio: ['ignore', handle.fd, handle.fd] + }); + child.once('exit', () => { + void handle.close().catch(() => {}); + }); + return { child, handle, origin: `http://127.0.0.1:${port}` }; +} + +async function stopServer(server) { + if (!server?.child || server.child.exitCode !== null) return; + server.child.kill('SIGTERM'); + await Promise.race([ + new Promise((resolve) => server.child.once('exit', resolve)), + new Promise((resolve) => setTimeout(resolve, 5_000)) + ]); + if (server.child.exitCode === null) server.child.kill('SIGKILL'); +} + +async function waitForRequestSettlement(outstanding, timeoutMs = 20_000) { + const started = Date.now(); + let idleSince = outstanding.size ? null : Date.now(); + while (Date.now() - started < timeoutMs) { + if (!outstanding.size) { + idleSince ||= Date.now(); + if (Date.now() - idleSince >= 250) return; + } else { + idleSince = null; + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`Tile delivery did not settle; ${outstanding.size} requests remain.`); +} + +async function waitForInViewTileSettlement(page, timeoutMs = 30_000) { + const started = Date.now(); + let state = await mapState(page); + while (Date.now() - started < timeoutMs) { + const tiles = state.inViewTiles; + if (tiles.length > 0 && tiles.every((tile) => tile.state !== 'loading')) { + await pumpRenderFrames(page, 4, 2_000); + return mapState(page); + } + await pumpRenderFrames(page, 1, 500); + await page.waitForTimeout(50); + state = await mapState(page); + } + const loading = state.inViewTiles.filter((tile) => tile.state === 'loading'); + throw new Error( + `In-view tile parsing did not settle; ${loading.length} of ` + + `${state.inViewTiles.length} tiles remain loading.` + ); +} + +async function waitForRenderedCamera(page, camera) { + await page.evaluate(({ center, zoom }) => { + const map = globalThis.__OCCUMED_MAP__; + map.stop(); + map.jumpTo({ center, zoom, bearing: 0, pitch: 0 }); + }, camera); + await page.waitForFunction(({ center, zoom }) => { + const map = globalThis.__OCCUMED_MAP__; + if (!map) return false; + const actual = map.getCenter(); + const longitudeError = Math.abs((((actual.lng - center[0]) + 540) % 360) - 180); + return longitudeError < 0.01 && + Math.abs(actual.lat - center[1]) < 0.01 && + Math.abs(map.getZoom() - zoom) < 0.01; + }, camera, { timeout: 30_000 }); + await pumpRenderFrames(page); +} + +async function pumpRenderFrames(page, minimumFrames = 4, timeoutMs = 2_000) { + await page.evaluate(({ minimumFrames, timeoutMs }) => new Promise((resolve) => { + const map = globalThis.__OCCUMED_MAP__; + let frames = 0; + let settled = false; + const finish = () => { + if (settled) return; + settled = true; + clearInterval(repaint); + clearTimeout(timeout); + map.off('render', onRender); + resolve(); + }; + const onRender = () => { + frames += 1; + if (frames >= minimumFrames) finish(); + }; + map.on('render', onRender); + const repaint = setInterval(() => map.triggerRepaint(), 50); + const timeout = setTimeout(finish, timeoutMs); + map.triggerRepaint(); + }), { minimumFrames, timeoutMs }); +} + +async function mapState(page) { + return page.evaluate(() => { + const map = globalThis.__OCCUMED_MAP__; + const style = map.getStyle(); + const tileManager = map.style?.tileManagers?.['occumed-open']; + const sourceIds = Object.keys(style.sources || {}).sort(); + const sourceLayers = new Set(); + const width = map.getContainer().clientWidth; + const height = map.getContainer().clientHeight; + for (let row = 0; row < 3; row += 1) { + for (let column = 0; column < 3; column += 1) { + const x = ((column + 0.5) / 3) * width; + const y = ((row + 0.5) / 3) * height; + const features = map.queryRenderedFeatures([ + [Math.max(0, x - 45), Math.max(0, y - 45)], + [Math.min(width, x + 45), Math.min(height, y + 45)] + ]); + for (const feature of features) { + if (feature.source === 'occumed-open' && feature.sourceLayer) { + sourceLayers.add(feature.sourceLayer); + } + } + } + } + return { + sourceIds, + sourceLayers: [...sourceLayers].sort(), + center: [map.getCenter().lng, map.getCenter().lat], + zoom: map.getZoom(), + moving: map.isMoving(), + styleLoaded: map.isStyleLoaded(), + inViewTiles: tileManager?._inViewTiles?.getAllIds?.().map((id) => { + const tile = tileManager._inViewTiles.getTileById(id); + return { + z: tile?.tileID?.canonical?.z, + x: tile?.tileID?.canonical?.x, + y: tile?.tileID?.canonical?.y, + state: tile?.state, + hasData: Boolean(tile?.hasData?.()) + }; + }) || [] + }; + }); +} + +async function analyzeScreenshot(_page, screenshot, zoom) { + const { data, width, height } = PNG.sync.read(screenshot); + const offset = (x, y) => (y * width + x) * 4; + const delta = (left, right) => + Math.abs(data[left] - data[right]) + + Math.abs(data[left + 1] - data[right + 1]) + + Math.abs(data[left + 2] - data[right + 2]); + const luma = (value) => + (data[value] * 299 + data[value + 1] * 587 + data[value + 2] * 114) / 1_000; + const waterLike = (value) => + data[value + 2] >= 170 && + data[value + 1] >= 125 && + data[value + 2] - data[value] >= 35 && + data[value + 2] - data[value + 1] >= 20; + const surfaceLike = (value) => + data[value + 3] >= 250 && luma(value) >= 75 && luma(value) <= 245; + + const histogram = new Map(); + let sampled = 0; + let nonDark = 0; + for (let y = 0; y < height; y += 4) { + for (let x = 0; x < width; x += 4) { + const value = offset(x, y); + const key = + ((data[value] >> 4) << 8) | + ((data[value + 1] >> 4) << 4) | + (data[value + 2] >> 4); + histogram.set(key, (histogram.get(key) || 0) + 1); + sampled += 1; + if (luma(value) > 25) nonDark += 1; + } + } + let dominantKey = 0; + let dominantCount = 0; + for (const [key, count] of histogram) { + if (count > dominantCount) { + dominantKey = key; + dominantCount = count; + } + } + const dominant = dominantCount / sampled; + const dominantColor = [ + ((dominantKey >> 8) & 0x0f) * 16 + 8, + ((dominantKey >> 4) & 0x0f) * 16 + 8, + (dominantKey & 0x0f) * 16 + 8 + ]; + + function scanVertical(x, predicate) { + let best = { start: 0, end: 0, length: 0 }; + let start = 0; + let length = 0; + let gaps = 0; + for (let y = 2; y < height - 2; y += 1) { + const accepted = predicate(offset(x - 1, y), offset(x, y)); + if (accepted) { + if (!length) start = y; + length += 1 + gaps; + gaps = 0; + } else if (length && gaps < 2) { + gaps += 1; + } else { + if (length > best.length) best = { start, end: y - gaps - 1, length }; + start = 0; + length = 0; + gaps = 0; + } + } + if (length > best.length) best = { start, end: height - gaps - 2, length }; + return { axis: x, ...best }; + } + + function scanHorizontal(y, predicate) { + let best = { start: 0, end: 0, length: 0 }; + let start = 0; + let length = 0; + let gaps = 0; + for (let x = 2; x < width - 2; x += 1) { + const accepted = predicate(offset(x, y - 1), offset(x, y)); + if (accepted) { + if (!length) start = x; + length += 1 + gaps; + gaps = 0; + } else if (length && gaps < 2) { + gaps += 1; + } else { + if (length > best.length) best = { start, end: x - gaps - 1, length }; + start = 0; + length = 0; + gaps = 0; + } + } + if (length > best.length) best = { start, end: width - gaps - 2, length }; + return { axis: y, ...best }; + } + + const strongEdge = (left, right) => { + const difference = delta(left, right); + return surfaceLike(left) && surfaceLike(right) && difference >= 45 && difference <= 260; + }; + const waterEdge = (left, right) => { + const difference = delta(left, right); + return waterLike(left) && waterLike(right) && difference >= 5 && difference <= 75; + }; + const footprintEdge = (left, right) => { + const difference = delta(left, right); + return surfaceLike(left) && surfaceLike(right) && difference >= 8 && difference <= 260; + }; + + const verticalStrong = []; + const horizontalStrong = []; + const verticalWater = []; + const horizontalWater = []; + const verticalFootprints = []; + const horizontalFootprints = []; + for (let x = 4; x < width - 4; x += 1) { + const strong = scanVertical(x, strongEdge); + if (strong.length >= height * 0.62) verticalStrong.push(strong); + if (zoom >= 5) { + const water = scanVertical(x, waterEdge); + if (water.length >= height * 0.32) verticalWater.push(water); + const footprint = scanVertical(x, footprintEdge); + if (footprint.length >= 90) verticalFootprints.push(footprint); + } + } + for (let y = 4; y < height - 4; y += 1) { + const strong = scanHorizontal(y, strongEdge); + if (strong.length >= width * 0.62) horizontalStrong.push(strong); + if (zoom >= 5) { + const water = scanHorizontal(y, waterEdge); + if (water.length >= width * 0.32) horizontalWater.push(water); + const footprint = scanHorizontal(y, footprintEdge); + if (footprint.length >= 110) horizontalFootprints.push(footprint); + } + } + + function collapse(runs) { + const ordered = [...runs].sort((left, right) => left.axis - right.axis || right.length - left.length); + const output = []; + for (const run of ordered) { + const previous = output.at(-1); + if (previous && run.axis - previous.axis <= 3) { + if (run.length > previous.length) output[output.length - 1] = run; + } else { + output.push(run); + } + } + return output; + } + + const verticalCandidates = collapse(verticalFootprints).slice(0, 80); + const horizontalCandidates = collapse(horizontalFootprints).slice(0, 80); + let rectangles = 0; + let rectangleBounds = null; + function hasUniformInterior(left, right, top, bottom) { + const counts = new Map(); + let total = 0; + for (let y = top + 8; y < bottom - 8; y += 8) { + for (let x = left + 8; x < right - 8; x += 8) { + const value = offset(x, y); + const key = + ((data[value] >> 4) << 8) | + ((data[value + 1] >> 4) << 4) | + (data[value + 2] >> 4); + counts.set(key, (counts.get(key) || 0) + 1); + total += 1; + } + } + return total > 40 && Math.max(...counts.values()) / total >= 0.68; + } + + for (let leftIndex = 0; leftIndex < verticalCandidates.length; leftIndex += 1) { + const left = verticalCandidates[leftIndex]; + for (let rightIndex = leftIndex + 1; rightIndex < verticalCandidates.length; rightIndex += 1) { + const right = verticalCandidates[rightIndex]; + if (right.axis - left.axis < 90) continue; + const topLimit = Math.max(left.start, right.start); + const bottomLimit = Math.min(left.end, right.end); + if (bottomLimit - topLimit < 90) continue; + const top = horizontalCandidates.find((run) => + run.axis >= topLimit - 5 && + run.axis <= bottomLimit + 5 && + run.start <= left.axis + 5 && + run.end >= right.axis - 5 + ); + if (!top) continue; + const bottom = horizontalCandidates.find((run) => + run.axis - top.axis >= 90 && + run.axis <= bottomLimit + 5 && + run.start <= left.axis + 5 && + run.end >= right.axis - 5 + ); + const candidateArea = bottom + ? (right.axis - left.axis) * (bottom.axis - top.axis) + : 0; + if ( + bottom && + candidateArea >= width * height * 0.08 && + hasUniformInterior(left.axis, right.axis, top.axis, bottom.axis) + ) { + rectangles += 1; + rectangleBounds = { + left: left.axis, + right: right.axis, + top: top.axis, + bottom: bottom.axis, + areaRatio: candidateArea / (width * height) + }; + break; + } + } + if (rectangles) break; + } + + const verticalSeams = collapse([...verticalStrong, ...verticalWater]); + const horizontalSeams = collapse([...horizontalStrong, ...horizontalWater]); + const blank = histogram.size < 12 || dominant > 0.985 || nonDark / sampled < 0.02; + return { + width, + height, + sampledPixels: sampled, + quantizedColors: histogram.size, + dominantColorRatio: dominant, + dominantColor, + nonDarkRatio: nonDark / sampled, + blank, + rectangularFootprints: rectangles, + rectangleBounds, + verticalSeams: verticalSeams.length, + horizontalSeams: horizontalSeams.length, + stretchedPolygons: rectangles, + inconsistentNeighbors: verticalSeams.length + horizontalSeams.length, + longestVerticalEdge: Math.max( + 0, + ...verticalStrong.map((run) => run.length), + ...verticalWater.map((run) => run.length) + ), + longestHorizontalEdge: Math.max( + 0, + ...horizontalStrong.map((run) => run.length), + ...horizontalWater.map((run) => run.length) + ) + }; +} + +function validateSourceState(state, label, failures) { + if (state.sourceIds.length !== 1 || state.sourceIds[0] !== 'occumed-open') { + failures.push(`${label}: source switched to [${state.sourceIds.join(', ')}].`); + } +} + +function validatePixels(analysis, label, failures) { + if (analysis.blank) failures.push(`${label}: blank rendered frame.`); + if (analysis.rectangularFootprints) failures.push(`${label}: rectangular tile footprint detected.`); + if (analysis.verticalSeams) failures.push(`${label}: vertical seam detected.`); + if (analysis.horizontalSeams) failures.push(`${label}: horizontal seam detected.`); + if (analysis.stretchedPolygons) failures.push(`${label}: stretched polygon detected.`); + if (analysis.inconsistentNeighbors) failures.push(`${label}: neighboring tiles are inconsistent.`); +} + +function applyFoundationBlankContract(pixels, state) { + pixels.pixelBlank = pixels.blank; + const oceanOnly = + state.sourceLayers.length > 0 && + state.sourceLayers.every((layer) => + ['depth', 'water', 'water_name', 'waterway'].includes(layer) + ); + const foundationLand = [232, 232, 216]; + const landColorDistance = pixels.dominantColor.reduce( + (sum, channel, index) => sum + Math.abs(channel - foundationLand[index]), + 0 + ); + const uniformLand = + state.sourceLayers.includes('land') && + pixels.dominantColorRatio >= 0.985 && + landColorDistance <= 30; + if (oceanOnly || uniformLand) { + pixels.blank = false; + pixels.blankContract = oceanOnly ? 'ocean-foundation' : 'uniform-land-foundation'; + } +} + +function validateRequiredLayers(name, state, failures) { + const available = new Set(state.sourceLayers); + const requireOne = (choices, label) => { + if (!choices.some((choice) => available.has(choice))) { + failures.push(`${name}: no rendered ${label} bucket (${choices.join(' or ')}).`); + } + }; + if (['pacific', 'antimeridian'].includes(name)) { + requireOne(['depth', 'land'], 'foundation'); + return; + } + requireOne(['land', 'landcover'], 'land foundation'); + if (name.startsWith('fresno-')) { + requireOne(['land'], 'prebuilt land foundation'); + } + if (name === 'fresno-city' || name === 'fresno-street') { + requireOne(['transportation', 'building', 'place'], 'regional cartography'); + } +} + +function staticStateIsReady(name, state) { + const available = new Set(state.sourceLayers); + const hasOne = (choices) => choices.some((choice) => available.has(choice)); + if (['pacific', 'antimeridian'].includes(name)) { + return hasOne(['depth', 'land']); + } + if (!hasOne(['land', 'landcover'])) return false; + if (name.startsWith('fresno-') && !available.has('land')) return false; + if (name === 'fresno-city' || name === 'fresno-street') { + return hasOne(['transportation', 'building', 'place']); + } + return true; +} + +async function waitForStaticState(page, name, timeoutMs = 30_000) { + const started = Date.now(); + let state = await mapState(page); + while (!staticStateIsReady(name, state) && Date.now() - started < timeoutMs) { + await pumpRenderFrames(page, 3, 1_000); + await page.waitForTimeout(100); + state = await mapState(page); + } + return state; +} + +const options = parseArguments(process.argv.slice(2)); +const phase = options.phase || 'all'; +if (!['all', 'static', 'motion'].includes(phase)) { + throw new Error(`Unsupported validation phase: ${phase}`); +} +const manifest = path.resolve(options.manifest); +const targets = JSON.parse(await fs.readFile(path.resolve(options.targets), 'utf8')); +const outputDir = path.resolve(options['output-dir']); +await fs.mkdir(path.dirname(outputDir), { recursive: true }); +await fs.mkdir(outputDir, { recursive: false }); +const priorStaticReport = options['static-report'] + ? JSON.parse(await fs.readFile(path.resolve(options['static-report']), 'utf8')) + : null; +if (phase === 'motion' && !priorStaticReport) { + throw new Error('Motion-only validation requires --static-report.'); +} +if (priorStaticReport && priorStaticReport.summary?.passed !== true) { + throw new Error('The supplied static validation report did not pass.'); +} + +const report = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + manifest, + phase, + staticReport: options['static-report'] ? path.resolve(options['static-report']) : null, + viewport: VIEWPORT, + failClosedCheck: null, + staticViews: priorStaticReport?.staticViews || [], + motionFrames: [], + pageErrors: [], + networkErrors: [], + requestCancellations: [], + tileResponses: [], + failures: [] +}; + +let validationServer; +let browser; +try { + const failClosedServer = await startServer({ + manifest, + port: 4317, + allowPartial: false, + logPath: path.join(outputDir, 'server-fail-closed.log') + }); + try { + const rejected = await waitForReadiness(failClosedServer.origin, 503); + report.failClosedCheck = rejected; + } finally { + await stopServer(failClosedServer); + } + + validationServer = await startServer({ + manifest, + port: 4318, + allowPartial: true, + logPath: path.join(outputDir, 'server.log') + }); + const ready = await waitForReadiness(validationServer.origin, 200); + report.readiness = ready; + + browser = await chromium.launch({ + headless: true, + args: ['--disable-dev-shm-usage', '--no-sandbox'] + }); + const context = await browser.newContext({ + viewport: VIEWPORT, + deviceScaleFactor: 1 + }); + const page = await context.newPage(); + const outstandingTiles = new Set(); + page.on('pageerror', (error) => report.pageErrors.push(error.message)); + page.on('console', (message) => { + if (message.type() === 'error') report.pageErrors.push(message.text()); + }); + page.on('request', (request) => { + if (request.url().includes('/tiles/')) outstandingTiles.add(request); + }); + page.on('requestfinished', (request) => outstandingTiles.delete(request)); + page.on('requestfailed', (request) => { + outstandingTiles.delete(request); + const failure = request.failure()?.errorText || 'request failed'; + const entry = { url: request.url(), failure }; + if (failure.includes('ERR_ABORTED')) report.requestCancellations.push(entry); + else report.networkErrors.push(entry); + }); + page.on('response', (response) => { + if (response.url().includes('/tiles/')) { + report.tileResponses.push({ + url: response.url(), + status: response.status() + }); + } + if ( + response.url().startsWith(validationServer.origin) && + response.status() >= 400 + ) { + report.networkErrors.push({ + url: response.url(), + status: response.status() + }); + } + }); + + await page.goto(validationServer.origin, { + waitUntil: 'domcontentloaded', + timeout: 60_000 + }); + await page.addStyleTag({ + content: '.map-header,.maplibregl-control-container{display:none!important}' + }); + await page.waitForFunction(() => { + const map = globalThis.__OCCUMED_MAP__; + return map && map.getStyle()?.sources?.['occumed-open']; + }, { timeout: 60_000 }); + await page.evaluate(() => { + const map = globalThis.__OCCUMED_MAP__; + if (typeof map.setPixelRatio === 'function') map.setPixelRatio(1); + map.resize(); + map.triggerRepaint(); + }); + report.validatorPixelRatio = await page.evaluate(() => + globalThis.__OCCUMED_MAP__.getPixelRatio?.() || 1 + ); + + if (phase !== 'motion') { + for (const view of targets.staticViews) { + await waitForRenderedCamera(page, view); + await page.waitForTimeout(200); + await waitForRequestSettlement(outstandingTiles); + await pumpRenderFrames(page); + const state = await waitForStaticState(page, view.name); + const screenshot = await page.screenshot({ + path: path.join(outputDir, `static-${safeName(view.name)}.png`) + }); + const pixels = await analyzeScreenshot(page, screenshot, state.zoom); + applyFoundationBlankContract(pixels, state); + validateSourceState(state, `static ${view.name}`, report.failures); + validateRequiredLayers(view.name, state, report.failures); + validatePixels(pixels, `static ${view.name}`, report.failures); + report.staticViews.push({ + ...view, + state, + pixels, + file: `static-${safeName(view.name)}.png` + }); + console.log( + `[static] ${view.name}: sources=${state.sourceIds.length}, ` + + `layers=${state.sourceLayers.length}, seams=${pixels.verticalSeams + pixels.horizontalSeams}, ` + + `rectangles=${pixels.rectangularFootprints}, blank=${pixels.blank}.` + ); + } + } + + // The nine static views intentionally traverse the complete zoom pyramid and + // can leave hundreds of megabytes of decoded vector buckets in a software + // browser. Start the motion phase with a fresh page and an empty renderer + // cache; HTTP and PMTiles inputs remain unchanged. + if (phase === 'all') { + await waitForRequestSettlement(outstandingTiles); + await page.reload({ waitUntil: 'domcontentloaded', timeout: 60_000 }); + await page.addStyleTag({ + content: '.map-header,.maplibregl-control-container{display:none!important}' + }); + await page.waitForFunction(() => { + const map = globalThis.__OCCUMED_MAP__; + return map && map.getStyle()?.sources?.['occumed-open']; + }, { timeout: 60_000 }); + await page.evaluate(() => { + const map = globalThis.__OCCUMED_MAP__; + if (typeof map.setPixelRatio === 'function') map.setPixelRatio(1); + map.resize(); + map.triggerRepaint(); + }); + await page.waitForTimeout(200); + await waitForRequestSettlement(outstandingTiles); + } + + if (phase !== 'static') { + const selectedFrames = options.motion + ? targets.validationFrames.filter((frame) => frame.motion === options.motion) + : targets.validationFrames; + const framesByMotion = Map.groupBy(selectedFrames, (frame) => frame.motion); + for (const [motionName, frames] of framesByMotion) { + const ordered = [...frames].sort((left, right) => left.index - right.index); + const start = ordered[0]; + await waitForRenderedCamera(page, start); + await page.waitForTimeout(200); + await waitForRequestSettlement(outstandingTiles); + await pumpRenderFrames(page); + await waitForInViewTileSettlement(page); + + for (let index = 0; index < ordered.length; index += 1) { + if (index > 0) { + await page.evaluate(({ frame, duration }) => new Promise((resolve) => { + const map = globalThis.__OCCUMED_MAP__; + const finish = () => { + map.off('moveend', finish); + resolve(); + }; + map.on('moveend', finish); + map.easeTo({ + center: frame.center, + zoom: frame.zoom, + bearing: 0, + pitch: 0, + duration, + easing: (value) => value + }); + }), { + frame: ordered[index], + duration: MOTION_DURATION_MS / (ordered.length - 1) + }); + await waitForRequestSettlement(outstandingTiles); + } + + const state = await waitForInViewTileSettlement(page); + const filename = `motion-${safeName(motionName)}-${String(index).padStart(2, '0')}.png`; + const screenshot = await page.screenshot({ path: path.join(outputDir, filename) }); + const pixels = await analyzeScreenshot(page, screenshot, state.zoom); + applyFoundationBlankContract(pixels, state); + validateSourceState(state, `motion ${motionName} frame ${index}`, report.failures); + validatePixels(pixels, `motion ${motionName} frame ${index}`, report.failures); + report.motionFrames.push({ + motion: motionName, + index, + expectedCamera: ordered[index], + state, + pixels, + file: filename + }); + } + console.log(`[motion] ${motionName}: ${ordered.length} exact-camera transition frames captured.`); + } + } + + if (report.pageErrors.length) { + report.failures.push(`${report.pageErrors.length} browser/page errors occurred.`); + } + if (report.networkErrors.length) { + report.failures.push(`${report.networkErrors.length} network/resource errors occurred.`); + } + report.summary = { + staticViewCount: report.staticViews.length, + motionFrameCount: report.motionFrames.length, + blankFrames: [...report.staticViews, ...report.motionFrames] + .filter((capture) => capture.pixels.blank).length, + rectangularFootprints: [...report.staticViews, ...report.motionFrames] + .reduce((sum, capture) => sum + capture.pixels.rectangularFootprints, 0), + verticalSeams: [...report.staticViews, ...report.motionFrames] + .reduce((sum, capture) => sum + capture.pixels.verticalSeams, 0), + horizontalSeams: [...report.staticViews, ...report.motionFrames] + .reduce((sum, capture) => sum + capture.pixels.horizontalSeams, 0), + stretchedPolygons: [...report.staticViews, ...report.motionFrames] + .reduce((sum, capture) => sum + capture.pixels.stretchedPolygons, 0), + inconsistentNeighbors: [...report.staticViews, ...report.motionFrames] + .reduce((sum, capture) => sum + capture.pixels.inconsistentNeighbors, 0), + sourceSwitches: [...report.staticViews, ...report.motionFrames] + .filter((capture) => + capture.state.sourceIds.length !== 1 || + capture.state.sourceIds[0] !== 'occumed-open' + ).length, + pageErrors: report.pageErrors.length, + networkErrors: report.networkErrors.length, + requestCancellations: report.requestCancellations.length, + passed: report.failures.length === 0 + }; +} catch (error) { + report.failures.push(error?.stack || error?.message || String(error)); +} finally { + await browser?.close().catch(() => {}); + await stopServer(validationServer); + await fs.writeFile( + path.join(outputDir, 'immutable-visual-report.json'), + `${JSON.stringify(report, null, 2)}\n` + ); +} + +if (report.failures.length) { + console.error('Immutable visual validation failed:'); + for (const failure of report.failures) console.error(`- ${failure}`); + process.exit(1); +} +console.log( + `Immutable visual validation passed: ${report.summary.staticViewCount} static views, ` + + `${report.summary.motionFrameCount} exact-camera motion checkpoints, one source, and zero visual/network defects.` +); diff --git a/server.mjs b/server.mjs index e1d22297..70bb6855 100644 --- a/server.mjs +++ b/server.mjs @@ -3,26 +3,12 @@ import { createReadStream } from 'node:fs'; import fs from 'node:fs/promises'; import http from 'node:http'; import path from 'node:path'; -import { promisify } from 'node:util'; import { fileURLToPath } from 'node:url'; -import { gzip } from 'node:zlib'; -import { createNeonNavigationTileCacheFromEnv } from './src/server/neon-navigation-tile-cache.js'; -import { - GatewayOverloadedError, - WorldTileGateway -} from './src/server/world-tile-gateway.js'; -import { normalizeTileCoordinates } from './src/server/world-tile-routing.js'; -import { validateVectorTilePayload } from './src/server/tile-safety.js'; - -const gzipAsync = promisify(gzip); +import { ImmutableWorldTileset } from './src/server/immutable-world-tileset.js'; + const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), 'dist'); const port = Number(process.env.PORT || 4173); const host = process.env.HOST?.trim() || '0.0.0.0'; -const worldReleaseRepository = process.env.OCCUMED_WORLD_RELEASE_REPOSITORY?.trim() || 'Occumed79/Map'; -const worldReleaseTag = process.env.OCCUMED_WORLD_RELEASE_TAG?.trim() || 'occumed-world-v1'; -const worldManifestAsset = 'world-virtual-manifest.json'; -const worldSurfaceAsset = 'occumed-world-surface.pmtiles'; -const worldSurfaceUrl = process.env.OCCUMED_WORLD_SURFACE_URL?.trim(); const maxConcurrentTileRequests = Number(process.env.OCCUMED_MAX_CONCURRENT_TILE_REQUESTS || 64); const tileRequestTimeoutMs = Number(process.env.OCCUMED_TILE_REQUEST_TIMEOUT_MS || 30_000); const maxResolvedTileBytes = Number(process.env.OCCUMED_MAX_RESOLVED_TILE_BYTES || 24 * 1024 * 1024); @@ -137,7 +123,7 @@ function sendHealth(request, response) { async function sendReadiness(request, response) { try { const ready = await Promise.race([ - worldTileGateway.ready(), + immutableWorldTileset.ready(), new Promise((_, reject) => { const timer = setTimeout(() => reject(new Error('Readiness check timed out.')), 10_000); timer.unref?.(); @@ -167,19 +153,9 @@ async function serveStyle(request, response) { ); } -function releaseAssetUrl(assetName) { - if (assetName === worldSurfaceAsset && worldSurfaceUrl) return worldSurfaceUrl; - return `https://github.com/${worldReleaseRepository}/releases/download/${encodeURIComponent(worldReleaseTag)}/${encodeURIComponent(assetName)}`; -} - -const navigationTileCache = createNeonNavigationTileCacheFromEnv(); -const worldTileGateway = new WorldTileGateway({ - manifestUrl: - process.env.OCCUMED_WORLD_MANIFEST_URL?.trim() || - releaseAssetUrl(worldManifestAsset), - releaseAssetUrl, - persistentTileCache: navigationTileCache, - maxResolvedTileBytes: safeInteger( +const immutableWorldTileset = new ImmutableWorldTileset({ + root: path.resolve(path.dirname(fileURLToPath(import.meta.url))), + maxTileBytes: safeInteger( maxResolvedTileBytes, 24 * 1024 * 1024, 1_024, @@ -206,37 +182,45 @@ function tileEtag(tile) { async function serveVirtualTile(request, response, coordinates) { const concurrencyLimit = safeInteger(maxConcurrentTileRequests, 64, 4, 512); if (activeTileRequests >= concurrencyLimit) { - throw new GatewayOverloadedError('The HTTP tile concurrency limit has been reached.'); + const error = new Error('The HTTP tile concurrency limit has been reached.'); + error.code = 'OCCUMED_TILE_SERVER_OVERLOADED'; + error.statusCode = 503; + throw error; } activeTileRequests += 1; const startedAt = performance.now(); try { - const tile = await Promise.race([ - worldTileGateway.resolveTile(coordinates.z, coordinates.x, coordinates.y), + const resolved = await Promise.race([ + immutableWorldTileset.resolveTile(coordinates.z, coordinates.x, coordinates.y), timeoutAfter( safeInteger(tileRequestTimeoutMs, 30_000, 1_000, 120_000), `Tile ${coordinates.z}/${coordinates.x}/${coordinates.y} timed out.` ) ]); - validateVectorTilePayload(tile, { - label: `resolved tile ${coordinates.z}/${coordinates.x}/${coordinates.y}`, - maxBytes: safeInteger(maxResolvedTileBytes, 24 * 1024 * 1024, 1_024, 96 * 1024 * 1024) - }); + const tile = resolved.data; + if (!tile) { + writeHeaders(response, 204, { + 'Cache-Control': 'public, max-age=3600, must-revalidate, stale-if-error=86400', + 'X-Occumed-Tileset': resolved.artifactVersion, + 'X-Occumed-Tile-Owner': resolved.owner.id + }); + response.end(); + return; + } const etag = tileEtag(tile); if (request.headers['if-none-match'] === etag) { writeHeaders(response, 304, { 'Cache-Control': 'public, max-age=300, must-revalidate, stale-if-error=86400', ETag: etag, - 'X-Occumed-Tileset': 'virtual-worldwide-v2' + 'X-Occumed-Tileset': resolved.artifactVersion, + 'X-Occumed-Tile-Owner': resolved.owner.id }); response.end(); return; } - const acceptsGzip = /(?:^|,)\s*gzip\s*(?:,|$)/i.test(String(request.headers['accept-encoding'] || '')); - const payload = acceptsGzip ? await gzipAsync(tile, { level: 5 }) : tile; const cacheControl = 'public, max-age=300, must-revalidate, stale-while-revalidate=60, stale-if-error=86400'; const duration = Math.max(0, performance.now() - startedAt); @@ -244,16 +228,21 @@ async function serveVirtualTile(request, response, coordinates) { 'Cache-Control': cacheControl, 'CDN-Cache-Control': cacheControl, 'Surrogate-Control': cacheControl, - ...(acceptsGzip ? { 'Content-Encoding': 'gzip' } : {}), - 'Content-Length': payload.byteLength, + 'Content-Length': tile.byteLength, 'Content-Type': contentTypes['.pbf'], + ...(resolved.contentEncoding + ? { + 'Content-Encoding': resolved.contentEncoding, + Vary: 'Accept-Encoding' + } + : {}), ETag: etag, 'Server-Timing': `tile;dur=${duration.toFixed(1)}`, - Vary: 'Accept-Encoding', - 'X-Occumed-Tileset': 'virtual-worldwide-v2' + 'X-Occumed-Tileset': resolved.artifactVersion, + 'X-Occumed-Tile-Owner': resolved.owner.id }); if (request.method === 'HEAD') response.end(); - else response.end(payload); + else response.end(tile); } finally { activeTileRequests -= 1; } @@ -416,7 +405,6 @@ async function serveStatic(request, response, pathname) { } function tileErrorStatus(error) { - if (error instanceof GatewayOverloadedError) return 503; if (error instanceof RangeError) return 404; if (Number.isSafeInteger(error?.statusCode)) return error.statusCode; if (String(error?.code || '').startsWith('OCCUMED_')) return 503; @@ -456,10 +444,8 @@ async function handleRequest(request, response) { return; } if (url.pathname === '/internal/tile-health' && diagnosticsEnabled) { - sendJson(response, 200, { - activeTileRequests, - ...worldTileGateway.getHealthSnapshot() - }, 'no-store', method); + const ready = await immutableWorldTileset.ready().catch(() => ({ ready: false })); + sendJson(response, 200, { activeTileRequests, ...ready }, 'no-store', method); return; } if (url.pathname === '/style/occumed-open.json') { @@ -469,8 +455,23 @@ async function handleRequest(request, response) { const tileMatch = /^\/tiles\/(\d+)\/(\d+)\/(\d+)\.pbf$/.exec(url.pathname); if (tileMatch) { - const coordinates = normalizeTileCoordinates(tileMatch[1], tileMatch[2], tileMatch[3]); - if (!coordinates) { + const coordinates = { + z: Number(tileMatch[1]), + x: Number(tileMatch[2]), + y: Number(tileMatch[3]) + }; + const width = 2 ** coordinates.z; + if ( + !Number.isSafeInteger(coordinates.z) || + !Number.isSafeInteger(coordinates.x) || + !Number.isSafeInteger(coordinates.y) || + coordinates.z < 0 || + coordinates.z > 16 || + coordinates.x < 0 || + coordinates.y < 0 || + coordinates.x >= width || + coordinates.y >= width + ) { send(response, 404, 'Tile not found', 'text/plain; charset=utf-8', 'no-store', method); return; } @@ -558,7 +559,7 @@ function shutdown(signal) { shuttingDown = true; console.log(`Occu-Med Map received ${signal}; draining connections.`); server.close((error) => { - void worldTileGateway.close().finally(() => { + void immutableWorldTileset.close().finally(() => { if (error) { console.error('Occu-Med Map shutdown error:', error); process.exitCode = 1; @@ -567,7 +568,7 @@ function shutdown(signal) { }); const timer = setTimeout(() => { server.closeAllConnections?.(); - void worldTileGateway.close(); + void immutableWorldTileset.close(); process.exitCode = 1; }, 25_000); timer.unref?.(); @@ -579,16 +580,11 @@ process.once('SIGINT', () => shutdown('SIGINT')); server.listen(port, host, () => { console.log(`Occu-Med Map listening on ${host}:${port}.`); console.log(`Health endpoint ready at http://127.0.0.1:${port}/health.`); - if (navigationTileCache) { - const snapshot = navigationTileCache.snapshot(); - console.log(`Neon navigation cache configured with ${snapshot.configuredShards} of ${snapshot.expectedShards} shards.`); - void worldTileGateway.initializePersistentCache().then( - (initialized) => console.log(`Neon navigation cache initialized ${initialized} shard(s).`), - (error) => console.error(`Neon navigation cache initialization failed: ${error?.code || error?.name || 'UNKNOWN'}`) - ); - } - void worldTileGateway.ready().then( - (ready) => console.log(`Worldwide gateway ready with ${ready.regions} regional shards.`), - (error) => console.error('Worldwide gateway readiness failed:', error) + void immutableWorldTileset.ready().then( + (ready) => console.log( + `Immutable worldwide tileset ${ready.artifactVersion} ready ` + + `(${ready.builtOwnerCount}/${ready.plannedOwnerCount} owners).` + ), + (error) => console.error('Immutable worldwide tileset readiness failed:', error) ); }); diff --git a/src/occumed-map.js b/src/occumed-map.js index 96491093..556d47c1 100644 --- a/src/occumed-map.js +++ b/src/occumed-map.js @@ -11,7 +11,6 @@ const BLOOM_FADE_START_ZOOM = 2.85; const BLOOM_FADE_END_ZOOM = 4.25; const WORLD_MIN_ZOOM = 0; const WORLD_MAX_ZOOM = 16; -const WORLD_ZOOM_PYRAMID_LEVELS = WORLD_MAX_ZOOM - WORLD_MIN_ZOOM + 1; export function resolveOccumedPixelRatio() { const deviceRatio = Number(globalThis.devicePixelRatio); @@ -86,92 +85,22 @@ export function installOccumedAtmosphereBloom(map) { } /** - * Keeps decoded substitute tiles renderable across the complete 0–16 pyramid. + * Enforces exact prebuilt z/x/y addressing. * - * MapLibre's normal retention only searches in-view tiles while an ideal tile - * is loading. Fully decoded parents and children in its out-of-view cache are - * consequently skipped, leaving no renderable tile until the ideal request is - * parsed. Keep the same-source global foundation decoded as a last resort and - * reattach the nearest cached substitute before cleanup removes it. + * The immutable tileset contains every production zoom, so a missing ideal tile + * must remain missing rather than being painted from a stretched parent or + * child. This changes MapLibre's fallback depth only; it does not replace or + * wrap its retained-tile algorithm. */ -export function installContinuousTileRetention(map) { - let removed = false; - +export function installExactTileAddressing(map) { const configure = () => { - if (removed) return; const tileManager = map.style?.tileManagers?.['occumed-open']; if (!tileManager?.constructor) return; - tileManager.constructor.maxUnderzooming = Math.max( - Number(tileManager.constructor.maxUnderzooming || 0), - WORLD_ZOOM_PYRAMID_LEVELS - ); - tileManager.constructor.maxOverzooming = Math.max( - Number(tileManager.constructor.maxOverzooming || 0), - WORLD_ZOOM_PYRAMID_LEVELS - ); - if (tileManager.__occumedContinuousRetention) return; - - const updateRetainedTiles = tileManager._updateRetainedTiles.bind(tileManager); - let globalFoundationID = null; - tileManager._updateRetainedTiles = function retainCachedFoundation(idealTileIDs, zoom) { - const retained = updateRetainedTiles(idealTileIDs, zoom); - if (idealTileIDs.length && !globalFoundationID) { - globalFoundationID = idealTileIDs[0].scaledTo(0); - } - if (globalFoundationID) { - this._addTile(globalFoundationID); - retained[globalFoundationID.key] = globalFoundationID; - } - - for (const idealID of idealTileIDs) { - if (this.getTileByID(idealID.key)?.hasData()) continue; - - let foundAncestor = false; - for (let parentZoom = idealID.overscaledZ - 1; parentZoom >= 0; parentZoom -= 1) { - const parentID = idealID.scaledTo(parentZoom); - let parent = this.getTileByID(parentID.key); - if (!parent && this._outOfViewCache.has(parentID)) { - parent = this._addTile(parentID); - } - if (parent?.hasData()) { - retained[parentID.key] = parentID; - foundAncestor = true; - break; - } - } - if (foundAncestor) continue; - - const cachedChildren = Object.values(this._outOfViewCache.data) - .flat() - .map(({ value }) => value) - .filter((tile) => - tile.hasData() && - tile.tileID.isChildOf(idealID) && - tile.tileID.overscaledZ - idealID.overscaledZ <= WORLD_ZOOM_PYRAMID_LEVELS - ) - .map((tile) => tile.tileID.clone()); - if (!cachedChildren.length) continue; - - const nearestZoom = Math.min(...cachedChildren.map((tileID) => tileID.overscaledZ)); - for (const childID of cachedChildren) { - if (childID.overscaledZ !== nearestZoom) continue; - const child = this._addTile(childID); - if (child.hasData()) retained[childID.key] = childID; - } - } - - return retained; - }; - tileManager.__occumedContinuousRetention = true; + tileManager.constructor.maxUnderzooming = 0; + tileManager.constructor.maxOverzooming = 0; }; - - const remove = () => { - removed = true; - map.off('styledata', configure); - }; - map.on('styledata', configure); - map.once('remove', remove); + map.once('remove', () => map.off('styledata', configure)); configure(); } @@ -254,11 +183,7 @@ export async function createOccumedMap({ hash: false, pixelRatio: resolveOccumedPixelRatio(), antialias: true, - // MapLibre only retains pending smaller-zoom requests during zoom-in when - // cancellation is disabled. Reverse zooms also require the already-loaded - // parent pyramid to remain in cache, so retain every zoom level from 0–16. - cancelPendingTileRequestsWhileZooming: false, - maxTileCacheZoomLevels: WORLD_ZOOM_PYRAMID_LEVELS, + cancelPendingTileRequestsWhileZooming: true, refreshExpiredTiles: false, fadeDuration: 300, renderWorldCopies: false, @@ -267,7 +192,7 @@ export async function createOccumedMap({ ...mapOptions }); - installContinuousTileRetention(map); + installExactTileAddressing(map); installOccumedAtmosphereBloom(map); if (controls) { diff --git a/src/server/immutable-world-tileset.js b/src/server/immutable-world-tileset.js new file mode 100644 index 00000000..f32d7739 --- /dev/null +++ b/src/server/immutable-world-tileset.js @@ -0,0 +1,504 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + Compression, + FetchSource, + findTile, + PMTiles, + SharedPromiseCache, + zxyToTileId +} from 'pmtiles'; + +const DEFAULT_MANIFEST = 'dist/immutable-world/manifest.json'; +const MANIFEST_SCHEMA_VERSION = 1; +const MAX_MANIFEST_BYTES = 8 * 1024 * 1024; +const MAX_OWNER_COUNT = 16_384; + +class LocalArchiveSource { + constructor(filename) { + this.filename = path.resolve(filename); + this.handlePromise = fs.open(this.filename, 'r'); + this.sizePromise = this.handlePromise.then((handle) => handle.stat()).then((stat) => stat.size); + this.closed = false; + } + + getKey() { + return this.filename; + } + + async getBytes(offset, length) { + if (this.closed) throw new Error(`Immutable PMTiles source is closed: ${this.filename}`); + const start = Number(offset); + const requested = Number(length); + const [handle, fileSize] = await Promise.all([this.handlePromise, this.sizePromise]); + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(requested) || + start < 0 || + requested <= 0 || + start >= fileSize + ) { + throw new RangeError(`Invalid immutable PMTiles range ${offset}+${length}.`); + } + const size = Math.min(requested, fileSize - start); + const buffer = Buffer.allocUnsafe(size); + const { bytesRead } = await handle.read(buffer, 0, size, start); + if (bytesRead !== size) { + throw new Error( + `Short immutable PMTiles read from ${this.filename}: expected ${size}, received ${bytesRead}.` + ); + } + return { + data: buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) + }; + } + + async close() { + if (this.closed) return; + this.closed = true; + await (await this.handlePromise).close(); + } +} + +function isHttpUrl(value) { + try { + const url = new URL(value); + return ['http:', 'https:'].includes(url.protocol); + } catch { + return false; + } +} + +function normalizeCoordinate(value, label) { + const number = Number(value); + if (!Number.isSafeInteger(number) || number < 0) { + throw new TypeError(`${label} must be a non-negative safe integer.`); + } + return number; +} + +function normalizeTile(zValue, xValue, yValue, maxZoom) { + const z = normalizeCoordinate(zValue, 'Tile z'); + const x = normalizeCoordinate(xValue, 'Tile x'); + const y = normalizeCoordinate(yValue, 'Tile y'); + if (z > maxZoom) return null; + const width = 2 ** z; + if (x >= width || y >= width) return null; + return { z, x, y }; +} + +function canonicalDocument(value) { + if (Array.isArray(value)) return value.map(canonicalDocument); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries( + Object.keys(value) + .sort() + .filter((key) => !['artifactVersion', 'generatedAt'].includes(key)) + .map((key) => [key, canonicalDocument(value[key])]) + ); +} + +export function computeImmutableArtifactVersion(manifest) { + return createHash('sha256') + .update(`${JSON.stringify(canonicalDocument(manifest))}\n`) + .digest('hex'); +} + +function validateSha(value, label) { + if (!/^[a-f0-9]{64}$/.test(String(value || ''))) { + throw new TypeError(`${label} must be a lowercase SHA-256 digest.`); + } +} + +function validateAsset(asset, label) { + if (!asset || typeof asset !== 'object') throw new TypeError(`${label} is missing.`); + if (!/^[a-z0-9][a-z0-9._/-]*\.pmtiles$/.test(String(asset.file || ''))) { + throw new TypeError(`${label}.file is not a safe PMTiles path.`); + } + if (String(asset.file).includes('..') || path.isAbsolute(String(asset.file))) { + throw new TypeError(`${label}.file must stay beneath the immutable asset root.`); + } + if (!Number.isSafeInteger(asset.bytes) || asset.bytes < 127) { + throw new TypeError(`${label}.bytes is invalid.`); + } + validateSha(asset.sha256, `${label}.sha256`); +} + +function prefixContains(ancestor, candidate) { + if (ancestor.z > candidate.z) return false; + const divisor = 2 ** (candidate.z - ancestor.z); + return ( + Math.floor(candidate.x / divisor) === ancestor.x && + Math.floor(candidate.y / divisor) === ancestor.y + ); +} + +function normalizePrefix(prefix, label) { + const z = normalizeCoordinate(prefix?.z, `${label}.z`); + const x = normalizeCoordinate(prefix?.x, `${label}.x`); + const y = normalizeCoordinate(prefix?.y, `${label}.y`); + const width = 2 ** z; + if (z > 16 || x >= width || y >= width) { + throw new TypeError(`${label} is outside the z0-z16 tile pyramid.`); + } + return { z, x, y }; +} + +export function validateImmutableManifest(document, { + allowPartial = false +} = {}) { + if (!document || typeof document !== 'object') throw new TypeError('Immutable tileset manifest is missing.'); + if (document.schemaVersion !== MANIFEST_SCHEMA_VERSION) { + throw new TypeError(`Unsupported immutable manifest schema: ${document.schemaVersion}`); + } + if (document.browserSourceId !== 'occumed-open') { + throw new TypeError('Immutable manifest browser source must remain occumed-open.'); + } + if (document.maxZoom !== 16 || document.minZoom !== 0) { + throw new TypeError('Immutable manifest must cover the z0-z16 pyramid.'); + } + if ( + document.authorities?.land !== 'world-surface' || + document.authorities?.depth !== 'world-surface' || + document.authorities?.landcover !== 'world-overview' || + document.authorities?.cartography !== 'regional-owner' + ) { + throw new TypeError('Immutable manifest has an invalid source authority matrix.'); + } + for (const forbidden of [ + 'neonTileCache', + 'runtimeShardMerge', + 'runtimeLandcoverSynthesis', + 'runtimeGeometry', + 'parentChildStretching' + ]) { + if (document.runtimePolicy?.[forbidden] !== false) { + throw new TypeError(`Immutable manifest does not disable ${forbidden}.`); + } + } + + validateAsset(document.foundation, 'foundation'); + if (!Number.isSafeInteger(document.foundation.maxZoom) || + document.foundation.maxZoom < 0 || + document.foundation.maxZoom > 16) { + throw new TypeError('foundation.maxZoom is invalid.'); + } + const owners = Array.isArray(document.owners) ? document.owners : []; + if (owners.length > MAX_OWNER_COUNT) throw new TypeError('Immutable owner count is unsafe.'); + if (!Number.isSafeInteger(document.plannedOwnerCount) || document.plannedOwnerCount < 0) { + throw new TypeError('plannedOwnerCount is invalid.'); + } + if (!Number.isSafeInteger(document.builtOwnerCount) || + document.builtOwnerCount !== owners.length) { + throw new TypeError('builtOwnerCount does not match the owner inventory.'); + } + if (!document.complete && !allowPartial) { + throw new Error( + `Immutable tileset is incomplete: ${owners.length} of ${document.plannedOwnerCount} owners.` + ); + } + if (document.complete && owners.length !== document.plannedOwnerCount) { + throw new Error('A complete immutable tileset must include every planned owner.'); + } + if (document.defaultOwner !== document.foundation.id) { + throw new TypeError('The worldwide default owner must be the prebuilt foundation.'); + } + + const ids = new Set([document.foundation.id]); + const files = new Set([document.foundation.file]); + const prefixes = []; + const exactTileOwners = new Map(); + const normalizedOwners = owners.map((owner, index) => { + validateAsset(owner, `owners[${index}]`); + if (!/^[a-z0-9][a-z0-9-]*$/.test(String(owner.id || ''))) { + throw new TypeError(`owners[${index}].id is invalid.`); + } + if (ids.has(owner.id)) throw new TypeError(`Duplicate immutable owner id: ${owner.id}`); + if (files.has(owner.file)) throw new TypeError(`Duplicate immutable owner file: ${owner.file}`); + ids.add(owner.id); + files.add(owner.file); + const prefix = normalizePrefix(owner.prefix, `owners[${index}].prefix`); + for (const existing of prefixes) { + if (prefixContains(existing, prefix) || prefixContains(prefix, existing)) { + throw new TypeError( + `Overlapping immutable owner prefixes: ${existing.z}/${existing.x}/${existing.y} and ` + + `${prefix.z}/${prefix.x}/${prefix.y}.` + ); + } + } + prefixes.push(prefix); + const exactTiles = (owner.exactTiles || []).map((tile, tileIndex) => { + const exact = normalizePrefix(tile, `owners[${index}].exactTiles[${tileIndex}]`); + if (exact.z <= document.foundation.maxZoom) { + throw new TypeError( + `Owner exact tile ${exact.z}/${exact.x}/${exact.y} overlaps the foundation.` + ); + } + const exactKey = `${exact.z}/${exact.x}/${exact.y}`; + if (exactTileOwners.has(exactKey)) { + throw new TypeError( + `Exact tile ${exactKey} is assigned to both ${exactTileOwners.get(exactKey)} and ${owner.id}.` + ); + } + exactTileOwners.set(exactKey, owner.id); + return exact; + }); + return { ...owner, prefix, exactTiles }; + }); + for (const [exactKey, exactOwner] of exactTileOwners) { + const [z, x, y] = exactKey.split('/').map(Number); + const matchingPrefixes = normalizedOwners.filter((owner) => + prefixContains(owner.prefix, { z, x, y }) + ); + if (matchingPrefixes.length) { + throw new TypeError( + `Exact tile ${exactKey} for ${exactOwner} overlaps prefix owner ` + + `${matchingPrefixes[0].id}.` + ); + } + } + + const computedVersion = computeImmutableArtifactVersion(document); + if (document.artifactVersion !== computedVersion) { + throw new Error( + `Immutable artifact version mismatch: expected ${computedVersion}, ` + + `received ${document.artifactVersion}.` + ); + } + return { + ...document, + owners: normalizedOwners + }; +} + +async function readManifest(location) { + if (isHttpUrl(location)) { + const response = await fetch(location, { + redirect: 'follow', + headers: { 'User-Agent': 'Occu-Med-Map/immutable-tileset' } + }); + if (!response.ok) throw new Error(`Immutable manifest returned HTTP ${response.status}.`); + const contentLength = Number(response.headers.get('content-length')); + if (Number.isFinite(contentLength) && contentLength > MAX_MANIFEST_BYTES) { + throw new Error('Immutable manifest exceeds the size budget.'); + } + const text = await response.text(); + if (Buffer.byteLength(text) > MAX_MANIFEST_BYTES) { + throw new Error('Immutable manifest exceeds the size budget.'); + } + return JSON.parse(text); + } + + const stat = await fs.stat(location); + if (!stat.isFile() || stat.size > MAX_MANIFEST_BYTES) { + throw new Error('Immutable manifest is missing or exceeds the size budget.'); + } + return JSON.parse(await fs.readFile(location, 'utf8')); +} + +function resolveRemoteAsset(baseUrl, file) { + const base = new URL(baseUrl); + if (!['http:', 'https:'].includes(base.protocol)) { + throw new TypeError('Immutable assetBaseUrl must use HTTP or HTTPS.'); + } + return new URL(file, base.href.endsWith('/') ? base.href : `${base.href}/`).href; +} + +function buildOwnerIndex(manifest) { + const exact = new Map(); + const prefixes = new Map(); + for (const owner of manifest.owners) { + for (const tile of owner.exactTiles) { + exact.set(`${tile.z}/${tile.x}/${tile.y}`, owner); + } + const zoomOwners = prefixes.get(owner.prefix.z) || new Map(); + zoomOwners.set(`${owner.prefix.x}/${owner.prefix.y}`, owner); + prefixes.set(owner.prefix.z, zoomOwners); + } + return { + exact, + prefixes, + prefixZooms: [...prefixes.keys()].sort((left, right) => right - left) + }; +} + +export class ImmutableWorldTileset { + constructor({ + root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'), + manifestLocation = process.env.OCCUMED_IMMUTABLE_TILESET_MANIFEST?.trim(), + allowPartial = process.env.OCCUMED_ALLOW_PARTIAL_TILESET_FIXTURE === 'true', + maxTileBytes = 32 * 1024 * 1024 + } = {}) { + this.root = path.resolve(root); + this.manifestLocation = manifestLocation + ? (isHttpUrl(manifestLocation) ? manifestLocation : path.resolve(manifestLocation)) + : path.join(this.root, DEFAULT_MANIFEST); + this.allowPartial = allowPartial; + this.maxTileBytes = maxTileBytes; + this.manifestPromise = null; + this.ownerIndex = null; + this.archives = new Map(); + this.closed = false; + } + + async loadManifest() { + if (this.closed) throw new Error('Immutable tileset is closed.'); + if (!this.manifestPromise) { + this.manifestPromise = readManifest(this.manifestLocation) + .then((document) => validateImmutableManifest(document, { + allowPartial: this.allowPartial + })) + .then((manifest) => { + this.ownerIndex = buildOwnerIndex(manifest); + return manifest; + }) + .catch((error) => { + this.manifestPromise = null; + this.ownerIndex = null; + throw error; + }); + } + return this.manifestPromise; + } + + assetLocation(manifest, asset) { + if (manifest.assetBaseUrl) return resolveRemoteAsset(manifest.assetBaseUrl, asset.file); + if (isHttpUrl(this.manifestLocation)) { + return new URL(asset.file, this.manifestLocation).href; + } + const assetRoot = path.dirname(this.manifestLocation); + const resolved = path.resolve(assetRoot, asset.file); + if (!resolved.startsWith(`${assetRoot}${path.sep}`)) { + throw new Error(`Immutable asset escapes its manifest directory: ${asset.file}`); + } + return resolved; + } + + async openArchive(manifest, asset) { + const location = this.assetLocation(manifest, asset); + if (this.archives.has(location)) return this.archives.get(location); + const source = isHttpUrl(location) + ? new FetchSource(location) + : new LocalArchiveSource(location); + const archive = new PMTiles(source, new SharedPromiseCache(256)); + const opened = Promise.all([ + archive.getHeader(), + isHttpUrl(location) ? Promise.resolve(null) : fs.stat(location) + ]).then(([header, stat]) => { + if (header.specVersion !== 3 || header.tileType !== 1) { + throw new Error(`Immutable asset is not PMTiles v3 MVT: ${asset.file}`); + } + if (stat && stat.size !== asset.bytes) { + throw new Error( + `Immutable asset size mismatch for ${asset.file}: expected ${asset.bytes}, received ${stat.size}.` + ); + } + return { archive, source, header, asset, location }; + }); + this.archives.set(location, opened); + return opened; + } + + selectOwner(manifest, tile) { + if (tile.z <= manifest.foundation.maxZoom) return manifest.foundation; + if (!this.ownerIndex) throw new Error('Immutable owner index is not initialized.'); + const exact = this.ownerIndex.exact.get(`${tile.z}/${tile.x}/${tile.y}`); + if (exact) return exact; + for (const prefixZoom of this.ownerIndex.prefixZooms) { + if (prefixZoom > tile.z) continue; + const divisor = 2 ** (tile.z - prefixZoom); + const owner = this.ownerIndex.prefixes.get(prefixZoom).get( + `${Math.floor(tile.x / divisor)}/${Math.floor(tile.y / divisor)}` + ); + if (owner) return owner; + } + return manifest.foundation; + } + + async resolveTile(zValue, xValue, yValue) { + const manifest = await this.loadManifest(); + const tile = normalizeTile(zValue, xValue, yValue, manifest.maxZoom); + if (!tile) throw new RangeError('Invalid immutable tile coordinates.'); + const owner = this.selectOwner(manifest, tile); + const opened = await this.openArchive(manifest, owner); + const tileId = zxyToTileId(tile.z, tile.x, tile.y); + let directoryOffset = opened.header.rootDirectoryOffset; + let directoryLength = opened.header.rootDirectoryLength; + let data = null; + for (let depth = 0; depth <= 3; depth += 1) { + const directory = await opened.archive.cache.getDirectory( + opened.source, + directoryOffset, + directoryLength, + opened.header + ); + const entry = findTile(directory, tileId); + if (!entry) break; + if (entry.runLength > 0) { + const stored = await opened.source.getBytes( + opened.header.tileDataOffset + entry.offset, + entry.length + ); + data = Buffer.from(stored.data); + break; + } + directoryOffset = opened.header.leafDirectoryOffset + entry.offset; + directoryLength = entry.length; + } + if (!data) return { data: null, owner, tile, artifactVersion: manifest.artifactVersion }; + if (data.byteLength > this.maxTileBytes) { + throw new Error( + `Immutable tile ${tile.z}/${tile.x}/${tile.y} exceeds the production byte limit.` + ); + } + let contentEncoding = null; + if (opened.header.tileCompression === Compression.Gzip) contentEncoding = 'gzip'; + else if ( + opened.header.tileCompression !== Compression.None && + opened.header.tileCompression !== Compression.Unknown + ) { + throw new Error( + `Immutable tile compression is not HTTP-safe: ${opened.header.tileCompression}.` + ); + } + return { + data, + contentEncoding, + owner, + tile, + artifactVersion: manifest.artifactVersion + }; + } + + async ready() { + const manifest = await this.loadManifest(); + const foundation = await this.openArchive(manifest, manifest.foundation); + return { + ready: true, + architecture: 'immutable-prebuilt-pmtiles-v1', + artifactVersion: manifest.artifactVersion, + complete: manifest.complete, + builtOwnerCount: manifest.builtOwnerCount, + plannedOwnerCount: manifest.plannedOwnerCount, + foundationTiles: foundation.header.numAddressedTiles, + sourceCount: 1 + }; + } + + async close() { + if (this.closed) return; + this.closed = true; + const opened = await Promise.allSettled([...this.archives.values()]); + await Promise.allSettled( + opened + .filter((result) => result.status === 'fulfilled') + .map(({ value }) => value.source?.close?.()) + ); + this.archives.clear(); + } +}