diff --git a/.github/workflows/bootstrap-custom-flat-pmtiles.yml b/.github/workflows/bootstrap-custom-flat-pmtiles.yml new file mode 100644 index 00000000..cca7ace8 --- /dev/null +++ b/.github/workflows/bootstrap-custom-flat-pmtiles.yml @@ -0,0 +1,97 @@ +name: Harden Custom Flat Motion Fixture + +on: + push: + branches: + - build/custom-flat-from-globe-tiles + +permissions: + contents: write + +concurrency: + group: harden-custom-flat-motion-fixture + cancel-in-progress: false + +jobs: + patch: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + ref: build/custom-flat-from-globe-tiles + fetch-depth: 0 + + - name: Check whether the motion fixture is already owner-complete + id: guard + run: | + if grep -q 'const fresnoGlobal =' scripts/offline-tileset/generate-representative-targets.mjs; then + echo "run=false" >> "$GITHUB_OUTPUT" + echo "Owner-complete motion fixture is already installed." + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi + + - name: Keep world-to-street motion inside the Fresno owner + if: steps.guard.outputs.run == 'true' + run: | + python - <<'PY' + from pathlib import Path + + path = Path('scripts/offline-tileset/generate-representative-targets.mjs') + source = path.read_text() + + old_setup = '''const global = { center: [-20, 18], zoom: 2.2 }; + const fresnoStreet = { center: [-119.7871, 36.7378], zoom: 16 }; + const antimeridianEast = { center: [179.65, 8], zoom: 8 };''' + new_setup = '''const global = { center: [-20, 18], zoom: 2.2 }; + const fresnoStreet = { center: [-119.7871, 36.7378], zoom: 16 }; + const fresnoGlobal = { center: fresnoStreet.center, zoom: global.zoom }; + const antimeridianEast = { center: [179.65, 8], zoom: 8 };''' + if old_setup not in source: + raise SystemExit('Could not find representative camera setup.') + source = source.replace(old_setup, new_setup, 1) + + old_motions = '''const motions = [ + { name: 'global-to-street', start: global, end: fresnoStreet }, + { name: 'street-to-global', start: fresnoStreet, end: global }, + { name: 'antimeridian-crossing', start: antimeridianEast, end: antimeridianWest } + ];''' + new_motions = '''const motions = [ + { name: 'global-to-street', start: fresnoGlobal, end: fresnoStreet }, + { name: 'street-to-global', start: fresnoStreet, end: fresnoGlobal }, + { name: 'antimeridian-crossing', start: antimeridianEast, end: antimeridianWest } + ];''' + if old_motions not in source: + raise SystemExit('Could not find representative motion definitions.') + source = source.replace(old_motions, new_motions, 1) + + old_owner = ''' const owner = [...ownerById.values()].find((candidate) => + prefixContains(candidate.prefix, tile) + ); + const target = owner ? targetMaps[owner.id] : targetMaps.foundation; + target.set(tileKey(tile), tile);''' + new_owner = ''' const owner = [...ownerById.values()].find((candidate) => + prefixContains(candidate.prefix, tile) + ); + if (!owner) { + throw new Error( + `Representative high-zoom tile ${tileKey(tile)} is outside every selected owner.` + ); + } + targetMaps[owner.id].set(tileKey(tile), tile);''' + if old_owner not in source: + raise SystemExit('Could not find representative owner fallback.') + source = source.replace(old_owner, new_owner, 1) + + path.write_text(source) + PY + + - name: Commit owner-complete motion fixture + if: steps.guard.outputs.run == 'true' + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add scripts/offline-tileset/generate-representative-targets.mjs + git commit -m 'Require owner-complete flat-map motion fixtures [motion-owner-lock]' + git push origin HEAD:build/custom-flat-from-globe-tiles diff --git a/.github/workflows/build-complete-flat-production.yml b/.github/workflows/build-complete-flat-production.yml new file mode 100644 index 00000000..5080b825 --- /dev/null +++ b/.github/workflows/build-complete-flat-production.yml @@ -0,0 +1,312 @@ +name: Build Complete Flat Production Tileset + +on: + pull_request: + branches: + - main + paths: + - .github/workflows/build-complete-flat-production.yml + - package.json + - package-lock.json + - scripts/offline-tileset/** + - src/server/immutable-world-tileset.js + - server.mjs + +permissions: + contents: write + +concurrency: + group: complete-flat-production-v1 + cancel-in-progress: false + +env: + PRODUCTION_TAG: occumed-flat-v1 + PRODUCTION_ROOT: build/complete-flat-production + +jobs: + prepare: + if: github.head_ref == 'build/custom-flat-from-globe-tiles' + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + wave0: ${{ steps.matrix.outputs.wave0 }} + wave1: ${{ steps.matrix.outputs.wave1 }} + wave2: ${{ steps.matrix.outputs.wave2 }} + batch_count: ${{ steps.matrix.outputs.batch_count }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + + - run: npm ci + + - name: Create immutable world and spatial batch plans + run: | + mkdir -p "$PRODUCTION_ROOT/plans" + npm run tiles:plan-world -- \ + --output "$PRODUCTION_ROOT/plans/immutable-owner-plan.json" + npm run tiles:plan-production-batches -- \ + --plan "$PRODUCTION_ROOT/plans/immutable-owner-plan.json" \ + --batch-count 768 \ + --tag "$PRODUCTION_TAG" \ + --output "$PRODUCTION_ROOT/plans/production-batch-plan.json" + + - name: Create three legal GitHub matrix waves + id: matrix + run: | + node - <<'NODE' + const fs = require('fs'); + const plan = JSON.parse(fs.readFileSync(process.env.PRODUCTION_ROOT + '/plans/production-batch-plan.json')); + const output = process.env.GITHUB_OUTPUT; + for (let wave = 0; wave < 3; wave += 1) { + const include = plan.batches + .slice(wave * 256, (wave + 1) * 256) + .map(({ index, id, file }) => ({ index, id, file })); + fs.appendFileSync(output, `wave${wave}=${JSON.stringify({ include })}\n`); + } + fs.appendFileSync(output, `batch_count=${plan.batchCount}\n`); + if (plan.batchCount > 768) throw new Error(`Batch plan exceeds workflow capacity: ${plan.batchCount}`); + if (!plan.batchCount) throw new Error('Batch plan is empty.'); + NODE + + - name: Create public prerelease for immutable production assets + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release view "$PRODUCTION_TAG" >/dev/null 2>&1 || \ + gh release create "$PRODUCTION_TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --prerelease \ + --title "Occu-Med immutable flat map v1" \ + --notes "Complete offline-compiled worldwide PMTiles. Do not activate until manifest validation passes." + + - uses: actions/upload-artifact@v4 + with: + name: complete-flat-production-plans + path: ${{ env.PRODUCTION_ROOT }}/plans/** + retention-days: 7 + + foundation: + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + + - run: npm ci + + - uses: actions/download-artifact@v4 + with: + name: complete-flat-production-plans + path: ${{ env.PRODUCTION_ROOT }}/plans + + - name: Build immutable worldwide foundation + run: | + mkdir -p "$PRODUCTION_ROOT/foundation" + npm run tiles:targets-foundation -- \ + --plan "$PRODUCTION_ROOT/plans/immutable-owner-plan.json" \ + --output "$PRODUCTION_ROOT/foundation-targets.json" + npm run tiles:build-production-foundation -- \ + --plan "$PRODUCTION_ROOT/plans/immutable-owner-plan.json" \ + --targets "$PRODUCTION_ROOT/foundation-targets.json" \ + --output-dir "$PRODUCTION_ROOT/foundation" \ + --concurrency 8 + + - name: Publish SHA-locked foundation + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release upload "$PRODUCTION_TAG" \ + "$PRODUCTION_ROOT/foundation/foundation.pmtiles#foundation.pmtiles" \ + --repo "$GITHUB_REPOSITORY" \ + --clobber + + build-wave-0: + needs: prepare + strategy: + fail-fast: false + max-parallel: 16 + matrix: ${{ fromJSON(needs.prepare.outputs.wave0) }} + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + - run: npm ci + - uses: actions/download-artifact@v4 + with: + name: complete-flat-production-plans + path: ${{ env.PRODUCTION_ROOT }}/plans + - name: Build exact spatial batch ${{ matrix.id }} + run: | + mkdir -p "$PRODUCTION_ROOT/jobs/${{ matrix.id }}" + npm run tiles:targets-production-batch -- \ + --plan "$PRODUCTION_ROOT/plans/immutable-owner-plan.json" \ + --batch-plan "$PRODUCTION_ROOT/plans/production-batch-plan.json" \ + --batch-index "${{ matrix.index }}" \ + --output "$PRODUCTION_ROOT/jobs/${{ matrix.id }}/targets.json" + npm run tiles:build-production-batch -- \ + --plan "$PRODUCTION_ROOT/plans/immutable-owner-plan.json" \ + --batch-plan "$PRODUCTION_ROOT/plans/production-batch-plan.json" \ + --batch-index "${{ matrix.index }}" \ + --targets "$PRODUCTION_ROOT/jobs/${{ matrix.id }}/targets.json" \ + --output-dir "$PRODUCTION_ROOT/jobs/${{ matrix.id }}" \ + --max-bytes 1900000000 \ + --concurrency 8 + - name: Publish ${{ matrix.file }} + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release upload "$PRODUCTION_TAG" \ + "$PRODUCTION_ROOT/jobs/${{ matrix.id }}/batches/${{ matrix.file }}#${{ matrix.file }}" \ + --repo "$GITHUB_REPOSITORY" \ + --clobber + + build-wave-1: + needs: prepare + strategy: + fail-fast: false + max-parallel: 16 + matrix: ${{ fromJSON(needs.prepare.outputs.wave1) }} + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + - run: npm ci + - uses: actions/download-artifact@v4 + with: + name: complete-flat-production-plans + path: ${{ env.PRODUCTION_ROOT }}/plans + - name: Build exact spatial batch ${{ matrix.id }} + run: | + mkdir -p "$PRODUCTION_ROOT/jobs/${{ matrix.id }}" + npm run tiles:targets-production-batch -- \ + --plan "$PRODUCTION_ROOT/plans/immutable-owner-plan.json" \ + --batch-plan "$PRODUCTION_ROOT/plans/production-batch-plan.json" \ + --batch-index "${{ matrix.index }}" \ + --output "$PRODUCTION_ROOT/jobs/${{ matrix.id }}/targets.json" + npm run tiles:build-production-batch -- \ + --plan "$PRODUCTION_ROOT/plans/immutable-owner-plan.json" \ + --batch-plan "$PRODUCTION_ROOT/plans/production-batch-plan.json" \ + --batch-index "${{ matrix.index }}" \ + --targets "$PRODUCTION_ROOT/jobs/${{ matrix.id }}/targets.json" \ + --output-dir "$PRODUCTION_ROOT/jobs/${{ matrix.id }}" \ + --max-bytes 1900000000 \ + --concurrency 8 + - name: Publish ${{ matrix.file }} + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release upload "$PRODUCTION_TAG" \ + "$PRODUCTION_ROOT/jobs/${{ matrix.id }}/batches/${{ matrix.file }}#${{ matrix.file }}" \ + --repo "$GITHUB_REPOSITORY" \ + --clobber + + build-wave-2: + needs: prepare + strategy: + fail-fast: false + max-parallel: 16 + matrix: ${{ fromJSON(needs.prepare.outputs.wave2) }} + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + - run: npm ci + - uses: actions/download-artifact@v4 + with: + name: complete-flat-production-plans + path: ${{ env.PRODUCTION_ROOT }}/plans + - name: Build exact spatial batch ${{ matrix.id }} + run: | + mkdir -p "$PRODUCTION_ROOT/jobs/${{ matrix.id }}" + npm run tiles:targets-production-batch -- \ + --plan "$PRODUCTION_ROOT/plans/immutable-owner-plan.json" \ + --batch-plan "$PRODUCTION_ROOT/plans/production-batch-plan.json" \ + --batch-index "${{ matrix.index }}" \ + --output "$PRODUCTION_ROOT/jobs/${{ matrix.id }}/targets.json" + npm run tiles:build-production-batch -- \ + --plan "$PRODUCTION_ROOT/plans/immutable-owner-plan.json" \ + --batch-plan "$PRODUCTION_ROOT/plans/production-batch-plan.json" \ + --batch-index "${{ matrix.index }}" \ + --targets "$PRODUCTION_ROOT/jobs/${{ matrix.id }}/targets.json" \ + --output-dir "$PRODUCTION_ROOT/jobs/${{ matrix.id }}" \ + --max-bytes 1900000000 \ + --concurrency 8 + - name: Publish ${{ matrix.file }} + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release upload "$PRODUCTION_TAG" \ + "$PRODUCTION_ROOT/jobs/${{ matrix.id }}/batches/${{ matrix.file }}#${{ matrix.file }}" \ + --repo "$GITHUB_REPOSITORY" \ + --clobber + + finalize: + needs: + - prepare + - foundation + - build-wave-0 + - build-wave-1 + - build-wave-2 + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + + - run: npm ci + + - uses: actions/download-artifact@v4 + with: + name: complete-flat-production-plans + path: ${{ env.PRODUCTION_ROOT }}/plans + + - name: Assemble and validate the complete production manifest + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + npm run tiles:finalize-production -- \ + --plan "$PRODUCTION_ROOT/plans/immutable-owner-plan.json" \ + --batch-plan "$PRODUCTION_ROOT/plans/production-batch-plan.json" \ + --repository "$GITHUB_REPOSITORY" \ + --tag "$PRODUCTION_TAG" \ + --output "$PRODUCTION_ROOT/manifest.json" + + - name: Publish complete production manifest + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release upload "$PRODUCTION_TAG" \ + "$PRODUCTION_ROOT/manifest.json#manifest.json" \ + --repo "$GITHUB_REPOSITORY" \ + --clobber + + - uses: actions/upload-artifact@v4 + with: + name: complete-flat-production-manifest + path: ${{ env.PRODUCTION_ROOT }}/manifest.json + retention-days: 30 diff --git a/.github/workflows/validate-custom-flat-pmtiles.yml b/.github/workflows/validate-custom-flat-pmtiles.yml new file mode 100644 index 00000000..0191376c --- /dev/null +++ b/.github/workflows/validate-custom-flat-pmtiles.yml @@ -0,0 +1,177 @@ +name: Validate Custom Flat PMTiles + +on: + pull_request: + branches: + - main + paths: + - .github/workflows/validate-custom-flat-pmtiles.yml + - package-lock.json + - package.json + - server.mjs + - style.json + - src/** + - scripts/offline-tileset/** + - scripts/apply-flat-custom-parity.mjs + - scripts/check-custom-flat-tiles.mjs + - scripts/build-runtime-style.mjs + - scripts/validate-immutable-visuals.mjs + +permissions: + contents: write + +concurrency: + group: custom-flat-pmtiles-${{ github.ref }} + cancel-in-progress: true + +jobs: + harden-motion-fixture: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + patched: ${{ steps.guard.outputs.patched }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 + + - name: Check unclamped Fresno motion start + id: guard + run: | + if grep -q "const fresnoGlobal = { center: fresnoStreet.center, zoom: 3.5 };" scripts/offline-tileset/generate-representative-targets.mjs; then + echo "patched=false" >> "$GITHUB_OUTPUT" + echo "Fresno motion starts at an unclamped flat-map zoom." + else + echo "patched=true" >> "$GITHUB_OUTPUT" + fi + + - name: Move Fresno motion above the single-world clamp + if: steps.guard.outputs.patched == 'true' + run: | + python - <<'PY' + from pathlib import Path + + path = Path('scripts/offline-tileset/generate-representative-targets.mjs') + source = path.read_text() + old = "const fresnoGlobal = { center: fresnoStreet.center, zoom: global.zoom };" + new = "const fresnoGlobal = { center: fresnoStreet.center, zoom: 3.5 };" + if old not in source: + raise SystemExit('Could not find the Fresno motion start camera.') + path.write_text(source.replace(old, new, 1)) + PY + + - name: Commit unclamped Fresno motion fixture + if: steps.guard.outputs.patched == 'true' + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add scripts/offline-tileset/generate-representative-targets.mjs + git commit -m 'Start Fresno zoom motion above flat-map camera clamp' + git push origin HEAD:${{ github.event.pull_request.head.ref }} + + production-build: + needs: harden-motion-fixture + if: needs.harden-motion-fixture.outputs.patched != 'true' + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + + - name: Install locked dependencies + run: npm ci + + - name: Build exact production application + run: npm run build + + - name: Preserve generated flat custom style + uses: actions/upload-artifact@v4 + if: always() + with: + name: custom-flat-production-style-${{ github.sha }} + path: | + public/style/occumed-open.json + public/style/compatibility-report.json + dist/** + if-no-files-found: warn + retention-days: 14 + + representative-visuals: + needs: harden-motion-fixture + if: needs.harden-motion-fixture.outputs.patched != 'true' + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + PLAYWRIGHT_BROWSERS_PATH: build/custom-flat/playwright + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + + - name: Install locked dependencies + run: npm ci + + - name: Build exact production application + run: npm run build + + - name: Create SHA-locked world owner plan + run: | + mkdir -p build/custom-flat + npm run tiles:plan-world -- \ + --output build/custom-flat/immutable-owner-plan.json + + - name: Enumerate representative world and motion targets + run: | + npm run tiles:targets-fixture -- \ + --plan build/custom-flat/immutable-owner-plan.json \ + --output build/custom-flat/representative-targets.json + + - name: Download only the locked globe inputs required by the fixture + run: | + npm run tiles:localize-fixture -- \ + --plan build/custom-flat/immutable-owner-plan.json \ + --targets build/custom-flat/representative-targets.json \ + --output-dir build/custom-flat/immutable-inputs \ + --report build/custom-flat/representative-inputs.json + + - name: Build representative immutable flat tileset + run: | + npm run tiles:build-fixture -- \ + --plan build/custom-flat/immutable-owner-plan.json \ + --targets build/custom-flat/representative-targets.json \ + --input-report build/custom-flat/representative-inputs.json \ + --output-dir build/custom-flat/artifact + + - name: Install Chromium + run: npx playwright install --with-deps chromium + + - name: Validate static and continuous-motion views + run: | + npm run check:immutable-visuals -- \ + --manifest build/custom-flat/artifact/manifest.json \ + --targets build/custom-flat/representative-targets.json \ + --output-dir build/custom-flat/visual-validation + + - name: Preserve custom tile artifact and screenshots + uses: actions/upload-artifact@v4 + if: always() + with: + name: custom-flat-globe-tiles-${{ github.sha }} + path: | + build/custom-flat/immutable-owner-plan.json + build/custom-flat/representative-targets.json + build/custom-flat/representative-inputs.json + build/custom-flat/artifact/manifest.json + build/custom-flat/artifact/representative-build-report.json + build/custom-flat/artifact/reports/** + build/custom-flat/visual-validation/** + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/validate-flat-authoritative-surface.yml b/.github/workflows/validate-flat-authoritative-surface.yml deleted file mode 100644 index 19a4a43e..00000000 --- a/.github/workflows/validate-flat-authoritative-surface.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: Validate Flat Authoritative Surface - -on: - pull_request: - branches: - - main - paths: - - .github/workflows/validate-flat-authoritative-surface.yml - - package-lock.json - - package.json - - scripts/apply-flat-overview-mode.mjs - - scripts/check-flat-overview-mode.mjs - - scripts/start-flat-overview.mjs - - scripts/validate-flat-authoritative-surface.mjs - - server-flat.mjs - - src/** - -permissions: - contents: read - -concurrency: - group: occumed-flat-surface-${{ github.ref }} - cancel-in-progress: true - -jobs: - rendered-land: - runs-on: ubuntu-latest - timeout-minutes: 35 - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: npm - - - run: npm ci - - - name: Build the exact Render application - run: npm run build - - - name: Install Chromium - run: npx playwright install --with-deps chromium - - - name: Start the exact Render command - run: | - set -euo pipefail - npm start > flat-surface-server.log 2>&1 & - echo "$!" > flat-surface-server.pid - for attempt in {1..600}; do - if curl --fail --silent http://127.0.0.1:4173/readyz > flat-surface-ready.json; then - exit 0 - fi - if ! kill -0 "$(cat flat-surface-server.pid)" 2>/dev/null; then - cat flat-surface-server.log - exit 1 - fi - sleep 1 - done - cat flat-surface-server.log - exit 1 - - - name: Prove land renders in the flat viewport - env: - OCCUMED_PREVIEW_OUTPUT: flat-surface-validation - run: node scripts/validate-flat-authoritative-surface.mjs - - - name: Stop server - if: always() - run: | - test -f flat-surface-server.pid && kill "$(cat flat-surface-server.pid)" 2>/dev/null || true - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: flat-authoritative-surface-${{ github.sha }} - path: | - flat-surface-validation/** - flat-surface-ready.json - flat-surface-server.log - if-no-files-found: warn - retention-days: 14 diff --git a/.github/workflows/validate-new-flat-map-v2.yml b/.github/workflows/validate-new-flat-map-v2.yml deleted file mode 100644 index efed7017..00000000 --- a/.github/workflows/validate-new-flat-map-v2.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Validate Clean Worldwide Map V2 - -on: - pull_request: - branches: - - main - paths: - - .github/workflows/validate-new-flat-map-v2.yml - - package-lock.json - - package.json - - server-new-map-v2.mjs - - scripts/check-new-map-v2.mjs - - scripts/validate-new-map-v2.mjs - - src/main.js - - src/new-map-v2.js - - src/new-map-v2.css - -permissions: - contents: read - -concurrency: - group: occumed-clean-map-v2-${{ github.ref }} - cancel-in-progress: true - -jobs: - exact-render: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: npm - - - name: Install locked dependencies - run: npm ci - - - name: Build the exact Render application - run: npm run build - - - name: Install Chromium - run: npx playwright install --with-deps chromium - - - name: Start the exact Render command - run: | - set -euo pipefail - npm start > new-map-v2-server.log 2>&1 & - echo "$!" > new-map-v2-server.pid - for attempt in {1..120}; do - if curl --fail --silent http://127.0.0.1:4173/readyz > new-map-v2-ready.json; then - exit 0 - fi - if ! kill -0 "$(cat new-map-v2-server.pid)" 2>/dev/null; then - cat new-map-v2-server.log - exit 1 - fi - sleep 1 - done - cat new-map-v2-server.log - exit 1 - - - name: Validate whole-world and regional rendering - env: - OCCUMED_PREVIEW_OUTPUT: new-map-v2-validation - run: node scripts/validate-new-map-v2.mjs - - - name: Stop server - if: always() - run: | - test -f new-map-v2-server.pid && kill "$(cat new-map-v2-server.pid)" 2>/dev/null || true - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: clean-worldwide-map-v2-${{ github.sha }} - path: | - new-map-v2-validation/** - new-map-v2-ready.json - new-map-v2-server.log - if-no-files-found: warn - retention-days: 14 diff --git a/.gitignore b/.gitignore index ebc14aed..653bff2e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules/ +node_modules dist/ .generated/ public/style/occumed-open.json @@ -11,3 +12,8 @@ public/sprites/occumed@2x.png .env.* !.env.example .DS_Store +build/offline-global/ +config/immutable-owner-plan.json +immutable-inputs/ +.build-tools/ +visual-validation/immutable-*/ diff --git a/README.md b/README.md index 596ec160..b32a786c 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,7 @@ The goal is a close visual replica without a Mapbox token or Mapbox-hosted runti - MapLibre GL JS - one permanent MapLibre vector source at `/tiles/{z}/{x}/{y}.pbf` -- a server-side virtual worldwide tileset backed by the 754 PMTiles storage archives -- one consolidated zoom 0–5 overview and one generalized worldwide physical surface -- open elevation data used only for hillshade, not as a second basemap +- one immutable worldwide foundation plus deterministic non-overlapping PMTiles owners - locally compiled sprites - browser-local glyph rendering - generated runtime style at `/style/occumed-open.json` @@ -32,17 +30,19 @@ No `VITE_MAPBOX_ACCESS_TOKEN`, `mapbox-gl`, `mapbox://` URL, Mapbox API endpoint MapLibre sees only `occumed-open`, whose URL is permanent from zoom 0 through 16. The browser does not load the world manifest, select an archive, register a PMTiles protocol, or replace a source while the map moves. -The Node tile gateway resolves each Z/X/Y request on the server: +The existing worldwide and regional PMTiles archives are offline inputs only. +The offline builder assigns one authority per layer family, clips and normalizes +geometry, removes duplicates and contained overlaps, rejects malformed, +oversized, and tile-shaped surface polygons, and writes each final Z/X/Y once. -- zoom 0–5 comes from a consolidated overview built from the same regional schema; -- zoom 6–16 is resolved against every storage shard intersecting the requested tile; -- boundary tiles are decoded, deduplicated by stable feature ID, and re-encoded as one MVT; -- the worldwide `land` layer is merged into the same response at every zoom; -- nested Natural Earth bathymetry bands are served as the `depth` layer at globe - and regional zooms, then fade before detailed navigation zooms; -- completed virtual tiles are held in a bounded in-memory cache and exposed with CDN cache headers. +Production loads a versioned ownership manifest, performs one deterministic +owner lookup, and returns the selected archive's stored MVT bytes unchanged. +It does not connect to Neon, merge shards, synthesize landcover, create +geometry, or stretch parent/child tiles. The browser never sees the owner +inventory and never switches its single source. -The PMTiles archives and routing manifest are storage implementation details. Their URLs never appear in the MapLibre style. +See [the immutable tileset build guide](docs/offline-global-tileset.md) for +the complete offline build and mandatory visual validation workflow. ## Source of truth @@ -73,6 +73,12 @@ Optional Render variable: PUBLIC_ORIGIN=https://map-yxjb.onrender.com ``` +Required tileset location (unless deployed at `dist/immutable-world/manifest.json`): + +```text +OCCUMED_IMMUTABLE_TILESET_MANIFEST=/absolute/path/to/immutable-world/manifest.json +``` + ## Reuse Install the repository in the consuming application: @@ -105,10 +111,11 @@ The build verifies: - the original export remains intact; - the generated style passes the MapLibre style specification; - no active source, sprite, or glyph URL points to Mapbox; -- globe, terrain, landcover, water, labels, and viewer-quality settings remain calibrated to the screenshot reference set; +- globe, landcover, water, labels, and viewer-quality settings remain calibrated to the screenshot reference set; - only one permanent vector source and one same-origin Z/X/Y template exist in the style; - browser-side PMTiles routing, `source.setUrl()`, fallback URLs, and OpenFreeMap are absent; -- the routing index includes every intersecting shard, including antimeridian segments; -- duplicate features are removed while boundary geometry is preserved; -- the overview, physical surface, regional merge, in-memory cache, and virtual release workflow remain wired; +- the immutable manifest is complete, non-overlapping, and fail-closed; +- every split-prefix ancestor and descendant tile has one deterministic prebuilt owner; +- production contains no Neon tile cache, runtime merge, geometry creation, landcover synthesis, or parent/child stretch path; +- mandatory static and exact-camera motion captures reject seams, tile footprints, stretched polygons, inconsistent neighbors, blank frames, and source switching; - no application-specific overlay data is included. diff --git a/docs/custom-flat-from-globe-tiles.md b/docs/custom-flat-from-globe-tiles.md new file mode 100644 index 00000000..7367787f --- /dev/null +++ b/docs/custom-flat-from-globe-tiles.md @@ -0,0 +1,16 @@ +# Custom flat map from the existing globe PMTiles + +This branch reuses the existing worldwide surface, overview, and regional PMTiles as offline inputs. The immutable owner pipeline from PR #45 is replayed on current main and the browser projection is locked to flat Mercator. + +Production contract: + +- one browser vector source at `/tiles/{z}/{x}/{y}.pbf`; +- exact prebuilt z0-z16 addressing; +- no runtime shard merge, geometry synthesis, Neon cache, or regional source switching; +- `land` and `depth` from the worldwide surface; +- `landcover` from the worldwide overview at foundation zooms and one deterministic regional owner above the handoff; +- roads, buildings, boundaries, labels, and other detail from one deterministic regional owner; +- no globe, atmosphere, fog, terrain, hillshade, or external basemap; +- one visible world at global zooms, with horizontal wrapping enabled only above zoom 3 so regional antimeridian navigation remains continuous without duplicating the low-zoom world view; +- the global static view remains at zoom 2.2, while Fresno continuous zoom validation starts at zoom 3.5 to avoid the intentional single-world camera clamp; +- representative zoom motion remains centered on Fresno, and generation fails on overlapping regional ownership while ownerless ocean tiles remain exact physical-foundation tiles. diff --git a/docs/new-flat-map-v2.md b/docs/new-flat-map-v2.md deleted file mode 100644 index 779bb5d3..00000000 --- a/docs/new-flat-map-v2.md +++ /dev/null @@ -1,32 +0,0 @@ -# Clean worldwide flat map v2 - -This production path replaces the PMTiles containment implementation with a clean MapLibre application using one complete worldwide vector source. - -## Runtime contract - -- Flat Mercator projection. -- Exactly one worldwide vector source. -- No PMTiles archive download or localization. -- No regional shard lookup. -- No Neon tile cache. -- No runtime tile merge, geometry synthesis, or parent/child transformation. -- No globe, atmosphere, terrain, hillshade, or 3D building path. - -## Occu-Med cartography - -- Water: `#79BCEC`. -- Parks and green space: `#A5CC8E`. -- Roads: `#F2F2F2`. -- Administrative boundaries: `#A65966`. -- Full-world initial view. - -## Merge gate - -The exact production build and start commands must render and save screenshots for: - -1. whole world; -2. North America; -3. Europe and Africa; -4. Fresno street-level view. - -The branch must not merge until those screenshots are manually inspected and show continuous map coverage without rectangular blanks or corrupted geometry. diff --git a/docs/offline-global-tileset-recovery.md b/docs/offline-global-tileset-recovery.md new file mode 100644 index 00000000..3c0c968f --- /dev/null +++ b/docs/offline-global-tileset-recovery.md @@ -0,0 +1,118 @@ +# Offline Global Tileset Recovery Handoff + +Status captured after Codex usage credits were exhausted on 2026-07-28/29. + +## Critical state + +- Target branch: `rebuild/offline-global-tileset` +- Remote branch was still identical to `main` at `ba835de833b7c62d26f7767a0cf06ed8dec287ee` when this handoff was written. +- The implementation described below had not yet been committed or pushed by Codex. +- The Codex workspace is therefore the only known location of the uncommitted source changes and generated artifacts. +- Do not delete or reset that Codex workspace before exporting/committing its changes. + +## Intended architecture + +- One browser vector source. +- Immutable worldwide PMTiles output. +- Deterministic non-overlapping owner partitions when a single physical file is too large. +- Every `z/x/y` tile belongs to exactly one owner. +- No Neon tile cache in the active production path. +- No runtime shard merging. +- No runtime landcover synthesis. +- No runtime parent/child stretching or overscaling. +- No production geometry creation. +- Offline-only clipping, schema normalization, deduplication, ancestor materialization, and malformed-feature rejection. + +## Input inventory discovered + +- 754 regional PMTiles archives plus worldwide overview and surface archives. +- Published regional input total reported by Codex: approximately 258.0 GB. +- Inputs were resolved from the existing GitHub Release and SHA-256 locked. +- Local `dist/virtual-assets` PMTiles files were only 325-byte fixtures and were correctly rejected as production inputs. + +## Representative artifact results + +Codex completed a representative validation build covering 17,901 exact `z/x/y` tiles plus three deterministic owner partitions for Fresno and both sides of the antimeridian. + +### Foundation + +- Final stable size: `250,938,363` bytes. +- Reported SHA-256 prefix: `b76dfa...`. +- Independent PMTiles verification passed after staged write, fsync, pending-file verification, and atomic rename. + +### Representative manifest + +- Total size across foundation and three non-overlapping owners: `272,198,460` bytes. +- A later manifest/artifact version after Fresno landcover correction was reported as `729e166...`. + +## Builder defects already found and corrected locally + +1. Node `Buffer` was returned where the PMTiles reader required an exact-range `ArrayBuffer`. +2. Polygon bounds used `Math.min(...points)` / `Math.max(...points)` and exceeded the JavaScript argument limit on detailed polygons; replaced with bounded linear scans. +3. Same-property polygon containment/deduplication was quadratic on dense tiles; replaced with deterministic spatial bucket indexing. +4. Legitimate buffered overview polygons exceeded the strict tile box; offline normalization was changed to clip to the exact tile boundary and revalidate encoded output. +5. The inherited runtime overscale helper validated intermediate MVT too early; ancestor materialization was moved into the offline builder and changed to feature-granular transform, clip, reject, then encode. +6. PMTiles owner metadata inherited an invalid center zoom; metadata was changed to derive center and zoom bounds from the owner’s actual addressed tiles. +7. Initial foundation output changed after the success report; writer was hardened to staging-path generation, verification, fsync, pending-copy verification, descriptor close, and atomic promotion. +8. Fresno z16 tiles lacked landcover because the overview z6 parent had no Fresno landcover. Authority was corrected so overview owns landcover at z0-6 and exactly one deterministic regional archive owns landcover above z6 for each partition. +9. Legacy AWS terrain/hillshade remained as a second browser source and caused readiness/network failures; Codex removed that source and dependent hillshade layer from the generated runtime style to enforce one browser source. + +## Validation state reached before credit exhaustion + +### Static views + +Codex reported all nine static views passing after the Fresno landcover rebuild: + +- Global +- North America +- South America +- Europe +- Pacific +- Antimeridian +- Fresno regional +- Fresno city +- Fresno street + +Reported results: + +- Required land rendered. +- Regional landcover rendered at Fresno street level. +- Transportation rendered. +- Exactly one style source. +- Zero network failures. +- Zero page errors. +- Zero detected seams. +- Zero rectangular tile footprints. +- Zero stretching detections. +- Zero blank-frame detections. +- Zero source changes. +- Zero neighboring-tile inconsistency detections. + +### Motion validation + +Motion validation was not finished. + +The last reported change corrected only the first motion setup predicate: global starting views should require immutable-source readiness at the target camera, not visible landcover, because the global contract is land plus ocean depth. Motion frames were still intended to undergo every pixel/source gate. + +## Immediate recovery instruction for the next Codex turn + +Do not rebuild first. Preserve the workspace immediately: + +1. Confirm the workspace still contains the uncommitted changes and generated reports. +2. Confirm the current branch is `rebuild/offline-global-tileset`. +3. Review `git status --short` and `git diff --stat`. +4. Exclude `node_modules`, downloaded PMTiles inputs, browser binaries, temporary build directories, and generated PMTiles artifacts from Git. +5. Commit all source, configuration, workflow, manifest, validator, report, and tracked screenshot changes. +6. Push the branch. +7. Open a draft PR. +8. Report the commit SHA and PR number before resuming any build or validation. + +## Work still required after preservation + +- Finish all continuous motion validations. +- Inspect saved screenshots manually, not only numeric gates. +- Prove complete deterministic worldwide ownership for every intended tile. +- Execute the production-scale immutable partition build, not only the 17,901-tile representative artifact. +- Keep every final PMTiles release asset below GitHub’s per-asset limit. +- Upload only final immutable partitions and the versioned ownership manifest. +- Keep the PR draft until the actual worldwide artifact set exists and is visually verified. diff --git a/docs/offline-global-tileset.md b/docs/offline-global-tileset.md new file mode 100644 index 00000000..e8c15da8 --- /dev/null +++ b/docs/offline-global-tileset.md @@ -0,0 +1,137 @@ +# Immutable worldwide PMTiles build + +The production map uses one browser source and an immutable PMTiles ownership +manifest. Existing worldwide and regional archives are build inputs only. +Production never decodes MVT geometry, merges shards, queries Neon, synthesizes +landcover, or substitutes parent/child tiles. + +## Authority and ownership + +| Output layers | Sole offline authority | +|---|---| +| `land`, `depth` | `occumed-world-surface.pmtiles` | +| `landcover` | `occumed-world-overview.pmtiles` | +| roads, buildings, boundaries, labels, other cartography | one regional owner | + +The deterministic plan starts with z6 logical cells. A cell whose locked +candidate input set is too large is split through z7 or z8. Leaf prefixes never +overlap. When a prefix is split, its own ancestor tile is assigned explicitly +to the first descendant owner in stable child order. Consequently every +addressed z0–z16 tile resolves to exactly one foundation, exact-tile, or prefix +owner. + +Missing entries are immutable empty results from that owner; production never +tries a second archive. + +## Full offline build + +The complete published input inventory is about 270 GB. Use a build host with +enough space for the locked input cache, work files, and final owners. All +outputs are created at staging paths, verified as PMTiles, fsynced, copied to a +pending file, verified again, and atomically promoted. + +Create the SHA-locked owner plan and worldwide foundation target list: + +```bash +npm run tiles:plan-world -- \ + --output config/immutable-owner-plan.json + +npm run tiles:targets-foundation -- \ + --plan config/immutable-owner-plan.json \ + --output build/offline-global/foundation-targets.json +``` + +Localize the two worldwide inputs and build the z0–z6 foundation: + +```bash +npm run tiles:localize -- \ + --plan config/immutable-owner-plan.json \ + --targets build/offline-global/foundation-targets.json \ + --output-dir immutable-inputs \ + --report build/offline-global/foundation-inputs.json + +npm run tiles:build-foundation -- \ + --plan config/immutable-owner-plan.json \ + --targets build/offline-global/foundation-targets.json \ + --input-report build/offline-global/foundation-inputs.json \ + --output-dir build/offline-global/artifact +``` + +For every `owners[].id` in the plan, localize its locked candidates, enumerate +the exact addresses in their PMTiles directories, and build that owner. The +input directory is a shared digest-verified cache, so repeated candidates are +reused. + +```bash +OWNER_ID=z6-10-24 + +npm run tiles:localize -- \ + --plan config/immutable-owner-plan.json \ + --owner-id "$OWNER_ID" \ + --output-dir immutable-inputs \ + --report "build/offline-global/$OWNER_ID-inputs.json" + +npm run tiles:targets-owner -- \ + --plan config/immutable-owner-plan.json \ + --owner-id "$OWNER_ID" \ + --input-report "build/offline-global/$OWNER_ID-inputs.json" \ + --output "build/offline-global/$OWNER_ID-targets.json" + +npm run tiles:build-owner -- \ + --plan config/immutable-owner-plan.json \ + --owner-id "$OWNER_ID" \ + --targets "build/offline-global/$OWNER_ID-targets.json" \ + --input-report "build/offline-global/$OWNER_ID-inputs.json" \ + --output-dir build/offline-global/artifact +``` + +Finalize only after every planned owner report exists: + +```bash +npm run tiles:finalize -- \ + --plan config/immutable-owner-plan.json \ + --foundation build/offline-global/artifact/reports/foundation.json \ + --owner-dir build/offline-global/artifact/reports/owners \ + --output build/offline-global/artifact/manifest.json +``` + +The finalizer marks an incomplete inventory as `validationFixture: true`. +Production rejects it. `OCCUMED_ALLOW_PARTIAL_TILESET_FIXTURE=true` exists only +for the bounded local visual fixture and must not be set in deployment. + +## Production + +Set `OCCUMED_IMMUTABLE_TILESET_MANIFEST` to the local manifest path or an HTTPS +manifest URL. Local assets are resolved beneath the manifest directory. Remote +assets use the manifest's optional `assetBaseUrl`. + +The server validates completeness, artifact identity, SHA metadata, safe asset +paths, non-overlapping prefixes, and unique exact-tile assignments before +serving tiles. Each request performs a bounded index lookup and a single +PMTiles byte-range read. Stored gzip MVT bytes are returned unchanged. + +## Mandatory validation + +Run the structural and production checks: + +```bash +npm run build +``` + +Run the browser gate against the finalized manifest and validation target +document: + +```bash +PLAYWRIGHT_BROWSERS_PATH=build/offline-global/playwright \ +npm run check:immutable-visuals -- \ + --manifest build/offline-global/artifact/manifest.json \ + --targets build/offline-global/representative-targets.json \ + --output-dir visual-validation/immutable-final +``` + +The target document must cover global, North America, South America, Europe, +Pacific, antimeridian, regional, city, and street views, plus 30 exact-camera +motion checkpoints. Validation fails for rectangular footprints, vertical or +horizontal seams, stretched polygons, inconsistent neighbors, blank frames, +source switching, page errors, or tile delivery failures. Saved screenshots +must also be inspected at full size before an artifact is accepted. diff --git a/package-lock.json b/package-lock.json index 065d3364..c987e504 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@chrispahm/spritezero": "8.1.0", "@maplibre/maplibre-gl-style-spec": "24.8.1", "playwright": "1.55.0", + "pngjs": "7.0.0", "vite": "8.1.5" } }, @@ -1414,6 +1415,16 @@ "fflate": "^0.8.2" } }, + "node_modules/pngjs": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", + "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.19.0" + } + }, "node_modules/postcss": { "version": "8.5.23", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz", diff --git a/package.json b/package.json index 8cdbd09b..5454ffc9 100644 --- a/package.json +++ b/package.json @@ -5,23 +5,34 @@ "type": "module", "scripts": { "prepare:sprites": "node scripts/build-sprites.mjs", - "prepare:style": "node scripts/build-runtime-style.mjs && node scripts/normalize-runtime-fonts.mjs && node scripts/normalize-runtime-filters.mjs && node scripts/normalize-boundary-filters.mjs && node scripts/force-runtime-filter-expressions.mjs && node scripts/apply-schema-parity.mjs && node scripts/apply-globe-parity.mjs && node scripts/apply-photo-reference.mjs && node scripts/restore-exported-cartography.mjs && node scripts/calibrate-reference-colors.mjs && node scripts/lock-exact-exported-swatches.mjs && node scripts/lock-reference-atmosphere.mjs && node scripts/normalize-runtime-fonts.mjs && node scripts/apply-mapbox-rendering-contract.mjs && node scripts/use-local-maplibre-glyphs.mjs && node scripts/apply-flat-overview-mode.mjs", + "prepare:style": "node scripts/build-runtime-style.mjs && node scripts/normalize-runtime-fonts.mjs && node scripts/normalize-runtime-filters.mjs && node scripts/normalize-boundary-filters.mjs && node scripts/force-runtime-filter-expressions.mjs && node scripts/apply-schema-parity.mjs && node scripts/apply-photo-reference.mjs && node scripts/restore-exported-cartography.mjs && node scripts/calibrate-reference-colors.mjs && node scripts/lock-exact-exported-swatches.mjs && node scripts/normalize-runtime-fonts.mjs && node scripts/apply-mapbox-rendering-contract.mjs && node scripts/use-local-maplibre-glyphs.mjs && node scripts/apply-flat-custom-parity.mjs", "prepare:assets": "npm run prepare:sprites && npm run prepare:style", "tiles:build": "bash planetiler/build-region.sh", - "tiles:plan-world": "node scripts/plan-world-shards.mjs --scope all", + "tiles:plan-world": "node scripts/offline-tileset/plan-owners.mjs", + "tiles:plan-production-batches": "node scripts/offline-tileset/plan-production-batches.mjs", + "tiles:targets-foundation": "node scripts/offline-tileset/generate-foundation-targets.mjs", + "tiles:targets-owner": "node scripts/offline-tileset/generate-owner-targets.mjs", + "tiles:targets-production-batch": "node scripts/offline-tileset/generate-production-batch-targets.mjs", + "tiles:localize": "node scripts/offline-tileset/localize-representative-inputs.mjs", + "tiles:build-foundation": "node scripts/offline-tileset/build-foundation.mjs", + "tiles:build-production-foundation": "node scripts/offline-tileset/build-production-foundation.mjs", + "tiles:build-owner": "node scripts/offline-tileset/build-owner.mjs", + "tiles:build-production-batch": "node scripts/offline-tileset/build-production-batch.mjs", + "tiles:finalize": "node scripts/offline-tileset/finalize-manifest.mjs", + "tiles:finalize-production": "node scripts/offline-tileset/finalize-published-production.mjs", + "tiles:targets-fixture": "node scripts/offline-tileset/generate-representative-targets.mjs", + "tiles:localize-fixture": "node scripts/offline-tileset/localize-representative-inputs.mjs", + "tiles:build-fixture": "node scripts/offline-tileset/build-representative.mjs", "check:export": "node scripts/check-export.mjs", - "check:hardening": "node scripts/check-world-hardening.mjs", - "check:foundation": "node scripts/check-continuous-foundation-lock.mjs", - "check:neon-cache": "node scripts/check-neon-navigation-cache.mjs", - "check:runtime": "node scripts/check-runtime.mjs && node scripts/validate-maplibre-style.mjs && node scripts/check-globe-parity.mjs && node scripts/check-cartography-parity.mjs && node scripts/check-viewer-quality.mjs && node scripts/check-photo-reference.mjs && node scripts/check-exact-exported-swatches.mjs && node scripts/check-render-clarity.mjs && node scripts/check-pmtiles-integration.mjs && node scripts/check-world-tile-gateway.mjs && npm run check:neon-cache && npm run check:hardening && npm run check:foundation", + "check:architecture": "node scripts/check-immutable-architecture.mjs", + "check:runtime": "node scripts/check-immutable-architecture.mjs && node scripts/check-custom-flat-tiles.mjs && node scripts/validate-maplibre-style.mjs && node scripts/check-cartography-parity.mjs && node scripts/check-render-clarity.mjs", + "check:immutable-visuals": "node scripts/validate-immutable-visuals.mjs", "check:server": "node scripts/check-server-health.mjs", - "check:flat": "node scripts/check-flat-overview-mode.mjs", - "check:new-map": "node scripts/check-new-map-v2.mjs", - "check": "npm run check:new-map", - "dev": "vite", - "build": "npm run check:new-map && vite build", + "check": "npm run check:export && npm run check:runtime", + "dev": "npm run prepare:assets && vite", + "build": "npm run prepare:assets && npm run check && vite build && npm run check:server", "preview": "vite preview", - "start": "node server-new-map-v2.mjs" + "start": "sh -c 'OCCUMED_IMMUTABLE_TILESET_MANIFEST=\"${OCCUMED_IMMUTABLE_TILESET_MANIFEST:-https://github.com/Occumed79/Map/releases/download/occumed-flat-v1/manifest.json}\" node server.mjs'" }, "dependencies": { "@mapbox/vector-tile": "2.0.4", @@ -33,6 +44,7 @@ "devDependencies": { "@chrispahm/spritezero": "8.1.0", "@maplibre/maplibre-gl-style-spec": "24.8.1", + "pngjs": "7.0.0", "playwright": "1.55.0", "vite": "8.1.5" } diff --git a/scripts/apply-flat-custom-parity.mjs b/scripts/apply-flat-custom-parity.mjs new file mode 100644 index 00000000..bfb226ac --- /dev/null +++ b/scripts/apply-flat-custom-parity.mjs @@ -0,0 +1,40 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const stylePath = path.join(root, 'public/style/occumed-open.json'); +const style = JSON.parse(await fs.readFile(stylePath, 'utf8')); +const sources = Object.entries(style.sources || {}); + +if (sources.length !== 1 || sources[0][0] !== 'occumed-open' || sources[0][1]?.type !== 'vector') { + throw new Error(`Custom flat map requires exactly one occumed-open vector source; found ${sources.length}.`); +} + +style.projection = { type: 'mercator' }; +delete style.sky; +delete style.fog; +delete style.terrain; +delete style.light; +style.layers = (style.layers || []).filter((layer) => !['sky', 'hillshade', 'model'].includes(layer.type)); + +for (const layer of style.layers) { + if (layer.type === 'background') { + layer.paint = { ...(layer.paint || {}), 'background-color': '#79BCEC', 'background-opacity': 1 }; + } +} + +style.metadata = { + ...(style.metadata || {}), + 'occumed:architecture': 'immutable-custom-flat-pmtiles', + 'occumed:projection': 'mercator', + 'occumed:source-count': 1, + 'occumed:globe': false, + 'occumed:runtime-merge': false, + 'occumed:regional-routing': false, + 'occumed:neon': false, + 'occumed:exact-prebuilt-addressing': true +}; + +await fs.writeFile(stylePath, `${JSON.stringify(style, null, 2)}\n`); +console.log('Locked the preserved Occu-Med custom tileset to flat Mercator.'); diff --git a/scripts/apply-globe-parity.mjs b/scripts/apply-globe-parity.mjs index bfba9293..a0f50f6a 100644 --- a/scripts/apply-globe-parity.mjs +++ b/scripts/apply-globe-parity.mjs @@ -49,35 +49,14 @@ runtime.sky = { delete runtime.fog; delete runtime.light; -const hillshade = runtime.layers.find((layer) => layer.id === 'occumed-hillshade'); -if (!hillshade) throw new Error('The generated open hillshade layer is missing.'); - -hillshade.paint = { - 'hillshade-exaggeration': [ - 'interpolate', - ['linear'], - ['zoom'], - 2, - 0.04, - 8, - 0.16, - 15, - 0.11 - ], - 'hillshade-shadow-color': 'hsla(215, 22%, 28%, 0.42)', - 'hillshade-highlight-color': 'hsla(48, 38%, 96%, 0.34)', - 'hillshade-accent-color': 'hsla(95, 26%, 46%, 0.32)', - 'hillshade-illumination-direction': 335, - 'hillshade-illumination-anchor': 'map' -}; - runtime.metadata = { ...(runtime.metadata || {}), 'occumed:mapbox-fog-translated-to-maplibre-sky': true, 'occumed:globe-parity-pass': 3, 'occumed:stable-orientation-neutral-atmosphere': true, + 'occumed:external-terrain-disabled': true, 'occumed:visual-quality-pass': true }; await fs.writeFile(runtimePath, `${JSON.stringify(runtime, null, 2)}\n`); -console.log('Applied the refined Occu-Med globe rim and stable hillshade.'); +console.log('Applied the refined Occu-Med globe rim without an external terrain source.'); diff --git a/scripts/build-runtime-style.mjs b/scripts/build-runtime-style.mjs index 8ef6001e..06a9d57d 100644 --- a/scripts/build-runtime-style.mjs +++ b/scripts/build-runtime-style.mjs @@ -14,10 +14,6 @@ const vectorTilesUrl = const glyphsUrl = process.env.OCCUMED_GLYPHS_URL || 'https://demotiles.maplibre.org/font/{fontstack}/{range}.pbf'; -const terrainUrl = - process.env.OCCUMED_TERRAIN_URL || - 'https://s3.amazonaws.com/elevation-tiles-prod/terrarium/{z}/{x}/{y}.png'; - const original = JSON.parse(await fs.readFile(sourcePath, 'utf8')); const fontMap = new Map([ @@ -296,8 +292,6 @@ function rewriteLayer(layer, targetSourceLayer) { const convertedLayers = []; const skippedLayers = []; const sourceLayerMappings = {}; -let hillshadeInserted = false; - for (const layer of original.layers || []) { if (!layer.source) { convertedLayers.push(clone(layer)); @@ -324,26 +318,10 @@ for (const layer of original.layers || []) { const sourceLayer = layer['source-layer']; if (sourceLayer === 'hillshade') { - if (!hillshadeInserted) { - convertedLayers.push({ - id: 'occumed-hillshade', - type: 'hillshade', - source: 'occumed-terrain', - minzoom: 2, - maxzoom: 16, - paint: { - 'hillshade-exaggeration': ['interpolate', ['linear'], ['zoom'], 2, 0.16, 8, 0.34, 15, 0.24], - 'hillshade-shadow-color': 'hsl(215, 18%, 30%)', - 'hillshade-highlight-color': 'hsl(48, 40%, 94%)', - 'hillshade-accent-color': 'hsl(95, 18%, 55%)', - 'hillshade-illumination-direction': 335, - 'hillshade-illumination-anchor': 'viewport' - }, - metadata: { 'occumed:original-source-layer': 'hillshade' } - }); - hillshadeInserted = true; - } - skippedLayers.push({ id: layer.id, reason: 'replaced by open raster DEM hillshade' }); + skippedLayers.push({ + id: layer.id, + reason: 'one-source immutable architecture has no external terrain source' + }); continue; } @@ -381,15 +359,6 @@ const runtimeStyle = { maxzoom: 16, attribution: '© OpenStreetMap contributors' - }, - 'occumed-terrain': { - type: 'raster-dem', - tiles: [terrainUrl], - encoding: 'terrarium', - tileSize: 256, - minzoom: 0, - maxzoom: 15, - attribution: 'Elevation data via the AWS Terrain Tiles public dataset' } }, sprite: '__OCCUMED_PUBLIC_ORIGIN__/sprites/occumed', @@ -411,7 +380,7 @@ const report = { endpoints: { vectorTiles: vectorTilesUrl, glyphs: glyphsUrl, - terrain: terrainUrl, + terrain: null, relief: null, sprite: '__OCCUMED_PUBLIC_ORIGIN__/sprites/occumed' } diff --git a/scripts/calibrate-reference-colors.mjs b/scripts/calibrate-reference-colors.mjs index 165c7a53..d5174ca4 100644 --- a/scripts/calibrate-reference-colors.mjs +++ b/scripts/calibrate-reference-colors.mjs @@ -190,30 +190,6 @@ if (!water) throw new Error('The exported water layer is missing.'); water.paint ||= {}; water.paint['fill-opacity'] = 1; -// Strengthen physical form without changing any vector structure color. -const hillshade = layer('occumed-hillshade'); -if (!hillshade) throw new Error('The open hillshade layer is missing.'); -hillshade.minzoom = 1.5; -hillshade.maxzoom = 16; -hillshade.paint = { - 'hillshade-exaggeration': [ - 'interpolate', - ['linear'], - ['zoom'], - 1.5, 0.07, - 4.5, 0.13, - 7.5, 0.2, - 10, 0.27, - 13, 0.31, - 16, 0.2 - ], - 'hillshade-shadow-color': '#52685B', - 'hillshade-highlight-color': '#FFF8E8', - 'hillshade-accent-color': '#6E8D69', - 'hillshade-illumination-direction': 335, - 'hillshade-illumination-anchor': 'map' -}; - // Bring the exported road and boundary hierarchy into view at the same scales as // the supplied Studio references. Their individual line colors remain untouched. const entryZooms = new Map([ diff --git a/scripts/check-custom-flat-tiles.mjs b/scripts/check-custom-flat-tiles.mjs new file mode 100644 index 00000000..6813f114 --- /dev/null +++ b/scripts/check-custom-flat-tiles.mjs @@ -0,0 +1,43 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const style = JSON.parse(await fs.readFile(path.join(root, 'public/style/occumed-open.json'), 'utf8')); +const mapSource = await fs.readFile(path.join(root, 'src/occumed-map.js'), 'utf8'); +const main = await fs.readFile(path.join(root, 'src/main.js'), 'utf8'); +const server = await fs.readFile(path.join(root, 'server.mjs'), 'utf8'); +const failures = []; +const expect = (condition, message) => { if (!condition) failures.push(message); }; +const sources = Object.entries(style.sources || {}); +const source = style.sources?.['occumed-open']; + +expect(style.projection?.type === 'mercator', 'Projection is not flat Mercator.'); +expect(sources.length === 1, `Expected one browser source; found ${sources.length}.`); +expect(source?.type === 'vector', 'occumed-open is not a vector source.'); +expect(JSON.stringify(source?.tiles) === JSON.stringify(['__OCCUMED_PUBLIC_ORIGIN__/tiles/{z}/{x}/{y}.pbf']), 'Custom source does not use the immutable tile endpoint.'); +expect(source?.minzoom === 0 && source?.maxzoom === 16, 'Custom source does not cover z0-z16.'); +expect(!style.sky && !style.fog && !style.terrain && !style.light, 'Globe, fog, terrain, or lighting remains active.'); +expect(!(style.layers || []).some((layer) => ['sky', 'hillshade', 'model'].includes(layer.type)), 'Forbidden globe/terrain layers remain.'); +expect((style.layers || []).every((layer) => !layer.source || layer.source === 'occumed-open'), 'A layer references a second browser source.'); +expect(style.metadata?.['occumed:architecture'] === 'immutable-custom-flat-pmtiles', 'Custom flat architecture metadata is missing.'); +expect(style.metadata?.['occumed:exact-prebuilt-addressing'] === true, 'Exact addressing metadata is missing.'); +expect(main.includes("./occumed-map.js"), 'Application is not using the preserved custom map renderer.'); +expect(!main.includes('new-map-v2'), 'Generic basemap renderer is still active.'); +expect(!mapSource.includes('installOccumedAtmosphereBloom(map);'), 'Globe atmosphere is still installed.'); +expect(mapSource.includes('installExactTileAddressing(map);'), 'Exact prebuilt tile addressing is not installed.'); +expect(mapSource.includes('installAdaptiveFlatWorldWrap(map);'), 'Adaptive high-zoom antimeridian wrapping is not installed.'); +expect(server.includes('ImmutableWorldTileset'), 'Production server is not using the immutable PMTiles store.'); +expect(!server.includes('NAV_DATABASE_URL_'), 'Production server still references Neon tile cache variables.'); + +const sourceLayers = new Set((style.layers || []).map((layer) => layer['source-layer']).filter(Boolean)); +for (const layer of ['land', 'landcover', 'depth', 'water', 'transportation', 'boundary', 'place']) { + expect(sourceLayers.has(layer), `Generated style is missing required ${layer} layer usage.`); +} + +if (failures.length) { + console.error('Custom flat PMTiles contract failed:'); + for (const failure of failures) console.error(`- ${failure}`); + process.exit(1); +} +console.log('Custom flat PMTiles contract passed: one immutable z0-z16 source, Mercator, no runtime merge, no Neon, exact prebuilt addressing.'); diff --git a/scripts/check-exact-exported-swatches.mjs b/scripts/check-exact-exported-swatches.mjs index 32f0574e..705da777 100644 --- a/scripts/check-exact-exported-swatches.mjs +++ b/scripts/check-exact-exported-swatches.mjs @@ -33,9 +33,9 @@ const EXACT = { wetland: '#A5CAD6', water: '#79BCEC', waterShadow: '#7293EE', - depthShallow: '#79BCEC59', - depthMid: '#5AACE759', - depthDeep: '#3B9DE359' + depthShallow: '#79BCEC', + depthMid: '#6EB6EA', + depthDeep: '#63B1E9' }; expect(color('land', 'background-color') === EXACT.ocean, 'The permanent ocean background changed.'); @@ -117,10 +117,8 @@ expect(color('landuse', 'fill-opacity') === 1, 'Detailed landuse colors are weak expect(color('water', 'fill-opacity') === 1, 'Water is not fully opaque.'); expect(!runtime.layers.some((candidate) => candidate.type === 'raster'), 'A raster basemap was reintroduced.'); -const hillshade = layer('occumed-hillshade'); -expect(hillshade?.paint?.['hillshade-shadow-color'] === '#0000004D', 'Hillshade shadows are tinting terrain.'); -expect(hillshade?.paint?.['hillshade-highlight-color'] === '#FFFFFF4D', 'Hillshade highlights are tinting terrain.'); -expect(hillshade?.paint?.['hillshade-accent-color'] === '#00000026', 'Hillshade accents are tinting terrain.'); +expect(!runtime.layers.some((candidate) => candidate.type === 'hillshade'), 'An external hillshade layer was reintroduced.'); +expect(Object.keys(runtime.sources || {}).length === 1, 'The palette style no longer has exactly one browser source.'); expect(runtime.metadata?.['occumed:reference-color-system'] === 'continuous-world-v10', 'The continuous-world color pass did not run.'); expect(runtime.metadata?.['occumed:raster-relief-disabled'] === true, 'Raster relief protection is missing.'); diff --git a/scripts/check-globe-parity.mjs b/scripts/check-globe-parity.mjs index 43b8435a..37600eb8 100644 --- a/scripts/check-globe-parity.mjs +++ b/scripts/check-globe-parity.mjs @@ -102,16 +102,13 @@ if (!Array.isArray(depthOpacity) || depthOpacity[0] !== 'max' || Number(depthOpa fail('Bathymetry can still collapse to zero at detailed navigation zooms.'); } const waterDepthColors = JSON.stringify(waterDepth?.paint?.['fill-color'] || []); -for (const value of ['#79BCEC59', '#5AACE759', '#3B9DE359']) { +for (const value of ['#79BCEC', '#6EB6EA', '#63B1E9']) { if (!waterDepthColors.includes(value)) fail(`The exported bathymetry swatch ${value} is missing.`); } - -const hillshade = layer('occumed-hillshade'); -if (!hillshade) fail('The open hillshade layer is missing.'); -if (hillshade?.minzoom !== 1.5) fail('Hillshade begins too late to shape the globe.'); -if (hillshade?.paint?.['hillshade-illumination-anchor'] !== 'map') fail('Hillshade must remain geographically anchored.'); -if (hillshade?.paint?.['hillshade-shadow-color'] !== '#0000004D') fail('Hillshade shadows are tinting exported colors.'); -if (hillshade?.paint?.['hillshade-highlight-color'] !== '#FFFFFF4D') fail('Hillshade highlights are tinting exported colors.'); +if (runtime.layers.some((candidate) => candidate.type === 'hillshade')) { + fail('The one-source style contains an external hillshade layer.'); +} +if (Object.keys(runtime.sources || {}).length !== 1) fail('The globe style does not use exactly one browser source.'); if ((layer('road-motorway-trunk')?.minzoom ?? 99) > 2) fail('Major highways enter too late for the supplied regional hierarchy.'); if ((layer('admin-0-boundary')?.minzoom ?? 99) > 0) fail('Country boundaries are unavailable at globe zoom.'); diff --git a/scripts/check-immutable-architecture.mjs b/scripts/check-immutable-architecture.mjs new file mode 100644 index 00000000..a69c8eea --- /dev/null +++ b/scripts/check-immutable-architecture.mjs @@ -0,0 +1,233 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + computeImmutableArtifactVersion, + validateImmutableManifest +} from '../src/server/immutable-world-tileset.js'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const read = (filename) => fs.readFile(path.join(root, filename), 'utf8'); +const [ + server, + browser, + styleBuilder, + normalizer, + packageText +] = await Promise.all([ + read('server.mjs'), + read('src/occumed-map.js'), + read('scripts/build-runtime-style.mjs'), + read('scripts/offline-tileset/mvt-normalizer.mjs'), + read('package.json') +]); + +for (const forbidden of [ + 'neon-navigation-cache', + 'world-tile-gateway', + 'mergeVectorTiles', + 'world-routing', + 'overscale-vector-tile' +]) { + assert( + !server.includes(forbidden), + `Production server still references forbidden runtime module or operation: ${forbidden}` + ); +} +assert( + server.includes("from './src/server/immutable-world-tileset.js'"), + 'Production server does not use the immutable PMTiles store.' +); +assert( + !server.includes('validateVectorTilePayload'), + 'Production server still decodes or validates geometry instead of returning prebuilt bytes.' +); +assert( + server.includes("'Content-Encoding': resolved.contentEncoding"), + 'Production server does not preserve the stored PMTiles tile compression.' +); + +for (const forbidden of [ + 'installContinuousTileRetention', + '_updateRetainedTiles', + 'maxTileCacheZoomLevels' +]) { + assert(!browser.includes(forbidden), `Browser still stretches or pins parent/child tiles: ${forbidden}`); +} +assert( + browser.includes('tileManager.constructor.maxUnderzooming = 0') && + browser.includes('tileManager.constructor.maxOverzooming = 0'), + 'Browser does not explicitly disable MapLibre parent/child fallback depths.' +); +assert( + browser.includes('cancelPendingTileRequestsWhileZooming: true'), + 'Browser must use MapLibre normal request cancellation without a retained-tile patch.' +); + +assert(!styleBuilder.includes('occumed-terrain'), 'Style builder still creates a second terrain source.'); +assert(!styleBuilder.includes('raster-dem'), 'Style builder still creates a raster DEM source.'); +assert( + styleBuilder.includes("reason: 'one-source immutable architecture has no external terrain source'"), + 'Style builder does not document why exported hillshade layers are excluded.' +); + +for (const authority of [ + "land: 'world-surface'", + "depth: 'world-surface'", + "landcover: 'world-overview'", + "cartography: 'regional-owner'" +]) { + assert(normalizer.includes(authority), `Offline authority is missing: ${authority}`); +} +assert( + normalizer.includes('isLargeAxisAlignedRectangle'), + 'Offline normalizer does not reject large tile-shaped surface polygons.' +); +assert( + normalizer.includes('rejectedMalformed') && normalizer.includes('rejectedOversized'), + 'Offline normalizer does not count malformed and oversized feature rejection.' +); + +const packageJson = JSON.parse(packageText); +assert.equal(packageJson.scripts.start, 'node server.mjs', 'Production start command bypasses the immutable server.'); +assert( + packageJson.scripts['check:runtime'].includes('check-immutable-architecture.mjs'), + 'Runtime validation does not include the immutable architecture lock.' +); + +function fixtureManifest(overrides = {}) { + const manifest = { + schemaVersion: 1, + generatedAt: '2000-01-01T00:00:00.000Z', + planVersion: 'a'.repeat(64), + browserSourceId: 'occumed-open', + minZoom: 0, + maxZoom: 16, + complete: false, + validationFixture: true, + logicalOwnerCount: 1, + plannedOwnerCount: 2, + builtOwnerCount: 1, + defaultOwner: 'foundation', + totalBytes: 508, + authorities: { + land: 'world-surface', + depth: 'world-surface', + landcover: 'world-overview', + cartography: 'regional-owner' + }, + runtimePolicy: { + neonTileCache: false, + runtimeShardMerge: false, + runtimeLandcoverSynthesis: false, + runtimeGeometry: false, + parentChildStretching: false + }, + foundation: { + id: 'foundation', + file: 'foundation.pmtiles', + bytes: 254, + sha256: 'b'.repeat(64), + maxZoom: 6 + }, + owners: [{ + id: 'z6-10-24', + prefix: { z: 6, x: 10, y: 24 }, + file: 'owners/z6-10-24.pmtiles', + bytes: 254, + sha256: 'c'.repeat(64) + }], + ...overrides + }; + manifest.artifactVersion = computeImmutableArtifactVersion(manifest); + return manifest; +} + +const partial = fixtureManifest(); +assert.throws( + () => validateImmutableManifest(partial), + /incomplete/, + 'Production accepted a partial validation fixture without an explicit override.' +); +assert.doesNotThrow( + () => validateImmutableManifest(partial, { allowPartial: true }), + 'The explicit validation-fixture override did not work.' +); + +const overlapping = fixtureManifest({ + plannedOwnerCount: 2, + builtOwnerCount: 2, + complete: true, + validationFixture: false, + owners: [ + partial.owners[0], + { + id: 'z7-20-48', + prefix: { z: 7, x: 20, y: 48 }, + file: 'owners/z7-20-48.pmtiles', + bytes: 254, + sha256: 'd'.repeat(64) + } + ] +}); +overlapping.artifactVersion = computeImmutableArtifactVersion(overlapping); +assert.throws( + () => validateImmutableManifest(overlapping), + /Overlapping immutable owner prefixes/, + 'Manifest accepted overlapping prebuilt owners.' +); + +const splitAncestor = fixtureManifest({ + plannedOwnerCount: 1, + owners: [{ + id: 'z8-40-96', + prefix: { z: 8, x: 40, y: 96 }, + exactTiles: [{ z: 7, x: 20, y: 48 }], + file: 'owners/z8-40-96.pmtiles', + bytes: 254, + sha256: 'd'.repeat(64) + }] +}); +splitAncestor.artifactVersion = computeImmutableArtifactVersion(splitAncestor); +assert.doesNotThrow( + () => validateImmutableManifest(splitAncestor, { allowPartial: true }), + 'A deterministic split-prefix ancestor assignment was rejected.' +); + +const duplicateExact = fixtureManifest({ + plannedOwnerCount: 2, + builtOwnerCount: 2, + complete: true, + validationFixture: false, + owners: [ + splitAncestor.owners[0], + { + id: 'z8-41-96', + prefix: { z: 8, x: 41, y: 96 }, + exactTiles: [{ z: 7, x: 20, y: 48 }], + file: 'owners/z8-41-96.pmtiles', + bytes: 254, + sha256: 'e'.repeat(64) + } + ] +}); +duplicateExact.artifactVersion = computeImmutableArtifactVersion(duplicateExact); +assert.throws( + () => validateImmutableManifest(duplicateExact), + /assigned to both/, + 'Manifest accepted two owners for one exact split-prefix ancestor tile.' +); + +const regenerated = fixtureManifest({ generatedAt: '2030-01-01T00:00:00.000Z' }); +assert.equal( + partial.artifactVersion, + regenerated.artifactVersion, + 'Artifact identity changes with a non-semantic generation timestamp.' +); + +console.log( + 'Immutable architecture locked: one source, prebuilt PMTiles reads only, deterministic identity, ' + + 'fail-closed completeness, exact split-ancestor ownership, non-overlapping owners, and no ' + + 'runtime merge/synthesis/stretching.' +); diff --git a/scripts/check-new-map-v2.mjs b/scripts/check-new-map-v2.mjs deleted file mode 100644 index d257e534..00000000 --- a/scripts/check-new-map-v2.mjs +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env node - -import fs from 'node:fs/promises'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); -const failures = []; -const expect = (condition, message) => { - if (!condition) failures.push(message); -}; - -const [main, mapSource, css, server, packageJson] = await Promise.all([ - fs.readFile(path.join(root, 'src', 'main.js'), 'utf8'), - fs.readFile(path.join(root, 'src', 'new-map-v2.js'), 'utf8'), - fs.readFile(path.join(root, 'src', 'new-map-v2.css'), 'utf8'), - fs.readFile(path.join(root, 'server-new-map-v2.mjs'), 'utf8'), - fs.readFile(path.join(root, 'package.json'), 'utf8').then(JSON.parse) -]); - -expect(main.includes("./new-map-v2.js"), 'Application entry point does not use the clean map v2 renderer.'); -expect(main.includes("./new-map-v2.css"), 'Application entry point does not use the clean map v2 stylesheet.'); -expect(!main.includes('occumed-map.js'), 'Legacy PMTiles-aware map renderer is still active.'); -expect(!main.includes('flat-overview.css'), 'Legacy flat-overview stylesheet is still active.'); - -expect(mapSource.includes("https://tiles.openfreemap.org/styles/liberty"), 'Clean map does not use the selected complete worldwide style.'); -expect(mapSource.includes('vectorSources.length !== 1'), 'Clean map does not enforce exactly one vector source.'); -expect(mapSource.includes("projection = { type: 'mercator' }"), 'Clean map does not force Mercator projection.'); -expect(mapSource.includes("water: '#79BCEC'"), 'Occu-Med water color is not locked.'); -expect(mapSource.includes("park: '#A5CC8E'"), 'Occu-Med park color is not locked.'); -expect(mapSource.includes("road: '#F2F2F2'"), 'Occu-Med road color is not locked.'); -expect(mapSource.includes("boundary: '#A65966'"), 'Occu-Med boundary color is not locked.'); -expect(mapSource.includes("'occumed:source-count': 1"), 'One-source metadata lock is missing.'); -expect(mapSource.includes("architecture: 'clean-worldwide-vector-v2'"), 'Browser readiness contract is missing.'); -expect(mapSource.includes('renderedFeatureCount'), 'Browser readiness contract does not record rendered features.'); -expect(mapSource.includes('renderedSourceLayers'), 'Browser readiness contract does not record rendered source layers.'); - -expect(css.includes('#79BCEC'), 'Map canvas fallback is not locked to the Occu-Med water color.'); -expect(css.includes('.occumed-atmosphere-bloom'), 'Clean stylesheet does not explicitly suppress the old atmosphere layer.'); - -expect(packageJson.scripts?.build === 'npm run check:new-map && vite build', 'Production build still runs the legacy PMTiles/style pipeline.'); -expect(packageJson.scripts?.start === 'node server-new-map-v2.mjs', 'Production start command does not use the clean static server.'); -expect(packageJson.scripts?.dev === 'vite', 'Development command still runs the legacy asset pipeline.'); - -for (const [name, content] of [['renderer', mapSource], ['server', server], ['entry', main]]) { - for (const forbidden of [ - 'PMTiles(', - '/tiles/{z}/{x}/{y}', - 'world-tile-gateway', - 'mergeVectorTiles', - 'overscaleVectorLayer', - 'NAV_DATABASE_URL_', - 'neon-navigation-tile-cache', - 'start-flat-overview', - 'occumed-world-overview.pmtiles', - 'occumed-world-surface.pmtiles' - ]) { - expect(!content.includes(forbidden), `${name} still references forbidden legacy path: ${forbidden}`); - } -} - -expect(server.includes("mode: 'clean-worldwide-vector-v2'"), 'Readiness endpoint does not identify the new architecture.'); -expect(server.includes('runtimeMerging: false'), 'Readiness endpoint does not lock runtime merging off.'); -expect(server.includes('regionalRouting: false'), 'Readiness endpoint does not lock regional routing off.'); -expect(server.includes('neon: false'), 'Readiness endpoint does not lock Neon off.'); - -if (failures.length) { - console.error('Clean worldwide map v2 architecture validation failed:'); - for (const failure of failures) console.error(`- ${failure}`); - process.exit(1); -} - -console.log('Clean worldwide map v2 validated: one complete vector source, Mercator, Occu-Med palette, no PMTiles, no Neon, no routing, and no runtime merging.'); diff --git a/scripts/check-photo-reference.mjs b/scripts/check-photo-reference.mjs index 471f4820..0dae6a68 100644 --- a/scripts/check-photo-reference.mjs +++ b/scripts/check-photo-reference.mjs @@ -83,7 +83,7 @@ assert(JSON.stringify(landcover?.paint?.['fill-opacity'] || []).includes('1'), ' assert(layer('landuse')?.paint?.['fill-opacity'] === 1, 'Detailed landuse swatches are being weakened by extra opacity.'); assert(layer('water')?.paint?.['fill-opacity'] === 1, 'Water must remain fully opaque.'); assert(layer('water-depth')?.['source-layer'] === 'depth', 'The reference bathymetry layer is missing.'); -for (const required of ['#79BCEC59', '#5AACE759', '#3B9DE359']) { +for (const required of ['#79BCEC', '#6EB6EA', '#63B1E9']) { assert( JSON.stringify(layer('water-depth')?.paint?.['fill-color'] || []).includes(required), `The exported bathymetry swatch ${required} is missing.` @@ -92,11 +92,8 @@ for (const required of ['#79BCEC59', '#5AACE759', '#3B9DE359']) { assert(!runtime.layers.some((candidate) => candidate.type === 'raster'), 'A raster fallback basemap was reintroduced.'); -const hillshade = layer('occumed-hillshade'); -assert(hillshade?.minzoom === 1.5, 'Hillshade must begin early enough to shape the globe.'); -assert(hillshade?.paint?.['hillshade-illumination-anchor'] === 'map', 'Hillshade must stay fixed to geography.'); -assert(hillshade?.paint?.['hillshade-shadow-color'] === '#0000004D', 'Hillshade shadows are tinting the exported palette.'); -assert(hillshade?.paint?.['hillshade-highlight-color'] === '#FFFFFF4D', 'Hillshade highlights are tinting the exported palette.'); +assert(!runtime.layers.some((candidate) => candidate.type === 'hillshade'), 'An external hillshade layer was reintroduced.'); +assert(Object.keys(runtime.sources || {}).length === 1, 'The reference style no longer has exactly one browser source.'); const paintedLayers = runtime.layers.filter((candidate) => candidate.paint); const allColors = collectHex(paintedLayers.map((candidate) => candidate.paint)); diff --git a/scripts/check-runtime.mjs b/scripts/check-runtime.mjs index 11ac673b..6f8421c5 100644 --- a/scripts/check-runtime.mjs +++ b/scripts/check-runtime.mjs @@ -31,9 +31,9 @@ if (runtime.metadata?.['occumed:glyph-rendering'] !== 'local-maplibre') { } if (!String(runtime.sprite).includes('/sprites/occumed')) fail('Local Occu-Med sprite endpoint is missing.'); -const allowedSources = new Set(['occumed-open', 'occumed-terrain']); -for (const sourceName of Object.keys(runtime.sources || {})) { - if (!allowedSources.has(sourceName)) fail(`Unexpected shared source: ${sourceName}`); +const sourceNames = Object.keys(runtime.sources || {}); +if (sourceNames.length !== 1 || sourceNames[0] !== 'occumed-open') { + fail(`Runtime must expose exactly one browser source (occumed-open); found ${sourceNames.join(', ') || 'none'}.`); } const requiredSourceLayers = new Set([ @@ -61,7 +61,7 @@ const minimumLayerCount = Math.max(100, Math.floor(original.layers.length * 0.7) if (runtime.layers.length < minimumLayerCount) { fail(`Runtime style is too incomplete: ${runtime.layers.length} layers; expected at least ${minimumLayerCount}.`); } -if (!runtime.layers.some((layer) => layer.id === 'occumed-hillshade')) fail('Open hillshade layer is missing.'); +if (runtime.layers.some((layer) => layer.type === 'hillshade')) fail('External terrain hillshade remains active.'); if (!runtime.layers.some((layer) => layer.id === 'occumed-land-surface')) fail('Worldwide land surface layer is missing.'); if (runtime.layers.some((layer) => layer.type === 'raster')) fail('The runtime contains a second raster basemap.'); if (runtime.layers.filter((layer) => layer.type === 'symbol').length < 20) { diff --git a/scripts/check-viewer-quality.mjs b/scripts/check-viewer-quality.mjs index cde6003a..697cb215 100644 --- a/scripts/check-viewer-quality.mjs +++ b/scripts/check-viewer-quality.mjs @@ -52,11 +52,8 @@ if (layer('water')?.paint?.['fill-opacity'] !== 1) fail('Water is translucent an if (runtime.layers.some((candidate) => candidate.type === 'raster')) fail('A raster fallback basemap was reintroduced.'); -const hillshade = layer('occumed-hillshade'); -if (hillshade?.minzoom !== 1.5) fail('Hillshade begins too late to give the globe physical form.'); -if (hillshade?.paint?.['hillshade-shadow-color'] !== '#0000004D') fail('Hillshade shadows are tinting exported colors.'); -if (hillshade?.paint?.['hillshade-highlight-color'] !== '#FFFFFF4D') fail('Hillshade highlights are tinting exported colors.'); -if (!JSON.stringify(hillshade?.paint?.['hillshade-exaggeration'] || []).includes('0.2')) fail('Regional terrain definition is too weak.'); +if (runtime.layers.some((candidate) => candidate.type === 'hillshade')) fail('An external hillshade layer was reintroduced.'); +if (Object.keys(runtime.sources || {}).length !== 1) fail('Viewer no longer uses exactly one browser source.'); const allColors = collectHex(runtime.layers.map((candidate) => candidate.paint || {})); if (allColors.size < 25) fail(`The exported per-structure palette was flattened to only ${allColors.size} colors.`); diff --git a/scripts/lock-exact-exported-swatches.mjs b/scripts/lock-exact-exported-swatches.mjs index 5018992b..f14ab0a1 100644 --- a/scripts/lock-exact-exported-swatches.mjs +++ b/scripts/lock-exact-exported-swatches.mjs @@ -30,9 +30,12 @@ const EXACT = Object.freeze({ wetland: '#A5CAD6', // supplied Studio chip, converted from Display-P3 water: '#79BCEC', // hsl(205, 75%, 70%) waterShadow: '#7293EE', // hsl(224, 79%, 69%) - depthShallow: '#79BCEC59', // hsla(205, 75%, 70%, 0.35) - depthMid: '#5AACE759', // hsla(205, 75%, 63%, 0.35) - depthDeep: '#3B9DE359' // hsla(205, 75%, 56%, 0.35) + // Preblended, opaque equivalents of the exported 35%-alpha bathymetry + // swatches over the permanent ocean background. Opaque depth is idempotent: + // nested bands cannot accumulate different alpha at tile boundaries. + depthShallow: '#79BCEC', + depthMid: '#6EB6EA', + depthDeep: '#63B1E9' }); function requireLayer(id) { @@ -151,31 +154,17 @@ water.paint['fill-color'] = EXACT.water; water.paint['fill-opacity'] = 1; const waterDepth = requireLayer('water-depth'); -waterDepth.maxzoom = 8; +delete waterDepth.maxzoom; waterDepth.paint['fill-antialias'] = false; waterDepth.paint['fill-color'] = [ 'interpolate', ['linear'], - ['zoom'], - 6, - [ - 'interpolate', - ['linear'], - ['get', 'min_depth'], - 0, EXACT.depthShallow, - 200, EXACT.depthMid, - 7000, EXACT.depthDeep - ], - 8, - [ - 'interpolate', - ['linear'], - ['get', 'min_depth'], - 0, '#79BCEC00', - 200, '#5AACE700', - 7000, '#2D96E100' - ] + ['get', 'min_depth'], + 0, EXACT.depthShallow, + 200, EXACT.depthMid, + 7000, EXACT.depthDeep ]; +waterDepth.paint['fill-opacity'] = 1; const waterway = requireLayer('waterway'); waterway.paint['line-color'] = EXACT.water; @@ -209,28 +198,9 @@ for (const group of REFERENCE_STUDIO_EXPRESSION_SWATCHES) { } } -// Use neutral light and shadow only. Terrain may change lightness, but never hue. -const hillshade = requireLayer('occumed-hillshade'); -hillshade.minzoom = 1.5; -hillshade.maxzoom = 16; -hillshade.paint = { - 'hillshade-exaggeration': [ - 'interpolate', - ['linear'], - ['zoom'], - 1.5, 0.04, - 5, 0.09, - 8, 0.15, - 11, 0.2, - 14, 0.24, - 16, 0.16 - ], - 'hillshade-shadow-color': '#0000004D', - 'hillshade-highlight-color': '#FFFFFF4D', - 'hillshade-accent-color': '#00000026', - 'hillshade-illumination-direction': 335, - 'hillshade-illumination-anchor': 'map' -}; +if (runtime.layers.some((candidate) => candidate.type === 'hillshade')) { + throw new Error('One-source runtime must not contain a hillshade layer.'); +} runtime.metadata = { ...(runtime.metadata || {}), @@ -241,6 +211,8 @@ runtime.metadata = { 'occumed:palette-source': 'supplied-mapbox-studio-screenshots-display-p3-to-srgb', 'occumed:layer-specific-palette': true, 'occumed:raster-relief-disabled': true, + 'occumed:external-terrain-disabled': true, + 'occumed:opaque-depth-palette': true, 'occumed:high-dpi-vector-clarity': true, 'occumed:exact-swatches': EXACT, 'occumed:unavailable-reference-swatches': REFERENCE_STUDIO_UNAVAILABLE_SWATCHES diff --git a/scripts/offline-tileset/build-archive.mjs b/scripts/offline-tileset/build-archive.mjs new file mode 100644 index 00000000..ced4d354 --- /dev/null +++ b/scripts/offline-tileset/build-archive.mjs @@ -0,0 +1,198 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { zxyToTileId } from 'pmtiles'; +import { compileImmutableTile } from './mvt-normalizer.mjs'; +import { openPmtiles } from './local-pmtiles.mjs'; +import { DeterministicPmtilesWriter } from './pmtiles-writer.mjs'; + +function parseArguments(argv) { + const result = { regional: [] }; + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (!token.startsWith('--')) throw new Error(`Unexpected argument: ${token}`); + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${token}.`); + const key = token.slice(2); + if (key === 'regional') result.regional.push(value); + else result[key] = value; + index += 1; + } + for (const required of ['targets', 'overview', 'surface', 'output', 'report', 'workdir']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +function parseRegional(value) { + const separator = value.indexOf('='); + if (separator <= 0) throw new Error(`Regional input must be asset=location: ${value}`); + return { + assetName: value.slice(0, separator), + location: value.slice(separator + 1) + }; +} + +function displayLocation(value) { + try { + return path.basename(new URL(value).pathname); + } catch { + return path.basename(value); + } +} + +function mergeCounts(target, report) { + for (const key of [ + 'rejectedMalformed', + 'rejectedOversized', + 'rejectedTileFootprints', + 'removedExactDuplicates', + 'removedContainedOverlaps', + 'clippedEmpty' + ]) { + target[key] = (target[key] || 0) + Number(report[key] || 0); + } + for (const [layer, counts] of Object.entries(report.layers || {})) { + target.layers[layer] ||= { featureCount: 0, pointCount: 0, tileCount: 0 }; + target.layers[layer].featureCount += Number(counts.featureCount || 0); + target.layers[layer].pointCount += Number(counts.pointCount || 0); + target.layers[layer].tileCount += 1; + } +} + +const options = parseArguments(process.argv.slice(2)); +const targetDocument = JSON.parse(await fs.readFile(path.resolve(options.targets), 'utf8')); +const targetName = options['target-name'] || 'foundation'; +const targets = targetDocument.targets?.[targetName] || targetDocument[targetName]; +if (!Array.isArray(targets) || !targets.length) { + throw new Error(`Target list is empty: ${targetName}`); +} +const ordered = [...targets] + .map(({ z, x, y }) => ({ z: Number(z), x: Number(x), y: Number(y) })) + .sort((left, right) => + zxyToTileId(left.z, left.x, left.y) - zxyToTileId(right.z, right.x, right.y) + ); +for (let index = 1; index < ordered.length; index += 1) { + if (zxyToTileId(ordered[index - 1].z, ordered[index - 1].x, ordered[index - 1].y) === + zxyToTileId(ordered[index].z, ordered[index].x, ordered[index].y)) { + throw new Error(`Duplicate target tile: ${ordered[index].z}/${ordered[index].x}/${ordered[index].y}`); + } +} + +const startedAt = performance.now(); +const overview = await openPmtiles(options.overview, { cacheEntries: 4_096 }); +const surface = await openPmtiles(options.surface, { cacheEntries: 4_096 }); +const regional = []; +for (const input of options.regional.map(parseRegional)) { + const opened = await openPmtiles(input.location, { cacheEntries: 4_096 }); + opened.assetName = input.assetName; + regional.push(opened); +} +const writer = await new DeterministicPmtilesWriter({ + output: path.resolve(options.output), + workDirectory: path.resolve(options.workdir), + metadata: { + name: `Occu-Med immutable owner ${targetName}`, + type: 'baselayer', + format: 'pbf', + compression: 'gzip', + authority_land: 'world-surface', + authority_depth: 'world-surface', + authority_landcover: 'world-overview', + authority_cartography: 'regional-owner', + runtime_merge: false, + runtime_geometry: false + } +}).initialize(); + +const aggregate = { + rejectedMalformed: 0, + rejectedOversized: 0, + rejectedTileFootprints: 0, + removedExactDuplicates: 0, + removedContainedOverlaps: 0, + clippedEmpty: 0, + layers: {} +}; +let completed = 0; +const concurrency = Math.max(1, Math.min(8, Number(options.concurrency || 4))); + +try { + for (let start = 0; start < ordered.length; start += concurrency) { + const batch = ordered.slice(start, start + concurrency); + const compiled = await Promise.all( + batch.map((tile) => compileImmutableTile({ + ...tile, + overview, + surface, + regional + })) + ); + for (let index = 0; index < batch.length; index += 1) { + await writer.addTile({ ...batch[index], data: compiled[index].data }); + mergeCounts(aggregate, compiled[index].report); + completed += 1; + } + if (completed % 250 === 0 || completed === ordered.length) { + console.log(`[${targetName}] ${completed}/${ordered.length} tiles normalized.`); + } + } + const finalized = await writer.finalize(); + const durationSeconds = (performance.now() - startedAt) / 1_000; + const report = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + targetName, + file: options['manifest-file'] || path.basename(options.output), + bytes: finalized.bytes, + sha256: finalized.sha256, + addressedTiles: finalized.addressedTiles, + tileEntries: finalized.tileEntries, + tileContents: finalized.tileContents, + minZoom: finalized.minZoom, + maxZoom: Number(options['foundation-max-zoom'] || finalized.maxZoom), + durationSeconds, + authorities: { + land: 'world-surface', + depth: 'world-surface', + landcover: 'world-overview', + cartography: 'regional-owner' + }, + inputs: { + overview: displayLocation(options.overview), + surface: displayLocation(options.surface), + regional: regional.map((opened) => opened.assetName) + }, + normalization: aggregate + }; + if (options['owner-id']) { + const [z, x, y] = String(options['owner-prefix']).split('/').map(Number); + const selectedOwner = targetDocument.selectedOwners?.find( + (owner) => owner.id === options['owner-id'] + ); + report.owner = { + id: options['owner-id'], + prefix: { z, x, y }, + exactTiles: selectedOwner?.exactTiles || [] + }; + } + const reportPath = path.resolve(options.report); + await fs.mkdir(path.dirname(reportPath), { recursive: true }); + const pending = `${reportPath}.pending-${process.pid}`; + await fs.writeFile(pending, `${JSON.stringify(report, null, 2)}\n`, { flag: 'wx' }); + await fs.rename(pending, reportPath); + console.log( + `[${targetName}] built ${finalized.bytes} bytes in ${durationSeconds.toFixed(1)} seconds; ` + + `SHA-256 ${finalized.sha256}.` + ); +} catch (error) { + await writer.abort(); + throw error; +} finally { + await Promise.allSettled([ + overview.close(), + surface.close(), + ...regional.map((opened) => opened.close()) + ]); +} diff --git a/scripts/offline-tileset/build-foundation.mjs b/scripts/offline-tileset/build-foundation.mjs new file mode 100644 index 00000000..4a44be3c --- /dev/null +++ b/scripts/offline-tileset/build-foundation.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`Invalid argument near ${key || ''}.`); + } + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'targets', 'input-report', 'output-dir']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +async function runNode(script, args) { + await new Promise((resolve, reject) => { + const child = spawn(process.execPath, [script, ...args], { stdio: 'inherit' }); + child.on('error', reject); + child.on('exit', (code, signal) => { + if (signal) reject(new Error(`${path.basename(script)} terminated by ${signal}.`)); + else if (code !== 0) reject(new Error(`${path.basename(script)} exited with ${code}.`)); + else resolve(); + }); + }); +} + +const options = parseArguments(process.argv.slice(2)); +const [plan, targets, inputReport] = await Promise.all([ + fs.readFile(path.resolve(options.plan), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options.targets), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options['input-report']), 'utf8').then(JSON.parse) +]); +if (plan.planVersion !== targets.planVersion || plan.planVersion !== inputReport.planVersion) { + throw new Error('Plan, targets, and localized inputs do not share one immutable plan version.'); +} +const inputByAsset = new Map(inputReport.inputs.map((input) => [input.asset, input.path])); +const overview = inputByAsset.get(plan.inputs.overview.asset); +const surface = inputByAsset.get(plan.inputs.surface.asset); +if (!overview || !surface) throw new Error('Localized overview or surface input is missing.'); + +const outputDir = path.resolve(options['output-dir']); +await fs.mkdir(path.join(outputDir, 'reports'), { recursive: true }); +await fs.mkdir(path.join(outputDir, 'work'), { recursive: true }); +const output = path.join(outputDir, 'foundation.pmtiles'); +const report = path.join(outputDir, 'reports/foundation.json'); +if (await fs.stat(output).catch(() => null) || await fs.stat(report).catch(() => null)) { + throw new Error('Immutable foundation output already exists.'); +} +await runNode(path.join(scriptDirectory, 'build-archive.mjs'), [ + '--targets', path.resolve(options.targets), + '--target-name', 'foundation', + '--overview', overview, + '--surface', surface, + '--output', output, + '--report', report, + '--workdir', path.join(outputDir, 'work/foundation'), + '--manifest-file', 'foundation.pmtiles', + '--foundation-max-zoom', String(plan.routingZoom), + '--concurrency', String(options.concurrency || 4) +]); diff --git a/scripts/offline-tileset/build-owner.mjs b/scripts/offline-tileset/build-owner.mjs new file mode 100644 index 00000000..762716ba --- /dev/null +++ b/scripts/offline-tileset/build-owner.mjs @@ -0,0 +1,83 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`Invalid argument near ${key || ''}.`); + } + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'owner-id', 'targets', 'input-report', 'output-dir']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +async function runNode(script, args) { + await new Promise((resolve, reject) => { + const child = spawn(process.execPath, [script, ...args], { stdio: 'inherit' }); + child.on('error', reject); + child.on('exit', (code, signal) => { + if (signal) reject(new Error(`${path.basename(script)} terminated by ${signal}.`)); + else if (code !== 0) reject(new Error(`${path.basename(script)} exited with ${code}.`)); + else resolve(); + }); + }); +} + +const options = parseArguments(process.argv.slice(2)); +const [plan, targets, inputReport] = await Promise.all([ + fs.readFile(path.resolve(options.plan), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options.targets), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options['input-report']), 'utf8').then(JSON.parse) +]); +if (plan.planVersion !== targets.planVersion || plan.planVersion !== inputReport.planVersion) { + throw new Error('Plan, targets, and localized inputs do not share one immutable plan version.'); +} +const owner = plan.owners.find((candidate) => candidate.id === options['owner-id']); +if (!owner) throw new Error(`Owner is not present in the plan: ${options['owner-id']}`); +if (!targets.targets?.[owner.id]?.length) throw new Error(`Owner targets are empty: ${owner.id}`); +const inputByAsset = new Map(inputReport.inputs.map((input) => [input.asset, input.path])); +const overview = inputByAsset.get(plan.inputs.overview.asset); +const surface = inputByAsset.get(plan.inputs.surface.asset); +if (!overview || !surface) throw new Error('Localized overview or surface input is missing.'); +const regional = owner.candidates.flatMap((candidate) => { + const filename = inputByAsset.get(candidate.asset); + if (!filename) throw new Error(`Localized owner input is missing: ${candidate.asset}`); + return ['--regional', `${candidate.asset}=${filename}`]; +}); + +const outputDir = path.resolve(options['output-dir']); +await fs.mkdir(path.join(outputDir, 'owners'), { recursive: true }); +await fs.mkdir(path.join(outputDir, 'reports/owners'), { recursive: true }); +await fs.mkdir(path.join(outputDir, 'work'), { recursive: true }); +const output = path.join(outputDir, `owners/${owner.id}.pmtiles`); +const report = path.join(outputDir, `reports/owners/${owner.id}.json`); +if (await fs.stat(output).catch(() => null) || await fs.stat(report).catch(() => null)) { + throw new Error(`Immutable owner output already exists: ${owner.id}`); +} +await runNode(path.join(scriptDirectory, 'build-archive.mjs'), [ + '--targets', path.resolve(options.targets), + '--target-name', owner.id, + '--overview', overview, + '--surface', surface, + ...regional, + '--output', output, + '--report', report, + '--workdir', path.join(outputDir, `work/${owner.id}`), + '--manifest-file', `owners/${owner.id}.pmtiles`, + '--owner-id', owner.id, + '--owner-prefix', `${owner.prefix.z}/${owner.prefix.x}/${owner.prefix.y}`, + '--concurrency', String(options.concurrency || 4) +]); diff --git a/scripts/offline-tileset/build-production-batch.mjs b/scripts/offline-tileset/build-production-batch.mjs new file mode 100644 index 00000000..e2906815 --- /dev/null +++ b/scripts/offline-tileset/build-production-batch.mjs @@ -0,0 +1,136 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`Invalid argument near ${key || ''}.`); + } + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'batch-plan', 'batch-index', 'targets', 'output-dir']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +async function runNode(script, args) { + await new Promise((resolve, reject) => { + const child = spawn(process.execPath, [script, ...args], { stdio: 'inherit' }); + child.on('error', reject); + child.on('exit', (code, signal) => { + if (signal) reject(new Error(`${path.basename(script)} terminated by ${signal}.`)); + else if (code !== 0) reject(new Error(`${path.basename(script)} exited with ${code}.`)); + else resolve(); + }); + }); +} + +const options = parseArguments(process.argv.slice(2)); +const [plan, batchPlan, targets] = await Promise.all([ + fs.readFile(path.resolve(options.plan), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options['batch-plan']), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options.targets), 'utf8').then(JSON.parse) +]); +if ( + plan.planVersion !== batchPlan.planVersion || + plan.planVersion !== targets.planVersion || + batchPlan.batchPlanVersion !== targets.batchPlanVersion +) { + throw new Error('Production plan, batch plan, and targets do not share one immutable version.'); +} +const batchIndex = Number(options['batch-index']); +const batch = batchPlan.batches?.find((candidate) => candidate.index === batchIndex); +if (!batch || targets.batch?.id !== batch.id) { + throw new Error(`Production batch metadata mismatch: ${batchIndex}.`); +} + +const batchOwnerIds = new Set(batch.ownerIds); +const selectedOwnerIds = new Set(targets.selectedOwners.map((owner) => owner.id)); +const activeOwnerIds = new Set(targets.batch.activeOwnerIds || []); +const emptyOwnerIds = new Set(targets.batch.emptyOwnerIds || []); +if ( + selectedOwnerIds.size !== activeOwnerIds.size || + [...selectedOwnerIds].some((id) => !activeOwnerIds.has(id)) +) { + throw new Error(`Active production owner inventory is inconsistent for ${batch.id}.`); +} +if ( + activeOwnerIds.size + emptyOwnerIds.size !== batchOwnerIds.size || + [...activeOwnerIds, ...emptyOwnerIds].some((id) => !batchOwnerIds.has(id)) +) { + throw new Error(`Active/empty production owner partition is incomplete for ${batch.id}.`); +} +const owners = plan.owners.filter((owner) => activeOwnerIds.has(owner.id)); +if (owners.length !== activeOwnerIds.size) { + throw new Error(`Active production owner lookup is incomplete for ${batch.id}.`); +} + +const regionalByAsset = new Map(); +for (const owner of owners) { + for (const candidate of owner.candidates) regionalByAsset.set(candidate.asset, candidate); +} +const outputDir = path.resolve(options['output-dir']); +await fs.mkdir(path.join(outputDir, 'batches'), { recursive: true }); +await fs.mkdir(path.join(outputDir, 'reports/batches'), { recursive: true }); +await fs.mkdir(path.join(outputDir, 'work'), { recursive: true }); +const output = path.join(outputDir, `batches/${batch.file}`); +const report = path.join(outputDir, `reports/batches/${batch.id}.json`); +if (await fs.stat(output).catch(() => null) || await fs.stat(report).catch(() => null)) { + throw new Error(`Immutable production output already exists: ${batch.id}`); +} + +const regionalArgs = [...regionalByAsset.values()] + .sort((left, right) => left.asset.localeCompare(right.asset)) + .flatMap((candidate) => ['--regional', `${candidate.asset}=${candidate.url}`]); +await runNode(path.join(scriptDirectory, 'build-archive.mjs'), [ + '--targets', path.resolve(options.targets), + '--target-name', batch.id, + '--overview', plan.inputs.overview.url, + '--surface', plan.inputs.surface.url, + ...regionalArgs, + '--output', output, + '--report', report, + '--workdir', path.join(outputDir, `work/${batch.id}`), + '--manifest-file', batch.file, + '--concurrency', String(options.concurrency || 8) +]); + +const built = JSON.parse(await fs.readFile(report, 'utf8')); +const maxBytes = Number(options['max-bytes'] || 1_900_000_000); +if (!Number.isSafeInteger(maxBytes) || maxBytes < 1_000_000) { + throw new Error('Invalid production batch byte limit.'); +} +if (built.bytes > maxBytes) { + throw new Error( + `${batch.id} produced ${built.bytes} bytes, exceeding the ${maxBytes}-byte release limit.` + ); +} +built.batch = batch; +built.planVersion = plan.planVersion; +built.batchPlanVersion = batchPlan.batchPlanVersion; +built.owners = owners.map((owner) => ({ + id: owner.id, + prefix: owner.prefix, + exactTiles: owner.exactTiles || [] +})); +built.emptyOwnerIds = [...emptyOwnerIds].sort(); +built.sourceOwnerCount = batch.ownerIds.length; +built.activeSourceOwnerCount = owners.length; +const pending = `${report}.pending-${process.pid}`; +await fs.writeFile(pending, `${JSON.stringify(built, null, 2)}\n`, { flag: 'wx' }); +await fs.rename(pending, report); +console.log( + `${batch.id} is release-ready: ${built.bytes} bytes, ${built.addressedTiles} exact tiles, ` + + `${owners.length} active and ${emptyOwnerIds.size} empty source owners.` +); diff --git a/scripts/offline-tileset/build-production-foundation.mjs b/scripts/offline-tileset/build-production-foundation.mjs new file mode 100644 index 00000000..1c7d10d2 --- /dev/null +++ b/scripts/offline-tileset/build-production-foundation.mjs @@ -0,0 +1,67 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`Invalid argument near ${key || ''}.`); + } + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'targets', 'output-dir']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +async function runNode(script, args) { + await new Promise((resolve, reject) => { + const child = spawn(process.execPath, [script, ...args], { stdio: 'inherit' }); + child.on('error', reject); + child.on('exit', (code, signal) => { + if (signal) reject(new Error(`${path.basename(script)} terminated by ${signal}.`)); + else if (code !== 0) reject(new Error(`${path.basename(script)} exited with ${code}.`)); + else resolve(); + }); + }); +} + +const options = parseArguments(process.argv.slice(2)); +const [plan, targets] = await Promise.all([ + fs.readFile(path.resolve(options.plan), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options.targets), 'utf8').then(JSON.parse) +]); +if (plan.planVersion !== targets.planVersion) { + throw new Error('Foundation targets do not match the immutable owner plan.'); +} +const outputDir = path.resolve(options['output-dir']); +await fs.mkdir(path.join(outputDir, 'reports'), { recursive: true }); +await fs.mkdir(path.join(outputDir, 'work'), { recursive: true }); +const output = path.join(outputDir, 'foundation.pmtiles'); +const report = path.join(outputDir, 'reports/foundation.json'); +if (await fs.stat(output).catch(() => null) || await fs.stat(report).catch(() => null)) { + throw new Error('Immutable production foundation already exists.'); +} +await runNode(path.join(scriptDirectory, 'build-archive.mjs'), [ + '--targets', path.resolve(options.targets), + '--target-name', 'foundation', + '--overview', plan.inputs.overview.url, + '--surface', plan.inputs.surface.url, + '--output', output, + '--report', report, + '--workdir', path.join(outputDir, 'work/foundation'), + '--manifest-file', 'foundation.pmtiles', + '--foundation-max-zoom', String(plan.routingZoom), + '--concurrency', String(options.concurrency || 8) +]); +console.log('Production foundation is release-ready.'); diff --git a/scripts/offline-tileset/build-representative.mjs b/scripts/offline-tileset/build-representative.mjs new file mode 100644 index 00000000..2a57334e --- /dev/null +++ b/scripts/offline-tileset/build-representative.mjs @@ -0,0 +1,140 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); +const FOUNDATION_WORLD_MAX_ZOOM = 6; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) throw new Error(`Unexpected argument: ${key}`); + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${key}.`); + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'targets', 'input-report', 'output-dir']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +async function runNode(script, args) { + await new Promise((resolve, reject) => { + const child = spawn(process.execPath, [script, ...args], { + stdio: 'inherit' + }); + child.on('error', reject); + child.on('exit', (code, signal) => { + if (signal) reject(new Error(`${path.basename(script)} terminated by ${signal}.`)); + else if (code !== 0) reject(new Error(`${path.basename(script)} exited with ${code}.`)); + else resolve(); + }); + }); +} + +const options = parseArguments(process.argv.slice(2)); +const planPath = path.resolve(options.plan); +const targetsPath = path.resolve(options.targets); +const inputReportPath = path.resolve(options['input-report']); +const outputDir = path.resolve(options['output-dir']); +const stat = await fs.stat(outputDir).catch(() => null); +if (stat) throw new Error(`Representative output directory already exists: ${outputDir}`); +await fs.mkdir(path.join(outputDir, 'owners'), { recursive: true }); +await fs.mkdir(path.join(outputDir, 'reports'), { recursive: true }); +await fs.mkdir(path.join(outputDir, 'work'), { recursive: true }); + +const [plan, targets, inputReport] = await Promise.all([ + fs.readFile(planPath, 'utf8').then(JSON.parse), + fs.readFile(targetsPath, 'utf8').then(JSON.parse), + fs.readFile(inputReportPath, 'utf8').then(JSON.parse) +]); +if (plan.planVersion !== inputReport.planVersion) { + throw new Error('Localized inputs do not match the owner plan.'); +} +const inputByAsset = new Map(inputReport.inputs.map((input) => [input.asset, input.path])); +const overview = inputByAsset.get(plan.inputs.overview.asset); +const surface = inputByAsset.get(plan.inputs.surface.asset); +if (!overview || !surface) throw new Error('Localized overview or surface input is missing.'); + +const buildScript = path.join(scriptDirectory, 'build-archive.mjs'); +const finalizeScript = path.join(scriptDirectory, 'finalize-manifest.mjs'); +const startedAt = performance.now(); +const foundationOutput = path.join(outputDir, 'foundation.pmtiles'); +const foundationReport = path.join(outputDir, 'reports/foundation.json'); +await runNode(buildScript, [ + '--targets', targetsPath, + '--target-name', 'foundation', + '--overview', overview, + '--surface', surface, + '--output', foundationOutput, + '--report', foundationReport, + '--workdir', path.join(outputDir, 'work/foundation'), + '--manifest-file', 'foundation.pmtiles', + '--foundation-max-zoom', String(FOUNDATION_WORLD_MAX_ZOOM) +]); + +const ownerReports = []; +for (const selected of targets.selectedOwners) { + const planOwner = plan.owners.find((owner) => owner.id === selected.id); + if (!planOwner) throw new Error(`Selected owner is not in the locked plan: ${selected.id}`); + const ownerOutput = path.join(outputDir, `owners/${selected.id}.pmtiles`); + const ownerReport = path.join(outputDir, `reports/${selected.id}.json`); + const regionalArgs = []; + const requiredNames = new Set( + (selected.requiredCandidates || planOwner.candidates).map((candidate) => candidate.asset) + ); + for (const candidate of planOwner.candidates.filter((item) => requiredNames.has(item.asset))) { + const filename = inputByAsset.get(candidate.asset); + if (!filename) throw new Error(`Localized owner input is missing: ${candidate.asset}`); + regionalArgs.push('--regional', `${candidate.asset}=${filename}`); + } + await runNode(buildScript, [ + '--targets', targetsPath, + '--target-name', selected.id, + '--overview', overview, + '--surface', surface, + ...regionalArgs, + '--output', ownerOutput, + '--report', ownerReport, + '--workdir', path.join(outputDir, `work/${selected.id}`), + '--manifest-file', `owners/${selected.id}.pmtiles`, + '--owner-id', selected.id, + '--owner-prefix', `${selected.prefix.z}/${selected.prefix.x}/${selected.prefix.y}` + ]); + ownerReports.push(ownerReport); +} + +const manifestPath = path.join(outputDir, 'manifest.json'); +await runNode(finalizeScript, [ + '--plan', planPath, + '--foundation', foundationReport, + ...ownerReports.flatMap((report) => ['--owner', report]), + '--output', manifestPath +]); +const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf8')); +const durationSeconds = (performance.now() - startedAt) / 1_000; +const buildReport = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + artifactVersion: manifest.artifactVersion, + durationSeconds, + totalBytes: manifest.totalBytes, + addressedTiles: targets.totalAddressedTiles, + foundationReport: path.relative(outputDir, foundationReport), + ownerReports: ownerReports.map((report) => path.relative(outputDir, report)), + manifest: path.relative(outputDir, manifestPath) +}; +await fs.writeFile( + path.join(outputDir, 'representative-build-report.json'), + `${JSON.stringify(buildReport, null, 2)}\n` +); +console.log( + `Representative immutable tileset ${manifest.artifactVersion} built in ` + + `${durationSeconds.toFixed(1)} seconds (${manifest.totalBytes} bytes).` +); diff --git a/scripts/offline-tileset/finalize-manifest.mjs b/scripts/offline-tileset/finalize-manifest.mjs new file mode 100644 index 00000000..a5bd1051 --- /dev/null +++ b/scripts/offline-tileset/finalize-manifest.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { computeImmutableArtifactVersion, validateImmutableManifest } from '../../src/server/immutable-world-tileset.js'; + +function parseArguments(argv) { + const result = { owners: [] }; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) throw new Error(`Unexpected argument: ${key}`); + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${key}.`); + const name = key.slice(2); + if (name === 'owner') result.owners.push(value); + else result[name] = value; + index += 1; + } + for (const required of ['plan', 'foundation', 'output']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +const options = parseArguments(process.argv.slice(2)); +const ownerReportFiles = [...options.owners]; +if (options['owner-dir']) { + const ownerDirectory = path.resolve(options['owner-dir']); + const entries = await fs.readdir(ownerDirectory, { withFileTypes: true }); + ownerReportFiles.push( + ...entries + .filter((entry) => entry.isFile() && entry.name.endsWith('.json')) + .map((entry) => path.join(ownerDirectory, entry.name)) + ); +} +const [plan, foundationReport, ...ownerReports] = await Promise.all([ + fs.readFile(path.resolve(options.plan), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options.foundation), 'utf8').then(JSON.parse), + ...[...new Set(ownerReportFiles)].map((filename) => + fs.readFile(path.resolve(filename), 'utf8').then(JSON.parse) + ) +]); +if (plan.schemaVersion !== 1 || !plan.planVersion) { + throw new Error('Immutable owner plan is invalid.'); +} + +const foundation = { + id: 'foundation', + file: foundationReport.file, + bytes: foundationReport.bytes, + sha256: foundationReport.sha256, + maxZoom: foundationReport.maxZoom +}; +const owners = ownerReports + .map((report) => ({ + id: report.owner.id, + prefix: report.owner.prefix, + exactTiles: report.owner.exactTiles || [], + file: report.file, + bytes: report.bytes, + sha256: report.sha256 + })) + .sort((left, right) => + left.prefix.z - right.prefix.z || + left.prefix.x - right.prefix.x || + left.prefix.y - right.prefix.y + ); +const complete = owners.length === plan.plannedOwnerCount; +const manifest = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + planVersion: plan.planVersion, + browserSourceId: 'occumed-open', + minZoom: 0, + maxZoom: 16, + complete, + validationFixture: !complete, + logicalOwnerCount: plan.logicalOwnerCount, + plannedOwnerCount: plan.plannedOwnerCount, + builtOwnerCount: owners.length, + defaultOwner: foundation.id, + totalBytes: foundation.bytes + owners.reduce((sum, owner) => sum + owner.bytes, 0), + authorities: { + land: 'world-surface', + depth: 'world-surface', + landcover: 'world-overview', + cartography: 'regional-owner' + }, + runtimePolicy: { + neonTileCache: false, + runtimeShardMerge: false, + runtimeLandcoverSynthesis: false, + runtimeGeometry: false, + parentChildStretching: false + }, + foundation, + owners +}; +if (options['asset-base-url']) manifest.assetBaseUrl = options['asset-base-url']; +manifest.artifactVersion = computeImmutableArtifactVersion(manifest); +validateImmutableManifest(manifest, { allowPartial: !complete }); + +const outputPath = path.resolve(options.output); +await fs.mkdir(path.dirname(outputPath), { recursive: true }); +const pending = `${outputPath}.pending-${process.pid}`; +await fs.writeFile(pending, `${JSON.stringify(manifest, null, 2)}\n`, { flag: 'wx' }); +await fs.rename(pending, outputPath); +console.log( + `Finalized immutable manifest ${manifest.artifactVersion}: ${owners.length} of ` + + `${plan.plannedOwnerCount} owners, ${manifest.totalBytes} bytes.` +); diff --git a/scripts/offline-tileset/finalize-published-production.mjs b/scripts/offline-tileset/finalize-published-production.mjs new file mode 100644 index 00000000..2a6379ad --- /dev/null +++ b/scripts/offline-tileset/finalize-published-production.mjs @@ -0,0 +1,146 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { + computeImmutableArtifactVersion, + validateImmutableManifest +} from '../../src/server/immutable-world-tileset.js'; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`Invalid argument near ${key || ''}.`); + } + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'batch-plan', 'repository', 'tag', 'output']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +async function fetchJson(url, token) { + const response = await fetch(url, { + redirect: 'follow', + headers: { + Accept: 'application/vnd.github+json', + 'User-Agent': 'Occu-Med-Map/production-finalizer', + ...(token ? { Authorization: `Bearer ${token}` } : {}) + } + }); + if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}.`); + return response.json(); +} + +async function releaseAssets(repository, tag, token) { + const release = await fetchJson( + `https://api.github.com/repos/${repository}/releases/tags/${encodeURIComponent(tag)}`, + token + ); + const assets = []; + for (let page = 1; ; page += 1) { + const batch = await fetchJson(`${release.assets_url}?per_page=100&page=${page}`, token); + assets.push(...batch); + if (batch.length < 100) break; + } + return assets; +} + +function lockedAsset(asset, expectedName) { + if (!asset) throw new Error(`Published production asset is missing: ${expectedName}`); + const sha256 = String(asset.digest || '').replace(/^sha256:/, ''); + if (!/^[a-f0-9]{64}$/.test(sha256)) { + throw new Error(`Published production asset lacks SHA-256 lock: ${expectedName}`); + } + return { file: expectedName, bytes: Number(asset.size), sha256 }; +} + +const options = parseArguments(process.argv.slice(2)); +const [plan, batchPlan, assets] = await Promise.all([ + fs.readFile(path.resolve(options.plan), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options['batch-plan']), 'utf8').then(JSON.parse), + releaseAssets(options.repository, options.tag, process.env.GITHUB_TOKEN) +]); +if (plan.planVersion !== batchPlan.planVersion) { + throw new Error('Production batch plan does not match the immutable owner plan.'); +} +const byName = new Map(assets.map((asset) => [asset.name, asset])); +const foundation = { + id: 'foundation', + ...lockedAsset(byName.get('foundation.pmtiles'), 'foundation.pmtiles'), + maxZoom: plan.routingZoom +}; +const sourceOwnerIds = new Set(); +const owners = batchPlan.batches.map((batch) => { + for (const ownerId of batch.ownerIds) { + if (sourceOwnerIds.has(ownerId)) throw new Error(`Source owner is assigned twice: ${ownerId}`); + sourceOwnerIds.add(ownerId); + } + return { + id: batch.id, + prefix: batch.prefix, + exactTiles: [], + ...lockedAsset(byName.get(batch.file), batch.file), + sourceOwnerCount: batch.sourceOwnerCount + }; +}); +if (sourceOwnerIds.size !== plan.owners.length) { + throw new Error(`Published source-owner coverage is incomplete: ${sourceOwnerIds.size}/${plan.owners.length}.`); +} +owners.sort((left, right) => + left.prefix.z - right.prefix.z || + left.prefix.x - right.prefix.x || + left.prefix.y - right.prefix.y +); +const manifest = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + planVersion: plan.planVersion, + batchPlanVersion: batchPlan.batchPlanVersion, + browserSourceId: 'occumed-open', + minZoom: 0, + maxZoom: 16, + complete: true, + validationFixture: false, + logicalOwnerCount: plan.logicalOwnerCount, + sourceOwnerCount: plan.plannedOwnerCount, + plannedOwnerCount: batchPlan.batchCount, + builtOwnerCount: owners.length, + batchCount: batchPlan.batchCount, + defaultOwner: foundation.id, + totalBytes: foundation.bytes + owners.reduce((sum, owner) => sum + owner.bytes, 0), + assetBaseUrl: `https://github.com/${options.repository}/releases/download/${encodeURIComponent(options.tag)}/`, + authorities: { + land: 'world-surface', + depth: 'world-surface', + landcover: 'world-overview', + cartography: 'regional-owner' + }, + runtimePolicy: { + neonTileCache: false, + runtimeShardMerge: false, + runtimeLandcoverSynthesis: false, + runtimeGeometry: false, + parentChildStretching: false + }, + foundation, + owners +}; +manifest.artifactVersion = computeImmutableArtifactVersion(manifest); +validateImmutableManifest(manifest); + +const output = path.resolve(options.output); +await fs.mkdir(path.dirname(output), { recursive: true }); +const pending = `${output}.pending-${process.pid}`; +await fs.writeFile(pending, `${JSON.stringify(manifest, null, 2)}\n`, { flag: 'wx' }); +await fs.rename(pending, output); +console.log( + `Finalized complete production manifest ${manifest.artifactVersion}: ` + + `${owners.length} non-overlapping archives covering ${plan.owners.length} source owners, ` + + `${manifest.totalBytes} bytes.` +); diff --git a/scripts/offline-tileset/generate-foundation-targets.mjs b/scripts/offline-tileset/generate-foundation-targets.mjs new file mode 100644 index 00000000..99caef25 --- /dev/null +++ b/scripts/offline-tileset/generate-foundation-targets.mjs @@ -0,0 +1,46 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`Invalid argument near ${key || ''}.`); + } + result[key.slice(2)] = value; + index += 1; + } + if (!result.plan || !result.output) throw new Error('--plan and --output are required.'); + return result; +} + +const options = parseArguments(process.argv.slice(2)); +const plan = JSON.parse(await fs.readFile(path.resolve(options.plan), 'utf8')); +const maxZoom = Number(options['max-zoom'] || plan.routingZoom); +if (!Number.isSafeInteger(maxZoom) || maxZoom < 0 || maxZoom > plan.routingZoom) { + throw new Error(`Foundation max zoom must be between 0 and ${plan.routingZoom}.`); +} + +const foundation = []; +for (let z = 0; z <= maxZoom; z += 1) { + const width = 2 ** z; + for (let y = 0; y < width; y += 1) { + for (let x = 0; x < width; x += 1) foundation.push({ z, x, y }); + } +} +const document = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + planVersion: plan.planVersion, + selectedOwners: [], + targets: { foundation }, + totalAddressedTiles: foundation.length +}; +const output = path.resolve(options.output); +await fs.mkdir(path.dirname(output), { recursive: true }); +await fs.writeFile(output, `${JSON.stringify(document, null, 2)}\n`, { flag: 'wx' }); +console.log(`Generated ${foundation.length} foundation targets through z${maxZoom}.`); diff --git a/scripts/offline-tileset/generate-owner-targets.mjs b/scripts/offline-tileset/generate-owner-targets.mjs new file mode 100644 index 00000000..1ccfc6a0 --- /dev/null +++ b/scripts/offline-tileset/generate-owner-targets.mjs @@ -0,0 +1,113 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { tileIdToZxy, zxyToTileId } from 'pmtiles'; +import { + openLocalPmtiles, + visitPmtilesTileAddresses +} from './local-pmtiles.mjs'; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`Invalid argument near ${key || ''}.`); + } + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'owner-id', 'input-report', 'output']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +function prefixContains(prefix, tile) { + if (tile.z < prefix.z) return false; + const divisor = 2 ** (tile.z - prefix.z); + return ( + Math.floor(tile.x / divisor) === prefix.x && + Math.floor(tile.y / divisor) === prefix.y + ); +} + +const options = parseArguments(process.argv.slice(2)); +const [plan, inputReport] = await Promise.all([ + fs.readFile(path.resolve(options.plan), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options['input-report']), 'utf8').then(JSON.parse) +]); +if (plan.planVersion !== inputReport.planVersion) { + throw new Error('Localized inputs do not match the immutable owner plan.'); +} +const owner = plan.owners.find((candidate) => candidate.id === options['owner-id']); +if (!owner) throw new Error(`Owner is not present in the plan: ${options['owner-id']}`); +if (!inputReport.ownerIds?.includes(owner.id)) { + throw new Error(`Input report was not localized for ${owner.id}.`); +} +const inputByAsset = new Map(inputReport.inputs.map((input) => [input.asset, input.path])); +const minZoom = Number(options['min-zoom'] || plan.routingZoom + 1); +const maxZoom = Number(options['max-zoom'] || 16); +const targetIds = new Set( + (owner.exactTiles || []).map(({ z, x, y }) => zxyToTileId(z, x, y)) +); +let scannedTiles = 0; +let scannedDirectories = 0; + +for (const candidate of owner.candidates) { + const filename = inputByAsset.get(candidate.asset); + if (!filename) throw new Error(`Localized owner input is missing: ${candidate.asset}`); + const opened = await openLocalPmtiles(filename, { cacheEntries: 2_048 }); + try { + const scanned = await visitPmtilesTileAddresses(opened, ({ z, x, y, tileId }) => { + if ( + z >= minZoom && + z <= maxZoom && + prefixContains(owner.prefix, { z, x, y }) + ) { + targetIds.add(tileId); + } + }); + scannedTiles += scanned.addressedTiles; + scannedDirectories += scanned.directoryCount; + } finally { + await opened.close(); + } +} + +const targets = [...targetIds] + .sort((left, right) => left - right) + .map((tileId) => { + const [z, x, y] = tileIdToZxy(tileId); + return { z, x, y }; + }); +if (!targets.length) throw new Error(`Owner ${owner.id} has no addressed output tiles.`); +const document = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + planVersion: plan.planVersion, + selectedOwners: [{ + id: owner.id, + prefix: owner.prefix, + exactTiles: owner.exactTiles || [], + requiredCandidates: owner.candidates + }], + targets: { + [owner.id]: targets + }, + totalAddressedTiles: targets.length, + inventory: { + candidateCount: owner.candidates.length, + scannedTiles, + scannedDirectories + } +}; +const output = path.resolve(options.output); +await fs.mkdir(path.dirname(output), { recursive: true }); +await fs.writeFile(output, `${JSON.stringify(document, null, 2)}\n`, { flag: 'wx' }); +console.log( + `Generated ${targets.length} exact targets for ${owner.id} from ` + + `${scannedTiles} addressed input tiles.` +); diff --git a/scripts/offline-tileset/generate-production-batch-targets.mjs b/scripts/offline-tileset/generate-production-batch-targets.mjs new file mode 100644 index 00000000..51edf71e --- /dev/null +++ b/scripts/offline-tileset/generate-production-batch-targets.mjs @@ -0,0 +1,201 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { tileIdToZxy, zxyToTileId } from 'pmtiles'; +import { openPmtiles, visitPmtilesTileAddresses } from './local-pmtiles.mjs'; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`Invalid argument near ${key || ''}.`); + } + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'batch-plan', 'batch-index', 'output']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +function prefixContains(prefix, tile) { + if (tile.z < prefix.z) return false; + const divisor = 2 ** (tile.z - prefix.z); + return ( + Math.floor(tile.x / divisor) === prefix.x && + Math.floor(tile.y / divisor) === prefix.y + ); +} + +function tileKey(tile) { + return `${tile.z}/${tile.x}/${tile.y}`; +} + +const options = parseArguments(process.argv.slice(2)); +const [plan, batchPlan] = await Promise.all([ + fs.readFile(path.resolve(options.plan), 'utf8').then(JSON.parse), + fs.readFile(path.resolve(options['batch-plan']), 'utf8').then(JSON.parse) +]); +if (plan.planVersion !== batchPlan.planVersion) { + throw new Error('Production batch plan does not match the immutable owner plan.'); +} +const batchIndex = Number(options['batch-index']); +const batch = batchPlan.batches?.find((candidate) => candidate.index === batchIndex); +if (!batch) throw new Error(`Production batch is missing: ${batchIndex}`); + +const ownerIds = new Set(batch.ownerIds); +const owners = plan.owners.filter((owner) => ownerIds.has(owner.id)); +if (owners.length !== ownerIds.size) { + throw new Error(`Batch ${batch.id} owner selection is incomplete.`); +} +const candidateOwners = new Map(); +for (const owner of owners) { + for (const candidate of owner.candidates) { + const existing = candidateOwners.get(candidate.asset) || { candidate, owners: [] }; + existing.owners.push(owner); + candidateOwners.set(candidate.asset, existing); + } +} + +const tileOwners = new Map(); +const ownerTileCounts = new Map(owners.map((owner) => [owner.id, 0])); +function assign(owner, tile) { + const id = zxyToTileId(tile.z, tile.x, tile.y); + const existing = tileOwners.get(id); + if (existing && existing !== owner.id) { + throw new Error( + `Exact production tile ${tileKey(tile)} belongs to both ${existing} and ${owner.id}.` + ); + } + if (!existing) { + tileOwners.set(id, owner.id); + ownerTileCounts.set(owner.id, ownerTileCounts.get(owner.id) + 1); + } +} + +for (const owner of owners) { + for (const tile of owner.exactTiles || []) assign(owner, tile); +} + +let scannedTiles = 0; +let scannedDirectories = 0; +const scannedInputs = []; + +async function scanInput({ label, location, candidateOwnerList }) { + const opened = await openPmtiles(location, { cacheEntries: 8_192 }); + try { + const scanned = await visitPmtilesTileAddresses(opened, ({ z, x, y }) => { + if (z <= plan.routingZoom || z > 16) return; + const tile = { z, x, y }; + const matches = candidateOwnerList.filter((owner) => prefixContains(owner.prefix, tile)); + if (matches.length > 1) { + throw new Error( + `${label} maps ${tileKey(tile)} to ${matches.length} owners in ${batch.id}.` + ); + } + if (matches.length === 1) assign(matches[0], tile); + }); + scannedTiles += scanned.addressedTiles; + scannedDirectories += scanned.directoryCount; + scannedInputs.push({ + label, + addressedTiles: scanned.addressedTiles, + directoryCount: scanned.directoryCount + }); + } finally { + await opened.close(); + } +} + +// Preserve authoritative physical coverage even where a regional bounding box +// intersects an owner but the regional archive has no addressed tile there. +await scanInput({ + label: 'world-surface', + location: plan.inputs.surface.url, + candidateOwnerList: owners +}); +await scanInput({ + label: 'world-overview', + location: plan.inputs.overview.url, + candidateOwnerList: owners +}); + +for (const { candidate, owners: candidateOwnerList } of [...candidateOwners.values()] + .sort((left, right) => left.candidate.asset.localeCompare(right.candidate.asset))) { + await scanInput({ + label: candidate.asset, + location: candidate.url, + candidateOwnerList + }); +} + +// A wholly empty batch can occur at a regional bounding-box fringe over open +// ocean. Emit one deterministic empty MVT address so the immutable batch has a +// valid PMTiles container; no geometry is synthesized. +let emptyBatchAnchor = null; +if (!tileOwners.size) { + const owner = owners[0]; + const z = Math.max(plan.routingZoom + 1, owner.prefix.z); + const scale = 2 ** (z - owner.prefix.z); + emptyBatchAnchor = { + z, + x: owner.prefix.x * scale, + y: owner.prefix.y * scale + }; + assign(owner, emptyBatchAnchor); +} + +const activeOwners = owners.filter((owner) => ownerTileCounts.get(owner.id) > 0); +const emptyOwnerIds = owners + .filter((owner) => ownerTileCounts.get(owner.id) === 0) + .map((owner) => owner.id); +const targets = [...tileOwners.keys()] + .sort((left, right) => left - right) + .map((tileId) => { + const [z, x, y] = tileIdToZxy(tileId); + return { z, x, y }; + }); +const document = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + planVersion: plan.planVersion, + batchPlanVersion: batchPlan.batchPlanVersion, + batch: { + ...batch, + activeOwnerIds: activeOwners.map((owner) => owner.id), + emptyOwnerIds, + emptyBatchAnchor, + ownerTileCounts: Object.fromEntries([...ownerTileCounts].sort()) + }, + selectedOwners: activeOwners.map((owner) => ({ + id: owner.id, + prefix: owner.prefix, + exactTiles: owner.exactTiles || [], + requiredCandidates: owner.candidates + })), + targets: { + [batch.id]: targets + }, + totalAddressedTiles: targets.length, + inventory: { + ownerCount: owners.length, + activeOwnerCount: activeOwners.length, + emptyOwnerCount: emptyOwnerIds.length, + candidateCount: candidateOwners.size, + scannedTiles, + scannedDirectories, + scannedInputs + } +}; +const output = path.resolve(options.output); +await fs.mkdir(path.dirname(output), { recursive: true }); +await fs.writeFile(output, `${JSON.stringify(document, null, 2)}\n`, { flag: 'wx' }); +console.log( + `Generated ${targets.length} exact tiles for ${batch.id} across ` + + `${activeOwners.length} active and ${emptyOwnerIds.length} empty source owners ` + + `from ${candidateOwners.size} regional inputs plus authoritative surface/overview.` +); diff --git a/scripts/offline-tileset/generate-representative-targets.mjs b/scripts/offline-tileset/generate-representative-targets.mjs new file mode 100644 index 00000000..276ceb61 --- /dev/null +++ b/scripts/offline-tileset/generate-representative-targets.mjs @@ -0,0 +1,243 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { zxyToTileId } from 'pmtiles'; + +const VIEWPORT = { width: 1440, height: 1000 }; +const TILE_SIZE = 512; +const OVERSCAN_TILES = 3; +const FOUNDATION_WORLD_MAX_ZOOM = 6; +const MOTION_FRAMES = 10; +const BUILD_MOTION_SAMPLES = 120; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) throw new Error(`Unexpected argument: ${key}`); + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${key}.`); + result[key.slice(2)] = value; + index += 1; + } + if (!result.plan || !result.output) throw new Error('Required: --plan and --output.'); + return result; +} + +function lonToTileX(lon, zoom) { + return ((lon + 180) / 360) * 2 ** zoom; +} + +function latToTileY(lat, zoom) { + const clipped = Math.min(85.05112878, Math.max(-85.05112878, lat)); + const radians = clipped * Math.PI / 180; + return ((1 - Math.asinh(Math.tan(radians)) / Math.PI) / 2) * 2 ** zoom; +} + +function tileKey(tile) { + return `${tile.z}/${tile.x}/${tile.y}`; +} + +function cameraTiles({ center, zoom }) { + const z = Math.max(0, Math.min(16, Math.floor(zoom))); + const width = 2 ** z; + const centerX = Math.floor(lonToTileX(center[0], z)); + const centerY = Math.floor(latToTileY(center[1], z)); + const radiusX = Math.ceil(VIEWPORT.width / TILE_SIZE / 2) + OVERSCAN_TILES; + const radiusY = Math.ceil(VIEWPORT.height / TILE_SIZE / 2) + OVERSCAN_TILES; + const tiles = []; + for (let deltaY = -radiusY; deltaY <= radiusY; deltaY += 1) { + const y = centerY + deltaY; + if (y < 0 || y >= width) continue; + for (let deltaX = -radiusX; deltaX <= radiusX; deltaX += 1) { + const x = ((centerX + deltaX) % width + width) % width; + tiles.push({ z, x, y }); + } + } + return tiles; +} + +function ease(value) { + return value < 0.5 + ? 4 * value ** 3 + : 1 - ((-2 * value + 2) ** 3) / 2; +} + +function interpolateLongitude(start, end, amount) { + let delta = end - start; + if (delta > 180) delta -= 360; + if (delta < -180) delta += 360; + let value = start + delta * amount; + while (value > 180) value -= 360; + while (value < -180) value += 360; + return value; +} + +function motionCameras(start, end, count) { + return Array.from({ length: count }, (_, index) => { + const amount = count === 1 ? 1 : ease(index / (count - 1)); + return { + center: [ + interpolateLongitude(start.center[0], end.center[0], amount), + start.center[1] + (end.center[1] - start.center[1]) * amount + ], + zoom: start.zoom + (end.zoom - start.zoom) * amount + }; + }); +} + +function prefixContains(prefix, tile) { + if (prefix.z > tile.z) return false; + const divisor = 2 ** (tile.z - prefix.z); + return ( + Math.floor(tile.x / divisor) === prefix.x && + Math.floor(tile.y / divisor) === prefix.y + ); +} + +function tileBounds(tile) { + const width = 2 ** tile.z; + const west = (tile.x / width) * 360 - 180; + const east = ((tile.x + 1) / width) * 360 - 180; + const north = Math.atan(Math.sinh(Math.PI * (1 - (2 * tile.y) / width))) * 180 / Math.PI; + const south = Math.atan(Math.sinh(Math.PI * (1 - (2 * (tile.y + 1)) / width))) * 180 / Math.PI; + return [west, south, east, north]; +} + +function splitAntimeridianBounds(bounds) { + const [west, south, east, north] = bounds; + return west <= east + ? [[west, south, east, north]] + : [[west, south, 180, north], [-180, south, east, north]]; +} + +function boundsIntersect(left, right) { + return !( + left[2] <= right[0] || + left[0] >= right[2] || + left[3] <= right[1] || + left[1] >= right[3] + ); +} + +function candidateIntersectsTile(candidate, tile) { + const target = tileBounds(tile); + return splitAntimeridianBounds(candidate.bounds).some((part) => + boundsIntersect(part, target) + ); +} + +function regionalOwnerForTile(plan, tile) { + const matches = plan.owners.filter((owner) => prefixContains(owner.prefix, tile)); + if (matches.length > 1) { + throw new Error(`Expected at most one regional owner for tile ${tileKey(tile)}; found ${matches.length}.`); + } + return matches[0] || null; +} + +function sortedTiles(map) { + return [...map.values()].sort((left, right) => + zxyToTileId(left.z, left.x, left.y) - zxyToTileId(right.z, right.x, right.y) + ); +} + +const options = parseArguments(process.argv.slice(2)); +const plan = JSON.parse(await fs.readFile(path.resolve(options.plan), 'utf8')); +const global = { center: [-20, 18], zoom: 2.2 }; +const fresnoStreet = { center: [-119.7871, 36.7378], zoom: 16 }; +const fresnoGlobal = { center: fresnoStreet.center, zoom: 3.5 }; +const antimeridianEast = { center: [179.65, 8], zoom: 8 }; +const antimeridianWest = { center: [-179.65, 8], zoom: 8 }; +const staticViews = [ + { name: 'global', ...global }, + { name: 'north-america', center: [-100, 40], zoom: 4 }, + { name: 'south-america', center: [-60, -15], zoom: 4 }, + { name: 'europe', center: [12, 50], zoom: 5 }, + { name: 'pacific', center: [155, 0], zoom: 5 }, + { name: 'antimeridian', center: [179.5, 8], zoom: 6 }, + { name: 'fresno-regional', center: fresnoStreet.center, zoom: 8 }, + { name: 'fresno-city', center: fresnoStreet.center, zoom: 12 }, + { name: 'fresno-street', ...fresnoStreet } +]; +const motions = [ + { name: 'global-to-street', start: fresnoGlobal, end: fresnoStreet }, + { name: 'street-to-global', start: fresnoStreet, end: fresnoGlobal }, + { name: 'antimeridian-crossing', start: antimeridianEast, end: antimeridianWest } +]; +const validationFrames = motions.flatMap((motion) => + motionCameras(motion.start, motion.end, MOTION_FRAMES) + .map((camera, index) => ({ motion: motion.name, index, ...camera })) +); +const buildCameras = [ + ...staticViews, + ...validationFrames, + ...motions.flatMap((motion) => + motionCameras(motion.start, motion.end, BUILD_MOTION_SAMPLES) + ) +]; + +const ownerById = new Map(); +for (const camera of buildCameras) { + for (const tile of cameraTiles(camera)) { + if (tile.z <= FOUNDATION_WORLD_MAX_ZOOM) continue; + const owner = regionalOwnerForTile(plan, tile); + if (owner) ownerById.set(owner.id, owner); + } +} +if (!ownerById.size) throw new Error('Representative target did not touch any high-zoom regional owners.'); + +const targetMaps = Object.fromEntries([ + ['foundation', new Map()], + ...[...ownerById].map(([id]) => [id, new Map()]) +]); + +for (let z = 0; z <= FOUNDATION_WORLD_MAX_ZOOM; z += 1) { + const width = 2 ** z; + for (let y = 0; y < width; y += 1) { + for (let x = 0; x < width; x += 1) { + const tile = { z, x, y }; + targetMaps.foundation.set(tileKey(tile), tile); + } + } +} +for (const camera of buildCameras) { + for (const tile of cameraTiles(camera)) { + if (tile.z <= FOUNDATION_WORLD_MAX_ZOOM) { + targetMaps.foundation.set(tileKey(tile), tile); + continue; + } + const owner = regionalOwnerForTile(plan, tile); + const target = owner ? targetMaps[owner.id] : targetMaps.foundation; + target.set(tileKey(tile), tile); + } +} + +const document = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + viewport: VIEWPORT, + staticViews, + validationFrames, + selectedOwners: [...ownerById.values()].map((owner) => ({ + ...owner, + requiredCandidates: owner.candidates.filter((candidate) => + [...targetMaps[owner.id].values()].some((tile) => + candidateIntersectsTile(candidate, tile) + ) + ) + })), + targets: Object.fromEntries( + Object.entries(targetMaps).map(([name, tiles]) => [name, sortedTiles(tiles)]) + ) +}; +document.totalAddressedTiles = Object.values(document.targets) + .reduce((sum, targets) => sum + targets.length, 0); + +const outputPath = path.resolve(options.output); +await fs.mkdir(path.dirname(outputPath), { recursive: true }); +await fs.writeFile(outputPath, `${JSON.stringify(document, null, 2)}\n`); +console.log( + `Generated ${document.totalAddressedTiles} exact representative tile targets across ` + + `${Object.keys(document.targets).length} non-overlapping owners.` +); diff --git a/scripts/offline-tileset/local-pmtiles.mjs b/scripts/offline-tileset/local-pmtiles.mjs new file mode 100644 index 00000000..06e0aad1 --- /dev/null +++ b/scripts/offline-tileset/local-pmtiles.mjs @@ -0,0 +1,144 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { FetchSource, PMTiles, SharedPromiseCache, tileIdToZxy } from 'pmtiles'; + +/** + * Exact-range local source for the PMTiles JavaScript reader. + * + * Node Buffers may expose a larger backing ArrayBuffer than the requested + * range. PMTiles requires the returned ArrayBuffer to contain exactly the + * requested bytes, so every read is sliced to its own byteOffset/byteLength. + */ +export class LocalPmtilesSource { + constructor(filename) { + this.filename = path.resolve(filename); + this.handlePromise = fs.open(this.filename, 'r'); + this.sizePromise = this.handlePromise.then((handle) => handle.stat()).then((stat) => stat.size); + this.closed = false; + } + + getKey() { + return this.filename; + } + + async getBytes(offset, length) { + if (this.closed) throw new Error(`PMTiles source is closed: ${this.filename}`); + const start = Number(offset); + const size = Number(length); + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(size) || + start < 0 || + size <= 0 + ) { + throw new RangeError(`Invalid PMTiles range ${offset}+${length}.`); + } + + const [handle, fileSize] = await Promise.all([this.handlePromise, this.sizePromise]); + if (start >= fileSize) { + throw new RangeError(`PMTiles range starts beyond EOF: ${start} >= ${fileSize}.`); + } + const actualSize = Math.min(size, fileSize - start); + const buffer = Buffer.allocUnsafe(actualSize); + const { bytesRead } = await handle.read(buffer, 0, actualSize, start); + if (bytesRead !== actualSize) { + throw new Error( + `Short PMTiles read from ${this.filename}: expected ${actualSize}, received ${bytesRead}.` + ); + } + + return { + data: buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) + }; + } + + async close() { + if (this.closed) return; + this.closed = true; + await (await this.handlePromise).close(); + } +} + +function isHttpLocation(value) { + try { + const url = new URL(value); + return url.protocol === 'https:' || url.protocol === 'http:'; + } catch { + return false; + } +} + +export async function openPmtiles(location, { cacheEntries = 256 } = {}) { + const normalized = isHttpLocation(location) ? location : path.resolve(location); + const source = isHttpLocation(normalized) + ? new FetchSource(normalized) + : new LocalPmtilesSource(normalized); + const archive = new PMTiles(source, new SharedPromiseCache(cacheEntries)); + const header = await archive.getHeader(); + return { + archive, + header, + source, + location: normalized, + async close() { + await source.close?.(); + } + }; +} + +export async function openLocalPmtiles(filename, options = {}) { + return openPmtiles(path.resolve(filename), options); +} + +export function tilePayload(result) { + if (!result?.data) return null; + return Buffer.from( + result.data, + result.data.byteOffset || 0, + result.data.byteLength + ); +} + +/** + * Visit every addressed z/x/y in a PMTiles archive without reading tile + * payloads. Run-length entries are expanded because every exact address must + * be assigned to one immutable output owner. + */ +export async function visitPmtilesTileAddresses(opened, visit) { + const { archive, header, source } = opened; + const visitedDirectories = new Set(); + let addressedTiles = 0; + + async function walk(offset, length, depth) { + if (depth > 4) throw new Error('PMTiles directory depth exceeds the v3 safety limit.'); + const key = `${offset}:${length}`; + if (visitedDirectories.has(key)) return; + visitedDirectories.add(key); + const entries = await archive.cache.getDirectory(source, offset, length, header); + for (const entry of entries) { + if (entry.runLength > 0) { + for (let delta = 0; delta < entry.runLength; delta += 1) { + const tileId = entry.tileId + delta; + const [z, x, y] = tileIdToZxy(tileId); + await visit({ z, x, y, tileId }); + addressedTiles += 1; + } + } else { + await walk( + header.leafDirectoryOffset + entry.offset, + entry.length, + depth + 1 + ); + } + } + } + + await walk(header.rootDirectoryOffset, header.rootDirectoryLength, 0); + return { + addressedTiles, + directoryCount: visitedDirectories.size + }; +} diff --git a/scripts/offline-tileset/localize-representative-inputs.mjs b/scripts/offline-tileset/localize-representative-inputs.mjs new file mode 100644 index 00000000..dabfbb26 --- /dev/null +++ b/scripts/offline-tileset/localize-representative-inputs.mjs @@ -0,0 +1,172 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) throw new Error(`Unexpected argument: ${key}`); + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${key}.`); + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'output-dir', 'report']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + if (!result.targets && !result['owner-id']) { + throw new Error('Either --targets or --owner-id is required.'); + } + return result; +} + +async function hashFile(filename) { + const hash = createHash('sha256'); + const handle = await fs.open(filename, 'r'); + try { + for await (const chunk of handle.createReadStream()) hash.update(chunk); + } finally { + await handle.close().catch(() => {}); + } + return hash.digest('hex'); +} + +function releaseUrl(plan, asset) { + return `https://github.com/${plan.repository}/releases/download/` + + `${encodeURIComponent(plan.releaseTag)}/${encodeURIComponent(asset)}`; +} + +async function validLockedFile(filename, lock) { + try { + const stat = await fs.stat(filename); + return ( + stat.isFile() && + stat.size === lock.bytes && + await hashFile(filename) === lock.sha256 + ); + } catch { + return false; + } +} + +async function downloadLocked(plan, lock, outputDir) { + const destination = path.join(outputDir, lock.asset); + if (await validLockedFile(destination, lock)) { + console.log(`Using SHA-locked input ${lock.asset}.`); + return { + asset: lock.asset, + path: destination, + bytes: lock.bytes, + sha256: lock.sha256, + reused: true + }; + } + + const temporary = `${destination}.pending-${process.pid}-${Date.now()}`; + const response = await fetch(lock.url || releaseUrl(plan, lock.asset), { + redirect: 'follow', + headers: { 'User-Agent': 'Occu-Med-Map/offline-input-localizer' } + }); + if (!response.ok || !response.body) { + throw new Error(`Unable to download ${lock.asset}: HTTP ${response.status}.`); + } + const declared = Number(response.headers.get('content-length')); + if (Number.isFinite(declared) && declared !== lock.bytes) { + throw new Error( + `Published byte count changed for ${lock.asset}: expected ${lock.bytes}, received ${declared}.` + ); + } + + const handle = await fs.open(temporary, 'wx'); + const hash = createHash('sha256'); + let bytes = 0; + let nextProgress = 256 * 1024 * 1024; + try { + for await (const chunk of response.body) { + const buffer = Buffer.from(chunk); + await handle.write(buffer); + hash.update(buffer); + bytes += buffer.byteLength; + if (bytes >= nextProgress) { + console.log(`[${lock.asset}] ${bytes}/${lock.bytes} bytes downloaded.`); + nextProgress += 256 * 1024 * 1024; + } + } + await handle.sync(); + } finally { + await handle.close(); + } + const digest = hash.digest('hex'); + if (bytes !== lock.bytes || digest !== lock.sha256) { + throw new Error( + `Input lock failed for ${lock.asset}: ${bytes}/${lock.bytes} bytes, ${digest}/${lock.sha256}.` + ); + } + await fs.rename(temporary, destination); + const verified = await validLockedFile(destination, lock); + if (!verified) throw new Error(`Promoted input failed verification: ${lock.asset}.`); + console.log(`Localized ${lock.asset}: ${bytes} bytes, SHA-256 ${digest}.`); + return { + asset: lock.asset, + path: destination, + bytes, + sha256: digest, + reused: false + }; +} + +const options = parseArguments(process.argv.slice(2)); +const plan = JSON.parse(await fs.readFile(path.resolve(options.plan), 'utf8')); +const targets = options.targets + ? JSON.parse(await fs.readFile(path.resolve(options.targets), 'utf8')) + : null; +const selectedIds = new Set( + options['owner-id'] + ? [options['owner-id']] + : (targets.selectedOwners || []).map((owner) => owner.id) +); +const selected = plan.owners.filter((owner) => selectedIds.has(owner.id)); +if (selected.length !== selectedIds.size) throw new Error('Representative owner selection is not in the locked plan.'); + +const locks = new Map(); +for (const input of [plan.inputs.overview, plan.inputs.surface]) locks.set(input.asset, input); +for (const selectedOwner of selected) { + const owner = selected.find((candidate) => candidate.id === selectedOwner.id); + if (!owner) throw new Error(`Selected owner is missing from plan: ${selectedOwner.id}`); + const requiredNames = new Set( + ( + targets?.selectedOwners?.find((candidate) => candidate.id === selectedOwner.id) + ?.requiredCandidates || + owner.candidates + ).map((candidate) => candidate.asset) + ); + for (const candidate of owner.candidates.filter((item) => requiredNames.has(item.asset))) { + locks.set(candidate.asset, { + ...candidate, + url: releaseUrl(plan, candidate.asset) + }); + } +} + +const outputDir = path.resolve(options['output-dir']); +await fs.mkdir(outputDir, { recursive: true }); +const results = []; +for (const lock of [...locks.values()].sort((left, right) => left.asset.localeCompare(right.asset))) { + results.push(await downloadLocked(plan, lock, outputDir)); +} +const report = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + planVersion: plan.planVersion, + ownerIds: [...selectedIds].sort(), + inputCount: results.length, + totalBytes: results.reduce((sum, result) => sum + result.bytes, 0), + inputs: results +}; +const reportPath = path.resolve(options.report); +await fs.mkdir(path.dirname(reportPath), { recursive: true }); +await fs.writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`); +console.log(`Localized ${results.length} locked inputs totaling ${report.totalBytes} bytes.`); diff --git a/scripts/offline-tileset/mvt-normalizer.mjs b/scripts/offline-tileset/mvt-normalizer.mjs new file mode 100644 index 00000000..82a85b73 --- /dev/null +++ b/scripts/offline-tileset/mvt-normalizer.mjs @@ -0,0 +1,765 @@ +import { createHash } from 'node:crypto'; +import { VectorTile } from '@mapbox/vector-tile'; +import Pbf from 'pbf'; +import vtpbf from 'vt-pbf'; +import { validateVectorTilePayload } from '../../src/server/tile-safety.js'; + +const DEFAULT_EXTENT = 4096; +const INPUT_COORDINATE_SCALE = 128; +const MAX_FEATURE_POINTS = 250_000; +const MAX_TILE_POINTS = 2_000_000; +const SURFACE_RECTANGLE_LAYERS = new Set([ + 'landcover', + 'landuse', + 'park', + 'protected_area', + 'water' +]); + +const LAYER_ALIASES = new Map([ + ['aerodrome', 'aerodrome_label'], + ['aerodrome_label', 'aerodrome_label'], + ['aeroway', 'aeroway'], + ['admin', 'boundary'], + ['boundary', 'boundary'], + ['building', 'building'], + ['buildings', 'building'], + ['depth', 'depth'], + ['housenum_label', 'housenumber'], + ['housenumber', 'housenumber'], + ['land', 'land'], + ['land_cover', 'landcover'], + ['landcover', 'landcover'], + ['landuse', 'landuse'], + ['landuse_overlay', 'landuse'], + ['mountain_peak', 'mountain_peak'], + ['park', 'park'], + ['place', 'place'], + ['place_label', 'place'], + ['poi', 'poi'], + ['poi_label', 'poi'], + ['protected_area', 'park'], + ['road', 'transportation'], + ['road_label', 'transportation_name'], + ['transportation', 'transportation'], + ['transportation_name', 'transportation_name'], + ['water', 'water'], + ['water_name', 'water_name'], + ['waterway', 'waterway'] +]); + +function canonicalLayerName(name) { + const normalized = String(name || '').trim().toLowerCase().replaceAll('-', '_'); + return LAYER_ALIASES.get(normalized) || normalized; +} + +function normalizePropertyValue(value) { + if (typeof value === 'bigint') return value.toString(); + if ( + typeof value === 'number' && + (!Number.isFinite(value) || (Number.isInteger(value) && !Number.isSafeInteger(value))) + ) { + return String(value); + } + return value; +} + +function normalizeProperties(layerName, properties = {}) { + const output = Object.fromEntries( + Object.entries(properties) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => [key, normalizePropertyValue(value)]) + ); + if (output['name_en'] && !output['name:en']) output['name:en'] = output['name_en']; + if (output.name && !output['name:latin']) output['name:latin'] = output.name; + if (layerName === 'landcover') { + if (output.class === 'farmland') output.class = 'crop'; + if (output.class === 'ice') output.class = 'snow'; + output.class ||= 'grass'; + } + if ( + ['transportation', 'transportation_name'].includes(layerName) && + !output.class && + output.type + ) { + output.class = output.type; + } + if (layerName === 'building') { + if (output.height !== undefined && output.render_height === undefined) { + output.render_height = output.height; + } + if (output.min_height !== undefined && output.render_min_height === undefined) { + output.render_min_height = output.min_height; + } + } + return output; +} + +function pointEqual(left, right) { + return left.x === right.x && left.y === right.y; +} + +function roundPoint(point) { + return { x: Math.round(point.x), y: Math.round(point.y) }; +} + +function dedupePoints(points, { close = false } = {}) { + const result = []; + for (const raw of points) { + const point = roundPoint(raw); + if (!result.length || !pointEqual(result.at(-1), point)) result.push(point); + } + while (result.length > 1 && pointEqual(result[0], result.at(-1))) result.pop(); + if (close && result.length >= 3) result.push({ ...result[0] }); + return result; +} + +function interpolateAtX(start, end, x) { + const delta = end.x - start.x; + if (delta === 0) return { x, y: start.y }; + const ratio = (x - start.x) / delta; + return { x, y: start.y + (end.y - start.y) * ratio }; +} + +function interpolateAtY(start, end, y) { + const delta = end.y - start.y; + if (delta === 0) return { x: start.x, y }; + const ratio = (y - start.y) / delta; + return { x: start.x + (end.x - start.x) * ratio, y }; +} + +function clipPolygonBoundary(points, inside, intersect) { + if (!points.length) return []; + const result = []; + let previous = points.at(-1); + let previousInside = inside(previous); + for (const current of points) { + const currentInside = inside(current); + if (currentInside) { + if (!previousInside) result.push(intersect(previous, current)); + result.push(current); + } else if (previousInside) { + result.push(intersect(previous, current)); + } + previous = current; + previousInside = currentInside; + } + return result; +} + +function clipPolygonRing(points, extent) { + if (points.length < 3) return []; + let ring = pointEqual(points[0], points.at(-1)) ? points.slice(0, -1) : [...points]; + ring = clipPolygonBoundary(ring, (point) => point.x >= 0, (a, b) => interpolateAtX(a, b, 0)); + ring = clipPolygonBoundary(ring, (point) => point.x <= extent, (a, b) => interpolateAtX(a, b, extent)); + ring = clipPolygonBoundary(ring, (point) => point.y >= 0, (a, b) => interpolateAtY(a, b, 0)); + ring = clipPolygonBoundary(ring, (point) => point.y <= extent, (a, b) => interpolateAtY(a, b, extent)); + ring = dedupePoints(ring, { close: true }); + return ring.length >= 4 ? ring : []; +} + +function regionCode(point, extent) { + let code = 0; + if (point.x < 0) code |= 1; + else if (point.x > extent) code |= 2; + if (point.y < 0) code |= 4; + else if (point.y > extent) code |= 8; + return code; +} + +function clipLineSegment(start, end, extent) { + let left = { ...start }; + let right = { ...end }; + let leftCode = regionCode(left, extent); + let rightCode = regionCode(right, extent); + while (true) { + if (!(leftCode | rightCode)) return [left, right]; + if (leftCode & rightCode) return null; + const code = leftCode || rightCode; + let point; + if (code & 8) point = interpolateAtY(left, right, extent); + else if (code & 4) point = interpolateAtY(left, right, 0); + else if (code & 2) point = interpolateAtX(left, right, extent); + else point = interpolateAtX(left, right, 0); + if (code === leftCode) { + left = point; + leftCode = regionCode(left, extent); + } else { + right = point; + rightCode = regionCode(right, extent); + } + } +} + +function clipLineString(points, extent) { + const fragments = []; + let current = []; + for (let index = 1; index < points.length; index += 1) { + const clipped = clipLineSegment(points[index - 1], points[index], extent); + if (!clipped) { + if (current.length >= 2) fragments.push(dedupePoints(current)); + current = []; + continue; + } + const [start, end] = clipped; + if (!current.length) current.push(start, end); + else if (pointEqual(roundPoint(current.at(-1)), roundPoint(start))) current.push(end); + else { + if (current.length >= 2) fragments.push(dedupePoints(current)); + current = [start, end]; + } + } + if (current.length >= 2) fragments.push(dedupePoints(current)); + return fragments.filter((fragment) => fragment.length >= 2); +} + +function transformAndClipGeometry(feature, { + scale, + offsetX, + offsetY, + extent +}) { + const transformed = []; + let pointCount = 0; + for (const part of feature.loadGeometry()) { + const points = []; + for (const point of part) { + if ( + !Number.isFinite(point.x) || + !Number.isFinite(point.y) || + Math.abs(point.x) > feature.extent * INPUT_COORDINATE_SCALE || + Math.abs(point.y) > feature.extent * INPUT_COORDINATE_SCALE + ) { + return { geometry: [], pointCount, malformed: true }; + } + points.push({ + x: point.x * scale - offsetX, + y: point.y * scale - offsetY + }); + pointCount += 1; + if (pointCount > MAX_FEATURE_POINTS) { + return { geometry: [], pointCount, oversized: true }; + } + } + if (feature.type === 1) { + transformed.push( + ...points + .filter((point) => point.x >= 0 && point.x <= extent && point.y >= 0 && point.y <= extent) + .map((point) => [roundPoint(point)]) + ); + } else if (feature.type === 2) { + transformed.push(...clipLineString(points, extent)); + } else if (feature.type === 3) { + const ring = clipPolygonRing(points, extent); + if (ring.length) transformed.push(ring); + } + } + return { geometry: transformed, pointCount }; +} + +function geometryBounds(geometry) { + let minX = Infinity; + let minY = Infinity; + let maxX = -Infinity; + let maxY = -Infinity; + let points = 0; + for (const part of geometry) { + for (const point of part) { + minX = Math.min(minX, point.x); + minY = Math.min(minY, point.y); + maxX = Math.max(maxX, point.x); + maxY = Math.max(maxY, point.y); + points += 1; + } + } + return points ? { minX, minY, maxX, maxY, points } : null; +} + +function isLargeAxisAlignedRectangle(feature, extent) { + if (feature.type !== 3 || !SURFACE_RECTANGLE_LAYERS.has(feature.layerName)) return false; + const bounds = feature.bounds; + if (!bounds) return false; + const width = bounds.maxX - bounds.minX; + const height = bounds.maxY - bounds.minY; + if (width <= 0 || height <= 0 || (width * height) / (extent * extent) < 0.06) { + return false; + } + + const points = feature.geometry.flat(); + if (points.length < 4) return false; + const epsilon = 1; + const onBoundary = (point) => + Math.abs(point.x - bounds.minX) <= epsilon || + Math.abs(point.x - bounds.maxX) <= epsilon || + Math.abs(point.y - bounds.minY) <= epsilon || + Math.abs(point.y - bounds.maxY) <= epsilon; + if (!points.every(onBoundary)) return false; + for (let index = 1; index < points.length; index += 1) { + const previous = points[index - 1]; + const current = points[index]; + if ( + Math.abs(previous.x - current.x) > epsilon && + Math.abs(previous.y - current.y) > epsilon + ) { + return false; + } + } + return true; +} + +function stableProperties(properties) { + return JSON.stringify( + Object.entries(properties) + .sort(([left], [right]) => left.localeCompare(right)) + ); +} + +function geometrySignature(feature) { + const hash = createHash('sha1'); + hash.update(`${feature.type}|${stableProperties(feature.properties)}|`); + for (const part of feature.geometry) { + hash.update('['); + for (const point of part) hash.update(`${point.x},${point.y};`); + hash.update(']'); + } + return hash.digest('base64url'); +} + +function boxesIntersect(left, right) { + return !( + left.maxX < right.minX || + left.minX > right.maxX || + left.maxY < right.minY || + left.minY > right.maxY + ); +} + +function orientation(a, b, c) { + return Math.sign((b.y - a.y) * (c.x - b.x) - (b.x - a.x) * (c.y - b.y)); +} + +function pointOnSegment(a, b, point) { + return ( + point.x >= Math.min(a.x, b.x) && + point.x <= Math.max(a.x, b.x) && + point.y >= Math.min(a.y, b.y) && + point.y <= Math.max(a.y, b.y) && + orientation(a, b, point) === 0 + ); +} + +function segmentsIntersect(a, b, c, d) { + const o1 = orientation(a, b, c); + const o2 = orientation(a, b, d); + const o3 = orientation(c, d, a); + const o4 = orientation(c, d, b); + if (o1 !== o2 && o3 !== o4) return true; + return ( + (o1 === 0 && pointOnSegment(a, b, c)) || + (o2 === 0 && pointOnSegment(a, b, d)) || + (o3 === 0 && pointOnSegment(c, d, a)) || + (o4 === 0 && pointOnSegment(c, d, b)) + ); +} + +function pointInRing(point, ring) { + let inside = false; + for (let index = 0, previous = ring.length - 1; index < ring.length; previous = index++) { + const a = ring[index]; + const b = ring[previous]; + if (pointOnSegment(a, b, point)) return true; + const intersects = + (a.y > point.y) !== (b.y > point.y) && + point.x < ((b.x - a.x) * (point.y - a.y)) / (b.y - a.y || 1) + a.x; + if (intersects) inside = !inside; + } + return inside; +} + +function ringsCross(left, right) { + for (let leftIndex = 1; leftIndex < left.length; leftIndex += 1) { + for (let rightIndex = 1; rightIndex < right.length; rightIndex += 1) { + if ( + segmentsIntersect( + left[leftIndex - 1], + left[leftIndex], + right[rightIndex - 1], + right[rightIndex] + ) + ) { + return true; + } + } + } + return false; +} + +function polygonContains(container, candidate) { + if (!boxesIntersect(container.bounds, candidate.bounds)) return false; + if ( + container.bounds.minX > candidate.bounds.minX || + container.bounds.minY > candidate.bounds.minY || + container.bounds.maxX < candidate.bounds.maxX || + container.bounds.maxY < candidate.bounds.maxY + ) { + return false; + } + const containerRing = container.geometry[0]; + const candidateRing = candidate.geometry[0]; + if (!containerRing?.length || !candidateRing?.length) return false; + if (ringsCross(containerRing, candidateRing)) return false; + return candidateRing.slice(0, -1).every((point) => pointInRing(point, containerRing)); +} + +function bucketKeys(bounds, extent, bucketCount = 16) { + const size = extent / bucketCount; + const minX = Math.max(0, Math.min(bucketCount - 1, Math.floor(bounds.minX / size))); + const minY = Math.max(0, Math.min(bucketCount - 1, Math.floor(bounds.minY / size))); + const maxX = Math.max(0, Math.min(bucketCount - 1, Math.floor(bounds.maxX / size))); + const maxY = Math.max(0, Math.min(bucketCount - 1, Math.floor(bounds.maxY / size))); + const keys = []; + for (let x = minX; x <= maxX; x += 1) { + for (let y = minY; y <= maxY; y += 1) keys.push(`${x}/${y}`); + } + return keys; +} + +function dedupeAndRejectLayer(features, extent, report) { + const ordered = [...features].sort((left, right) => { + const properties = stableProperties(left.properties).localeCompare(stableProperties(right.properties)); + return properties || geometrySignature(left).localeCompare(geometrySignature(right)); + }); + const accepted = []; + const exact = new Set(); + const buckets = new Map(); + + for (const feature of ordered) { + if (isLargeAxisAlignedRectangle(feature, extent)) { + report.rejectedTileFootprints += 1; + continue; + } + const signature = geometrySignature(feature); + if (exact.has(signature)) { + report.removedExactDuplicates += 1; + continue; + } + + if (feature.type === 3) { + const properties = stableProperties(feature.properties); + const candidates = new Set(); + for (const key of bucketKeys(feature.bounds, extent)) { + for (const index of buckets.get(key) || []) candidates.add(index); + } + let contained = false; + for (const index of candidates) { + const existing = accepted[index]; + if (!existing || existing.removed || stableProperties(existing.properties) !== properties) continue; + if (polygonContains(existing, feature)) { + contained = true; + report.removedContainedOverlaps += 1; + break; + } + if (polygonContains(feature, existing)) { + existing.removed = true; + report.removedContainedOverlaps += 1; + } + } + if (contained) continue; + } + + exact.add(signature); + const index = accepted.length; + accepted.push(feature); + if (feature.type === 3) { + for (const key of bucketKeys(feature.bounds, extent)) { + const list = buckets.get(key) || []; + list.push(index); + buckets.set(key, list); + } + } + } + + const result = accepted.filter((feature) => !feature.removed); + if (result[0]?.layerName === 'depth') { + result.sort((left, right) => { + const leftDepth = Number(left.properties.min_depth || 0); + const rightDepth = Number(right.properties.min_depth || 0); + return leftDepth - rightDepth || geometrySignature(left).localeCompare(geometrySignature(right)); + }); + } + return result; +} + +class EncodedFeature { + constructor(feature) { + this.id = feature.id; + this.type = feature.type; + this.properties = feature.properties; + this.extent = feature.extent; + this.geometry = feature.geometry; + } + + loadGeometry() { + return this.geometry; + } +} + +class EncodedLayer { + constructor(name, features, extent = DEFAULT_EXTENT) { + this.name = name; + this.version = 2; + this.extent = extent; + this.features = features.map((feature) => new EncodedFeature(feature)); + this.length = this.features.length; + } + + feature(index) { + return this.features[index]; + } +} + +function decodeTile(payload, label) { + const bytes = validateVectorTilePayload(payload, { + label, + coordinateScale: INPUT_COORDINATE_SCALE, + maxBytes: 32 * 1024 * 1024 + }).bytes; + return new VectorTile(new Pbf(new Uint8Array(bytes))); +} + +/** + * Normalize selected source layers from one MVT into an exact target z/x/y. + */ +export function normalizeTileContribution(payload, { + sourceZoom, + targetZoom, + targetX, + targetY, + includeLayers, + excludeLayers, + authority, + report +}) { + if (!payload) return []; + const tile = decodeTile(payload, `offline ${authority} input`); + const include = includeLayers ? new Set(includeLayers.map(canonicalLayerName)) : null; + const exclude = excludeLayers ? new Set(excludeLayers.map(canonicalLayerName)) : null; + const scale = 2 ** (targetZoom - sourceZoom); + if (scale < 1) throw new Error('Offline materialization cannot downscale from a child tile.'); + const childX = ((targetX % scale) + scale) % scale; + const childY = ((targetY % scale) + scale) % scale; + const output = []; + let tilePoints = 0; + + for (const [inputName, layer] of Object.entries(tile.layers)) { + const layerName = canonicalLayerName(inputName); + if (include && !include.has(layerName)) continue; + if (exclude?.has(layerName)) continue; + for (let index = 0; index < layer.length; index += 1) { + const sourceFeature = layer.feature(index); + const normalized = transformAndClipGeometry(sourceFeature, { + scale, + offsetX: childX * layer.extent, + offsetY: childY * layer.extent, + extent: layer.extent + }); + if (normalized.malformed) { + report.rejectedMalformed += 1; + continue; + } + if (normalized.oversized) { + report.rejectedOversized += 1; + continue; + } + if (!normalized.geometry.length) { + report.clippedEmpty += 1; + continue; + } + tilePoints += normalized.geometry.reduce( + (sum, part) => sum + part.length, + 0 + ); + if (tilePoints > MAX_TILE_POINTS) { + throw new Error(`Offline tile exceeded ${MAX_TILE_POINTS} normalized geometry points.`); + } + const feature = { + id: sourceFeature.id, + type: sourceFeature.type, + properties: normalizeProperties(layerName, sourceFeature.properties), + extent: layer.extent, + geometry: normalized.geometry, + bounds: geometryBounds(normalized.geometry), + layerName, + authority + }; + output.push(feature); + } + } + return output; +} + +export function encodeNormalizedTile(contributions, { + label = 'offline normalized tile', + report = {} +} = {}) { + Object.assign(report, { + rejectedMalformed: report.rejectedMalformed || 0, + rejectedOversized: report.rejectedOversized || 0, + rejectedTileFootprints: report.rejectedTileFootprints || 0, + removedExactDuplicates: report.removedExactDuplicates || 0, + removedContainedOverlaps: report.removedContainedOverlaps || 0, + clippedEmpty: report.clippedEmpty || 0 + }); + const grouped = new Map(); + for (const feature of contributions) { + const list = grouped.get(feature.layerName) || []; + list.push(feature); + grouped.set(feature.layerName, list); + } + + const layers = {}; + for (const [layerName, features] of [...grouped.entries()].sort(([left], [right]) => left.localeCompare(right))) { + const extent = features[0]?.extent || DEFAULT_EXTENT; + const normalized = dedupeAndRejectLayer(features, extent, report); + if (normalized.length) layers[layerName] = new EncodedLayer(layerName, normalized, extent); + } + const encoded = Buffer.from(vtpbf.fromVectorTileJs({ layers })); + validateVectorTilePayload(encoded, { + label, + coordinateScale: 1, + maxBytes: 32 * 1024 * 1024, + maxTotalPoints: MAX_TILE_POINTS + }); + report.layers = Object.fromEntries( + Object.entries(layers).map(([name, layer]) => [ + name, + { + featureCount: layer.length, + pointCount: layer.features.reduce( + (sum, feature) => sum + feature.geometry.reduce((partSum, part) => partSum + part.length, 0), + 0 + ) + } + ]) + ); + report.bytes = encoded.byteLength; + return encoded; +} + +export async function readTileOrAncestor(openedArchive, z, x, y) { + const maxZoom = Number(openedArchive.header.maxZoom); + const sourceZoom = Math.min(z, maxZoom); + const divisor = 2 ** (z - sourceZoom); + const sourceX = Math.floor(x / divisor); + const sourceY = Math.floor(y / divisor); + const result = await openedArchive.archive.getZxy(sourceZoom, sourceX, sourceY); + return { + payload: result?.data ? Buffer.from(result.data) : null, + sourceZoom, + sourceX, + sourceY + }; +} + +/** + * Compile one final immutable tile with a strict offline authority matrix: + * surface -> land/depth, overview -> landcover, regional -> cartography. + */ +export async function compileImmutableTile({ + z, + x, + y, + surface, + overview, + regional = [] +}) { + const report = { + tile: { z, x, y }, + authorities: { + land: 'world-surface', + depth: 'world-surface', + landcover: 'world-overview', + cartography: 'regional-owner' + }, + policy: { + syntheticLandcover: false, + runtimeMerge: false, + runtimeGeometry: false, + parentChildStretching: false + }, + rejectedMalformed: 0, + rejectedOversized: 0, + rejectedTileFootprints: 0, + removedExactDuplicates: 0, + removedContainedOverlaps: 0, + clippedEmpty: 0 + }; + const [surfaceTile, overviewTile] = await Promise.all([ + readTileOrAncestor(surface, z, x, y), + readTileOrAncestor(overview, z, x, y) + ]); + const contributions = []; + if (surfaceTile.payload) { + contributions.push( + ...normalizeTileContribution(surfaceTile.payload, { + sourceZoom: surfaceTile.sourceZoom, + targetZoom: z, + targetX: x, + targetY: y, + includeLayers: ['land', 'depth'], + authority: 'world-surface', + report + }) + ); + } + if (overviewTile.payload) { + contributions.push( + ...normalizeTileContribution(overviewTile.payload, { + sourceZoom: overviewTile.sourceZoom, + targetZoom: z, + targetX: x, + targetY: y, + includeLayers: ['landcover'], + authority: 'world-overview', + report + }) + ); + if (z <= Number(overview.header.maxZoom)) { + contributions.push( + ...normalizeTileContribution(overviewTile.payload, { + sourceZoom: overviewTile.sourceZoom, + targetZoom: z, + targetX: x, + targetY: y, + excludeLayers: ['land', 'landcover', 'depth'], + authority: 'world-overview-cartography', + report + }) + ); + } + } + if (z > Number(overview.header.maxZoom)) { + for (const opened of regional) { + const result = await opened.archive.getZxy(z, x, y); + if (!result?.data) continue; + contributions.push( + ...normalizeTileContribution(Buffer.from(result.data), { + sourceZoom: z, + targetZoom: z, + targetX: x, + targetY: y, + excludeLayers: ['land', 'landcover', 'depth'], + authority: opened.assetName || opened.source?.filename || 'regional-cartography', + report + }) + ); + } + } + return { + data: encodeNormalizedTile(contributions, { + label: `immutable offline tile ${z}/${x}/${y}`, + report + }), + report + }; +} diff --git a/scripts/offline-tileset/plan-owners.mjs b/scripts/offline-tileset/plan-owners.mjs new file mode 100644 index 00000000..42744612 --- /dev/null +++ b/scripts/offline-tileset/plan-owners.mjs @@ -0,0 +1,246 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const DEFAULT_RELEASE = 'occumed-world-v1'; +const DEFAULT_ROUTING_ZOOM = 6; +const DEFAULT_MAX_PREFIX_ZOOM = 8; +const DEFAULT_MAX_CANDIDATE_BYTES = 2 * 1024 * 1024 * 1024; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) throw new Error(`Unexpected argument: ${key}`); + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${key}.`); + result[key.slice(2)] = value; + index += 1; + } + return result; +} + +function lonToTileX(lon, zoom) { + return Math.floor(((lon + 180) / 360) * 2 ** zoom); +} + +function latToTileY(lat, zoom) { + const clipped = Math.min(85.05112878, Math.max(-85.05112878, lat)); + const radians = clipped * Math.PI / 180; + return Math.floor( + ((1 - Math.asinh(Math.tan(radians)) / Math.PI) / 2) * 2 ** zoom + ); +} + +function tileBounds(z, x, y) { + const width = 2 ** z; + const west = (x / width) * 360 - 180; + const east = ((x + 1) / width) * 360 - 180; + const north = Math.atan(Math.sinh(Math.PI * (1 - (2 * y) / width))) * 180 / Math.PI; + const south = Math.atan(Math.sinh(Math.PI * (1 - (2 * (y + 1)) / width))) * 180 / Math.PI; + return [west, south, east, north]; +} + +function splitAntimeridianBounds(bounds) { + const [west, south, east, north] = bounds; + return west <= east + ? [[west, south, east, north]] + : [[west, south, 180, north], [-180, south, east, north]]; +} + +function boundsIntersect(left, right) { + return !( + left[2] <= right[0] || + left[0] >= right[2] || + left[3] <= right[1] || + left[1] >= right[3] + ); +} + +function regionIntersectsTile(region, tile) { + const bounds = tileBounds(tile.z, tile.x, tile.y); + return splitAntimeridianBounds(region.bounds).some((part) => boundsIntersect(part, bounds)); +} + +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries( + Object.keys(value) + .sort() + .filter((key) => key !== 'generatedAt') + .map((key) => [key, canonical(value[key])]) + ); +} + +function planHash(plan) { + return createHash('sha256') + .update(`${JSON.stringify(canonical(plan))}\n`) + .digest('hex'); +} + +async function fetchJson(url) { + const response = await fetch(url, { + redirect: 'follow', + headers: { 'User-Agent': 'Occu-Med-Map/offline-owner-plan' } + }); + if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}.`); + return response.json(); +} + +function splitOwner(prefix, candidates, options, output) { + const candidateBytes = candidates.reduce((sum, candidate) => sum + candidate.bytes, 0); + if ( + candidateBytes <= options.maxCandidateBytes || + prefix.z >= options.maxPrefixZoom + ) { + const owner = { + id: `z${prefix.z}-${prefix.x}-${prefix.y}`, + prefix, + exactTiles: [], + candidateBytes, + candidates: candidates + .map((candidate) => ({ + id: candidate.id, + asset: candidate.asset, + bounds: candidate.bounds, + bytes: candidate.bytes, + sha256: candidate.sha256, + url: candidate.url + })) + .sort((left, right) => left.asset.localeCompare(right.asset)) + }; + output.push(owner); + return owner; + } + + let deterministicAncestorOwner = null; + for (let deltaY = 0; deltaY < 2; deltaY += 1) { + for (let deltaX = 0; deltaX < 2; deltaX += 1) { + const child = { + z: prefix.z + 1, + x: prefix.x * 2 + deltaX, + y: prefix.y * 2 + deltaY + }; + const childCandidates = candidates.filter((candidate) => + regionIntersectsTile(candidate, child) + ); + if (childCandidates.length) { + const childOwner = splitOwner(child, childCandidates, options, output); + deterministicAncestorOwner ||= childOwner; + } + } + } + if (!deterministicAncestorOwner) { + throw new Error(`Split owner ${prefix.z}/${prefix.x}/${prefix.y} has no descendants.`); + } + if (prefix.z > options.routingZoom) { + deterministicAncestorOwner.exactTiles.push(prefix); + } + return deterministicAncestorOwner; +} + +const options = parseArguments(process.argv.slice(2)); +const repository = options.repository || 'Occumed79/Map'; +const tag = options.tag || DEFAULT_RELEASE; +const outputPath = path.resolve( + options.output || path.join(root, 'config/immutable-owner-plan.json') +); +const routingZoom = Number(options['routing-zoom'] || DEFAULT_ROUTING_ZOOM); +const maxPrefixZoom = Number(options['max-prefix-zoom'] || DEFAULT_MAX_PREFIX_ZOOM); +const maxCandidateBytes = Number( + options['max-candidate-bytes'] || DEFAULT_MAX_CANDIDATE_BYTES +); + +const releaseApi = `https://api.github.com/repos/${repository}/releases/tags/${tag}`; +const release = await fetchJson(releaseApi); +const byName = new Map( + (release.assets || []).map((asset) => [ + asset.name, + { + asset: asset.name, + bytes: Number(asset.size), + sha256: String(asset.digest || '').replace(/^sha256:/, ''), + url: asset.browser_download_url + } + ]) +); +const manifestAsset = byName.get('world-virtual-manifest.json'); +if (!manifestAsset) throw new Error('Published world-virtual-manifest.json is missing.'); +const published = await fetchJson(manifestAsset.url); +const overview = byName.get(published.virtualTiles?.overviewAsset); +const surface = byName.get(published.virtualTiles?.surfaceAsset); +if (!overview || !surface) throw new Error('Published overview or surface archive is missing.'); + +const regions = (published.regions || []).map((region) => { + const asset = byName.get(region.asset); + if (!asset || !asset.sha256 || asset.sha256.length !== 64) { + throw new Error(`Release asset lock is incomplete for ${region.asset}.`); + } + return { + id: region.id, + bounds: region.bounds.map(Number), + ...asset + }; +}); + +const owners = []; +let logicalOwnerCount = 0; +const width = 2 ** routingZoom; +for (let y = 0; y < width; y += 1) { + for (let x = 0; x < width; x += 1) { + const prefix = { z: routingZoom, x, y }; + const candidates = regions.filter((region) => regionIntersectsTile(region, prefix)); + if (candidates.length) { + logicalOwnerCount += 1; + splitOwner(prefix, candidates, { + maxCandidateBytes, + maxPrefixZoom, + routingZoom + }, owners); + } + } +} +owners.sort((left, right) => + left.prefix.z - right.prefix.z || + left.prefix.x - right.prefix.x || + left.prefix.y - right.prefix.y +); + +const plan = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + repository, + releaseTag: tag, + routingZoom, + maxPrefixZoom, + maxCandidateBytes, + authorities: { + land: 'world-surface', + depth: 'world-surface', + landcover: 'world-overview', + cartography: 'regional-owner' + }, + inputs: { + overview, + surface, + regionCount: regions.length, + regionalBytes: regions.reduce((sum, region) => sum + region.bytes, 0) + }, + owners, + logicalOwnerCount, + plannedOwnerCount: owners.length +}; +plan.planVersion = planHash(plan); + +await fs.mkdir(path.dirname(outputPath), { recursive: true }); +await fs.writeFile(outputPath, `${JSON.stringify(plan, null, 2)}\n`); +console.log( + `Planned ${owners.length} deterministic physical owners across ${logicalOwnerCount} logical z${routingZoom} cells ` + + `from ${regions.length} SHA-locked regional inputs ` + + `(${plan.inputs.regionalBytes} bytes); plan ${plan.planVersion}.` +); diff --git a/scripts/offline-tileset/plan-production-batches.mjs b/scripts/offline-tileset/plan-production-batches.mjs new file mode 100644 index 00000000..23c8c0bc --- /dev/null +++ b/scripts/offline-tileset/plan-production-batches.mjs @@ -0,0 +1,167 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`Invalid argument near ${key || ''}.`); + } + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['plan', 'output']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries( + Object.keys(value) + .sort() + .filter((key) => key !== 'generatedAt') + .map((key) => [key, canonical(value[key])]) + ); +} + +function hashDocument(document) { + return createHash('sha256') + .update(`${JSON.stringify(canonical(document))}\n`) + .digest('hex'); +} + +function prefixContains(prefix, candidate) { + if (prefix.z > candidate.z) return false; + const divisor = 2 ** (candidate.z - prefix.z); + return ( + Math.floor(candidate.x / divisor) === prefix.x && + Math.floor(candidate.y / divisor) === prefix.y + ); +} + +function children(prefix) { + const next = prefix.z + 1; + return [ + { z: next, x: prefix.x * 2, y: prefix.y * 2 }, + { z: next, x: prefix.x * 2 + 1, y: prefix.y * 2 }, + { z: next, x: prefix.x * 2, y: prefix.y * 2 + 1 }, + { z: next, x: prefix.x * 2 + 1, y: prefix.y * 2 + 1 } + ]; +} + +function uniqueCandidateBytes(owners) { + const assets = new Map(); + for (const owner of owners) { + for (const candidate of owner.candidates) assets.set(candidate.asset, candidate.bytes); + } + return [...assets.values()].reduce((sum, bytes) => sum + Number(bytes || 0), 0); +} + +function node(prefix, owners) { + return { + prefix, + owners, + ownerCount: owners.length, + candidateBytes: uniqueCandidateBytes(owners), + score: uniqueCandidateBytes(owners) + owners.length * 10_000_000 + }; +} + +const options = parseArguments(process.argv.slice(2)); +const plan = JSON.parse(await fs.readFile(path.resolve(options.plan), 'utf8')); +const requested = Number(options['batch-count'] || 768); +const startZoom = Number(options['start-zoom'] || 4); +if (!Number.isSafeInteger(requested) || requested < 1 || requested > 900) { + throw new Error('Production batch count must be between 1 and 900.'); +} +if (!Number.isSafeInteger(startZoom) || startZoom < 0 || startZoom > plan.routingZoom) { + throw new Error('Production start zoom is invalid.'); +} +if (!Array.isArray(plan.owners) || !plan.owners.length) { + throw new Error('Immutable owner plan is empty.'); +} + +const active = []; +const width = 2 ** startZoom; +for (let y = 0; y < width; y += 1) { + for (let x = 0; x < width; x += 1) { + const prefix = { z: startZoom, x, y }; + const owners = plan.owners.filter((owner) => prefixContains(prefix, owner.prefix)); + if (owners.length) active.push(node(prefix, owners)); + } +} + +while (active.length < requested) { + const candidates = active + .map((entry, index) => ({ entry, index })) + .filter(({ entry }) => entry.prefix.z < plan.maxPrefixZoom) + .map(({ entry, index }) => { + const split = children(entry.prefix) + .map((prefix) => node(prefix, entry.owners.filter((owner) => prefixContains(prefix, owner.prefix)))) + .filter((child) => child.owners.length); + return { entry, index, split, increase: split.length - 1 }; + }) + .filter(({ increase }) => increase > 0 && active.length + increase <= requested) + .sort((left, right) => + right.entry.score - left.entry.score || + left.entry.prefix.z - right.entry.prefix.z || + left.index - right.index + ); + if (!candidates.length) break; + const selected = candidates[0]; + active.splice(selected.index, 1, ...selected.split); +} + +active.sort((left, right) => + left.prefix.z - right.prefix.z || + left.prefix.x - right.prefix.x || + left.prefix.y - right.prefix.y +); +const assigned = new Set(); +const batches = active.map((entry, index) => { + for (const owner of entry.owners) { + if (assigned.has(owner.id)) throw new Error(`Production owner assigned twice: ${owner.id}`); + assigned.add(owner.id); + } + const id = `batch-${String(index).padStart(4, '0')}`; + return { + index, + id, + file: `${id}.pmtiles`, + prefix: entry.prefix, + ownerIds: entry.owners.map((owner) => owner.id), + sourceOwnerCount: entry.owners.length, + candidateBytes: entry.candidateBytes + }; +}); +if (assigned.size !== plan.owners.length) { + throw new Error(`Production batch coverage is incomplete: ${assigned.size}/${plan.owners.length}.`); +} + +const document = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + planVersion: plan.planVersion, + releaseTag: options.tag || 'occumed-flat-v1', + requestedBatchCount: requested, + batchCount: batches.length, + sourceOwnerCount: plan.owners.length, + batches +}; +document.batchPlanVersion = hashDocument(document); + +const output = path.resolve(options.output); +await fs.mkdir(path.dirname(output), { recursive: true }); +await fs.writeFile(output, `${JSON.stringify(document, null, 2)}\n`, { flag: 'wx' }); +console.log( + `Planned ${batches.length} non-overlapping spatial production batches for ` + + `${plan.owners.length} source owners; batch plan ${document.batchPlanVersion}.` +); diff --git a/scripts/offline-tileset/pmtiles-writer.mjs b/scripts/offline-tileset/pmtiles-writer.mjs new file mode 100644 index 00000000..c74092e8 --- /dev/null +++ b/scripts/offline-tileset/pmtiles-writer.mjs @@ -0,0 +1,378 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import path from 'node:path'; +import { pipeline } from 'node:stream/promises'; +import { gzipSync } from 'node:zlib'; +import { PMTiles, zxyToTileId } from 'pmtiles'; +import { LocalPmtilesSource, tilePayload } from './local-pmtiles.mjs'; + +const HEADER_BYTES = 127; +const ROOT_DIRECTORY_BUDGET = 16_384 - HEADER_BYTES; +const GZIP = 2; +const MVT = 1; + +function putUint64(view, offset, value) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`Unsafe PMTiles uint64 value: ${value}`); + } + view.setBigUint64(offset, BigInt(value), true); +} + +function encodeVarint(value) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`Unsafe PMTiles varint value: ${value}`); + } + const bytes = []; + let remaining = BigInt(value); + while (remaining >= 0x80n) { + bytes.push(Number((remaining & 0x7fn) | 0x80n)); + remaining >>= 7n; + } + bytes.push(Number(remaining)); + return Buffer.from(bytes); +} + +function serializeEntries(entries) { + const chunks = [encodeVarint(entries.length)]; + let previousId = 0; + for (const entry of entries) { + chunks.push(encodeVarint(entry.tileId - previousId)); + previousId = entry.tileId; + } + for (const entry of entries) chunks.push(encodeVarint(entry.runLength)); + for (const entry of entries) chunks.push(encodeVarint(entry.length)); + for (let index = 0; index < entries.length; index += 1) { + const entry = entries[index]; + const contiguous = + index > 0 && + entry.offset === entries[index - 1].offset + entries[index - 1].length; + chunks.push(encodeVarint(contiguous ? 0 : entry.offset + 1)); + } + return gzipSync(Buffer.concat(chunks), { level: 9 }); +} + +function buildRootAndLeaves(entries) { + if (entries.length < 16_384) { + const root = serializeEntries(entries); + if (root.byteLength <= ROOT_DIRECTORY_BUDGET) { + return { root, leaves: Buffer.alloc(0), leafCount: 0 }; + } + } + + let leafSize = Math.max(4_096, Math.ceil(entries.length / 3_500)); + while (true) { + const rootEntries = []; + const leafChunks = []; + let leafOffset = 0; + for (let index = 0; index < entries.length; index += leafSize) { + const leaf = serializeEntries(entries.slice(index, index + leafSize)); + rootEntries.push({ + tileId: entries[index].tileId, + offset: leafOffset, + length: leaf.byteLength, + runLength: 0 + }); + leafChunks.push(leaf); + leafOffset += leaf.byteLength; + } + const root = serializeEntries(rootEntries); + if (root.byteLength <= ROOT_DIRECTORY_BUDGET) { + return { + root, + leaves: Buffer.concat(leafChunks), + leafCount: leafChunks.length + }; + } + leafSize = Math.ceil(leafSize * 1.2); + } +} + +function clampE7(value, minimum, maximum) { + return Math.round(Math.min(maximum, Math.max(minimum, value)) * 10_000_000); +} + +function serializeHeader(header) { + const buffer = Buffer.alloc(HEADER_BYTES); + buffer.write('PMTiles', 0, 7, 'utf8'); + buffer[7] = 3; + const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength); + putUint64(view, 8, header.rootOffset); + putUint64(view, 16, header.rootLength); + putUint64(view, 24, header.metadataOffset); + putUint64(view, 32, header.metadataLength); + putUint64(view, 40, header.leafOffset); + putUint64(view, 48, header.leafLength); + putUint64(view, 56, header.tileDataOffset); + putUint64(view, 64, header.tileDataLength); + putUint64(view, 72, header.addressedTiles); + putUint64(view, 80, header.tileEntries); + putUint64(view, 88, header.tileContents); + buffer[96] = 1; + buffer[97] = GZIP; + buffer[98] = GZIP; + buffer[99] = MVT; + buffer[100] = header.minZoom; + buffer[101] = header.maxZoom; + view.setInt32(102, clampE7(header.bounds[0], -180, 180), true); + view.setInt32(106, clampE7(header.bounds[1], -85.0511288, 85.0511288), true); + view.setInt32(110, clampE7(header.bounds[2], -180, 180), true); + view.setInt32(114, clampE7(header.bounds[3], -85.0511288, 85.0511288), true); + buffer[118] = header.center[2]; + view.setInt32(119, clampE7(header.center[0], -180, 180), true); + view.setInt32(123, clampE7(header.center[1], -85.0511288, 85.0511288), true); + return buffer; +} + +async function sha256File(filename) { + const hash = createHash('sha256'); + await pipeline(fs.createReadStream(filename), hash); + return hash.digest('hex'); +} + +async function syncFile(filename) { + const handle = await fsp.open(filename, 'r'); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +export async function verifyPmtiles(filename, expected = {}) { + const stat = await fsp.stat(filename); + if (!stat.isFile() || stat.size < HEADER_BYTES) { + throw new Error(`Invalid PMTiles output: ${filename}`); + } + const source = new LocalPmtilesSource(filename); + try { + const archive = new PMTiles(source); + const header = await archive.getHeader(); + if (header.specVersion !== 3 || header.tileType !== MVT) { + throw new Error(`Unexpected PMTiles header for ${filename}.`); + } + if (expected.addressedTiles !== undefined && + header.numAddressedTiles !== expected.addressedTiles) { + throw new Error( + `PMTiles addressed-tile mismatch: expected ${expected.addressedTiles}, ` + + `received ${header.numAddressedTiles}.` + ); + } + for (const tile of expected.sampleTiles || []) { + const payload = tilePayload(await archive.getZxy(tile.z, tile.x, tile.y)); + if (!payload?.byteLength) { + throw new Error(`PMTiles verification sample is missing ${tile.z}/${tile.x}/${tile.y}.`); + } + } + return { + bytes: stat.size, + sha256: await sha256File(filename), + header + }; + } finally { + await source.close(); + } +} + +/** + * Deterministic PMTiles v3 writer for prebuilt MVT tiles. + * + * Tiles must be added in increasing Hilbert tile-id order. Payloads are + * compressed and content-deduplicated before the final directory is emitted. + */ +export class DeterministicPmtilesWriter { + constructor({ + output, + workDirectory, + metadata = {}, + bounds = [-180, -85.0511288, 180, 85.0511288] + }) { + this.output = path.resolve(output); + this.workDirectory = path.resolve(workDirectory); + this.metadata = metadata; + this.bounds = bounds; + this.entries = []; + this.content = new Map(); + this.dataBytes = 0; + this.addressedTiles = 0; + this.previousTileId = -1; + this.minZoom = 30; + this.maxZoom = 0; + this.coordinates = []; + this.dataPath = path.join( + this.workDirectory, + `tile-data-${process.pid}-${Date.now()}.bin` + ); + this.dataHandlePromise = null; + } + + async initialize() { + await fsp.mkdir(this.workDirectory, { recursive: true }); + await fsp.mkdir(path.dirname(this.output), { recursive: true }); + this.dataHandlePromise = fsp.open(this.dataPath, 'wx'); + return this; + } + + async addTile({ z, x, y, data }) { + if (!this.dataHandlePromise) throw new Error('PMTiles writer is not initialized.'); + const tileId = zxyToTileId(z, x, y); + if (tileId <= this.previousTileId) { + throw new Error(`PMTiles tiles are not strictly ordered at ${z}/${x}/${y}.`); + } + this.previousTileId = tileId; + this.minZoom = Math.min(this.minZoom, z); + this.maxZoom = Math.max(this.maxZoom, z); + this.coordinates.push({ z, x, y }); + + const raw = Buffer.from(data); + if (!raw.byteLength) throw new Error(`Cannot add an empty tile payload at ${z}/${x}/${y}.`); + const contentKey = createHash('sha256').update(raw).digest('hex'); + let stored = this.content.get(contentKey); + if (!stored) { + const compressed = + raw[0] === 0x1f && raw[1] === 0x8b + ? raw + : gzipSync(raw, { level: 9 }); + const handle = await this.dataHandlePromise; + await handle.write(compressed, 0, compressed.byteLength, this.dataBytes); + stored = { offset: this.dataBytes, length: compressed.byteLength }; + this.content.set(contentKey, stored); + this.dataBytes += compressed.byteLength; + } + + const previous = this.entries.at(-1); + if ( + previous && + tileId === previous.tileId + previous.runLength && + previous.offset === stored.offset && + previous.runLength < 0xffff_ffff + ) { + previous.runLength += 1; + } else { + this.entries.push({ + tileId, + offset: stored.offset, + length: stored.length, + runLength: 1 + }); + } + this.addressedTiles += 1; + } + + async finalize() { + if (!this.entries.length) throw new Error('Cannot finalize an empty PMTiles archive.'); + const dataHandle = await this.dataHandlePromise; + await dataHandle.sync(); + await dataHandle.close(); + this.dataHandlePromise = null; + + const { root, leaves, leafCount } = buildRootAndLeaves(this.entries); + const metadata = gzipSync( + Buffer.from(`${JSON.stringify(this.metadata)}\n`), + { level: 9 } + ); + const rootOffset = HEADER_BYTES; + const metadataOffset = rootOffset + root.byteLength; + const leafOffset = metadataOffset + metadata.byteLength; + const tileDataOffset = leafOffset + leaves.byteLength; + const center = [ + (this.bounds[0] + this.bounds[2]) / 2, + (this.bounds[1] + this.bounds[3]) / 2, + this.minZoom + ]; + const header = serializeHeader({ + rootOffset, + rootLength: root.byteLength, + metadataOffset, + metadataLength: metadata.byteLength, + leafOffset, + leafLength: leaves.byteLength, + tileDataOffset, + tileDataLength: this.dataBytes, + addressedTiles: this.addressedTiles, + tileEntries: this.entries.length, + tileContents: this.content.size, + minZoom: this.minZoom, + maxZoom: this.maxZoom, + bounds: this.bounds, + center + }); + + const unique = `${process.pid}-${Date.now()}`; + const staged = path.join(this.workDirectory, `archive-${unique}.staged.pmtiles`); + const pending = `${this.output}.pending-${unique}`; + const stagedHandle = await fsp.open(staged, 'wx'); + try { + await stagedHandle.write(header); + await stagedHandle.write(root); + await stagedHandle.write(metadata); + await stagedHandle.write(leaves); + let position = tileDataOffset; + for await (const chunk of fs.createReadStream(this.dataPath)) { + await stagedHandle.write(chunk, 0, chunk.byteLength, position); + position += chunk.byteLength; + } + if (position !== tileDataOffset + this.dataBytes) { + throw new Error('PMTiles tile-data copy ended at an unexpected offset.'); + } + await stagedHandle.sync(); + } finally { + await stagedHandle.close().catch(() => {}); + } + + const samples = [ + this.coordinates[0], + this.coordinates[Math.floor(this.coordinates.length / 2)], + this.coordinates.at(-1) + ]; + const stagedVerification = await verifyPmtiles(staged, { + addressedTiles: this.addressedTiles, + sampleTiles: samples + }); + await fsp.copyFile(staged, pending, fs.constants.COPYFILE_EXCL); + await syncFile(pending); + const pendingVerification = await verifyPmtiles(pending, { + addressedTiles: this.addressedTiles, + sampleTiles: samples + }); + if ( + stagedVerification.bytes !== pendingVerification.bytes || + stagedVerification.sha256 !== pendingVerification.sha256 + ) { + throw new Error('PMTiles staged and pending files are not byte-identical.'); + } + + await fsp.rename(pending, this.output); + await syncFile(this.output); + const finalVerification = await verifyPmtiles(this.output, { + addressedTiles: this.addressedTiles, + sampleTiles: samples + }); + if (finalVerification.sha256 !== stagedVerification.sha256) { + throw new Error('PMTiles output changed during atomic promotion.'); + } + + await Promise.allSettled([ + fsp.unlink(staged), + fsp.unlink(this.dataPath) + ]); + return { + ...finalVerification, + addressedTiles: this.addressedTiles, + tileEntries: this.entries.length, + tileContents: this.content.size, + leafCount, + minZoom: this.minZoom, + maxZoom: this.maxZoom + }; + } + + async abort() { + if (this.dataHandlePromise) { + const handle = await this.dataHandlePromise.catch(() => null); + await handle?.close().catch(() => {}); + this.dataHandlePromise = null; + } + await fsp.unlink(this.dataPath).catch(() => {}); + } +} diff --git a/scripts/restore-exported-cartography.mjs b/scripts/restore-exported-cartography.mjs index 660c5056..6d96a1ae 100644 --- a/scripts/restore-exported-cartography.mjs +++ b/scripts/restore-exported-cartography.mjs @@ -375,26 +375,6 @@ water.paint['fill-opacity'] = [ 1 ]; -const hillshade = runtime.layers.find((layer) => layer.id === 'occumed-hillshade'); -if (!hillshade) throw new Error('The open hillshade layer is missing after cartography restoration.'); -hillshade.paint['hillshade-exaggeration'] = [ - 'interpolate', - ['linear'], - ['zoom'], - 3, - 0.06, - 6, - 0.16, - 8, - 0.24, - 10, - 0.28, - 14, - 0.24, - 16, - 0.16 -]; - for (const layer of runtime.layers || []) { if (layer.type !== 'symbol') continue; const haloWidth = layer.paint?.['text-halo-width']; diff --git a/scripts/validate-immutable-visuals.mjs b/scripts/validate-immutable-visuals.mjs new file mode 100644 index 00000000..fbdf5066 --- /dev/null +++ b/scripts/validate-immutable-visuals.mjs @@ -0,0 +1,829 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { chromium } from 'playwright'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const require = createRequire(import.meta.url); +const { PNG } = require('pngjs'); +const VIEWPORT = { width: 1440, height: 1000 }; +const MOTION_DURATION_MS = 12_600; + +function parseArguments(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) throw new Error(`Unexpected argument: ${key}`); + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${key}.`); + result[key.slice(2)] = value; + index += 1; + } + for (const required of ['manifest', 'targets', 'output-dir']) { + if (!result[required]) throw new Error(`Missing required --${required}.`); + } + return result; +} + +function safeName(value) { + return String(value).replace(/[^a-z0-9-]+/gi, '-').replace(/^-|-$/g, '').toLowerCase(); +} + +async function waitForReadiness(origin, expectedStatus, timeoutMs = 30_000) { + const started = Date.now(); + let lastStatus = null; + while (Date.now() - started < timeoutMs) { + try { + const response = await fetch(`${origin}/readyz`, { cache: 'no-store' }); + lastStatus = response.status; + if (response.status === expectedStatus) { + return { + status: response.status, + body: await response.json().catch(() => null) + }; + } + } catch { + // The child may still be binding its port. + } + await new Promise((resolve) => setTimeout(resolve, 150)); + } + throw new Error(`Readiness did not return ${expectedStatus}; last status was ${lastStatus}.`); +} + +async function startServer({ + manifest, + port, + allowPartial, + logPath +}) { + const handle = await fs.open(logPath, 'wx'); + const child = spawn(process.execPath, ['server.mjs'], { + cwd: root, + env: { + ...process.env, + HOST: '127.0.0.1', + PORT: String(port), + OCCUMED_IMMUTABLE_TILESET_MANIFEST: manifest, + OCCUMED_ALLOW_PARTIAL_TILESET_FIXTURE: allowPartial ? 'true' : 'false' + }, + stdio: ['ignore', handle.fd, handle.fd] + }); + child.once('exit', () => { + void handle.close().catch(() => {}); + }); + return { child, handle, origin: `http://127.0.0.1:${port}` }; +} + +async function stopServer(server) { + if (!server?.child || server.child.exitCode !== null) return; + server.child.kill('SIGTERM'); + await Promise.race([ + new Promise((resolve) => server.child.once('exit', resolve)), + new Promise((resolve) => setTimeout(resolve, 5_000)) + ]); + if (server.child.exitCode === null) server.child.kill('SIGKILL'); +} + +async function waitForRequestSettlement(outstanding, timeoutMs = 20_000) { + const started = Date.now(); + let idleSince = outstanding.size ? null : Date.now(); + while (Date.now() - started < timeoutMs) { + if (!outstanding.size) { + idleSince ||= Date.now(); + if (Date.now() - idleSince >= 250) return; + } else { + idleSince = null; + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`Tile delivery did not settle; ${outstanding.size} requests remain.`); +} + +async function waitForInViewTileSettlement(page, timeoutMs = 30_000) { + const started = Date.now(); + let state = await mapState(page); + while (Date.now() - started < timeoutMs) { + const tiles = state.inViewTiles; + if (tiles.length > 0 && tiles.every((tile) => tile.state !== 'loading')) { + await pumpRenderFrames(page, 4, 2_000); + return mapState(page); + } + await pumpRenderFrames(page, 1, 500); + await page.waitForTimeout(50); + state = await mapState(page); + } + const loading = state.inViewTiles.filter((tile) => tile.state === 'loading'); + throw new Error( + `In-view tile parsing did not settle; ${loading.length} of ` + + `${state.inViewTiles.length} tiles remain loading.` + ); +} + +async function waitForRenderedCamera(page, camera) { + await page.evaluate(({ center, zoom }) => { + const map = globalThis.__OCCUMED_MAP__; + map.stop(); + if (typeof map.setRenderWorldCopies === 'function') { + map.setRenderWorldCopies(zoom >= 3); + } + map.jumpTo({ center, zoom, bearing: 0, pitch: 0 }); + }, camera); + await page.waitForFunction(({ center, zoom }) => { + const map = globalThis.__OCCUMED_MAP__; + if (!map) return false; + const actual = map.getCenter(); + const longitudeError = Math.abs((((actual.lng - center[0]) + 540) % 360) - 180); + return longitudeError < 0.01 && + Math.abs(actual.lat - center[1]) < 0.01 && + Math.abs(map.getZoom() - zoom) < 0.01; + }, camera, { timeout: 30_000 }); + await pumpRenderFrames(page); +} + +async function pumpRenderFrames(page, minimumFrames = 4, timeoutMs = 2_000) { + await page.evaluate(({ minimumFrames, timeoutMs }) => new Promise((resolve) => { + const map = globalThis.__OCCUMED_MAP__; + let frames = 0; + let settled = false; + const finish = () => { + if (settled) return; + settled = true; + clearInterval(repaint); + clearTimeout(timeout); + map.off('render', onRender); + resolve(); + }; + const onRender = () => { + frames += 1; + if (frames >= minimumFrames) finish(); + }; + map.on('render', onRender); + const repaint = setInterval(() => map.triggerRepaint(), 50); + const timeout = setTimeout(finish, timeoutMs); + map.triggerRepaint(); + }), { minimumFrames, timeoutMs }); +} + +async function mapState(page) { + return page.evaluate(() => { + const map = globalThis.__OCCUMED_MAP__; + const style = map.getStyle(); + const tileManager = map.style?.tileManagers?.['occumed-open']; + const sourceIds = Object.keys(style.sources || {}).sort(); + const sourceLayers = new Set(); + const width = map.getContainer().clientWidth; + const height = map.getContainer().clientHeight; + for (let row = 0; row < 3; row += 1) { + for (let column = 0; column < 3; column += 1) { + const x = ((column + 0.5) / 3) * width; + const y = ((row + 0.5) / 3) * height; + const features = map.queryRenderedFeatures([ + [Math.max(0, x - 45), Math.max(0, y - 45)], + [Math.min(width, x + 45), Math.min(height, y + 45)] + ]); + for (const feature of features) { + if (feature.source === 'occumed-open' && feature.sourceLayer) { + sourceLayers.add(feature.sourceLayer); + } + } + } + } + return { + sourceIds, + sourceLayers: [...sourceLayers].sort(), + center: [map.getCenter().lng, map.getCenter().lat], + zoom: map.getZoom(), + moving: map.isMoving(), + styleLoaded: map.isStyleLoaded(), + inViewTiles: tileManager?._inViewTiles?.getAllIds?.().map((id) => { + const tile = tileManager._inViewTiles.getTileById(id); + return { + z: tile?.tileID?.canonical?.z, + x: tile?.tileID?.canonical?.x, + y: tile?.tileID?.canonical?.y, + state: tile?.state, + hasData: Boolean(tile?.hasData?.()) + }; + }) || [] + }; + }); +} + +async function analyzeScreenshot(_page, screenshot, zoom) { + const { data, width, height } = PNG.sync.read(screenshot); + const offset = (x, y) => (y * width + x) * 4; + const delta = (left, right) => + Math.abs(data[left] - data[right]) + + Math.abs(data[left + 1] - data[right + 1]) + + Math.abs(data[left + 2] - data[right + 2]); + const luma = (value) => + (data[value] * 299 + data[value + 1] * 587 + data[value + 2] * 114) / 1_000; + const waterLike = (value) => + data[value + 2] >= 170 && + data[value + 1] >= 125 && + data[value + 2] - data[value] >= 35 && + data[value + 2] - data[value + 1] >= 20; + const surfaceLike = (value) => + data[value + 3] >= 250 && luma(value) >= 75 && luma(value) <= 245; + + const histogram = new Map(); + let sampled = 0; + let nonDark = 0; + for (let y = 0; y < height; y += 4) { + for (let x = 0; x < width; x += 4) { + const value = offset(x, y); + const key = + ((data[value] >> 4) << 8) | + ((data[value + 1] >> 4) << 4) | + (data[value + 2] >> 4); + histogram.set(key, (histogram.get(key) || 0) + 1); + sampled += 1; + if (luma(value) > 25) nonDark += 1; + } + } + let dominantKey = 0; + let dominantCount = 0; + for (const [key, count] of histogram) { + if (count > dominantCount) { + dominantKey = key; + dominantCount = count; + } + } + const dominant = dominantCount / sampled; + const dominantColor = [ + ((dominantKey >> 8) & 0x0f) * 16 + 8, + ((dominantKey >> 4) & 0x0f) * 16 + 8, + (dominantKey & 0x0f) * 16 + 8 + ]; + + function scanVertical(x, predicate) { + let best = { start: 0, end: 0, length: 0 }; + let start = 0; + let length = 0; + let gaps = 0; + for (let y = 2; y < height - 2; y += 1) { + const accepted = predicate(offset(x - 1, y), offset(x, y)); + if (accepted) { + if (!length) start = y; + length += 1 + gaps; + gaps = 0; + } else if (length && gaps < 2) { + gaps += 1; + } else { + if (length > best.length) best = { start, end: y - gaps - 1, length }; + start = 0; + length = 0; + gaps = 0; + } + } + if (length > best.length) best = { start, end: height - gaps - 2, length }; + return { axis: x, ...best }; + } + + function scanHorizontal(y, predicate) { + let best = { start: 0, end: 0, length: 0 }; + let start = 0; + let length = 0; + let gaps = 0; + for (let x = 2; x < width - 2; x += 1) { + const accepted = predicate(offset(x, y - 1), offset(x, y)); + if (accepted) { + if (!length) start = x; + length += 1 + gaps; + gaps = 0; + } else if (length && gaps < 2) { + gaps += 1; + } else { + if (length > best.length) best = { start, end: x - gaps - 1, length }; + start = 0; + length = 0; + gaps = 0; + } + } + if (length > best.length) best = { start, end: width - gaps - 2, length }; + return { axis: y, ...best }; + } + + const strongEdge = (left, right) => { + const difference = delta(left, right); + return surfaceLike(left) && surfaceLike(right) && difference >= 45 && difference <= 260; + }; + const waterEdge = (left, right) => { + const difference = delta(left, right); + return waterLike(left) && waterLike(right) && difference >= 5 && difference <= 75; + }; + const footprintEdge = (left, right) => { + const difference = delta(left, right); + return surfaceLike(left) && surfaceLike(right) && difference >= 8 && difference <= 260; + }; + + const verticalStrong = []; + const horizontalStrong = []; + const verticalWater = []; + const horizontalWater = []; + const verticalFootprints = []; + const horizontalFootprints = []; + for (let x = 4; x < width - 4; x += 1) { + const strong = scanVertical(x, strongEdge); + if (strong.length >= height * 0.62) verticalStrong.push(strong); + if (zoom >= 5) { + const water = scanVertical(x, waterEdge); + if (water.length >= height * 0.32) verticalWater.push(water); + const footprint = scanVertical(x, footprintEdge); + if (footprint.length >= 90) verticalFootprints.push(footprint); + } + } + for (let y = 4; y < height - 4; y += 1) { + const strong = scanHorizontal(y, strongEdge); + if (strong.length >= width * 0.62) horizontalStrong.push(strong); + if (zoom >= 5) { + const water = scanHorizontal(y, waterEdge); + if (water.length >= width * 0.32) horizontalWater.push(water); + const footprint = scanHorizontal(y, footprintEdge); + if (footprint.length >= 110) horizontalFootprints.push(footprint); + } + } + + function collapse(runs) { + const ordered = [...runs].sort((left, right) => left.axis - right.axis || right.length - left.length); + const output = []; + for (const run of ordered) { + const previous = output.at(-1); + if (previous && run.axis - previous.axis <= 3) { + if (run.length > previous.length) output[output.length - 1] = run; + } else { + output.push(run); + } + } + return output; + } + + const verticalCandidates = collapse(verticalFootprints).slice(0, 80); + const horizontalCandidates = collapse(horizontalFootprints).slice(0, 80); + let rectangles = 0; + let rectangleBounds = null; + function hasUniformInterior(left, right, top, bottom) { + const counts = new Map(); + let total = 0; + for (let y = top + 8; y < bottom - 8; y += 8) { + for (let x = left + 8; x < right - 8; x += 8) { + const value = offset(x, y); + const key = + ((data[value] >> 4) << 8) | + ((data[value + 1] >> 4) << 4) | + (data[value + 2] >> 4); + counts.set(key, (counts.get(key) || 0) + 1); + total += 1; + } + } + return total > 40 && Math.max(...counts.values()) / total >= 0.68; + } + + for (let leftIndex = 0; leftIndex < verticalCandidates.length; leftIndex += 1) { + const left = verticalCandidates[leftIndex]; + for (let rightIndex = leftIndex + 1; rightIndex < verticalCandidates.length; rightIndex += 1) { + const right = verticalCandidates[rightIndex]; + if (right.axis - left.axis < 90) continue; + const topLimit = Math.max(left.start, right.start); + const bottomLimit = Math.min(left.end, right.end); + if (bottomLimit - topLimit < 90) continue; + const top = horizontalCandidates.find((run) => + run.axis >= topLimit - 5 && + run.axis <= bottomLimit + 5 && + run.start <= left.axis + 5 && + run.end >= right.axis - 5 + ); + if (!top) continue; + const bottom = horizontalCandidates.find((run) => + run.axis - top.axis >= 90 && + run.axis <= bottomLimit + 5 && + run.start <= left.axis + 5 && + run.end >= right.axis - 5 + ); + const candidateArea = bottom + ? (right.axis - left.axis) * (bottom.axis - top.axis) + : 0; + if ( + bottom && + candidateArea >= width * height * 0.08 && + hasUniformInterior(left.axis, right.axis, top.axis, bottom.axis) + ) { + rectangles += 1; + rectangleBounds = { + left: left.axis, + right: right.axis, + top: top.axis, + bottom: bottom.axis, + areaRatio: candidateArea / (width * height) + }; + break; + } + } + if (rectangles) break; + } + + const verticalSeams = collapse([...verticalStrong, ...verticalWater]); + const horizontalSeams = collapse([...horizontalStrong, ...horizontalWater]); + const blank = histogram.size < 12 || dominant > 0.985 || nonDark / sampled < 0.02; + return { + width, + height, + sampledPixels: sampled, + quantizedColors: histogram.size, + dominantColorRatio: dominant, + dominantColor, + nonDarkRatio: nonDark / sampled, + blank, + rectangularFootprints: rectangles, + rectangleBounds, + verticalSeams: verticalSeams.length, + horizontalSeams: horizontalSeams.length, + stretchedPolygons: rectangles, + inconsistentNeighbors: verticalSeams.length + horizontalSeams.length, + longestVerticalEdge: Math.max( + 0, + ...verticalStrong.map((run) => run.length), + ...verticalWater.map((run) => run.length) + ), + longestHorizontalEdge: Math.max( + 0, + ...horizontalStrong.map((run) => run.length), + ...horizontalWater.map((run) => run.length) + ) + }; +} + +function validateSourceState(state, label, failures) { + if (state.sourceIds.length !== 1 || state.sourceIds[0] !== 'occumed-open') { + failures.push(`${label}: source switched to [${state.sourceIds.join(', ')}].`); + } +} + +function validatePixels(analysis, label, failures) { + if (analysis.blank) failures.push(`${label}: blank rendered frame.`); + if (analysis.rectangularFootprints) failures.push(`${label}: rectangular tile footprint detected.`); + if (analysis.verticalSeams) failures.push(`${label}: vertical seam detected.`); + if (analysis.horizontalSeams) failures.push(`${label}: horizontal seam detected.`); + if (analysis.stretchedPolygons) failures.push(`${label}: stretched polygon detected.`); + if (analysis.inconsistentNeighbors) failures.push(`${label}: neighboring tiles are inconsistent.`); +} + +function applyFoundationBlankContract(pixels, state) { + pixels.pixelBlank = pixels.blank; + const oceanOnly = + state.sourceLayers.length > 0 && + state.sourceLayers.every((layer) => + ['depth', 'water', 'water_name', 'waterway'].includes(layer) + ); + const foundationLand = [232, 232, 216]; + const landColorDistance = pixels.dominantColor.reduce( + (sum, channel, index) => sum + Math.abs(channel - foundationLand[index]), + 0 + ); + const uniformLand = + state.sourceLayers.includes('land') && + pixels.dominantColorRatio >= 0.985 && + landColorDistance <= 30; + if (oceanOnly || uniformLand) { + pixels.blank = false; + pixels.blankContract = oceanOnly ? 'ocean-foundation' : 'uniform-land-foundation'; + } +} + +function validateRequiredLayers(name, state, failures) { + const available = new Set(state.sourceLayers); + const requireOne = (choices, label) => { + if (!choices.some((choice) => available.has(choice))) { + failures.push(`${name}: no rendered ${label} bucket (${choices.join(' or ')}).`); + } + }; + if (['pacific', 'antimeridian'].includes(name)) { + requireOne(['depth', 'land'], 'foundation'); + return; + } + requireOne(['land', 'landcover'], 'land foundation'); + if (name.startsWith('fresno-')) { + requireOne(['land'], 'prebuilt land foundation'); + } + if (name === 'fresno-city' || name === 'fresno-street') { + requireOne(['transportation', 'building', 'place'], 'regional cartography'); + } +} + +function staticStateIsReady(name, state) { + const available = new Set(state.sourceLayers); + const hasOne = (choices) => choices.some((choice) => available.has(choice)); + if (['pacific', 'antimeridian'].includes(name)) { + return hasOne(['depth', 'land']); + } + if (!hasOne(['land', 'landcover'])) return false; + if (name.startsWith('fresno-') && !available.has('land')) return false; + if (name === 'fresno-city' || name === 'fresno-street') { + return hasOne(['transportation', 'building', 'place']); + } + return true; +} + +async function waitForStaticState(page, name, timeoutMs = 30_000) { + const started = Date.now(); + let state = await mapState(page); + while (!staticStateIsReady(name, state) && Date.now() - started < timeoutMs) { + await pumpRenderFrames(page, 3, 1_000); + await page.waitForTimeout(100); + state = await mapState(page); + } + return state; +} + +const options = parseArguments(process.argv.slice(2)); +const phase = options.phase || 'all'; +if (!['all', 'static', 'motion'].includes(phase)) { + throw new Error(`Unsupported validation phase: ${phase}`); +} +const manifest = path.resolve(options.manifest); +const targets = JSON.parse(await fs.readFile(path.resolve(options.targets), 'utf8')); +const outputDir = path.resolve(options['output-dir']); +await fs.mkdir(path.dirname(outputDir), { recursive: true }); +await fs.mkdir(outputDir, { recursive: false }); +const priorStaticReport = options['static-report'] + ? JSON.parse(await fs.readFile(path.resolve(options['static-report']), 'utf8')) + : null; +if (phase === 'motion' && !priorStaticReport) { + throw new Error('Motion-only validation requires --static-report.'); +} +if (priorStaticReport && priorStaticReport.summary?.passed !== true) { + throw new Error('The supplied static validation report did not pass.'); +} + +const report = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + manifest, + phase, + staticReport: options['static-report'] ? path.resolve(options['static-report']) : null, + viewport: VIEWPORT, + failClosedCheck: null, + staticViews: priorStaticReport?.staticViews || [], + motionFrames: [], + pageErrors: [], + networkErrors: [], + requestCancellations: [], + tileResponses: [], + failures: [] +}; + +let validationServer; +let browser; +try { + const failClosedServer = await startServer({ + manifest, + port: 4317, + allowPartial: false, + logPath: path.join(outputDir, 'server-fail-closed.log') + }); + try { + const rejected = await waitForReadiness(failClosedServer.origin, 503); + report.failClosedCheck = rejected; + } finally { + await stopServer(failClosedServer); + } + + validationServer = await startServer({ + manifest, + port: 4318, + allowPartial: true, + logPath: path.join(outputDir, 'server.log') + }); + const ready = await waitForReadiness(validationServer.origin, 200); + report.readiness = ready; + + browser = await chromium.launch({ + headless: true, + args: ['--disable-dev-shm-usage', '--no-sandbox'] + }); + const context = await browser.newContext({ + viewport: VIEWPORT, + deviceScaleFactor: 1 + }); + const page = await context.newPage(); + const outstandingTiles = new Set(); + page.on('pageerror', (error) => report.pageErrors.push(error.message)); + page.on('console', (message) => { + if (message.type() === 'error') report.pageErrors.push(message.text()); + }); + page.on('request', (request) => { + if (request.url().includes('/tiles/')) outstandingTiles.add(request); + }); + page.on('requestfinished', (request) => outstandingTiles.delete(request)); + page.on('requestfailed', (request) => { + outstandingTiles.delete(request); + const failure = request.failure()?.errorText || 'request failed'; + const entry = { url: request.url(), failure }; + if (failure.includes('ERR_ABORTED')) report.requestCancellations.push(entry); + else report.networkErrors.push(entry); + }); + page.on('response', (response) => { + if (response.url().includes('/tiles/')) { + report.tileResponses.push({ + url: response.url(), + status: response.status() + }); + } + if ( + response.url().startsWith(validationServer.origin) && + response.status() >= 400 + ) { + report.networkErrors.push({ + url: response.url(), + status: response.status() + }); + } + }); + + await page.goto(validationServer.origin, { + waitUntil: 'domcontentloaded', + timeout: 60_000 + }); + await page.addStyleTag({ + content: '.map-header,.maplibregl-control-container{display:none!important}' + }); + await page.waitForFunction(() => { + const map = globalThis.__OCCUMED_MAP__; + return map && map.getStyle()?.sources?.['occumed-open']; + }, { timeout: 60_000 }); + await page.evaluate(() => { + const map = globalThis.__OCCUMED_MAP__; + if (typeof map.setPixelRatio === 'function') map.setPixelRatio(1); + map.resize(); + map.triggerRepaint(); + }); + report.validatorPixelRatio = await page.evaluate(() => + globalThis.__OCCUMED_MAP__.getPixelRatio?.() || 1 + ); + + if (phase !== 'motion') { + for (const view of targets.staticViews) { + await waitForRenderedCamera(page, view); + await page.waitForTimeout(200); + await waitForRequestSettlement(outstandingTiles); + await pumpRenderFrames(page); + const state = await waitForStaticState(page, view.name); + const screenshot = await page.screenshot({ + path: path.join(outputDir, `static-${safeName(view.name)}.png`) + }); + const pixels = await analyzeScreenshot(page, screenshot, state.zoom); + applyFoundationBlankContract(pixels, state); + validateSourceState(state, `static ${view.name}`, report.failures); + validateRequiredLayers(view.name, state, report.failures); + validatePixels(pixels, `static ${view.name}`, report.failures); + report.staticViews.push({ + ...view, + state, + pixels, + file: `static-${safeName(view.name)}.png` + }); + console.log( + `[static] ${view.name}: sources=${state.sourceIds.length}, ` + + `layers=${state.sourceLayers.length}, seams=${pixels.verticalSeams + pixels.horizontalSeams}, ` + + `rectangles=${pixels.rectangularFootprints}, blank=${pixels.blank}.` + ); + } + } + + // The nine static views intentionally traverse the complete zoom pyramid and + // can leave hundreds of megabytes of decoded vector buckets in a software + // browser. Start the motion phase with a fresh page and an empty renderer + // cache; HTTP and PMTiles inputs remain unchanged. + if (phase === 'all') { + await waitForRequestSettlement(outstandingTiles); + await page.reload({ waitUntil: 'domcontentloaded', timeout: 60_000 }); + await page.addStyleTag({ + content: '.map-header,.maplibregl-control-container{display:none!important}' + }); + await page.waitForFunction(() => { + const map = globalThis.__OCCUMED_MAP__; + return map && map.getStyle()?.sources?.['occumed-open']; + }, { timeout: 60_000 }); + await page.evaluate(() => { + const map = globalThis.__OCCUMED_MAP__; + if (typeof map.setPixelRatio === 'function') map.setPixelRatio(1); + map.resize(); + map.triggerRepaint(); + }); + await page.waitForTimeout(200); + await waitForRequestSettlement(outstandingTiles); + } + + if (phase !== 'static') { + const selectedFrames = options.motion + ? targets.validationFrames.filter((frame) => frame.motion === options.motion) + : targets.validationFrames; + const framesByMotion = Map.groupBy(selectedFrames, (frame) => frame.motion); + for (const [motionName, frames] of framesByMotion) { + const ordered = [...frames].sort((left, right) => left.index - right.index); + const start = ordered[0]; + await waitForRenderedCamera(page, start); + await page.waitForTimeout(200); + await waitForRequestSettlement(outstandingTiles); + await pumpRenderFrames(page); + await waitForInViewTileSettlement(page); + + for (let index = 0; index < ordered.length; index += 1) { + if (index > 0) { + await page.evaluate(({ frame, duration }) => new Promise((resolve) => { + const map = globalThis.__OCCUMED_MAP__; + const finish = () => { + map.off('moveend', finish); + resolve(); + }; + map.on('moveend', finish); + map.easeTo({ + center: frame.center, + zoom: frame.zoom, + bearing: 0, + pitch: 0, + duration, + easing: (value) => value + }); + }), { + frame: ordered[index], + duration: MOTION_DURATION_MS / (ordered.length - 1) + }); + await waitForRequestSettlement(outstandingTiles); + } + + const state = await waitForInViewTileSettlement(page); + const filename = `motion-${safeName(motionName)}-${String(index).padStart(2, '0')}.png`; + const screenshot = await page.screenshot({ path: path.join(outputDir, filename) }); + const pixels = await analyzeScreenshot(page, screenshot, state.zoom); + applyFoundationBlankContract(pixels, state); + validateSourceState(state, `motion ${motionName} frame ${index}`, report.failures); + validatePixels(pixels, `motion ${motionName} frame ${index}`, report.failures); + report.motionFrames.push({ + motion: motionName, + index, + expectedCamera: ordered[index], + state, + pixels, + file: filename + }); + } + console.log(`[motion] ${motionName}: ${ordered.length} exact-camera transition frames captured.`); + } + } + + if (report.pageErrors.length) { + report.failures.push(`${report.pageErrors.length} browser/page errors occurred.`); + } + if (report.networkErrors.length) { + report.failures.push(`${report.networkErrors.length} network/resource errors occurred.`); + } + report.summary = { + staticViewCount: report.staticViews.length, + motionFrameCount: report.motionFrames.length, + blankFrames: [...report.staticViews, ...report.motionFrames] + .filter((capture) => capture.pixels.blank).length, + rectangularFootprints: [...report.staticViews, ...report.motionFrames] + .reduce((sum, capture) => sum + capture.pixels.rectangularFootprints, 0), + verticalSeams: [...report.staticViews, ...report.motionFrames] + .reduce((sum, capture) => sum + capture.pixels.verticalSeams, 0), + horizontalSeams: [...report.staticViews, ...report.motionFrames] + .reduce((sum, capture) => sum + capture.pixels.horizontalSeams, 0), + stretchedPolygons: [...report.staticViews, ...report.motionFrames] + .reduce((sum, capture) => sum + capture.pixels.stretchedPolygons, 0), + inconsistentNeighbors: [...report.staticViews, ...report.motionFrames] + .reduce((sum, capture) => sum + capture.pixels.inconsistentNeighbors, 0), + sourceSwitches: [...report.staticViews, ...report.motionFrames] + .filter((capture) => + capture.state.sourceIds.length !== 1 || + capture.state.sourceIds[0] !== 'occumed-open' + ).length, + pageErrors: report.pageErrors.length, + networkErrors: report.networkErrors.length, + requestCancellations: report.requestCancellations.length, + passed: report.failures.length === 0 + }; +} catch (error) { + report.failures.push(error?.stack || error?.message || String(error)); +} finally { + await browser?.close().catch(() => {}); + await stopServer(validationServer); + await fs.writeFile( + path.join(outputDir, 'immutable-visual-report.json'), + `${JSON.stringify(report, null, 2)}\n` + ); +} + +if (report.failures.length) { + console.error('Immutable visual validation failed:'); + for (const failure of report.failures) console.error(`- ${failure}`); + process.exit(1); +} +console.log( + `Immutable visual validation passed: ${report.summary.staticViewCount} static views, ` + + `${report.summary.motionFrameCount} exact-camera motion checkpoints, one source, and zero visual/network defects.` +); diff --git a/scripts/validate-new-map-v2.mjs b/scripts/validate-new-map-v2.mjs deleted file mode 100644 index 365ef3c0..00000000 --- a/scripts/validate-new-map-v2.mjs +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env node - -import fs from 'node:fs/promises'; -import path from 'node:path'; -import { chromium } from 'playwright'; - -const baseUrl = process.env.OCCUMED_PREVIEW_URL || 'http://127.0.0.1:4173'; -const outputDir = path.resolve(process.env.OCCUMED_PREVIEW_OUTPUT || 'new-map-v2-validation'); -await fs.mkdir(outputDir, { recursive: true }); - -const views = [ - { - id: 'world', - center: [0, 18], - zoom: 1.25, - requiredLandPoints: [ - [-112.074, 33.448], - [-99.133, 19.432], - [-47.882, -15.794], - [2.352, 48.857], - [38.758, 8.981], - [28.047, -26.204], - [77.209, 28.614], - [116.407, 39.904], - [149.13, -35.28] - ] - }, - { id: 'north-america', center: [-100, 39], zoom: 3.35, requiredLandPoints: [[-112.074, 33.448], [-99.133, 19.432], [-104.99, 39.739]] }, - { id: 'europe-africa', center: [15, 8], zoom: 2.55, requiredLandPoints: [[2.352, 48.857], [38.758, 8.981], [28.047, -26.204]] }, - { id: 'fresno', center: [-119.7871, 36.7378], zoom: 11, requiredLandPoints: [[-119.7871, 36.7378]] } -]; - -const browser = await chromium.launch({ headless: true }); -const context = await browser.newContext({ - viewport: { width: 1600, height: 1000 }, - deviceScaleFactor: 1 -}); -const page = await context.newPage(); -const pageErrors = []; -const consoleErrors = []; -const failedRequests = []; -const badResponses = []; - -page.on('pageerror', (error) => pageErrors.push(error.message)); -page.on('console', (message) => { - if (message.type() === 'error') consoleErrors.push(message.text()); -}); -page.on('requestfailed', (request) => { - const failure = request.failure()?.errorText || 'request failed'; - if (!/ERR_ABORTED|NS_BINDING_ABORTED/.test(failure)) failedRequests.push(`${failure} ${request.url()}`); -}); -page.on('response', (response) => { - if (response.status() >= 400) badResponses.push(`${response.status()} ${response.url()}`); -}); - -const response = await page.goto(baseUrl, { waitUntil: 'domcontentloaded', timeout: 45_000 }); -if (!response?.ok()) throw new Error(`Map page returned HTTP ${response?.status() || 'unknown'}.`); - -await page.waitForFunction( - () => globalThis.__OCCUMED_MAP_V2__?.ready === true, - undefined, - { timeout: 60_000 } -); - -async function settleView(center, zoom) { - await page.evaluate(({ center, zoom }) => new Promise((resolve, reject) => { - const map = globalThis.__OCCUMED_MAP__; - if (!map) { - reject(new Error('Map instance missing.')); - return; - } - const timeout = setTimeout(() => reject(new Error('View did not become idle.')), 45_000); - const finish = () => { - if (!map.loaded() || !map.areTilesLoaded()) return; - clearTimeout(timeout); - map.off('idle', finish); - requestAnimationFrame(() => requestAnimationFrame(resolve)); - }; - map.on('idle', finish); - map.jumpTo({ center, zoom, bearing: 0, pitch: 0 }); - finish(); - }), { center, zoom }); -} - -const results = []; -for (const view of views) { - await settleView(view.center, view.zoom); - const inspection = await page.evaluate(({ requiredLandPoints }) => { - const map = globalThis.__OCCUMED_MAP__; - const style = map.getStyle(); - const canvas = map.getCanvas(); - const viewportFeatures = map.queryRenderedFeatures(); - const pointChecks = requiredLandPoints.map((coordinate) => { - const point = map.project(coordinate); - const visible = point.x >= 0 && point.y >= 0 && point.x <= canvas.clientWidth && point.y <= canvas.clientHeight; - const features = visible ? map.queryRenderedFeatures(point) : []; - const sourceLayers = [...new Set(features.map((feature) => feature.sourceLayer).filter(Boolean))].sort(); - const layerIds = [...new Set(features.map((feature) => feature.layer?.id).filter(Boolean))].sort(); - const keys = [...sourceLayers, ...layerIds].map((value) => String(value).toLowerCase()); - return { - coordinate, - visible, - featureCount: features.length, - sourceLayers, - layerIds, - resolvesToWater: keys.some((key) => /water|ocean|lake|river|marine|bay/.test(key)) - }; - }); - - return { - center: map.getCenter().toArray(), - zoom: map.getZoom(), - projection: style.projection?.type || map.getProjection?.()?.type || null, - sourceCount: Object.keys(style.sources || {}).length, - vectorSourceCount: Object.values(style.sources || {}).filter((source) => source?.type === 'vector').length, - sourceLoaded: Object.keys(style.sources || {}).every((sourceId) => map.isSourceLoaded(sourceId)), - tilesLoaded: map.areTilesLoaded(), - renderedFeatureCount: viewportFeatures.length, - renderedSourceLayers: [...new Set(viewportFeatures.map((feature) => feature.sourceLayer).filter(Boolean))].sort(), - renderedLayerIds: [...new Set(viewportFeatures.map((feature) => feature.layer?.id).filter(Boolean))].sort(), - forbiddenLayerCount: (style.layers || []).filter((layer) => ['sky', 'hillshade', 'model', 'fill-extrusion'].includes(layer.type)).length, - terrain: style.terrain || null, - fog: style.fog || null, - architecture: style.metadata?.['occumed:architecture'] || null, - pointChecks - }; - }, { requiredLandPoints: view.requiredLandPoints }); - - await page.screenshot({ - path: path.join(outputDir, `${view.id}.png`), - fullPage: true - }); - - const invisibleRequiredPoints = inspection.pointChecks.filter((point) => !point.visible); - const waterRequiredPoints = inspection.pointChecks.filter((point) => point.visible && point.resolvesToWater); - const minimumFeatureCount = view.id === 'fresno' ? 80 : 25; - const minimumSourceLayerCount = view.id === 'fresno' ? 4 : 3; - - if (inspection.projection !== 'mercator') throw new Error(`${view.id}: projection is ${inspection.projection}.`); - if (inspection.sourceCount !== 1 || inspection.vectorSourceCount !== 1) { - throw new Error(`${view.id}: expected one vector source, found ${inspection.sourceCount} total and ${inspection.vectorSourceCount} vector.`); - } - if (!inspection.sourceLoaded || !inspection.tilesLoaded) throw new Error(`${view.id}: worldwide source or tiles did not fully load.`); - if (inspection.renderedFeatureCount < minimumFeatureCount) { - throw new Error(`${view.id}: only ${inspection.renderedFeatureCount} features rendered.`); - } - if (inspection.renderedSourceLayers.length < minimumSourceLayerCount) { - throw new Error(`${view.id}: only ${inspection.renderedSourceLayers.length} source layers rendered.`); - } - if (inspection.forbiddenLayerCount || inspection.terrain || inspection.fog) { - throw new Error(`${view.id}: globe, terrain, or forbidden 3D layers remain active.`); - } - if (inspection.architecture !== 'clean-worldwide-vector-v2') { - throw new Error(`${view.id}: architecture metadata is ${inspection.architecture}.`); - } - if (invisibleRequiredPoints.length) { - throw new Error(`${view.id}: required land points fell outside the viewport: ${JSON.stringify(invisibleRequiredPoints)}.`); - } - if (waterRequiredPoints.length) { - throw new Error(`${view.id}: required land points resolved to water layers: ${JSON.stringify(waterRequiredPoints)}.`); - } - - results.push({ id: view.id, ...inspection }); -} - -const contract = await page.evaluate(() => globalThis.__OCCUMED_MAP_V2__); -const report = { - baseUrl, - contract, - views: results, - pageErrors, - consoleErrors, - failedRequests, - badResponses -}; -await fs.writeFile(path.join(outputDir, 'report.json'), `${JSON.stringify(report, null, 2)}\n`); - -await browser.close(); - -if (pageErrors.length || consoleErrors.length || failedRequests.length || badResponses.length) { - throw new Error(`Browser/network errors detected: ${JSON.stringify({ pageErrors, consoleErrors, failedRequests, badResponses })}`); -} - -console.log(JSON.stringify({ - validated: true, - architecture: contract.architecture, - sourceCount: contract.sourceCount, - screenshots: views.map((view) => `${view.id}.png`), - viewFeatureCounts: Object.fromEntries(results.map((view) => [view.id, view.renderedFeatureCount])) -}, null, 2)); diff --git a/server-new-map-v2.mjs b/server-new-map-v2.mjs deleted file mode 100644 index 28375e9e..00000000 --- a/server-new-map-v2.mjs +++ /dev/null @@ -1,172 +0,0 @@ -import fs from 'node:fs/promises'; -import http from 'node:http'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const root = path.dirname(fileURLToPath(import.meta.url)); -const publicRoot = path.join(root, 'dist'); -const port = Number(process.env.PORT || 4173); -const host = process.env.HOST?.trim() || '0.0.0.0'; - -const contentTypes = { - '.css': 'text/css; charset=utf-8', - '.html': 'text/html; charset=utf-8', - '.ico': 'image/x-icon', - '.js': 'text/javascript; charset=utf-8', - '.json': 'application/json; charset=utf-8', - '.map': 'application/json; charset=utf-8', - '.png': 'image/png', - '.svg': 'image/svg+xml; charset=utf-8', - '.webp': 'image/webp', - '.woff': 'font/woff', - '.woff2': 'font/woff2' -}; - -const commonHeaders = { - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Methods': 'GET, HEAD, OPTIONS', - 'Access-Control-Allow-Headers': 'Content-Type, Cache-Control', - 'Cross-Origin-Opener-Policy': 'same-origin', - 'Cross-Origin-Resource-Policy': 'cross-origin', - 'Permissions-Policy': 'camera=(), microphone=(), geolocation=(), payment=(), usb=()', - 'Referrer-Policy': 'strict-origin-when-cross-origin', - 'X-Content-Type-Options': 'nosniff', - 'X-Frame-Options': 'SAMEORIGIN' -}; - -function write(response, status, body, contentType = 'text/plain; charset=utf-8', method = 'GET', cacheControl = 'no-store') { - const payload = Buffer.isBuffer(body) ? body : Buffer.from(String(body ?? '')); - response.writeHead(status, { - ...commonHeaders, - 'Cache-Control': cacheControl, - 'Content-Length': payload.byteLength, - 'Content-Type': contentType - }); - if (method === 'HEAD') response.end(); - else response.end(payload); -} - -function writeJson(response, status, value, method = 'GET') { - write(response, status, `${JSON.stringify(value)}\n`, contentTypes['.json'], method); -} - -function shouldServeSpa(request, pathname) { - return pathname === '/' || (!path.extname(pathname) && String(request.headers.accept || '').includes('text/html')); -} - -async function serveStatic(request, response, pathname) { - const requested = pathname === '/' ? '/index.html' : pathname; - let decoded; - try { - decoded = decodeURIComponent(requested); - } catch { - write(response, 400, 'Invalid URL encoding', undefined, request.method); - return; - } - - if (decoded.includes('\0')) { - write(response, 400, 'Invalid path', undefined, request.method); - return; - } - - const absolute = path.resolve(publicRoot, `.${decoded}`); - if (!absolute.startsWith(`${publicRoot}${path.sep}`) && absolute !== path.join(publicRoot, 'index.html')) { - write(response, 403, 'Forbidden', undefined, request.method); - return; - } - - try { - const stat = await fs.stat(absolute); - if (!stat.isFile()) throw new Error('Not a file'); - const extension = path.extname(absolute).toLowerCase(); - const body = await fs.readFile(absolute); - const immutable = decoded.startsWith('/assets/'); - write( - response, - 200, - body, - contentTypes[extension] || 'application/octet-stream', - request.method, - immutable ? 'public, max-age=31536000, immutable' : 'no-store, max-age=0' - ); - } catch { - if (!shouldServeSpa(request, decoded)) { - write(response, 404, 'Not found', undefined, request.method); - return; - } - try { - const index = await fs.readFile(path.join(publicRoot, 'index.html')); - write(response, 200, index, contentTypes['.html'], request.method, 'no-store, max-age=0'); - } catch { - write(response, 503, 'Application build unavailable', undefined, request.method); - } - } -} - -let shuttingDown = false; - -const server = http.createServer((request, response) => { - void (async () => { - const method = request.method || 'GET'; - if (method === 'OPTIONS') { - response.writeHead(204, { ...commonHeaders, 'Cache-Control': 'no-store' }); - response.end(); - return; - } - if (!['GET', 'HEAD'].includes(method)) { - write(response, 405, 'Method not allowed', undefined, method); - return; - } - if (shuttingDown) { - write(response, 503, 'Server is shutting down', undefined, method); - return; - } - - const url = new URL(request.url || '/', `http://${request.headers.host || 'localhost'}`); - if (url.pathname === '/health' || url.pathname === '/healthz') { - write(response, 200, 'ok', undefined, method); - return; - } - if (url.pathname === '/readyz') { - writeJson(response, 200, { - ready: true, - mode: 'clean-worldwide-vector-v2', - projection: 'mercator', - sourceCount: 1, - runtimeMerging: false, - neon: false, - regionalRouting: false, - shuttingDown - }, method); - return; - } - - await serveStatic(request, response, url.pathname); - })().catch((error) => { - console.error('Request failed:', error); - if (!response.headersSent) write(response, 500, 'Internal server error', undefined, request.method); - else response.destroy(); - }); -}); - -server.requestTimeout = 45_000; -server.headersTimeout = 20_000; -server.keepAliveTimeout = 5_000; -server.maxHeadersCount = 100; - -function shutdown(signal) { - if (shuttingDown) return; - shuttingDown = true; - console.log(`Map v2 server received ${signal}; draining connections.`); - server.close(() => process.exit(0)); - const timer = setTimeout(() => server.closeAllConnections?.(), 15_000); - timer.unref?.(); -} - -process.once('SIGTERM', () => shutdown('SIGTERM')); -process.once('SIGINT', () => shutdown('SIGINT')); - -server.listen(port, host, () => { - console.log(`Occu-Med clean worldwide map v2 listening on ${host}:${port}.`); - console.log('No PMTiles localization, Neon cache, regional routing, or runtime tile merging is active.'); -}); diff --git a/server.mjs b/server.mjs index e1d22297..70bb6855 100644 --- a/server.mjs +++ b/server.mjs @@ -3,26 +3,12 @@ import { createReadStream } from 'node:fs'; import fs from 'node:fs/promises'; import http from 'node:http'; import path from 'node:path'; -import { promisify } from 'node:util'; import { fileURLToPath } from 'node:url'; -import { gzip } from 'node:zlib'; -import { createNeonNavigationTileCacheFromEnv } from './src/server/neon-navigation-tile-cache.js'; -import { - GatewayOverloadedError, - WorldTileGateway -} from './src/server/world-tile-gateway.js'; -import { normalizeTileCoordinates } from './src/server/world-tile-routing.js'; -import { validateVectorTilePayload } from './src/server/tile-safety.js'; - -const gzipAsync = promisify(gzip); +import { ImmutableWorldTileset } from './src/server/immutable-world-tileset.js'; + const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), 'dist'); const port = Number(process.env.PORT || 4173); const host = process.env.HOST?.trim() || '0.0.0.0'; -const worldReleaseRepository = process.env.OCCUMED_WORLD_RELEASE_REPOSITORY?.trim() || 'Occumed79/Map'; -const worldReleaseTag = process.env.OCCUMED_WORLD_RELEASE_TAG?.trim() || 'occumed-world-v1'; -const worldManifestAsset = 'world-virtual-manifest.json'; -const worldSurfaceAsset = 'occumed-world-surface.pmtiles'; -const worldSurfaceUrl = process.env.OCCUMED_WORLD_SURFACE_URL?.trim(); const maxConcurrentTileRequests = Number(process.env.OCCUMED_MAX_CONCURRENT_TILE_REQUESTS || 64); const tileRequestTimeoutMs = Number(process.env.OCCUMED_TILE_REQUEST_TIMEOUT_MS || 30_000); const maxResolvedTileBytes = Number(process.env.OCCUMED_MAX_RESOLVED_TILE_BYTES || 24 * 1024 * 1024); @@ -137,7 +123,7 @@ function sendHealth(request, response) { async function sendReadiness(request, response) { try { const ready = await Promise.race([ - worldTileGateway.ready(), + immutableWorldTileset.ready(), new Promise((_, reject) => { const timer = setTimeout(() => reject(new Error('Readiness check timed out.')), 10_000); timer.unref?.(); @@ -167,19 +153,9 @@ async function serveStyle(request, response) { ); } -function releaseAssetUrl(assetName) { - if (assetName === worldSurfaceAsset && worldSurfaceUrl) return worldSurfaceUrl; - return `https://github.com/${worldReleaseRepository}/releases/download/${encodeURIComponent(worldReleaseTag)}/${encodeURIComponent(assetName)}`; -} - -const navigationTileCache = createNeonNavigationTileCacheFromEnv(); -const worldTileGateway = new WorldTileGateway({ - manifestUrl: - process.env.OCCUMED_WORLD_MANIFEST_URL?.trim() || - releaseAssetUrl(worldManifestAsset), - releaseAssetUrl, - persistentTileCache: navigationTileCache, - maxResolvedTileBytes: safeInteger( +const immutableWorldTileset = new ImmutableWorldTileset({ + root: path.resolve(path.dirname(fileURLToPath(import.meta.url))), + maxTileBytes: safeInteger( maxResolvedTileBytes, 24 * 1024 * 1024, 1_024, @@ -206,37 +182,45 @@ function tileEtag(tile) { async function serveVirtualTile(request, response, coordinates) { const concurrencyLimit = safeInteger(maxConcurrentTileRequests, 64, 4, 512); if (activeTileRequests >= concurrencyLimit) { - throw new GatewayOverloadedError('The HTTP tile concurrency limit has been reached.'); + const error = new Error('The HTTP tile concurrency limit has been reached.'); + error.code = 'OCCUMED_TILE_SERVER_OVERLOADED'; + error.statusCode = 503; + throw error; } activeTileRequests += 1; const startedAt = performance.now(); try { - const tile = await Promise.race([ - worldTileGateway.resolveTile(coordinates.z, coordinates.x, coordinates.y), + const resolved = await Promise.race([ + immutableWorldTileset.resolveTile(coordinates.z, coordinates.x, coordinates.y), timeoutAfter( safeInteger(tileRequestTimeoutMs, 30_000, 1_000, 120_000), `Tile ${coordinates.z}/${coordinates.x}/${coordinates.y} timed out.` ) ]); - validateVectorTilePayload(tile, { - label: `resolved tile ${coordinates.z}/${coordinates.x}/${coordinates.y}`, - maxBytes: safeInteger(maxResolvedTileBytes, 24 * 1024 * 1024, 1_024, 96 * 1024 * 1024) - }); + const tile = resolved.data; + if (!tile) { + writeHeaders(response, 204, { + 'Cache-Control': 'public, max-age=3600, must-revalidate, stale-if-error=86400', + 'X-Occumed-Tileset': resolved.artifactVersion, + 'X-Occumed-Tile-Owner': resolved.owner.id + }); + response.end(); + return; + } const etag = tileEtag(tile); if (request.headers['if-none-match'] === etag) { writeHeaders(response, 304, { 'Cache-Control': 'public, max-age=300, must-revalidate, stale-if-error=86400', ETag: etag, - 'X-Occumed-Tileset': 'virtual-worldwide-v2' + 'X-Occumed-Tileset': resolved.artifactVersion, + 'X-Occumed-Tile-Owner': resolved.owner.id }); response.end(); return; } - const acceptsGzip = /(?:^|,)\s*gzip\s*(?:,|$)/i.test(String(request.headers['accept-encoding'] || '')); - const payload = acceptsGzip ? await gzipAsync(tile, { level: 5 }) : tile; const cacheControl = 'public, max-age=300, must-revalidate, stale-while-revalidate=60, stale-if-error=86400'; const duration = Math.max(0, performance.now() - startedAt); @@ -244,16 +228,21 @@ async function serveVirtualTile(request, response, coordinates) { 'Cache-Control': cacheControl, 'CDN-Cache-Control': cacheControl, 'Surrogate-Control': cacheControl, - ...(acceptsGzip ? { 'Content-Encoding': 'gzip' } : {}), - 'Content-Length': payload.byteLength, + 'Content-Length': tile.byteLength, 'Content-Type': contentTypes['.pbf'], + ...(resolved.contentEncoding + ? { + 'Content-Encoding': resolved.contentEncoding, + Vary: 'Accept-Encoding' + } + : {}), ETag: etag, 'Server-Timing': `tile;dur=${duration.toFixed(1)}`, - Vary: 'Accept-Encoding', - 'X-Occumed-Tileset': 'virtual-worldwide-v2' + 'X-Occumed-Tileset': resolved.artifactVersion, + 'X-Occumed-Tile-Owner': resolved.owner.id }); if (request.method === 'HEAD') response.end(); - else response.end(payload); + else response.end(tile); } finally { activeTileRequests -= 1; } @@ -416,7 +405,6 @@ async function serveStatic(request, response, pathname) { } function tileErrorStatus(error) { - if (error instanceof GatewayOverloadedError) return 503; if (error instanceof RangeError) return 404; if (Number.isSafeInteger(error?.statusCode)) return error.statusCode; if (String(error?.code || '').startsWith('OCCUMED_')) return 503; @@ -456,10 +444,8 @@ async function handleRequest(request, response) { return; } if (url.pathname === '/internal/tile-health' && diagnosticsEnabled) { - sendJson(response, 200, { - activeTileRequests, - ...worldTileGateway.getHealthSnapshot() - }, 'no-store', method); + const ready = await immutableWorldTileset.ready().catch(() => ({ ready: false })); + sendJson(response, 200, { activeTileRequests, ...ready }, 'no-store', method); return; } if (url.pathname === '/style/occumed-open.json') { @@ -469,8 +455,23 @@ async function handleRequest(request, response) { const tileMatch = /^\/tiles\/(\d+)\/(\d+)\/(\d+)\.pbf$/.exec(url.pathname); if (tileMatch) { - const coordinates = normalizeTileCoordinates(tileMatch[1], tileMatch[2], tileMatch[3]); - if (!coordinates) { + const coordinates = { + z: Number(tileMatch[1]), + x: Number(tileMatch[2]), + y: Number(tileMatch[3]) + }; + const width = 2 ** coordinates.z; + if ( + !Number.isSafeInteger(coordinates.z) || + !Number.isSafeInteger(coordinates.x) || + !Number.isSafeInteger(coordinates.y) || + coordinates.z < 0 || + coordinates.z > 16 || + coordinates.x < 0 || + coordinates.y < 0 || + coordinates.x >= width || + coordinates.y >= width + ) { send(response, 404, 'Tile not found', 'text/plain; charset=utf-8', 'no-store', method); return; } @@ -558,7 +559,7 @@ function shutdown(signal) { shuttingDown = true; console.log(`Occu-Med Map received ${signal}; draining connections.`); server.close((error) => { - void worldTileGateway.close().finally(() => { + void immutableWorldTileset.close().finally(() => { if (error) { console.error('Occu-Med Map shutdown error:', error); process.exitCode = 1; @@ -567,7 +568,7 @@ function shutdown(signal) { }); const timer = setTimeout(() => { server.closeAllConnections?.(); - void worldTileGateway.close(); + void immutableWorldTileset.close(); process.exitCode = 1; }, 25_000); timer.unref?.(); @@ -579,16 +580,11 @@ process.once('SIGINT', () => shutdown('SIGINT')); server.listen(port, host, () => { console.log(`Occu-Med Map listening on ${host}:${port}.`); console.log(`Health endpoint ready at http://127.0.0.1:${port}/health.`); - if (navigationTileCache) { - const snapshot = navigationTileCache.snapshot(); - console.log(`Neon navigation cache configured with ${snapshot.configuredShards} of ${snapshot.expectedShards} shards.`); - void worldTileGateway.initializePersistentCache().then( - (initialized) => console.log(`Neon navigation cache initialized ${initialized} shard(s).`), - (error) => console.error(`Neon navigation cache initialization failed: ${error?.code || error?.name || 'UNKNOWN'}`) - ); - } - void worldTileGateway.ready().then( - (ready) => console.log(`Worldwide gateway ready with ${ready.regions} regional shards.`), - (error) => console.error('Worldwide gateway readiness failed:', error) + void immutableWorldTileset.ready().then( + (ready) => console.log( + `Immutable worldwide tileset ${ready.artifactVersion} ready ` + + `(${ready.builtOwnerCount}/${ready.plannedOwnerCount} owners).` + ), + (error) => console.error('Immutable worldwide tileset readiness failed:', error) ); }); diff --git a/src/flat-overview.css b/src/flat-overview.css deleted file mode 100644 index 83e920e4..00000000 --- a/src/flat-overview.css +++ /dev/null @@ -1,3 +0,0 @@ -.occumed-atmosphere-bloom { - display: none !important; -} diff --git a/src/main.js b/src/main.js index 69999a34..96c1347d 100644 --- a/src/main.js +++ b/src/main.js @@ -1,12 +1,50 @@ -import { startOccumedMapV2 } from './new-map-v2.js'; -import './new-map-v2.css'; +import { createOccumedMap } from './occumed-map.js'; +import './styles.css'; const statusElement = document.querySelector('#map-status'); +let mapReady = false; -startOccumedMapV2().catch((error) => { +function markReady(message) { + if (mapReady) return; + mapReady = true; + if (statusElement) statusElement.textContent = message; + document.documentElement.classList.add('map-is-ready'); +} + +function markUnavailable(error) { const message = error instanceof Error ? error.message : String(error); - console.error('Occu-Med map v2 startup failure:', message); + console.error('Occu-Med basemap startup failure:', message); if (statusElement) statusElement.textContent = 'Map unavailable'; - document.documentElement.dataset.mapState = 'error'; document.documentElement.classList.remove('map-is-ready'); -}); +} + +async function initialize() { + try { + const map = await createOccumedMap({ + container: 'map', + styleUrl: import.meta.env.VITE_OCCUMED_STYLE_URL || '/style/occumed-open.json' + }); + + // Exposed only as a stable integration and visual-QA hook. Applications still + // own their overlays and should use createOccumedMap directly. + globalThis.__OCCUMED_MAP__ = map; + + map.once('render', () => markReady('Occu-Med map ready')); + map.once('load', () => markReady('Occu-Med map ready')); + map.once('idle', () => markReady('Occu-Med map ready')); + + map.on('error', (event) => { + console.warn('Occu-Med basemap resource warning:', { + message: event?.error?.message || event?.message || '', + sourceId: event?.sourceId || null, + sourceDataType: event?.sourceDataType || null, + dataType: event?.dataType || null, + tile: event?.tile || null + }); + }); + } catch (error) { + markUnavailable(error); + } +} + +initialize(); diff --git a/src/new-map-v2.css b/src/new-map-v2.css deleted file mode 100644 index f3a63e9a..00000000 --- a/src/new-map-v2.css +++ /dev/null @@ -1,109 +0,0 @@ -:root { - color-scheme: light; - font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; - background: #79BCEC; -} - -* { - box-sizing: border-box; -} - -html, -body, -#app, -#map { - width: 100%; - height: 100%; - margin: 0; - overflow: hidden; -} - -body { - background: #79BCEC; -} - -#map { - position: absolute; - inset: 0; - background: #79BCEC; -} - -.map-header { - position: fixed; - top: 16px; - left: 16px; - z-index: 10; - display: flex; - align-items: center; - gap: 10px; - min-height: 36px; - padding: 8px 12px; - border: 1px solid rgba(255, 255, 255, 0.58); - border-radius: 12px; - background: rgba(255, 255, 255, 0.82); - box-shadow: 0 10px 30px rgba(31, 72, 100, 0.14); - color: #27313A; - backdrop-filter: blur(14px); - -webkit-backdrop-filter: blur(14px); - pointer-events: none; -} - -.eyebrow { - font-size: 11px; - font-weight: 800; - letter-spacing: 0.14em; -} - -.status { - font-size: 12px; - font-weight: 600; - opacity: 0.8; -} - -html[data-map-state="ready"] .map-header { - opacity: 0; - transform: translateY(-8px); - transition: opacity 240ms ease, transform 240ms ease; -} - -.maplibregl-ctrl-top-right { - top: 8px; - right: 8px; -} - -.maplibregl-ctrl-group { - overflow: hidden; - border: 1px solid rgba(39, 49, 58, 0.16); - border-radius: 12px; - box-shadow: 0 8px 24px rgba(31, 72, 100, 0.15); -} - -.maplibregl-ctrl-group button { - width: 36px; - height: 36px; - background-color: rgba(255, 255, 255, 0.94); -} - -.maplibregl-ctrl-attrib { - border-radius: 8px 0 0 0; - background: rgba(255, 255, 255, 0.86) !important; - color: #27313A; -} - -.maplibregl-canvas { - outline: none; -} - -.occumed-atmosphere-bloom, -.maplibregl-globe, -.maplibregl-fog { - display: none !important; -} - -@media (max-width: 720px) { - .map-header { - top: 10px; - left: 10px; - max-width: calc(100vw - 68px); - } -} diff --git a/src/new-map-v2.js b/src/new-map-v2.js deleted file mode 100644 index 5175b0c4..00000000 --- a/src/new-map-v2.js +++ /dev/null @@ -1,243 +0,0 @@ -import maplibregl from 'maplibre-gl'; -import 'maplibre-gl/dist/maplibre-gl.css'; - -const STYLE_URL = 'https://tiles.openfreemap.org/styles/liberty'; -const WORLD_BOUNDS = [[-179.8, -78], [179.8, 82]]; -const PALETTE = Object.freeze({ - water: '#79BCEC', - waterDeep: '#5EA9DF', - land: '#C1DAAB', - landSoft: '#D4E3C1', - park: '#A5CC8E', - parkDark: '#91BD78', - road: '#F2F2F2', - roadCasing: '#D5D7D8', - boundary: '#A65966', - building: '#DDD8CC', - text: '#27313A', - waterText: '#286E99', - parkText: '#3D6D45', - halo: '#F5FDFF' -}); - -function setStatus(message, state = 'loading') { - const element = document.querySelector('#map-status'); - if (element) element.textContent = message; - document.documentElement.dataset.mapState = state; -} - -async function fetchStyle(attempts = 3) { - let lastError; - for (let attempt = 1; attempt <= attempts; attempt += 1) { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), 20_000); - try { - const response = await fetch(`${STYLE_URL}?occumed=${Date.now()}`, { - cache: 'no-store', - signal: controller.signal, - headers: { Accept: 'application/json' } - }); - if (!response.ok) throw new Error(`Worldwide style returned HTTP ${response.status}.`); - return await response.json(); - } catch (error) { - lastError = error; - if (attempt < attempts) await new Promise((resolve) => setTimeout(resolve, attempt * 800)); - } finally { - clearTimeout(timer); - } - } - throw lastError || new Error('Unable to load the worldwide vector style.'); -} - -function textKey(layer) { - return `${layer.id || ''} ${layer['source-layer'] || ''}`.toLowerCase(); -} - -function isWater(key) { - return /water|ocean|lake|river|marine|bay/.test(key); -} - -function isPark(key) { - return /park|grass|wood|forest|landcover|landuse|nature|green|garden|golf|pitch|cemetery/.test(key); -} - -function isRoad(key) { - return /road|street|transport|motorway|trunk|primary|secondary|tertiary|minor|path|bridge|tunnel|rail/.test(key); -} - -function isBoundary(key) { - return /boundary|admin|border/.test(key); -} - -function recolorLayer(layer) { - const next = structuredClone(layer); - const key = textKey(next); - next.paint = { ...(next.paint || {}) }; - - if (next.type === 'background') { - next.paint['background-color'] = PALETTE.land; - next.paint['background-opacity'] = 1; - return next; - } - - if (next.type === 'fill') { - if (isWater(key)) { - next.paint['fill-color'] = PALETTE.water; - next.paint['fill-opacity'] = 1; - } else if (isPark(key)) { - next.paint['fill-color'] = /wood|forest/.test(key) ? PALETTE.parkDark : PALETTE.park; - next.paint['fill-opacity'] = 0.86; - } else if (/building/.test(key)) { - next.paint['fill-color'] = PALETTE.building; - next.paint['fill-opacity'] = 0.9; - } else { - next.paint['fill-color'] = /residential|suburb|neighbourhood|industrial|commercial/.test(key) - ? PALETTE.landSoft - : PALETTE.land; - next.paint['fill-opacity'] = 1; - } - if ('fill-outline-color' in next.paint) next.paint['fill-outline-color'] = 'rgba(0,0,0,0.08)'; - return next; - } - - if (next.type === 'line') { - if (isBoundary(key)) { - next.paint['line-color'] = PALETTE.boundary; - next.paint['line-opacity'] = 0.82; - } else if (isWater(key)) { - next.paint['line-color'] = PALETTE.waterDeep; - next.paint['line-opacity'] = 0.92; - } else if (isRoad(key)) { - next.paint['line-color'] = /case|casing|outline/.test(key) ? PALETTE.roadCasing : PALETTE.road; - next.paint['line-opacity'] = 0.98; - } - return next; - } - - if (next.type === 'symbol') { - if (next.layout?.['text-field']) { - next.paint['text-color'] = isWater(key) ? PALETTE.waterText : isPark(key) ? PALETTE.parkText : PALETTE.text; - next.paint['text-halo-color'] = PALETTE.halo; - next.paint['text-halo-width'] = 1.2; - next.paint['text-halo-blur'] = 0.35; - } - return next; - } - - return next; -} - -function buildOccumedStyle(rawStyle) { - const style = structuredClone(rawStyle); - const vectorSources = Object.entries(style.sources || {}).filter(([, source]) => source?.type === 'vector'); - if (vectorSources.length !== 1) { - throw new Error(`The replacement map requires exactly one worldwide vector source; received ${vectorSources.length}.`); - } - - const [sourceId, source] = vectorSources[0]; - style.sources = { [sourceId]: source }; - style.layers = (style.layers || []) - .filter((layer) => !layer.source || layer.source === sourceId) - .filter((layer) => !['sky', 'hillshade', 'model', 'fill-extrusion'].includes(layer.type)) - .map(recolorLayer); - style.projection = { type: 'mercator' }; - delete style.terrain; - delete style.fog; - delete style.light; - style.metadata = { - ...(style.metadata || {}), - 'occumed:architecture': 'clean-worldwide-vector-v2', - 'occumed:projection': 'mercator', - 'occumed:source-count': 1, - 'occumed:runtime-merge': false, - 'occumed:regional-routing': false, - 'occumed:neon': false - }; - return { style, sourceId }; -} - -function waitForIdle(map, timeoutMs = 45_000) { - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => reject(new Error('The worldwide map did not become idle in time.')), timeoutMs); - const complete = () => { - if (!map.loaded() || !map.areTilesLoaded()) return; - clearTimeout(timeout); - map.off('idle', complete); - resolve(); - }; - map.on('idle', complete); - complete(); - }); -} - -export async function startOccumedMapV2() { - setStatus('Loading worldwide map…'); - const rawStyle = await fetchStyle(); - const { style, sourceId } = buildOccumedStyle(rawStyle); - const errors = []; - - const map = new maplibregl.Map({ - container: 'map', - style, - center: [0, 20], - zoom: 1.35, - minZoom: 1, - maxZoom: 18, - pitch: 0, - bearing: 0, - hash: false, - antialias: true, - renderWorldCopies: false, - refreshExpiredTiles: false, - fadeDuration: 120, - attributionControl: false, - cooperativeGestures: false, - maxTileCacheZoomLevels: 8 - }); - - map.addControl(new maplibregl.NavigationControl({ visualizePitch: false }), 'top-right'); - map.addControl(new maplibregl.AttributionControl({ compact: true }), 'bottom-right'); - - map.on('error', (event) => { - const message = event?.error?.message || event?.message || 'Unknown map error'; - errors.push(message); - console.warn('Occu-Med map resource warning:', message); - }); - - map.once('load', () => { - map.fitBounds(WORLD_BOUNDS, { padding: 18, duration: 0, maxZoom: 2.2 }); - }); - - globalThis.__OCCUMED_MAP__ = map; - globalThis.__OCCUMED_MAP_V2__ = { - ready: false, - sourceId, - sourceCount: Object.keys(style.sources).length, - projection: 'mercator', - architecture: 'clean-worldwide-vector-v2', - errors - }; - - await waitForIdle(map); - const renderedFeatures = map.queryRenderedFeatures(); - const sourceLoaded = map.isSourceLoaded(sourceId); - const sourceLayers = new Set(renderedFeatures.map((feature) => feature.sourceLayer).filter(Boolean)); - const contract = globalThis.__OCCUMED_MAP_V2__; - Object.assign(contract, { - ready: true, - sourceLoaded, - renderedFeatureCount: renderedFeatures.length, - renderedSourceLayers: [...sourceLayers].sort(), - zoom: map.getZoom(), - center: map.getCenter().toArray(), - tilesLoaded: map.areTilesLoaded() - }); - - if (!sourceLoaded || renderedFeatures.length < 25 || sourceLayers.size < 3) { - throw new Error('The replacement worldwide source loaded without enough visible map detail.'); - } - - setStatus('Occu-Med map ready', 'ready'); - document.documentElement.classList.add('map-is-ready'); - return map; -} diff --git a/src/occumed-map.js b/src/occumed-map.js index 96491093..4e81116f 100644 --- a/src/occumed-map.js +++ b/src/occumed-map.js @@ -11,7 +11,6 @@ const BLOOM_FADE_START_ZOOM = 2.85; const BLOOM_FADE_END_ZOOM = 4.25; const WORLD_MIN_ZOOM = 0; const WORLD_MAX_ZOOM = 16; -const WORLD_ZOOM_PYRAMID_LEVELS = WORLD_MAX_ZOOM - WORLD_MIN_ZOOM + 1; export function resolveOccumedPixelRatio() { const deviceRatio = Number(globalThis.devicePixelRatio); @@ -37,13 +36,8 @@ function resolveGlobeRadius(zoom) { } /** - * Adds a true outward atmosphere bloom around the globe limb. - * - * MapLibre's sky properties provide the crisp horizon rim, but increasing their - * blend values also brightens the visible hemisphere. This DOM halo tracks the - * rendered globe radius and adds only an exterior white-blue bloom, leaving the - * map surface neutral. It fades away before the projection reads as a regional - * map rather than a complete globe. + * Retained for backward-compatible imports only. Flat Mercator production does + * not install the globe atmosphere bloom. */ export function installOccumedAtmosphereBloom(map) { const canvasContainer = map.getCanvasContainer(); @@ -86,95 +80,39 @@ export function installOccumedAtmosphereBloom(map) { } /** - * Keeps decoded substitute tiles renderable across the complete 0–16 pyramid. + * Enforces exact prebuilt z/x/y addressing. * - * MapLibre's normal retention only searches in-view tiles while an ideal tile - * is loading. Fully decoded parents and children in its out-of-view cache are - * consequently skipped, leaving no renderable tile until the ideal request is - * parsed. Keep the same-source global foundation decoded as a last resort and - * reattach the nearest cached substitute before cleanup removes it. + * The immutable tileset contains every production zoom, so a missing ideal tile + * must remain missing rather than being painted from a stretched parent or + * child. This changes MapLibre's fallback depth only; it does not replace or + * wrap its retained-tile algorithm. */ -export function installContinuousTileRetention(map) { - let removed = false; - +export function installExactTileAddressing(map) { const configure = () => { - if (removed) return; const tileManager = map.style?.tileManagers?.['occumed-open']; if (!tileManager?.constructor) return; - tileManager.constructor.maxUnderzooming = Math.max( - Number(tileManager.constructor.maxUnderzooming || 0), - WORLD_ZOOM_PYRAMID_LEVELS - ); - tileManager.constructor.maxOverzooming = Math.max( - Number(tileManager.constructor.maxOverzooming || 0), - WORLD_ZOOM_PYRAMID_LEVELS - ); - if (tileManager.__occumedContinuousRetention) return; - - const updateRetainedTiles = tileManager._updateRetainedTiles.bind(tileManager); - let globalFoundationID = null; - tileManager._updateRetainedTiles = function retainCachedFoundation(idealTileIDs, zoom) { - const retained = updateRetainedTiles(idealTileIDs, zoom); - if (idealTileIDs.length && !globalFoundationID) { - globalFoundationID = idealTileIDs[0].scaledTo(0); - } - if (globalFoundationID) { - this._addTile(globalFoundationID); - retained[globalFoundationID.key] = globalFoundationID; - } - - for (const idealID of idealTileIDs) { - if (this.getTileByID(idealID.key)?.hasData()) continue; - - let foundAncestor = false; - for (let parentZoom = idealID.overscaledZ - 1; parentZoom >= 0; parentZoom -= 1) { - const parentID = idealID.scaledTo(parentZoom); - let parent = this.getTileByID(parentID.key); - if (!parent && this._outOfViewCache.has(parentID)) { - parent = this._addTile(parentID); - } - if (parent?.hasData()) { - retained[parentID.key] = parentID; - foundAncestor = true; - break; - } - } - if (foundAncestor) continue; - - const cachedChildren = Object.values(this._outOfViewCache.data) - .flat() - .map(({ value }) => value) - .filter((tile) => - tile.hasData() && - tile.tileID.isChildOf(idealID) && - tile.tileID.overscaledZ - idealID.overscaledZ <= WORLD_ZOOM_PYRAMID_LEVELS - ) - .map((tile) => tile.tileID.clone()); - if (!cachedChildren.length) continue; - - const nearestZoom = Math.min(...cachedChildren.map((tileID) => tileID.overscaledZ)); - for (const childID of cachedChildren) { - if (childID.overscaledZ !== nearestZoom) continue; - const child = this._addTile(childID); - if (child.hasData()) retained[childID.key] = childID; - } - } - - return retained; - }; - tileManager.__occumedContinuousRetention = true; + tileManager.constructor.maxUnderzooming = 0; + tileManager.constructor.maxOverzooming = 0; }; - - const remove = () => { - removed = true; - map.off('styledata', configure); - }; - map.on('styledata', configure); - map.once('remove', remove); + map.once('remove', () => map.off('styledata', configure)); configure(); } +export function installAdaptiveFlatWorldWrap(map) { + const sync = () => { + if (typeof map.setRenderWorldCopies !== 'function') return; + map.setRenderWorldCopies(map.getZoom() >= 3); + }; + map.on('zoom', sync); + map.on('zoomend', sync); + map.once('remove', () => { + map.off('zoom', sync); + map.off('zoomend', sync); + }); + sync(); +} + function resolvePublicOrigin(style, styleUrl) { const resolved = structuredClone(style); const styleOrigin = new URL(styleUrl, window.location.href).origin; @@ -229,8 +167,8 @@ export async function loadOccumedStyle(styleUrl = DEFAULT_STYLE_URL) { export async function createOccumedMap({ container, styleUrl = DEFAULT_STYLE_URL, - center = [-98.5, 25], - zoom = 2.43, + center = [0, 20], + zoom = 1.25, minZoom = WORLD_MIN_ZOOM, maxZoom = WORLD_MAX_ZOOM, controls = true, @@ -254,11 +192,7 @@ export async function createOccumedMap({ hash: false, pixelRatio: resolveOccumedPixelRatio(), antialias: true, - // MapLibre only retains pending smaller-zoom requests during zoom-in when - // cancellation is disabled. Reverse zooms also require the already-loaded - // parent pyramid to remain in cache, so retain every zoom level from 0–16. - cancelPendingTileRequestsWhileZooming: false, - maxTileCacheZoomLevels: WORLD_ZOOM_PYRAMID_LEVELS, + cancelPendingTileRequestsWhileZooming: true, refreshExpiredTiles: false, fadeDuration: 300, renderWorldCopies: false, @@ -267,11 +201,11 @@ export async function createOccumedMap({ ...mapOptions }); - installContinuousTileRetention(map); - installOccumedAtmosphereBloom(map); + installExactTileAddressing(map); + installAdaptiveFlatWorldWrap(map); if (controls) { - map.addControl(new maplibregl.NavigationControl({ visualizePitch: true }), 'top-right'); + map.addControl(new maplibregl.NavigationControl({ visualizePitch: false }), 'top-right'); } if (scaleControl) { diff --git a/src/server/immutable-world-tileset.js b/src/server/immutable-world-tileset.js new file mode 100644 index 00000000..f32d7739 --- /dev/null +++ b/src/server/immutable-world-tileset.js @@ -0,0 +1,504 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + Compression, + FetchSource, + findTile, + PMTiles, + SharedPromiseCache, + zxyToTileId +} from 'pmtiles'; + +const DEFAULT_MANIFEST = 'dist/immutable-world/manifest.json'; +const MANIFEST_SCHEMA_VERSION = 1; +const MAX_MANIFEST_BYTES = 8 * 1024 * 1024; +const MAX_OWNER_COUNT = 16_384; + +class LocalArchiveSource { + constructor(filename) { + this.filename = path.resolve(filename); + this.handlePromise = fs.open(this.filename, 'r'); + this.sizePromise = this.handlePromise.then((handle) => handle.stat()).then((stat) => stat.size); + this.closed = false; + } + + getKey() { + return this.filename; + } + + async getBytes(offset, length) { + if (this.closed) throw new Error(`Immutable PMTiles source is closed: ${this.filename}`); + const start = Number(offset); + const requested = Number(length); + const [handle, fileSize] = await Promise.all([this.handlePromise, this.sizePromise]); + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(requested) || + start < 0 || + requested <= 0 || + start >= fileSize + ) { + throw new RangeError(`Invalid immutable PMTiles range ${offset}+${length}.`); + } + const size = Math.min(requested, fileSize - start); + const buffer = Buffer.allocUnsafe(size); + const { bytesRead } = await handle.read(buffer, 0, size, start); + if (bytesRead !== size) { + throw new Error( + `Short immutable PMTiles read from ${this.filename}: expected ${size}, received ${bytesRead}.` + ); + } + return { + data: buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) + }; + } + + async close() { + if (this.closed) return; + this.closed = true; + await (await this.handlePromise).close(); + } +} + +function isHttpUrl(value) { + try { + const url = new URL(value); + return ['http:', 'https:'].includes(url.protocol); + } catch { + return false; + } +} + +function normalizeCoordinate(value, label) { + const number = Number(value); + if (!Number.isSafeInteger(number) || number < 0) { + throw new TypeError(`${label} must be a non-negative safe integer.`); + } + return number; +} + +function normalizeTile(zValue, xValue, yValue, maxZoom) { + const z = normalizeCoordinate(zValue, 'Tile z'); + const x = normalizeCoordinate(xValue, 'Tile x'); + const y = normalizeCoordinate(yValue, 'Tile y'); + if (z > maxZoom) return null; + const width = 2 ** z; + if (x >= width || y >= width) return null; + return { z, x, y }; +} + +function canonicalDocument(value) { + if (Array.isArray(value)) return value.map(canonicalDocument); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries( + Object.keys(value) + .sort() + .filter((key) => !['artifactVersion', 'generatedAt'].includes(key)) + .map((key) => [key, canonicalDocument(value[key])]) + ); +} + +export function computeImmutableArtifactVersion(manifest) { + return createHash('sha256') + .update(`${JSON.stringify(canonicalDocument(manifest))}\n`) + .digest('hex'); +} + +function validateSha(value, label) { + if (!/^[a-f0-9]{64}$/.test(String(value || ''))) { + throw new TypeError(`${label} must be a lowercase SHA-256 digest.`); + } +} + +function validateAsset(asset, label) { + if (!asset || typeof asset !== 'object') throw new TypeError(`${label} is missing.`); + if (!/^[a-z0-9][a-z0-9._/-]*\.pmtiles$/.test(String(asset.file || ''))) { + throw new TypeError(`${label}.file is not a safe PMTiles path.`); + } + if (String(asset.file).includes('..') || path.isAbsolute(String(asset.file))) { + throw new TypeError(`${label}.file must stay beneath the immutable asset root.`); + } + if (!Number.isSafeInteger(asset.bytes) || asset.bytes < 127) { + throw new TypeError(`${label}.bytes is invalid.`); + } + validateSha(asset.sha256, `${label}.sha256`); +} + +function prefixContains(ancestor, candidate) { + if (ancestor.z > candidate.z) return false; + const divisor = 2 ** (candidate.z - ancestor.z); + return ( + Math.floor(candidate.x / divisor) === ancestor.x && + Math.floor(candidate.y / divisor) === ancestor.y + ); +} + +function normalizePrefix(prefix, label) { + const z = normalizeCoordinate(prefix?.z, `${label}.z`); + const x = normalizeCoordinate(prefix?.x, `${label}.x`); + const y = normalizeCoordinate(prefix?.y, `${label}.y`); + const width = 2 ** z; + if (z > 16 || x >= width || y >= width) { + throw new TypeError(`${label} is outside the z0-z16 tile pyramid.`); + } + return { z, x, y }; +} + +export function validateImmutableManifest(document, { + allowPartial = false +} = {}) { + if (!document || typeof document !== 'object') throw new TypeError('Immutable tileset manifest is missing.'); + if (document.schemaVersion !== MANIFEST_SCHEMA_VERSION) { + throw new TypeError(`Unsupported immutable manifest schema: ${document.schemaVersion}`); + } + if (document.browserSourceId !== 'occumed-open') { + throw new TypeError('Immutable manifest browser source must remain occumed-open.'); + } + if (document.maxZoom !== 16 || document.minZoom !== 0) { + throw new TypeError('Immutable manifest must cover the z0-z16 pyramid.'); + } + if ( + document.authorities?.land !== 'world-surface' || + document.authorities?.depth !== 'world-surface' || + document.authorities?.landcover !== 'world-overview' || + document.authorities?.cartography !== 'regional-owner' + ) { + throw new TypeError('Immutable manifest has an invalid source authority matrix.'); + } + for (const forbidden of [ + 'neonTileCache', + 'runtimeShardMerge', + 'runtimeLandcoverSynthesis', + 'runtimeGeometry', + 'parentChildStretching' + ]) { + if (document.runtimePolicy?.[forbidden] !== false) { + throw new TypeError(`Immutable manifest does not disable ${forbidden}.`); + } + } + + validateAsset(document.foundation, 'foundation'); + if (!Number.isSafeInteger(document.foundation.maxZoom) || + document.foundation.maxZoom < 0 || + document.foundation.maxZoom > 16) { + throw new TypeError('foundation.maxZoom is invalid.'); + } + const owners = Array.isArray(document.owners) ? document.owners : []; + if (owners.length > MAX_OWNER_COUNT) throw new TypeError('Immutable owner count is unsafe.'); + if (!Number.isSafeInteger(document.plannedOwnerCount) || document.plannedOwnerCount < 0) { + throw new TypeError('plannedOwnerCount is invalid.'); + } + if (!Number.isSafeInteger(document.builtOwnerCount) || + document.builtOwnerCount !== owners.length) { + throw new TypeError('builtOwnerCount does not match the owner inventory.'); + } + if (!document.complete && !allowPartial) { + throw new Error( + `Immutable tileset is incomplete: ${owners.length} of ${document.plannedOwnerCount} owners.` + ); + } + if (document.complete && owners.length !== document.plannedOwnerCount) { + throw new Error('A complete immutable tileset must include every planned owner.'); + } + if (document.defaultOwner !== document.foundation.id) { + throw new TypeError('The worldwide default owner must be the prebuilt foundation.'); + } + + const ids = new Set([document.foundation.id]); + const files = new Set([document.foundation.file]); + const prefixes = []; + const exactTileOwners = new Map(); + const normalizedOwners = owners.map((owner, index) => { + validateAsset(owner, `owners[${index}]`); + if (!/^[a-z0-9][a-z0-9-]*$/.test(String(owner.id || ''))) { + throw new TypeError(`owners[${index}].id is invalid.`); + } + if (ids.has(owner.id)) throw new TypeError(`Duplicate immutable owner id: ${owner.id}`); + if (files.has(owner.file)) throw new TypeError(`Duplicate immutable owner file: ${owner.file}`); + ids.add(owner.id); + files.add(owner.file); + const prefix = normalizePrefix(owner.prefix, `owners[${index}].prefix`); + for (const existing of prefixes) { + if (prefixContains(existing, prefix) || prefixContains(prefix, existing)) { + throw new TypeError( + `Overlapping immutable owner prefixes: ${existing.z}/${existing.x}/${existing.y} and ` + + `${prefix.z}/${prefix.x}/${prefix.y}.` + ); + } + } + prefixes.push(prefix); + const exactTiles = (owner.exactTiles || []).map((tile, tileIndex) => { + const exact = normalizePrefix(tile, `owners[${index}].exactTiles[${tileIndex}]`); + if (exact.z <= document.foundation.maxZoom) { + throw new TypeError( + `Owner exact tile ${exact.z}/${exact.x}/${exact.y} overlaps the foundation.` + ); + } + const exactKey = `${exact.z}/${exact.x}/${exact.y}`; + if (exactTileOwners.has(exactKey)) { + throw new TypeError( + `Exact tile ${exactKey} is assigned to both ${exactTileOwners.get(exactKey)} and ${owner.id}.` + ); + } + exactTileOwners.set(exactKey, owner.id); + return exact; + }); + return { ...owner, prefix, exactTiles }; + }); + for (const [exactKey, exactOwner] of exactTileOwners) { + const [z, x, y] = exactKey.split('/').map(Number); + const matchingPrefixes = normalizedOwners.filter((owner) => + prefixContains(owner.prefix, { z, x, y }) + ); + if (matchingPrefixes.length) { + throw new TypeError( + `Exact tile ${exactKey} for ${exactOwner} overlaps prefix owner ` + + `${matchingPrefixes[0].id}.` + ); + } + } + + const computedVersion = computeImmutableArtifactVersion(document); + if (document.artifactVersion !== computedVersion) { + throw new Error( + `Immutable artifact version mismatch: expected ${computedVersion}, ` + + `received ${document.artifactVersion}.` + ); + } + return { + ...document, + owners: normalizedOwners + }; +} + +async function readManifest(location) { + if (isHttpUrl(location)) { + const response = await fetch(location, { + redirect: 'follow', + headers: { 'User-Agent': 'Occu-Med-Map/immutable-tileset' } + }); + if (!response.ok) throw new Error(`Immutable manifest returned HTTP ${response.status}.`); + const contentLength = Number(response.headers.get('content-length')); + if (Number.isFinite(contentLength) && contentLength > MAX_MANIFEST_BYTES) { + throw new Error('Immutable manifest exceeds the size budget.'); + } + const text = await response.text(); + if (Buffer.byteLength(text) > MAX_MANIFEST_BYTES) { + throw new Error('Immutable manifest exceeds the size budget.'); + } + return JSON.parse(text); + } + + const stat = await fs.stat(location); + if (!stat.isFile() || stat.size > MAX_MANIFEST_BYTES) { + throw new Error('Immutable manifest is missing or exceeds the size budget.'); + } + return JSON.parse(await fs.readFile(location, 'utf8')); +} + +function resolveRemoteAsset(baseUrl, file) { + const base = new URL(baseUrl); + if (!['http:', 'https:'].includes(base.protocol)) { + throw new TypeError('Immutable assetBaseUrl must use HTTP or HTTPS.'); + } + return new URL(file, base.href.endsWith('/') ? base.href : `${base.href}/`).href; +} + +function buildOwnerIndex(manifest) { + const exact = new Map(); + const prefixes = new Map(); + for (const owner of manifest.owners) { + for (const tile of owner.exactTiles) { + exact.set(`${tile.z}/${tile.x}/${tile.y}`, owner); + } + const zoomOwners = prefixes.get(owner.prefix.z) || new Map(); + zoomOwners.set(`${owner.prefix.x}/${owner.prefix.y}`, owner); + prefixes.set(owner.prefix.z, zoomOwners); + } + return { + exact, + prefixes, + prefixZooms: [...prefixes.keys()].sort((left, right) => right - left) + }; +} + +export class ImmutableWorldTileset { + constructor({ + root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'), + manifestLocation = process.env.OCCUMED_IMMUTABLE_TILESET_MANIFEST?.trim(), + allowPartial = process.env.OCCUMED_ALLOW_PARTIAL_TILESET_FIXTURE === 'true', + maxTileBytes = 32 * 1024 * 1024 + } = {}) { + this.root = path.resolve(root); + this.manifestLocation = manifestLocation + ? (isHttpUrl(manifestLocation) ? manifestLocation : path.resolve(manifestLocation)) + : path.join(this.root, DEFAULT_MANIFEST); + this.allowPartial = allowPartial; + this.maxTileBytes = maxTileBytes; + this.manifestPromise = null; + this.ownerIndex = null; + this.archives = new Map(); + this.closed = false; + } + + async loadManifest() { + if (this.closed) throw new Error('Immutable tileset is closed.'); + if (!this.manifestPromise) { + this.manifestPromise = readManifest(this.manifestLocation) + .then((document) => validateImmutableManifest(document, { + allowPartial: this.allowPartial + })) + .then((manifest) => { + this.ownerIndex = buildOwnerIndex(manifest); + return manifest; + }) + .catch((error) => { + this.manifestPromise = null; + this.ownerIndex = null; + throw error; + }); + } + return this.manifestPromise; + } + + assetLocation(manifest, asset) { + if (manifest.assetBaseUrl) return resolveRemoteAsset(manifest.assetBaseUrl, asset.file); + if (isHttpUrl(this.manifestLocation)) { + return new URL(asset.file, this.manifestLocation).href; + } + const assetRoot = path.dirname(this.manifestLocation); + const resolved = path.resolve(assetRoot, asset.file); + if (!resolved.startsWith(`${assetRoot}${path.sep}`)) { + throw new Error(`Immutable asset escapes its manifest directory: ${asset.file}`); + } + return resolved; + } + + async openArchive(manifest, asset) { + const location = this.assetLocation(manifest, asset); + if (this.archives.has(location)) return this.archives.get(location); + const source = isHttpUrl(location) + ? new FetchSource(location) + : new LocalArchiveSource(location); + const archive = new PMTiles(source, new SharedPromiseCache(256)); + const opened = Promise.all([ + archive.getHeader(), + isHttpUrl(location) ? Promise.resolve(null) : fs.stat(location) + ]).then(([header, stat]) => { + if (header.specVersion !== 3 || header.tileType !== 1) { + throw new Error(`Immutable asset is not PMTiles v3 MVT: ${asset.file}`); + } + if (stat && stat.size !== asset.bytes) { + throw new Error( + `Immutable asset size mismatch for ${asset.file}: expected ${asset.bytes}, received ${stat.size}.` + ); + } + return { archive, source, header, asset, location }; + }); + this.archives.set(location, opened); + return opened; + } + + selectOwner(manifest, tile) { + if (tile.z <= manifest.foundation.maxZoom) return manifest.foundation; + if (!this.ownerIndex) throw new Error('Immutable owner index is not initialized.'); + const exact = this.ownerIndex.exact.get(`${tile.z}/${tile.x}/${tile.y}`); + if (exact) return exact; + for (const prefixZoom of this.ownerIndex.prefixZooms) { + if (prefixZoom > tile.z) continue; + const divisor = 2 ** (tile.z - prefixZoom); + const owner = this.ownerIndex.prefixes.get(prefixZoom).get( + `${Math.floor(tile.x / divisor)}/${Math.floor(tile.y / divisor)}` + ); + if (owner) return owner; + } + return manifest.foundation; + } + + async resolveTile(zValue, xValue, yValue) { + const manifest = await this.loadManifest(); + const tile = normalizeTile(zValue, xValue, yValue, manifest.maxZoom); + if (!tile) throw new RangeError('Invalid immutable tile coordinates.'); + const owner = this.selectOwner(manifest, tile); + const opened = await this.openArchive(manifest, owner); + const tileId = zxyToTileId(tile.z, tile.x, tile.y); + let directoryOffset = opened.header.rootDirectoryOffset; + let directoryLength = opened.header.rootDirectoryLength; + let data = null; + for (let depth = 0; depth <= 3; depth += 1) { + const directory = await opened.archive.cache.getDirectory( + opened.source, + directoryOffset, + directoryLength, + opened.header + ); + const entry = findTile(directory, tileId); + if (!entry) break; + if (entry.runLength > 0) { + const stored = await opened.source.getBytes( + opened.header.tileDataOffset + entry.offset, + entry.length + ); + data = Buffer.from(stored.data); + break; + } + directoryOffset = opened.header.leafDirectoryOffset + entry.offset; + directoryLength = entry.length; + } + if (!data) return { data: null, owner, tile, artifactVersion: manifest.artifactVersion }; + if (data.byteLength > this.maxTileBytes) { + throw new Error( + `Immutable tile ${tile.z}/${tile.x}/${tile.y} exceeds the production byte limit.` + ); + } + let contentEncoding = null; + if (opened.header.tileCompression === Compression.Gzip) contentEncoding = 'gzip'; + else if ( + opened.header.tileCompression !== Compression.None && + opened.header.tileCompression !== Compression.Unknown + ) { + throw new Error( + `Immutable tile compression is not HTTP-safe: ${opened.header.tileCompression}.` + ); + } + return { + data, + contentEncoding, + owner, + tile, + artifactVersion: manifest.artifactVersion + }; + } + + async ready() { + const manifest = await this.loadManifest(); + const foundation = await this.openArchive(manifest, manifest.foundation); + return { + ready: true, + architecture: 'immutable-prebuilt-pmtiles-v1', + artifactVersion: manifest.artifactVersion, + complete: manifest.complete, + builtOwnerCount: manifest.builtOwnerCount, + plannedOwnerCount: manifest.plannedOwnerCount, + foundationTiles: foundation.header.numAddressedTiles, + sourceCount: 1 + }; + } + + async close() { + if (this.closed) return; + this.closed = true; + const opened = await Promise.allSettled([...this.archives.values()]); + await Promise.allSettled( + opened + .filter((result) => result.status === 'fulfilled') + .map(({ value }) => value.source?.close?.()) + ); + this.archives.clear(); + } +}