From 118f24f8961c7c215544a67bbf50c5a0161daf59 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 22 Aug 2026 13:57:21 +0200 Subject: [PATCH 1/2] feat(ev): use Spain official charger registry instead of crowdsourced data Mapa REVE is the registry every Spanish charge point operator is required to file into, so it is authoritative where Open Charge Map is crowdsourced. The API allows 5 requests per hour at 100 locations per page, so a full pass over the ~14.5k-location registry takes about 30 hours. The scraper does not attempt a full sync: each hourly run fetches a few pages and the database fills in the background. The page window advances by wall clock rather than a stored cursor, so it needs no new table and resumes in place after a restart or redeploy. 92% of REVE locations sit within 50m of an existing Open Charge Map row, so the two sources must never both run for Spain. Open Charge Map stops being scraped as soon as a REVE key is configured, and the rows it leaves behind are removed once REVE reaches 95% coverage - not before, or Spain would show almost no chargers for the length of the backfill. Without PUMPERLY_REVE_API_KEY nothing changes. --- .env.example | 13 + .github/workflows/reve-key-renewal.yml | 91 ++++++ README.md | 7 +- src/components/nav/legal-modal.tsx | 1 + src/instrumentation.ts | 21 ++ src/scrapers/cli.ts | 3 + src/scrapers/reve.test.ts | 244 +++++++++++++++ src/scrapers/reve.ts | 417 +++++++++++++++++++++++++ 8 files changed, 796 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/reve-key-renewal.yml create mode 100644 src/scrapers/reve.test.ts create mode 100644 src/scrapers/reve.ts diff --git a/.env.example b/.env.example index f7f83ff..3299462 100644 --- a/.env.example +++ b/.env.example @@ -68,6 +68,19 @@ PUMPERLY_DEFAULT_COUNTRY=ES # OpenChargeMap — EV charging data, free key from https://openchargemap.org # PUMPERLY_OCM_API_KEY= +# Spain EV chargers (Mapa REVE) — the official registry every Spanish charge +# point operator must file into. Free key, request it at +# https://www.mapareve.es/api-contacto (keys expire after ~1 year). +# When set, Spain's EV data comes from REVE instead of OpenChargeMap, and the +# OpenChargeMap rows it replaces are deleted once the backfill is complete. +# Note: the API allows only 5 requests/hour of 100 locations each, so the +# first full load of ~14.5k chargers takes roughly 30 hours. It fills in the +# background — nothing to do but wait. +# PUMPERLY_REVE_API_KEY= + +# Pages fetched per hourly run (default 4, of the 5/hour the API allows) +# PUMPERLY_REVE_PAGES_PER_RUN=4 + # Denmark (FuelPrices.dk) # FUELPRICES_DK_API_KEY= diff --git a/.github/workflows/reve-key-renewal.yml b/.github/workflows/reve-key-renewal.yml new file mode 100644 index 0000000..3c47b18 --- /dev/null +++ b/.github/workflows/reve-key-renewal.yml @@ -0,0 +1,91 @@ +name: REVE API key renewal reminder + +# The Mapa REVE API key that feeds Spain's EV charging data expires once a year. +# When it does, the REVE scraper starts failing and Spanish charger data quietly +# goes stale — nothing else breaks, so it would go unnoticed for a long time. +# +# This opens an issue a week before expiry with the renewal steps in it. +# +# Renewing? Change BOTH the `schedule` cron below and KEY_EXPIRY to the new +# date, so next year's reminder still lands a week early. +# +# Current key expires: 2027-08-21 → reminder fires 2027-08-14. + +on: + schedule: + # 08:00 UTC on 14 August, every year (a week before the 21 August expiry). + - cron: "0 8 14 8 *" + workflow_dispatch: + +permissions: + contents: read + issues: write + +env: + KEY_EXPIRY: "2027-08-21" + +jobs: + remind: + name: Open renewal issue + runs-on: ubuntu-latest + steps: + - name: Open (or update) the renewal issue + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + set -euo pipefail + + TITLE="Renew the Mapa REVE API key (expires ${KEY_EXPIRY})" + + # Don't stack a new issue on top of last year's if one is still open. + EXISTING=$(gh issue list --state open --search '"Renew the Mapa REVE API key" in:title' \ + --json number --jq '.[0].number // empty') + + BODY=$(cat <<'EOF' + The Mapa REVE API key behind Spain's EV charging data expires on __EXPIRY__. + + Nothing breaks loudly when it lapses: the scraper logs an HTTP error and + Spanish charger data simply stops being refreshed. Renew before the date. + + ## How to renew + + 1. Request a new key at https://www.mapareve.es/api-contacto — it is free. + Say the use is non-commercial and name Pumperly. Approval arrives by + email, usually within a few days, with the key and its new expiry. + 2. Update the running deployment's `PUMPERLY_REVE_API_KEY` and redeploy. + The key is an environment variable — it is never committed. + 3. Confirm it works. A single request is enough: + + ``` + curl -sS -o /dev/null -w '%{http_code}\n' \ + -H "x-api-key: $NEW_KEY" \ + 'https://www.mapareve.es/api/external/v1/locations?limit=1&page=1' + ``` + + `200` means good, `401`/`403` means the key is wrong. + 4. Update `.github/workflows/reve-key-renewal.yml` — both the `schedule` + cron and `KEY_EXPIRY` — to a week before the new expiry date, so this + reminder keeps working. + 5. Close this issue. + + ## Worth knowing + + - The API allows **5 requests per hour**, 100 locations per page. A full + reload of the ~14.5k-location registry takes about 30 hours, so a lapsed + key is not instantly recoverable — the data refills gradually. + - Terms of use: non-commercial only, Red Eléctrica de España must be + credited as the source, and the data must not be altered or + misrepresented. The credit lives in the in-app legal modal. + + _Opened automatically by `.github/workflows/reve-key-renewal.yml`._ + EOF + ) + BODY=${BODY//__EXPIRY__/$KEY_EXPIRY} + + if [ -n "$EXISTING" ]; then + echo "Issue #$EXISTING is already open — adding a comment instead." + gh issue comment "$EXISTING" --body "$BODY" + else + gh issue create --title "$TITLE" --body "$BODY" + fi diff --git a/README.md b/README.md index e503935..907ce2b 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Pumperly combines route planning with real-time fuel prices and EV charging stat - **Route planning** — Geocoding via [Photon](https://github.com/komoot/photon), routing via [Valhalla](https://github.com/valhalla/valhalla), with alternative routes - **Real-time fuel prices** — From government open data APIs and community sources -- **EV charging stations** — Via [Open Charge Map](https://openchargemap.org) across all supported countries +- **EV charging stations** — Via [Open Charge Map](https://openchargemap.org) across all supported countries, and the official [Mapa REVE](https://www.mapareve.es) registry in Spain - **Detour calculation** — Each station shows estimated detour time from your route - **"Cheapest within N min"** — Slider filters stations by maximum detour, highlights the best deal - **Corridor station list** — Sorted by position along route, with price deltas vs average @@ -97,6 +97,9 @@ Pumperly combines route planning with real-time fuel prices and EV charging stat | Source | Coverage | License | |---|---|---| | [Open Charge Map](https://openchargemap.org) | All supported countries + United States (EV-only) | ODbL | +| [Mapa REVE](https://www.mapareve.es) (Red Eléctrica de España) | Spain — official operator-reported registry | Non-commercial, attribution required | + +Spain uses Mapa REVE when `PUMPERLY_REVE_API_KEY` is set: it is the registry every Spanish charge point operator files into, so it is authoritative where Open Charge Map is crowdsourced. It then replaces the Open Charge Map rows for Spain rather than adding to them. ### Map & routing @@ -381,6 +384,7 @@ services: # API keys — get your own: # TANKERKOENIG_API_KEY: "" # https://creativecommons.tankerkoenig.de # PUMPERLY_OCM_API_KEY: "" # https://openchargemap.org (free, for EV data) + # PUMPERLY_REVE_API_KEY: "" # https://www.mapareve.es/api-contacto (free, Spain EV) # FUELPRICES_DK_API_KEY: "" # Denmark fuel prices ports: - "3000:3000" @@ -420,6 +424,7 @@ Some data sources require API keys (all free): |---|---|---| | `TANKERKOENIG_API_KEY` | Germany fuel prices | Register at [creativecommons.tankerkoenig.de](https://creativecommons.tankerkoenig.de) | | `PUMPERLY_OCM_API_KEY` | EV charging stations | Register at [openchargemap.org](https://openchargemap.org), go to My Profile > My API Keys | +| `PUMPERLY_REVE_API_KEY` | Spain EV chargers (official registry) | Request at [mapareve.es/api-contacto](https://www.mapareve.es/api-contacto). Keys expire after about a year. The API allows 5 requests/hour, so the first full load takes ~30 hours and fills in the background. | | `FUELPRICES_DK_API_KEY` | Denmark fuel prices | Contact [fuelprices.dk](https://fuelprices.dk) | Most countries work without any API key — they use open government data. diff --git a/src/components/nav/legal-modal.tsx b/src/components/nav/legal-modal.tsx index 69c8a75..a5e89b5 100644 --- a/src/components/nav/legal-modal.tsx +++ b/src/components/nav/legal-modal.tsx @@ -158,6 +158,7 @@ function SourcesContent() {

EV charging data

Map and routing

diff --git a/src/instrumentation.ts b/src/instrumentation.ts index 06ecd18..99b2fb4 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -49,6 +49,9 @@ const DEFAULT_INTERVALS: Record = { EV_RS: 24, EV_FI: 24, EV_EE: 24, EV_LV: 24, EV_LT: 24, EV_BA: 24, EV_MK: 24, EV_TR: 24, EV_MD: 24, EV_AU: 24, EV_AR: 24, EV_MX: 24, EV_US: 24, + // Spain Mapa REVE — the API allows only 5 requests/hour, so this crawls a + // few pages at a time and must run hourly to get through the registry. + EV_ES_REVE: 1, }; export async function register() { @@ -100,6 +103,7 @@ export async function register() { const { ArgentinaScraper } = await import("./scrapers/argentina"); const { MexicoScraper } = await import("./scrapers/mexico"); const { OCMScraper } = await import("./scrapers/ocm"); + const { REVEScraper } = await import("./scrapers/reve"); const { StaticScraper } = await import("./scrapers/static"); const { STATIC_DATASETS } = await import("./scrapers/data"); @@ -180,6 +184,8 @@ export async function register() { EV_MX: () => new OCMScraper("MX"), // US has no national fuel-price API — EV-only coverage via OCM (#85) EV_US: () => new OCMScraper("US"), + // Spain's official EV registry — supersedes EV_ES when a key is set (#121) + EV_ES_REVE: () => new REVEScraper(), }; // Register community-contributed static datasets (see scrapers/data/README.md). @@ -223,6 +229,21 @@ export async function register() { : Object.keys(scraperFactories).filter((c) => !c.startsWith("EV_")); } + // Spain: the official Mapa REVE registry supersedes OpenChargeMap whenever a + // key is configured. They must never both run — 92% of REVE locations sit + // within 50m of an existing OpenChargeMap row, so the map would double-pin + // every Spanish charger. REVE also retires the rows it replaces once its + // backfill is complete (see scrapers/reve.ts). + if (process.env.PUMPERLY_REVE_API_KEY && countries.includes("EV_ES")) { + countries = countries.filter((c) => c !== "EV_ES"); + if (!countries.includes("EV_ES_REVE")) countries.push("EV_ES_REVE"); + console.log( + "[scraper] Spain EV: using Mapa REVE (official registry) instead of OpenChargeMap", + ); + } else { + countries = countries.filter((c) => c !== "EV_ES_REVE"); + } + // Resolve per-country intervals for (const code of countries) { // Priority: PUMPERLY_SCRAPE_INTERVAL_XX > PUMPERLY_SCRAPE_INTERVAL_HOURS > DEFAULT_INTERVALS diff --git a/src/scrapers/cli.ts b/src/scrapers/cli.ts index 73991b9..9df2bed 100644 --- a/src/scrapers/cli.ts +++ b/src/scrapers/cli.ts @@ -38,6 +38,7 @@ import { AustraliaNSWScraper } from "./australia-nsw"; import { ArgentinaScraper } from "./argentina"; import { MexicoScraper } from "./mexico"; import { OCMScraper } from "./ocm"; +import { REVEScraper } from "./reve"; // --------------------------------------------------------------------------- // Scraper CLI @@ -128,6 +129,8 @@ const SCRAPERS: Record BaseScraper>> = { EV_MX: [() => new OCMScraper("MX")], // US has no national fuel-price API — EV-only coverage via OCM (#85) EV_US: [() => new OCMScraper("US")], + // Spain's official EV registry — supersedes EV_ES when a key is set (#121) + EV_ES_REVE: [() => new REVEScraper()], }; function usage(): never { diff --git a/src/scrapers/reve.test.ts b/src/scrapers/reve.test.ts new file mode 100644 index 0000000..05a0eb6 --- /dev/null +++ b/src/scrapers/reve.test.ts @@ -0,0 +1,244 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +vi.mock("@prisma/adapter-pg", () => ({ PrismaPg: vi.fn() })); +vi.mock("../generated/prisma/client", () => ({ PrismaClient: vi.fn() })); + +const HOUR_MS = 60 * 60 * 1000; + +function okResponse(body: unknown, headers: Record = {}) { + const lower = Object.fromEntries(Object.entries(headers).map(([k, v]) => [k.toLowerCase(), v])); + return { + ok: true, + status: 200, + json: async () => body, + text: async () => JSON.stringify(body), + headers: { get: (k: string) => lower[k.toLowerCase()] ?? null }, + } as unknown as Response; +} + +function rateLimitedResponse() { + return { + ok: false, + status: 429, + text: async () => "Retry later", + json: async () => "Retry later", + headers: { get: () => null }, + } as unknown as Response; +} + +// One location shaped exactly like a real /locations entry. +function location(overrides: Record = {}) { + return { + id: "a3d9dfbb-5f99-467c-a903-8bd9ec1af2d1", + country_code: "ES", + party_id: "AEQ", + cpo_name: "QWELLO España SL", + version: "V221", + address: "Calle Nubledo 77", + city: "Nubledo", + postal_code: "33416", + region: "33", + state: "03", + country: "ESP", + coordinates: { latitude: "43.526146", longitude: "-5.874451" }, + evses: [ + { + id: "6078d6f0-2e6d-4b9a-9c6c-bf6f48cef542", + connectors: [{ standard: "IEC_62196_T2", max_electric_power: 22080 }], + }, + ], + owner: "Qwello - www.qwello.es", + time_zone: "Europe/Madrid", + last_updated: "2026-08-22T01:33:03.532Z", + ...overrides, + }; +} + +const PAGE_HEADERS = { "total-count": "14513", "total-pages": "146" }; + +describe("REVEScraper", () => { + beforeEach(() => { + vi.stubGlobal("fetch", vi.fn()); + vi.stubEnv("PUMPERLY_REVE_API_KEY", "test-reve-key"); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.resetModules(); + vi.unstubAllEnvs(); + }); + + it("has correct source and country", async () => { + const { REVEScraper } = await import("./reve"); + const scraper = new REVEScraper(); + expect(scraper.country).toBe("ES"); + expect(scraper.source).toBe("reve"); + }); + + it("skips entirely when no API key is configured", async () => { + vi.stubEnv("PUMPERLY_REVE_API_KEY", ""); + const { REVEScraper } = await import("./reve"); + const result = await new REVEScraper().fetch(); + expect(result.stations).toEqual([]); + expect(result.prices).toEqual([]); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("maps a location into an EV charger station with no prices", async () => { + vi.stubEnv("PUMPERLY_REVE_PAGES_PER_RUN", "1"); + const { REVEScraper } = await import("./reve"); + vi.mocked(fetch).mockResolvedValue(okResponse([location()], PAGE_HEADERS)); + + const { stations, prices } = await new REVEScraper().fetch(); + + expect(prices).toEqual([]); + expect(stations).toHaveLength(1); + expect(stations[0]).toEqual({ + externalId: "reve-a3d9dfbb-5f99-467c-a903-8bd9ec1af2d1", + name: "Qwello — 22 kW", + brand: "Qwello", + address: "Calle Nubledo 77, 33416", + city: "Nubledo", + province: "Asturias", + latitude: 43.526146, + longitude: -5.874451, + stationType: "ev_charger", + }); + }); + + it("falls back to the legal CPO name when owner is missing", async () => { + vi.stubEnv("PUMPERLY_REVE_PAGES_PER_RUN", "1"); + const { REVEScraper } = await import("./reve"); + vi.mocked(fetch).mockResolvedValue( + okResponse([location({ owner: null, evses: [] })], PAGE_HEADERS), + ); + + const { stations } = await new REVEScraper().fetch(); + expect(stations[0].brand).toBe("QWELLO España SL"); + // No connectors → no power suffix + expect(stations[0].name).toBe("QWELLO España SL"); + }); + + it("stops cleanly on HTTP 429 and keeps what it already fetched", async () => { + const { REVEScraper } = await import("./reve"); + vi.mocked(fetch) + .mockResolvedValueOnce(okResponse([location()], PAGE_HEADERS)) + .mockResolvedValue(rateLimitedResponse()); + + const { stations } = await new REVEScraper().fetch(); + + // Page 1 survived; the rate-limited pages did not throw. + expect(stations).toHaveLength(1); + expect(stations[0].externalId).toBe("reve-a3d9dfbb-5f99-467c-a903-8bd9ec1af2d1"); + }); + + it("drops locations with unusable coordinates instead of throwing", async () => { + vi.stubEnv("PUMPERLY_REVE_PAGES_PER_RUN", "1"); + const { REVEScraper } = await import("./reve"); + vi.mocked(fetch).mockResolvedValue( + okResponse( + [ + location({ id: "bad-lat", coordinates: { latitude: "not-a-number", longitude: "-5.0" } }), + location({ id: "out-of-range", coordinates: { latitude: "99.9", longitude: "-5.0" } }), + location({ id: "no-coords", coordinates: null }), + location({ id: "good" }), + ], + PAGE_HEADERS, + ), + ); + + const { stations } = await new REVEScraper().fetch(); + expect(stations.map((s) => s.externalId)).toEqual(["reve-good"]); + }); + + it("throws on a non-array payload rather than wiping data", async () => { + const { REVEScraper } = await import("./reve"); + vi.mocked(fetch).mockResolvedValue(okResponse({ error: "nope" }, PAGE_HEADERS)); + await expect(new REVEScraper().fetch()).rejects.toThrow(/expected a JSON array/); + }); + + it("sends the API key as the x-api-key header", async () => { + vi.stubEnv("PUMPERLY_REVE_PAGES_PER_RUN", "1"); + const { REVEScraper } = await import("./reve"); + vi.mocked(fetch).mockResolvedValue(okResponse([location()], PAGE_HEADERS)); + + await new REVEScraper().fetch(); + + const [url, init] = vi.mocked(fetch).mock.calls[0]; + expect(String(url)).toContain("limit=100"); + expect( + (init as RequestInit & { headers: Record }).headers["x-api-key"], + ).toBe("test-reve-key"); + }); +}); + +describe("pagesForRun", () => { + it("advances the window every hour so no cursor is needed", async () => { + const { pagesForRun } = await import("./reve"); + const hour = 1000 * HOUR_MS; + const first = pagesForRun(146, 4, hour); + const next = pagesForRun(146, 4, hour + HOUR_MS); + expect(first).toHaveLength(4); + expect(next).toHaveLength(4); + expect(next).not.toEqual(first); + // Consecutive hours cover consecutive blocks — nothing skipped. + expect(next[0]).toBe((first[3] % 146) + 1); + }); + + it("is a pure function of time, so a restart resumes in place", async () => { + const { pagesForRun } = await import("./reve"); + const t = 123456 * HOUR_MS; + expect(pagesForRun(146, 4, t)).toEqual(pagesForRun(146, 4, t)); + }); + + it("covers every page across a full cycle", async () => { + const { pagesForRun } = await import("./reve"); + const seen = new Set(); + for (let h = 0; h < 200; h++) { + for (const p of pagesForRun(146, 4, h * HOUR_MS)) seen.add(p); + } + expect(seen.size).toBe(146); + expect(Math.min(...seen)).toBe(1); + expect(Math.max(...seen)).toBe(146); + }); + + it("wraps within range and never emits page 0", async () => { + const { pagesForRun } = await import("./reve"); + for (let h = 0; h < 50; h++) { + for (const p of pagesForRun(3, 4, h * HOUR_MS)) { + expect(p).toBeGreaterThanOrEqual(1); + expect(p).toBeLessThanOrEqual(3); + } + } + }); + + it("falls back to page 1 when the page count is unknown", async () => { + const { pagesForRun } = await import("./reve"); + expect(pagesForRun(0, 4, 0)).toEqual([1]); + }); +}); + +describe("shouldRetireOcmRows", () => { + it("holds the OpenChargeMap rows until the backfill is nearly complete", async () => { + const { shouldRetireOcmRows } = await import("./reve"); + expect(shouldRetireOcmRows(0, 19067, 14513, 0.95)).toBe(false); + expect(shouldRetireOcmRows(5000, 19067, 14513, 0.95)).toBe(false); + expect(shouldRetireOcmRows(13786, 19067, 14513, 0.95)).toBe(false); + }); + + it("retires them once coverage passes the threshold", async () => { + const { shouldRetireOcmRows } = await import("./reve"); + expect(shouldRetireOcmRows(13787, 19067, 14513, 0.95)).toBe(true); + expect(shouldRetireOcmRows(14513, 19067, 14513, 0.95)).toBe(true); + }); + + it("never deletes when the registry size is unknown", async () => { + const { shouldRetireOcmRows } = await import("./reve"); + expect(shouldRetireOcmRows(14513, 19067, 0, 0.95)).toBe(false); + }); + + it("does nothing once there is nothing left to retire", async () => { + const { shouldRetireOcmRows } = await import("./reve"); + expect(shouldRetireOcmRows(14513, 0, 14513, 0.95)).toBe(false); + }); +}); diff --git a/src/scrapers/reve.ts b/src/scrapers/reve.ts new file mode 100644 index 0000000..19a4d07 --- /dev/null +++ b/src/scrapers/reve.ts @@ -0,0 +1,417 @@ +import { z } from "zod"; +import { PrismaPg } from "@prisma/adapter-pg"; +import { PrismaClient } from "../generated/prisma/client"; +import { BaseScraper, type RawFuelPrice, type RawStation, type ScraperResult } from "./base"; + +// --------------------------------------------------------------------------- +// Mapa REVE — official Spanish EV charging point registry +// --------------------------------------------------------------------------- +// API: https://www.mapareve.es/api/external/v1/locations (OCPI-shaped) +// Docs: https://www.mapareve.es/docs/api/external/v1 +// Key: free, request at https://www.mapareve.es/api-contacto +// Source: Red Electrica de Espana (REE) — the operator-reported registry every +// Spanish CPO must file into, so it is authoritative where OpenChargeMap is +// crowdsourced. Non-commercial use only; attribution to REE is required. +// +// THE CONSTRAINT THAT SHAPES THIS WHOLE FILE: the API allows **5 requests per +// hour** and caps `limit` at 100, so the ceiling is 500 locations/hour. The +// registry holds ~14.5k locations, so one complete pass takes ~30 hours. There +// is no bulk export and no way around it — a full sync in a single run is +// impossible, not merely slow. +// +// So this scraper does not try. Each run fetches a small window of pages and +// upserts them; the DB fills over successive runs. Two properties make that +// safe: station upserts are idempotent, and `base.run()` never orphan-deletes +// EV chargers (only price-less `fuel` rows), so partial fetches accumulate +// instead of wiping each other out. +// +// The page window advances by wall clock rather than a stored cursor: +// +// startIndex = (hoursSinceEpoch * PAGES_PER_RUN) mod totalPages +// +// That is a pure function of time, so it needs no state anywhere. A container +// restart, a redeploy or a fresh self-host resumes exactly where the clock says +// — no cursor to persist, no migration, and no risk of a restart loop pinning +// the crawl to page 1 and never reaching the tail. Once a full pass completes +// it simply keeps wrapping, which is also the refresh cycle. +// --------------------------------------------------------------------------- + +const BASE_URL = "https://www.mapareve.es/api/external/v1/locations"; +const API_KEY = process.env.PUMPERLY_REVE_API_KEY ?? ""; +const PAGE_LIMIT = 100; // API maximum — do not raise, larger values are ignored + +// Pages fetched per run. The hourly budget is 5; the default of 4 leaves one +// request spare for a manual `scraper:run` or a probe without tripping the +// limit. Runs are scheduled hourly (see instrumentation.ts). +const rawPagesPerRun = Number(process.env.PUMPERLY_REVE_PAGES_PER_RUN ?? "4"); +const PAGES_PER_RUN = + Number.isFinite(rawPagesPerRun) && rawPagesPerRun >= 1 ? Math.floor(rawPagesPerRun) : 4; + +// Fraction of the registry that must be stored locally before the OpenChargeMap +// rows this data replaces are retired (see retireSupersededOcmRows). +const rawCutover = Number(process.env.PUMPERLY_REVE_CUTOVER_RATIO ?? "0.95"); +const CUTOVER_RATIO = + Number.isFinite(rawCutover) && rawCutover > 0 && rawCutover <= 1 ? rawCutover : 0.95; + +const HOUR_MS = 60 * 60 * 1000; + +// INE province codes (`region`) → the province names the Spanish fuel scraper +// already writes, so both sources agree on how a province is spelled. +const INE_PROVINCES: Record = { + "01": "Álava", + "02": "Albacete", + "03": "Alicante", + "04": "Almería", + "05": "Ávila", + "06": "Badajoz", + "07": "Baleares", + "08": "Barcelona", + "09": "Burgos", + "10": "Cáceres", + "11": "Cádiz", + "12": "Castellón", + "13": "Ciudad Real", + "14": "Córdoba", + "15": "A Coruña", + "16": "Cuenca", + "17": "Girona", + "18": "Granada", + "19": "Guadalajara", + "20": "Gipuzkoa", + "21": "Huelva", + "22": "Huesca", + "23": "Jaén", + "24": "León", + "25": "Lleida", + "26": "La Rioja", + "27": "Lugo", + "28": "Madrid", + "29": "Málaga", + "30": "Murcia", + "31": "Navarra", + "32": "Ourense", + "33": "Asturias", + "34": "Palencia", + "35": "Las Palmas", + "36": "Pontevedra", + "37": "Salamanca", + "38": "Santa Cruz de Tenerife", + "39": "Cantabria", + "40": "Segovia", + "41": "Sevilla", + "42": "Soria", + "43": "Tarragona", + "44": "Teruel", + "45": "Toledo", + "46": "Valencia", + "47": "Valladolid", + "48": "Bizkaia", + "49": "Zamora", + "50": "Zaragoza", + "51": "Ceuta", + "52": "Melilla", +}; + +// Only the fields we consume. REVE marks `name` as optional and in practice +// never sends it, so everything user-visible is derived from the CPO and +// address instead. Coordinates arrive as strings. +const ConnectorSchema = z.object({ + max_electric_power: z.number().nullish(), // watts +}); + +const EvseSchema = z.object({ + connectors: z.array(ConnectorSchema).nullish(), +}); + +const LocationSchema = z.object({ + id: z.string(), + cpo_name: z.string().nullish(), + owner: z.string().nullish(), + name: z.string().nullish(), + address: z.string().nullish(), + city: z.string().nullish(), + postal_code: z.string().nullish(), + region: z.string().nullish(), // INE province code + coordinates: z + .object({ + latitude: z.string(), + longitude: z.string(), + }) + .nullish(), + evses: z.array(EvseSchema).nullish(), +}); + +type REVELocation = z.infer; + +/** + * Pages to fetch this run, 1-based, wrapping at `totalPages`. + * + * Derived from the hour bucket so the crawl advances without stored state and + * survives restarts. Exported for tests. + */ +export function pagesForRun(totalPages: number, pagesPerRun: number, nowMs: number): number[] { + if (totalPages < 1) return [1]; + const count = Math.min(pagesPerRun, totalPages); + const start = (Math.floor(nowMs / HOUR_MS) * pagesPerRun) % totalPages; + return Array.from({ length: count }, (_, i) => ((start + i) % totalPages) + 1); +} + +/** + * CPO display name. `owner` is "Trade name - website" (e.g. "Qwello - + * www.qwello.es"), which reads better on the map than the legal entity in + * `cpo_name` ("QWELLO España SL"), so prefer it and fall back. + */ +export function cpoDisplayName(loc: REVELocation): string | null { + const owner = loc.owner?.split(" - ")[0]?.trim(); + if (owner) return owner; + return loc.cpo_name?.trim() || null; +} + +/** + * Whether the OpenChargeMap rows for Spain can be retired yet. + * + * Kept pure and exported because it gates a bulk DELETE: everything about when + * ~19k rows disappear is decided here, where a test can pin it down. + */ +export function shouldRetireOcmRows( + reveCount: number, + ocmCount: number, + totalCount: number, + ratio: number, +): boolean { + if (ocmCount <= 0) return false; // nothing left to retire + if (totalCount <= 0) return false; // registry size unknown — never guess + return reveCount >= Math.floor(totalCount * ratio); +} + +/** Highest connector power at a location, in kW (rounded). */ +function maxPowerKw(loc: REVELocation): number | null { + let maxW = 0; + for (const evse of loc.evses ?? []) { + for (const connector of evse.connectors ?? []) { + const w = connector.max_electric_power; + if (typeof w === "number" && Number.isFinite(w) && w > maxW) maxW = w; + } + } + return maxW > 0 ? Math.round(maxW / 1000) : null; +} + +export class REVEScraper extends BaseScraper { + readonly country = "ES"; + readonly source = "reve"; + + /** `total-count` from the most recent successful response (0 = unknown). */ + private lastTotalCount = 0; + + /** Learned from `total-pages`; seeds the page rotation on later runs. */ + private static knownTotalPages = 0; + + private async fetchPage( + page: number, + ): Promise<{ locations: REVELocation[]; totalCount: number; totalPages: number } | "rate-limited"> { + const url = new URL(BASE_URL); + url.searchParams.set("page", String(page)); + url.searchParams.set("limit", String(PAGE_LIMIT)); + + const res = await fetch(url.toString(), { + headers: { + "x-api-key": API_KEY, + Accept: "application/json", + "User-Agent": "Pumperly/1.0 (+https://pumperly.com)", + }, + signal: AbortSignal.timeout(120_000), + }); + + // 429 carries no Retry-After and the window is a whole hour, so retrying + // inside this run cannot succeed. Stop and let the next run continue. + if (res.status === 429) return "rate-limited"; + + if (!res.ok) { + throw new Error(`REVE HTTP ${res.status}: ${await res.text().catch(() => "")}`); + } + + const raw: unknown = await res.json(); + if (!Array.isArray(raw)) { + throw new Error(`REVE: expected a JSON array, got ${typeof raw}`); + } + + const locations: REVELocation[] = []; + let dropped = 0; + for (const entry of raw) { + const parsed = LocationSchema.safeParse(entry); + if (parsed.success) { + locations.push(parsed.data); + } else { + dropped++; + } + } + if (dropped > 0) { + console.warn(`[${this.source}] page ${page}: dropped ${dropped} malformed location(s)`); + } + + return { + locations, + totalCount: Number(res.headers.get("total-count")) || 0, + totalPages: Number(res.headers.get("total-pages")) || 0, + }; + } + + async fetch(): Promise<{ stations: RawStation[]; prices: RawFuelPrice[] }> { + if (!API_KEY) { + console.warn(`[${this.source}] PUMPERLY_REVE_API_KEY not set, skipping`); + return { stations: [], prices: [] }; + } + + // First run in this process does not yet know how many pages exist. Page 1 + // both answers that and returns real data, so nothing is wasted. + const pages = + REVEScraper.knownTotalPages > 0 + ? pagesForRun(REVEScraper.knownTotalPages, PAGES_PER_RUN, Date.now()) + : [1]; + + const byId = new Map(); + let fetched = 0; + + for (let i = 0; i < pages.length; i++) { + const page = pages[i]; + const result = await this.fetchPage(page); + + if (result === "rate-limited") { + console.warn( + `[${this.source}] HTTP 429 after ${fetched} page(s) — hourly budget spent, resuming next run`, + ); + break; + } + + fetched++; + for (const loc of result.locations) byId.set(loc.id, loc); + if (result.totalCount > 0) this.lastTotalCount = result.totalCount; + if (result.totalPages > 0) REVEScraper.knownTotalPages = result.totalPages; + + // Page 1 was a probe because the page count was unknown; now that it is + // known, spend the rest of this run's budget on the real rotation. + if (i === 0 && pages.length === 1 && REVEScraper.knownTotalPages > 0) { + const rest = pagesForRun(REVEScraper.knownTotalPages, PAGES_PER_RUN, Date.now()).filter( + (p) => p !== 1, + ); + pages.push(...rest.slice(0, PAGES_PER_RUN - 1)); + } + } + + const stations: RawStation[] = []; + for (const loc of byId.values()) { + const coords = loc.coordinates; + if (!coords) continue; + + const latitude = Number(coords.latitude); + const longitude = Number(coords.longitude); + if (!Number.isFinite(latitude) || !Number.isFinite(longitude)) continue; + if (latitude < -90 || latitude > 90) continue; + if (longitude < -180 || longitude > 180) continue; + + const brand = cpoDisplayName(loc); + const kw = maxPowerKw(loc); + // REVE never sends `name`, so build one: the popup shows `brand`, and + // `name` is the search fallback, which is where the power is useful. + const name = + loc.name?.trim() || + [brand ?? "Punto de recarga", kw ? `${kw} kW` : null].filter(Boolean).join(" — "); + + const address = + [loc.address?.trim(), loc.postal_code?.trim()].filter(Boolean).join(", ") || name; + + stations.push({ + externalId: `reve-${loc.id}`, + name, + brand, + address, + city: loc.city?.trim() || "", + province: INE_PROVINCES[loc.region?.trim() ?? ""] ?? null, + latitude, + longitude, + stationType: "ev_charger", + }); + } + + const coverage = + this.lastTotalCount > 0 ? ` of ~${this.lastTotalCount} in the registry` : ""; + console.log( + `[${this.source}] ES: ${fetched} page(s) [${pages.slice(0, fetched).join(", ")}] → ` + + `${stations.length} stations${coverage}`, + ); + + // EV chargers have no per-litre fuel price. REVE does publish connector + // tariffs on a separate endpoint, but they are per-kWh and per-session, + // which the fuel_prices model cannot express — left out deliberately. + return { stations, prices: [] }; + } + + /** + * Run the normal pipeline, then retire the OpenChargeMap rows this data + * replaces once local coverage is high enough. + * + * Why this exists: 92% of REVE locations sit within 50m of an existing + * `ocm-` row for Spain, so running both sources leaves the map double-pinned. + * OCM stops being scraped for ES the moment a REVE key is configured, but its + * rows are never orphan-cleaned (that only touches price-less `fuel` rows), + * so something has to remove them — and it cannot happen up front, because + * the ~30h backfill would leave Spain nearly empty of chargers in the + * meantime. Deleting them only once REVE has replaced them keeps the map + * populated throughout the handover, and makes the cutover automatic for + * self-hosters instead of a manual step nobody remembers. + */ + async run(): Promise { + const result = await super.run(); + if (result.errors.length === 0 && this.lastTotalCount > 0) { + try { + await this.retireSupersededOcmRows(this.lastTotalCount); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + console.error(`[${this.source}] OCM retirement check failed: ${msg}`); + result.errors.push(`OCM retirement check: ${msg}`); + } + } + return result; + } + + private async retireSupersededOcmRows(totalCount: number): Promise { + const adapter = new PrismaPg({ connectionString: process.env.DATABASE_URL! }); + const prisma = new PrismaClient({ adapter }); + try { + const rows: Array<{ reve: bigint; ocm: bigint }> = await prisma.$queryRawUnsafe( + `SELECT + count(*) FILTER (WHERE external_id LIKE 'reve-%') AS reve, + count(*) FILTER (WHERE external_id LIKE 'ocm-%') AS ocm + FROM stations + WHERE country = 'ES' AND station_type = 'ev_charger'`, + ); + const reveCount = Number(rows[0]?.reve ?? 0); + const ocmCount = Number(rows[0]?.ocm ?? 0); + if (ocmCount === 0) return; // handover already done + + if (!shouldRetireOcmRows(reveCount, ocmCount, totalCount, CUTOVER_RATIO)) { + const target = Math.floor(totalCount * CUTOVER_RATIO); + console.log( + `[${this.source}] backfill ${reveCount}/${totalCount} — keeping ${ocmCount} OpenChargeMap row(s) until ${target}`, + ); + return; + } + + const deleted: Array<{ count: bigint }> = await prisma.$queryRawUnsafe( + `WITH deleted AS ( + DELETE FROM stations + WHERE country = 'ES' + AND station_type = 'ev_charger' + AND external_id LIKE 'ocm-%' + RETURNING id + ) SELECT count(*) FROM deleted`, + ); + console.log( + `[${this.source}] backfill complete (${reveCount}/${totalCount}) — retired ` + + `${Number(deleted[0]?.count ?? 0)} superseded OpenChargeMap row(s) for ES`, + ); + } finally { + await prisma.$disconnect().catch(() => {}); + } + } +} From 14b379499cf65e42ffd8916b8ec7fdefc3f4fadc Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 22 Aug 2026 14:12:40 +0200 Subject: [PATCH 2/2] fix(ev): clamp the REVE page budget to the API hourly limit PUMPERLY_REVE_PAGES_PER_RUN above 5 guaranteed that every surplus request was spent on an HTTP 429 - pointless, and rude to a free public service. Values are now capped at the published limit. Also corrects the backfill estimate. The ~30 hour figure is the floor at 5 requests/hour; the shipped default of 4 pages per run takes about 37. And the docs said the OpenChargeMap rows are removed "once the backfill is complete" when the actual trigger is 95% coverage, with both sources visible until then. --- .env.example | 13 ++++++++----- README.md | 4 ++-- src/scrapers/reve.test.ts | 21 +++++++++++++++++++++ src/scrapers/reve.ts | 18 +++++++++++++----- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/.env.example b/.env.example index 3299462..ae0faf5 100644 --- a/.env.example +++ b/.env.example @@ -71,14 +71,17 @@ PUMPERLY_DEFAULT_COUNTRY=ES # Spain EV chargers (Mapa REVE) — the official registry every Spanish charge # point operator must file into. Free key, request it at # https://www.mapareve.es/api-contacto (keys expire after ~1 year). -# When set, Spain's EV data comes from REVE instead of OpenChargeMap, and the -# OpenChargeMap rows it replaces are deleted once the backfill is complete. +# When set, Spain's EV data comes from REVE instead of OpenChargeMap. The +# existing OpenChargeMap rows for Spain stay put during the backfill (so the +# map is never empty) and are deleted once REVE reaches 95% of the registry. # Note: the API allows only 5 requests/hour of 100 locations each, so the -# first full load of ~14.5k chargers takes roughly 30 hours. It fills in the -# background — nothing to do but wait. +# first full load of ~14.5k chargers takes about 37 hours at the default 4 +# pages per run (~30 hours if you raise it to 5). It fills in the background +# — nothing to do but wait, and expect duplicate Spanish pins until it lands. # PUMPERLY_REVE_API_KEY= -# Pages fetched per hourly run (default 4, of the 5/hour the API allows) +# Pages fetched per hourly run (default 4). Values above 5 are clamped to 5, +# the API's published hourly limit. # PUMPERLY_REVE_PAGES_PER_RUN=4 # Denmark (FuelPrices.dk) diff --git a/README.md b/README.md index 907ce2b..5b098dc 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Pumperly combines route planning with real-time fuel prices and EV charging stat | [Open Charge Map](https://openchargemap.org) | All supported countries + United States (EV-only) | ODbL | | [Mapa REVE](https://www.mapareve.es) (Red Eléctrica de España) | Spain — official operator-reported registry | Non-commercial, attribution required | -Spain uses Mapa REVE when `PUMPERLY_REVE_API_KEY` is set: it is the registry every Spanish charge point operator files into, so it is authoritative where Open Charge Map is crowdsourced. It then replaces the Open Charge Map rows for Spain rather than adding to them. +Spain uses Mapa REVE when `PUMPERLY_REVE_API_KEY` is set: it is the registry every Spanish charge point operator files into, so it is authoritative where Open Charge Map is crowdsourced. The two overlap heavily, so Open Charge Map stops being scraped for Spain immediately, its existing Spanish rows stay visible while REVE backfills, and they are deleted once REVE reaches 95% of the registry. Expect duplicate Spanish pins until then. ### Map & routing @@ -424,7 +424,7 @@ Some data sources require API keys (all free): |---|---|---| | `TANKERKOENIG_API_KEY` | Germany fuel prices | Register at [creativecommons.tankerkoenig.de](https://creativecommons.tankerkoenig.de) | | `PUMPERLY_OCM_API_KEY` | EV charging stations | Register at [openchargemap.org](https://openchargemap.org), go to My Profile > My API Keys | -| `PUMPERLY_REVE_API_KEY` | Spain EV chargers (official registry) | Request at [mapareve.es/api-contacto](https://www.mapareve.es/api-contacto). Keys expire after about a year. The API allows 5 requests/hour, so the first full load takes ~30 hours and fills in the background. | +| `PUMPERLY_REVE_API_KEY` | Spain EV chargers (official registry) | Request at [mapareve.es/api-contacto](https://www.mapareve.es/api-contacto). Keys expire after about a year. The API allows 5 requests/hour, so the first full load takes about 37 hours at the default 4 pages per run, and fills in the background. | | `FUELPRICES_DK_API_KEY` | Denmark fuel prices | Contact [fuelprices.dk](https://fuelprices.dk) | Most countries work without any API key — they use open government data. diff --git a/src/scrapers/reve.test.ts b/src/scrapers/reve.test.ts index 05a0eb6..2f662d2 100644 --- a/src/scrapers/reve.test.ts +++ b/src/scrapers/reve.test.ts @@ -132,6 +132,27 @@ describe("REVEScraper", () => { expect(stations[0].externalId).toBe("reve-a3d9dfbb-5f99-467c-a903-8bd9ec1af2d1"); }); + it("never issues more requests per run than the API allows", async () => { + // 6 pages/run would spend the sixth request on a guaranteed 429. + vi.stubEnv("PUMPERLY_REVE_PAGES_PER_RUN", "99"); + const { REVEScraper } = await import("./reve"); + vi.mocked(fetch).mockResolvedValue(okResponse([location()], PAGE_HEADERS)); + + await new REVEScraper().fetch(); + + expect(vi.mocked(fetch).mock.calls.length).toBe(5); + }); + + it("falls back to the default page budget when the override is nonsense", async () => { + vi.stubEnv("PUMPERLY_REVE_PAGES_PER_RUN", "not-a-number"); + const { REVEScraper } = await import("./reve"); + vi.mocked(fetch).mockResolvedValue(okResponse([location()], PAGE_HEADERS)); + + await new REVEScraper().fetch(); + + expect(vi.mocked(fetch).mock.calls.length).toBe(4); + }); + it("drops locations with unusable coordinates instead of throwing", async () => { vi.stubEnv("PUMPERLY_REVE_PAGES_PER_RUN", "1"); const { REVEScraper } = await import("./reve"); diff --git a/src/scrapers/reve.ts b/src/scrapers/reve.ts index 19a4d07..d22e5e2 100644 --- a/src/scrapers/reve.ts +++ b/src/scrapers/reve.ts @@ -15,7 +15,8 @@ import { BaseScraper, type RawFuelPrice, type RawStation, type ScraperResult } f // // THE CONSTRAINT THAT SHAPES THIS WHOLE FILE: the API allows **5 requests per // hour** and caps `limit` at 100, so the ceiling is 500 locations/hour. The -// registry holds ~14.5k locations, so one complete pass takes ~30 hours. There +// registry holds ~14.5k locations across ~146 pages, so a complete pass takes +// ~30 hours at that ceiling, and ~37 hours at the default 4 pages/run. There // is no bulk export and no way around it — a full sync in a single run is // impossible, not merely slow. // @@ -40,12 +41,19 @@ const BASE_URL = "https://www.mapareve.es/api/external/v1/locations"; const API_KEY = process.env.PUMPERLY_REVE_API_KEY ?? ""; const PAGE_LIMIT = 100; // API maximum — do not raise, larger values are ignored -// Pages fetched per run. The hourly budget is 5; the default of 4 leaves one -// request spare for a manual `scraper:run` or a probe without tripping the -// limit. Runs are scheduled hourly (see instrumentation.ts). +// Hard ceiling published by the API. Config may tune below it but never above +// it: a run that asks for more than this is guaranteed to burn the surplus on +// HTTP 429s, which is both pointless and rude to a free public service. +const RATE_LIMIT_PER_HOUR = 5; + +// Pages fetched per run. The default of 4 leaves one request spare for a manual +// `scraper:run` or a probe without tripping the limit. Runs are scheduled hourly +// (see instrumentation.ts). const rawPagesPerRun = Number(process.env.PUMPERLY_REVE_PAGES_PER_RUN ?? "4"); const PAGES_PER_RUN = - Number.isFinite(rawPagesPerRun) && rawPagesPerRun >= 1 ? Math.floor(rawPagesPerRun) : 4; + Number.isFinite(rawPagesPerRun) && rawPagesPerRun >= 1 + ? Math.min(Math.floor(rawPagesPerRun), RATE_LIMIT_PER_HOUR) + : 4; // Fraction of the registry that must be stored locally before the OpenChargeMap // rows this data replaces are retired (see retireSupersededOcmRows).