From fddabe1514500962f903e4ce18e0c50f244415f0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 22 Aug 2026 14:43:56 +0200 Subject: [PATCH] fix(ev): stop a manual scraper run from undoing the Spain EV handover The rule that Spain must use exactly one EV source lived only in the scheduler. The manual CLI did not have it, so `scraper:run --country=all` would run OpenChargeMap for Spain alongside REVE - and after REVE retires those rows, a single such run puts all ~19k of them straight back. Rather than copy the rule into the CLI, both now call one function. Two copies would drift, and the drift is silent: the map just quietly fills up with duplicate Spanish chargers again. Naming EV_ES explicitly on the CLI still runs it. The CLI is a manual override and an operator asking for it by name means it. --- src/instrumentation.ts | 15 +++----- src/scrapers/cli.ts | 5 ++- src/scrapers/spain-ev-source.test.ts | 53 ++++++++++++++++++++++++++++ src/scrapers/spain-ev-source.ts | 31 ++++++++++++++++ 4 files changed, 93 insertions(+), 11 deletions(-) create mode 100644 src/scrapers/spain-ev-source.test.ts create mode 100644 src/scrapers/spain-ev-source.ts diff --git a/src/instrumentation.ts b/src/instrumentation.ts index 99b2fb4..f0ff32a 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -104,6 +104,7 @@ export async function register() { const { MexicoScraper } = await import("./scrapers/mexico"); const { OCMScraper } = await import("./scrapers/ocm"); const { REVEScraper } = await import("./scrapers/reve"); + const { resolveSpainEvSource } = await import("./scrapers/spain-ev-source"); const { StaticScraper } = await import("./scrapers/static"); const { STATIC_DATASETS } = await import("./scrapers/data"); @@ -229,19 +230,13 @@ export async function register() { : Object.keys(scraperFactories).filter((c) => !c.startsWith("EV_")); } - // Spain: the official Mapa REVE registry supersedes OpenChargeMap whenever a - // key is configured. They must never both run — 92% of REVE locations sit - // within 50m of an existing OpenChargeMap row, so the map would double-pin - // every Spanish charger. REVE also retires the rows it replaces once its - // backfill is complete (see scrapers/reve.ts). - if (process.env.PUMPERLY_REVE_API_KEY && countries.includes("EV_ES")) { - countries = countries.filter((c) => c !== "EV_ES"); - if (!countries.includes("EV_ES_REVE")) countries.push("EV_ES_REVE"); + // Spain has two possible EV sources and must never run both — see + // scrapers/spain-ev-source.ts for why, and scrapers/reve.ts for the handover. + countries = resolveSpainEvSource(countries); + if (countries.includes("EV_ES_REVE")) { console.log( "[scraper] Spain EV: using Mapa REVE (official registry) instead of OpenChargeMap", ); - } else { - countries = countries.filter((c) => c !== "EV_ES_REVE"); } // Resolve per-country intervals diff --git a/src/scrapers/cli.ts b/src/scrapers/cli.ts index 9df2bed..ccccfb7 100644 --- a/src/scrapers/cli.ts +++ b/src/scrapers/cli.ts @@ -39,6 +39,7 @@ import { ArgentinaScraper } from "./argentina"; import { MexicoScraper } from "./mexico"; import { OCMScraper } from "./ocm"; import { REVEScraper } from "./reve"; +import { resolveSpainEvSource } from "./spain-ev-source"; // --------------------------------------------------------------------------- // Scraper CLI @@ -153,9 +154,11 @@ function parseArgs(argv: string[]): { countries: string[] } { usage(); } + // `all` must not run both Spanish EV sources. Naming EV_ES explicitly still + // works — the CLI is a manual override, and asking for it by name means it. const countries = countryArg === "ALL" - ? Object.keys(SCRAPERS) + ? resolveSpainEvSource(Object.keys(SCRAPERS)) : countryArg.split(",").map((c) => c.trim().toUpperCase()); // Validate diff --git a/src/scrapers/spain-ev-source.test.ts b/src/scrapers/spain-ev-source.test.ts new file mode 100644 index 0000000..7d26828 --- /dev/null +++ b/src/scrapers/spain-ev-source.test.ts @@ -0,0 +1,53 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import { resolveSpainEvSource } from "./spain-ev-source"; + +// The scheduler and the manual CLI both route Spain's EV scraping through this, +// so this is the test that stops them drifting apart. Drift is silent: the map +// just quietly refills with duplicate Spanish chargers. + +describe("resolveSpainEvSource", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + // What the scheduler produces once EV_XX codes are derived. + const SCHEDULED = ["ES", "FR", "EV_ES", "EV_FR", "EV_US"]; + // What `--country=all` produces: both Spanish sources are registered. + const ALL = ["ES", "FR", "EV_ES", "EV_FR", "EV_ES_REVE", "EV_US"]; + + it("uses REVE for Spain when a key is configured", () => { + vi.stubEnv("PUMPERLY_REVE_API_KEY", "a-key"); + expect(resolveSpainEvSource(SCHEDULED)).toContain("EV_ES_REVE"); + expect(resolveSpainEvSource(SCHEDULED)).not.toContain("EV_ES"); + }); + + it("keeps OpenChargeMap for Spain when no key is configured", () => { + vi.stubEnv("PUMPERLY_REVE_API_KEY", ""); + expect(resolveSpainEvSource(SCHEDULED)).toContain("EV_ES"); + expect(resolveSpainEvSource(SCHEDULED)).not.toContain("EV_ES_REVE"); + }); + + it("collapses --country=all down to one Spanish EV source", () => { + for (const key of ["a-key", ""]) { + vi.stubEnv("PUMPERLY_REVE_API_KEY", key); + const out = resolveSpainEvSource(ALL); + expect(out.filter((c) => c === "EV_ES" || c === "EV_ES_REVE")).toHaveLength(1); + } + }); + + it("leaves every other country untouched", () => { + vi.stubEnv("PUMPERLY_REVE_API_KEY", "a-key"); + expect(resolveSpainEvSource(ALL).filter((c) => !c.startsWith("EV_ES"))).toEqual([ + "ES", + "FR", + "EV_FR", + "EV_US", + ]); + }); + + it("adds nothing when Spain has no EV scraper enabled", () => { + vi.stubEnv("PUMPERLY_REVE_API_KEY", "a-key"); + const fuelOnly = ["ES", "FR", "IT"]; + expect(resolveSpainEvSource(fuelOnly)).toEqual(fuelOnly); + }); +}); diff --git a/src/scrapers/spain-ev-source.ts b/src/scrapers/spain-ev-source.ts new file mode 100644 index 0000000..44860a6 --- /dev/null +++ b/src/scrapers/spain-ev-source.ts @@ -0,0 +1,31 @@ +// --------------------------------------------------------------------------- +// Which source supplies Spain's EV chargers +// --------------------------------------------------------------------------- +// Spain has two: OpenChargeMap (`EV_ES`, crowdsourced, every country) and Mapa +// REVE (`EV_ES_REVE`, the official registry every Spanish CPO files into). +// +// They must never both run. 92% of REVE locations sit within 50m of an existing +// OpenChargeMap row, so running both double-pins ~13k Spanish chargers — and +// once REVE has retired those rows (see scrapers/reve.ts), a single stray run of +// the OpenChargeMap scraper would put all ~19k of them straight back. +// +// This rule is needed by both the scheduler (instrumentation.ts) and the manual +// CLI (scrapers/cli.ts), so it lives here rather than in either of them. Two +// copies would drift, and the way it drifts is silent: the map just quietly +// fills up with duplicates again. +// --------------------------------------------------------------------------- + +/** + * Collapse Spain's two EV scrapers down to whichever one is configured. + * + * Returns `codes` unchanged when Spain has no EV scraper enabled at all (e.g. + * PUMPERLY_EV_ENABLED=0). Otherwise exactly one of `EV_ES` / `EV_ES_REVE` + * survives: REVE when an API key is set, OpenChargeMap when it is not. + */ +export function resolveSpainEvSource(codes: string[]): string[] { + const hasSpainEv = codes.includes("EV_ES") || codes.includes("EV_ES_REVE"); + const rest = codes.filter((c) => c !== "EV_ES" && c !== "EV_ES_REVE"); + if (!hasSpainEv) return rest; + rest.push(process.env.PUMPERLY_REVE_API_KEY ? "EV_ES_REVE" : "EV_ES"); + return rest; +}