diff --git a/.changeset/roofmodel-building-picker.md b/.changeset/roofmodel-building-picker.md new file mode 100644 index 00000000..4514ba21 --- /dev/null +++ b/.changeset/roofmodel-building-picker.md @@ -0,0 +1,38 @@ +--- +"ftw": minor +--- + +Pick your building on the map and read the panel angles off Lantmäteriet's laser +scan, instead of measuring your own roof. + +**Settings → Weather** gains a Geotorget credential form and a building picker. +Press *Find buildings here* and the footprints near the marker are drawn on the +map and listed beside it; click yours, press *Read roof from LiDAR*, and the PV +arrays fill in with one entry per usable roof face. The form is filled but +nothing is saved: the operator sees the numbers, corrects what is wrong and +presses Save. FTW never rewrites a panel configuration on its own, because the +derivation is a guess from a scan that may be years old and only the operator +knows whether that face has panels on it at all. + +Picking a building is not cosmetic. Without a footprint the module segments +whatever stands inside its search radius, and the plane fitting is global — a +fitted plane is infinite, so a roof at azimuth 180° is `z = f(y)` with no `x` +term and extends across the whole tile. A second building sharing that ridge +orientation lands inside its inlier band however far away it is, and the two lose +returns to each other. Measured on a synthetic pair: a detached garage recovered +93% of its true area and split into two fragments while coplanar with the house, +against 100% and one clean face once clipped to its own footprint. Clipping also +buffers the outline by a metre so the eaves, where the lowest roof returns are, +are not shaved off. + +New `GET /api/roofmodel/buildings` lists footprints as GeoJSON, honouring an +explicit `lat`/`lon` so the picker can search where the marker is rather than +where the last save put it. `POST /api/roofmodel/derive` accepts a `building_id`. +`GET /api/roofmodel` reports `has_credentials` so the UI can stop asking. + +The Geotorget token now masks and restores like every other secret: it never +appears in an API response, and saving an unrelated setting no longer wipes it. + +Documented in [docs/roof-geometry.md](../docs/roof-geometry.md), including how to +order the two Geotorget products, what the derived kWp does and does not mean, +and what each failure message is telling you. diff --git a/.changeset/roofmodel-lantmateriet.md b/.changeset/roofmodel-lantmateriet.md new file mode 100644 index 00000000..4143658b --- /dev/null +++ b/.changeset/roofmodel-lantmateriet.md @@ -0,0 +1,33 @@ +--- +"ftw": minor +--- + +New optional roof-geometry module derives PV array tilt, azimuth and kWp from +Lantmäteriet open geodata, so a Swedish site can stop typing panel angles in by +hand. + +`roofmodel/` is a separate Python package alongside `optimizer/`, invoked at +arm's length: core spawns it, hands it coordinates and the operator's own +Geotorget credentials, and reads back one versioned `roof_model.json`. LiDAR +segmentation drags in a compiled point-cloud stack and runs for minutes, so +keeping it in a time-boxed subprocess means it cannot stall the control tick or +leak into the daemon — and it can be absent entirely, which is the normal case, +since the data only exists for Sweden. + +The pipeline follows the SPAN method (Yavuzdoğan, *Renewable Energy* 2023): +iterative RANSAC plane fitting pulls one roof surface at a time out of the point +cloud, then DBSCAN splits faces that share a plane equation but not a location — +two wings of a building fit the same plane and are not the same roof. Method +only; no code is taken from SPAN's GPL QGIS plugin, and the module depends on +numpy, scikit-learn and requests, all BSD. + +`GET /api/roofmodel` reports availability and coverage; `POST +/api/roofmodel/derive` runs a derive and returns the proposed arrays. Applying +them to `weather.pv_arrays` is deliberately left as a separate explicit act: +derivation is a best guess from a point cloud that may be years old, and +silently rewriting an operator's panel config is a change they should make +knowingly. Lantmäteriet also joins the `/api/data-sources` registry as a +Sweden-only, credential-gated source. + +Off by default. Absent credentials, absent module or a non-Swedish site all +produce a clean explanation rather than a failure. diff --git a/.changeset/roofmodel-stac-formats.md b/.changeset/roofmodel-stac-formats.md new file mode 100644 index 00000000..874e48c0 --- /dev/null +++ b/.changeset/roofmodel-stac-formats.md @@ -0,0 +1,26 @@ +--- +"ftw": minor +--- + +Read Lantmäteriet's data in the formats it is actually published in. + +Both roof-geometry products are STAC APIs behind one Geotorget account, and they +differ only in what their items point at: *Byggnad Nedladdning, vektor* delivers +**GeoPackage**, *Laserdata Nedladdning, Skog* delivers **LAZ organised as COPC** +(Cloud Optimized Point Cloud). Assets are now chosen by their declared media +type instead of by guessing at asset names, so a catalogue that calls its asset +`punktmoln` rather than `data` still works, and a thumbnail is never mistaken +for a point cloud. + +Building footprints are read straight out of the GeoPackage with the standard +library — a GeoPackage is a SQLite database holding geometry as WKB, both +published formats with fixed layouts, so this needs no GDAL and installs on a Pi +unchanged. Previously only inline STAC geometry was handled, which meant the +normal asset-backed case returned nothing at all. + +Because COPC indexes its points into an octree, picking a building now also +makes the download small: FTW range-requests only the octree nodes covering that +footprint instead of pulling a 2.5 km tile that runs to hundreds of megabytes. +Plain `.laz` assets, servers that ignore `Range`, and builds of laspy without +COPC support all fall back to reading the tile whole — slower, same answer. The +derived model records which path ran as `source.fetch`. diff --git a/.changeset/roofmodel-vostok-shading.md b/.changeset/roofmodel-vostok-shading.md new file mode 100644 index 00000000..cea310da --- /dev/null +++ b/.changeset/roofmodel-vostok-shading.md @@ -0,0 +1,28 @@ +--- +"ftw": minor +--- + +Roof faces can now carry a shading factor from neighbouring buildings and trees, +computed with vostok. + +Roof segmentation gives each face a tilt and an azimuth, which predicts yield for +an unobstructed roof and says nothing about the spruce to the south. `vostok` +("Voxel Octree Solar Toolkit") computes per-point solar potential against +voxelised occlusion geometry, so pointing it at the same LiDAR tile the roof came +from supplies exactly the missing number. Each face is run twice — once against +the real geometry, once with shadowing off — and the ratio is a shading factor +that is independent of vostok's absolute units, sky model and chosen year, since +all of those cancel. + +**vostok is GPL-3.0 and FTW is not.** It is therefore treated as an external +program at arm's length: a separate process communicating through files and a +command line, never linked, with no code copied in either direction. Two rules +keep that boundary intact — vostok is never bundled or redistributed with FTW, +and FTW never installs it. The operator installs it themselves and sets +`roofmodel.vostok_binary`. + +Absent or unset, shading is simply not evaluated: faces report `shading_factor` +as *absent* rather than 1.0, because "we did not look" and "we looked and it is +unobstructed" are different claims and only one of them justifies trusting the +yield estimate. A missing or failing binary never fails a derive — the roof +geometry is still good without a shading number. diff --git a/.changeset/strang-poa-pv-forecast.md b/.changeset/strang-poa-pv-forecast.md new file mode 100644 index 00000000..687c8a7e --- /dev/null +++ b/.changeset/strang-poa-pv-forecast.md @@ -0,0 +1,15 @@ +--- +"ftw": minor +--- + +PV forecasts are now orientation-aware. When per-plane geometry is configured +(the Weather tab's PV arrays: tilt/azimuth/kWp), a radiation-bearing forecast +provider's global horizontal irradiance is projected onto each panel plane via +the physics `sunpos` model and summed, instead of the previous flat +`rated × (W/m² / 1000)` estimate that ignored panel orientation. Sites with no +arrays configured keep the existing behaviour, and providers that already return +site-calibrated watts (Forecast.Solar) are left untouched. + +Providers that publish only global horizontal irradiance get an Erbs correlation +to split it into direct and diffuse components before projection, so a +south-facing 35° roof and a flat one no longer receive the same forecast. diff --git a/.changeset/strang-source-coverage.md b/.changeset/strang-source-coverage.md new file mode 100644 index 00000000..6fc1a2b0 --- /dev/null +++ b/.changeset/strang-source-coverage.md @@ -0,0 +1,46 @@ +--- +"ftw": minor +--- + +SMHI STRÅNG becomes a first-class irradiance source, every external data source +now declares where in the world it works, and the location picker moves to +MapLibre GL JS. + +- **STRÅNG as an irradiance source.** The client now covers the model's full + parameter set and knows its own domain. A nightly backfill scores measured + production against the DC energy the configured arrays should have produced + under that irradiance, exposed at `GET /api/pv/performance` and drawn as a + dashed "expected (STRÅNG)" overlay on the Produced tile. The resulting + performance ratio feeds back as a calibration factor on the forward forecast, + refused outright when it lands outside a plausible band — a site reading at + 10% or 160% of nameplate is a configuration fault, and silently rescaling the + forecast would hide it. + +- **Cloud cover, derived.** STRÅNG publishes no cloud-cover parameter; it is a + radiation model. It does publish sunshine duration (minutes per hour above the + WMO beam threshold), so cloudiness is recovered as `1 − minutes/60`. That is an + observed quantity rather than an inferred cloud field, but coarser: blind to + thin cirrus, and undefined at night. The API returns an explicit *unknown* + rather than defaulting to *clear*, because those lead to opposite decisions. + +- **Coverage metadata.** New `GET /api/data-sources` reports every forecast, + irradiance and price source with its coverage area, country list, licence and + whether it reaches this specific site; the Weather tab renders it under the + map and flags sources that do not. This makes an existing silence explicit: + STRÅNG is Nordic-only and every price provider is European, so sites elsewhere + were getting empty results with no explanation. Bounds are advisory — a + rotated model grid means a lat/lon box can only be a superset — so `false` is + definitive and `true` means "worth trying". PV performance scoring now declines + to start outside the STRÅNG domain instead of retrying nightly forever. + +- **MapLibre GL JS location picker.** The Weather tab's map is now MapLibre GL + JS 6 (BSD-3) instead of Leaflet, still lazy-loaded only when the tab opens. v6 + is ESM-only and code-split, so it loads via a pinned dynamic import; the + stylesheet keeps its integrity hash, while the JS relies on version pinning + (an integrity hash on the entry point would not cover the shared chunk it + imports anyway). The style is built inline from the same OpenStreetMap raster + tiles as before, so neither the tile source nor the attribution changed. The + numeric latitude/longitude fields remain authoritative, so a CDN or WebGL + failure costs the picker and nothing else. + +Nothing here touches the control tick, dispatch or the optimizer contract. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 160aa976..4ffc3666 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,6 +22,7 @@ jobs: outputs: core: ${{ steps.paths.outputs.core }} optimizer: ${{ steps.paths.outputs.optimizer }} + roofmodel: ${{ steps.paths.outputs.roofmodel }} web: ${{ steps.paths.outputs.web }} drivers: ${{ steps.paths.outputs.drivers }} compose: ${{ steps.paths.outputs.compose }} @@ -43,6 +44,7 @@ jobs: set -euo pipefail core=false optimizer=false + roofmodel=false web=false drivers=false compose=false @@ -63,6 +65,12 @@ jobs: optimizer/*|Dockerfile.optimizer|go/internal/mpc/*|go/cmd/ftw/main.go) optimizer=true ;; + # Only the Python module. Its Go counterpart lives under go/, + # which the core suite already covers -- and a `go/...` pattern + # here would never fire anyway, since the `go/*` arm above wins. + roofmodel/*) + roofmodel=true + ;; web/*|package.json|package-lock.json) web=true ;; @@ -75,6 +83,7 @@ jobs: Makefile|.github/workflows/test.yml) core=true optimizer=true + roofmodel=true web=true drivers=true compose=true @@ -96,6 +105,7 @@ jobs: { echo "core=${core}" echo "optimizer=${optimizer}" + echo "roofmodel=${roofmodel}" echo "web=${web}" echo "drivers=${drivers}" echo "compose=${compose}" @@ -157,6 +167,28 @@ jobs: go test -count=1 ./internal/mpc -run 'TestExternalOptimizer(EndToEnd|PlansMultipleLoadpoints|PlansAndValidatesMultipleStorages)$' + # The roof-geometry module is optional at runtime but not optional to verify: + # it decides the tilt and azimuth every PV forecast is then built on, and a + # silently wrong plane fit looks exactly like a working one. Installed without + # the `geo` extra on purpose -- LAZ decoding pulls a compiled backend, and + # everything except the point-cloud read is exercised without it. + roofmodel: + name: roofmodel (Python) + needs: changes + if: needs.changes.outputs.roofmodel == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + cache: pip + cache-dependency-path: roofmodel/pyproject.toml + - name: Install + run: pip install -e 'roofmodel[test]' + - name: Python tests + run: pytest -q roofmodel/tests + web: name: web needs: changes @@ -412,12 +444,13 @@ jobs: test: name: go test + vet if: always() - needs: [changes, core, optimizer, web, drivers, device-support-contract, compose, e2e] + needs: [changes, core, optimizer, roofmodel, web, drivers, device-support-contract, compose, e2e] runs-on: ubuntu-latest env: RESULTS: >- ${{ needs.changes.result }} ${{ needs.core.result }} - ${{ needs.optimizer.result }} ${{ needs.web.result }} + ${{ needs.optimizer.result }} ${{ needs.roofmodel.result }} + ${{ needs.web.result }} ${{ needs.drivers.result }} ${{ needs.device-support-contract.result }} ${{ needs.compose.result }} ${{ needs.e2e.result }} diff --git a/.gitignore b/.gitignore index 127e881d..c2893bc0 100644 --- a/.gitignore +++ b/.gitignore @@ -65,7 +65,11 @@ node_modules/ release-notes.md optimizer/.venv/ optimizer/.pytest_cache/ -optimizer/**/__pycache__/ +# Python build artefacts from any module, not just the optimizer — roofmodel/ +# is a second one, and a third would otherwise repeat this again. +**/__pycache__/ +**/.pytest_cache/ +*.pyc # TLS material — never commit. *.pem diff --git a/README.md b/README.md index bf287419..03a2eb72 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,7 @@ metadata are the detailed reference. - [Product roadmap](docs/roadmap.md) - [Power sign convention](docs/site-convention.md) - [Safety invariants](docs/safety.md) +- [Geographic coverage of external data](docs/data-coverage.md) - [Operations and recovery](docs/operations.md) - [Full backup and safe restore](docs/backup-and-restore.md) - [Writing a driver](docs/writing-a-driver.md) diff --git a/docs/data-coverage.md b/docs/data-coverage.md new file mode 100644 index 00000000..a38a678a --- /dev/null +++ b/docs/data-coverage.md @@ -0,0 +1,154 @@ +# Geographic coverage of external data sources + +FTW controls hardware anywhere, but it depends on external data for three +things: **spot prices**, **weather/PV forecasts** and **PV performance +scoring**. Those three have very different geographic reach, and the difference +decides how much of FTW is useful at a given site. + +Short version: + +- **Weather and PV forecasting works worldwide.** +- **Price-driven planning works in Europe only.** +- **PV performance scoring works in the Nordic region only.** +- **Roof geometry (planned) is Sweden only.** + +A site outside Europe can still run FTW for monitoring, safety and control — but +the economic optimisation that motivates most of the planner has no price source +to work from. + +`GET /api/data-sources` answers this per site: it returns every source with its +coverage area and, when the site location is known, whether that source reaches +it. The Weather settings tab renders the same data under the map. This file is +the prose; `go/internal/coverage` is the machine-readable source of truth, and +the two are meant to stay in step. + +> **Coverage bounds are advisory.** Each bounded source declares a lat/lon box, +> but STRÅNG's model grid is rotated relative to lat/lon, so its box is a +> *superset* of the real domain — points near a corner pass the box test and +> still return nothing. Treat `covers: false` as definitive and `covers: true` +> as "worth trying". The upstream API is always the final word. + +## Spot prices — Europe only + +Configured under `price.provider`. + +| Provider | Coverage | API key | Notes | +|---|---|---|---| +| `sourceful` | European day-ahead markets | No | Default. Sourceful's cached ENTSO-E API. | +| `elprisetjustnu` | **Sweden only** — zones SE1–SE4 | No | 15-minute PTU since late 2025. | +| `entsoe` | ENTSO-E member markets (most of Europe) | Yes | Direct from the Transparency Platform. | +| `none` | — | — | Disables price fetching entirely. | + +There is **no provider for any market outside Europe**. North America (CAISO, +ERCOT, PJM, ISO-NE, NYISO, MISO, SPP, AESO, IESO), Australia (AEMO/NEM), Japan +(JEPX) and everywhere else are unsupported, and there is no manual or +fixed-tariff provider to stand in for them. + +Two further Europe-centric assumptions live in the price layer: prices are +stored internally in **öre** (1 SEK = 100 öre), and ENTSO-E's EUR/MWh figures +are converted using **ECB** daily FX rates. + +> The Tibber driver (`drivers/tibber.lua`) is telemetry only — it reports meter +> readings, not prices, so it is not a fourth price source. + +## Weather and PV forecasts — worldwide + +Configured under `weather.provider`. All four work at any latitude/longitude. + +| Provider | Coverage | API key | Signal quality | +|---|---|---|---| +| `met_no` | Global | No | Cloud cover only — weakest PV signal. | +| `openweather` | Global | Yes | Cloud cover only. | +| `open_meteo` | Global | No | Shortwave radiation (GHI) — good. | +| `forecast_solar` | Global | No (free tier) | Site-calibrated watts from panel geometry — best. | + +Accuracy varies by region because the underlying numerical weather models do, +but none of these are geographically gated. Outside the Nordics, prefer +`open_meteo` or `forecast_solar`: they carry an irradiance signal, which is what +the orientation-aware plane-of-array model needs. + +## PV performance scoring — Nordic region only + +The scorer (`GET /api/pv/performance`) compares measured production against a +physics baseline built from **SMHI STRÅNG** irradiance. STRÅNG is a mesoscale +analysis product covering the **Nordic region** hourly at ~2.5 km from 1999 to +roughly one day ago. It is free, keyless and CC BY 4.0. + +Outside that domain STRÅNG returns no data, so scoring simply never produces +rows and the dashboard overlay stays hidden. Nothing fails loudly; the feature +is just unavailable. + +Because the same scoring feeds the **forecast calibration factor**, sites +outside the STRÅNG domain also do not get measured calibration of their PV +forecast — they fall back to the uncalibrated physics estimate. + +STRÅNG has **no forward horizon**. It is never used as a forecast provider; see +[architecture.md](architecture.md) for where it sits. + +### What STRÅNG actually publishes + +Probing the live API on 2026-07-31 (SMHI's own apidocs pages currently 404) +returned data for exactly seven parameters and 404 for everything else. Names +were confirmed from their magnitudes on a clear day rather than from docs: + +| Code | Quantity | Unit | Noon value, Stockholm 2026-06-21 | +|---|---|---|---| +| 116 | CIE-weighted UV irradiance | mW/m² | 146.6 | +| 117 | **Global horizontal (GHI)** | W/m² | 810.5 | +| 118 | Direct normal (DNI) | W/m² | 917.2 | +| 119 | **Sunshine duration** | min/h | 60.0 | +| 120 | Photosynthetically active radiation | W/m² | 357.9 | +| 121 | Direct horizontal | W/m² | 723.0 | +| 122 | **Diffuse horizontal (DHI)** | W/m² | 87.5 | + +Two checks confirm the identification: 121 + 122 = 723.0 + 87.5 = 810.5, exactly +parameter 117 (direct + diffuse = global), and 119 caps at exactly 60, i.e. +minutes within the hour. + +**STRÅNG publishes no cloud cover.** It is a radiation model; cloudiness is not +among its outputs. It is however *derivable*: parameter 119 counts the minutes +in each hour during which direct beam irradiance exceeded the WMO sunshine +threshold, so `1 − minutes/60` is the fraction of the hour the sun spent +obscured. That is an observed quantity rather than an inferred cloud field, but +it is coarser than a forecast provider's cloud percentage — it cannot see thin +cirrus that dims without blocking. FTW exposes it via +`strang.IrradianceHour.CloudCover(lat, lon)`, which returns an explicit +"unknown" rather than defaulting to "clear". + +The location argument is not decoration. Sunshine duration is zero at night for +the trivial reason that there is no sun, and zero again near sunrise and sunset +because the beam crosses ten or more air masses and cannot reach the 120 W/m² +threshold even under a spotless sky. Both would read as "100% overcast" if taken +at face value. `CloudCover` therefore declines to answer unless the sun clears +**5° of elevation** at some point in the hour, sampling the hour's start, +midpoint and end so the hour in which the sun crosses that line is still counted. + +Live data from 2026-06-21 at Stockholm shows the distinction: + +| Hour (UTC) | GHI W/m² | Sunshine | Cloud cover | +|---|---|---|---| +| 00:00 | 0.0 | 0 min | *unknown* — sun below horizon | +| 04:00 | 163.2 | 60 min | 0% | +| 12:00 | 810.5 | 60 min | 0% | +| 20:00 | 2.5 | 0 min | *unknown* — sun minutes from setting | + +## Roof geometry — Sweden only (planned) + +The roof-derivation module proposed in +[RFC #717](https://github.com/srcfl/ftw/discussions/717) reads **Lantmäteriet** +building footprints and LiDAR, which exist for **Sweden only** and require a +Geotorget account. Everywhere else, panel tilt/azimuth/kWp stays a manual entry +in the Weather settings tab — which is the fallback by design, not a +degraded mode. + +## What a non-European site loses + +| Capability | Works outside Europe? | +|---|---| +| Device control, safety, dispatch | Yes | +| Telemetry, history, dashboard | Yes | +| Weather + PV forecasting | Yes | +| Self-learning PV twin | Yes | +| Price-driven planning / optimisation | **No** — no price source | +| PV performance scoring + calibration | **No** — outside STRÅNG's domain | +| Automatic roof geometry | **No** — Sweden only | diff --git a/docs/roof-geometry.md b/docs/roof-geometry.md new file mode 100644 index 00000000..f6d90d10 --- /dev/null +++ b/docs/roof-geometry.md @@ -0,0 +1,134 @@ +# Roof geometry from Lantmäteriet + +FTW's PV forecast needs the tilt and azimuth of each roof face. Typing them in +means measuring your own roof, and most people estimate. In Sweden the state +already flew a laser over it, so FTW can read the numbers instead. + +This is **optional and Sweden-only**. Everywhere else, and whenever anything +below is missing, the numeric fields in **Settings → Weather → PV arrays** stay +the way they work today. + +## What you need + +A free [Geotorget](https://geotorget.lantmateriet.se) account with access +ordered to two products. Both are open data under CC BY 4.0; the account exists +so Lantmäteriet can see who is downloading, not to charge you. + +| Product | Delivered as | What FTW uses it for | +|---|---|---| +| [Byggnad Nedladdning, vektor](https://geotorget.lantmateriet.se/geodataprodukter/byggnad-nedladdning-vektor-api) | STAC → **GeoPackage** | Building footprints, so you can point at your house | +| [Laserdata Nedladdning, Skog](https://geotorget.lantmateriet.se/geodataprodukter/laserdata-nedladdning-skog-api) | STAC → **LAZ as COPC** | The laser scan the roof planes are fitted to | + +Both are STAC APIs behind the same account, so one set of credentials covers +both and FTW searches them the same way. They differ only in what the items +point at, and FTW picks the right asset by its declared media type rather than +by its name — a catalogue that renames `data` to `punktmoln` keeps working. + +Ordering access is not instant — Lantmäteriet approves it — so do it before you +plan to use this. + +You also need the `roofmodel` module's dependencies on the FTW host: + +```bash +pip install -e roofmodel[geo] +``` + +The `geo` extra pulls the LAZ reader. Without it everything except reading the +point cloud works, which is enough to run the tests but not enough to derive a +real roof. GeoPackage needs nothing extra: it is a SQLite file, and FTW reads it +with the standard library. + +### Why picking a building also makes it fast + +COPC — Cloud Optimized Point Cloud — is LAZ with the points ordered into an +octree and an index at a known offset, so a reader can ask for a region and +fetch only the parts that cover it. Once you have picked a building, FTW asks +for the bounding box of *that footprint* instead of the tile: a Laserdata Skog +tile covers 2.5 km and runs to hundreds of megabytes, and a house is a few tens +of metres across. + +This is best-effort. A plain (non-COPC) `.laz` asset, a server that ignores +`Range` requests, or a laspy without COPC support all fall back to reading the +tile whole — slower, same answer. The result records which path ran as +`source.fetch`: `copc-window` or `whole-tile`. + +## Using it + +1. Open **Settings → Weather**. +2. Put the map marker on your building. +3. Under **Roof geometry from Lantmäteriet**, tick **Enable roof derivation**, + enter your Geotorget username and token, and **Save**. +4. Press **Find buildings here**. Footprints appear on the map and as a list. +5. Click your building. It highlights green. +6. Press **Read roof from LiDAR**. + +The PV arrays above fill in with one entry per usable roof face. **Nothing is +saved yet** — look at the numbers, correct anything that is wrong, then press +Save. FTW never rewrites your panel configuration on its own: the derivation is +a good guess from a scan that may be several years old, and you are the one who +knows whether there are panels on that face at all. + +## What the numbers mean, and what they do not + +- **Tilt and azimuth** come from a plane fitted to the laser returns on your + roof. These are the values worth trusting. +- **kWp** is an *upper bound on what fits*: roof area × a packing factor (0.70 + by default, covering ridges, eaves, chimneys and walkways) × 200 W/m². It is + not what you have installed. If you have six panels on a face that could hold + twenty, correct it. +- **North-facing pitched faces are dropped.** At Swedish latitudes they yield + too little to be worth proposing. Flat roofs are kept, since panels on them + get mounted facing south regardless of which way the building points. +- **Capture date** is shown with the result. Lantmäteriet is still backfilling + the STAC `datetime` field through 2026, so it sometimes reads "date unknown". + A roof built after the scan will not be in the data at all — you will get a + clear error rather than a wrong answer. + +## Why you pick a building + +Without a footprint the module segments everything within its radius: your +neighbour's roof, the garage, the trees. Worse, the plane fitting is *global* — +a fitted plane is infinite, so a roof at azimuth 180° is described by `z = f(y)` +with no `x` term at all and does not stop at your wall. A second building +sharing your pitch and ridge orientation is not on a *similar* plane; it is on +the same one. + +Measured on a synthetic pair, a single fitting pass over a house and a garage +40 m apart swallowed **all** of both south faces — 576 returns from one and 256 +from the other — as one surface. The result was identical at every separation +from 3 m to 40 m, which is what tells you it is a global effect and not a +proximity one. Only the clustering step afterwards told the two buildings apart. + +Clipping to your footprint first gives exactly the same face you would get by +scanning your building alone. Picking a building is what makes the answer yours. + +## Optional: shading + +Tilt and azimuth predict an unobstructed roof. They say nothing about the spruce +to the south, which can cost more than any amount of azimuth. + +If you install [vostok](https://github.com/3dgeo-heidelberg/vostok) and set +`roofmodel.vostok_binary`, each face is additionally run against the surrounding +geometry and gets a `shading_factor`. + +vostok is **GPL-3.0 and is not part of FTW**. FTW never bundles it, never ships +it and never installs it; you install it yourself and point FTW at it. It is run +as a separate process communicating through files, which is what keeps the +licences apart. Without it, faces carry no shading factor at all — deliberately +distinct from a factor of 1.0, because "we did not look" and "we looked and it +is clear" are different claims. + +## When it does not work + +| What you see | What it means | +|---|---| +| "Roof derivation is off" | Tick the box, add credentials, Save, retry | +| "Geotorget rejected the credentials" | Wrong token, or the account has not been granted that product | +| "No buildings found here" | The marker is not on a building, or you are outside Sweden | +| "only N LiDAR returns fall on building" | The building is newer than the scan, or you picked the wrong footprint | +| "No roof faces worth mounting panels on" | Everything found was north-facing or under 8 m² | +| "the building tile could not be read" | The GeoPackage asset was not a GeoPackage — usually a changed download URL | + +Failures never change your configuration. + +*Data © Lantmäteriet, CC BY 4.0.* diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 509f7946..dc53e770 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -53,6 +53,8 @@ import ( "github.com/srcfl/ftw/go/internal/prices" "github.com/srcfl/ftw/go/internal/proxy" "github.com/srcfl/ftw/go/internal/pvmodel" + "github.com/srcfl/ftw/go/internal/pvperf" + "github.com/srcfl/ftw/go/internal/roofmodel" "github.com/srcfl/ftw/go/internal/selftune" "github.com/srcfl/ftw/go/internal/selfupdate" "github.com/srcfl/ftw/go/internal/state" @@ -992,6 +994,30 @@ func main() { "lat", forecastSvc.Lat, "lon", forecastSvc.Lon, "rated_pv_w", ratedPVW) } + // ---- Start PV performance scoring (optional) ---- + // Nightly backfill of SMHI STRÅNG historical irradiance + expected-vs-actual + // PV scoring. Nil when the site has no PV geometry to score against. This is + // read-only with respect to control: it only fetches weather data and writes + // the irradiance_history + pv_performance_daily tables. + // Optional roof-geometry module: nil unless explicitly enabled, and + // stateless — it only runs when an operator asks for a derive. + roofModelSvc := roofmodel.FromConfig(cfg.RoofModel) + + pvPerfSvc := pvperf.FromConfig(cfg.Weather, ratedPVW, st, + "ftw/"+Version+" github.com/srcfl/ftw") + if pvPerfSvc != nil { + pvPerfSvc.Start(ctx) + defer pvPerfSvc.Stop() + // Close the loop: measured performance calibrates the forward + // forecast. The hook is read at fetch time, so it starts correcting + // as soon as enough days are scored — no restart needed. + if forecastSvc != nil { + forecastSvc.Calibration = pvPerfSvc.CalibrationFactor + } + slog.Info("pv performance scoring started", + "lat", pvPerfSvc.Lat, "lon", pvPerfSvc.Lon, "arrays", len(pvPerfSvc.Arrays)) + } + // ---- Start PV digital twin (optional, requires weather config) ---- // pvSvc is pre-declared above so the reload Applier can update it. if cfg.Weather != nil && cfg.Weather.Provider != "" && cfg.Weather.Provider != "none" { @@ -2081,6 +2107,8 @@ func main() { SnapshotDir: filepath.Join(filepath.Dir(statePath), "snapshots"), Prices: priceSvc, Forecast: forecastSvc, + PVPerf: pvPerfSvc, + RoofModel: roofModelSvc, MPC: mpcSvc, PVModel: pvSvc, LoadModel: loadSvc, diff --git a/go/internal/api/api.go b/go/internal/api/api.go index fa5ed888..0507bdba 100644 --- a/go/internal/api/api.go +++ b/go/internal/api/api.go @@ -27,6 +27,8 @@ import ( "github.com/srcfl/ftw/go/internal/battery" "github.com/srcfl/ftw/go/internal/calendar" "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/coverage" + "github.com/srcfl/ftw/go/internal/roofmodel" "github.com/srcfl/ftw/go/internal/control" "github.com/srcfl/ftw/go/internal/driverrepo" "github.com/srcfl/ftw/go/internal/drivers" @@ -40,6 +42,7 @@ import ( "github.com/srcfl/ftw/go/internal/notifications" "github.com/srcfl/ftw/go/internal/prices" "github.com/srcfl/ftw/go/internal/pvmodel" + "github.com/srcfl/ftw/go/internal/pvperf" "github.com/srcfl/ftw/go/internal/scanner" "github.com/srcfl/ftw/go/internal/selftune" "github.com/srcfl/ftw/go/internal/selfupdate" @@ -106,6 +109,14 @@ type Deps struct { Prices *prices.Service Forecast *forecast.Service + // Optional: STRÅNG-based PV performance scoring. Nil when the site has no + // PV geometry to score against (surfaced as {enabled:false}). + PVPerf *pvperf.Service + + // RoofModel is the optional Lantmäteriet roof-geometry module. nil when the + // module is absent or disabled, which is the normal case outside Sweden. + RoofModel *roofmodel.Service + // Optional: MPC planner. Nil if disabled. MPC *mpc.Service @@ -344,6 +355,11 @@ func (s *Server) routes() { s.handle("POST /api/pv/manual_hold", s.handlePVManualHold) s.handle("DELETE /api/pv/manual_hold", s.handlePVManualHoldClear) s.handle("GET /api/pv/manual_hold", s.handlePVManualHoldGet) + s.handle("GET /api/pv/performance", s.handlePVPerformance) + s.handle("GET /api/data-sources", s.handleDataSources) + s.handle("GET /api/roofmodel", s.handleRoofModel) + s.handle("GET /api/roofmodel/buildings", s.handleRoofModelBuildings) + s.handle("POST /api/roofmodel/derive", s.handleRoofModelDerive) s.handle("GET /api/version/check", s.handleVersionCheck) s.handle("POST /api/version/channel", s.handleVersionChannel) s.handle("POST /api/version/skip", s.handleVersionSkip) @@ -2018,6 +2034,272 @@ func (s *Server) handleForecast(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, map[string]any{"items": rows, "enabled": true}) } +// ---- /api/roofmodel ---- +// +// GET reports whether roof derivation is available for this site and why not +// when it is unavailable. GET /api/roofmodel/buildings lists footprints to pick +// from; POST /api/roofmodel/derive fits the roof of the picked one. +// +// The derived arrays are returned, never written. The settings form fills +// itself in with them and the operator saves -- so the panel config still +// changes only when someone looks at the numbers and agrees. Derivation is a +// best guess from a point cloud that may be years old. +func (s *Server) handleRoofModel(w http.ResponseWriter, r *http.Request) { + lat, lon, haveSite := s.siteLocation() + resp := map[string]any{"enabled": s.deps.RoofModel.Enabled()} + if haveSite { + resp["covers"] = coverage.Covers("lantmateriet", lat, lon) + resp["latitude"], resp["longitude"] = lat, lon + } + resp["has_credentials"] = s.roofModelHasCredentials() + if src, ok := coverage.ByID("lantmateriet"); ok { + resp["area"] = src.Area + resp["license"] = src.License + resp["note"] = src.Note + } + writeJSON(w, 200, resp) +} + +func (s *Server) handleRoofModelDerive(w http.ResponseWriter, r *http.Request) { + if !s.deps.RoofModel.Enabled() { + writeJSON(w, 200, map[string]any{ + "enabled": false, + "error": "roof model module is not enabled", + }) + return + } + lat, lon, haveSite := s.siteLocation() + if !haveSite { + writeJSON(w, 400, map[string]any{"error": "site latitude/longitude is not configured"}) + return + } + // Optional: which footprint to clip the LiDAR to. An absent or empty body + // keeps the old behaviour of segmenting the whole search radius. + var body struct { + BuildingID string `json:"building_id"` + } + if r.Body != nil { + _ = json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&body) + } + + // A derive downloads and segments LiDAR tiles; the service time-boxes it, + // but the request should also die with the client rather than outliving it. + model, err := s.deps.RoofModel.Derive(r.Context(), lat, lon, body.BuildingID) + if err != nil { + writeJSON(w, roofModelErrorStatus(err), map[string]any{"error": err.Error()}) + return + } + writeJSON(w, 200, map[string]any{ + "enabled": true, + "model": model, + "proposed_arrays": model.ToPVArrays(), + }) +} + +// handleRoofModelBuildings lists building footprints near the site for the +// picker. Read-only and cheap relative to a derive: it is one STAC search with +// no LiDAR download behind it. +func (s *Server) handleRoofModelBuildings(w http.ResponseWriter, r *http.Request) { + if !s.deps.RoofModel.Enabled() { + writeJSON(w, 200, map[string]any{ + "enabled": false, + "error": "roof model module is not enabled", + }) + return + } + lat, lon, haveSite := s.siteLocation() + // The map lets you drag the pin before saving, so honour an explicit + // coordinate over the stored one. + if v, err := strconv.ParseFloat(r.URL.Query().Get("lat"), 64); err == nil { + if w2, err2 := strconv.ParseFloat(r.URL.Query().Get("lon"), 64); err2 == nil { + lat, lon, haveSite = v, w2, true + } + } + if !haveSite { + writeJSON(w, 400, map[string]any{"error": "site latitude/longitude is not configured"}) + return + } + + list, err := s.deps.RoofModel.Buildings(r.Context(), lat, lon) + if err != nil { + writeJSON(w, roofModelErrorStatus(err), map[string]any{"error": err.Error()}) + return + } + writeJSON(w, 200, map[string]any{ + "enabled": true, + "latitude": lat, + "longitude": lon, + "buildings": list.Buildings, + }) +} + +// roofModelErrorStatus separates "you asked for something impossible" from +// "the module or Lantmateriet failed", so the UI can tell the operator to fix +// their input rather than to retry. +func roofModelErrorStatus(err error) int { + if errors.Is(err, roofmodel.ErrOutsideCoverage) || errors.Is(err, roofmodel.ErrNoCredentials) { + return 400 + } + return 502 +} + +// roofModelHasCredentials reports whether a Geotorget token is stored, without +// revealing it. +func (s *Server) roofModelHasCredentials() bool { + if s.deps.CfgMu == nil { + return false + } + s.deps.CfgMu.RLock() + defer s.deps.CfgMu.RUnlock() + if s.deps.Cfg == nil || s.deps.Cfg.RoofModel == nil { + return false + } + rm := s.deps.Cfg.RoofModel + return rm.GeotorgetUsername != "" && rm.GeotorgetToken != "" +} + +// siteLocation returns the configured site coordinates. +func (s *Server) siteLocation() (lat, lon float64, ok bool) { + if s.deps.CfgMu == nil { + return 0, 0, false + } + s.deps.CfgMu.RLock() + defer s.deps.CfgMu.RUnlock() + if s.deps.Cfg == nil || s.deps.Cfg.Weather == nil { + return 0, 0, false + } + lat, lon = s.deps.Cfg.Weather.Latitude, s.deps.Cfg.Weather.Longitude + return lat, lon, lat != 0 || lon != 0 +} + +// ---- /api/data-sources ---- +// +// Where each external data source works, and whether it covers this site. +// Response: {latitude, longitude, sources:[{id, kind, label, area, countries, +// worldwide, requires_key, license, note, covers}]}. `covers` is advisory: for +// a bounded source it is a lat/lon box test, and STRÅNG's grid is rotated, so a +// true near a corner still means "worth trying", not "guaranteed". False is +// reliable — that location is definitely not served. +// +// This exists because several sources are regional (STRÅNG is Nordic-only, +// every price provider is European) and nothing previously said so: a site +// outside those areas got an empty result and no explanation. See #726. +func (s *Server) handleDataSources(w http.ResponseWriter, r *http.Request) { + var lat, lon float64 + var haveSite bool + // Weather is an optional config section, so it is nil on a site that has + // never configured one — which is exactly the site most likely to be + // looking at this endpoint. + lat, lon, haveSite = s.siteLocation() + + // An explicit ?lat=&lon= overrides the configured site so the Weather tab + // can preview coverage for a pin the operator is still dragging around, + // before they save it. + if v := r.URL.Query().Get("lat"); v != "" { + if f, err := strconv.ParseFloat(v, 64); err == nil { + lat, haveSite = f, true + } + } + if v := r.URL.Query().Get("lon"); v != "" { + if f, err := strconv.ParseFloat(v, 64); err == nil { + lon, haveSite = f, true + } + } + + items := make([]map[string]any, 0, len(coverage.All())) + for _, src := range coverage.All() { + item := map[string]any{ + "id": src.ID, + "kind": string(src.Kind), + "label": src.Label, + "area": src.Area, + "worldwide": src.Worldwide(), + "requires_key": src.RequiresKey, + } + if len(src.Countries) > 0 { + item["countries"] = src.Countries + } + if src.License != "" { + item["license"] = src.License + } + if src.Note != "" { + item["note"] = src.Note + } + // Without a site location there is nothing to test against, so omit + // `covers` entirely rather than defaulting it to a misleading true. + if haveSite { + item["covers"] = src.Covers(lat, lon) + } + items = append(items, item) + } + resp := map[string]any{"sources": items} + if haveSite { + resp["latitude"], resp["longitude"] = lat, lon + } + writeJSON(w, 200, resp) +} + +// ---- /api/pv/performance ---- +// +// STRÅNG-based expected-vs-actual PV performance scoring. Query param days=N +// (default 30, max 365) selects the lookback window. Response: +// {enabled, items:[{day, expected_wh, actual_wh, pr, ...}], performance_ratio, +// attribution}. Returns {enabled:false} when scoring is unavailable (no PV +// geometry configured). +func (s *Server) handlePVPerformance(w http.ResponseWriter, r *http.Request) { + if s.deps.PVPerf == nil { + writeJSON(w, 200, map[string]any{"items": []any{}, "enabled": false}) + return + } + days := 30 + if v := r.URL.Query().Get("days"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + days = n + } + } + if days > 365 { + days = 365 + } + now := time.Now() + loc := now.Location() + today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, loc) + sinceDay := today.AddDate(0, 0, -days).Format("2006-01-02") + untilDay := today.Format("2006-01-02") + items, err := s.deps.PVPerf.Load(sinceDay, untilDay) + if err != nil { + writeJSON(w, 500, map[string]string{"error": err.Error()}) + return + } + // Energy-weighted overall performance ratio across the window — only days + // with a meaningful expected baseline (pr != null) contribute. + var sumExpected, sumActual float64 + for _, it := range items { + if it.PR != nil { + sumExpected += it.ExpectedWh + sumActual += it.ActualWh + } + } + // The calibration is reported over its own fixed window rather than the + // caller's, so a short ?days= request cannot make the site look + // uncalibrated. "applied" is what actually reaches the forward forecast. + cal := s.deps.PVPerf.Calibration() + resp := map[string]any{ + "items": items, + "enabled": true, + "attribution": "Irradiance: SMHI STRÅNG (CC BY 4.0)", + "calibration": map[string]any{ + "factor": cal.Factor, + "sigma_rel": cal.SigmaRel, + "days": cal.Days, + "applied": cal.Valid, + }, + } + if sumExpected > 0 { + resp["performance_ratio"] = sumActual / sumExpected + } + writeJSON(w, 200, resp) +} + // ---- MPC planner ---- func (s *Server) handleMPCPlan(w http.ResponseWriter, r *http.Request) { diff --git a/go/internal/api/api_datasources_test.go b/go/internal/api/api_datasources_test.go new file mode 100644 index 00000000..74fd3dde --- /dev/null +++ b/go/internal/api/api_datasources_test.go @@ -0,0 +1,157 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "github.com/srcfl/ftw/go/internal/config" +) + +type dataSource struct { + ID string `json:"id"` + Kind string `json:"kind"` + Label string `json:"label"` + Area string `json:"area"` + Countries []string `json:"countries"` + Worldwide bool `json:"worldwide"` + RequiresKey bool `json:"requires_key"` + Note string `json:"note"` + Covers *bool `json:"covers"` +} + +type dataSourcesResp struct { + Latitude *float64 `json:"latitude"` + Longitude *float64 `json:"longitude"` + Sources []dataSource `json:"sources"` +} + +func getDataSources(t *testing.T, deps *Deps, query string) dataSourcesResp { + t.Helper() + srv := New(deps) + req := httptest.NewRequest(http.MethodGet, "/api/data-sources"+query, nil) + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, req) + if rr.Code != 200 { + t.Fatalf("status = %d, want 200", rr.Code) + } + var resp dataSourcesResp + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + return resp +} + +func depsAt(lat, lon float64) *Deps { + cfg := &config.Config{Weather: &config.Weather{Latitude: lat, Longitude: lon}} + return &Deps{Cfg: cfg, CfgMu: &sync.RWMutex{}} +} + +func find(t *testing.T, resp dataSourcesResp, id string) dataSource { + t.Helper() + for _, s := range resp.Sources { + if s.ID == id { + return s + } + } + t.Fatalf("source %q missing from response", id) + return dataSource{} +} + +func TestDataSourcesListsEverySource(t *testing.T) { + resp := getDataSources(t, depsAt(59.33, 18.07), "") + for _, id := range []string{ + "met_no", "openweather", "open_meteo", "forecast_solar", + "strang", "sourceful", "elprisetjustnu", "entsoe", + } { + find(t, resp, id) // fails the test if absent + } +} + +// A Nordic site: STRÅNG and the Swedish price feed both apply. +func TestDataSourcesCoversNordicSite(t *testing.T) { + resp := getDataSources(t, depsAt(59.33, 18.07), "") + for _, id := range []string{"strang", "elprisetjustnu", "sourceful", "open_meteo"} { + s := find(t, resp, id) + if s.Covers == nil || !*s.Covers { + t.Errorf("%s: want covers=true for Stockholm", id) + } + } +} + +// The case that motivated this endpoint: outside Europe the forecast still +// works, but irradiance scoring and every price provider do not. +func TestDataSourcesExplainsWhySydneyIsLimited(t *testing.T) { + resp := getDataSources(t, depsAt(-33.87, 151.21), "") + + for _, id := range []string{"met_no", "openweather", "open_meteo", "forecast_solar"} { + s := find(t, resp, id) + if s.Covers == nil || !*s.Covers { + t.Errorf("%s: forecast providers are worldwide, want covers=true", id) + } + } + for _, id := range []string{"strang", "sourceful", "elprisetjustnu", "entsoe"} { + s := find(t, resp, id) + if s.Covers == nil || *s.Covers { + t.Errorf("%s: want covers=false in Sydney", id) + } + if s.Note == "" && s.Area == "" { + t.Errorf("%s: an uncovered source must still explain its area", id) + } + } +} + +// The Weather tab previews a pin before it is saved, so an explicit lat/lon +// must override the configured site. +func TestDataSourcesQueryOverridesConfiguredSite(t *testing.T) { + deps := depsAt(59.33, 18.07) // configured: Stockholm + resp := getDataSources(t, deps, "?lat=-33.87&lon=151.21") + if s := find(t, resp, "strang"); s.Covers == nil || *s.Covers { + t.Error("query lat/lon should override config and report not covered") + } + if resp.Latitude == nil || *resp.Latitude != -33.87 { + t.Errorf("latitude = %v, want the overridden -33.87", resp.Latitude) + } +} + +// With no location configured there is nothing to test against, so `covers` +// must be absent rather than defaulting to a misleading true. +func TestDataSourcesOmitsCoversWithoutASite(t *testing.T) { + resp := getDataSources(t, &Deps{}, "") + if len(resp.Sources) == 0 { + t.Fatal("sources should still be listed without a site") + } + for _, s := range resp.Sources { + if s.Covers != nil { + t.Errorf("%s: covers should be omitted when no site is known", s.ID) + } + } + if resp.Latitude != nil || resp.Longitude != nil { + t.Error("latitude/longitude should be omitted when no site is known") + } +} + +// Metadata is the whole point of the endpoint; assert it actually arrives. +func TestDataSourcesCarriesRegionMetadata(t *testing.T) { + resp := getDataSources(t, depsAt(59.33, 18.07), "") + + strang := find(t, resp, "strang") + if strang.Worldwide { + t.Error("strang must not be reported worldwide") + } + if strang.Area == "" || len(strang.Countries) == 0 { + t.Error("strang should carry an area and country list") + } + if strang.RequiresKey { + t.Error("strang needs no API key") + } + + if ow := find(t, resp, "openweather"); !ow.RequiresKey { + t.Error("openweather requires an API key") + } + if mn := find(t, resp, "met_no"); !mn.Worldwide { + t.Error("met_no is worldwide") + } +} diff --git a/go/internal/api/api_pvperf_test.go b/go/internal/api/api_pvperf_test.go new file mode 100644 index 00000000..17107bb1 --- /dev/null +++ b/go/internal/api/api_pvperf_test.go @@ -0,0 +1,91 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/pvperf" + "github.com/srcfl/ftw/go/internal/state" +) + +func TestPVPerformanceDisabled(t *testing.T) { + srv := New(&Deps{}) // no PVPerf service + req := httptest.NewRequest(http.MethodGet, "/api/pv/performance", nil) + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, req) + + if rr.Code != 200 { + t.Fatalf("status = %d, want 200", rr.Code) + } + var resp struct { + Enabled bool `json:"enabled"` + Items []any `json:"items"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.Enabled { + t.Error("enabled should be false when no PVPerf service is wired") + } + if len(resp.Items) != 0 { + t.Errorf("items should be empty, got %d", len(resp.Items)) + } +} + +func TestPVPerformanceEnabledReturnsScores(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + + // Seed two recent scored days (relative to now so the default window covers them). + now := time.Now() + pr := 0.9 + for i := 1; i <= 2; i++ { + day := now.AddDate(0, 0, -i).Format("2006-01-02") + if err := st.SavePVPerformance(state.PVPerformanceDay{ + Day: day, ExpectedWh: 10000, ActualWh: 9000, PR: &pr, + }); err != nil { + t.Fatal(err) + } + } + + srv := New(&Deps{PVPerf: &pvperf.Service{Store: st}}) + req := httptest.NewRequest(http.MethodGet, "/api/pv/performance?days=30", nil) + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, req) + + if rr.Code != 200 { + t.Fatalf("status = %d, want 200", rr.Code) + } + var resp struct { + Enabled bool `json:"enabled"` + PerformanceRatio *float64 `json:"performance_ratio"` + Attribution string `json:"attribution"` + Items []struct { + Day string `json:"day"` + ExpectedWh float64 `json:"expected_wh"` + ActualWh float64 `json:"actual_wh"` + } `json:"items"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if !resp.Enabled { + t.Error("enabled should be true") + } + if len(resp.Items) != 2 { + t.Fatalf("want 2 scored days, got %d", len(resp.Items)) + } + if resp.PerformanceRatio == nil || *resp.PerformanceRatio < 0.89 || *resp.PerformanceRatio > 0.91 { + t.Errorf("energy-weighted PR should be ~0.9, got %v", resp.PerformanceRatio) + } + if resp.Attribution == "" { + t.Error("attribution (SMHI STRÅNG CC BY 4.0) should be present") + } +} diff --git a/go/internal/api/api_roofmodel_test.go b/go/internal/api/api_roofmodel_test.go new file mode 100644 index 00000000..8e65f8ea --- /dev/null +++ b/go/internal/api/api_roofmodel_test.go @@ -0,0 +1,174 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/roofmodel" +) + +func getJSON(t *testing.T, deps *Deps, method, path string) (int, map[string]any) { + t.Helper() + srv := New(deps) + req := httptest.NewRequest(method, path, nil) + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, req) + var body map[string]any + if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { + t.Fatalf("unreadable body %q: %v", rr.Body.String(), err) + } + return rr.Code, body +} + +// Absent module: the endpoint must answer calmly, the way every other optional +// service does, rather than 404 or 500. +func TestRoofModelDisabledReportsCleanly(t *testing.T) { + code, body := getJSON(t, depsAt(59.33, 18.07), http.MethodGet, "/api/roofmodel") + if code != 200 { + t.Fatalf("status = %d, want 200", code) + } + if body["enabled"] != false { + t.Errorf("enabled = %v, want false", body["enabled"]) + } +} + +// The metadata is the useful part when it is unavailable: it says where the +// data exists at all. +func TestRoofModelReportsCoverageForTheSite(t *testing.T) { + _, sthlm := getJSON(t, depsAt(59.33, 18.07), http.MethodGet, "/api/roofmodel") + if sthlm["covers"] != true { + t.Errorf("Stockholm covers = %v, want true", sthlm["covers"]) + } + if sthlm["area"] != "Sweden" { + t.Errorf("area = %v, want Sweden", sthlm["area"]) + } + + _, berlin := getJSON(t, depsAt(52.52, 13.40), http.MethodGet, "/api/roofmodel") + if berlin["covers"] != false { + t.Errorf("Berlin covers = %v, want false", berlin["covers"]) + } + // Even uncovered, it must still say where the data does exist. + if berlin["area"] != "Sweden" { + t.Errorf("area = %v, want Sweden even when uncovered", berlin["area"]) + } +} + +func TestRoofModelDeriveDisabledDoesNotError(t *testing.T) { + code, body := getJSON(t, depsAt(59.33, 18.07), http.MethodPost, "/api/roofmodel/derive") + if code != 200 { + t.Fatalf("status = %d, want 200", code) + } + if body["enabled"] != false { + t.Errorf("enabled = %v, want false", body["enabled"]) + } + if body["error"] == nil { + t.Error("a disabled derive should say why") + } +} + +// Without a location there is nothing to derive, and that is the operator's +// mistake to fix — so it is a 400, not a silent empty result. +func TestRoofModelDeriveWithoutASiteIsARequestError(t *testing.T) { + code, body := getJSON(t, &Deps{}, http.MethodPost, "/api/roofmodel/derive") + if code == 200 && body["enabled"] == false { + return // module disabled takes precedence, which is also correct + } + if code != 400 { + t.Errorf("status = %d, want 400", code) + } +} + +func TestRoofModelBuildingsDisabledReportsCleanly(t *testing.T) { + code, body := getJSON(t, depsAt(59.33, 18.07), http.MethodGet, "/api/roofmodel/buildings") + if code != 200 { + t.Fatalf("status = %d, want 200", code) + } + if body["enabled"] != false { + t.Errorf("enabled = %v, want false", body["enabled"]) + } + if body["error"] == nil { + t.Error("a disabled building search should say why") + } +} + +// The map lets the pin be dragged before anything is saved, so the picker has +// to be able to search where the pin is rather than where the config says. +func TestRoofModelBuildingsAcceptsAnExplicitCoordinate(t *testing.T) { + deps := depsAt(59.33, 18.07) + deps.RoofModel = roofmodel.FromConfig(&config.RoofModel{ + Enabled: true, Command: "definitely-not-a-real-command", + GeotorgetUsername: "u", GeotorgetToken: "t", + }) + + // Berlin is outside Lantmateriet coverage; if the query coordinate were + // ignored the stored Stockholm one would be used and this would not be a 400. + code, body := getJSON(t, deps, http.MethodGet, + "/api/roofmodel/buildings?lat=52.52&lon=13.40") + if code != 400 { + t.Fatalf("status = %d, want 400 for a site outside Sweden (body %v)", code, body) + } + + // A malformed pair must fall back to the configured site rather than + // searching at (0, 0), which is in the Atlantic. + _, sthlm := getJSON(t, deps, http.MethodGet, "/api/roofmodel/buildings?lat=abc&lon=def") + if sthlm["error"] == nil { + t.Fatal("want the spawn to fail, since the command does not exist") + } + if msg, _ := sthlm["error"].(string); strings.Contains(msg, "not in Sweden") { + t.Errorf("bad coordinates were used instead of the configured site: %v", msg) + } +} + +// The Geotorget token is the operator's credential. Status may be reported; +// the secret itself must never appear in a response. +func TestRoofModelNeverEchoesTheToken(t *testing.T) { + deps := depsAt(59.33, 18.07) + deps.Cfg.RoofModel = &config.RoofModel{ + Enabled: true, GeotorgetUsername: "operator", GeotorgetToken: "gt_secret_value", + } + deps.RoofModel = roofmodel.FromConfig(deps.Cfg.RoofModel) + + srv := New(deps) + for _, path := range []string{"/api/roofmodel", "/api/roofmodel/buildings"} { + req := httptest.NewRequest(http.MethodGet, path, nil) + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, req) + if strings.Contains(rr.Body.String(), "gt_secret_value") { + t.Errorf("%s leaked the Geotorget token: %s", path, rr.Body.String()) + } + } + + _, status := getJSON(t, deps, http.MethodGet, "/api/roofmodel") + if status["has_credentials"] != true { + t.Errorf("has_credentials = %v, want true so the UI can stop asking", + status["has_credentials"]) + } +} + +// Lantmäteriet must appear in the coverage listing alongside every other +// source, so an operator finds it without knowing it exists. +func TestLantmaterietAppearsInDataSources(t *testing.T) { + resp := getDataSources(t, depsAt(59.33, 18.07), "") + lm := find(t, resp, "lantmateriet") + if lm.Kind != "geodata" { + t.Errorf("kind = %q, want geodata", lm.Kind) + } + if !lm.RequiresKey { + t.Error("Geotorget access is credential-gated") + } + if lm.Worldwide { + t.Error("Lantmäteriet is Sweden only") + } + if lm.Covers == nil || !*lm.Covers { + t.Error("should cover Stockholm") + } + + away := getDataSources(t, depsAt(-33.87, 151.21), "") + if s := find(t, away, "lantmateriet"); s.Covers == nil || *s.Covers { + t.Error("must not claim to cover Sydney") + } +} diff --git a/go/internal/config/config.go b/go/internal/config/config.go index 074d84e8..ab5bbe4b 100644 --- a/go/internal/config/config.go +++ b/go/internal/config/config.go @@ -30,6 +30,7 @@ type Config struct { State *StateConf `yaml:"state,omitempty" json:"state,omitempty"` Price *Price `yaml:"price,omitempty" json:"price,omitempty"` Weather *Weather `yaml:"weather,omitempty" json:"weather,omitempty"` + RoofModel *RoofModel `yaml:"roofmodel,omitempty" json:"roofmodel,omitempty"` Planner *Planner `yaml:"planner,omitempty" json:"planner,omitempty"` Batteries map[string]Battery `yaml:"batteries,omitempty" json:"batteries,omitempty"` EVCharger *EVCharger `yaml:"ev_charger,omitempty" json:"ev_charger,omitempty"` @@ -951,6 +952,48 @@ type Price struct { ExportFloorOreKwh *float64 `yaml:"export_floor_ore_kwh,omitempty" json:"export_floor_ore_kwh,omitempty"` } +// RoofModel configures the optional Lantmäteriet roof-geometry module. +// +// Off by default and off the control tick entirely: it runs only when an +// operator asks for a derive during setup, in its own time-boxed subprocess, +// and its output only ever pre-fills the editable weather.pv_arrays. Absent or +// disabled, everything else behaves normally. +// +// GeotorgetToken is the operator's own credential. It is redacted in API +// responses by the existing sensitive-key rule (any key containing "token"). +type RoofModel struct { + Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // Command is the interpreter used for the module; defaults to "python3". + Command string `yaml:"command,omitempty" json:"command,omitempty"` + ModuleDir string `yaml:"module_dir,omitempty" json:"module_dir,omitempty"` + + GeotorgetUsername string `yaml:"geotorget_username,omitempty" json:"geotorget_username,omitempty"` + GeotorgetToken string `yaml:"geotorget_token,omitempty" json:"geotorget_token,omitempty"` + // HasGeotorgetToken is set only on the masked copy the API returns, so the + // UI can show that a token is stored without ever receiving it. Never + // written to YAML and never read from an incoming config. + HasGeotorgetToken bool `yaml:"-" json:"has_geotorget_token,omitempty"` + + // RadiusM is how far around the site to pull LiDAR (default 40 m). + RadiusM float64 `yaml:"radius_m,omitempty" json:"radius_m,omitempty"` + // PackingFactor is the usable fraction of a roof face after ridges, eaves, + // chimneys and walkways (default 0.70). + PackingFactor float64 `yaml:"packing_factor,omitempty" json:"packing_factor,omitempty"` + // TimeoutS bounds a derive. LiDAR tiles are large and this runs on a Pi, + // so an unbounded run could sit on memory indefinitely (default 600). + TimeoutS int `yaml:"timeout_s,omitempty" json:"timeout_s,omitempty"` + + // VostokBinary optionally enables shadow-aware irradiance, giving each roof + // face a shading factor from neighbouring buildings and trees. + // + // vostok is GPL-3.0 and is NOT shipped with FTW: the operator installs it + // themselves and points this at it. FTW invokes it at arm's length as a + // separate process, never links it, and never installs it. Empty means + // shading is not evaluated — which is reported as unknown, not as + // unobstructed. + VostokBinary string `yaml:"vostok_binary,omitempty" json:"vostok_binary,omitempty"` +} + // Weather is the weather-forecast source config. type Weather struct { Provider string `yaml:"provider" json:"provider"` // met_no | openweather | open_meteo | forecast_solar | none @@ -1041,6 +1084,15 @@ func (c Config) MaskSecrets() Config { cp.APIKey = "" out.Weather = &cp } + if out.RoofModel != nil { + cp := *out.RoofModel + // The UI has to distinguish "no credential stored" from "one is stored + // but masked", or an operator cannot tell whether they still need to + // paste their Geotorget token in. + cp.HasGeotorgetToken = strings.TrimSpace(cp.GeotorgetToken) != "" + cp.GeotorgetToken = "" + out.RoofModel = &cp + } if out.Notifications != nil { cp := *out.Notifications if cp.Ntfy != nil { @@ -1112,6 +1164,9 @@ func (incoming *Config) PreserveMaskedSecrets(existing *Config) { if incoming.Weather != nil && existing.Weather != nil && incoming.Weather.APIKey == "" { incoming.Weather.APIKey = existing.Weather.APIKey } + if incoming.RoofModel != nil && existing.RoofModel != nil && incoming.RoofModel.GeotorgetToken == "" { + incoming.RoofModel.GeotorgetToken = existing.RoofModel.GeotorgetToken + } if incoming.Notifications != nil && existing.Notifications != nil && incoming.Notifications.Ntfy != nil && existing.Notifications.Ntfy != nil { if incoming.Notifications.Ntfy.AccessToken == "" { diff --git a/go/internal/config/roofmodel_secrets_test.go b/go/internal/config/roofmodel_secrets_test.go new file mode 100644 index 00000000..fe1e36d7 --- /dev/null +++ b/go/internal/config/roofmodel_secrets_test.go @@ -0,0 +1,89 @@ +package config + +import "testing" + +// The Geotorget token is the operator's own credential for Lantmateriet. It +// must never come back out of the API, and -- the failure that actually bites -- +// saving the settings form must not wipe it, because the form only ever sends +// back the blank it was given. + +func TestRoofModelMaskSecretsHidesTheTokenButSaysOneExists(t *testing.T) { + c := Config{RoofModel: &RoofModel{ + Enabled: true, + GeotorgetUsername: "operator@example.com", + GeotorgetToken: "gt_secret_value", + }} + m := c.MaskSecrets() + + if m.RoofModel.GeotorgetToken != "" { + t.Errorf("token leaked through the API: %q", m.RoofModel.GeotorgetToken) + } + if !m.RoofModel.HasGeotorgetToken { + t.Error("UI cannot tell a stored token from a missing one") + } + // The username is not a secret, and blanking it would make the form look + // empty when it is not. + if m.RoofModel.GeotorgetUsername != "operator@example.com" { + t.Errorf("username got blanked: %q", m.RoofModel.GeotorgetUsername) + } + if c.RoofModel.GeotorgetToken != "gt_secret_value" { + t.Error("masking mutated the original config") + } +} + +func TestRoofModelMaskSecretsReportsNoTokenWhenUnset(t *testing.T) { + for _, tok := range []string{"", " "} { + c := Config{RoofModel: &RoofModel{Enabled: true, GeotorgetToken: tok}} + if c.MaskSecrets().RoofModel.HasGeotorgetToken { + t.Errorf("token %q reported as stored", tok) + } + } +} + +// Saving any unrelated setting round-trips the whole config, so an empty token +// from the UI means "unchanged", not "delete it". +func TestRoofModelPreserveMaskedSecretsKeepsTheStoredToken(t *testing.T) { + existing := &Config{RoofModel: &RoofModel{ + Enabled: true, GeotorgetUsername: "operator", GeotorgetToken: "gt_secret_value", + }} + incoming := &Config{RoofModel: &RoofModel{ + Enabled: true, GeotorgetUsername: "operator", GeotorgetToken: "", RadiusM: 60, + }} + + incoming.PreserveMaskedSecrets(existing) + + if incoming.RoofModel.GeotorgetToken != "gt_secret_value" { + t.Errorf("token = %q, want it preserved", incoming.RoofModel.GeotorgetToken) + } + if incoming.RoofModel.RadiusM != 60 { + t.Error("the edit being saved was lost") + } +} + +// Pasting a new token has to replace the old one, or a rotated credential +// could never be entered. +func TestRoofModelPreserveMaskedSecretsAcceptsANewToken(t *testing.T) { + existing := &Config{RoofModel: &RoofModel{GeotorgetToken: "old_token"}} + incoming := &Config{RoofModel: &RoofModel{GeotorgetToken: "new_token"}} + + incoming.PreserveMaskedSecrets(existing) + + if incoming.RoofModel.GeotorgetToken != "new_token" { + t.Errorf("token = %q, want the newly entered one", incoming.RoofModel.GeotorgetToken) + } +} + +// Enabling the module for the first time has no existing section to copy from. +func TestRoofModelPreserveMaskedSecretsSurvivesAMissingSection(t *testing.T) { + incoming := &Config{RoofModel: &RoofModel{GeotorgetToken: "first_token"}} + incoming.PreserveMaskedSecrets(&Config{}) + if incoming.RoofModel.GeotorgetToken != "first_token" { + t.Errorf("token = %q", incoming.RoofModel.GeotorgetToken) + } + + none := &Config{} + none.PreserveMaskedSecrets(&Config{RoofModel: &RoofModel{GeotorgetToken: "x"}}) + if none.RoofModel != nil { + t.Error("a section the operator never configured was invented") + } +} diff --git a/go/internal/coverage/coverage.go b/go/internal/coverage/coverage.go new file mode 100644 index 00000000..9b65e6cc --- /dev/null +++ b/go/internal/coverage/coverage.go @@ -0,0 +1,201 @@ +// Package coverage records where each external data source FTW talks to +// actually returns usable data. +// +// FTW runs outside the Nordics, but several of its sources are regional: +// STRÅNG models only the Nordic domain, and every price provider is European. +// Nothing in the code said so, so a site in Australia would get an empty price +// curve and an unscored PV history with no explanation. This package is that +// missing explanation, in one place, so the API and the UI can tell an operator +// *before* they select a source that it cannot serve their location. +// +// Bounds here are ADVISORY, and deliberately generous. Coverage is declared as +// a lat/lon box, but STRÅNG's grid is rotated relative to lat/lon, so the box is +// a superset of the real domain: a point near a corner can pass Covers and still +// return no data. Read Covers()==false as "definitely not supported, do not +// bother asking" and Covers()==true as "worth trying" — the upstream API stays +// authoritative. Nothing here is a safety input; it only decides what we show +// and whether we skip a pointless fetch. +package coverage + +// Kind groups sources by what they supply, so the UI can present forecast, +// irradiance and price coverage separately. +type Kind string + +const ( + KindForecast Kind = "forecast" + KindIrradiance Kind = "irradiance" + KindPrice Kind = "price" + KindGeodata Kind = "geodata" +) + +// BBox is an inclusive latitude/longitude bounding box in WGS84 degrees. +type BBox struct { + MinLat float64 `json:"min_lat"` + MinLon float64 `json:"min_lon"` + MaxLat float64 `json:"max_lat"` + MaxLon float64 `json:"max_lon"` +} + +// Contains reports whether (lat, lon) falls inside the box. Longitude is not +// wrapped: no source described here spans the antimeridian, and silently +// wrapping would turn a nonsense coordinate into a plausible-looking hit. +func (b BBox) Contains(lat, lon float64) bool { + return lat >= b.MinLat && lat <= b.MaxLat && lon >= b.MinLon && lon <= b.MaxLon +} + +// Source describes one external data source and where it works. +type Source struct { + ID string `json:"id"` + Kind Kind `json:"kind"` + Label string `json:"label"` + // Area is the human-readable coverage, shown in the UI. + Area string `json:"area"` + // Countries lists ISO 3166-1 alpha-2 codes when the source is bounded to a + // known set. Empty means either worldwide or "bounded by BBox, not by + // borders" — check Worldwide() rather than inferring from length. + Countries []string `json:"countries,omitempty"` + // BBox bounds the source geographically. nil means worldwide. + BBox *BBox `json:"bbox,omitempty"` + // RequiresKey is true when the operator must supply their own credential. + RequiresKey bool `json:"requires_key"` + License string `json:"license,omitempty"` + Note string `json:"note,omitempty"` +} + +// Worldwide reports whether the source is unbounded geographically. +func (s Source) Worldwide() bool { return s.BBox == nil } + +// Covers reports whether the source plausibly serves (lat, lon). Worldwide +// sources always do. See the package doc: a true result is advisory. +func (s Source) Covers(lat, lon float64) bool { + if s.BBox == nil { + return true + } + return s.BBox.Contains(lat, lon) +} + +// strangDomain is the STRÅNG model domain, measured against the live API rather +// than taken from documentation (SMHI's apidocs pages 404 as of 2026-07). +// Probing parameter 117 found data at lat 53.5 and 72.5 but not 53.0 or 73.5, +// and at lon 0 and 30 but not -2 or 35. The corner (53.5, -4) returns nothing +// even though it is inside this box, which is the rotated grid showing through — +// hence "advisory superset" in the package doc. +var strangDomain = &BBox{MinLat: 53.0, MinLon: -1.0, MaxLat: 73.0, MaxLon: 33.0} + +// sources is the registry. Keep it ordered by kind then id so the API response +// is stable and diffs stay readable. +var sources = []Source{ + { + ID: "met_no", Kind: KindForecast, Label: "MET Norway", + Area: "Worldwide", + License: "NLOD / CC BY 4.0", + Note: "Cloud cover only — no irradiance, so PV is derived from a cloud-derated clear-sky prior.", + }, + { + ID: "openweather", Kind: KindForecast, Label: "OpenWeather", + Area: "Worldwide", + RequiresKey: true, + Note: "Cloud cover only — same cloud-derated prior as MET Norway.", + }, + { + ID: "open_meteo", Kind: KindForecast, Label: "Open-Meteo", + Area: "Worldwide", + License: "CC BY 4.0", + Note: "Publishes shortwave radiation, so PV is irradiance-derived rather than cloud-derated.", + }, + { + ID: "forecast_solar", Kind: KindForecast, Label: "Forecast.Solar", + Area: "Worldwide", + Note: "Returns site-calibrated watts from the configured array geometry; free tier is rate-limited.", + }, + { + ID: "strang", Kind: KindIrradiance, Label: "SMHI STRÅNG", + Area: "Nordic region", + Countries: []string{"SE", "NO", "FI", "DK", "EE", "LV", "LT"}, + BBox: strangDomain, + License: "CC BY 4.0", + Note: "Historical only (1999 to ~1 day ago). Used for PV performance scoring and forecast calibration, never as a forward forecast.", + }, + { + ID: "lantmateriet", Kind: KindGeodata, Label: "Lantmäteriet (buildings + LiDAR)", + Area: "Sweden", + Countries: []string{"SE"}, + // Sweden's national extent. Generous at the edges for the same reason + // as STRÅNG: a box cannot trace a coastline, and the upstream STAC + // search returning no tiles is the authoritative answer. + BBox: &BBox{MinLat: 55.0, MinLon: 10.5, MaxLat: 69.5, MaxLon: 24.5}, + RequiresKey: true, + License: "CC BY 4.0", + Note: "Free open data, but gated behind a Geotorget account the operator orders themselves. Used to derive roof tilt/azimuth; everywhere else that stays a manual entry.", + }, + { + ID: "sourceful", Kind: KindPrice, Label: "Sourceful (cached ENTSO-E)", + Area: "Europe", + Countries: europeanPriceCountries, + BBox: &BBox{MinLat: 34.0, MinLon: -25.0, MaxLat: 72.0, MaxLon: 45.0}, + Note: "European day-ahead bidding zones. No key required.", + }, + { + ID: "elprisetjustnu", Kind: KindPrice, Label: "Elpriset just nu", + Area: "Sweden", + Countries: []string{"SE"}, + BBox: &BBox{MinLat: 55.0, MinLon: 10.0, MaxLat: 69.5, MaxLon: 24.5}, + Note: "Swedish bidding zones SE1-SE4 only. No key required.", + }, + { + ID: "entsoe", Kind: KindPrice, Label: "ENTSO-E Transparency", + Area: "Europe", + Countries: europeanPriceCountries, + BBox: &BBox{MinLat: 34.0, MinLon: -25.0, MaxLat: 72.0, MaxLon: 45.0}, + RequiresKey: true, + Note: "All ENTSO-E member bidding zones.", + }, +} + +// europeanPriceCountries are the ENTSO-E member states whose day-ahead prices +// the European providers can serve. Shared by sourceful and entsoe because both +// resolve to the same underlying bidding zones. +var europeanPriceCountries = []string{ + "AT", "BE", "BG", "CH", "CZ", "DE", "DK", "EE", "ES", "FI", + "FR", "GR", "HR", "HU", "IE", "IT", "LT", "LU", "LV", "NL", + "NO", "PL", "PT", "RO", "RS", "SE", "SI", "SK", +} + +// All returns every known source. +func All() []Source { + out := make([]Source, len(sources)) + copy(out, sources) + return out +} + +// ByID returns the source with the given id. +func ByID(id string) (Source, bool) { + for _, s := range sources { + if s.ID == id { + return s, true + } + } + return Source{}, false +} + +// ForKind returns every source of one kind, in registry order. +func ForKind(k Kind) []Source { + var out []Source + for _, s := range sources { + if s.Kind == k { + out = append(out, s) + } + } + return out +} + +// Covers reports whether the named source plausibly serves (lat, lon). An +// unknown id returns false: callers ask about a source they intend to use, and +// answering "sure" for a source we know nothing about is the wrong default. +func Covers(id string, lat, lon float64) bool { + s, ok := ByID(id) + if !ok { + return false + } + return s.Covers(lat, lon) +} diff --git a/go/internal/coverage/coverage_test.go b/go/internal/coverage/coverage_test.go new file mode 100644 index 00000000..36de8d82 --- /dev/null +++ b/go/internal/coverage/coverage_test.go @@ -0,0 +1,174 @@ +package coverage + +import "testing" + +// The STRÅNG cases below are the live-API probe results recorded on 2026-07-31 +// (parameter 117, 2026-06-21). They are the reason the box has the bounds it +// does, so if someone widens it these fail and say why. +func TestStrangCoversProbedNordicPoints(t *testing.T) { + in := []struct { + name string + lat, lon float64 + }{ + {"Stockholm", 59.33, 18.07}, + {"Tromsø", 69.65, 18.96}, + {"Helsinki", 60.17, 24.94}, + {"Copenhagen", 55.68, 12.57}, + } + for _, c := range in { + if !Covers("strang", c.lat, c.lon) { + t.Errorf("%s (%.2f,%.2f): want covered, got not covered", c.name, c.lat, c.lon) + } + } +} + +func TestStrangRejectsProbedOutsidePoints(t *testing.T) { + // Every one of these returned no data from the live API. + out := []struct { + name string + lat, lon float64 + }{ + {"Berlin", 52.52, 13.40}, + {"London", 51.51, -0.13}, + {"Paris", 48.86, 2.35}, + {"Reykjavík", 64.15, -21.94}, + {"Sydney", -33.87, 151.21}, + {"New York", 40.71, -74.01}, + } + for _, c := range out { + if Covers("strang", c.lat, c.lon) { + t.Errorf("%s (%.2f,%.2f): want not covered, got covered", c.name, c.lat, c.lon) + } + } +} + +// The declared box is a superset of the rotated grid: these four corners are +// inside the box yet every one returned no data when probed live on 2026-07-31. +// That gap is intentional and documented — Covers()==true means "worth asking", +// not "guaranteed". Pinned so nobody tightens the box into a false promise, or +// starts treating a true result as a guarantee. +func TestStrangBoxIsAdvisorySupersetAtCorners(t *testing.T) { + corners := [][2]float64{ + {53.5, -0.5}, {53.5, 32.0}, {72.5, -0.5}, {72.5, 32.0}, + } + for _, c := range corners { + if !Covers("strang", c[0], c[1]) { + t.Errorf("(%.1f,%.1f): corner should pass the advisory box test", c[0], c[1]) + } + } +} + +func TestForecastProvidersAreWorldwide(t *testing.T) { + for _, id := range []string{"met_no", "openweather", "open_meteo", "forecast_solar"} { + s, ok := ByID(id) + if !ok { + t.Fatalf("%s: not registered", id) + } + if !s.Worldwide() { + t.Errorf("%s: want worldwide", id) + } + // A worldwide source must cover anywhere, including the far south. + if !s.Covers(-33.87, 151.21) { + t.Errorf("%s: worldwide source must cover Sydney", id) + } + } +} + +// The whole point of #726: price data is Europe-only. If someone adds a global +// price provider this test should be updated deliberately, not incidentally. +func TestPriceProvidersAreEuropeOnly(t *testing.T) { + prices := ForKind(KindPrice) + if len(prices) == 0 { + t.Fatal("no price sources registered") + } + for _, s := range prices { + if s.Worldwide() { + t.Errorf("%s: price sources are not worldwide", s.ID) + } + if s.Covers(-33.87, 151.21) { + t.Errorf("%s: must not claim to cover Sydney", s.ID) + } + if s.Covers(40.71, -74.01) { + t.Errorf("%s: must not claim to cover New York", s.ID) + } + } +} + +func TestSwedishPriceProviderIsNarrowerThanEuropean(t *testing.T) { + // Berlin: served by the European providers, not by the Swedish one. + if Covers("elprisetjustnu", 52.52, 13.40) { + t.Error("elprisetjustnu must not claim Berlin") + } + if !Covers("sourceful", 52.52, 13.40) { + t.Error("sourceful should cover Berlin") + } + if !Covers("elprisetjustnu", 59.33, 18.07) { + t.Error("elprisetjustnu should cover Stockholm") + } +} + +// An unknown id must not be treated as universally available. +func TestUnknownSourceIsNotCovered(t *testing.T) { + if Covers("does_not_exist", 59.33, 18.07) { + t.Error("unknown source must report not covered") + } + if _, ok := ByID("does_not_exist"); ok { + t.Error("unknown source must not resolve") + } +} + +func TestBBoxContainsIsInclusive(t *testing.T) { + b := BBox{MinLat: 10, MinLon: 20, MaxLat: 30, MaxLon: 40} + for _, c := range []struct { + lat, lon float64 + want bool + }{ + {10, 20, true}, // min corner + {30, 40, true}, // max corner + {20, 30, true}, // interior + {9.99, 30, false}, // just south + {20, 40.01, false}, // just east + } { + if got := b.Contains(c.lat, c.lon); got != c.want { + t.Errorf("Contains(%v,%v) = %v, want %v", c.lat, c.lon, got, c.want) + } + } +} + +// Longitude is intentionally not wrapped; a nonsense coordinate must stay a +// miss rather than being folded into range. +func TestBBoxDoesNotWrapLongitude(t *testing.T) { + b := BBox{MinLat: -90, MinLon: -180, MaxLat: 90, MaxLon: 180} + if b.Contains(0, 200) { + t.Error("lon 200 must not wrap to -160") + } +} + +func TestRegistryIsInternallyConsistent(t *testing.T) { + seen := map[string]bool{} + for _, s := range All() { + if s.ID == "" || s.Label == "" || s.Area == "" { + t.Errorf("%+v: id, label and area are all required", s) + } + if seen[s.ID] { + t.Errorf("%s: duplicate id", s.ID) + } + seen[s.ID] = true + if s.BBox != nil { + if s.BBox.MinLat > s.BBox.MaxLat || s.BBox.MinLon > s.BBox.MaxLon { + t.Errorf("%s: inverted bbox %+v", s.ID, *s.BBox) + } + } + } +} + +// All() must hand out a copy: a caller mutating the result must not corrupt the +// registry for everyone else in the process. +func TestAllReturnsACopy(t *testing.T) { + got := All() + original := got[0].ID + got[0].ID = "mutated" + if All()[0].ID != original { + t.Fatal("All() exposed the backing array") + } +} diff --git a/go/internal/forecast/forecast.go b/go/internal/forecast/forecast.go index 08a55efb..5c52095c 100644 --- a/go/internal/forecast/forecast.go +++ b/go/internal/forecast/forecast.go @@ -29,6 +29,7 @@ import ( "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/sunpos" ) // Provider is implemented by each weather source. @@ -243,10 +244,25 @@ func EstimatePVW(lat, lon float64, t time.Time, cloudPct *float64, ratedW float6 // Service wraps a provider + store + scheduler for forecasts. type Service struct { - Provider Provider - Store *state.Store - Lat, Lon float64 - RatedPVW float64 // total rated PV across all arrays (used for estimate) + Provider Provider + Store *state.Store + Lat, Lon float64 + RatedPVW float64 // total rated PV across all arrays (used for estimate) + + // Arrays holds per-plane geometry (tilt/azimuth/kWp) mirrored from the + // weather config. When set, a radiation-bearing provider's horizontal + // GHI is projected onto each plane via sunpos and summed, instead of the + // orientation-blind flat rated×(W/m²/1000) estimate. Empty → flat estimate. + Arrays []Array + + // Calibration optionally supplies a measured site correction factor, + // wired to the STRÅNG performance scorer. The irradiance-derived + // estimates below are deliberately loss-free (no inverter, wiring or + // temperature derate), so a site settles at a stable ratio slightly under + // 1; feeding that measured ratio back closes the gap. The second return + // is false whenever the factor must not be applied, and nil means the + // scorer isn't wired at all — both leave the estimate untouched. + Calibration func() (float64, bool) stop chan struct{} done chan struct{} @@ -286,10 +302,21 @@ func FromConfig(cfg *config.Weather, ratedPVW float64, st *state.Store, userAgen default: return nil } + // Mirror per-plane geometry for the POA path. Shared across all + // providers: forecast_solar already applies geometry server-side (so + // this stays unused there), but open_meteo / STRÅNG-style GHI providers + // use it to project irradiance onto each plane in fetchAndStore. + var arrays []Array + for _, a := range cfg.PVArrays { + if a.KWp > 0 { + arrays = append(arrays, Array{TiltDeg: a.TiltDeg, AzimuthDeg: a.AzimuthDeg, KWp: a.KWp}) + } + } return &Service{ Provider: p, Store: st, Lat: cfg.Latitude, Lon: cfg.Longitude, RatedPVW: ratedPVW, + Arrays: arrays, stop: make(chan struct{}), done: make(chan struct{}), } @@ -331,6 +358,15 @@ func (s *Service) fetchAndStore(ctx context.Context) { } if len(rows) == 0 { return } nowMs := time.Now().UnixMilli() + + // Resolved once per fetch so every point in a batch shares one factor. + calibration, calibrated := 1.0, false + if s.Calibration != nil { + if f, ok := s.Calibration(); ok && f > 0 { + calibration, calibrated = f, true + } + } + points := make([]state.ForecastPoint, 0, len(rows)) for _, r := range rows { // Pick the most direct PV signal the provider gave us. Forecast.Solar @@ -338,14 +374,30 @@ func (s *Service) fetchAndStore(ctx context.Context) { // radiation we turn into watts via rated × W/m²/1000; met.no only has // cloud fraction, so we fall through to the naive cloud-derated prior. var pvW float64 + // Only the two irradiance-derived branches are calibrated. They share + // the loss-free "irradiance × nameplate" baseline the performance ratio + // was measured against, so the correction is the same quantity. A + // provider-native figure is already site-calibrated upstream and the + // cloud-derated prior carries its own empirical derate; scaling either + // would double-count. + applyCalibration := false switch { case r.PVWEstimated != nil: pvW = *r.PVWEstimated + case r.SolarWm2 != nil && len(s.Arrays) > 0: + // Orientation-aware: project GHI onto each configured plane and sum. + pvW = poaPVWattsFromGHI(s.Lat, s.Lon, r.HourStart, *r.SolarWm2, s.Arrays) + applyCalibration = true case r.SolarWm2 != nil && s.RatedPVW > 0: + // No geometry configured — flat, orientation-blind estimate. pvW = s.RatedPVW * (*r.SolarWm2) / 1000.0 + applyCalibration = true default: pvW = EstimatePVW(s.Lat, s.Lon, r.HourStart, r.CloudCoverPct, s.RatedPVW) } + if applyCalibration && calibrated { + pvW *= calibration + } pvPtr := &pvW points = append(points, state.ForecastPoint{ SlotTsMs: r.HourStart.UnixMilli(), @@ -362,7 +414,27 @@ func (s *Service) fetchAndStore(ctx context.Context) { slog.Warn("forecast save failed", "err", err) return } - slog.Info("forecast fetched", "count", len(points), "provider", s.Provider.Name()) + slog.Info("forecast fetched", "count", len(points), "provider", s.Provider.Name(), + "pv_calibration", calibration, "calibrated", calibrated) +} + +// poaPVWattsFromGHI converts a global-horizontal irradiance (W/m², positive) +// into expected DC PV output (W, positive) by projecting it onto each +// configured array's plane via sunpos and scaling by nameplate. This is the +// orientation-aware replacement for the flat rated×(W/m²/1000) estimate; it +// is used whenever the provider supplies GHI and the site has per-plane +// geometry. Returns 0 when the sun is down or no arrays produce output. +func poaPVWattsFromGHI(lat, lon float64, t time.Time, ghiWm2 float64, arrays []Array) float64 { + var total float64 + for _, a := range arrays { + if a.KWp <= 0 { + continue + } + poa := sunpos.POAFromGHI(t, lat, lon, ghiWm2, a.TiltDeg, a.AzimuthDeg) + // kWp×1000 = nameplate W at STC (1000 W/m²); scale by POA/1000. + total += a.KWp * 1000.0 * (poa / 1000.0) + } + return total } // Load returns forecasts in [sinceMs, untilMs]. diff --git a/go/internal/forecast/forecast_test.go b/go/internal/forecast/forecast_test.go index 73450fa0..19231b68 100644 --- a/go/internal/forecast/forecast_test.go +++ b/go/internal/forecast/forecast_test.go @@ -247,3 +247,177 @@ func TestFromConfigBuildsMetNo(t *testing.T) { if s.Lat != 59 { t.Errorf("lat: %f", s.Lat) } if s.RatedPVW != 10000 { t.Errorf("rated: %f", s.RatedPVW) } } + +func TestFromConfigPopulatesArrays(t *testing.T) { + st, _ := state.Open(filepath.Join(t.TempDir(), "t.db")) + defer st.Close() + cfg := &config.Weather{ + Provider: "open_meteo", Latitude: 59, Longitude: 18, + PVArrays: []config.PVArray{ + {Name: "south", KWp: 6, TiltDeg: 35, AzimuthDeg: 180}, + {Name: "east", KWp: 4, TiltDeg: 30, AzimuthDeg: 90}, + {Name: "empty", KWp: 0, TiltDeg: 10, AzimuthDeg: 200}, // skipped (kWp 0) + }, + } + s := FromConfig(cfg, 10000, st, "ua") + if s == nil { t.Fatal("expected service") } + if len(s.Arrays) != 2 { + t.Fatalf("expected 2 arrays (kWp>0 only), got %d", len(s.Arrays)) + } + if s.Arrays[0].KWp != 6 || s.Arrays[1].AzimuthDeg != 90 { + t.Errorf("array geometry mismatch: %+v", s.Arrays) + } +} + +// ---- POA-per-array (orientation-aware) estimate ---- + +func TestPOAPVWattsSumsArrays(t *testing.T) { + tt := time.Date(2026, 6, 21, 11, 0, 0, 0, time.UTC) + one := poaPVWattsFromGHI(59.3293, 18.0686, tt, 700, []Array{{TiltDeg: 35, AzimuthDeg: 180, KWp: 5}}) + two := poaPVWattsFromGHI(59.3293, 18.0686, tt, 700, []Array{ + {TiltDeg: 35, AzimuthDeg: 180, KWp: 5}, + {TiltDeg: 35, AzimuthDeg: 180, KWp: 5}, + }) + if one <= 0 { + t.Fatalf("expected positive POA watts, got %.1f", one) + } + if math.Abs(two-2*one) > 1e-6 { + t.Errorf("two identical arrays should double output: one=%.2f two=%.2f", one, two) + } +} + +func TestPOAPVWattsZeroAtNight(t *testing.T) { + tt := time.Date(2026, 12, 21, 23, 0, 0, 0, time.UTC) + w := poaPVWattsFromGHI(59.3293, 18.0686, tt, 500, []Array{{TiltDeg: 35, AzimuthDeg: 180, KWp: 10}}) + if w != 0 { + t.Errorf("night POA watts should be 0, got %.2f", w) + } +} + +// End-to-end: with per-plane geometry, a GHI-bearing provider's stored PV +// estimate comes from the POA path and differs from the flat rated×GHI/1000. +func TestServicePOAPathDiffersFromFlat(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := map[string]any{ + "hourly": map[string]any{ + "time": []string{"2026-06-21T11:00"}, + "shortwave_radiation": []float64{700}, + "cloud_cover": []float64{5}, + "temperature_2m": []float64{20}, + }, + } + _ = json.NewEncoder(w).Encode(resp) + }) + srv := httptest.NewServer(handler) + defer srv.Close() + + st, _ := state.Open(filepath.Join(t.TempDir(), "state.db")) + defer st.Close() + + p := NewOpenMeteo() + p.BaseURL = srv.URL + s := &Service{ + Provider: p, Store: st, Lat: 59.3293, Lon: 18.0686, RatedPVW: 10000, + Arrays: []Array{{TiltDeg: 35, AzimuthDeg: 180, KWp: 10}}, + } + s.fetchAndStore(context.Background()) + + tt := time.Date(2026, 6, 21, 11, 0, 0, 0, time.UTC) + rows, err := st.LoadForecasts(tt.UnixMilli(), tt.Add(time.Hour).UnixMilli()) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 || rows[0].PVWEstimated == nil { + t.Fatalf("expected 1 forecast with PV estimate, got %+v", rows) + } + got := *rows[0].PVWEstimated + flat := 10000 * 700.0 / 1000.0 // orientation-blind estimate = 7000 W + want := poaPVWattsFromGHI(59.3293, 18.0686, tt, 700, s.Arrays) + if math.Abs(got-want) > 1.0 { + t.Errorf("service should use POA path: got %.1f want %.1f", got, want) + } + if math.Abs(got-flat) < 1.0 { + t.Errorf("POA estimate should differ from flat %.0f, got %.1f", flat, got) + } + t.Logf("POA-per-array estimate %.0fW vs flat %.0fW", got, flat) +} + +// ---- STRÅNG calibration hook ---- + +// radiationForecastPVW runs one fetch against a stub shortwave-radiation +// provider and returns the stored PV estimate, so calibration variants can be +// compared against an otherwise identical run. +func radiationForecastPVW(t *testing.T, calibration func() (float64, bool)) float64 { + t.Helper() + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := map[string]any{ + "hourly": map[string]any{ + "time": []string{"2026-06-21T11:00"}, + "shortwave_radiation": []float64{700}, + "cloud_cover": []float64{5}, + "temperature_2m": []float64{20}, + }, + } + _ = json.NewEncoder(w).Encode(resp) + }) + srv := httptest.NewServer(handler) + defer srv.Close() + + st, _ := state.Open(filepath.Join(t.TempDir(), "state.db")) + defer st.Close() + + p := NewOpenMeteo() + p.BaseURL = srv.URL + s := &Service{ + Provider: p, Store: st, Lat: 59.3293, Lon: 18.0686, RatedPVW: 10000, + Arrays: []Array{{TiltDeg: 35, AzimuthDeg: 180, KWp: 10}}, + Calibration: calibration, + } + s.fetchAndStore(context.Background()) + + tt := time.Date(2026, 6, 21, 11, 0, 0, 0, time.UTC) + rows, err := st.LoadForecasts(tt.UnixMilli(), tt.Add(time.Hour).UnixMilli()) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 || rows[0].PVWEstimated == nil { + t.Fatalf("expected 1 forecast with PV estimate, got %+v", rows) + } + return *rows[0].PVWEstimated +} + +// The point of scoring: a site measured at 80% of its physics baseline should +// have its forward forecast scaled to match. +func TestServiceAppliesCalibrationToIrradianceEstimate(t *testing.T) { + uncalibrated := radiationForecastPVW(t, nil) + calibrated := radiationForecastPVW(t, func() (float64, bool) { return 0.8, true }) + + want := uncalibrated * 0.8 + if math.Abs(calibrated-want) > 1.0 { + t.Errorf("calibrated estimate = %.1f W, want %.1f W (0.8 × %.1f)", calibrated, want, uncalibrated) + } + t.Logf("uncalibrated %.0fW → calibrated %.0fW", uncalibrated, calibrated) +} + +// An untrusted factor must change nothing: too few days, or a ratio outside the +// plausible band, leaves the physics estimate exactly as it was. +func TestServiceIgnoresUntrustedCalibration(t *testing.T) { + uncalibrated := radiationForecastPVW(t, nil) + rejected := radiationForecastPVW(t, func() (float64, bool) { return 0.05, false }) + + if math.Abs(rejected-uncalibrated) > 1e-9 { + t.Errorf("estimate = %.1f W, want the uncalibrated %.1f W", rejected, uncalibrated) + } +} + +// A zero or negative factor would silently zero out the site's whole forecast, +// so it is refused even when the source claims it is usable. +func TestServiceIgnoresNonPositiveCalibration(t *testing.T) { + uncalibrated := radiationForecastPVW(t, nil) + for _, factor := range []float64{0, -0.5} { + got := radiationForecastPVW(t, func() (float64, bool) { return factor, true }) + if math.Abs(got-uncalibrated) > 1e-9 { + t.Errorf("factor %v: estimate = %.1f W, want the uncalibrated %.1f W", factor, got, uncalibrated) + } + } +} diff --git a/go/internal/mpc/external_optimizer.go b/go/internal/mpc/external_optimizer.go index 1a7078b8..68e2c73b 100644 --- a/go/internal/mpc/external_optimizer.go +++ b/go/internal/mpc/external_optimizer.go @@ -486,9 +486,9 @@ func (o *ExternalOptimizer) buildRequest(slots []Slot, p Params) externalRequest basePV[i] = slot.PVW if slot.PVW < 0 { hasDaylight = true - generation := -slot.PVW - downsidePV[i] = -math.Max(0, generation-spread) - upsidePV[i] = -(generation + spread) + // PVBand owns the site-sign mapping: "low" is more negative + // (more generation, optimistic), "high" is nearer zero. + upsidePV[i], downsidePV[i] = PVBand(slot.PVW, spread) } } if hasDaylight { diff --git a/go/internal/mpc/pvband.go b/go/internal/mpc/pvband.go new file mode 100644 index 00000000..f25c42d8 --- /dev/null +++ b/go/internal/mpc/pvband.go @@ -0,0 +1,37 @@ +package mpc + +import "math" + +// PVBand returns an uncertainty band around a slot's expected PV power. +// +// Everything here is in the site convention: power into the site is positive, +// so generation is NEGATIVE. That inverts the intuitive reading of the return +// values, which is exactly why this lives in one tested function instead of +// being open-coded at each call site: +// +// - low is the numerically smaller (more negative) bound — MORE generation, +// i.e. the OPTIMISTIC case. +// - high is the numerically larger bound (closer to zero) — LESS generation, +// i.e. the PESSIMISTIC case. +// +// The optimizer validates `low <= base <= high <= 0` and rejects the request +// outright if any of that fails, so two clamps are load-bearing: high can never +// cross zero (a spread wider than expected generation degrades to "produces +// nothing", never to positive PV, which would be read as load), and a +// non-positive spread collapses the band onto the base forecast exactly. +// +// Slots with no expected generation return a zero-width band at zero, matching +// how night slots are left untouched. +func PVBand(basePVW, spreadW float64) (low, high float64) { + // `!(x < 0)` rather than `x >= 0` so NaN falls into this branch too. + if !(basePVW < 0) || math.IsInf(basePVW, 0) { + return 0, 0 + } + if !(spreadW > 0) || math.IsInf(spreadW, 0) { + return basePVW, basePVW + } + generation := -basePVW // > 0 + low = -(generation + spreadW) + high = -math.Max(0, generation-spreadW) + return low, high +} diff --git a/go/internal/mpc/pvband_test.go b/go/internal/mpc/pvband_test.go new file mode 100644 index 00000000..7dab3fc4 --- /dev/null +++ b/go/internal/mpc/pvband_test.go @@ -0,0 +1,86 @@ +package mpc + +import ( + "math" + "testing" +) + +// The optimizer rejects any request where the band fails to bracket the base +// forecast or where either bound crosses zero. This is the invariant the whole +// helper exists to guarantee, so it is asserted over a spread of inputs +// including the degenerate ones. +func TestPVBandHoldsSiteSignInvariant(t *testing.T) { + bases := []float64{0, -1, -250, -2000, -9500, math.NaN(), math.Inf(-1)} + spreads := []float64{0, 1, 250, 3000, 100000, -50, math.NaN()} + + for _, base := range bases { + for _, spread := range spreads { + low, high := PVBand(base, spread) + + if math.IsNaN(low) || math.IsNaN(high) { + t.Fatalf("PVBand(%v, %v) produced NaN: low=%v high=%v", base, spread, low, high) + } + if low > 0 || high > 0 { + t.Errorf("PVBand(%v, %v) = (%v, %v); bounds must stay <= 0 (site convention)", + base, spread, low, high) + } + if low > high { + t.Errorf("PVBand(%v, %v) = (%v, %v); low must not exceed high", base, spread, low, high) + } + // The base is only bracketed when it is itself a valid generation + // figure; invalid bases are normalised to a zero-width band. + if base < 0 && !math.IsInf(base, 0) { + if low > base || base > high { + t.Errorf("PVBand(%v, %v) = (%v, %v); band must bracket the base forecast", + base, spread, low, high) + } + } + } + } +} + +// A site with no measured uncertainty must plan against the plain forecast -- +// the band collapses to a point rather than quietly widening. +func TestPVBandZeroSpreadCollapsesOntoBase(t *testing.T) { + const base = -3200.0 + low, high := PVBand(base, 0) + if low != base || high != base { + t.Errorf("PVBand(%v, 0) = (%v, %v), want both == %v", base, low, high, base) + } +} + +// The clamp that keeps the pessimistic bound from crossing into positive +// territory: a spread wider than the expected generation means "might produce +// nothing", never "might consume". +func TestPVBandClampsPessimisticBoundAtZero(t *testing.T) { + low, high := PVBand(-1000, 4000) + if high != 0 { + t.Errorf("high = %v, want 0 (clamped); a positive bound would read as load", high) + } + if low != -5000 { + t.Errorf("low = %v, want -5000 (optimistic bound is unclamped)", low) + } +} + +// Direction check stated in production terms, so a future sign flip fails here +// with an obvious message rather than as an opaque optimizer ProtocolError. +func TestPVBandLowIsOptimisticHighIsPessimistic(t *testing.T) { + const base = -5000.0 + low, high := PVBand(base, 1500) + if generation := -low; generation != 6500 { + t.Errorf("optimistic generation = %v W, want 6500 W", generation) + } + if generation := -high; generation != 3500 { + t.Errorf("pessimistic generation = %v W, want 3500 W", generation) + } +} + +// Night slots carry no expected generation, so they get no band at all. +func TestPVBandNoGenerationYieldsNoBand(t *testing.T) { + for _, base := range []float64{0, 250} { + low, high := PVBand(base, 900) + if low != 0 || high != 0 { + t.Errorf("PVBand(%v, 900) = (%v, %v), want (0, 0)", base, low, high) + } + } +} diff --git a/go/internal/pvperf/calibration.go b/go/internal/pvperf/calibration.go new file mode 100644 index 00000000..fd2bfea6 --- /dev/null +++ b/go/internal/pvperf/calibration.go @@ -0,0 +1,117 @@ +package pvperf + +import ( + "math" + "sort" + "time" + + "github.com/srcfl/ftw/go/internal/state" +) + +// Calibration is the site-specific correction learned by comparing measured +// production against the STRÅNG physics baseline. It is the payoff of scoring: +// once a site has enough closed days, its typical performance ratio *is* the +// derate the forward forecast should apply, and the spread of that ratio is a +// measured uncertainty that beats any hand-tuned constant. +type Calibration struct { + // Factor is the representative performance ratio — multiply a loss-free + // physics PV estimate by this to get an expected real-world figure. + Factor float64 + // SigmaRel is the robust spread of the daily ratio. PR is dimensionless, + // so this is already a relative uncertainty. + SigmaRel float64 + // Days is how many scored days carried a usable ratio. + Days int + // Valid reports whether Factor is trustworthy enough to apply. A factor + // outside the plausible band means the geometry or the meter is wrong, + // not that the panels are dirty — better to leave the forecast alone. + Valid bool +} + +const ( + // minCalibrationDays is the smallest sample that can outvote a single + // snowy or curtailed day. + minCalibrationDays = 7 + + // Plausible bounds for a real site's ratio. Below the floor points at + // broken telemetry or an array that never ran; above the ceiling means + // the declared kWp understates the installation. Both are configuration + // faults that a silent forecast rescale would only hide. + minCalibrationFactor = 0.30 + maxCalibrationFactor = 1.30 + + // maxCalibrationSigma caps the reported spread so a pathological sample + // cannot widen the planner's uncertainty without bound. + maxCalibrationSigma = 1.0 + + // madToSigma converts a median absolute deviation into a standard + // deviation for normally distributed data. + madToSigma = 1.4826 + + // calibrationWindowDays is how far back Service.Calibration looks. + calibrationWindowDays = 30 +) + +// Calibrate summarises scored days into a site calibration. It uses the median +// and a median-absolute-deviation sigma rather than mean/stddev: a single +// snow-covered or curtailed day is a large outlier, and the mean would chase it. +func Calibrate(days []state.PVPerformanceDay) Calibration { + ratios := make([]float64, 0, len(days)) + for _, d := range days { + if d.PR != nil && !math.IsNaN(*d.PR) && !math.IsInf(*d.PR, 0) { + ratios = append(ratios, *d.PR) + } + } + c := Calibration{Days: len(ratios)} + if len(ratios) < minCalibrationDays { + return c + } + c.Factor = median(ratios) + + deviations := make([]float64, len(ratios)) + for i, r := range ratios { + deviations[i] = math.Abs(r - c.Factor) + } + c.SigmaRel = math.Min(maxCalibrationSigma, madToSigma*median(deviations)) + c.Valid = c.Factor >= minCalibrationFactor && c.Factor <= maxCalibrationFactor + return c +} + +// median returns the middle value of xs without mutating the caller's slice. +func median(xs []float64) float64 { + if len(xs) == 0 { + return 0 + } + sorted := append([]float64(nil), xs...) + sort.Float64s(sorted) + n := len(sorted) + if n%2 == 1 { + return sorted[n/2] + } + return (sorted[n/2-1] + sorted[n/2]) / 2 +} + +// Calibration returns the site calibration over the recent scored window. +// A zero value (Valid false) is returned whenever the service is unwired or +// the history is too thin to draw a conclusion. +func (s *Service) Calibration() Calibration { + if s == nil || s.Store == nil { + return Calibration{} + } + now := time.Now() + days, err := s.Store.LoadPVPerformance( + now.AddDate(0, 0, -calibrationWindowDays).Format("2006-01-02"), + now.Format("2006-01-02"), + ) + if err != nil { + return Calibration{} + } + return Calibrate(days) +} + +// CalibrationFactor adapts Calibration to the hook shape the forecast service +// consumes. The second return is false when the factor must not be applied. +func (s *Service) CalibrationFactor() (float64, bool) { + c := s.Calibration() + return c.Factor, c.Valid +} diff --git a/go/internal/pvperf/calibration_test.go b/go/internal/pvperf/calibration_test.go new file mode 100644 index 00000000..76c7155e --- /dev/null +++ b/go/internal/pvperf/calibration_test.go @@ -0,0 +1,153 @@ +package pvperf + +import ( + "fmt" + "math" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/state" +) + +// scoredDays builds a run of days carrying the given performance ratios. +func scoredDays(ratios ...float64) []state.PVPerformanceDay { + days := make([]state.PVPerformanceDay, 0, len(ratios)) + for i, r := range ratios { + ratio := r + days = append(days, state.PVPerformanceDay{ + Day: fmt.Sprintf("2026-07-%02d", i+1), + ExpectedWh: 50000, + ActualWh: 50000 * ratio, + PR: &ratio, + }) + } + return days +} + +// A handful of days is not evidence. Applying a factor from a thin sample would +// let one cloudy week permanently depress the forecast. +func TestCalibrateNeedsEnoughDays(t *testing.T) { + c := Calibrate(scoredDays(0.9, 0.9, 0.9)) + if c.Valid { + t.Error("calibration should not be valid on 3 days") + } + if c.Days != 3 { + t.Errorf("Days = %d, want 3", c.Days) + } +} + +// Days without a usable ratio (polar night, no history) must not be counted as +// evidence, and must not drag the factor toward zero. +func TestCalibrateIgnoresDaysWithoutRatio(t *testing.T) { + days := scoredDays(0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9) + days = append(days, state.PVPerformanceDay{Day: "2026-07-30", ExpectedWh: 10, ActualWh: 0}) + c := Calibrate(days) + if c.Days != 7 { + t.Errorf("Days = %d, want 7 (the ratio-less day is not evidence)", c.Days) + } + if math.Abs(c.Factor-0.9) > 1e-9 { + t.Errorf("Factor = %v, want 0.9", c.Factor) + } +} + +// The reason for median-over-mean: a snow day is a huge outlier, and a site +// that normally runs at 0.90 must still calibrate to ~0.90 despite one. +func TestCalibrateResistsOutlierDays(t *testing.T) { + c := Calibrate(scoredDays(0.90, 0.91, 0.89, 0.90, 0.92, 0.88, 0.90, 0.05)) + if !c.Valid { + t.Fatal("expected a valid calibration") + } + if math.Abs(c.Factor-0.90) > 0.02 { + t.Errorf("Factor = %v, want ~0.90; a single snow day should not move it", c.Factor) + } +} + +// A factor this low means the geometry or the meter is misconfigured. Silently +// rescaling the forecast would bake the fault in instead of surfacing it. +func TestCalibrateRejectsImplausiblyLowFactor(t *testing.T) { + c := Calibrate(scoredDays(0.10, 0.11, 0.09, 0.10, 0.12, 0.08, 0.10, 0.11)) + if c.Valid { + t.Errorf("factor %v should be rejected as implausible", c.Factor) + } + if c.Days != 8 { + t.Errorf("Days = %d, want 8 — the sample is still reported", c.Days) + } +} + +// Likewise for a site consistently beating its own nameplate: that is an +// understated kWp, not a bonus to hand to the planner. +func TestCalibrateRejectsImplausiblyHighFactor(t *testing.T) { + c := Calibrate(scoredDays(1.6, 1.7, 1.55, 1.62, 1.58, 1.7, 1.65, 1.6)) + if c.Valid { + t.Errorf("factor %v should be rejected as implausible", c.Factor) + } +} + +// A perfectly steady site reports no uncertainty, so the planner's band +// collapses onto the forecast rather than inventing spread. +func TestCalibrateSigmaIsZeroForSteadySite(t *testing.T) { + c := Calibrate(scoredDays(0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9)) + if !c.Valid { + t.Fatal("expected a valid calibration") + } + if c.SigmaRel != 0 { + t.Errorf("SigmaRel = %v, want 0 for an unvarying site", c.SigmaRel) + } +} + +// A variable site reports a real spread, which is what makes the measured band +// better than a hand-tuned constant. +func TestCalibrateSigmaGrowsWithSpread(t *testing.T) { + steady := Calibrate(scoredDays(0.90, 0.91, 0.89, 0.90, 0.91, 0.89, 0.90, 0.90)) + variable := Calibrate(scoredDays(0.60, 0.95, 0.70, 0.99, 0.65, 0.92, 0.75, 0.88)) + if !(variable.SigmaRel > steady.SigmaRel) { + t.Errorf("variable SigmaRel %v should exceed steady %v", variable.SigmaRel, steady.SigmaRel) + } +} + +// The forecast hook is wired unconditionally at startup, so it must be safe on +// a service that was never built. +func TestCalibrationFactorNilServiceIsSafe(t *testing.T) { + var s *Service + factor, ok := s.CalibrationFactor() + if ok { + t.Errorf("nil service reported a usable factor (%v)", factor) + } +} + +// End-to-end through the store, since that is how the API and the forecast +// hook actually reach it. +func TestServiceCalibrationReadsPersistedDays(t *testing.T) { + st := openStore(t) + svc := &Service{Store: st} + + if c := svc.Calibration(); c.Valid || c.Days != 0 { + t.Fatalf("empty store should yield no calibration, got %+v", c) + } + + now := time.Now() + for i := 1; i <= 10; i++ { + ratio := 0.88 + day := now.AddDate(0, 0, -i).Format("2006-01-02") + if err := st.SavePVPerformance(state.PVPerformanceDay{ + Day: day, ExpectedWh: 40000, ActualWh: 40000 * ratio, PR: &ratio, + }); err != nil { + t.Fatalf("save %s: %v", day, err) + } + } + c := svc.Calibration() + if !c.Valid { + t.Fatalf("expected a valid calibration, got %+v", c) + } + if math.Abs(c.Factor-0.88) > 1e-9 { + t.Errorf("Factor = %v, want 0.88", c.Factor) + } + if c.Days != 10 { + t.Errorf("Days = %d, want 10", c.Days) + } + + factor, ok := svc.CalibrationFactor() + if !ok || math.Abs(factor-0.88) > 1e-9 { + t.Errorf("CalibrationFactor() = (%v, %v), want (0.88, true)", factor, ok) + } +} diff --git a/go/internal/pvperf/pvperf.go b/go/internal/pvperf/pvperf.go new file mode 100644 index 00000000..e492bd9b --- /dev/null +++ b/go/internal/pvperf/pvperf.go @@ -0,0 +1,88 @@ +// Package pvperf scores measured PV production against a weather-expected +// baseline derived from SMHI STRÅNG irradiance and the site's per-plane +// geometry. +// +// It is read-only analytics with no control-path coupling: given historical +// irradiance and the panel arrays, it computes the DC energy the arrays should +// have produced under that irradiance, then compares to the measured energy to +// yield a performance ratio (PR). A sustained PR below ~1 hints at soiling, +// snow, shading or degradation; the raw PR series is the signal, the diagnosis +// is left to the caller/UI. +// +// The expected baseline deliberately omits system losses (inverter, wiring, +// temperature). The performance ratio absorbs the site's fixed derate, so a +// healthy site simply sits at a stable PR < 1 and anomalies show as departures +// from that baseline. +package pvperf + +import ( + "time" + + "github.com/srcfl/ftw/go/internal/sunpos" +) + +// Array is one panel plane: nameplate DC kWp at a tilt/azimuth (same +// conventions as sunpos — tilt 0=flat..90=wall, azimuth 0=N,90=E,180=S,270=W). +type Array struct { + KWp float64 + TiltDeg float64 + AzimuthDeg float64 +} + +// Irradiance is one hour of horizontal irradiance (W/m²). DHIWm2 is nil when +// the diffuse component is unavailable, in which case an Erbs split is used. +type Irradiance struct { + HourStart time.Time + GHIWm2 float64 + DHIWm2 *float64 +} + +// ExpectedWh returns the DC energy (Wh, positive) the given arrays would +// produce under the supplied hourly irradiance at (lat, lon). Each hour's +// plane-of-array irradiance is projected via sunpos and integrated over one +// hour. When measured diffuse is present it is used directly (more accurate); +// otherwise the diffuse fraction is estimated from the clearness index. +func ExpectedWh(lat, lon float64, arrays []Array, hours []Irradiance) float64 { + var wh float64 + for _, h := range hours { + if h.GHIWm2 <= 0 { + continue + } + sun := sunpos.At(h.HourStart, lat, lon) + for _, a := range arrays { + if a.KWp <= 0 { + continue + } + var poa float64 + if h.DHIWm2 != nil { + poa = sunpos.POAFromComponents(sun, h.GHIWm2, *h.DHIWm2, a.TiltDeg, a.AzimuthDeg) + } else { + poa = sunpos.POAFromGHI(h.HourStart, lat, lon, h.GHIWm2, a.TiltDeg, a.AzimuthDeg) + } + // kWp×1000 W at STC (1000 W/m²) × (POA/1000) × 1 h = Wh this hour. + wh += a.KWp * 1000.0 * (poa / 1000.0) + } + } + return wh +} + +// minExpectedWh is the floor below which a performance ratio is meaningless +// (polar-night / near-dark days) — reported as "not available" instead. +const minExpectedWh = 100.0 + +// PerformanceRatio returns actualWh / expectedWh, clamped to [0, 2]. The second +// return is false when expected production is too small to form a meaningful +// ratio, so callers can render "n/a" rather than a divide-by-tiny artifact. +func PerformanceRatio(expectedWh, actualWh float64) (float64, bool) { + if expectedWh < minExpectedWh { + return 0, false + } + pr := actualWh / expectedWh + if pr < 0 { + pr = 0 + } + if pr > 2 { + pr = 2 + } + return pr, true +} diff --git a/go/internal/pvperf/pvperf_test.go b/go/internal/pvperf/pvperf_test.go new file mode 100644 index 00000000..8408c617 --- /dev/null +++ b/go/internal/pvperf/pvperf_test.go @@ -0,0 +1,83 @@ +package pvperf + +import ( + "math" + "testing" + "time" +) + +// A few clear-ish midday summer hours at Stockholm, GHI + DHI populated. +func summerHours() []Irradiance { + base := time.Date(2024, 6, 21, 8, 0, 0, 0, time.UTC) + ghi := []float64{300, 450, 600, 700, 680, 550, 400, 250} + out := make([]Irradiance, len(ghi)) + for i, g := range ghi { + d := g * 0.25 + out[i] = Irradiance{HourStart: base.Add(time.Duration(i) * time.Hour), GHIWm2: g, DHIWm2: &d} + } + return out +} + +func TestExpectedWhScalesWithKWp(t *testing.T) { + hours := summerHours() + e5 := ExpectedWh(59.33, 18.07, []Array{{KWp: 5, TiltDeg: 35, AzimuthDeg: 180}}, hours) + e10 := ExpectedWh(59.33, 18.07, []Array{{KWp: 10, TiltDeg: 35, AzimuthDeg: 180}}, hours) + if e5 <= 0 { + t.Fatalf("expected positive energy, got %.1f", e5) + } + if math.Abs(e10/e5-2) > 1e-9 { + t.Errorf("10 kWp should be 2× 5 kWp, ratio %.4f", e10/e5) + } +} + +func TestExpectedWhSumsArrays(t *testing.T) { + hours := summerHours() + south := ExpectedWh(59.33, 18.07, []Array{{KWp: 5, TiltDeg: 35, AzimuthDeg: 180}}, hours) + east := ExpectedWh(59.33, 18.07, []Array{{KWp: 5, TiltDeg: 35, AzimuthDeg: 90}}, hours) + both := ExpectedWh(59.33, 18.07, []Array{ + {KWp: 5, TiltDeg: 35, AzimuthDeg: 180}, + {KWp: 5, TiltDeg: 35, AzimuthDeg: 90}, + }, hours) + if math.Abs(both-(south+east)) > 1e-6 { + t.Errorf("multi-array should sum: both=%.2f south+east=%.2f", both, south+east) + } +} + +// Measured diffuse should be honoured: an all-diffuse hour yields less on a +// south-tilted panel than the GHI-only Erbs path (which infers more beam). +func TestExpectedWhUsesMeasuredDiffuse(t *testing.T) { + base := time.Date(2024, 6, 21, 11, 0, 0, 0, time.UTC) + ghiOnly := []Irradiance{{HourStart: base, GHIWm2: 600}} + full := 600.0 + allDiffuse := []Irradiance{{HourStart: base, GHIWm2: 600, DHIWm2: &full}} + arr := []Array{{KWp: 10, TiltDeg: 35, AzimuthDeg: 180}} + + eErbs := ExpectedWh(59.33, 18.07, arr, ghiOnly) + eDiffuse := ExpectedWh(59.33, 18.07, arr, allDiffuse) + if !(eDiffuse < eErbs) { + t.Errorf("all-diffuse (%.1f) should be < Erbs-inferred-beam (%.1f)", eDiffuse, eErbs) + } +} + +func TestExpectedWhZeroWhenDark(t *testing.T) { + night := []Irradiance{{HourStart: time.Date(2024, 12, 21, 23, 0, 0, 0, time.UTC), GHIWm2: 0}} + if e := ExpectedWh(59.33, 18.07, []Array{{KWp: 10, TiltDeg: 35, AzimuthDeg: 180}}, night); e != 0 { + t.Errorf("dark hours should yield 0 Wh, got %.2f", e) + } +} + +func TestPerformanceRatio(t *testing.T) { + if _, ok := PerformanceRatio(50, 40); ok { + t.Error("tiny expected should be n/a") + } + pr, ok := PerformanceRatio(1000, 850) + if !ok || math.Abs(pr-0.85) > 1e-9 { + t.Errorf("pr=%.3f ok=%v, want 0.85 true", pr, ok) + } + if pr, _ := PerformanceRatio(1000, 5000); pr != 2 { + t.Errorf("PR should clamp high to 2, got %.2f", pr) + } + if pr, _ := PerformanceRatio(1000, -10); pr != 0 { + t.Errorf("PR should clamp low to 0, got %.2f", pr) + } +} diff --git a/go/internal/pvperf/service.go b/go/internal/pvperf/service.go new file mode 100644 index 00000000..cc9b0cce --- /dev/null +++ b/go/internal/pvperf/service.go @@ -0,0 +1,221 @@ +package pvperf + +import ( + "context" + "log/slog" + "time" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/coverage" + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/strang" +) + +// irradianceSource labels rows persisted by this service. +const irradianceSource = "strang" + +// defaultLookbackDays is how far back the backfill scores on each run. Older +// closed days already have an immutable cached score and are skipped. +const defaultLookbackDays = 30 + +// rescoreTailDays is how many of the most recent days are recomputed on every +// run even if already scored, so the STRÅNG ~1-day analysis lag is corrected as +// data lands (a day scored against partial irradiance is refreshed next run). +const rescoreTailDays = 3 + +// Service backfills historical STRÅNG irradiance and scores realised PV +// production against it, once at startup and nightly thereafter. It is +// read-only with respect to control: it only fetches weather data and writes +// the irradiance_history + pv_performance_daily tables. Nil when the site has +// no usable PV geometry (scoring is impossible), which the API surfaces as +// {enabled:false}. +type Service struct { + Store *state.Store + Strang *strang.Client + Lat, Lon float64 + Arrays []Array + LookbackDays int + + stop chan struct{} + done chan struct{} +} + +// FromConfig builds a scoring Service from the weather config, mirroring how +// forecast.FromConfig derives per-plane geometry (explicit pv_arrays, else a +// single synthesized array from the legacy flat fields). Returns nil when no +// geometry is available — without arrays there is nothing to score against. +func FromConfig(cfg *config.Weather, ratedPVW float64, st *state.Store, userAgent string) *Service { + if cfg == nil || st == nil { + return nil + } + var arrays []Array + for _, a := range cfg.PVArrays { + if a.KWp > 0 { + arrays = append(arrays, Array{KWp: a.KWp, TiltDeg: a.TiltDeg, AzimuthDeg: a.AzimuthDeg}) + } + } + if len(arrays) == 0 && ratedPVW > 0 { + arrays = append(arrays, Array{KWp: ratedPVW / 1000.0, TiltDeg: cfg.PVTiltDeg, AzimuthDeg: cfg.PVAzimuthDeg}) + } + if len(arrays) == 0 { + return nil + } + // STRÅNG only models the Nordic domain. Outside it every nightly backfill + // would spend three HTTP requests to be told nothing, forever, so decline + // to start at all. GET /api/data-sources is where an operator finds out + // why — this is a silent no-op by design, not a hidden failure. + if !coverage.Covers("strang", cfg.Latitude, cfg.Longitude) { + slog.Info("pvperf: site is outside the STRÅNG domain, PV performance scoring disabled", + "lat", cfg.Latitude, "lon", cfg.Longitude) + return nil + } + return &Service{ + Store: st, + Strang: strang.NewClient(userAgent), + Lat: cfg.Latitude, + Lon: cfg.Longitude, + Arrays: arrays, + LookbackDays: defaultLookbackDays, + stop: make(chan struct{}), + done: make(chan struct{}), + } +} + +// Start runs an initial backfill shortly after boot, then nightly. +func (s *Service) Start(ctx context.Context) { + go s.loop(ctx) +} + +// Stop terminates the backfill loop and waits for it to drain. +func (s *Service) Stop() { + close(s.stop) + <-s.done +} + +func (s *Service) loop(ctx context.Context) { + defer close(s.done) + // Delay the first run so boot isn't competing with a network fetch, and + // so telemetry has a moment to settle before we read history. + first := time.NewTimer(3 * time.Minute) + defer first.Stop() + tick := time.NewTicker(24 * time.Hour) + defer tick.Stop() + for { + select { + case <-s.stop: + return + case <-ctx.Done(): + return + case <-first.C: + s.runBackfill(ctx) + case <-tick.C: + s.runBackfill(ctx) + } + } +} + +// runBackfill fetches the lookback window from STRÅNG, persists the irradiance, +// and scores each closed day that is missing or within the rescore tail. +func (s *Service) runBackfill(ctx context.Context) { + fetchCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + + now := time.Now() + loc := now.Location() + todayMidnight := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, loc) + windowStart := todayMidnight.AddDate(0, 0, -s.LookbackDays) + + // STRÅNG takes calendar dates (UTC). Pad by a day each side so local-day + // boundaries are fully covered regardless of the UTC offset. + hours, err := s.Strang.FetchWindow(fetchCtx, s.Lat, s.Lon, + windowStart.UTC().AddDate(0, 0, -1), todayMidnight.UTC()) + if err != nil { + slog.Warn("pvperf: STRÅNG fetch failed", "err", err) + return + } + if len(hours) == 0 { + slog.Info("pvperf: STRÅNG returned no data for window", "lat", s.Lat, "lon", s.Lon) + return + } + + fetchedAtMs := now.UnixMilli() + rows := make([]state.IrradianceRow, 0, len(hours)) + for _, h := range hours { + rows = append(rows, state.IrradianceRow{ + SlotTsMs: h.HourStart.UnixMilli(), + GHIWm2: h.GHIWm2, + DHIWm2: h.DHIWm2, + Source: irradianceSource, + FetchedAtMs: fetchedAtMs, + }) + } + if err := s.Store.SaveIrradiance(rows); err != nil { + slog.Warn("pvperf: save irradiance failed", "err", err) + return + } + + scored := 0 + // Score closed days only (strictly before today's midnight). + for i := s.LookbackDays; i >= 1; i-- { + dayStart := todayMidnight.AddDate(0, 0, -i) + dayEnd := dayStart.AddDate(0, 0, 1) + day := dayStart.Format("2006-01-02") + + // Skip days already scored, except the recent tail we always refresh. + if i > rescoreTailDays { + if _, ok, _ := s.Store.LoadPVPerformanceDay(day); ok { + continue + } + } + if s.scoreDay(day, dayStart, dayEnd, hours, fetchedAtMs) { + scored++ + } + } + slog.Info("pvperf: backfill complete", "irradiance_rows", len(rows), "days_scored", scored) +} + +// scoreDay computes and persists one day's performance score. Returns false +// (and persists nothing) when there is no measured PV history for the day. +func (s *Service) scoreDay(day string, dayStart, dayEnd time.Time, hours []strang.IrradianceHour, fetchedAtMs int64) bool { + startMs, endMs := dayStart.UnixMilli(), dayEnd.UnixMilli() + + dayHours := make([]Irradiance, 0, 24) + for _, h := range hours { + ms := h.HourStart.UnixMilli() + if ms < startMs || ms >= endMs { + continue + } + dayHours = append(dayHours, Irradiance{HourStart: h.HourStart, GHIWm2: h.GHIWm2, DHIWm2: h.DHIWm2}) + } + + de, err := s.Store.DailyEnergy(startMs, endMs-1) + if err != nil { + slog.Warn("pvperf: read actual energy failed", "day", day, "err", err) + return false + } + if de.Intervals == 0 { + // No measured history for this day — nothing to score against. + return false + } + + expectedWh := ExpectedWh(s.Lat, s.Lon, s.Arrays, dayHours) + rec := state.PVPerformanceDay{ + Day: day, + ExpectedWh: expectedWh, + ActualWh: de.PVWh, + StrangDataDateMs: &fetchedAtMs, + } + if pr, ok := PerformanceRatio(expectedWh, de.PVWh); ok { + rec.PR = &pr + } + if err := s.Store.SavePVPerformance(rec); err != nil { + slog.Warn("pvperf: save score failed", "day", day, "err", err) + return false + } + return true +} + +// Load returns scored days in [sinceDay, untilDay] (inclusive YYYY-MM-DD). +func (s *Service) Load(sinceDay, untilDay string) ([]state.PVPerformanceDay, error) { + return s.Store.LoadPVPerformance(sinceDay, untilDay) +} diff --git a/go/internal/pvperf/service_test.go b/go/internal/pvperf/service_test.go new file mode 100644 index 00000000..e14628ce --- /dev/null +++ b/go/internal/pvperf/service_test.go @@ -0,0 +1,132 @@ +package pvperf + +import ( + "math" + "path/filepath" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/strang" +) + +func openStore(t *testing.T) *state.Store { + t.Helper() + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + return st +} + +func TestFromConfigGating(t *testing.T) { + st := openStore(t) + + if FromConfig(nil, 5000, st, "ua") != nil { + t.Error("nil weather config should yield nil service") + } + if FromConfig(&config.Weather{Latitude: 59, Longitude: 18}, 0, st, "ua") != nil { + t.Error("no arrays and no rated PV should yield nil service") + } + + // Explicit arrays win. + svc := FromConfig(&config.Weather{ + Latitude: 59, Longitude: 18, + PVArrays: []config.PVArray{{KWp: 5, TiltDeg: 35, AzimuthDeg: 180}, {KWp: 3, TiltDeg: 20, AzimuthDeg: 90}}, + }, 0, st, "ua") + if svc == nil || len(svc.Arrays) != 2 { + t.Fatalf("expected 2 arrays, got %+v", svc) + } + if svc.Arrays[0].KWp != 5 || svc.Arrays[1].AzimuthDeg != 90 { + t.Errorf("array geometry mismatch: %+v", svc.Arrays) + } + + // Flat fallback synthesizes one array from rated + legacy tilt/azimuth. + flat := FromConfig(&config.Weather{ + Latitude: 59, Longitude: 18, PVTiltDeg: 30, PVAzimuthDeg: 180, + }, 8000, st, "ua") + if flat == nil || len(flat.Arrays) != 1 { + t.Fatalf("flat fallback should synthesize one array, got %+v", flat) + } + if flat.Arrays[0].KWp != 8 || flat.Arrays[0].TiltDeg != 30 { + t.Errorf("synthesized array mismatch: %+v", flat.Arrays[0]) + } +} + +// A bell-ish clear day of hourly GHI centered on solar noon. +func syntheticDay(dayStart time.Time) []strang.IrradianceHour { + out := []strang.IrradianceHour{} + for hr := 4; hr <= 20; hr++ { + // crude parabola peaking ~700 W/m² at hour 12 + g := 700.0 - 12.0*float64((hr-12)*(hr-12)) + if g < 0 { + g = 0 + } + out = append(out, strang.IrradianceHour{ + HourStart: dayStart.Add(time.Duration(hr) * time.Hour), + GHIWm2: g, + }) + } + return out +} + +func TestScoreDayPersistsExpectedVsActual(t *testing.T) { + st := openStore(t) + svc := &Service{ + Store: st, + Lat: 59.33, + Lon: 18.07, + Arrays: []Array{{KWp: 10, TiltDeg: 35, AzimuthDeg: 180}}, + } + + dayStart := time.Date(2024, 6, 21, 0, 0, 0, 0, time.UTC) + dayEnd := dayStart.AddDate(0, 0, 1) + day := dayStart.Format("2006-01-02") + + // Seed measured PV history: -2000 W constant across 8h → 16000 Wh produced + // (PV is stored site-signed negative; DailyEnergy integrates SUM(-pv_w·Δt)). + if err := st.RecordHistory(state.HistoryPoint{TsMs: dayStart.Add(8 * time.Hour).UnixMilli(), PVW: -2000}); err != nil { + t.Fatal(err) + } + if err := st.RecordHistory(state.HistoryPoint{TsMs: dayStart.Add(16 * time.Hour).UnixMilli(), PVW: -2000}); err != nil { + t.Fatal(err) + } + + hours := syntheticDay(dayStart) + if !svc.scoreDay(day, dayStart, dayEnd, hours, 12345) { + t.Fatal("scoreDay should succeed with history present") + } + + got, ok, err := st.LoadPVPerformanceDay(day) + if err != nil || !ok { + t.Fatalf("score not persisted: ok=%v err=%v", ok, err) + } + if math.Abs(got.ActualWh-16000) > 1 { + t.Errorf("actual Wh: want ~16000, got %.1f", got.ActualWh) + } + if got.ExpectedWh <= 0 { + t.Errorf("expected Wh should be positive, got %.1f", got.ExpectedWh) + } + if got.PR == nil { + t.Error("PR should be set when expected is above the floor") + } + if got.StrangDataDateMs == nil || *got.StrangDataDateMs != 12345 { + t.Errorf("provenance not stamped: %+v", got.StrangDataDateMs) + } +} + +func TestScoreDaySkipsWhenNoHistory(t *testing.T) { + st := openStore(t) + svc := &Service{Store: st, Lat: 59.33, Lon: 18.07, Arrays: []Array{{KWp: 10, TiltDeg: 35, AzimuthDeg: 180}}} + + dayStart := time.Date(2024, 6, 21, 0, 0, 0, 0, time.UTC) + day := dayStart.Format("2006-01-02") + if svc.scoreDay(day, dayStart, dayStart.AddDate(0, 0, 1), syntheticDay(dayStart), 1) { + t.Error("scoreDay should return false with no measured history") + } + if _, ok, _ := st.LoadPVPerformanceDay(day); ok { + t.Error("nothing should be persisted when there's no history") + } +} diff --git a/go/internal/roofmodel/roofmodel.go b/go/internal/roofmodel/roofmodel.go new file mode 100644 index 00000000..ac6a484c --- /dev/null +++ b/go/internal/roofmodel/roofmodel.go @@ -0,0 +1,304 @@ +// Package roofmodel invokes the optional Lantmäteriet roof-geometry module and +// turns its output into candidate PV arrays. +// +// The module is a separate Python package (roofmodel/, sibling to optimizer/) +// reached at arm's length: core spawns it, hands it coordinates and the +// operator's Geotorget credentials on the command line, and reads one versioned +// JSON document back from stdout. Core knows nothing about STAC, LAZ or plane +// fitting, and the module knows nothing about FTW. +// +// That boundary is the point. LiDAR segmentation drags in a compiled point-cloud +// stack, runs for minutes, and is onboarding-time work rather than runtime work. +// Keeping it in a subprocess means it cannot stall the control tick, cannot leak +// memory into the daemon, and can be absent entirely — which is the normal case, +// since the data only exists for Sweden. +// +// Nothing here is authoritative. Derived arrays *pre-fill* the operator's +// editable weather.pv_arrays; the numeric editor stays the final word. +package roofmodel + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os/exec" + "time" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/coverage" +) + +var ( + // ErrDisabled means no roofmodel section is configured, or it is off. + ErrDisabled = errors.New("roof model module is not enabled") + // ErrOutsideCoverage means Lantmäteriet has no data for this site. + ErrOutsideCoverage = errors.New("outside Lantmäteriet coverage") + // ErrNoCredentials means Geotorget credentials are missing. + ErrNoCredentials = errors.New("Geotorget credentials are required") +) + +const ( + defaultCommand = "python3" + defaultRadiusM = 40.0 + defaultPackingFactor = 0.70 + defaultTimeout = 10 * time.Minute + // A roof model is small; anything larger is a runaway or a wrong command. + maxOutputBytes = 1 << 20 +) + +// Array is one derived candidate PV array. Field names mirror config.PVArray so +// the document can pre-fill weather.pv_arrays directly. +type Array struct { + Name string `json:"name"` + KWp float64 `json:"kwp"` + TiltDeg float64 `json:"tilt_deg"` + AzimuthDeg float64 `json:"azimuth_deg"` + AreaM2 float64 `json:"area_m2"` + SegmentID string `json:"segment_id"` + // ShadingFactor is the fraction of open-sky irradiation this face receives + // once neighbouring geometry is accounted for. nil means shading was not + // evaluated, which is deliberately distinct from 1.0 ("evaluated, and + // unobstructed"). + ShadingFactor *float64 `json:"shading_factor,omitempty"` +} + +// BuildingList is what `--mode buildings` emits: GeoJSON features a map can +// draw directly, nearest first. Geometry is passed through as raw JSON because +// core has no business interpreting a polygon -- it only ferries it to the UI. +type BuildingList struct { + SchemaVersion int `json:"schema_version"` + Site struct { + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + } `json:"site"` + Buildings []json.RawMessage `json:"buildings"` +} + +// Building records which footprint a model was derived from, and how much of +// the surrounding cloud survived the clip -- the honest measure of whether the +// footprint and the scan actually agree. +type Building struct { + BuildingID string `json:"building_id"` + AreaM2 float64 `json:"area_m2"` + Footprint json.RawMessage `json:"footprint"` + ReturnsUsed int `json:"returns_used"` + ReturnsInRadius int `json:"returns_in_radius"` +} + +// Model is the versioned document the module emits. +type Model struct { + SchemaVersion int `json:"schema_version"` + Arrays []Array `json:"arrays"` + PlanesFound int `json:"planes_found"` + // Building is null when the whole search radius was segmented rather than + // one picked footprint. + Building *Building `json:"building"` + // Shading reports whether the optional vostok pass ran, and why not when it + // did not. Absent evaluation leaves every ShadingFactor nil. + Shading struct { + Evaluated bool `json:"evaluated"` + Reason string `json:"reason"` + } `json:"shading"` + Site struct { + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + RadiusM float64 `json:"radius_m"` + } `json:"site"` + Source struct { + Provider string `json:"provider"` + Collection string `json:"collection"` + ItemCount int `json:"item_count"` + DatasetDatetime string `json:"dataset_datetime"` + // Fetch is "copc-window" when only the picked building's neighbourhood + // was pulled from the LiDAR tile, or "whole-tile" when the whole thing + // came across. It also qualifies Building.ReturnsInRadius below, whose + // denominator is the fetched window in the first case. + Fetch string `json:"fetch"` + } `json:"source"` + // CapturedAtMs is when Lantmäteriet flew the LiDAR. Null while their STAC + // datetime backfill is incomplete, which is a missing provenance date and + // not a failure — the UI shows "capture date unknown". + CapturedAtMs *int64 `json:"captured_at_ms"` + DerivedAtMs int64 `json:"derived_at_ms"` +} + +// moduleError is the JSON the module writes to stderr when it fails. +type moduleError struct { + Error string `json:"error"` + Kind string `json:"kind"` +} + +// Service derives roof models. The zero value is unusable; use FromConfig. +type Service struct { + cfg *config.RoofModel +} + +// FromConfig returns a Service, or nil when the module is not configured. A nil +// Service is safe to call: every method reports ErrDisabled. +func FromConfig(cfg *config.RoofModel) *Service { + if cfg == nil || !cfg.Enabled { + return nil + } + return &Service{cfg: cfg} +} + +// Enabled reports whether derives are possible. +func (s *Service) Enabled() bool { return s != nil && s.cfg != nil && s.cfg.Enabled } + +func (s *Service) timeout() time.Duration { + if s.cfg.TimeoutS > 0 { + return time.Duration(s.cfg.TimeoutS) * time.Second + } + return defaultTimeout +} + +func (s *Service) command() string { + if s.cfg.Command != "" { + return s.cfg.Command + } + return defaultCommand +} + +func (s *Service) radius() float64 { + if s.cfg.RadiusM > 0 { + return s.cfg.RadiusM + } + return defaultRadiusM +} + +func (s *Service) packingFactor() float64 { + if s.cfg.PackingFactor > 0 { + return s.cfg.PackingFactor + } + return defaultPackingFactor +} + +// Buildings lists candidate building footprints near a site, nearest first, so +// the operator can pick the one their panels are going on. +// +// Without this step a derive segments everything inside its radius: the +// neighbour's roof, the garage, the trees. Worse, RANSAC fits infinite planes, +// so a second building sharing the ridge orientation lands inside the first +// one's inlier band however far away it is and steals its returns. +func (s *Service) Buildings(ctx context.Context, lat, lon float64) (*BuildingList, error) { + out, err := s.run(ctx, lat, lon, "buildings", "") + if err != nil { + return nil, err + } + var list BuildingList + if err := json.Unmarshal(out, &list); err != nil { + return nil, fmt.Errorf("roof model returned unreadable output: %w", err) + } + if list.SchemaVersion != 1 { + return nil, fmt.Errorf("roof model schema_version %d is not supported", list.SchemaVersion) + } + slog.Info("roof model buildings", "lat", lat, "lon", lon, "found", len(list.Buildings)) + return &list, nil +} + +// Derive runs the module for one site. +// +// buildingID is optional; pass one from Buildings to clip the LiDAR to that +// footprint before segmenting, which is what makes the derived tilt and azimuth +// belong to the operator's own roof rather than to whatever else stood in range. +// +// Coverage and credentials are checked before spawning anything: a site outside +// Sweden can never succeed, and a missing credential fails the same way every +// time, so neither is worth an interpreter start and a network round trip. +func (s *Service) Derive(ctx context.Context, lat, lon float64, buildingID string) (*Model, error) { + out, err := s.run(ctx, lat, lon, "derive", buildingID) + if err != nil { + return nil, err + } + var m Model + if err := json.Unmarshal(out, &m); err != nil { + return nil, fmt.Errorf("roof model returned unreadable output: %w", err) + } + if m.SchemaVersion != 1 { + return nil, fmt.Errorf("roof model schema_version %d is not supported", m.SchemaVersion) + } + slog.Info("roof model derived", + "lat", lat, "lon", lon, "arrays", len(m.Arrays), + "planes", m.PlanesFound, "building", buildingID) + return &m, nil +} + +// run spawns the module and returns its stdout. +func (s *Service) run(ctx context.Context, lat, lon float64, mode, buildingID string) ([]byte, error) { + if !s.Enabled() { + return nil, ErrDisabled + } + if !coverage.Covers("lantmateriet", lat, lon) { + return nil, fmt.Errorf("%w: (%.4f, %.4f) is not in Sweden", ErrOutsideCoverage, lat, lon) + } + if s.cfg.GeotorgetUsername == "" || s.cfg.GeotorgetToken == "" { + return nil, ErrNoCredentials + } + + ctx, cancel := context.WithTimeout(ctx, s.timeout()) + defer cancel() + + args := []string{ + "-m", "ftw_roofmodel", + "--mode", mode, + "--lat", fmt.Sprintf("%.6f", lat), + "--lon", fmt.Sprintf("%.6f", lon), + "--username", s.cfg.GeotorgetUsername, + "--token", s.cfg.GeotorgetToken, + "--radius-m", fmt.Sprintf("%.1f", s.radius()), + "--packing-factor", fmt.Sprintf("%.3f", s.packingFactor()), + } + if buildingID != "" { + args = append(args, "--building-id", buildingID) + } + if s.cfg.VostokBinary != "" { + args = append(args, "--vostok", s.cfg.VostokBinary) + } + cmd := exec.CommandContext(ctx, s.command(), args...) + if s.cfg.ModuleDir != "" { + cmd.Env = append(cmd.Environ(), "PYTHONPATH="+s.cfg.ModuleDir) + } + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + + if ctx.Err() == context.DeadlineExceeded { + return nil, fmt.Errorf("roof model timed out after %s", s.timeout()) + } + if err != nil { + // The module reports failures as JSON on stderr so an operator sees a + // reason ("credentials rejected") rather than a Python traceback. + var me moduleError + if jsonErr := json.Unmarshal(bytes.TrimSpace(stderr.Bytes()), &me); jsonErr == nil && me.Error != "" { + return nil, fmt.Errorf("roof model: %s", me.Error) + } + return nil, fmt.Errorf("roof model failed: %w", err) + } + if stdout.Len() > maxOutputBytes { + return nil, fmt.Errorf("roof model returned %d bytes, refusing", stdout.Len()) + } + return stdout.Bytes(), nil +} + +// ToPVArrays converts derived arrays into config entries ready to be written +// into weather.pv_arrays. +func (m *Model) ToPVArrays() []config.PVArray { + if m == nil { + return nil + } + out := make([]config.PVArray, 0, len(m.Arrays)) + for _, a := range m.Arrays { + out = append(out, config.PVArray{ + Name: a.Name, + KWp: a.KWp, + TiltDeg: a.TiltDeg, + AzimuthDeg: a.AzimuthDeg, + }) + } + return out +} diff --git a/go/internal/roofmodel/roofmodel_test.go b/go/internal/roofmodel/roofmodel_test.go new file mode 100644 index 00000000..7c219f9d --- /dev/null +++ b/go/internal/roofmodel/roofmodel_test.go @@ -0,0 +1,499 @@ +package roofmodel + +import ( + "context" + "encoding/json" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/config" +) + +// The module is a subprocess, so exercising the contract -- argument passing, +// stdout parsing, stderr error reporting, timeouts -- needs something to spawn. +// This test binary stands in for it: with stubEnvVar set, TestMain impersonates +// the Python module instead of running tests. +// +// The first version of this harness wrote little sh and .bat scripts instead. +// It passed on Windows and failed on Linux in two separate ways: the service +// invokes the command as ` -m ftw_roofmodel ...`, which dash reads as +// "run the script named ftw_roofmodel" (exit 2, the stub never ran at all), and +// an unquoted JSON document loses its double quotes to shell word-splitting. +// Re-executing a compiled binary has no shell in the path, so there is no +// quoting to get wrong and nothing that can behave differently per platform. +const ( + stubModeVar = "FTW_ROOFMODEL_TEST_STUB" + stubPayloadVar = "FTW_ROOFMODEL_TEST_PAYLOAD" +) + +func TestMain(m *testing.M) { + if mode, ok := os.LookupEnv(stubModeVar); ok { + os.Exit(runStub(mode, os.Getenv(stubPayloadVar))) + } + os.Exit(m.Run()) +} + +// stubInvocation is what the stub records about how it was called, so a test +// can assert what core actually handed the module. +type stubInvocation struct { + Args []string `json:"args"` + PythonPath string `json:"pythonpath"` +} + +// runStub plays the part of `python3 -m ftw_roofmodel`. +func runStub(mode, payload string) int { + switch mode { + case "stdout": + os.Stdout.WriteString(payload) + return 0 + case "stderr": + // How the real module reports failure: JSON on stderr, non-zero exit. + os.Stderr.WriteString(payload) + return 1 + case "record": + enc, err := json.Marshal(stubInvocation{ + Args: os.Args[1:], + PythonPath: os.Getenv("PYTHONPATH"), + }) + if err != nil { + return 3 + } + if err := os.WriteFile(payload, enc, 0o600); err != nil { + return 3 + } + os.Stdout.WriteString(minimalModel) + return 0 + case "hang": + time.Sleep(30 * time.Second) + return 0 + } + os.Stderr.WriteString("unknown stub mode " + mode) + return 2 +} + +// stubModule points the service at this test binary running in stub mode. +func stubModule(t *testing.T, mode, payload string) string { + t.Helper() + t.Setenv(stubModeVar, mode) + t.Setenv(stubPayloadVar, payload) + exe, err := os.Executable() + if err != nil { + t.Fatalf("locating the test binary: %v", err) + } + return exe +} + +const minimalModel = `{"schema_version":1,"arrays":[]}` + +func svc(t *testing.T, cfg *config.RoofModel) *Service { + t.Helper() + s := FromConfig(cfg) + if s == nil { + t.Fatal("FromConfig returned nil for an enabled config") + } + return s +} + +const stockholmLat, stockholmLon = 59.33, 18.07 + +func TestDisabledWhenAbsentOrOff(t *testing.T) { + if FromConfig(nil) != nil { + t.Error("nil config must not produce a service") + } + if FromConfig(&config.RoofModel{Enabled: false}) != nil { + t.Error("disabled config must not produce a service") + } + // A nil *Service must be safe to call, not a panic. + var s *Service + if s.Enabled() { + t.Error("nil service must report disabled") + } + if _, err := s.Derive(context.Background(), stockholmLat, stockholmLon, ""); !errors.Is(err, ErrDisabled) { + t.Errorf("err = %v, want ErrDisabled", err) + } +} + +// A site outside Sweden can never succeed, so it must fail before spawning an +// interpreter and hitting the network. +func TestDeriveRefusesOutsideSwedenWithoutSpawning(t *testing.T) { + s := svc(t, &config.RoofModel{ + Enabled: true, + Command: "definitely-not-a-real-command", + GeotorgetUsername: "u", GeotorgetToken: "t", + }) + for _, c := range []struct { + name string + lat, lon float64 + }{ + {"Berlin", 52.52, 13.40}, + {"Sydney", -33.87, 151.21}, + {"New York", 40.71, -74.01}, + {"north of Sweden", 71.0, 20.0}, + } { + _, err := s.Derive(context.Background(), c.lat, c.lon, "") + if !errors.Is(err, ErrOutsideCoverage) { + t.Errorf("%s: err = %v, want ErrOutsideCoverage", c.name, err) + } + } +} + +// Sweden is a long diagonal, so no lat/lon rectangle can trace its border -- +// any box containing Sweden also contains parts of Norway and Finland. Oslo is +// the clearest example: it is west of Sweden but inside the box. +// +// This is the same advisory-superset property the coverage package documents +// for STRÅNG, and it is resolved upstream rather than geometrically: the STAC +// search returns no tiles and the module reports "Sweden only". Pinned so the +// box is not "tightened" into something that starts excluding real Swedish +// addresses near the border. +func TestSwedishBoxAdmitsSomeNonSwedishPointsByDesign(t *testing.T) { + s := svc(t, &config.RoofModel{ + Enabled: true, + Command: "definitely-not-a-real-command", + GeotorgetUsername: "u", GeotorgetToken: "t", + }) + _, err := s.Derive(context.Background(), 59.91, 10.75, "") // Oslo + if errors.Is(err, ErrOutsideCoverage) { + t.Skip("box now excludes Oslo; verify it still admits Strömstad and Haparanda") + } + if err == nil { + t.Fatal("want the spawn to fail, since the command does not exist") + } +} + +// The border towns are the reason the box stays generous: tightening it to +// exclude Oslo would start excluding real Swedish sites. +func TestSwedishBoxCoversBorderTowns(t *testing.T) { + s := svc(t, &config.RoofModel{ + Enabled: true, + Command: "definitely-not-a-real-command", + GeotorgetUsername: "u", GeotorgetToken: "t", + }) + for _, c := range []struct { + name string + lat, lon float64 + }{ + {"Strömstad (west coast, near Norway)", 58.94, 11.17}, + {"Haparanda (east, near Finland)", 65.83, 24.14}, + {"Karesuando (far north)", 68.44, 22.49}, + {"Smygehuk (far south)", 55.34, 13.36}, + } { + _, err := s.Derive(context.Background(), c.lat, c.lon, "") + if errors.Is(err, ErrOutsideCoverage) { + t.Errorf("%s: must not be excluded", c.name) + } + } +} + +func TestDeriveRequiresCredentials(t *testing.T) { + for _, c := range []struct { + name, user, token string + }{ + {"no username", "", "t"}, + {"no token", "u", ""}, + {"neither", "", ""}, + } { + s := svc(t, &config.RoofModel{Enabled: true, Command: "no-such-command", GeotorgetUsername: c.user, GeotorgetToken: c.token}) + _, err := s.Derive(context.Background(), stockholmLat, stockholmLon, "") + if !errors.Is(err, ErrNoCredentials) { + t.Errorf("%s: err = %v, want ErrNoCredentials", c.name, err) + } + } +} + +func TestDeriveParsesAModel(t *testing.T) { + doc := `{"schema_version":1,"planes_found":3,` + + `"site":{"latitude":59.33,"longitude":18.07,"radius_m":40},` + + `"source":{"provider":"lantmateriet","item_count":2,"dataset_datetime":"2018-03-01T00:00:00+00:00"},` + + `"arrays":[{"name":"Roof south","kwp":7.2,"tilt_deg":35,"azimuth_deg":180,"area_m2":51.4,"segment_id":"seg-0"}],` + + `"captured_at_ms":1519862400000,"derived_at_ms":1785456000000}` + cmd := stubModule(t, "stdout", doc) + s := svc(t, &config.RoofModel{Enabled: true, Command: cmd, GeotorgetUsername: "u", GeotorgetToken: "t"}) + + m, err := s.Derive(context.Background(), stockholmLat, stockholmLon, "") + if err != nil { + t.Fatal(err) + } + if m.SchemaVersion != 1 || m.PlanesFound != 3 { + t.Errorf("unexpected model: %+v", m) + } + if len(m.Arrays) != 1 || m.Arrays[0].Name != "Roof south" { + t.Fatalf("arrays = %+v", m.Arrays) + } + if m.CapturedAtMs == nil || *m.CapturedAtMs != 1519862400000 { + t.Errorf("captured_at_ms = %v", m.CapturedAtMs) + } +} + +// Whether the module streamed a window out of the COPC tile or pulled the whole +// thing changes what ReturnsInRadius counts, so the answer has to reach core +// rather than being inferred from the numbers. +func TestDeriveCarriesHowTheLidarWasFetched(t *testing.T) { + doc := `{"schema_version":1,"planes_found":2,` + + `"source":{"provider":"lantmateriet","collection":"laserdata-nedladdning-skog",` + + `"item_count":1,"fetch":"copc-window"},` + + `"building":{"building_id":"b-1","area_m2":144,"returns_used":220,"returns_in_radius":260},` + + `"arrays":[]}` + cmd := stubModule(t, "stdout", doc) + s := svc(t, &config.RoofModel{Enabled: true, Command: cmd, GeotorgetUsername: "u", GeotorgetToken: "t"}) + + m, err := s.Derive(context.Background(), stockholmLat, stockholmLon, "b-1") + if err != nil { + t.Fatal(err) + } + if m.Source.Fetch != "copc-window" { + t.Errorf("fetch = %q, want copc-window", m.Source.Fetch) + } + if m.Building == nil || m.Building.ReturnsUsed != 220 { + t.Fatalf("building = %+v", m.Building) + } +} + +// The site and the operator's credentials have to survive the process boundary, +// or the module derives a roof somewhere else entirely. +func TestDerivePassesTheSiteAndCredentials(t *testing.T) { + dir := t.TempDir() + record := dir + string(os.PathSeparator) + "invocation.json" + cmd := stubModule(t, "record", record) + s := svc(t, &config.RoofModel{ + Enabled: true, Command: cmd, ModuleDir: dir, + GeotorgetUsername: "operator", GeotorgetToken: "secret-token", + RadiusM: 25, + }) + + if _, err := s.Derive(context.Background(), stockholmLat, stockholmLon, ""); err != nil { + t.Fatal(err) + } + + raw, err := os.ReadFile(record) + if err != nil { + t.Fatalf("stub recorded nothing: %v", err) + } + var got stubInvocation + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatal(err) + } + line := strings.Join(got.Args, " ") + for _, want := range []string{ + "-m ftw_roofmodel", + "--lat 59.330000", + "--lon 18.070000", + "--username operator", + "--token secret-token", + "--radius-m 25.0", + } { + if !strings.Contains(line, want) { + t.Errorf("args %q missing %q", line, want) + } + } + // Without PYTHONPATH the module is only importable if it happens to be + // installed system-wide, which on a Pi it is not. + if got.PythonPath != dir { + t.Errorf("PYTHONPATH = %q, want %q", got.PythonPath, dir) + } + // vostok is opt-in: absent config must not silently enable a GPL tool. + if strings.Contains(line, "--vostok") { + t.Errorf("args %q passed --vostok without configuration", line) + } +} + +// Picking a building is the whole point of the picker: the id has to reach the +// module, or the derive silently segments the neighbourhood instead. +func TestDerivePassesThePickedBuilding(t *testing.T) { + dir := t.TempDir() + record := dir + string(os.PathSeparator) + "invocation.json" + cmd := stubModule(t, "record", record) + s := svc(t, &config.RoofModel{ + Enabled: true, Command: cmd, + GeotorgetUsername: "u", GeotorgetToken: "t", + }) + + if _, err := s.Derive(context.Background(), stockholmLat, stockholmLon, "bldg-42"); err != nil { + t.Fatal(err) + } + + line := strings.Join(readInvocation(t, record).Args, " ") + if !strings.Contains(line, "--building-id bldg-42") { + t.Errorf("args %q did not carry the picked building", line) + } + if !strings.Contains(line, "--mode derive") { + t.Errorf("args %q did not select derive mode", line) + } +} + +// Not picking one must not send an empty flag the module would treat as a +// building named "". +func TestDeriveOmitsTheBuildingFlagWhenNoneIsPicked(t *testing.T) { + dir := t.TempDir() + record := dir + string(os.PathSeparator) + "invocation.json" + cmd := stubModule(t, "record", record) + s := svc(t, &config.RoofModel{ + Enabled: true, Command: cmd, GeotorgetUsername: "u", GeotorgetToken: "t", + }) + + if _, err := s.Derive(context.Background(), stockholmLat, stockholmLon, ""); err != nil { + t.Fatal(err) + } + + if line := strings.Join(readInvocation(t, record).Args, " "); strings.Contains(line, "--building-id") { + t.Errorf("args %q passed an empty building id", line) + } +} + +func TestBuildingsListsFootprints(t *testing.T) { + doc := `{"schema_version":1,"site":{"latitude":59.33,"longitude":18.07},"buildings":[` + + `{"type":"Feature","id":"b1","geometry":{"type":"Polygon","coordinates":[[[18.0,59.3],[18.001,59.3],[18.001,59.301],[18.0,59.3]]]},"properties":{"area_m2":120.5}},` + + `{"type":"Feature","id":"b2","geometry":{"type":"Polygon","coordinates":[[[18.01,59.3],[18.011,59.3],[18.011,59.301],[18.01,59.3]]]},"properties":{"area_m2":64.0}}]}` + cmd := stubModule(t, "stdout", doc) + s := svc(t, &config.RoofModel{Enabled: true, Command: cmd, GeotorgetUsername: "u", GeotorgetToken: "t"}) + + list, err := s.Buildings(context.Background(), stockholmLat, stockholmLon) + if err != nil { + t.Fatal(err) + } + if len(list.Buildings) != 2 { + t.Fatalf("got %d buildings", len(list.Buildings)) + } + // Geometry is ferried to the map untouched, so it must survive intact. + if !strings.Contains(string(list.Buildings[0]), `"id":"b1"`) { + t.Errorf("first feature = %s", list.Buildings[0]) + } +} + +func TestBuildingsUsesBuildingsMode(t *testing.T) { + dir := t.TempDir() + record := dir + string(os.PathSeparator) + "invocation.json" + cmd := stubModule(t, "record", record) + s := svc(t, &config.RoofModel{Enabled: true, Command: cmd, GeotorgetUsername: "u", GeotorgetToken: "t"}) + + // The stub answers with a roof model, not a building list; only the + // invocation matters here. + _, _ = s.Buildings(context.Background(), stockholmLat, stockholmLon) + + if line := strings.Join(readInvocation(t, record).Args, " "); !strings.Contains(line, "--mode buildings") { + t.Errorf("args %q did not select buildings mode", line) + } +} + +// Every guard that protects a derive has to protect a building search too -- +// it is the same credentials and the same country. +func TestBuildingsRefusesOutsideCoverageAndWithoutCredentials(t *testing.T) { + s := svc(t, &config.RoofModel{ + Enabled: true, Command: "definitely-not-a-real-command", + GeotorgetUsername: "u", GeotorgetToken: "t", + }) + if _, err := s.Buildings(context.Background(), 52.52, 13.40); !errors.Is(err, ErrOutsideCoverage) { + t.Errorf("Berlin: err = %v, want ErrOutsideCoverage", err) + } + + noCreds := svc(t, &config.RoofModel{Enabled: true, Command: "no-such-command"}) + if _, err := noCreds.Buildings(context.Background(), stockholmLat, stockholmLon); !errors.Is(err, ErrNoCredentials) { + t.Errorf("err = %v, want ErrNoCredentials", err) + } + + var nilService *Service + if _, err := nilService.Buildings(context.Background(), stockholmLat, stockholmLon); !errors.Is(err, ErrDisabled) { + t.Errorf("err = %v, want ErrDisabled", err) + } +} + +func readInvocation(t *testing.T, path string) stubInvocation { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("stub recorded nothing: %v", err) + } + var got stubInvocation + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatal(err) + } + return got +} + +// The module signals failure as JSON on stderr precisely so an operator sees a +// cause rather than a traceback. +func TestDeriveSurfacesTheModuleErrorMessage(t *testing.T) { + cmd := stubModule(t, "stderr", + `{"error":"Geotorget rejected the credentials","kind":"MissingCredentials"}`) + s := svc(t, &config.RoofModel{Enabled: true, Command: cmd, GeotorgetUsername: "u", GeotorgetToken: "t"}) + + _, err := s.Derive(context.Background(), stockholmLat, stockholmLon, "") + if err == nil { + t.Fatal("want an error") + } + if !strings.Contains(err.Error(), "Geotorget rejected the credentials") { + t.Errorf("err = %v, want the module's own message", err) + } +} + +// A crash that is not the module's own JSON must still surface as an error +// rather than being mistaken for a successful empty model. +func TestDeriveReportsNonJSONFailure(t *testing.T) { + cmd := stubModule(t, "stderr", "Traceback (most recent call last):\n MemoryError\n") + s := svc(t, &config.RoofModel{Enabled: true, Command: cmd, GeotorgetUsername: "u", GeotorgetToken: "t"}) + + _, err := s.Derive(context.Background(), stockholmLat, stockholmLon, "") + if err == nil || !strings.Contains(err.Error(), "roof model failed") { + t.Errorf("err = %v, want a plain failure", err) + } +} + +func TestDeriveRejectsUnknownSchemaVersion(t *testing.T) { + cmd := stubModule(t, "stdout", `{"schema_version":99,"arrays":[]}`) + s := svc(t, &config.RoofModel{Enabled: true, Command: cmd, GeotorgetUsername: "u", GeotorgetToken: "t"}) + + _, err := s.Derive(context.Background(), stockholmLat, stockholmLon, "") + if err == nil || !strings.Contains(err.Error(), "schema_version") { + t.Errorf("err = %v, want a schema-version rejection", err) + } +} + +func TestDeriveRejectsUnreadableOutput(t *testing.T) { + cmd := stubModule(t, "stdout", "not-json-at-all") + s := svc(t, &config.RoofModel{Enabled: true, Command: cmd, GeotorgetUsername: "u", GeotorgetToken: "t"}) + + _, err := s.Derive(context.Background(), stockholmLat, stockholmLon, "") + if err == nil || !strings.Contains(err.Error(), "unreadable") { + t.Errorf("err = %v, want an unreadable-output error", err) + } +} + +// LiDAR tiles are large and this runs on a Pi; an unbounded derive could hold +// memory indefinitely. +func TestDeriveIsTimeBoxed(t *testing.T) { + cmd := stubModule(t, "hang", "") + s := svc(t, &config.RoofModel{ + Enabled: true, Command: cmd, + GeotorgetUsername: "u", GeotorgetToken: "t", TimeoutS: 1, + }) + + start := time.Now() + _, err := s.Derive(context.Background(), stockholmLat, stockholmLon, "") + if err == nil || !strings.Contains(err.Error(), "timed out") { + t.Errorf("err = %v, want a timeout", err) + } + if elapsed := time.Since(start); elapsed > 10*time.Second { + t.Errorf("took %s, timeout was not enforced", elapsed) + } +} + +func TestToPVArraysMatchesConfigShape(t *testing.T) { + m := &Model{Arrays: []Array{ + {Name: "Roof south", KWp: 7.2, TiltDeg: 35, AzimuthDeg: 180, AreaM2: 51.4}, + {Name: "Roof west", KWp: 4.1, TiltDeg: 35, AzimuthDeg: 270, AreaM2: 29.3}, + }} + got := m.ToPVArrays() + if len(got) != 2 { + t.Fatalf("got %d arrays", len(got)) + } + if got[0].Name != "Roof south" || got[0].KWp != 7.2 || got[0].TiltDeg != 35 || got[0].AzimuthDeg != 180 { + t.Errorf("array 0 = %+v", got[0]) + } + var nilModel *Model + if nilModel.ToPVArrays() != nil { + t.Error("nil model must yield nil arrays") + } +} diff --git a/go/internal/state/pvperf.go b/go/internal/state/pvperf.go new file mode 100644 index 00000000..59476f1e --- /dev/null +++ b/go/internal/state/pvperf.go @@ -0,0 +1,138 @@ +package state + +import ( + "database/sql" + "time" +) + +// ---- Irradiance history (cache.db) ---- + +// IrradianceRow is one hour of historical horizontal irradiance (W/m²). +// DHIWm2 is nil when the source did not provide a diffuse component. +type IrradianceRow struct { + SlotTsMs int64 `json:"slot_ts_ms"` + GHIWm2 float64 `json:"ghi_wm2"` + DHIWm2 *float64 `json:"dhi_wm2,omitempty"` + Source string `json:"source"` + FetchedAtMs int64 `json:"fetched_at_ms"` +} + +// SaveIrradiance upserts a batch of historical-irradiance rows (keyed by +// slot_ts_ms). Re-fetching a window overwrites the existing rows. +func (s *Store) SaveIrradiance(rows []IrradianceRow) error { + if len(rows) == 0 { + return nil + } + tx, err := s.cache.Begin() + if err != nil { + return err + } + defer tx.Rollback() + stmt, err := tx.Prepare(`INSERT INTO irradiance_history + (slot_ts_ms, ghi_wm2, dhi_wm2, source, fetched_at_ms) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (slot_ts_ms) DO UPDATE SET + ghi_wm2 = excluded.ghi_wm2, + dhi_wm2 = excluded.dhi_wm2, + source = excluded.source, + fetched_at_ms = excluded.fetched_at_ms`) + if err != nil { + return err + } + defer stmt.Close() + for _, r := range rows { + if _, err := stmt.Exec(r.SlotTsMs, r.GHIWm2, r.DHIWm2, r.Source, r.FetchedAtMs); err != nil { + return err + } + } + return tx.Commit() +} + +// LoadIrradiance returns irradiance rows in [sinceMs, untilMs], ascending. +func (s *Store) LoadIrradiance(sinceMs, untilMs int64) ([]IrradianceRow, error) { + rows, err := s.cache.Query(`SELECT slot_ts_ms, ghi_wm2, dhi_wm2, source, fetched_at_ms + FROM irradiance_history + WHERE slot_ts_ms BETWEEN ? AND ? + ORDER BY slot_ts_ms ASC`, sinceMs, untilMs) + if err != nil { + return nil, err + } + defer rows.Close() + out := []IrradianceRow{} + for rows.Next() { + var r IrradianceRow + if err := rows.Scan(&r.SlotTsMs, &r.GHIWm2, &r.DHIWm2, &r.Source, &r.FetchedAtMs); err != nil { + return out, err + } + out = append(out, r) + } + return out, rows.Err() +} + +// ---- PV performance daily (state.db) ---- + +// PVPerformanceDay is one day's PV performance score: expected DC energy under +// measured irradiance versus the site's actual generation, plus their ratio. +// PR is nil when expected production was below a meaningful floor (n/a). +type PVPerformanceDay struct { + Day string `json:"day"` // YYYY-MM-DD, local date + ExpectedWh float64 `json:"expected_wh"` + ActualWh float64 `json:"actual_wh"` + PR *float64 `json:"pr,omitempty"` + StrangDataDateMs *int64 `json:"strang_data_date_ms,omitempty"` + ComputedAtMs int64 `json:"computed_at_ms"` +} + +// SavePVPerformance upserts one day's PV performance score (keyed by day). +func (s *Store) SavePVPerformance(p PVPerformanceDay) error { + const q = ` + INSERT INTO pv_performance_daily( + day, expected_wh, actual_wh, pr, strang_data_date_ms, computed_at_ms + ) VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(day) DO UPDATE SET + expected_wh = excluded.expected_wh, + actual_wh = excluded.actual_wh, + pr = excluded.pr, + strang_data_date_ms = excluded.strang_data_date_ms, + computed_at_ms = excluded.computed_at_ms + ` + _, err := s.db.Exec(q, p.Day, p.ExpectedWh, p.ActualWh, p.PR, p.StrangDataDateMs, time.Now().UnixMilli()) + return err +} + +// LoadPVPerformanceDay returns one day's score, or ok=false on a cache miss. +func (s *Store) LoadPVPerformanceDay(day string) (PVPerformanceDay, bool, error) { + const q = `SELECT day, expected_wh, actual_wh, pr, strang_data_date_ms, computed_at_ms + FROM pv_performance_daily WHERE day = ?` + var p PVPerformanceDay + err := s.db.QueryRow(q, day).Scan(&p.Day, &p.ExpectedWh, &p.ActualWh, &p.PR, &p.StrangDataDateMs, &p.ComputedAtMs) + if err == sql.ErrNoRows { + return PVPerformanceDay{}, false, nil + } + if err != nil { + return PVPerformanceDay{}, false, err + } + return p, true, nil +} + +// LoadPVPerformance returns scored days in [sinceDay, untilDay] (inclusive, +// YYYY-MM-DD string compare), ascending by day. +func (s *Store) LoadPVPerformance(sinceDay, untilDay string) ([]PVPerformanceDay, error) { + rows, err := s.db.Query(`SELECT day, expected_wh, actual_wh, pr, strang_data_date_ms, computed_at_ms + FROM pv_performance_daily + WHERE day BETWEEN ? AND ? + ORDER BY day ASC`, sinceDay, untilDay) + if err != nil { + return nil, err + } + defer rows.Close() + out := []PVPerformanceDay{} + for rows.Next() { + var p PVPerformanceDay + if err := rows.Scan(&p.Day, &p.ExpectedWh, &p.ActualWh, &p.PR, &p.StrangDataDateMs, &p.ComputedAtMs); err != nil { + return out, err + } + out = append(out, p) + } + return out, rows.Err() +} diff --git a/go/internal/state/pvperf_test.go b/go/internal/state/pvperf_test.go new file mode 100644 index 00000000..ef1becad --- /dev/null +++ b/go/internal/state/pvperf_test.go @@ -0,0 +1,119 @@ +package state + +import ( + "testing" +) + +func f64(v float64) *float64 { return &v } +func i64(v int64) *int64 { return &v } + +func TestSaveIrradianceRoundtrip(t *testing.T) { + s := freshStore(t) + rows := []IrradianceRow{ + {SlotTsMs: 1000, GHIWm2: 300, DHIWm2: f64(80), Source: "strang", FetchedAtMs: 5000}, + {SlotTsMs: 2000, GHIWm2: 450, DHIWm2: nil, Source: "strang", FetchedAtMs: 5000}, + } + if err := s.SaveIrradiance(rows); err != nil { + t.Fatal(err) + } + got, err := s.LoadIrradiance(0, 10000) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("want 2 rows, got %d", len(got)) + } + if got[0].SlotTsMs != 1000 || got[0].GHIWm2 != 300 || got[0].DHIWm2 == nil || *got[0].DHIWm2 != 80 { + t.Errorf("row0 mismatch: %+v", got[0]) + } + if got[1].DHIWm2 != nil { + t.Errorf("row1 diffuse should be nil, got %v", *got[1].DHIWm2) + } + + // Upsert overwrites, no duplicate. + if err := s.SaveIrradiance([]IrradianceRow{{SlotTsMs: 1000, GHIWm2: 999, Source: "strang", FetchedAtMs: 6000}}); err != nil { + t.Fatal(err) + } + got, _ = s.LoadIrradiance(0, 10000) + if len(got) != 2 { + t.Fatalf("upsert should not add a row, got %d", len(got)) + } + if got[0].GHIWm2 != 999 { + t.Errorf("upsert should overwrite ghi, got %.0f", got[0].GHIWm2) + } +} + +func TestLoadIrradianceRangeFilters(t *testing.T) { + s := freshStore(t) + _ = s.SaveIrradiance([]IrradianceRow{ + {SlotTsMs: 100, GHIWm2: 1, Source: "strang", FetchedAtMs: 1}, + {SlotTsMs: 200, GHIWm2: 2, Source: "strang", FetchedAtMs: 1}, + {SlotTsMs: 300, GHIWm2: 3, Source: "strang", FetchedAtMs: 1}, + }) + got, err := s.LoadIrradiance(150, 250) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].SlotTsMs != 200 { + t.Fatalf("range filter failed: %+v", got) + } +} + +func TestPVPerformanceRoundtrip(t *testing.T) { + s := freshStore(t) + day := "2026-06-21" + p := PVPerformanceDay{ + Day: day, + ExpectedWh: 12000, + ActualWh: 10800, + PR: f64(0.9), + StrangDataDateMs: i64(1719000000000), + } + if err := s.SavePVPerformance(p); err != nil { + t.Fatal(err) + } + got, ok, err := s.LoadPVPerformanceDay(day) + if err != nil || !ok { + t.Fatalf("load ok=%v err=%v", ok, err) + } + if got.ExpectedWh != 12000 || got.ActualWh != 10800 || got.PR == nil || *got.PR != 0.9 { + t.Errorf("mismatch: %+v", got) + } + if got.StrangDataDateMs == nil || *got.StrangDataDateMs != 1719000000000 { + t.Errorf("provenance mismatch: %+v", got.StrangDataDateMs) + } + if got.ComputedAtMs == 0 { + t.Error("computed_at_ms should be stamped") + } + + // Upsert with n/a PR (nil) overwrites. + p.PR = nil + p.ExpectedWh = 50 + if err := s.SavePVPerformance(p); err != nil { + t.Fatal(err) + } + got, _, _ = s.LoadPVPerformanceDay(day) + if got.PR != nil { + t.Errorf("PR should be nil after upsert, got %v", *got.PR) + } + if got.ExpectedWh != 50 { + t.Errorf("expected_wh should overwrite, got %.0f", got.ExpectedWh) + } +} + +func TestLoadPVPerformanceMissAndRange(t *testing.T) { + s := freshStore(t) + if _, ok, err := s.LoadPVPerformanceDay("2000-01-01"); ok || err != nil { + t.Fatalf("miss should be ok=false err=nil, got ok=%v err=%v", ok, err) + } + for _, d := range []string{"2026-06-19", "2026-06-20", "2026-06-21"} { + _ = s.SavePVPerformance(PVPerformanceDay{Day: d, ExpectedWh: 1000, ActualWh: 900, PR: f64(0.9)}) + } + got, err := s.LoadPVPerformance("2026-06-20", "2026-06-21") + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].Day != "2026-06-20" || got[1].Day != "2026-06-21" { + t.Fatalf("range/order wrong: %+v", got) + } +} diff --git a/go/internal/state/store.go b/go/internal/state/store.go index 1aea5115..e0d98613 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -1030,6 +1030,22 @@ func (s *Store) migrate() error { ts_ms INTEGER NOT NULL, PRIMARY KEY(asset_id, flow, cursor_kind) ) WITHOUT ROWID, STRICT`, + // Persistent per-day PV performance score: the DC energy the + // configured arrays should have produced under measured STRÅNG + // irradiance (expected_wh) versus what the site actually generated + // (actual_wh, from history), and their ratio. Precious like + // energy_daily — closed days are immutable, so a computed score is + // cached here forever and never recomputed. pr is null when expected + // production was below a meaningful floor (polar-night / near-dark). + // strang_data_date_ms records the STRÅNG fetch time for provenance. + `CREATE TABLE IF NOT EXISTS pv_performance_daily ( + day TEXT PRIMARY KEY, + expected_wh REAL NOT NULL, + actual_wh REAL NOT NULL, + pr REAL, + strang_data_date_ms INTEGER, + computed_at_ms INTEGER NOT NULL + ) STRICT`, } for _, stmt := range stmts { if _, err := s.db.Exec(stmt); err != nil { @@ -1068,6 +1084,18 @@ func (s *Store) migrate() error { source TEXT NOT NULL, fetched_at_ms INTEGER NOT NULL )`, + // Historical solar irradiance — one row per hour. Backfilled from + // SMHI STRÅNG (an analysis product, ~1-day lag) to score realised PV + // performance against a weather-expected baseline. Disposable and + // re-fetchable, so it lives in cache.db alongside forecasts. dhi_wm2 + // (diffuse) is null when the source doesn't provide it. + `CREATE TABLE IF NOT EXISTS irradiance_history ( + slot_ts_ms INTEGER PRIMARY KEY, + ghi_wm2 REAL NOT NULL, + dhi_wm2 REAL, + source TEXT NOT NULL, + fetched_at_ms INTEGER NOT NULL + )`, } for _, stmt := range cacheStmts { if _, err := s.cache.Exec(stmt); err != nil { diff --git a/go/internal/strang/strang.go b/go/internal/strang/strang.go new file mode 100644 index 00000000..579d5136 --- /dev/null +++ b/go/internal/strang/strang.go @@ -0,0 +1,232 @@ +// Package strang fetches historical solar irradiance from SMHI's STRÅNG +// mesoscale model (https://strang.smhi.se/). +// +// STRÅNG is an analysis/reanalysis product: it covers the Nordic region hourly +// at ~2.5 km resolution from 1999 to ~1 day ago. It has NO forward horizon, so +// it is used here for historical PV-performance scoring and model calibration, +// never as a forward forecast provider (those stay in the forecast package). +// +// Data is free and licensed CC BY 4.0 — attribution to SMHI required. No API +// key. The public point time-series endpoint is: +// +// {base}/geotype/point/lon/{lon}/lat/{lat}/parameter/{p}/data.json?from=&to=&interval=hourly +package strang + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "sort" + "time" + + "github.com/srcfl/ftw/go/internal/coverage" + "github.com/srcfl/ftw/go/internal/sunpos" +) + +// ErrOutsideDomain is returned when a request falls outside STRÅNG's Nordic +// model domain. Callers should treat it as "this site can never be scored by +// STRÅNG" and stop asking, rather than as a transient failure to retry. +var ErrOutsideDomain = errors.New("outside STRÅNG domain") + +// STRÅNG parameter codes (category "strang1g", version 1). +// +// These are the complete set — probing 100..130 against the live API on +// 2026-07-31 returned 200 for exactly 116..122 and 404 for everything else. +// Names were confirmed by their magnitudes on a clear day at Stockholm rather +// than from documentation, since SMHI's apidocs pages currently 404: parameter +// 119 caps at exactly 60 (minutes in an hour), 118 exceeds 117 the way direct +// *normal* irradiance exceeds global, and at solar noon 121 + 122 = 723.0 + +// 87.5 = 810.5, which is exactly 117 — the direct-plus-diffuse identity that +// tells us 121 is direct *horizontal* and not something else. +// +// Note what is absent: STRÅNG models radiation only and publishes no cloud +// cover. See CloudCover for how cloudiness is recovered from parameter 119. +const ( + ParamCIEUV = 116 // CIE-weighted UV irradiance, mW/m² + ParamGlobalIrradiance = 117 // Global (horizontal) irradiance, W/m² — GHI + ParamDirectNormal = 118 // Direct normal irradiance, W/m² — DNI + ParamSunshineDuration = 119 // Sunshine duration within the hour, minutes 0..60 + ParamPAR = 120 // Photosynthetically active radiation, W/m² + ParamDirectHorizontal = 121 // Direct horizontal irradiance, W/m² + ParamDiffuseIrradiance = 122 // Diffuse (horizontal) irradiance, W/m² — DHI +) + +// minutesPerHour is the full-sun value of ParamSunshineDuration. +const minutesPerHour = 60.0 + +// DefaultBaseURL is SMHI's open-data meteorological-analysis host for STRÅNG. +const DefaultBaseURL = "https://opendata-download-metanalys.smhi.se/api/category/strang1g/version/1" + +// IrradianceHour is one hour of horizontal irradiance at a point. DHIWm2 is nil +// when the diffuse component is unavailable (e.g. windows before 2017-04-18, or +// a transient diffuse-parameter error) — callers then estimate the split. +// SunshineMin is nil on the same terms and carries parameter 119. +type IrradianceHour struct { + HourStart time.Time + GHIWm2 float64 + DHIWm2 *float64 + SunshineMin *float64 +} + +// CloudCover derives a cloud-cover fraction (0 = clear, 1 = overcast) for the +// hour at (lat, lon), and reports whether it could be derived at all. +// +// STRÅNG publishes no cloud-cover parameter, but parameter 119 is sunshine +// duration: the number of minutes in the hour during which direct beam +// irradiance exceeded the WMO sunshine threshold (120 W/m²). One minus that +// fraction is the share of the hour the sun spent obscured, which is what +// "cloud cover" means for a solar model's purposes. +// +// This is an *observed* quantity, not an inference from a cloud field, so it is +// better grounded than a forecast provider's cloud percentage. It is coarser in +// one direction: it cannot see thin cirrus that dims without blocking. +// +// The location is required because sunshine duration is zero at night for the +// trivial reason that there is no sun — reading that as "100% overcast" would be +// confidently wrong every single night. When the sun is below the horizon for +// the whole hour this returns not-ok, and callers must treat that as unknown +// rather than as clear or as overcast. +func (h IrradianceHour) CloudCover(lat, lon float64) (float64, bool) { + if h.SunshineMin == nil { + return 0, false + } + if !h.daylight(lat, lon) { + return 0, false + } + m := *h.SunshineMin + if m < 0 { + return 0, false + } + if m > minutesPerHour { + m = minutesPerHour + } + return 1 - m/minutesPerHour, true +} + +// minSunElevationDeg is how high the sun must get during the hour before a +// sunshine-duration reading says anything about cloud. +// +// The WMO sunshine threshold is 120 W/m² of direct beam. Near the horizon the +// beam crosses roughly ten or more air masses and cannot reach that threshold +// even under a spotless sky, so a zero reading there is a statement about +// geometry, not about cloud. Five degrees is where the beam can plausibly clear +// the threshold; below it we decline to answer rather than report a twilight +// hour as fully overcast. +const minSunElevationDeg = 5.0 + +// daylight reports whether the sun climbs above minSunElevationDeg at any point +// in the hour. Sampling start, middle and end catches the sunrise and sunset +// hours, where the midpoint alone would misclassify half the hour. +func (h IrradianceHour) daylight(lat, lon float64) bool { + for _, off := range []time.Duration{0, 30 * time.Minute, 59 * time.Minute} { + if sunpos.At(h.HourStart.Add(off), lat, lon).ZenithDeg < 90-minSunElevationDeg { + return true + } + } + return false +} + +// Client is a thin STRÅNG point-series HTTP client. +type Client struct { + HTTP *http.Client + BaseURL string + UserAgent string +} + +// NewClient returns a Client with sane defaults. A descriptive User-Agent is +// required by SMHI's fair-use policy, mirroring the forecast providers. +func NewClient(userAgent string) *Client { + if userAgent == "" { + userAgent = "FTW github.com/srcfl/ftw" + } + return &Client{ + HTTP: &http.Client{Timeout: 30 * time.Second}, + BaseURL: DefaultBaseURL, + UserAgent: userAgent, + } +} + +// FetchWindow returns hourly irradiance for [start, end] (dates, UTC) at +// (lat, lon). Global irradiance is required; diffuse is best-effort — a diffuse +// error (common for pre-2017 windows) leaves DHIWm2 nil rather than failing the +// whole window. Rows are returned ascending by hour. +func (c *Client) FetchWindow(ctx context.Context, lat, lon float64, start, end time.Time) ([]IrradianceHour, error) { + if !coverage.Covers("strang", lat, lon) { + return nil, fmt.Errorf("strang: %w: (%.4f, %.4f) is outside the Nordic model domain", ErrOutsideDomain, lat, lon) + } + ghi, err := c.fetchParam(ctx, lat, lon, ParamGlobalIrradiance, start, end) + if err != nil { + return nil, fmt.Errorf("strang: global irradiance: %w", err) + } + // Best-effort extras: never fail the window because a secondary parameter + // errored. Both leave their field nil and callers fall back — the diffuse + // split is estimated, and cloud cover simply reports unknown. + dhi, _ := c.fetchParam(ctx, lat, lon, ParamDiffuseIrradiance, start, end) + sun, _ := c.fetchParam(ctx, lat, lon, ParamSunshineDuration, start, end) + + hours := make([]int64, 0, len(ghi)) + for ms := range ghi { + hours = append(hours, ms) + } + sort.Slice(hours, func(i, j int) bool { return hours[i] < hours[j] }) + + out := make([]IrradianceHour, 0, len(hours)) + for _, ms := range hours { + h := IrradianceHour{HourStart: time.UnixMilli(ms).UTC(), GHIWm2: ghi[ms]} + if d, ok := dhi[ms]; ok { + dv := d + h.DHIWm2 = &dv + } + if s, ok := sun[ms]; ok { + sv := s + h.SunshineMin = &sv + } + out = append(out, h) + } + return out, nil +} + +// fetchParam returns hour-start-ms → value for one STRÅNG parameter. +func (c *Client) fetchParam(ctx context.Context, lat, lon float64, param int, start, end time.Time) (map[int64]float64, error) { + url := fmt.Sprintf("%s/geotype/point/lon/%.4f/lat/%.4f/parameter/%d/data.json?from=%s&to=%s&interval=hourly", + c.BaseURL, lon, lat, param, + start.UTC().Format("2006-01-02"), end.UTC().Format("2006-01-02")) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", c.UserAgent) + req.Header.Set("Accept", "application/json") + resp, err := c.HTTP.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return nil, fmt.Errorf("status %d: %s", resp.StatusCode, string(body)) + } + // STRÅNG data.json is an array of {date_time, value}. + var doc []struct { + DateTime string `json:"date_time"` + Value *float64 `json:"value"` + } + if err := json.NewDecoder(resp.Body).Decode(&doc); err != nil { + return nil, fmt.Errorf("decode: %w", err) + } + out := make(map[int64]float64, len(doc)) + for _, d := range doc { + if d.Value == nil { + continue + } + t, err := time.Parse(time.RFC3339, d.DateTime) + if err != nil { + continue + } + out[t.UTC().Truncate(time.Hour).UnixMilli()] = *d.Value + } + return out, nil +} diff --git a/go/internal/strang/strang_test.go b/go/internal/strang/strang_test.go new file mode 100644 index 00000000..1596fbff --- /dev/null +++ b/go/internal/strang/strang_test.go @@ -0,0 +1,306 @@ +package strang + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/sunpos" +) + +func TestFetchWindowMergesGHIAndDHI(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var arr []map[string]any + switch { + case strings.Contains(r.URL.Path, "/parameter/117/"): + arr = []map[string]any{ + {"date_time": "2024-06-01T10:00:00Z", "value": 500.0}, + {"date_time": "2024-06-01T11:00:00Z", "value": 650.0}, + } + case strings.Contains(r.URL.Path, "/parameter/122/"): + arr = []map[string]any{ + {"date_time": "2024-06-01T10:00:00Z", "value": 120.0}, + {"date_time": "2024-06-01T11:00:00Z", "value": 150.0}, + } + } + _ = json.NewEncoder(w).Encode(arr) + }) + srv := httptest.NewServer(handler) + defer srv.Close() + + c := NewClient("test") + c.BaseURL = srv.URL + hours, err := c.FetchWindow(context.Background(), 59.33, 18.07, + time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + time.Date(2024, 6, 2, 0, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatal(err) + } + if len(hours) != 2 { + t.Fatalf("got %d hours, want 2", len(hours)) + } + if hours[0].GHIWm2 != 500 || hours[1].GHIWm2 != 650 { + t.Errorf("GHI mismatch: %+v", hours) + } + if hours[0].DHIWm2 == nil || *hours[0].DHIWm2 != 120 { + t.Errorf("DHI[0] mismatch: %+v", hours[0]) + } + if !hours[0].HourStart.Before(hours[1].HourStart) { + t.Error("hours should be ascending") + } +} + +// Diffuse (122) unavailable — common for pre-2017 windows — must not fail the +// window; GHI still returns with nil DHI. +func TestFetchWindowDiffuseErrorTolerated(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/parameter/122/") { + w.WriteHeader(500) + return + } + _ = json.NewEncoder(w).Encode([]map[string]any{ + {"date_time": "2016-06-01T10:00:00Z", "value": 480.0}, + }) + }) + srv := httptest.NewServer(handler) + defer srv.Close() + + c := NewClient("test") + c.BaseURL = srv.URL + hours, err := c.FetchWindow(context.Background(), 59, 18, + time.Date(2016, 6, 1, 0, 0, 0, 0, time.UTC), + time.Date(2016, 6, 2, 0, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatalf("diffuse error should not fail window: %v", err) + } + if len(hours) != 1 || hours[0].DHIWm2 != nil { + t.Errorf("expected 1 hour with nil DHI, got %+v", hours) + } +} + +// Global (117) error must fail the window — GHI is required. +func TestFetchWindowGlobalErrorFails(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + })) + defer srv.Close() + + c := NewClient("t") + c.BaseURL = srv.URL + _, err := c.FetchWindow(context.Background(), 59, 18, + time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC)) + if err == nil { + t.Error("global irradiance error should fail the window") + } +} + +// Null values in the series are skipped, not decoded as 0. +func TestFetchWindowSkipsNullValues(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/parameter/122/") { + _ = json.NewEncoder(w).Encode([]map[string]any{}) + return + } + _ = json.NewEncoder(w).Encode([]map[string]any{ + {"date_time": "2024-06-01T10:00:00Z", "value": 500.0}, + {"date_time": "2024-06-01T11:00:00Z", "value": nil}, + }) + }) + srv := httptest.NewServer(handler) + defer srv.Close() + + c := NewClient("t") + c.BaseURL = srv.URL + hours, err := c.FetchWindow(context.Background(), 59, 18, + time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + time.Date(2024, 6, 2, 0, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatal(err) + } + if len(hours) != 1 || hours[0].GHIWm2 != 500 { + t.Errorf("null value should be skipped, got %+v", hours) + } +} + +// --- cloud cover derived from sunshine duration (parameter 119) --- + +func minutesPtr(v float64) *float64 { return &v } + +const ( + sthlmLat = 59.33 + sthlmLon = 18.07 +) + +// Midsummer noon and midnight at Stockholm: unambiguously day and night. +var ( + noonUTC = time.Date(2026, 6, 21, 12, 0, 0, 0, time.UTC) + midnightUTC = time.Date(2026, 12, 21, 23, 0, 0, 0, time.UTC) +) + +func TestCloudCoverFromSunshineDuration(t *testing.T) { + cases := []struct { + name string + minutes *float64 + want float64 + wantOK bool + }{ + {"full hour of sun is clear sky", minutesPtr(60), 0, true}, + {"no sun at all is overcast", minutesPtr(0), 1, true}, + {"half an hour is half cover", minutesPtr(30), 0.5, true}, + {"quarter hour is three quarters cover", minutesPtr(15), 0.75, true}, + {"missing parameter is unknown, not clear", nil, 0, false}, + {"negative is rejected as unknown", minutesPtr(-1), 0, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + h := IrradianceHour{HourStart: noonUTC, SunshineMin: c.minutes} + got, ok := h.CloudCover(sthlmLat, sthlmLon) + if ok != c.wantOK { + t.Fatalf("ok = %v, want %v", ok, c.wantOK) + } + if ok && got != c.want { + t.Errorf("cover = %v, want %v", got, c.want) + } + }) + } +} + +// A value above 60 would push cover negative and read as "brighter than clear", +// which is meaningless. Clamp instead. +func TestCloudCoverClampsAboveFullHour(t *testing.T) { + h := IrradianceHour{HourStart: noonUTC, SunshineMin: minutesPtr(75)} + got, ok := h.CloudCover(sthlmLat, sthlmLon) + if !ok { + t.Fatal("want derivable") + } + if got != 0 { + t.Errorf("cover = %v, want 0 (clamped)", got) + } +} + +// The distinction that matters at a call site: unknown must never be mistaken +// for clear, because they lead to opposite decisions. +func TestCloudCoverUnknownIsDistinguishableFromClear(t *testing.T) { + unknown, okU := IrradianceHour{HourStart: noonUTC}.CloudCover(sthlmLat, sthlmLon) + clear, okC := IrradianceHour{HourStart: noonUTC, SunshineMin: minutesPtr(60)}.CloudCover(sthlmLat, sthlmLon) + if okU { + t.Error("absent sunshine must report not-ok") + } + if !okC { + t.Error("full sun must report ok") + } + if unknown != clear { + t.Log("values differ, but callers must branch on the boolean, not the value") + } +} + +// Outside the Nordic domain STRÅNG can never return data, so the client must +// refuse locally rather than spend three HTTP requests learning that. +func TestFetchWindowRefusesOutsideDomain(t *testing.T) { + c := NewClient("test") + c.BaseURL = "http://127.0.0.1:1" // must never be dialled + _, err := c.FetchWindow(context.Background(), -33.87, 151.21, + time.Date(2026, 6, 21, 0, 0, 0, 0, time.UTC), + time.Date(2026, 6, 22, 0, 0, 0, 0, time.UTC)) + if err == nil { + t.Fatal("want an error for Sydney") + } + if !errors.Is(err, ErrOutsideDomain) { + t.Errorf("err = %v, want ErrOutsideDomain", err) + } +} + +func TestFetchWindowAcceptsInsideDomain(t *testing.T) { + // Stockholm is in-domain, so this must get past the guard and fail on the + // unreachable transport instead. + c := NewClient("test") + c.BaseURL = "http://127.0.0.1:1" + _, err := c.FetchWindow(context.Background(), 59.33, 18.07, + time.Date(2026, 6, 21, 0, 0, 0, 0, time.UTC), + time.Date(2026, 6, 22, 0, 0, 0, 0, time.UTC)) + if errors.Is(err, ErrOutsideDomain) { + t.Fatal("Stockholm must not be rejected as outside the domain") + } +} + +// Sunshine duration is zero at night because there is no sun, not because it is +// overcast. Reporting 100% cover would be confidently wrong every single night, +// which is exactly what the live API returned before this guard existed. +func TestCloudCoverIsUnknownAtNight(t *testing.T) { + h := IrradianceHour{HourStart: midnightUTC, SunshineMin: minutesPtr(0)} + if _, ok := h.CloudCover(sthlmLat, sthlmLon); ok { + t.Error("midwinter midnight must report unknown, not 100% cloud") + } +} + +// The hour the sun climbs through the threshold must be answerable, and it is +// only answerable because all three sample points are checked. Midsummer at +// Stockholm, 02:00Z: elevation runs 1.59 deg at :00 and 4.34 deg at :30 — both +// below the 5 deg cutoff — reaching 7.25 deg by :59. Sampling the start or the +// midpoint alone would discard a genuinely observed half-hour of sunshine. +func TestCloudCoverCountsHourWhereSunCrossesThreshold(t *testing.T) { + start := time.Date(2026, 6, 21, 2, 0, 0, 0, time.UTC) + if sunpos.At(start, sthlmLat, sthlmLon).ZenithDeg < 90-minSunElevationDeg { + t.Fatal("premise broken: the sun should start this hour below the cutoff") + } + h := IrradianceHour{HourStart: start, SunshineMin: minutesPtr(30)} + got, ok := h.CloudCover(sthlmLat, sthlmLon) + if !ok { + t.Fatal("the hour the sun crosses the cutoff should be derivable") + } + if got != 0.5 { + t.Errorf("cover = %v, want 0.5", got) + } +} + +// Polar night: the sun never rises, so no hour of the day is derivable. +func TestCloudCoverUnknownThroughPolarNight(t *testing.T) { + const tromsoLat, tromsoLon = 69.65, 18.96 + for hour := 0; hour < 24; hour++ { + h := IrradianceHour{ + HourStart: time.Date(2026, 12, 21, hour, 0, 0, 0, time.UTC), + SunshineMin: minutesPtr(0), + } + if _, ok := h.CloudCover(tromsoLat, tromsoLon); ok { + t.Errorf("hour %02d: polar night must report unknown", hour) + } + } +} + +// Near sunrise/sunset the beam crosses too much atmosphere to clear the WMO +// threshold even under a clear sky, so a zero reading there describes geometry +// rather than cloud. Verified against the live API: 2026-06-21 20:00Z at +// Stockholm has GHI 2.5 W/m2 and 0 minutes of sunshine — the sun is minutes +// from setting, and calling that "100% overcast" would be wrong. +func TestCloudCoverDeclinesNearTheHorizon(t *testing.T) { + h := IrradianceHour{ + HourStart: time.Date(2026, 6, 21, 20, 0, 0, 0, time.UTC), + SunshineMin: minutesPtr(0), + } + if _, ok := h.CloudCover(sthlmLat, sthlmLon); ok { + t.Error("a sun about to set must report unknown, not fully overcast") + } +} + +// ...but a genuinely low-yet-usable sun must still be answerable, otherwise the +// guard would silently discard most of a Nordic winter. +func TestCloudCoverStillAnswersWhenSunIsUsablyUp(t *testing.T) { + // 2026-06-21 04:00Z at Stockholm: live GHI 163.2 W/m2, 60 min sunshine. + h := IrradianceHour{ + HourStart: time.Date(2026, 6, 21, 4, 0, 0, 0, time.UTC), + SunshineMin: minutesPtr(60), + } + got, ok := h.CloudCover(sthlmLat, sthlmLon) + if !ok { + t.Fatal("a usable morning sun must be derivable") + } + if got != 0 { + t.Errorf("cover = %v, want 0 (full sunshine)", got) + } +} diff --git a/go/internal/sunpos/sunpos.go b/go/internal/sunpos/sunpos.go index 8ce95cc3..787e7c27 100644 --- a/go/internal/sunpos/sunpos.go +++ b/go/internal/sunpos/sunpos.go @@ -125,35 +125,117 @@ func ClearSkyW(t time.Time, lat, lon float64) float64 { } // POA estimates plane-of-array irradiance for one tilted panel using the -// isotropic-sky model. Splits clear-sky horizontal irradiance into beam -// (DNI) and diffuse (DHI) components via a simple Erbs correlation, then -// projects each onto the panel. +// isotropic-sky model, driven by the package's own clear-sky prior. It splits +// that clear-sky horizontal irradiance into beam (DNI) and diffuse (DHI) +// components with a fixed 20% diffuse fraction, then projects each onto the +// panel. Used as the prior signal for the PV twin when no measured irradiance +// is available. // // Returns W/m² on the panel surface; clamped to ≥ 0. +// +// When a data source supplies measured irradiance, prefer the two variants +// below: POAFromComponents (GHI + DHI both known, e.g. SMHI STRÅNG params +// 117 + 122) or POAFromGHI (only GHI known, e.g. Open-Meteo shortwave). func POA(t time.Time, lat, lon, panelTiltDeg, panelAzDeg float64) float64 { sun := At(t, lat, lon) - if sun.ZenithDeg >= 90 { - return 0 - } ghi := ClearSkyW(t, lat, lon) - if ghi <= 0 { + // No measured diffuse component from the clear-sky prior, so keep the + // historical fixed 20% diffuse fraction for this variant. + return POAFromComponents(sun, ghi, 0.2*ghi, panelTiltDeg, panelAzDeg) +} + +// POAFromComponents projects measured global (GHI) and diffuse (DHI) +// horizontal irradiance onto a tilted panel using the isotropic-sky model, +// reusing AOI for the beam projection. All irradiances in W/m²; returns the +// plane-of-array irradiance in W/m², clamped ≥ 0. +// +// Use this when a source gives both GHI and DHI directly (e.g. SMHI STRÅNG +// parameters 117 + 122). When only GHI is available use POAFromGHI, which +// estimates the diffuse split via the Erbs correlation first. +func POAFromComponents(sun Position, ghi, dhi, panelTiltDeg, panelAzDeg float64) float64 { + if sun.ZenithDeg >= 90 || ghi <= 0 { return 0 } - // Erbs et al. (1982) clearness-based diffuse fraction. We don't have - // real GHI measurements, so kt comes from our own clear-sky model → - // always ~0.7-0.75 → diffuse ratio ~0.2. Conservative. - kd := 0.2 - dhi := ghi * kd - dni := (ghi - dhi) / math.Cos(sun.ZenithDeg*math.Pi/180) - + if dhi < 0 { + dhi = 0 + } + if dhi > ghi { + dhi = ghi + } + tiltR := panelTiltDeg * math.Pi / 180 + diffusePOA := dhi * (1 + math.Cos(tiltR)) / 2 // isotropic sky dome aoi := AOI(sun, panelTiltDeg, panelAzDeg) if aoi > 90 { - // Sun behind panel — only diffuse counts. - return dhi * (1 + math.Cos(panelTiltDeg*math.Pi/180)) / 2 + // Sun behind the panel — only diffuse reaches the surface. + return diffusePOA } + cosZ := math.Cos(sun.ZenithDeg * math.Pi / 180) + if cosZ < 0.01 { + // Sun on the horizon: beam projection is ill-conditioned + // (divide-by-~0) and diffuse dominates anyway. + return diffusePOA + } + dni := (ghi - dhi) / cosZ beamPOA := dni * math.Cos(aoi*math.Pi/180) - diffusePOA := dhi * (1 + math.Cos(panelTiltDeg*math.Pi/180)) / 2 out := beamPOA + diffusePOA - if out < 0 { out = 0 } + if out < 0 { + out = 0 + } return out } + +// POAFromGHI projects a measured/forecast global horizontal irradiance (GHI, +// W/m²) onto a tilted panel when no diffuse component is available. It +// estimates the diffuse fraction from the hourly clearness index via the Erbs +// et al. (1982) correlation, then delegates to POAFromComponents. +// +// Use this for radiation providers that expose shortwave/GHI but not diffuse +// (e.g. Open-Meteo shortwave_radiation, or SMHI STRÅNG global-only windows). +func POAFromGHI(t time.Time, lat, lon, ghi, panelTiltDeg, panelAzDeg float64) float64 { + sun := At(t, lat, lon) + if sun.ZenithDeg >= 90 || ghi <= 0 { + return 0 + } + cosZ := math.Cos(sun.ZenithDeg * math.Pi / 180) + i0h := extraterrestrialHorizontalW(t, cosZ) + kt := 0.0 + if i0h > 0 { + kt = ghi / i0h + } + dhi := ghi * ErbsDiffuseFraction(kt) + return POAFromComponents(sun, ghi, dhi, panelTiltDeg, panelAzDeg) +} + +// ErbsDiffuseFraction returns the diffuse fraction (DHI/GHI) for an hourly +// clearness index kt, per Erbs, Klein & Duffie (1982). Result is in +// [0.165, 1]: overcast skies (low kt) are almost entirely diffuse, clear +// skies (high kt) settle near 16.5% diffuse. +func ErbsDiffuseFraction(kt float64) float64 { + switch { + case kt <= 0: + return 1 + case kt <= 0.22: + return 1 - 0.09*kt + case kt <= 0.80: + return 0.9511 - 0.1604*kt + 4.388*kt*kt - 16.638*kt*kt*kt + 12.336*kt*kt*kt*kt + default: + return 0.165 + } +} + +// extraterrestrialHorizontalW returns top-of-atmosphere irradiance on a +// horizontal surface (W/m²) at time t for a solar cosine-zenith cosZ. Used as +// the denominator of the clearness index. Returns 0 when the sun is at/below +// the horizon. +func extraterrestrialHorizontalW(t time.Time, cosZ float64) float64 { + if cosZ <= 0 { + return 0 + } + doy := float64(t.UTC().YearDay()) + gamma := 2 * math.Pi * (doy - 1) / 365 + e0 := 1.000110 + + 0.034221*math.Cos(gamma) + 0.001280*math.Sin(gamma) + + 0.000719*math.Cos(2*gamma) + 0.000077*math.Sin(2*gamma) + const i0 = 1361.0 // solar constant W/m² + return i0 * e0 * cosZ +} diff --git a/go/internal/sunpos/sunpos_test.go b/go/internal/sunpos/sunpos_test.go index efe085fa..fb48f8d5 100644 --- a/go/internal/sunpos/sunpos_test.go +++ b/go/internal/sunpos/sunpos_test.go @@ -66,3 +66,90 @@ func TestPOAFlatEqualsGHI(t *testing.T) { t.Errorf("flat POA should equal GHI: ghi=%.1f poa=%.1f", ghi, poa) } } + +// A flat panel receives all of GHI regardless of the diffuse split, because +// beam-on-horizontal + diffuse-on-horizontal reconstructs GHI exactly. +func TestPOAFromComponentsFlatEqualsGHI(t *testing.T) { + tt := time.Date(2026, 6, 21, 11, 0, 0, 0, time.UTC) + sun := At(tt, 59.33, 18.07) + const ghi = 700.0 + for _, dhi := range []float64{0, 140, 350, 700} { + poa := POAFromComponents(sun, ghi, dhi, 0, 180) + if math.Abs(poa-ghi) > 0.5 { + t.Errorf("flat POA should equal GHI for dhi=%.0f: got %.2f want %.0f", dhi, poa, ghi) + } + } +} + +// A south-tilted panel at Stockholm winter noon (low sun) collects more than a +// flat one — the whole point of projecting onto the plane. +func TestPOAFromComponentsSouthTiltBeatsFlatInWinter(t *testing.T) { + tt := time.Date(2026, 12, 21, 11, 0, 0, 0, time.UTC) // ~noon local, low sun + sun := At(tt, 59.33, 18.07) + if sun.ZenithDeg >= 90 { + t.Skip("sun below horizon") + } + const ghi, dhi = 200.0, 60.0 + flat := POAFromComponents(sun, ghi, dhi, 0, 180) + tilt := POAFromComponents(sun, ghi, dhi, 45, 180) // 45° south + if tilt <= flat { + t.Errorf("south-tilted winter POA (%.1f) should beat flat (%.1f)", tilt, flat) + } +} + +// Sun behind the panel → only the diffuse dome contributes (no negative beam). +func TestPOAFromComponentsSunBehindPanelIsDiffuseOnly(t *testing.T) { + tt := time.Date(2026, 6, 21, 5, 0, 0, 0, time.UTC) // morning, sun in the east + sun := At(tt, 59.33, 18.07) + if sun.ZenithDeg >= 90 { + t.Skip("sun below horizon") + } + const ghi, dhi = 300.0, 90.0 + // A steep west-facing wall can't see the eastern morning sun's beam. + poa := POAFromComponents(sun, ghi, dhi, 90, 270) + diffuseOnly := dhi * (1 + math.Cos(90*math.Pi/180)) / 2 + if math.Abs(poa-diffuseOnly) > 0.5 { + t.Errorf("sun-behind panel should be diffuse-only %.2f, got %.2f", diffuseOnly, poa) + } +} + +// Erbs diffuse fraction: overcast → ~all diffuse, clear → floor at 0.165, +// monotonically non-increasing across the mid range. +func TestErbsDiffuseFraction(t *testing.T) { + if f := ErbsDiffuseFraction(0); f != 1 { + t.Errorf("kt=0 → fraction 1, got %.3f", f) + } + if f := ErbsDiffuseFraction(1.0); math.Abs(f-0.165) > 1e-9 { + t.Errorf("kt>0.8 → 0.165, got %.3f", f) + } + clear := ErbsDiffuseFraction(0.75) + murky := ErbsDiffuseFraction(0.30) + if !(clear < murky) { + t.Errorf("clearer sky should have less diffuse: clear=%.3f murky=%.3f", clear, murky) + } + if clear < 0.165 || clear > 1 { + t.Errorf("fraction out of [0.165,1]: %.3f", clear) + } +} + +// POAFromGHI (Erbs split) on a flat panel still reconstructs ~GHI. +func TestPOAFromGHIFlatApproxGHI(t *testing.T) { + tt := time.Date(2026, 6, 21, 11, 0, 0, 0, time.UTC) + const ghi = 600.0 + poa := POAFromGHI(tt, 59.33, 18.07, ghi, 0, 180) + if math.Abs(poa-ghi) > 0.5 { + t.Errorf("flat POAFromGHI should equal GHI: got %.2f want %.0f", poa, ghi) + } +} + +// Night → zero from both measured-irradiance variants regardless of input. +func TestPOAVariantsZeroAtNight(t *testing.T) { + tt := time.Date(2026, 12, 21, 23, 0, 0, 0, time.UTC) + sun := At(tt, 59.33, 18.07) + if p := POAFromComponents(sun, 500, 100, 35, 180); p != 0 { + t.Errorf("night POAFromComponents should be 0, got %.2f", p) + } + if p := POAFromGHI(tt, 59.33, 18.07, 500, 35, 180); p != 0 { + t.Errorf("night POAFromGHI should be 0, got %.2f", p) + } +} diff --git a/roofmodel/ftw_roofmodel/__init__.py b/roofmodel/ftw_roofmodel/__init__.py new file mode 100644 index 00000000..37aaafd6 --- /dev/null +++ b/roofmodel/ftw_roofmodel/__init__.py @@ -0,0 +1,18 @@ +"""FTW roof-geometry module. + +Derives PV array geometry (tilt, azimuth, kWp) from Lantmaeteriet open geodata. +Optional and independently versioned: core reads only the versioned +`roof_model.json` this module emits, and works normally without it. +""" + +from .pipeline import SCHEMA_VERSION, RoofModelError, derive, planes_to_arrays +from .segment import RoofPlane, segment_roof + +__all__ = [ + "SCHEMA_VERSION", + "RoofModelError", + "RoofPlane", + "derive", + "planes_to_arrays", + "segment_roof", +] diff --git a/roofmodel/ftw_roofmodel/__main__.py b/roofmodel/ftw_roofmodel/__main__.py new file mode 100644 index 00000000..78385420 --- /dev/null +++ b/roofmodel/ftw_roofmodel/__main__.py @@ -0,0 +1,87 @@ +"""CLI entry point: python -m ftw_roofmodel --lat .. --lon .. + +Core invokes this as a subprocess and reads roof_model.json from stdout, the +same arm's-length pattern the optimizer uses. Errors go to stderr as JSON so +the caller can surface a reason rather than a stack trace. +""" + +from __future__ import annotations + +import argparse +import json +import sys + +from .buildings import DEFAULT_SEARCH_RADIUS_M, search_buildings +from .geotorget import Credentials, GeotorgetClient, GeotorgetError +from .pipeline import SCHEMA_VERSION, RoofModelError, derive + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(prog="ftw_roofmodel") + p.add_argument( + "--mode", + choices=("derive", "buildings"), + default="derive", + help="'buildings' lists footprints to pick from; 'derive' fits the roof", + ) + p.add_argument("--lat", type=float, required=True) + p.add_argument("--lon", type=float, required=True) + p.add_argument( + "--building-id", + default="", + help="footprint to clip the LiDAR to, from a --mode buildings run", + ) + p.add_argument("--search-radius-m", type=float, default=DEFAULT_SEARCH_RADIUS_M) + p.add_argument("--username", default="", help="Geotorget username") + p.add_argument("--token", default="", help="Geotorget token/password") + p.add_argument("--radius-m", type=float, default=40.0) + p.add_argument("--packing-factor", type=float, default=0.70) + p.add_argument("--module-w-per-m2", type=float, default=200.0) + p.add_argument( + "--vostok", + default="", + help=( + "path to a vostok binary for shadow-aware irradiance. vostok is a " + "separate GPL-3.0 tool that FTW never bundles or installs; omit this " + "and shading is simply not evaluated." + ), + ) + args = p.parse_args(argv) + credentials = Credentials(args.username, args.token) + + try: + if args.mode == "buildings": + client = GeotorgetClient(credentials) + found = search_buildings( + client, + latitude=args.lat, + longitude=args.lon, + radius_m=args.search_radius_m, + ) + payload = { + "schema_version": SCHEMA_VERSION, + "site": {"latitude": args.lat, "longitude": args.lon}, + "buildings": [b.to_geojson() for b in found], + } + else: + payload = derive( + latitude=args.lat, + longitude=args.lon, + credentials=credentials, + radius_m=args.radius_m, + packing_factor=args.packing_factor, + module_w_per_m2=args.module_w_per_m2, + vostok_binary=args.vostok or None, + building_id=args.building_id or None, + ) + except (GeotorgetError, RoofModelError) as exc: + json.dump({"error": str(exc), "kind": type(exc).__name__}, sys.stderr) + sys.stderr.write("\n") + return 1 + json.dump(payload, sys.stdout) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/roofmodel/ftw_roofmodel/buildings.py b/roofmodel/ftw_roofmodel/buildings.py new file mode 100644 index 00000000..595a64f1 --- /dev/null +++ b/roofmodel/ftw_roofmodel/buildings.py @@ -0,0 +1,350 @@ +"""Building footprints from Lantmaeteriet, and clipping LiDAR to one of them. + +Picking a building matters more than it sounds. Searching LiDAR by a radius +around a coordinate returns the neighbours' roofs, the garage and whatever trees +stand in the garden, and the segmenter has no way to know which returns belong +to the operator's own house. Worse, RANSAC fits *infinite* planes over the whole +tile: an azimuth-180 roof plane is z = f(y) with no x term, so it does not stop +at the wall, and a second building sharing the pitch and ridge orientation lies +on *the same* plane rather than a similar one. + +Measured on a synthetic pair, one RANSAC pass over a house and a garage 40 m +apart consumed all 576 of the house's south-face returns and all 256 of the +garage's as a single surface -- and produced identical output at every +separation from 3 m to 40 m, which is the signature of a global fit rather than +a neighbourhood effect. Only the DBSCAN pass afterwards told the two buildings +apart, leaving a clustering parameter as the sole thing standing between a +garage and its neighbour's roof. + +Clipping to a chosen footprint removes that whole class of error: the returns +that reach the segmenter are the ones standing on the operator's building, and +the derived face is then identical to segmenting that building in isolation. + +Coordinate frames +----------------- +GeoJSON mandates WGS84, but Lantmaeteriet publishes this catalogue in +SWEREF 99 TM (EPSG:3006) and its STAC search takes a SWEREF bbox. Rather than +guess which one a given deployment returns, the frame is detected from the +magnitude of the numbers -- SWEREF eastings and northings are six and seven +figures, WGS84 degrees never are. +""" + +from __future__ import annotations + +import dataclasses +import math +from typing import Any, Iterable + +from . import sweref +from .geopackage import GeoPackageError, read_features +from .geotorget import ( + COLLECTION_BUILDINGS, + MEDIA_GEOJSON, + MEDIA_GEOPACKAGE, + GeotorgetClient, + GeotorgetError, + StacItem, +) + +# How far around the site to look for candidate buildings. Wide enough to reach +# a house set back from its coordinate, narrow enough not to return a village. +DEFAULT_SEARCH_RADIUS_M = 150.0 + +# A footprint larger than this is a tile boundary or a whole city block, not a +# building someone is about to mount panels on. +MAX_FOOTPRINT_AREA_M2 = 20000.0 +# Below this it is a shed, a bin store or a digitising artefact. +MIN_FOOTPRINT_AREA_M2 = 8.0 + +# Roofs overhang their walls. Clipping exactly on the footprint would shave the +# eaves off every face, and the eaves are where the lowest roof returns are. +DEFAULT_EAVES_BUFFER_M = 1.0 + + +class BuildingLookupError(GeotorgetError): + """The building search succeeded but produced nothing usable.""" + + +@dataclasses.dataclass +class Building: + """One candidate building, ready to hand to a map.""" + + building_id: str + # Ring of (easting, northing) in SWEREF 99 TM -- the frame clipping happens + # in, since the LiDAR arrives in it too. + ring_sweref: list[tuple[float, float]] + area_m2: float + distance_m: float + properties: dict[str, Any] = dataclasses.field(default_factory=dict) + + def centroid_sweref(self) -> tuple[float, float]: + return _centroid(self.ring_sweref) + + def centroid_wgs84(self) -> tuple[float, float]: + e, n = self.centroid_sweref() + return sweref.sweref99tm_to_wgs84(e, n) + + def ring_wgs84(self) -> list[list[float]]: + """GeoJSON ring: [lon, lat] pairs, closed.""" + out = [] + for e, n in self.ring_sweref: + lat, lon = sweref.sweref99tm_to_wgs84(e, n) + out.append([round(lon, 7), round(lat, 7)]) + if out and out[0] != out[-1]: + out.append(out[0]) + return out + + def to_geojson(self) -> dict[str, Any]: + lat, lon = self.centroid_wgs84() + return { + "type": "Feature", + "id": self.building_id, + "geometry": {"type": "Polygon", "coordinates": [self.ring_wgs84()]}, + "properties": { + "building_id": self.building_id, + "area_m2": round(self.area_m2, 1), + "distance_m": round(self.distance_m, 1), + "latitude": round(lat, 6), + "longitude": round(lon, 6), + **self.properties, + }, + } + + +def _looks_like_sweref(ring: Iterable[Iterable[float]]) -> bool: + """True when a ring is projected metres rather than degrees. + + SWEREF 99 TM eastings run roughly 200 000-900 000 and northings 6 100 000- + 7 700 000. No WGS84 coordinate can reach either, so one look at the + magnitude settles which frame a ring is in. + """ + for point in ring: + pt = list(point) + if len(pt) < 2: + continue + if abs(pt[0]) > 180.0 or abs(pt[1]) > 90.0: + return True + return False + + +def _shoelace_area(ring: list[tuple[float, float]]) -> float: + """Planar polygon area in square metres. Ring must be projected.""" + n = len(ring) + if n < 3: + return 0.0 + total = 0.0 + for i in range(n): + x1, y1 = ring[i] + x2, y2 = ring[(i + 1) % n] + total += x1 * y2 - x2 * y1 + return abs(total) / 2.0 + + +def _centroid(ring: list[tuple[float, float]]) -> tuple[float, float]: + """Area centroid of a projected ring, falling back to the vertex mean.""" + n = len(ring) + if n == 0: + return (0.0, 0.0) + if n < 3: + return (sum(p[0] for p in ring) / n, sum(p[1] for p in ring) / n) + cx = cy = a = 0.0 + for i in range(n): + x1, y1 = ring[i] + x2, y2 = ring[(i + 1) % n] + cross = x1 * y2 - x2 * y1 + a += cross + cx += (x1 + x2) * cross + cy += (y1 + y2) * cross + if abs(a) < 1e-9: # degenerate (collinear) ring + return (sum(p[0] for p in ring) / n, sum(p[1] for p in ring) / n) + a *= 0.5 + return (cx / (6.0 * a), cy / (6.0 * a)) + + +def _rings_from_geometry(geometry: dict[str, Any]) -> list[list[tuple[float, float]]]: + """Outer rings of a GeoJSON Polygon or MultiPolygon, in their own frame.""" + if not isinstance(geometry, dict): + return [] + kind = geometry.get("type") + coords = geometry.get("coordinates") or [] + rings: list[list[tuple[float, float]]] = [] + if kind == "Polygon" and coords: + rings.append([(float(p[0]), float(p[1])) for p in coords[0] if len(p) >= 2]) + elif kind == "MultiPolygon": + for poly in coords: + if poly: + rings.append([(float(p[0]), float(p[1])) for p in poly[0] if len(p) >= 2]) + return [r for r in rings if len(r) >= 3] + + +def _to_sweref(ring: list[tuple[float, float]]) -> list[tuple[float, float]]: + if _looks_like_sweref(ring): + return ring + # GeoJSON order is [lon, lat]. + return [sweref.wgs84_to_sweref99tm(lat, lon) for lon, lat in ring] + + +def _features_from_item(item: StacItem, client: GeotorgetClient | None = None) -> list[dict[str, Any]]: + """Every building-like feature an item carries. + + A STAC item may *be* the building -- geometry inline, no download -- or it + may be a tile whose asset holds thousands of them. Lantmaeteriet publishes + *Byggnad Nedladdning, vektor* as **GeoPackage**, so the asset path is the + normal one and the inline path is the exception. + """ + geom = (item.raw or {}).get("geometry") + if geom: + return [{"geometry": geom, "properties": (item.raw or {}).get("properties") or {}, + "id": item.item_id}] + if client is None: + return [] + asset = item.pick(MEDIA_GEOPACKAGE, MEDIA_GEOJSON) + if asset is None or not asset.href: + return [] + media = asset.effective_media_type + payload = client.download(asset.href) + if media == MEDIA_GEOJSON: + return _features_from_geojson(payload) + try: + return read_features(payload) + except GeoPackageError as exc: + raise BuildingLookupError( + f"the building tile for this site could not be read: {exc}" + ) from exc + + +def _features_from_geojson(payload: bytes) -> list[dict[str, Any]]: + """A GeoJSON FeatureCollection asset, for catalogues that publish one.""" + import json + + try: + doc = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, ValueError) as exc: + raise BuildingLookupError( + f"the building tile was announced as GeoJSON but did not parse: {exc}" + ) from exc + if isinstance(doc, dict) and doc.get("type") == "FeatureCollection": + return list(doc.get("features") or []) + if isinstance(doc, dict) and doc.get("type") == "Feature": + return [doc] + return [] + + +def buildings_from_features( + features: Iterable[dict[str, Any]], + *, + latitude: float, + longitude: float, + fallback_id: str = "building", +) -> list[Building]: + """Turn GeoJSON-ish features into ranked Building candidates.""" + site_e, site_n = sweref.wgs84_to_sweref99tm(latitude, longitude) + out: list[Building] = [] + for i, feat in enumerate(features): + for j, ring in enumerate(_rings_from_geometry(feat.get("geometry") or {})): + ring_sweref = _to_sweref(ring) + area = _shoelace_area(ring_sweref) + if area < MIN_FOOTPRINT_AREA_M2 or area > MAX_FOOTPRINT_AREA_M2: + continue + cx, cy = _centroid(ring_sweref) + props = dict(feat.get("properties") or {}) + bid = str(feat.get("id") or props.get("objektidentitet") or f"{fallback_id}-{i}") + if j: + bid = f"{bid}-{j}" + out.append(Building( + building_id=bid, + ring_sweref=ring_sweref, + area_m2=area, + distance_m=math.hypot(cx - site_e, cy - site_n), + properties={k: v for k, v in props.items() if isinstance(v, (str, int, float))}, + )) + out.sort(key=lambda b: b.distance_m) + return out + + +def search_buildings( + client: GeotorgetClient, + *, + latitude: float, + longitude: float, + radius_m: float = DEFAULT_SEARCH_RADIUS_M, + limit: int = 50, +) -> list[Building]: + """Building footprints near a site, nearest first.""" + south, west, north, east = sweref.metre_box_around(latitude, longitude, radius_m) + bbox = sweref.bbox_wgs84_to_sweref99tm(south, west, north, east) + items = client.search(COLLECTION_BUILDINGS, bbox, limit=limit) + features: list[dict[str, Any]] = [] + for item in items: + features.extend(_features_from_item(item, client)) + if not features: + raise BuildingLookupError( + "no building footprints were returned for this site. The Geotorget " + "account needs access to 'Byggnad Nedladdning, vektor', and the data " + "covers Sweden only." + ) + return buildings_from_features( + features, latitude=latitude, longitude=longitude + ) + + +def point_in_ring(x: float, y: float, ring: list[tuple[float, float]]) -> bool: + """Ray-casting point-in-polygon, in the ring's own projected frame.""" + inside = False + n = len(ring) + for i in range(n): + x1, y1 = ring[i] + x2, y2 = ring[(i + 1) % n] + if (y1 > y) != (y2 > y): + if y2 != y1 and x < x1 + (y - y1) * (x2 - x1) / (y2 - y1): + inside = not inside + return inside + + +def inflate_ring(ring: list[tuple[float, float]], metres: float) -> list[tuple[float, float]]: + """Push a ring outward from its centroid by roughly `metres`. + + Approximate on purpose: a true polygon offset needs a geometry library, and + this only has to catch the eaves. For the compact, roughly convex outlines + houses actually have, scaling about the centroid is within a few centimetres + of a proper offset; for a long thin wing it over-buffers the short sides, + which costs a little neighbouring ground rather than losing roof. + """ + if metres <= 0 or len(ring) < 3: + return ring + cx, cy = _centroid(ring) + out = [] + for x, y in ring: + dx, dy = x - cx, y - cy + d = math.hypot(dx, dy) + if d < 1e-9: + out.append((x, y)) + continue + scale = (d + metres) / d + out.append((cx + dx * scale, cy + dy * scale)) + return out + + +def clip_to_footprint(points, ring: list[tuple[float, float]], + *, buffer_m: float = DEFAULT_EAVES_BUFFER_M): + """Keep only the returns standing on one building. + + `points` is (N, 3) in SWEREF 99 TM metres, the frame the LiDAR arrives in. + """ + import numpy as np + + pts = np.asarray(points, dtype=float) + if len(pts) == 0 or len(ring) < 3: + return pts + outline = inflate_ring(ring, buffer_m) + + # Cheap rejection first: most of a tile is nowhere near the building. + xs = [p[0] for p in outline] + ys = [p[1] for p in outline] + box = ( + (pts[:, 0] >= min(xs)) & (pts[:, 0] <= max(xs)) + & (pts[:, 1] >= min(ys)) & (pts[:, 1] <= max(ys)) + ) + candidates = np.nonzero(box)[0] + keep = [i for i in candidates if point_in_ring(pts[i, 0], pts[i, 1], outline)] + return pts[keep] diff --git a/roofmodel/ftw_roofmodel/geopackage.py b/roofmodel/ftw_roofmodel/geopackage.py new file mode 100644 index 00000000..9a7ad342 --- /dev/null +++ b/roofmodel/ftw_roofmodel/geopackage.py @@ -0,0 +1,228 @@ +"""Read polygon features out of a GeoPackage, using only the standard library. + +Lantmaeteriet publishes *Byggnad Nedladdning, vektor* as GeoPackage, so a STAC +item for a building tile carries a `.gpkg` asset rather than inline GeoJSON. + +A GeoPackage is a SQLite database with an agreed set of metadata tables, and +geometry stored as a small binary header followed by standard WKB. Both are +published specifications with fixed layouts, so decoding them here costs about +a hundred lines of `sqlite3` and `struct` -- against a GDAL/fiona/geopandas +dependency that would not install on a Pi without a compiler and pulls in a +second projection stack we already decided not to carry (see sweref.py, which +implements SWEREF 99 TM directly for the same reason). + +Only what a roof model needs is read: polygon and multipolygon rings, in the +file's own coordinate reference system. Curves, triangulated surfaces and the +extended (`GPB` "extended geometry") binary types are rejected explicitly rather +than mis-parsed, because a wrong ring silently clips the wrong LiDAR. + +References +---------- +GeoPackage Encoding Standard (OGC 12-128r19), clause 2.1.3 "BLOB Format". +OpenGIS Simple Features (OGC 06-103r4), clause 8.2 "Well-known Binary". +""" + +from __future__ import annotations + +import os +import sqlite3 +import struct +import tempfile +from typing import Any, Iterator + +# "GP" -- the two magic bytes every GeoPackage geometry blob starts with. +GPKG_MAGIC = b"GP" + +# Envelope sizes in doubles, indexed by the header's envelope indicator. +# 0 = absent, 1 = xy, 2 = xyz, 3 = xym, 4 = xyzm. 5-7 are reserved. +_ENVELOPE_DOUBLES = {0: 0, 1: 4, 2: 6, 3: 6, 4: 8} + +# WKB geometry type codes we can use. The ISO variants add 1000 for Z, 2000 for +# M and 3000 for ZM, so the base code is recovered with % 1000. +_WKB_POLYGON = 3 +_WKB_MULTIPOLYGON = 6 + +# The EWKB flag bits PostGIS adds to the type word. GeoPackage forbids them, but +# files written by other tools do turn up, and reading a flagged type as a +# geometry code would silently produce nonsense. +_EWKB_Z = 0x80000000 +_EWKB_M = 0x40000000 +_EWKB_SRID = 0x20000000 + + +class GeoPackageError(ValueError): + """The file is not a GeoPackage, or holds geometry we will not guess at.""" + + +def _unpack(fmt: str, data: bytes, offset: int) -> tuple[Any, ...]: + size = struct.calcsize(fmt) + if offset + size > len(data): + raise GeoPackageError("geometry blob ended mid-value") + return struct.unpack_from(fmt, data, offset) + + +def _dimensions(type_word: int) -> tuple[int, int]: + """(base geometry code, coordinates per point) for a WKB type word.""" + has_z = bool(type_word & _EWKB_Z) + has_m = bool(type_word & _EWKB_M) + code = type_word & ~(_EWKB_Z | _EWKB_M | _EWKB_SRID) + # ISO style: 1000/2000/3000 offsets carry the same information. + if code >= 3000: + code, has_z, has_m = code - 3000, True, True + elif code >= 2000: + code, has_m = code - 2000, True + elif code >= 1000: + code, has_z = code - 1000, True + return code, 2 + int(has_z) + int(has_m) + + +def _read_ring(data: bytes, offset: int, endian: str, coords: int) -> tuple[list[tuple[float, float]], int]: + (count,) = _unpack(endian + "I", data, offset) + offset += 4 + stride = 8 * coords + ring: list[tuple[float, float]] = [] + for _ in range(count): + x, y = _unpack(endian + "dd", data, offset) + ring.append((x, y)) + offset += stride + return ring, offset + + +def _read_polygon(data: bytes, offset: int, endian: str, coords: int) -> tuple[list[list[tuple[float, float]]], int]: + (n_rings,) = _unpack(endian + "I", data, offset) + offset += 4 + rings = [] + for _ in range(n_rings): + ring, offset = _read_ring(data, offset, endian, coords) + rings.append(ring) + return rings, offset + + +def _read_geometry(data: bytes, offset: int) -> tuple[list[list[list[tuple[float, float]]]], int]: + """Read one WKB geometry, returning it as a list of polygons.""" + (byte_order,) = _unpack("B", data, offset) + endian = "<" if byte_order == 1 else ">" + (type_word,) = _unpack(endian + "I", data, offset + 1) + offset += 5 + if type_word & _EWKB_SRID: + offset += 4 # embedded SRID, which we take from the header instead + code, coords = _dimensions(type_word) + if code == _WKB_POLYGON: + rings, offset = _read_polygon(data, offset, endian, coords) + return [rings], offset + if code == _WKB_MULTIPOLYGON: + (n,) = _unpack(endian + "I", data, offset) + offset += 4 + polys = [] + for _ in range(n): + # Each part carries its own byte order and type word. + part, offset = _read_geometry(data, offset) + polys.extend(part) + return polys, offset + raise GeoPackageError( + f"WKB geometry type {code} is not a polygon; a building footprint must be " + "a Polygon or MultiPolygon" + ) + + +def parse_geometry_blob(blob: bytes) -> dict[str, Any] | None: + """Decode a GeoPackage geometry BLOB into a GeoJSON-shaped geometry. + + Returns None for the empty geometry, which GeoPackage represents with a flag + rather than an absent row. + """ + if len(blob) < 8 or blob[:2] != GPKG_MAGIC: + raise GeoPackageError("not a GeoPackage geometry blob (bad magic)") + flags = blob[3] + if flags & 0x20: + raise GeoPackageError( + "extended (ExtendedGeoPackageBinary) geometry is not supported" + ) + envelope_indicator = (flags >> 1) & 0x07 + if envelope_indicator not in _ENVELOPE_DOUBLES: + raise GeoPackageError(f"reserved envelope indicator {envelope_indicator}") + if flags & 0x10: # empty geometry + return None + offset = 8 + 8 * _ENVELOPE_DOUBLES[envelope_indicator] + polygons, _ = _read_geometry(blob, offset) + if not polygons: + return None + if len(polygons) == 1: + return {"type": "Polygon", "coordinates": [[list(p) for p in r] for r in polygons[0]]} + return { + "type": "MultiPolygon", + "coordinates": [[[list(p) for p in r] for r in poly] for poly in polygons], + } + + +def _feature_tables(conn: sqlite3.Connection) -> list[tuple[str, str]]: + """(table, geometry column) for every feature table in the file.""" + try: + rows = conn.execute( + "SELECT c.table_name, g.column_name FROM gpkg_contents c " + "JOIN gpkg_geometry_columns g ON g.table_name = c.table_name " + "WHERE c.data_type = 'features'" + ).fetchall() + except sqlite3.DatabaseError as exc: + raise GeoPackageError(f"not a readable GeoPackage: {exc}") from exc + return [(str(t), str(c)) for t, c in rows] + + +def read_features(data: bytes, *, limit: int = 5000) -> list[dict[str, Any]]: + """Every polygon feature in a GeoPackage, as GeoJSON-shaped dicts. + + Attributes travel alongside the geometry so the picker can label a building + with whatever the source calls it. + """ + return list(iter_features(data, limit=limit)) + + +def iter_features(data: bytes, *, limit: int = 5000) -> Iterator[dict[str, Any]]: + if not data.startswith(b"SQLite format 3\x00"): + raise GeoPackageError( + "asset is not a GeoPackage (missing the SQLite file header)" + ) + # sqlite3 opens paths, not buffers, and a GeoPackage is random-access by + # design, so the bytes land in a temp file for the life of the read. + fd, path = tempfile.mkstemp(suffix=".gpkg") + try: + with os.fdopen(fd, "wb") as fh: + fh.write(data) + conn = sqlite3.connect(path) + try: + conn.row_factory = sqlite3.Row + yielded = 0 + for table, geom_col in _feature_tables(conn): + cursor = conn.execute(f'SELECT * FROM "{table}"') + for row in cursor: + if yielded >= limit: + return + blob = row[geom_col] + if not isinstance(blob, (bytes, bytearray)): + continue + try: + geometry = parse_geometry_blob(bytes(blob)) + except GeoPackageError: + # One unreadable row must not lose the other buildings + # in the tile. + continue + if geometry is None: + continue + props = { + k: row[k] + for k in row.keys() + if k != geom_col and isinstance(row[k], (str, int, float)) + } + yield { + "id": str(props.get("objektidentitet") or f"{table}-{yielded}"), + "geometry": geometry, + "properties": props, + } + yielded += 1 + finally: + conn.close() + finally: + try: + os.unlink(path) + except OSError: # pragma: no cover - Windows may hold the handle briefly + pass diff --git a/roofmodel/ftw_roofmodel/geotorget.py b/roofmodel/ftw_roofmodel/geotorget.py new file mode 100644 index 00000000..72553ea0 --- /dev/null +++ b/roofmodel/ftw_roofmodel/geotorget.py @@ -0,0 +1,278 @@ +"""Lantmaeteriet Geotorget access: authentication and STAC search. + +Two products are used, both free open data (CC BY 4.0) but both gated behind a +Geotorget account the operator orders themselves: + + * *Byggnad Nedladdning, vektor* -- building footprint polygons. + * *Laserdata Nedladdning, Skog* -- airborne LiDAR, 1-2 points/m2, from 2018. + +Credentials are the operator's own and are never shipped, logged or echoed back +through the API. FTW stores them the same way it stores `weather.api_key`, and +redacts them in config responses. + +Only `requests` is used. The STAC API is plain JSON over HTTP, so pulling in +pystac-client would add a dependency for a search body we can write in six +lines -- and a thinner surface is easier to keep working when Lantmaeteriet +moves an endpoint. +""" + +from __future__ import annotations + +import dataclasses +import datetime as dt +from typing import Any, Iterable + +DEFAULT_BASE_URL = "https://api.lantmateriet.se" + +# Collection ids as published in Lantmaeteriet's STAC catalogue. Both products +# are STAC APIs over the same base URL and the same credentials; they differ +# only in what their items point at, which is what the media types below say. +COLLECTION_BUILDINGS = "byggnad-nedladdning-vektor" +COLLECTION_LIDAR = "laserdata-nedladdning-skog" + +# Media types, so an asset is chosen by *what it is* rather than by hoping the +# publisher named the key "data". Byggnad-vektor delivers GeoPackage; Laserdata +# Skog delivers LAZ organised as COPC (Cloud Optimized Point Cloud). +MEDIA_GEOPACKAGE = "application/geopackage+sqlite3" +MEDIA_COPC = "application/vnd.laszip+copc" +MEDIA_LAZ = "application/vnd.laszip" +MEDIA_LAS = "application/vnd.las" +MEDIA_GEOJSON = "application/geo+json" + +# Longest suffix first: a COPC file is also a .laz, and reading it as a plain +# one would download the whole tile instead of the part we asked for. +_EXTENSION_MEDIA: tuple[tuple[str, str], ...] = ( + (".copc.laz", MEDIA_COPC), + (".gpkg", MEDIA_GEOPACKAGE), + (".geojson", MEDIA_GEOJSON), + (".laz", MEDIA_LAZ), + (".las", MEDIA_LAS), +) + + +class GeotorgetError(RuntimeError): + """Any failure talking to Geotorget.""" + + +class MissingCredentials(GeotorgetError): + """No usable credentials were supplied.""" + + +@dataclasses.dataclass(frozen=True) +class Credentials: + username: str + password: str + + def validate(self) -> None: + if not self.username or not self.password: + raise MissingCredentials( + "Geotorget username and token are both required; order access at " + "https://geotorget.lantmateriet.se and set roofmodel.geotorget_username " + "and roofmodel.geotorget_token" + ) + + +def media_type_for(href: str) -> str | None: + """Media type implied by a URL's extension, or None if it says nothing.""" + path = href.split("?", 1)[0].split("#", 1)[0].lower() + for suffix, media in _EXTENSION_MEDIA: + if path.endswith(suffix): + return media + return None + + +@dataclasses.dataclass(frozen=True) +class Asset: + """One STAC asset: where it is, and what it is.""" + + href: str + media_type: str | None = None + roles: tuple[str, ...] = () + title: str = "" + + @property + def effective_media_type(self) -> str | None: + """Declared media type, or the one the extension implies. + + Catalogues are inconsistent about `type`, and an asset with no declared + type is common enough that refusing to guess would mean refusing most + real items. The extension is only consulted when nothing was declared. + """ + return self.media_type or media_type_for(self.href) + + +@dataclasses.dataclass +class StacItem: + """One STAC item, reduced to what the pipeline needs.""" + + item_id: str + collection: str + assets: dict[str, Asset] + captured_at: dt.datetime | None + raw: dict[str, Any] = dataclasses.field(default_factory=dict, repr=False) + + def __post_init__(self) -> None: + # A bare href is accepted wherever an Asset is, so callers and tests can + # write {"data": "http://.../tile.copc.laz"} without losing the typing + # that selection depends on -- the extension supplies it. + self.assets = { + name: value if isinstance(value, Asset) else Asset(href=str(value)) + for name, value in (self.assets or {}).items() + } + + def pick(self, *media_types: str) -> Asset | None: + """Best asset for a wanted media type, most preferred type first. + + Where nothing declares a usable type the search widens: an asset with + the `data` role, then a lone asset, since an item carrying exactly one + asset is unambiguous however it is labelled. + + Both fallbacks consider only assets of *unknown* type. Guessing in the + absence of information is reasonable; guessing against it is not, and an + item whose single asset is a thumbnail must not be handed back as a + point cloud. + """ + for wanted in media_types: + for asset in self.assets.values(): + if asset.effective_media_type == wanted: + return asset + untyped = [a for a in self.assets.values() if a.effective_media_type is None] + for asset in untyped: + if "data" in asset.roles: + return asset + if len(untyped) == 1: + return untyped[0] + return None + + def asset_url(self, *preferred: str) -> str | None: + """First matching asset href, trying each preferred key in order.""" + for key in preferred: + if key in self.assets: + return self.assets[key].href + first = next(iter(self.assets.values()), None) + return first.href if first else None + + +def _parse_datetime(value: str | None) -> dt.datetime | None: + """Parse a STAC RFC 3339 timestamp. + + Lantmaeteriet is backfilling `properties.datetime` across 2026, so it is + routinely absent. That is a missing provenance date, not an error -- the UI + degrades to "capture date unknown" rather than refusing the model. + """ + if not value: + return None + try: + return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +def _item_from_feature(feature: dict[str, Any]) -> StacItem: + props = feature.get("properties") or {} + assets = { + name: Asset( + href=asset.get("href", ""), + media_type=asset.get("type") or None, + roles=tuple(asset.get("roles") or ()), + title=str(asset.get("title") or ""), + ) + for name, asset in (feature.get("assets") or {}).items() + if asset.get("href") + } + captured = _parse_datetime(props.get("datetime")) or _parse_datetime( + props.get("start_datetime") + ) + if captured is None: + # Laser strips carry an acquisition date as `datum` (e.g. "20180301") + # even where the STAC datetime has not been backfilled yet. + datum = props.get("datum") + if datum: + try: + captured = dt.datetime.strptime(str(datum), "%Y%m%d").replace( + tzinfo=dt.timezone.utc + ) + except ValueError: + captured = None + return StacItem( + item_id=feature.get("id", ""), + collection=feature.get("collection", ""), + assets=assets, + captured_at=captured, + raw=feature, + ) + + +class GeotorgetClient: + """Thin STAC client for Lantmaeteriet's download APIs.""" + + def __init__( + self, + credentials: Credentials, + session: Any = None, + base_url: str = DEFAULT_BASE_URL, + timeout: float = 60.0, + ) -> None: + credentials.validate() + self._credentials = credentials + self._base_url = base_url.rstrip("/") + self._timeout = timeout + if session is None: + import requests # imported lazily so tests can inject a fake session + + session = requests.Session() + session.auth = (credentials.username, credentials.password) + self._session = session + + @property + def session(self) -> Any: + """The authenticated session, for readers that stream their own ranges.""" + return self._session + + def search( + self, + collection: str, + bbox_sweref: tuple[float, float, float, float], + limit: int = 20, + ) -> list[StacItem]: + """POST /stac/search for one collection over a SWEREF 99 TM bbox. + + bbox is (min_easting, min_northing, max_easting, max_northing); the + catalogue is published in EPSG:3006, so no reprojection happens here. + """ + body = { + "collections": [collection], + "bbox": list(bbox_sweref), + "limit": limit, + } + url = f"{self._base_url}/stac/search" + try: + resp = self._session.post(url, json=body, timeout=self._timeout) + except Exception as exc: # network, DNS, TLS + raise GeotorgetError(f"STAC search failed: {exc}") from exc + if resp.status_code in (401, 403): + raise MissingCredentials( + f"Geotorget rejected the credentials for {collection} " + f"(HTTP {resp.status_code}). Check the account has ordered access " + "to this product." + ) + if resp.status_code != 200: + raise GeotorgetError(f"STAC search returned HTTP {resp.status_code}") + payload = resp.json() + return [_item_from_feature(f) for f in payload.get("features", [])] + + def download(self, url: str) -> bytes: + """Fetch one asset.""" + try: + resp = self._session.get(url, timeout=self._timeout) + except Exception as exc: + raise GeotorgetError(f"asset download failed: {exc}") from exc + if resp.status_code != 200: + raise GeotorgetError(f"asset download returned HTTP {resp.status_code}") + return resp.content + + +def newest_capture(items: Iterable[StacItem]) -> dt.datetime | None: + """Most recent known capture date across items, or None if none carry one.""" + dates = [i.captured_at for i in items if i.captured_at is not None] + return max(dates) if dates else None diff --git a/roofmodel/ftw_roofmodel/pipeline.py b/roofmodel/ftw_roofmodel/pipeline.py new file mode 100644 index 00000000..88b8fc08 --- /dev/null +++ b/roofmodel/ftw_roofmodel/pipeline.py @@ -0,0 +1,300 @@ +"""Derive roof geometry for a site: STAC search -> LiDAR -> planes -> arrays. + +The output contract is a versioned `roof_model.json`, which is the whole reason +this lives in a separate module rather than inside core: core only ever reads +that document, so the heavy geospatial dependencies, their failure modes and +their update cadence stay on this side of the boundary. + +Nothing here is authoritative. The derived arrays *pre-fill* the operator's +editable `weather.pv_arrays`; they are hints, and the numeric editor stays the +final word. A failure produces a clean error and leaves the existing config +untouched. +""" + +from __future__ import annotations + +import dataclasses +import datetime as dt +from typing import Any + +from . import geotorget, pointcloud, sweref +from .buildings import ( + DEFAULT_EAVES_BUFFER_M, + Building, + clip_to_footprint, + search_buildings, +) +from .geotorget import ( + COLLECTION_LIDAR, + Credentials, + GeotorgetClient, + GeotorgetError, + StacItem, + newest_capture, +) +from .segment import ( + DEFAULT_MODULE_W_PER_M2, + DEFAULT_PACKING_FACTOR, + RoofPlane, + segment_roof, +) + +SCHEMA_VERSION = 1 + +# How far around the site to pull LiDAR. 40 m comfortably contains a detached +# house and its outbuildings without dragging in the neighbours' roofs. +DEFAULT_RADIUS_M = 40.0 + +# Roof faces smaller than this are dormers, porches and sheds: real surfaces, +# but not worth proposing as a PV array. +MIN_ARRAY_AREA_M2 = 8.0 + +# Lantmaeteriet's Laserdata Skog is specified at 1-2 points/m2. +NOMINAL_POINT_DENSITY = 1.5 + +# Below this, a clipped footprint cannot support a plane fit -- segment_roof +# needs 40 points for a single face, and a roof has at least two. +MIN_POINTS_AFTER_CLIP = 80 + + +class RoofModelError(RuntimeError): + """Derivation failed.""" + + +@dataclasses.dataclass +class DerivedArray: + name: str + kwp: float + tilt_deg: float + azimuth_deg: float + area_m2: float + segment_id: str + # Fraction of open-sky irradiation this face actually receives once + # neighbouring geometry is accounted for. None when shading was not + # evaluated, which is different from 1.0 ("evaluated, unobstructed"). + shading_factor: float | None = None + + def to_json(self) -> dict[str, Any]: + out = { + "name": self.name, + "kwp": round(self.kwp, 2), + "tilt_deg": round(self.tilt_deg, 1), + "azimuth_deg": round(self.azimuth_deg, 1), + "area_m2": round(self.area_m2, 1), + "segment_id": self.segment_id, + } + if self.shading_factor is not None: + out["shading_factor"] = round(self.shading_factor, 3) + return out + + +def _compass_name(azimuth_deg: float, tilt_deg: float) -> str: + """Human-readable face name, e.g. "Roof south".""" + if tilt_deg < 5.0: + return "Roof flat" + points = [ + (0, "north"), (45, "north-east"), (90, "east"), (135, "south-east"), + (180, "south"), (225, "south-west"), (270, "west"), (315, "north-west"), + (360, "north"), + ] + best = min(points, key=lambda p: abs(p[0] - azimuth_deg)) + return f"Roof {best[1]}" + + +def planes_to_arrays( + planes: list[RoofPlane], + *, + packing_factor: float = DEFAULT_PACKING_FACTOR, + module_w_per_m2: float = DEFAULT_MODULE_W_PER_M2, + min_area_m2: float = MIN_ARRAY_AREA_M2, +) -> list[DerivedArray]: + """Convert roof planes into candidate PV arrays. + + North-facing pitched roofs are dropped: at Swedish latitudes a north face + at any real pitch yields so little that proposing it as an array would be + noise in the operator's config. Flat roofs are kept -- they are mounted to + face south regardless of which way the building points. + """ + arrays: list[DerivedArray] = [] + used: dict[str, int] = {} + for idx, plane in enumerate(planes): + if plane.area_m2 < min_area_m2: + continue + if plane.tilt_deg >= 5.0 and (plane.azimuth_deg <= 45.0 or plane.azimuth_deg >= 315.0): + continue + name = _compass_name(plane.azimuth_deg, plane.tilt_deg) + used[name] = used.get(name, 0) + 1 + if used[name] > 1: + name = f"{name} {used[name]}" + arrays.append( + DerivedArray( + name=name, + kwp=plane.kwp(packing_factor, module_w_per_m2), + tilt_deg=plane.tilt_deg, + azimuth_deg=plane.azimuth_deg, + area_m2=plane.area_m2, + segment_id=f"seg-{idx}", + ) + ) + return arrays + + +def load_points(data: bytes) -> Any: + """Decode a whole LAZ/LAS payload into (N, 3) SWEREF 99 TM metres. + + Re-exported from pointcloud so that laspy stays lazily imported: everything + above -- projection, segmentation, array derivation -- is importable and + testable without the geospatial stack installed. + """ + return pointcloud.load_points(data) + + +def _read_lidar( + client: GeotorgetClient, + items: list[StacItem], + chosen: Building | None, +) -> tuple[Any, str]: + """Points for the first readable LiDAR asset, and how they were fetched. + + Laserdata Skog is LAZ organised as COPC, so when the operator has already + picked a building there is no reason to move the rest of a 2.5 km tile + across the network: the footprint's bounding box is exactly the query COPC + is built to answer. Everything about that is best-effort -- a plain LAZ + asset, a host that ignores `Range`, or a laspy without COPC support all + fall back to reading the tile whole. + """ + for item in items: + asset = item.pick( + geotorget.MEDIA_COPC, geotorget.MEDIA_LAZ, geotorget.MEDIA_LAS + ) + if asset is None or not asset.href: + continue + if chosen is not None and asset.effective_media_type == geotorget.MEDIA_COPC: + session = getattr(client, "session", None) + if session is not None: + bounds = pointcloud.bounds_of(chosen.ring_sweref, DEFAULT_EAVES_BUFFER_M) + try: + return pointcloud.read_copc_window(session, asset.href, bounds), "copc-window" + except pointcloud.PointCloudError: + # A slow success beats a failure; the operator gets their + # roof either way, and `fetch` records which path ran. + pass + return load_points(client.download(asset.href)), "whole-tile" + raise RoofModelError("LiDAR tiles carried no readable point data") + + +def derive( + *, + latitude: float, + longitude: float, + credentials: Credentials, + client: GeotorgetClient | None = None, + radius_m: float = DEFAULT_RADIUS_M, + packing_factor: float = DEFAULT_PACKING_FACTOR, + module_w_per_m2: float = DEFAULT_MODULE_W_PER_M2, + vostok_binary: str | None = None, + building_id: str | None = None, + now: dt.datetime | None = None, +) -> dict[str, Any]: + """Derive a roof model for one site and return it as a JSON-ready dict. + + Pass `building_id` -- one the operator picked from `search_buildings` -- to + clip the LiDAR to that footprint before segmenting. Without it the whole + radius is segmented, which will happily return the neighbour's roof and lets + coplanar buildings steal each other's points; see buildings.py. + """ + if client is None: + client = GeotorgetClient(credentials) + + chosen: Building | None = None + if building_id: + candidates = search_buildings( + client, latitude=latitude, longitude=longitude, radius_m=radius_m + ) + chosen = next((b for b in candidates if b.building_id == building_id), None) + if chosen is None: + raise RoofModelError( + f"building {building_id!r} was not found near this site; it may " + "have been picked against a different coordinate" + ) + + south, west, north, east = sweref.metre_box_around(latitude, longitude, radius_m) + bbox = sweref.bbox_wgs84_to_sweref99tm(south, west, north, east) + + try: + lidar_items: list[StacItem] = client.search(COLLECTION_LIDAR, bbox) + except GeotorgetError: + raise + if not lidar_items: + raise RoofModelError( + f"no LiDAR tiles cover ({latitude:.5f}, {longitude:.5f}); " + "Lantmaeteriet data is Sweden only" + ) + + points, fetch = _read_lidar(client, lidar_items, chosen) + if points is None or len(points) == 0: + raise RoofModelError("LiDAR tiles carried no readable point data") + + total_returns = len(points) + if chosen is not None: + points = clip_to_footprint(points, chosen.ring_sweref) + if len(points) < MIN_POINTS_AFTER_CLIP: + raise RoofModelError( + f"only {len(points)} LiDAR returns fall on building " + f"{chosen.building_id!r}. The footprint and the point cloud may " + "be from different years, or the building is newer than the scan." + ) + + planes = segment_roof(points, point_density=NOMINAL_POINT_DENSITY) + arrays = planes_to_arrays( + planes, packing_factor=packing_factor, module_w_per_m2=module_w_per_m2 + ) + + # Optional shadow-aware pass. vostok is GPL-3.0 and separately installed by + # the operator; absent, every face keeps shading_factor None, which reads as + # "not evaluated" rather than "unobstructed". + shading_info: dict[str, Any] = {"evaluated": False, "reason": "not requested"} + if vostok_binary: + from .shading import compute_shading + + result = compute_shading( + points, planes, latitude=latitude, longitude=longitude, + binary=vostok_binary, + ) + shading_info = {"evaluated": result.evaluated, "reason": result.reason} + if result.evaluated: + # planes_to_arrays drops faces, so map back through segment_id + # rather than assuming the two lists line up. + for arr in arrays: + idx = int(arr.segment_id.split("-")[-1]) + if idx in result.factors: + arr.shading_factor = result.factors[idx] + + captured = newest_capture(lidar_items) + stamp = now or dt.datetime.now(dt.timezone.utc) + return { + "schema_version": SCHEMA_VERSION, + "site": {"latitude": latitude, "longitude": longitude, "radius_m": radius_m}, + "source": { + "provider": "lantmateriet", + "collection": COLLECTION_LIDAR, + "item_count": len(lidar_items), + "dataset_datetime": captured.isoformat() if captured else None, + # "copc-window" means only the footprint's neighbourhood was moved + # across the network, which also makes returns_in_radius below a + # count over that window rather than over the whole search radius. + "fetch": fetch, + }, + "building": { + "building_id": chosen.building_id, + "area_m2": round(chosen.area_m2, 1), + "footprint": chosen.to_geojson()["geometry"], + "returns_used": len(points), + "returns_in_radius": total_returns, + } if chosen is not None else None, + "arrays": [a.to_json() for a in arrays], + "planes_found": len(planes), + "shading": shading_info, + "captured_at_ms": int(captured.timestamp() * 1000) if captured else None, + "derived_at_ms": int(stamp.timestamp() * 1000), + } diff --git a/roofmodel/ftw_roofmodel/pointcloud.py b/roofmodel/ftw_roofmodel/pointcloud.py new file mode 100644 index 00000000..622c9efc --- /dev/null +++ b/roofmodel/ftw_roofmodel/pointcloud.py @@ -0,0 +1,226 @@ +"""Read Lantmaeteriet LiDAR, fetching only the part of the tile we need. + +*Laserdata Nedladdning, Skog* is delivered as LAZ organised as **COPC** (Cloud +Optimized Point Cloud): the points are ordered into an octree and the node index +lives in a VLR at a known offset, so a reader that can issue HTTP range requests +can pull the handful of octree nodes covering one building instead of the whole +2.5 km tile. + +That is worth real money on a Pi. A Laserdata Skog tile is hundreds of megabytes; +a detached house is a few tens of metres across. Since the operator has already +told us *which building* they mean, the bounding box of that footprint is exactly +the query COPC exists to answer. + +The fallbacks are deliberate and ordered, because none of the preconditions are +guaranteed: + + 1. COPC asset + a bounding box + a server that honours `Range` -> spatial query. + 2. Anything else -> download the asset and read it whole. + +A server that ignores `Range` returns 200 with the entire body, which would +otherwise be mistaken for a successful partial read, so that case is detected on +the status code rather than assumed away. +""" + +from __future__ import annotations + +import io +from typing import Any + +__all__ = [ + "PointCloudError", + "HttpRangeFile", + "bounds_of", + "load_points", + "read_copc_window", +] + +# Read this much per range request. COPC chunks are small, and a request per +# chunk would spend more time in round trips than in transfer. +DEFAULT_CHUNK_BYTES = 1 << 20 + + +class PointCloudError(RuntimeError): + """The LiDAR asset could not be read.""" + + +class HttpRangeFile(io.RawIOBase): + """A seekable read-only file over HTTP `Range` requests. + + laspy's COPC reader needs `seek`/`read` and nothing else, so this is the + whole adapter: it turns an HTTP URL into something that behaves like an open + file without ever holding the tile in memory. + """ + + def __init__(self, session: Any, url: str, *, timeout: float = 60.0, + chunk_bytes: int = DEFAULT_CHUNK_BYTES) -> None: + self._session = session + self._url = url + self._timeout = timeout + self._chunk = max(1, chunk_bytes) + self._pos = 0 + self._size: int | None = None + # One cached chunk. COPC reads are clustered -- header, then index, then + # the nodes -- so a single block absorbs most of the repeat traffic. + self._cache: tuple[int, bytes] | None = None + self.requests = 0 + self.bytes_fetched = 0 + + # -- io.RawIOBase ---------------------------------------------------- + def readable(self) -> bool: + return True + + def seekable(self) -> bool: + return True + + def tell(self) -> int: + return self._pos + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + if whence == io.SEEK_SET: + self._pos = offset + elif whence == io.SEEK_CUR: + self._pos += offset + elif whence == io.SEEK_END: + self._pos = self.size + offset + else: # pragma: no cover - io module only defines the three + raise ValueError(f"invalid whence {whence}") + self._pos = max(0, self._pos) + return self._pos + + def read(self, size: int = -1) -> bytes: + if size is None or size < 0: + size = max(0, self.size - self._pos) + if size == 0: + return b"" + data = self._read_at(self._pos, size) + self._pos += len(data) + return data + + def readall(self) -> bytes: + return self.read(-1) + + def readinto(self, buffer) -> int: # type: ignore[override] + data = self.read(len(buffer)) + buffer[: len(data)] = data + return len(data) + + # -- range plumbing -------------------------------------------------- + @property + def size(self) -> int: + if self._size is None: + self._size = self._head_size() + return self._size + + def _head_size(self) -> int: + try: + resp = self._session.head(self._url, timeout=self._timeout) + except Exception as exc: + raise PointCloudError(f"could not stat {self._url}: {exc}") from exc + length = (getattr(resp, "headers", None) or {}).get("Content-Length") + if getattr(resp, "status_code", 0) != 200 or not length: + raise PointCloudError( + "the LiDAR host did not report a size, so it cannot be read in ranges" + ) + return int(length) + + def _read_at(self, offset: int, size: int) -> bytes: + cached = self._from_cache(offset, size) + if cached is not None: + return cached + want = max(size, self._chunk) + end = offset + want - 1 + try: + resp = self._session.get( + self._url, + headers={"Range": f"bytes={offset}-{end}"}, + timeout=self._timeout, + ) + except Exception as exc: + raise PointCloudError(f"range request failed: {exc}") from exc + status = getattr(resp, "status_code", 0) + if status == 200: + # The server ignored Range and sent everything. Honest failure: the + # caller falls back to a whole-tile read rather than silently + # paying for the full download on every seek. + raise PointCloudError("the LiDAR host does not support range requests") + if status != 206: + raise PointCloudError(f"range request returned HTTP {status}") + body = resp.content + self.requests += 1 + self.bytes_fetched += len(body) + self._cache = (offset, body) + return body[:size] + + def _from_cache(self, offset: int, size: int) -> bytes | None: + if self._cache is None: + return None + start, body = self._cache + if offset < start or offset + size > start + len(body): + return None + rel = offset - start + return body[rel : rel + size] + + +def bounds_of(ring: list[tuple[float, float]], pad_m: float = 2.0) -> tuple[float, float, float, float]: + xs = [p[0] for p in ring] + ys = [p[1] for p in ring] + return (min(xs) - pad_m, min(ys) - pad_m, max(xs) + pad_m, max(ys) + pad_m) + + +def _points_from_las(las: Any) -> Any: + import numpy as np + + return np.column_stack([np.asarray(las.x), np.asarray(las.y), np.asarray(las.z)]) + + +def load_points(data: bytes) -> Any: + """Decode a whole LAZ/LAS payload into (N, 3) SWEREF 99 TM metres.""" + laspy = _import_laspy() + with laspy.open(io.BytesIO(data)) as reader: + las = reader.read() + return _points_from_las(las) + + +def _import_laspy(): + try: + import laspy + except ImportError as exc: # pragma: no cover - depends on the install + raise PointCloudError( + "laspy is required to read Lantmaeteriet LiDAR. Install the module's " + "extras: pip install -e roofmodel[geo]" + ) from exc + return laspy + + +def read_copc_window( + session: Any, + url: str, + bounds: tuple[float, float, float, float], + *, + timeout: float = 60.0, +) -> Any: + """Points inside `bounds` from a COPC file, over HTTP range requests. + + Raises PointCloudError if the file or the host cannot support it, so the + caller can fall back to a whole-tile read. + """ + laspy = _import_laspy() + try: + from laspy.copc import Bounds, CopcReader + except ImportError as exc: + raise PointCloudError( + "this laspy build has no COPC support; install laspy[lazrs] 2.5 or newer" + ) from exc + + min_x, min_y, max_x, max_y = bounds + handle = HttpRangeFile(session, url, timeout=timeout) + try: + with CopcReader.open(handle) as reader: + query = Bounds(mins=[min_x, min_y], maxs=[max_x, max_y]) + points = reader.query(query) + except PointCloudError: + raise + except Exception as exc: + raise PointCloudError(f"COPC read failed: {exc}") from exc + return _points_from_las(points) diff --git a/roofmodel/ftw_roofmodel/segment.py b/roofmodel/ftw_roofmodel/segment.py new file mode 100644 index 00000000..33d6a13d --- /dev/null +++ b/roofmodel/ftw_roofmodel/segment.py @@ -0,0 +1,243 @@ +"""Roof-plane segmentation from a LiDAR point cloud. + +Implements the method described in the SPAN paper (Yavuzdogan, *Renewable +Energy* 2023, doi:10.1016/j.renene.2023.119022): iterative RANSAC plane fitting +to pull one roof surface at a time out of the cloud, then DBSCAN over each +plane's inliers to split faces that share a plane but not a location -- the two +halves of a gable on opposite wings of a building fit the same equation and are +not the same roof face. + +The method is reimplemented from its description. No code is taken from SPAN's +QGIS plugin, which is GPL; everything here uses numpy and scikit-learn, both +BSD, so this module carries no copyleft obligation. + +Coordinates are SWEREF 99 TM metres as (easting, northing, height). Azimuth +follows FTW's convention: 0 = north, 90 = east, 180 = south, 270 = west. +""" + +from __future__ import annotations + +import dataclasses +import math + +import numpy as np + +# A roof plane must hold at least this many returns to be believed. Below this +# a "plane" is usually a chimney, an aerial, or three points of noise that +# happen to be collinear. +MIN_PLANE_POINTS = 40 + +# Surfaces flatter than this are treated as flat roofs: their azimuth is +# meaningless (the normal is essentially vertical, so its horizontal component +# is numerical noise that can point anywhere). +FLAT_TILT_DEG = 5.0 + +# Roofs steeper than this are walls, dormer cheeks or mis-fits. +MAX_TILT_DEG = 80.0 + +# Fraction of a roof plane's area that can carry modules once you subtract +# ridges, eaves, chimneys, vents and walkways. +DEFAULT_PACKING_FACTOR = 0.70 + +# Module DC rating per square metre of module. ~20% efficiency at 1000 W/m2. +DEFAULT_MODULE_W_PER_M2 = 200.0 + + +@dataclasses.dataclass +class RoofPlane: + """One contiguous roof surface.""" + + tilt_deg: float + azimuth_deg: float + area_m2: float + point_count: int + mean_height_m: float + + def kwp( + self, + packing_factor: float = DEFAULT_PACKING_FACTOR, + module_w_per_m2: float = DEFAULT_MODULE_W_PER_M2, + ) -> float: + """Installable DC capacity for this surface, in kWp.""" + return self.area_m2 * packing_factor * module_w_per_m2 / 1000.0 + + +def _fit_plane(points: np.ndarray) -> np.ndarray: + """Least-squares plane through points; returns a unit normal pointing up. + + Uses the smallest singular vector of the mean-centred points, which is the + total-least-squares fit -- it minimises perpendicular distance rather than + vertical distance, so a steep roof is not biased the way an ordinary + z = ax + by + c regression would bias it. + """ + centred = points - points.mean(axis=0) + _, _, vh = np.linalg.svd(centred, full_matrices=False) + normal = vh[-1] + if normal[2] < 0: + normal = -normal + return normal / np.linalg.norm(normal) + + +def _tilt_azimuth(normal: np.ndarray) -> tuple[float, float]: + """Convert an upward unit normal to (tilt_deg, azimuth_deg).""" + tilt = math.degrees(math.acos(max(-1.0, min(1.0, float(normal[2]))))) + if tilt < FLAT_TILT_DEG: + # Horizontal component is noise at this point; report due south, which + # is what a flat-roof array is normally mounted to face anyway. + return tilt, 180.0 + east, north = float(normal[0]), float(normal[1]) + azimuth = math.degrees(math.atan2(east, north)) % 360.0 + return tilt, azimuth + + +def _convex_hull_area(xy: np.ndarray) -> float: + """Area of the convex hull of 2D points (monotone chain, shoelace). + + Hand-rolled to avoid a scipy dependency for one small routine. The hull + overestimates a concave roof outline, so it is only used as a fallback + when the point count is too low for the density estimate to be stable. + """ + pts = np.unique(xy, axis=0) + if len(pts) < 3: + return 0.0 + order = np.lexsort((pts[:, 1], pts[:, 0])) + pts = pts[order] + + def cross(o, a, b): + return (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]) + + lower: list = [] + for p in pts: + while len(lower) >= 2 and cross(lower[-2], lower[-1], p) <= 0: + lower.pop() + lower.append(p) + upper: list = [] + for p in pts[::-1]: + while len(upper) >= 2 and cross(upper[-2], upper[-1], p) <= 0: + upper.pop() + upper.append(p) + hull = np.array(lower[:-1] + upper[:-1]) + if len(hull) < 3: + return 0.0 + x, y = hull[:, 0], hull[:, 1] + return 0.5 * abs(float(np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1)))) + + +def _surface_area(points: np.ndarray, tilt_deg: float, point_density: float | None) -> float: + """Sloped surface area of a roof face, in m2. + + LiDAR density is quoted per square metre of *ground*, so a known density + gives the horizontal footprint directly from the point count; dividing by + cos(tilt) lifts that onto the slope. Without a density we fall back to the + convex hull of the horizontal projection, which is looser -- it fills in + L-shapes and courtyards. + """ + if point_density and point_density > 0: + horizontal = len(points) / point_density + else: + horizontal = _convex_hull_area(points[:, :2]) + cos_t = math.cos(math.radians(min(tilt_deg, MAX_TILT_DEG))) + if cos_t <= 1e-6: + return horizontal + return horizontal / cos_t + + +def _ransac_plane( + points: np.ndarray, + threshold_m: float, + iterations: int, + rng: np.random.Generator, +) -> np.ndarray | None: + """Return a boolean inlier mask for the best plane found, or None. + + Plain RANSAC over point triples. scikit-learn's RANSACRegressor is not used + because it regresses z on (x, y) and so cannot represent a vertical or + near-vertical surface, and weights errors vertically rather than + perpendicular to the plane. + """ + n = len(points) + if n < 3: + return None + best_mask = None + best_count = 0 + for _ in range(iterations): + idx = rng.choice(n, size=3, replace=False) + a, b, c = points[idx] + normal = np.cross(b - a, c - a) + norm = np.linalg.norm(normal) + if norm < 1e-9: + continue # degenerate (collinear) sample + normal = normal / norm + distances = np.abs((points - a) @ normal) + mask = distances < threshold_m + count = int(mask.sum()) + if count > best_count: + best_count, best_mask = count, mask + if best_mask is None or best_count < 3: + return None + return best_mask + + +def segment_roof( + points: np.ndarray, + *, + threshold_m: float = 0.25, + max_planes: int = 8, + min_plane_points: int = MIN_PLANE_POINTS, + cluster_eps_m: float = 1.5, + point_density: float | None = None, + ransac_iterations: int = 200, + seed: int = 0, +) -> list[RoofPlane]: + """Segment a roof point cloud into planes. + + `points` is an (N, 3) array of SWEREF 99 TM (easting, northing, height). + Returns planes ordered by descending area. Determinism is deliberate: the + same cloud must always yield the same arrays, or an operator re-running a + derive would see the geometry shuffle for no reason. + """ + from sklearn.cluster import DBSCAN # imported here to keep import cost off the CLI path + + pts = np.asarray(points, dtype=float) + if pts.ndim != 2 or pts.shape[1] != 3: + raise ValueError(f"points must be (N, 3), got {pts.shape}") + + rng = np.random.default_rng(seed) + remaining = pts + planes: list[RoofPlane] = [] + + for _ in range(max_planes): + if len(remaining) < min_plane_points: + break + mask = _ransac_plane(remaining, threshold_m, ransac_iterations, rng) + if mask is None or int(mask.sum()) < min_plane_points: + break + inliers = remaining[mask] + remaining = remaining[~mask] + + # One plane equation can describe several disjoint faces. Split them. + labels = DBSCAN(eps=cluster_eps_m, min_samples=10).fit(inliers[:, :2]).labels_ + for label in sorted(set(labels)): + if label == -1: + continue # DBSCAN noise + cluster = inliers[labels == label] + if len(cluster) < min_plane_points: + continue + normal = _fit_plane(cluster) + tilt, azimuth = _tilt_azimuth(normal) + if tilt > MAX_TILT_DEG: + continue # a wall, not a roof + planes.append( + RoofPlane( + tilt_deg=round(tilt, 1), + # Round before normalising: 359.97 rounds to 360.0, which is + # the same direction as 0 but reads as an out-of-range value. + azimuth_deg=round(azimuth, 1) % 360.0, + area_m2=round(_surface_area(cluster, tilt, point_density), 1), + point_count=len(cluster), + mean_height_m=round(float(cluster[:, 2].mean()), 2), + ) + ) + + planes.sort(key=lambda p: p.area_m2, reverse=True) + return planes diff --git a/roofmodel/ftw_roofmodel/shading.py b/roofmodel/ftw_roofmodel/shading.py new file mode 100644 index 00000000..b7eabf7b --- /dev/null +++ b/roofmodel/ftw_roofmodel/shading.py @@ -0,0 +1,328 @@ +"""Optional shadow-aware irradiance via vostok. + +Roof segmentation gives each face a tilt and an azimuth, which is enough to +predict yield for an unobstructed roof. It says nothing about the spruce to the +south or the neighbour's gable, and those can cost a real installation far more +than a few degrees of azimuth ever will. + +`vostok` ("Voxel Octree Solar Toolkit", 3DGeo Heidelberg) computes per-point +solar potential against voxelised occlusion geometry, so pointing it at the same +LiDAR tile the roof came from yields exactly the missing number. + +LICENSING -- read before changing anything here +----------------------------------------------- +vostok is **GPL-3.0**. FTW is not. This module therefore treats it as an +external program invoked at arm's length: a separate process, communicating +through files and a command line, with no linking and no code copied in either +direction. Under the FSF's own mere-aggregation reading that leaves FTW's +licence unaffected. + +Two rules keep it that way, and neither is negotiable: + + 1. **vostok is never bundled or redistributed with FTW.** The operator installs + it themselves. Adding it to an image, an installer or a dependency list + would turn aggregation into distribution. + 2. **FTW never installs it automatically.** No download, no build, no package + manager invocation. If the binary is absent, shading is skipped. + +Absent vostok, every roof face keeps a shading factor of 1.0 and the roof model +says shading was not evaluated -- an honest "unknown", not an assumed "unshaded". +""" + +from __future__ import annotations + +import dataclasses +import os +import shutil +import subprocess +import tempfile + +import numpy as np + +DEFAULT_BINARY = "vostok" +DEFAULT_VOXEL_SIZE_M = 1.0 +DEFAULT_TIMEOUT_S = 900 + +# Sampling cap per roof face. vostok cost scales with query points and a roof +# needs a representative sample, not every return. +MAX_QUERY_POINTS_PER_PLANE = 200 + + +class VostokUnavailable(RuntimeError): + """vostok is not installed, or not runnable.""" + + +class VostokFailed(RuntimeError): + """vostok ran but did not produce usable output.""" + + +@dataclasses.dataclass +class ShadingResult: + """Per-plane shading factors, keyed by the plane's index.""" + + factors: dict[int, float] + evaluated: bool + reason: str = "" + + def factor_for(self, index: int) -> float: + """Shading factor for a plane; 1.0 (no derate) when not evaluated.""" + return self.factors.get(index, 1.0) + + +def find_vostok(binary: str = DEFAULT_BINARY) -> str | None: + """Absolute path to the vostok executable, or None if not installed.""" + if os.path.isabs(binary): + return binary if os.path.isfile(binary) and os.access(binary, os.X_OK) else None + return shutil.which(binary) + + +def available(binary: str = DEFAULT_BINARY) -> bool: + return find_vostok(binary) is not None + + +def _plane_normal(tilt_deg: float, azimuth_deg: float) -> tuple[float, float, float]: + """Upward unit normal for a surface with the given tilt and azimuth. + + Inverse of segment._tilt_azimuth. vostok requires a normal on every query + point, and the fitted plane normal is a cleaner input than re-deriving one + per point from noisy neighbours. + """ + import math + + t = math.radians(tilt_deg) + a = math.radians(azimuth_deg) + return (math.sin(t) * math.sin(a), math.sin(t) * math.cos(a), math.cos(t)) + + +def _write_xyz(path: str, points: np.ndarray, normal=None) -> None: + """Write an ASCII point cloud, with normals when given.""" + with open(path, "w", encoding="ascii") as fh: + if normal is None: + for p in points: + fh.write(f"{p[0]:.3f} {p[1]:.3f} {p[2]:.3f}\n") + else: + nx, ny, nz = normal + for p in points: + fh.write(f"{p[0]:.3f} {p[1]:.3f} {p[2]:.3f} {nx:.6f} {ny:.6f} {nz:.6f}\n") + + +def _write_sol( + path: str, + *, + shadow_cloud: str, + query_cloud: str, + output: str, + latitude: float, + longitude: float, + voxel_size_m: float, + year: int, + shadowing: bool, +) -> None: + """Write a vostok .sol configuration. + + `shadowing` is the whole trick: the same query points are run twice, once + against the real occlusion geometry and once with shadowing off. The ratio + of the two is a shading factor that is independent of vostok's absolute + units, its sky model and the year chosen -- all of which cancel. + """ + lines = [ + f"shadow_cloud = {shadow_cloud}", + f"query_cloud = {query_cloud}", + f"output = {output}", + f"voxel_size = {voxel_size_m}", + f"latitude = {latitude:.6f}", + f"longitude = {longitude:.6f}", + f"year = {year}", + "start_day = 1", + "end_day = 365", + "time_step_minutes = 60", + f"shadowing = {'true' if shadowing else 'false'}", + ] + with open(path, "w", encoding="ascii") as fh: + fh.write("\n".join(lines) + "\n") + + +def _read_potential(path: str) -> np.ndarray: + """Read the per-point solar potential column from a vostok output file. + + vostok appends the potential (Wh/m2/day) after the xyz and nxnynz columns, + so it is the last numeric field on each line. Reading it positionally from + the end rather than by a fixed index keeps this working whether or not the + build emits normals it was not given. + """ + values: list[float] = [] + with open(path, "r", encoding="ascii", errors="ignore") as fh: + for line in fh: + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + try: + values.append(float(parts[-1])) + except (ValueError, IndexError): + continue + if not values: + raise VostokFailed(f"no usable values in vostok output {path}") + return np.asarray(values, dtype=float) + + +def _run(binary: str, sol_path: str, timeout_s: int) -> None: + try: + proc = subprocess.run( + [binary, sol_path], + capture_output=True, + timeout=timeout_s, + check=False, + ) + except FileNotFoundError as exc: + raise VostokUnavailable(f"vostok not found: {binary}") from exc + except subprocess.TimeoutExpired as exc: + raise VostokFailed(f"vostok timed out after {timeout_s}s") from exc + if proc.returncode != 0: + tail = (proc.stderr or b"").decode("utf-8", "replace").strip()[-400:] + raise VostokFailed(f"vostok exited {proc.returncode}: {tail}") + + +def compute_shading( + points: np.ndarray, + planes: list, + *, + latitude: float, + longitude: float, + binary: str = DEFAULT_BINARY, + voxel_size_m: float = DEFAULT_VOXEL_SIZE_M, + timeout_s: int = DEFAULT_TIMEOUT_S, + year: int = 2024, + max_query_points: int = MAX_QUERY_POINTS_PER_PLANE, + seed: int = 0, +) -> ShadingResult: + """Shading factor per roof plane, in [0, 1]. + + 1.0 means the surface receives everything an unobstructed one would; 0.6 + means neighbouring geometry costs it 40% of its annual irradiation. + + `points` is the full local point cloud, which doubles as the occlusion + geometry -- the trees and neighbouring roofs that cast the shadows are + already in the tile the roof was segmented from. + + Never raises for an absent vostok: that is the normal case, and it returns + an unevaluated result instead so the caller can report "not evaluated" + rather than silently assuming no shading. + """ + if not planes: + return ShadingResult({}, evaluated=False, reason="no roof planes to evaluate") + + resolved = find_vostok(binary) + if resolved is None: + return ShadingResult( + {}, + evaluated=False, + reason=( + "vostok is not installed; shading was not evaluated. It is a " + "separate GPL-3.0 tool that FTW never bundles or installs -- see " + "https://github.com/3dgeo-heidelberg/vostok" + ), + ) + + rng = np.random.default_rng(seed) + pts = np.asarray(points, dtype=float) + + with tempfile.TemporaryDirectory(prefix="ftw-vostok-") as tmp: + shadow_path = os.path.join(tmp, "shadow.xyz") + _write_xyz(shadow_path, pts) + + # One query file per plane keeps the mapping from output rows back to + # planes trivial, and lets a single failed plane be dropped rather than + # invalidating the whole run. + query_specs: list[tuple[int, str, int]] = [] + for idx, plane in enumerate(planes): + sample = _sample_plane_points(pts, plane, rng, max_query_points) + if sample is None: + continue + qpath = os.path.join(tmp, f"query-{idx}.xyz") + _write_xyz(qpath, sample, normal=_plane_normal(plane.tilt_deg, plane.azimuth_deg)) + query_specs.append((idx, qpath, len(sample))) + + if not query_specs: + return ShadingResult({}, evaluated=False, reason="no query points could be sampled") + + factors: dict[int, float] = {} + for idx, qpath, _ in query_specs: + try: + shaded = _run_once( + resolved, tmp, idx, shadow_path, qpath, latitude, longitude, + voxel_size_m, year, timeout_s, shadowing=True, + ) + open_sky = _run_once( + resolved, tmp, idx, shadow_path, qpath, latitude, longitude, + voxel_size_m, year, timeout_s, shadowing=False, + ) + except (VostokFailed, VostokUnavailable): + continue + reference = float(np.mean(open_sky)) + if reference <= 0: + continue + factor = float(np.mean(shaded)) / reference + # Shadowing can only remove irradiance. A ratio above 1 means the + # two runs are not comparable, so clamp rather than report a gain. + factors[idx] = max(0.0, min(1.0, factor)) + + if not factors: + return ShadingResult({}, evaluated=False, reason="vostok produced no usable results") + return ShadingResult(factors, evaluated=True) + + +def _run_once( + binary: str, + tmp: str, + idx: int, + shadow_path: str, + query_path: str, + latitude: float, + longitude: float, + voxel_size_m: float, + year: int, + timeout_s: int, + *, + shadowing: bool, +) -> np.ndarray: + tag = "shaded" if shadowing else "open" + out_path = os.path.join(tmp, f"out-{idx}-{tag}.xyz") + sol_path = os.path.join(tmp, f"run-{idx}-{tag}.sol") + _write_sol( + sol_path, + shadow_cloud=shadow_path, + query_cloud=query_path, + output=out_path, + latitude=latitude, + longitude=longitude, + voxel_size_m=voxel_size_m, + year=year, + shadowing=shadowing, + ) + _run(binary, sol_path, timeout_s) + if not os.path.exists(out_path): + raise VostokFailed(f"vostok wrote no output for plane {idx} ({tag})") + return _read_potential(out_path) + + +def _sample_plane_points( + pts: np.ndarray, plane, rng: np.random.Generator, limit: int +) -> np.ndarray | None: + """Pick representative query points lying on one roof plane. + + Planes carry statistics rather than their member points, so the face is + recovered by selecting returns near its mean height. Coarse, but the query + points only need to sit on the surface being evaluated. + """ + if len(pts) == 0: + return None + band = 1.0 + mask = np.abs(pts[:, 2] - plane.mean_height_m) < band + candidates = pts[mask] + if len(candidates) < 10: + return None + if len(candidates) > limit: + idx = rng.choice(len(candidates), size=limit, replace=False) + candidates = candidates[idx] + return candidates diff --git a/roofmodel/ftw_roofmodel/sweref.py b/roofmodel/ftw_roofmodel/sweref.py new file mode 100644 index 00000000..3c2f699d --- /dev/null +++ b/roofmodel/ftw_roofmodel/sweref.py @@ -0,0 +1,173 @@ +"""SWEREF 99 TM <-> WGS84 conversion. + +Lantmaeteriet publishes everything in SWEREF 99 TM (EPSG:3006) while FTW stores +site location as WGS84 latitude/longitude, so every bounding box we send and +every point cloud we read has to cross this boundary. + +This is Lantmaeteriet's own published Gauss conformal projection algorithm +(Krueger series), implemented directly rather than pulled in via pyproj. The +reason is proportion: pyproj ships a full PROJ build for what is, here, exactly +one projection with fixed parameters. The series below is accurate to well under +a millimetre across Sweden, which is several orders of magnitude finer than the +1-2 points/m2 LiDAR it is used to place. + +SWEREF 99 TM is a transverse Mercator on GRS 80 with central meridian 15 deg E, +scale factor 0.9996, false easting 500 000 m and false northing 0. +""" + +from __future__ import annotations + +import math + +# GRS 80 ellipsoid. +_A = 6378137.0 +_F = 1.0 / 298.257222101 + +# SWEREF 99 TM projection parameters. +_CENTRAL_MERIDIAN = 15.0 +_SCALE = 0.9996 +_FALSE_EASTING = 500000.0 +_FALSE_NORTHING = 0.0 + +# Derived constants, computed once. +_E2 = _F * (2.0 - _F) +_N = _F / (2.0 - _F) +_A_HAT = _A / (1.0 + _N) * (1.0 + _N**2 / 4.0 + _N**4 / 64.0) + + +def _forward_coefficients() -> tuple[float, float, float, float]: + n = _N + return ( + n / 2.0 - 2.0 * n**2 / 3.0 + 5.0 * n**3 / 16.0 + 41.0 * n**4 / 180.0, + 13.0 * n**2 / 48.0 - 3.0 * n**3 / 5.0 + 557.0 * n**4 / 1440.0, + 61.0 * n**3 / 240.0 - 103.0 * n**4 / 140.0, + 49561.0 * n**4 / 161280.0, + ) + + +def _inverse_coefficients() -> tuple[float, float, float, float]: + n = _N + return ( + n / 2.0 - 2.0 * n**2 / 3.0 + 37.0 * n**3 / 96.0 - n**4 / 360.0, + n**2 / 48.0 + n**3 / 15.0 - 437.0 * n**4 / 1440.0, + 17.0 * n**3 / 480.0 - 37.0 * n**4 / 840.0, + 4397.0 * n**4 / 161280.0, + ) + + +def wgs84_to_sweref99tm(lat: float, lon: float) -> tuple[float, float]: + """Convert WGS84 degrees to SWEREF 99 TM (northing, easting) in metres.""" + phi = math.radians(lat) + lam = math.radians(lon) + lam0 = math.radians(_CENTRAL_MERIDIAN) + + e2 = _E2 + a_coef = e2 + b_coef = (5.0 * e2**2 - e2**3) / 6.0 + c_coef = (104.0 * e2**3 - 45.0 * e2**4) / 120.0 + d_coef = 1237.0 * e2**4 / 1260.0 + + sin_phi = math.sin(phi) + phi_star = phi - sin_phi * math.cos(phi) * ( + a_coef + + b_coef * sin_phi**2 + + c_coef * sin_phi**4 + + d_coef * sin_phi**6 + ) + + dlam = lam - lam0 + xi_p = math.atan2(math.tan(phi_star), math.cos(dlam)) + eta_p = math.atanh(math.cos(phi_star) * math.sin(dlam)) + + b1, b2, b3, b4 = _forward_coefficients() + scaled = _SCALE * _A_HAT + northing = _FALSE_NORTHING + scaled * ( + xi_p + + b1 * math.sin(2 * xi_p) * math.cosh(2 * eta_p) + + b2 * math.sin(4 * xi_p) * math.cosh(4 * eta_p) + + b3 * math.sin(6 * xi_p) * math.cosh(6 * eta_p) + + b4 * math.sin(8 * xi_p) * math.cosh(8 * eta_p) + ) + easting = _FALSE_EASTING + scaled * ( + eta_p + + b1 * math.cos(2 * xi_p) * math.sinh(2 * eta_p) + + b2 * math.cos(4 * xi_p) * math.sinh(4 * eta_p) + + b3 * math.cos(6 * xi_p) * math.sinh(6 * eta_p) + + b4 * math.cos(8 * xi_p) * math.sinh(8 * eta_p) + ) + return northing, easting + + +def sweref99tm_to_wgs84(northing: float, easting: float) -> tuple[float, float]: + """Convert SWEREF 99 TM (northing, easting) in metres to WGS84 degrees.""" + scaled = _SCALE * _A_HAT + xi = (northing - _FALSE_NORTHING) / scaled + eta = (easting - _FALSE_EASTING) / scaled + + d1, d2, d3, d4 = _inverse_coefficients() + xi_p = ( + xi + - d1 * math.sin(2 * xi) * math.cosh(2 * eta) + - d2 * math.sin(4 * xi) * math.cosh(4 * eta) + - d3 * math.sin(6 * xi) * math.cosh(6 * eta) + - d4 * math.sin(8 * xi) * math.cosh(8 * eta) + ) + eta_p = ( + eta + - d1 * math.cos(2 * xi) * math.sinh(2 * eta) + - d2 * math.cos(4 * xi) * math.sinh(4 * eta) + - d3 * math.cos(6 * xi) * math.sinh(6 * eta) + - d4 * math.cos(8 * xi) * math.sinh(8 * eta) + ) + + phi_star = math.asin(math.sin(xi_p) / math.cosh(eta_p)) + dlam = math.atan2(math.sinh(eta_p), math.cos(xi_p)) + + e2 = _E2 + a_star = e2 + e2**2 + e2**3 + e2**4 + b_star = -(7.0 * e2**2 + 17.0 * e2**3 + 30.0 * e2**4) / 6.0 + c_star = (224.0 * e2**3 + 889.0 * e2**4) / 120.0 + d_star = -(4279.0 * e2**4) / 1260.0 + + sin_ps = math.sin(phi_star) + phi = phi_star + sin_ps * math.cos(phi_star) * ( + a_star + + b_star * sin_ps**2 + + c_star * sin_ps**4 + + d_star * sin_ps**6 + ) + lam = math.radians(_CENTRAL_MERIDIAN) + dlam + return math.degrees(phi), math.degrees(lam) + + +def bbox_wgs84_to_sweref99tm( + min_lat: float, min_lon: float, max_lat: float, max_lon: float +) -> tuple[float, float, float, float]: + """Project a WGS84 bounding box to a SWEREF 99 TM (minE, minN, maxE, maxN). + + All four corners are projected and the extremes taken, rather than just the + two diagonal corners: the projection is not axis-aligned, so a box's + projected edges bow outward and the diagonal-only result would clip. + """ + corners = [ + wgs84_to_sweref99tm(min_lat, min_lon), + wgs84_to_sweref99tm(min_lat, max_lon), + wgs84_to_sweref99tm(max_lat, min_lon), + wgs84_to_sweref99tm(max_lat, max_lon), + ] + northings = [c[0] for c in corners] + eastings = [c[1] for c in corners] + return min(eastings), min(northings), max(eastings), max(northings) + + +def metre_box_around(lat: float, lon: float, radius_m: float) -> tuple[float, float, float, float]: + """Return a WGS84 (min_lat, min_lon, max_lat, max_lon) box of +/- radius_m. + + Built by projecting the centre, stepping in metres in SWEREF 99 TM, and + unprojecting: doing it that way keeps the box square on the ground instead + of stretching with latitude the way a naive degree offset would. + """ + n, e = wgs84_to_sweref99tm(lat, lon) + south, west = sweref99tm_to_wgs84(n - radius_m, e - radius_m) + north, east = sweref99tm_to_wgs84(n + radius_m, e + radius_m) + return south, west, north, east diff --git a/roofmodel/pyproject.toml b/roofmodel/pyproject.toml new file mode 100644 index 00000000..4e138a24 --- /dev/null +++ b/roofmodel/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "ftw-roofmodel" +version = "0.1.0" +description = "Derive PV array geometry from Lantmateriet building and LiDAR open data" +requires-python = ">=3.10" +# Core dependencies are permissive-licensed and light enough to install on a Pi. +dependencies = [ + "numpy>=1.24", + "scikit-learn>=1.3", + "requests>=2.31", +] + +[project.optional-dependencies] +# LAZ decoding pulls a compiled backend, so it is opt-in. Everything except the +# point-cloud read works without it, which keeps the module testable in CI. +geo = ["laspy[lazrs]>=2.5"] +test = ["pytest==8.4.2"] + +[tool.setuptools.packages.find] +include = ["ftw_roofmodel*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/roofmodel/tests/__init__.py b/roofmodel/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/roofmodel/tests/test_assets.py b/roofmodel/tests/test_assets.py new file mode 100644 index 00000000..107a5bae --- /dev/null +++ b/roofmodel/tests/test_assets.py @@ -0,0 +1,212 @@ +"""Choosing STAC assets by what they are, and reading them in ranges. + +Both Lantmaeteriet products are STAC APIs; they differ in what their items point +at. Byggnad-vektor delivers GeoPackage, Laserdata Skog delivers LAZ organised as +COPC. Selecting on media type rather than on an asset key is what keeps that +difference from becoming a pile of special cases. +""" + +from __future__ import annotations + +import io + +import pytest + +from ftw_roofmodel.geotorget import ( + MEDIA_COPC, + MEDIA_GEOJSON, + MEDIA_GEOPACKAGE, + MEDIA_LAZ, + Asset, + StacItem, + _item_from_feature, + media_type_for, +) +from ftw_roofmodel.pointcloud import HttpRangeFile, PointCloudError, bounds_of + + +def item(assets): + return StacItem("i", "c", assets, None, raw={}) + + +def test_a_bare_href_still_gets_a_type_from_its_extension(): + """Tests and simple catalogues pass strings; selection must still work.""" + it = item({"data": "https://x/tile.copc.laz"}) + assert isinstance(it.assets["data"], Asset) + assert it.pick(MEDIA_COPC).href == "https://x/tile.copc.laz" + + +def test_copc_is_recognised_before_plain_laz(): + """A COPC file is also a .laz; reading it as one costs the whole tile.""" + assert media_type_for("https://x/y/tile.copc.laz") == MEDIA_COPC + assert media_type_for("https://x/y/tile.laz") == MEDIA_LAZ + + +def test_query_strings_do_not_hide_the_extension(): + """Signed download URLs carry tokens after a '?'.""" + assert media_type_for("https://x/tile.gpkg?token=abc&x=1") == MEDIA_GEOPACKAGE + + +def test_a_declared_type_beats_the_extension(): + """The catalogue knows better than the filename.""" + a = Asset(href="https://x/download", media_type=MEDIA_GEOPACKAGE) + assert a.effective_media_type == MEDIA_GEOPACKAGE + + +def test_preference_order_is_honoured(): + it = item({ + "laz": Asset("https://x/t.laz", MEDIA_LAZ), + "copc": Asset("https://x/t.copc.laz", MEDIA_COPC), + }) + assert it.pick(MEDIA_COPC, MEDIA_LAZ).effective_media_type == MEDIA_COPC + assert it.pick(MEDIA_LAZ, MEDIA_COPC).effective_media_type == MEDIA_LAZ + + +def test_falls_back_to_the_data_role_when_the_type_is_unknown(): + it = item({ + "thumbnail": Asset("https://x/preview.png", "image/png", roles=("thumbnail",)), + "mystery": Asset("https://x/blob", None, roles=("data",)), + }) + assert it.pick(MEDIA_COPC).href == "https://x/blob" + + +def test_a_lone_asset_is_unambiguous_whatever_it_is_called(): + assert item({"whatever": Asset("https://x/blob")}).pick(MEDIA_COPC).href == "https://x/blob" + + +def test_several_unlabelled_assets_are_refused_rather_than_guessed(): + it = item({"a": Asset("https://x/a"), "b": Asset("https://x/b")}) + assert it.pick(MEDIA_COPC) is None + + +def test_stac_assets_keep_their_type_and_roles(): + feature = { + "id": "tile-1", + "collection": "laserdata-nedladdning-skog", + "assets": { + "data": { + "href": "https://x/t.copc.laz", + "type": MEDIA_COPC, + "roles": ["data"], + "title": "Punktmoln", + } + }, + "properties": {}, + } + parsed = _item_from_feature(feature) + asset = parsed.pick(MEDIA_COPC) + assert asset.media_type == MEDIA_COPC + assert asset.roles == ("data",) + assert asset.title == "Punktmoln" + + +def test_geojson_and_geopackage_are_both_selectable(): + it = item({"gj": Asset("https://x/b.geojson"), "gp": Asset("https://x/b.gpkg")}) + assert it.pick(MEDIA_GEOPACKAGE, MEDIA_GEOJSON).href == "https://x/b.gpkg" + assert it.pick(MEDIA_GEOJSON, MEDIA_GEOPACKAGE).href == "https://x/b.geojson" + + +# --- range reads ------------------------------------------------------------ + + +class FakeResponse: + def __init__(self, status, content=b"", headers=None): + self.status_code = status + self.content = content + self.headers = headers or {} + + +class RangeServer: + """Serves a byte string over Range, and counts what was actually moved.""" + + def __init__(self, body: bytes, *, supports_range: bool = True): + self.body = body + self.supports_range = supports_range + self.requests: list[str] = [] + + def head(self, url, timeout=None): + return FakeResponse(200, headers={"Content-Length": str(len(self.body))}) + + def get(self, url, headers=None, timeout=None): + rng = (headers or {}).get("Range") + if not self.supports_range or not rng: + self.requests.append("full") + return FakeResponse(200, self.body) + self.requests.append(rng) + spec = rng.split("=", 1)[1] + start, end = spec.split("-") + lo = int(start) + hi = min(int(end), len(self.body) - 1) + return FakeResponse(206, self.body[lo : hi + 1]) + + +BODY = bytes(range(256)) * 40 # 10 240 bytes, every offset distinguishable + + +def test_reads_a_window_without_moving_the_whole_file(): + server = RangeServer(BODY) + fh = HttpRangeFile(server, "https://x/t.copc.laz", chunk_bytes=512) + fh.seek(1000) + assert fh.read(16) == BODY[1000:1016] + assert fh.bytes_fetched == 512, "one chunk, not the whole file" + assert len(server.requests) == 1 + + +def test_seek_and_tell_track_the_position(): + fh = HttpRangeFile(RangeServer(BODY), "https://x/t", chunk_bytes=64) + assert fh.seek(100) == 100 and fh.tell() == 100 + fh.read(10) + assert fh.tell() == 110 + assert fh.seek(-10, io.SEEK_END) == len(BODY) - 10 + assert fh.seek(5, io.SEEK_CUR) == len(BODY) - 5 + + +def test_a_second_read_inside_the_chunk_costs_no_request(): + server = RangeServer(BODY) + fh = HttpRangeFile(server, "https://x/t", chunk_bytes=1024) + fh.seek(0) + fh.read(8) + before = len(server.requests) + fh.seek(64) + assert fh.read(8) == BODY[64:72] + assert len(server.requests) == before, "the chunk was already held" + + +def test_reading_across_the_chunk_boundary_fetches_again(): + server = RangeServer(BODY) + fh = HttpRangeFile(server, "https://x/t", chunk_bytes=128) + fh.seek(0) + assert fh.read(8) == BODY[0:8] + fh.seek(4096) + assert fh.read(8) == BODY[4096:4104] + assert len(server.requests) == 2 + + +def test_a_host_that_ignores_range_is_detected_not_trusted(): + """A 200 means the whole body arrived; treating it as partial corrupts.""" + fh = HttpRangeFile(RangeServer(BODY, supports_range=False), "https://x/t") + fh.seek(10) + with pytest.raises(PointCloudError, match="range requests"): + fh.read(4) + + +def test_a_host_that_will_not_report_a_size_is_refused(): + class NoLength: + def head(self, url, timeout=None): + return FakeResponse(200, headers={}) + + with pytest.raises(PointCloudError, match="size"): + HttpRangeFile(NoLength(), "https://x/t").size + + +def test_readinto_fills_the_buffer(): + fh = HttpRangeFile(RangeServer(BODY), "https://x/t", chunk_bytes=256) + fh.seek(32) + buf = bytearray(16) + assert fh.readinto(buf) == 16 + assert bytes(buf) == BODY[32:48] + + +def test_bounds_pad_the_footprint_so_the_eaves_survive(): + ring = [(100.0, 200.0), (110.0, 200.0), (110.0, 220.0), (100.0, 220.0)] + assert bounds_of(ring, 1.0) == (99.0, 199.0, 111.0, 221.0) diff --git a/roofmodel/tests/test_buildings.py b/roofmodel/tests/test_buildings.py new file mode 100644 index 00000000..27205ab9 --- /dev/null +++ b/roofmodel/tests/test_buildings.py @@ -0,0 +1,206 @@ +"""Building lookup, frame detection and footprint clipping.""" + +from __future__ import annotations + +import math + +import numpy as np +import pytest + +from ftw_roofmodel import sweref +from ftw_roofmodel.buildings import ( + Building, + BuildingLookupError, + buildings_from_features, + clip_to_footprint, + inflate_ring, + point_in_ring, + search_buildings, +) +from ftw_roofmodel.geotorget import COLLECTION_BUILDINGS, Credentials, GeotorgetClient + +STOCKHOLM = (59.33, 18.07) + + +def square_ring(cx, cy, side): + h = side / 2.0 + return [(cx - h, cy - h), (cx + h, cy - h), (cx + h, cy + h), (cx - h, cy + h)] + + +class FakeResponse: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + + def json(self): + return self._payload + + +class FakeSession: + """Records what was asked for and replays a canned STAC response.""" + + def __init__(self, payload): + self.payload = payload + self.posts = [] + + def post(self, url, json=None, timeout=None): + self.posts.append((url, json)) + return FakeResponse(self.payload) + + +def stac_feature(ring, feature_id="bldg-1", **props): + return { + "id": feature_id, + "collection": COLLECTION_BUILDINGS, + "geometry": {"type": "Polygon", "coordinates": [[list(p) for p in ring] + [list(ring[0])]]}, + "properties": props, + "assets": {}, + } + + +def test_area_and_centroid_of_a_known_square(): + ring = square_ring(674000.0, 6580000.0, 10.0) + [b] = buildings_from_features( + [{"geometry": {"type": "Polygon", "coordinates": [ring]}, "id": "sq"}], + latitude=STOCKHOLM[0], longitude=STOCKHOLM[1], + ) + assert b.area_m2 == pytest.approx(100.0) + cx, cy = b.centroid_sweref() + assert (cx, cy) == pytest.approx((674000.0, 6580000.0)) + + +def test_wgs84_rings_are_projected_before_measuring(): + """A ring in degrees must be recognised and converted, not measured raw.""" + lat, lon = STOCKHOLM + e, n = sweref.wgs84_to_sweref99tm(lat, lon) + ring_sweref = square_ring(e, n, 12.0) + ring_wgs84 = [] + for x, y in ring_sweref: + blat, blon = sweref.sweref99tm_to_wgs84(x, y) + ring_wgs84.append([blon, blat]) # GeoJSON is [lon, lat] + + [b] = buildings_from_features( + [{"geometry": {"type": "Polygon", "coordinates": [ring_wgs84]}, "id": "deg"}], + latitude=lat, longitude=lon, + ) + # 12 m square, recovered through a full round trip through degrees. + assert b.area_m2 == pytest.approx(144.0, abs=0.5) + + +def test_tiles_and_slivers_are_not_offered_as_buildings(): + lat, lon = STOCKHOLM + e, n = sweref.wgs84_to_sweref99tm(lat, lon) + feats = [ + {"geometry": {"type": "Polygon", "coordinates": [square_ring(e, n, 2500.0)]}, "id": "tile"}, + {"geometry": {"type": "Polygon", "coordinates": [square_ring(e, n, 1.0)]}, "id": "sliver"}, + {"geometry": {"type": "Polygon", "coordinates": [square_ring(e, n, 11.0)]}, "id": "house"}, + ] + got = buildings_from_features(feats, latitude=lat, longitude=lon) + assert [b.building_id for b in got] == ["house"] + + +def test_candidates_come_back_nearest_first(): + lat, lon = STOCKHOLM + e, n = sweref.wgs84_to_sweref99tm(lat, lon) + feats = [ + {"geometry": {"type": "Polygon", "coordinates": [square_ring(e + 60, n, 10.0)]}, "id": "far"}, + {"geometry": {"type": "Polygon", "coordinates": [square_ring(e + 5, n, 10.0)]}, "id": "near"}, + {"geometry": {"type": "Polygon", "coordinates": [square_ring(e + 25, n, 10.0)]}, "id": "mid"}, + ] + got = buildings_from_features(feats, latitude=lat, longitude=lon) + assert [b.building_id for b in got] == ["near", "mid", "far"] + assert got[0].distance_m < got[1].distance_m < got[2].distance_m + + +def test_multipolygon_yields_one_candidate_per_part(): + lat, lon = STOCKHOLM + e, n = sweref.wgs84_to_sweref99tm(lat, lon) + feat = { + "id": "pair", + "geometry": { + "type": "MultiPolygon", + "coordinates": [[square_ring(e, n, 10.0)], [square_ring(e + 30, n, 12.0)]], + }, + } + got = buildings_from_features([feat], latitude=lat, longitude=lon) + assert len(got) == 2 + assert len({b.building_id for b in got}) == 2, "parts must not share an id" + + +def test_search_queries_the_building_collection_and_maps_results(): + lat, lon = STOCKHOLM + e, n = sweref.wgs84_to_sweref99tm(lat, lon) + session = FakeSession({"features": [stac_feature(square_ring(e, n, 10.0), "b1")]}) + client = GeotorgetClient(Credentials("u", "t"), session=session) + + got = search_buildings(client, latitude=lat, longitude=lon) + + assert [b.building_id for b in got] == ["b1"] + (_, body), = session.posts + assert body["collections"] == [COLLECTION_BUILDINGS] + # The bbox must be the SWEREF box around the site, not raw degrees. + assert body["bbox"][0] > 1000 + + +def test_search_says_what_to_do_when_nothing_comes_back(): + client = GeotorgetClient(Credentials("u", "t"), session=FakeSession({"features": []})) + with pytest.raises(BuildingLookupError) as exc: + search_buildings(client, latitude=STOCKHOLM[0], longitude=STOCKHOLM[1]) + assert "Byggnad" in str(exc.value) + + +def test_point_in_ring_handles_edges_and_outside(): + ring = square_ring(0.0, 0.0, 10.0) + assert point_in_ring(0.0, 0.0, ring) + assert point_in_ring(4.9, 4.9, ring) + assert not point_in_ring(5.1, 0.0, ring) + assert not point_in_ring(0.0, 99.0, ring) + + +def test_inflate_ring_grows_the_outline(): + ring = square_ring(0.0, 0.0, 10.0) + bigger = inflate_ring(ring, 1.0) + # Corners sit at radius 7.07; pushing 1 m out puts them at 8.07. + assert math.hypot(*bigger[0]) == pytest.approx(math.hypot(*ring[0]) + 1.0) + assert all(point_in_ring(x, y, bigger) for x, y in ring) + + +def test_clip_keeps_the_building_and_drops_the_neighbours(): + rng = np.random.default_rng(3) + mine = np.column_stack([ + rng.uniform(-4, 4, 400), rng.uniform(-4, 4, 400), rng.uniform(0, 4, 400)]) + theirs = np.column_stack([ + rng.uniform(46, 54, 400), rng.uniform(-4, 4, 400), rng.uniform(0, 4, 400)]) + cloud = np.vstack([mine, theirs]) + + kept = clip_to_footprint(cloud, square_ring(0.0, 0.0, 10.0), buffer_m=0.0) + + assert len(kept) == len(mine) + assert kept[:, 0].max() < 10.0 + + +def test_clip_keeps_the_eaves(): + """Roof returns overhang the wall line; clipping exactly would shave them.""" + ring = square_ring(0.0, 0.0, 10.0) + eaves = np.array([[5.4, 0.0, 3.0], [-5.4, 0.0, 3.0], [0.0, 5.4, 3.0]]) + + assert len(clip_to_footprint(eaves, ring, buffer_m=0.0)) == 0 + assert len(clip_to_footprint(eaves, ring, buffer_m=1.0)) == 3 + + +def test_clip_of_an_empty_cloud_is_empty_not_an_error(): + assert len(clip_to_footprint(np.empty((0, 3)), square_ring(0, 0, 10))) == 0 + + +def test_geojson_feature_is_wgs84_and_closed(): + lat, lon = STOCKHOLM + e, n = sweref.wgs84_to_sweref99tm(lat, lon) + b = Building("b1", square_ring(e, n, 10.0), 100.0, 0.0) + feat = b.to_geojson() + + ring = feat["geometry"]["coordinates"][0] + assert ring[0] == ring[-1], "GeoJSON rings must close" + for x, y in ring: + assert -180 <= x <= 180 and -90 <= y <= 90 + assert feat["properties"]["latitude"] == pytest.approx(lat, abs=1e-3) + assert feat["properties"]["longitude"] == pytest.approx(lon, abs=1e-3) diff --git a/roofmodel/tests/test_derive_footprint.py b/roofmodel/tests/test_derive_footprint.py new file mode 100644 index 00000000..3932e7ef --- /dev/null +++ b/roofmodel/tests/test_derive_footprint.py @@ -0,0 +1,149 @@ +"""Deriving against a picked building footprint.""" + +from __future__ import annotations + +import math + +import numpy as np +import pytest + +from ftw_roofmodel import pipeline, sweref +from ftw_roofmodel.buildings import clip_to_footprint +from ftw_roofmodel.geotorget import Credentials, StacItem +from ftw_roofmodel.pipeline import RoofModelError, derive +from ftw_roofmodel.segment import segment_roof + +STOCKHOLM = (59.33, 18.07) + + +def roof_face(tilt, azimuth, w, d, origin, density=8, noise=0.04, seed=1): + """Sample a tilted rectangle. Written as the inverse of what segment.py + computes, so recovering the tilt is a real test rather than a tautology.""" + rng = np.random.default_rng(seed) + n = int(w * d * density) + x = rng.uniform(0, w, n) + y = rng.uniform(0, d, n) + az = math.radians(azimuth) + s = math.tan(math.radians(tilt)) + z = -(x * math.sin(az) + y * math.cos(az)) * s + rng.normal(0, noise, n) + return np.column_stack([x + origin[0], y + origin[1], z + origin[2]]) + + +def ring(cx, cy, w, d): + return [(cx, cy), (cx + w, cy), (cx + w, cy + d), (cx, cy + d)] + + +class FakeClient: + """Stands in for Geotorget: canned buildings, canned LiDAR.""" + + def __init__(self, buildings_payload, points): + self._buildings = buildings_payload + self._points = points + self.searched = [] + + def search(self, collection, bbox, limit=20): + self.searched.append(collection) + if collection == "byggnad-nedladdning-vektor": + return [StacItem(f["id"], collection, {}, None, raw=f) for f in self._buildings] + return [StacItem("lidar-1", collection, {"data": "http://x/tile.laz"}, None, raw={})] + + def download(self, url): + return b"laz-bytes" + + +@pytest.fixture +def scene(monkeypatch): + """A house and a neighbour that share a ridge orientation. + + The neighbour is the point: an azimuth-180 plane is z = f(y) with no x term, + so it extends across the whole tile and the two buildings compete for each + other's returns unless the cloud is clipped first. + """ + e, n = sweref.wgs84_to_sweref99tm(*STOCKHOLM) + mine = np.vstack([ + roof_face(35, 180, 12, 6, (e, n, 0), seed=2), + roof_face(35, 0, 12, 6, (e, n + 6, 4.2), seed=3), + ]) + neighbour = np.vstack([ + roof_face(35, 180, 12, 6, (e + 40, n, 0), seed=4), + roof_face(35, 0, 12, 6, (e + 40, n + 6, 4.2), seed=5), + ]) + cloud = np.vstack([mine, neighbour]) + + buildings = [ + {"id": "mine", "geometry": {"type": "Polygon", + "coordinates": [[list(p) for p in ring(e, n, 12, 12)]]}, "properties": {}}, + {"id": "neighbour", "geometry": {"type": "Polygon", + "coordinates": [[list(p) for p in ring(e + 40, n, 12, 12)]]}, "properties": {}}, + ] + monkeypatch.setattr(pipeline, "load_points", lambda data: cloud) + return FakeClient(buildings, cloud), cloud, (e, n) + + +def test_derive_clips_to_the_picked_building(scene): + client, cloud, _ = scene + model = derive( + latitude=STOCKHOLM[0], longitude=STOCKHOLM[1], + credentials=Credentials("u", "t"), client=client, building_id="mine", + ) + + b = model["building"] + assert b["building_id"] == "mine" + assert b["returns_in_radius"] == len(cloud) + # Roughly half the tile is the neighbour's, and it must be gone. + assert b["returns_used"] < b["returns_in_radius"] * 0.6 + assert model["arrays"], "a clipped house still has a south roof" + + +def test_clipping_recovers_the_true_area_the_neighbour_would_have_stolen(scene): + """The measurable payoff: coplanar buildings stop eating each other.""" + _, cloud, (e, n) = scene + truth = 12 * 6 / math.cos(math.radians(35)) + + whole_tile = [p for p in segment_roof(cloud, point_density=8.0) + if abs(p.azimuth_deg - 180) < 5] + clipped = [p for p in segment_roof(clip_to_footprint(cloud, ring(e, n, 12, 12)), + point_density=8.0) + if abs(p.azimuth_deg - 180) < 5] + + assert len(clipped) == 1, "one building has one south face" + assert clipped[0].area_m2 == pytest.approx(truth, rel=0.10) + # Unclipped, the two south faces are coplanar and merge into one oversized + # segment, so the site's own roof cannot be measured at all. + assert len(whole_tile) != 1 or whole_tile[0].area_m2 > truth * 1.5 + + +def test_derive_without_a_building_id_does_not_search_for_buildings(scene): + client, _, _ = scene + model = derive( + latitude=STOCKHOLM[0], longitude=STOCKHOLM[1], + credentials=Credentials("u", "t"), client=client, + ) + assert "byggnad-nedladdning-vektor" not in client.searched + assert model["building"] is None + + +def test_derive_rejects_a_building_id_that_is_not_there(scene): + client, _, _ = scene + with pytest.raises(RoofModelError) as exc: + derive( + latitude=STOCKHOLM[0], longitude=STOCKHOLM[1], + credentials=Credentials("u", "t"), client=client, building_id="not-a-building", + ) + assert "not found" in str(exc.value) + + +def test_derive_explains_a_footprint_with_no_returns_on_it(monkeypatch, scene): + """A building newer than the scan is a real case and needs a real message.""" + client, _, (e, n) = scene + empty = np.empty((0, 3)) + monkeypatch.setattr(pipeline, "load_points", lambda data: np.vstack([ + roof_face(35, 180, 12, 6, (e + 400, n, 0), seed=9)])) + + with pytest.raises(RoofModelError) as exc: + derive( + latitude=STOCKHOLM[0], longitude=STOCKHOLM[1], + credentials=Credentials("u", "t"), client=client, building_id="mine", + ) + msg = str(exc.value) + assert "fall on building" in msg and "newer than the scan" in msg diff --git a/roofmodel/tests/test_geopackage.py b/roofmodel/tests/test_geopackage.py new file mode 100644 index 00000000..54c49e42 --- /dev/null +++ b/roofmodel/tests/test_geopackage.py @@ -0,0 +1,201 @@ +"""Decoding GeoPackage, the format Lantmaeteriet ships building vectors in. + +The fixtures are built byte by byte from the published layouts rather than by +round-tripping the decoder, so a decoder that agrees with itself but not with +the standard still fails here. +""" + +from __future__ import annotations + +import os +import sqlite3 +import struct +import tempfile + +import pytest + +from ftw_roofmodel.geopackage import ( + GeoPackageError, + parse_geometry_blob, + read_features, +) + +SWEREF = 3006 + + +def wkb_polygon(rings, *, little=True, z=False): + """Standard WKB polygon, per OGC 06-103r4 clause 8.2.""" + e = "<" if little else ">" + code = 1003 if z else 3 + out = struct.pack("B", 1 if little else 0) + struct.pack(e + "I", code) + out += struct.pack(e + "I", len(rings)) + for ring in rings: + out += struct.pack(e + "I", len(ring)) + for point in ring: + out += struct.pack(e + "dd", point[0], point[1]) + if z: + out += struct.pack(e + "d", point[2] if len(point) > 2 else 0.0) + return out + + +def wkb_multipolygon(polygons, *, little=True): + e = "<" if little else ">" + out = struct.pack("B", 1 if little else 0) + struct.pack(e + "I", 6) + out += struct.pack(e + "I", len(polygons)) + for rings in polygons: + out += wkb_polygon(rings, little=little) + return out + + +def gpkg_blob(wkb, *, envelope=None, srs_id=SWEREF, little=True, empty=False): + """GeoPackage geometry BLOB, per OGC 12-128r19 clause 2.1.3. + + Header is magic(2) + version(1) + flags(1) + srs_id(4), then the envelope, + then the WKB. + """ + indicator = 0 if envelope is None else 1 + flags = (1 if little else 0) | (indicator << 1) | (0x10 if empty else 0) + e = "<" if little else ">" + header = b"GP" + bytes([0, flags]) + struct.pack(e + "i", srs_id) + assert len(header) == 8, "the GeoPackage header is 8 bytes before the envelope" + body = b"" + if envelope is not None: + body = struct.pack(e + "dddd", *envelope) + assert len(body) == 32, "an xy envelope is four doubles" + return header + body + wkb + + +SQUARE = [[(0.0, 0.0), (10.0, 0.0), (10.0, 10.0), (0.0, 10.0), (0.0, 0.0)]] + + +def test_reads_a_plain_polygon(): + geom = parse_geometry_blob(gpkg_blob(wkb_polygon(SQUARE))) + assert geom["type"] == "Polygon" + assert geom["coordinates"][0][0] == [0.0, 0.0] + assert geom["coordinates"][0][2] == [10.0, 10.0] + assert len(geom["coordinates"][0]) == 5 + + +def test_skips_the_envelope_when_one_is_present(): + """The envelope sits between the header and the WKB and must be stepped over.""" + with_env = parse_geometry_blob( + gpkg_blob(wkb_polygon(SQUARE), envelope=(0.0, 10.0, 0.0, 10.0)) + ) + without = parse_geometry_blob(gpkg_blob(wkb_polygon(SQUARE))) + assert with_env == without + + +def test_reads_big_endian_geometry(): + assert parse_geometry_blob( + gpkg_blob(wkb_polygon(SQUARE, little=False), little=False) + ) == parse_geometry_blob(gpkg_blob(wkb_polygon(SQUARE))) + + +def test_reads_3d_polygons_by_stepping_the_z(): + """Building footprints carry heights; the z must not shift the ring.""" + ring = [[(0.0, 0.0, 12.5), (10.0, 0.0, 12.5), (10.0, 10.0, 12.5), (0.0, 0.0, 12.5)]] + geom = parse_geometry_blob(gpkg_blob(wkb_polygon(ring, z=True))) + assert geom["coordinates"][0] == [[0.0, 0.0], [10.0, 0.0], [10.0, 10.0], [0.0, 0.0]] + + +def test_reads_a_multipolygon_as_several_rings(): + other = [[(20.0, 20.0), (30.0, 20.0), (30.0, 30.0), (20.0, 20.0)]] + geom = parse_geometry_blob(gpkg_blob(wkb_multipolygon([SQUARE, other]))) + assert geom["type"] == "MultiPolygon" + assert len(geom["coordinates"]) == 2 + + +def test_an_interior_ring_survives(): + """A courtyard is a second ring, and dropping it would inflate the roof.""" + hole = [(2.0, 2.0), (4.0, 2.0), (4.0, 4.0), (2.0, 2.0)] + geom = parse_geometry_blob(gpkg_blob(wkb_polygon(SQUARE + [hole]))) + assert len(geom["coordinates"]) == 2 + + +def test_empty_geometry_is_none_not_an_error(): + assert parse_geometry_blob(gpkg_blob(b"", empty=True)) is None + + +def test_rejects_a_blob_that_is_not_a_geopackage_geometry(): + with pytest.raises(GeoPackageError, match="magic"): + parse_geometry_blob(b"XX" + bytes(20)) + + +def test_refuses_extended_geometry_rather_than_guessing(): + blob = bytearray(gpkg_blob(wkb_polygon(SQUARE))) + blob[3] |= 0x20 + with pytest.raises(GeoPackageError, match="extended"): + parse_geometry_blob(bytes(blob)) + + +def test_refuses_a_non_polygon_rather_than_mis_clipping(): + point = struct.pack("B", 1) + struct.pack(" arrays ------------------------------------------------------ + +def test_north_facing_pitched_roofs_are_not_proposed(): + """At Swedish latitudes a north pitch yields too little to be worth adding + to an operator's config.""" + planes = [ + RoofPlane(tilt_deg=35, azimuth_deg=0, area_m2=60, point_count=200, mean_height_m=6), + RoofPlane(tilt_deg=35, azimuth_deg=180, area_m2=60, point_count=200, mean_height_m=6), + ] + arrays = planes_to_arrays(planes) + assert len(arrays) == 1 + assert arrays[0].azimuth_deg == 180 + + +def test_flat_roofs_are_kept_regardless_of_building_orientation(): + planes = [RoofPlane(tilt_deg=1.0, azimuth_deg=180, area_m2=90, point_count=300, mean_height_m=9)] + assert len(planes_to_arrays(planes)) == 1 + + +def test_tiny_faces_are_dropped(): + """Dormers and porches are real surfaces but not candidate arrays.""" + planes = [RoofPlane(tilt_deg=35, azimuth_deg=180, area_m2=3.0, point_count=50, mean_height_m=5)] + assert planes_to_arrays(planes) == [] + + +def test_arrays_get_readable_and_unique_names(): + planes = [ + RoofPlane(tilt_deg=35, azimuth_deg=180, area_m2=60, point_count=200, mean_height_m=6), + RoofPlane(tilt_deg=35, azimuth_deg=182, area_m2=40, point_count=150, mean_height_m=6), + RoofPlane(tilt_deg=35, azimuth_deg=270, area_m2=30, point_count=120, mean_height_m=6), + ] + arrays = planes_to_arrays(planes) + names = [a.name for a in arrays] + assert len(set(names)) == len(names), names + assert names[0] == "Roof south" + assert "Roof west" in names + + +def test_array_json_matches_the_config_field_names(): + """The document pre-fills weather.pv_arrays, so the keys must line up.""" + planes = [RoofPlane(tilt_deg=35, azimuth_deg=180, area_m2=60, point_count=200, mean_height_m=6)] + payload = planes_to_arrays(planes)[0].to_json() + assert set(payload) >= {"name", "kwp", "tilt_deg", "azimuth_deg"} + assert payload["kwp"] > 0 + + +# --- end to end ------------------------------------------------------------ + +def _patched_points(monkeypatch, cloud): + monkeypatch.setattr(pipeline, "load_points", lambda data: cloud) + + +def test_derive_produces_a_versioned_document(monkeypatch): + gable = np.vstack([ + make_plane(tilt_deg=35, azimuth_deg=180, width=12, depth=6, seed=11), + make_plane(tilt_deg=35, azimuth_deg=0, width=12, depth=6, origin=(0, 6, -4.2), seed=12), + ]) + _patched_points(monkeypatch, gable) + session = FakeSession(search={"features": [feature()]}, asset=b"laz-bytes") + client = GeotorgetClient(CREDS, session=session) + + model = pipeline.derive( + latitude=59.33, longitude=18.07, credentials=CREDS, client=client, + now=dt.datetime(2026, 7, 31, tzinfo=dt.timezone.utc), + ) + + assert model["schema_version"] == pipeline.SCHEMA_VERSION + assert model["source"]["provider"] == "lantmateriet" + assert model["captured_at_ms"] > 0 + assert model["derived_at_ms"] == 1785456000000 + # The gable's north face is dropped, so exactly the south face survives. + assert len(model["arrays"]) == 1 + south = model["arrays"][0] + assert south["azimuth_deg"] == pytest.approx(180.0, abs=3.0) + assert south["tilt_deg"] == pytest.approx(35.0, abs=3.0) + assert south["kwp"] > 0 + # Must be JSON-serialisable for the subprocess contract. + json.dumps(model) + + +def test_derive_searches_the_projected_bbox(monkeypatch): + _patched_points(monkeypatch, make_plane(tilt_deg=35, azimuth_deg=180)) + session = FakeSession(search={"features": [feature()]}, asset=b"x") + pipeline.derive( + latitude=59.33, longitude=18.07, credentials=CREDS, + client=GeotorgetClient(CREDS, session=session), radius_m=40.0, + ) + (_, body), = session.posts + min_e, min_n, max_e, max_n = body["bbox"] + # Stockholm in SWEREF 99 TM, and an 80 m box give or take projection bow. + assert 600_000 < min_e < 700_000, body["bbox"] + assert 6_500_000 < min_n < 6_600_000, body["bbox"] + assert 75 < (max_e - min_e) < 90 + assert 75 < (max_n - min_n) < 90 + + +def test_derive_outside_sweden_says_so(monkeypatch): + """No tiles come back for a site Lantmaeteriet does not cover.""" + session = FakeSession(search={"features": []}) + with pytest.raises(RoofModelError) as exc: + pipeline.derive( + latitude=-33.87, longitude=151.21, credentials=CREDS, + client=GeotorgetClient(CREDS, session=session), + ) + assert "Sweden only" in str(exc.value) + + +def test_derive_reports_unknown_capture_date_without_failing(monkeypatch): + _patched_points(monkeypatch, make_plane(tilt_deg=35, azimuth_deg=180)) + session = FakeSession(search={"features": [feature(datetime_value=None)]}, asset=b"x") + model = pipeline.derive( + latitude=59.33, longitude=18.07, credentials=CREDS, + client=GeotorgetClient(CREDS, session=session), + ) + assert model["captured_at_ms"] is None + assert model["source"]["dataset_datetime"] is None + assert model["arrays"], "a missing provenance date must not block the model" + + +# --- optional shading pass ------------------------------------------------- + +def test_derive_without_vostok_reports_shading_as_not_evaluated(monkeypatch): + """The default path: no vostok configured, so shading is unknown -- and + unknown must not be recorded as 1.0, which would read as 'unobstructed'.""" + _patched_points(monkeypatch, make_plane(tilt_deg=35, azimuth_deg=180)) + session = FakeSession(search={"features": [feature()]}, asset=b"x") + model = pipeline.derive( + latitude=59.33, longitude=18.07, credentials=CREDS, + client=GeotorgetClient(CREDS, session=session), + ) + assert model["shading"]["evaluated"] is False + for arr in model["arrays"]: + assert "shading_factor" not in arr + + +def test_derive_with_missing_vostok_degrades_cleanly(monkeypatch): + """A configured-but-absent binary must not fail the whole derive: the roof + geometry is still perfectly good without a shading number.""" + _patched_points(monkeypatch, make_plane(tilt_deg=35, azimuth_deg=180)) + session = FakeSession(search={"features": [feature()]}, asset=b"x") + model = pipeline.derive( + latitude=59.33, longitude=18.07, credentials=CREDS, + client=GeotorgetClient(CREDS, session=session), + vostok_binary="definitely-not-installed-vostok", + ) + assert model["arrays"], "geometry must survive an absent shading tool" + assert model["shading"]["evaluated"] is False + assert "not installed" in model["shading"]["reason"] + + +def test_derive_attaches_shading_factors_when_vostok_runs(monkeypatch): + """Shading factors must land on the surviving arrays, matched by segment id + rather than by list position -- planes_to_arrays drops faces, so the two + lists do not line up.""" + from ftw_roofmodel.shading import ShadingResult + + # Pin the planes rather than relying on how a synthetic cloud happens to + # segment: this test is about the factor-to-array mapping, and segmentation + # itself is covered in test_segment.py. + # + # Plane 0 faces north and is dropped by planes_to_arrays, so the single + # surviving array is seg-1. A positional mapping would hand it index 0's + # factor; the two values are far apart so that mistake cannot pass. + pinned = [ + RoofPlane(tilt_deg=35, azimuth_deg=0, area_m2=120.0, point_count=900, mean_height_m=6.0), + RoofPlane(tilt_deg=35, azimuth_deg=180, area_m2=80.0, point_count=600, mean_height_m=6.0), + ] + _patched_points(monkeypatch, make_plane(tilt_deg=35, azimuth_deg=180)) + monkeypatch.setattr(pipeline, "segment_roof", lambda *a, **k: pinned) + monkeypatch.setattr( + "ftw_roofmodel.shading.compute_shading", + lambda *a, **k: ShadingResult({0: 0.10, 1: 0.62}, evaluated=True), + ) + session = FakeSession(search={"features": [feature()]}, asset=b"x") + model = pipeline.derive( + latitude=59.33, longitude=18.07, credentials=CREDS, + client=GeotorgetClient(CREDS, session=session), + vostok_binary="stub-vostok", + ) + assert model["shading"]["evaluated"] is True + assert len(model["arrays"]) == 1, model["arrays"] + kept = model["arrays"][0] + assert kept["segment_id"] == "seg-1", "premise: the north face should sort first" + assert kept["shading_factor"] == pytest.approx(0.62), ( + "factor must be matched by segment id, not by list position" + ) + json.dumps(model) diff --git a/roofmodel/tests/test_segment.py b/roofmodel/tests/test_segment.py new file mode 100644 index 00000000..e8ec3545 --- /dev/null +++ b/roofmodel/tests/test_segment.py @@ -0,0 +1,188 @@ +"""Roof segmentation tests. + +Every case is a synthetic roof whose tilt, azimuth and area are known exactly by +construction, so the assertions check recovered geometry against ground truth +rather than against a previous run's output. +""" + +import math + +import numpy as np +import pytest + +from ftw_roofmodel.segment import RoofPlane, segment_roof + + +def make_plane( + *, + tilt_deg: float, + azimuth_deg: float, + width: float = 10.0, + depth: float = 8.0, + origin=(0.0, 0.0, 0.0), + density: float = 8.0, + noise_m: float = 0.0, + seed: int = 1, +) -> np.ndarray: + """Sample a rectangular sloped surface with a known tilt and azimuth. + + The surface is generated by taking a horizontal grid and raising it along + the downslope direction, which is the inverse of what segment_roof does, so + the test never shares an implementation with the code under test. + """ + rng = np.random.default_rng(seed) + n = max(int(width * depth * density), 60) + x = rng.uniform(0, width, n) + y = rng.uniform(0, depth, n) + + # Aspect points the way the surface faces; height falls off along it. + az = math.radians(azimuth_deg) + east_dir, north_dir = math.sin(az), math.cos(az) + slope = math.tan(math.radians(tilt_deg)) + z = -(x * east_dir + y * north_dir) * slope + + if noise_m: + z = z + rng.normal(0.0, noise_m, n) + + return np.column_stack([x + origin[0], y + origin[1], z + origin[2]]) + + +@pytest.mark.parametrize( + "tilt,azimuth", + [ + (35.0, 180.0), # classic south-facing pitched roof + (30.0, 90.0), # east + (30.0, 270.0), # west + (45.0, 0.0), # north + (20.0, 225.0), # south-west + (60.0, 135.0), # steep south-east + ], +) +def test_recovers_known_tilt_and_azimuth(tilt, azimuth): + cloud = make_plane(tilt_deg=tilt, azimuth_deg=azimuth) + planes = segment_roof(cloud, point_density=8.0) + assert planes, "expected at least one plane" + got = planes[0] + assert got.tilt_deg == pytest.approx(tilt, abs=1.5) + # Compare on the circle so 359.5 vs 0.5 is a 1-degree error, not 359. + delta = abs((got.azimuth_deg - azimuth + 180.0) % 360.0 - 180.0) + assert delta < 2.0, f"azimuth {got.azimuth_deg} vs {azimuth}" + + +def test_flat_roof_reports_flat_and_does_not_invent_an_azimuth(): + """A horizontal surface has no meaningful aspect: its normal is vertical, + so the horizontal component is pure noise and could point anywhere.""" + cloud = make_plane(tilt_deg=0.0, azimuth_deg=180.0, noise_m=0.02) + planes = segment_roof(cloud, point_density=8.0) + assert planes + assert planes[0].tilt_deg < 5.0 + assert planes[0].azimuth_deg == 180.0 + + +def test_gable_roof_splits_into_two_opposing_faces(): + """The canonical case: one ridge, two faces 180 degrees apart.""" + south = make_plane(tilt_deg=35, azimuth_deg=180, depth=6, origin=(0, 0, 0), seed=2) + north = make_plane(tilt_deg=35, azimuth_deg=0, depth=6, origin=(0, 6, -4.2), seed=3) + planes = segment_roof(np.vstack([south, north]), point_density=8.0) + + assert len(planes) >= 2, f"expected two faces, got {len(planes)}" + azimuths = sorted(p.azimuth_deg for p in planes[:2]) + opposed = abs((azimuths[1] - azimuths[0]) - 180.0) + assert opposed < 5.0, f"faces should oppose, got {azimuths}" + for p in planes[:2]: + assert p.tilt_deg == pytest.approx(35.0, abs=2.0) + + +def test_two_separate_buildings_on_the_same_plane_are_split(): + """Two roofs with identical pitch and aspect satisfy one plane equation. + RANSAC alone would merge them; the DBSCAN pass is what separates them.""" + a = make_plane(tilt_deg=30, azimuth_deg=180, origin=(0, 0, 0), seed=4) + b = make_plane(tilt_deg=30, azimuth_deg=180, origin=(60, 0, 0), seed=5) + planes = segment_roof(np.vstack([a, b]), point_density=8.0, cluster_eps_m=1.5) + + assert len(planes) >= 2, "spatially disjoint faces must not be merged" + for p in planes[:2]: + assert p.tilt_deg == pytest.approx(30.0, abs=2.0) + + +def test_area_is_the_sloped_area_not_the_footprint(): + """A 10x8 footprint at 60 degrees covers 80 / cos(60) = 160 m2 of roof.""" + cloud = make_plane(tilt_deg=60.0, azimuth_deg=180.0, width=10, depth=8, density=10) + planes = segment_roof(cloud, point_density=10.0) + assert planes + assert planes[0].area_m2 == pytest.approx(160.0, rel=0.15) + + +def test_area_of_a_flat_roof_matches_its_footprint(): + cloud = make_plane(tilt_deg=0.0, azimuth_deg=180.0, width=12, depth=10, density=10) + planes = segment_roof(cloud, point_density=10.0) + assert planes + assert planes[0].area_m2 == pytest.approx(120.0, rel=0.15) + + +def test_survives_realistic_measurement_noise(): + """1-2 pts/m2 airborne LiDAR carries several centimetres of range noise.""" + cloud = make_plane(tilt_deg=35.0, azimuth_deg=180.0, noise_m=0.05, density=6) + planes = segment_roof(cloud, point_density=6.0, threshold_m=0.3) + assert planes + assert planes[0].tilt_deg == pytest.approx(35.0, abs=3.0) + + +def test_walls_are_rejected(): + """A near-vertical surface is a wall or a dormer cheek, not a roof.""" + cloud = make_plane(tilt_deg=88.0, azimuth_deg=180.0) + planes = segment_roof(cloud, point_density=8.0) + assert all(p.tilt_deg <= 80.0 for p in planes) + + +def test_noise_alone_yields_nothing_believable(): + """A handful of scattered returns must not become a roof.""" + rng = np.random.default_rng(7) + cloud = rng.uniform(0, 10, size=(25, 3)) + assert segment_roof(cloud, point_density=8.0) == [] + + +def test_is_deterministic(): + """Re-running a derive must not shuffle an operator's arrays.""" + cloud = make_plane(tilt_deg=35, azimuth_deg=180) + a = segment_roof(cloud, point_density=8.0) + b = segment_roof(cloud, point_density=8.0) + assert a == b + + +def test_planes_are_ordered_by_area(): + big = make_plane(tilt_deg=30, azimuth_deg=180, width=14, depth=12, origin=(0, 0, 0), seed=8) + small = make_plane(tilt_deg=30, azimuth_deg=180, width=5, depth=4, origin=(70, 0, 0), seed=9) + planes = segment_roof(np.vstack([big, small]), point_density=8.0) + assert len(planes) >= 2 + assert planes[0].area_m2 >= planes[1].area_m2 + + +def test_rejects_malformed_input(): + with pytest.raises(ValueError): + segment_roof(np.zeros((10, 2))) + + +def test_kwp_derives_from_area_with_packing_losses(): + """A 100 m2 face cannot carry 100 m2 of modules.""" + plane = RoofPlane(tilt_deg=35, azimuth_deg=180, area_m2=100.0, point_count=800, mean_height_m=6.0) + kwp = plane.kwp() + assert kwp == pytest.approx(100 * 0.70 * 200 / 1000.0) + assert kwp < 100 * 200 / 1000.0, "packing factor must reduce the raw area" + + +def test_kwp_scales_with_area(): + a = RoofPlane(tilt_deg=35, azimuth_deg=180, area_m2=50.0, point_count=1, mean_height_m=1.0) + b = RoofPlane(tilt_deg=35, azimuth_deg=180, area_m2=100.0, point_count=1, mean_height_m=1.0) + assert b.kwp() == pytest.approx(2 * a.kwp()) + + +def test_azimuth_never_reports_360(): + """A north face fits at ~359.97 deg, which rounds to 360.0 -- the same + direction as 0 but an out-of-range-looking value for anything consuming it. + Caught by the demo, so pinned here.""" + cloud = make_plane(tilt_deg=35, azimuth_deg=0, seed=21) + planes = segment_roof(cloud, point_density=8.0) + assert planes + for p in planes: + assert 0.0 <= p.azimuth_deg < 360.0, p.azimuth_deg diff --git a/roofmodel/tests/test_shading.py b/roofmodel/tests/test_shading.py new file mode 100644 index 00000000..2061feb5 --- /dev/null +++ b/roofmodel/tests/test_shading.py @@ -0,0 +1,272 @@ +"""vostok shading-integration tests. + +vostok is GPL-3.0 and FTW never bundles or installs it, so it is not present in +CI or on a developer machine by default. These tests therefore drive a stub that +implements vostok's documented contract -- read a .sol, write the query points +back with an appended Wh/m2/day column -- which exercises everything on our side +of the boundary: config generation, the two-run ratio, output parsing, clamping, +and every way the tool can be missing or broken. +""" + +import os +import stat +import sys +import textwrap + +import numpy as np +import pytest + +from ftw_roofmodel import shading +from ftw_roofmodel.segment import RoofPlane + + +def plane(tilt=35.0, azimuth=180.0, height=6.0, area=60.0): + return RoofPlane( + tilt_deg=tilt, azimuth_deg=azimuth, area_m2=area, + point_count=400, mean_height_m=height, + ) + + +def cloud(n=500, height=6.0, seed=0): + rng = np.random.default_rng(seed) + return np.column_stack([ + rng.uniform(0, 12, n), + rng.uniform(0, 8, n), + rng.normal(height, 0.2, n), + ]) + + +def make_stub(tmp_path, *, shaded_value=600.0, open_value=1000.0, exit_code=0, write_output=True): + """A stand-in vostok: parses the .sol, emits a constant potential. + + Returning different values for the shadowed and open-sky runs is what makes + the ratio testable without a real solar model. + """ + script = tmp_path / "stub_vostok.py" + script.write_text(textwrap.dedent(f""" + import sys + cfg = {{}} + for line in open(sys.argv[1]): + if '=' in line: + k, v = line.split('=', 1) + cfg[k.strip()] = v.strip() + if {exit_code} != 0: + sys.stderr.write('stub failure\\n') + sys.exit({exit_code}) + if not {write_output}: + sys.exit(0) + value = {shaded_value} if cfg.get('shadowing') == 'true' else {open_value} + with open(cfg['output'], 'w') as out: + for line in open(cfg['query_cloud']): + parts = line.split() + if not parts: + continue + out.write(' '.join(parts) + f' {{value}}\\n') + """).strip(), encoding="ascii") + + # A launcher so the binary looks like an ordinary executable to the adapter. + if os.name == "nt": + launcher = tmp_path / "vostok.bat" + launcher.write_text(f'@echo off\r\n"{sys.executable}" "{script}" %1\r\n', encoding="ascii") + else: + launcher = tmp_path / "vostok" + launcher.write_text(f'#!/bin/sh\nexec "{sys.executable}" "{script}" "$1"\n', encoding="ascii") + launcher.chmod(launcher.stat().st_mode | stat.S_IEXEC) + return str(launcher) + + +# --- absence is the normal case -------------------------------------------- + +def test_absent_vostok_is_not_an_error(): + """FTW never installs it, so absence must degrade rather than fail.""" + result = shading.compute_shading( + cloud(), [plane()], latitude=59.33, longitude=18.07, + binary="definitely-not-installed-vostok", + ) + assert result.evaluated is False + assert result.factors == {} + + +def test_absent_vostok_explains_itself_without_assuming_no_shading(): + result = shading.compute_shading( + cloud(), [plane()], latitude=59.33, longitude=18.07, + binary="definitely-not-installed-vostok", + ) + assert "not installed" in result.reason + assert "GPL" in result.reason, "the reason should say why FTW does not ship it" + # An unevaluated plane must read as no derate, not as fully shaded. + assert result.factor_for(0) == 1.0 + + +def test_available_reports_missing_binary(): + assert shading.available("definitely-not-installed-vostok") is False + assert shading.find_vostok("definitely-not-installed-vostok") is None + + +def test_no_planes_short_circuits(): + result = shading.compute_shading(cloud(), [], latitude=59.33, longitude=18.07) + assert result.evaluated is False + assert "no roof planes" in result.reason + + +# --- the ratio ------------------------------------------------------------- + +def test_shading_factor_is_the_ratio_of_the_two_runs(tmp_path): + """600 Wh shadowed against 1000 Wh open sky is a factor of 0.6.""" + binary = make_stub(tmp_path, shaded_value=600.0, open_value=1000.0) + result = shading.compute_shading( + cloud(), [plane()], latitude=59.33, longitude=18.07, binary=binary, + ) + assert result.evaluated is True + assert result.factor_for(0) == pytest.approx(0.6, abs=0.01) + + +def test_unobstructed_roof_scores_one(tmp_path): + binary = make_stub(tmp_path, shaded_value=1000.0, open_value=1000.0) + result = shading.compute_shading( + cloud(), [plane()], latitude=59.33, longitude=18.07, binary=binary, + ) + assert result.factor_for(0) == pytest.approx(1.0, abs=0.01) + + +def test_fully_shaded_roof_scores_zero(tmp_path): + binary = make_stub(tmp_path, shaded_value=0.0, open_value=1000.0) + result = shading.compute_shading( + cloud(), [plane()], latitude=59.33, longitude=18.07, binary=binary, + ) + assert result.factor_for(0) == 0.0 + + +def test_factor_is_clamped_to_one(tmp_path): + """Shadowing can only remove irradiance. A ratio above 1 means the two runs + are not comparable, so it must clamp rather than report a gain.""" + binary = make_stub(tmp_path, shaded_value=1500.0, open_value=1000.0) + result = shading.compute_shading( + cloud(), [plane()], latitude=59.33, longitude=18.07, binary=binary, + ) + assert result.factor_for(0) == 1.0 + + +def test_each_plane_gets_its_own_factor(tmp_path): + binary = make_stub(tmp_path) + planes = [plane(height=6.0), plane(azimuth=0.0, height=6.0)] + result = shading.compute_shading( + cloud(), planes, latitude=59.33, longitude=18.07, binary=binary, + ) + assert result.evaluated + assert set(result.factors) <= {0, 1} + for f in result.factors.values(): + assert 0.0 <= f <= 1.0 + + +# --- failure handling ------------------------------------------------------ + +def test_nonzero_exit_is_reported_not_silently_assumed_unshaded(tmp_path): + binary = make_stub(tmp_path, exit_code=3) + result = shading.compute_shading( + cloud(), [plane()], latitude=59.33, longitude=18.07, binary=binary, + ) + assert result.evaluated is False + assert "no usable results" in result.reason + + +def test_missing_output_file_is_handled(tmp_path): + binary = make_stub(tmp_path, write_output=False) + result = shading.compute_shading( + cloud(), [plane()], latitude=59.33, longitude=18.07, binary=binary, + ) + assert result.evaluated is False + + +def test_a_plane_with_too_few_points_is_skipped_not_fatal(tmp_path): + binary = make_stub(tmp_path) + planes = [plane(height=6.0), plane(height=900.0)] # second matches nothing + result = shading.compute_shading( + cloud(), planes, latitude=59.33, longitude=18.07, binary=binary, + ) + assert result.evaluated is True + assert 1 not in result.factors + assert result.factor_for(1) == 1.0, "a skipped plane must not be derated" + + +# --- the .sol contract ----------------------------------------------------- + +def test_query_points_carry_normals(tmp_path): + """vostok requires a normal on every query point; without one it refuses.""" + captured = {} + + def fake_run(binary, sol_path, timeout_s): + cfg = dict( + line.split("=", 1) for line in open(sol_path) if "=" in line + ) + cfg = {k.strip(): v.strip() for k, v in cfg.items()} + captured.update(cfg) + with open(cfg["query_cloud"]) as fh: + captured["query_first_line"] = fh.readline().split() + with open(cfg["output"], "w") as out: + out.write("0 0 0 0 0 1 500\n") + + import ftw_roofmodel.shading as mod + original = mod._run + mod._run = fake_run + try: + mod.compute_shading( + cloud(), [plane(tilt=35, azimuth=180)], + latitude=59.33, longitude=18.07, binary=sys.executable, + ) + finally: + mod._run = original + + assert len(captured["query_first_line"]) == 6, "expected x y z nx ny nz" + nx, ny, nz = (float(v) for v in captured["query_first_line"][3:]) + # A 35-degree south-facing plane: normal tips south (negative northing) and + # still points up. + assert nz == pytest.approx(np.cos(np.radians(35)), abs=1e-3) + assert ny < 0 + assert abs(nx) < 1e-6 + + +def test_sol_carries_the_site_location(tmp_path): + captured = {} + + def fake_run(binary, sol_path, timeout_s): + cfg = {} + for line in open(sol_path): + if "=" in line: + k, v = line.split("=", 1) + cfg[k.strip()] = v.strip() + captured.setdefault("runs", []).append(cfg) + with open(cfg["output"], "w") as out: + out.write("0 0 0 0 0 1 500\n") + + import ftw_roofmodel.shading as mod + original = mod._run + mod._run = fake_run + try: + mod.compute_shading( + cloud(), [plane()], latitude=59.33, longitude=18.07, + binary=sys.executable, voxel_size_m=0.5, + ) + finally: + mod._run = original + + runs = captured["runs"] + assert len(runs) == 2, "one shadowed run and one open-sky run per plane" + assert {r["shadowing"] for r in runs} == {"true", "false"} + for r in runs: + assert float(r["latitude"]) == pytest.approx(59.33) + assert float(r["longitude"]) == pytest.approx(18.07) + assert float(r["voxel_size"]) == 0.5 + + +def test_normal_matches_the_segmenter_convention(): + """Round-trip: the normal handed to vostok must describe the same surface + the segmenter measured, or the shading would be computed for a different + roof than the one being derated.""" + from ftw_roofmodel.segment import _tilt_azimuth + + for tilt, azimuth in [(35, 180), (20, 90), (45, 270), (60, 135)]: + n = shading._plane_normal(tilt, azimuth) + back_tilt, back_az = _tilt_azimuth(np.array(n)) + assert back_tilt == pytest.approx(tilt, abs=0.01) + assert back_az == pytest.approx(azimuth, abs=0.01) diff --git a/roofmodel/tests/test_source_formats.py b/roofmodel/tests/test_source_formats.py new file mode 100644 index 00000000..1bcd434c --- /dev/null +++ b/roofmodel/tests/test_source_formats.py @@ -0,0 +1,235 @@ +"""The two Lantmaeteriet products end to end, in the formats they ship in. + +Both are STAC APIs over one client and one credential. What differs is the +payload: *Byggnad Nedladdning, vektor* is GeoPackage, *Laserdata Nedladdning, +Skog* is LAZ organised as COPC. +""" + +from __future__ import annotations + +import json +import math + +import numpy as np +import pytest + +from ftw_roofmodel import pipeline, sweref +from ftw_roofmodel.buildings import BuildingLookupError, search_buildings +from ftw_roofmodel.geotorget import ( + MEDIA_COPC, + MEDIA_GEOJSON, + MEDIA_GEOPACKAGE, + MEDIA_LAZ, + Asset, + Credentials, + StacItem, +) +from ftw_roofmodel.pipeline import derive +from ftw_roofmodel.pointcloud import PointCloudError + +from .test_geopackage import build_gpkg, gpkg_blob, wkb_polygon + +STOCKHOLM = (59.33, 18.07) +E, N = sweref.wgs84_to_sweref99tm(*STOCKHOLM) + + +def square(cx, cy, w, d): + return [[(cx, cy), (cx + w, cy), (cx + w, cy + d), (cx, cy + d), (cx, cy)]] + + +def roof_face(tilt, azimuth, w, d, origin, density=8, noise=0.04, seed=1): + rng = np.random.default_rng(seed) + n = int(w * d * density) + x = rng.uniform(0, w, n) + y = rng.uniform(0, d, n) + az = math.radians(azimuth) + s = math.tan(math.radians(tilt)) + z = -(x * math.sin(az) + y * math.cos(az)) * s + rng.normal(0, noise, n) + return np.column_stack([x + origin[0], y + origin[1], z + origin[2]]) + + +HOUSE = np.vstack([ + roof_face(35, 180, 12, 6, (E, N, 0), seed=2), + roof_face(35, 0, 12, 6, (E, N + 6, 4.2), seed=3), +]) + + +class FakeClient: + """A Geotorget stand-in that serves typed assets.""" + + def __init__(self, building_asset=None, lidar_asset=None, payloads=None, + session=None): + self._building_asset = building_asset + self._lidar_asset = lidar_asset + self._payloads = payloads or {} + self.session = session + self.downloaded: list[str] = [] + + def search(self, collection, bbox, limit=20): + if collection == "byggnad-nedladdning-vektor": + if self._building_asset is None: + return [] + return [StacItem("tile-b", collection, {"data": self._building_asset}, None, + raw={"id": "tile-b"})] + if self._lidar_asset is None: + return [] + return [StacItem("tile-l", collection, {"data": self._lidar_asset}, None, + raw={"id": "tile-l"})] + + def download(self, url): + self.downloaded.append(url) + return self._payloads[url] + + +def a_geopackage_of_two_buildings(): + return build_gpkg([ + ("house-1", "Bostad", gpkg_blob(wkb_polygon(square(E - 6, N - 3, 12, 12)))), + ("shed-1", "Komplementbyggnad", + gpkg_blob(wkb_polygon(square(E + 40, N + 40, 8, 8)))), + ]) + + +def test_buildings_come_out_of_a_geopackage_asset(): + """The normal Byggnad-vektor path: the item points at a .gpkg tile.""" + url = "https://api.lantmateriet.se/x/byggnad.gpkg" + client = FakeClient( + building_asset=Asset(url, MEDIA_GEOPACKAGE), + payloads={url: a_geopackage_of_two_buildings()}, + ) + found = search_buildings(client, latitude=STOCKHOLM[0], longitude=STOCKHOLM[1]) + + assert [b.building_id for b in found] == ["house-1", "shed-1"], "nearest first" + assert found[0].area_m2 == pytest.approx(144.0, rel=0.01) + assert found[0].properties["andamal"] == "Bostad" + assert client.downloaded == [url] + + +def test_buildings_come_out_of_a_geojson_asset_too(): + """Some catalogues publish GeoJSON; both are handled by media type.""" + url = "https://api.lantmateriet.se/x/byggnad.geojson" + doc = { + "type": "FeatureCollection", + "features": [{ + "type": "Feature", + "id": "gj-1", + "geometry": {"type": "Polygon", + "coordinates": [[list(p) for p in square(E - 6, N - 3, 12, 12)[0]]]}, + "properties": {"andamal": "Bostad"}, + }], + } + client = FakeClient( + building_asset=Asset(url, MEDIA_GEOJSON), + payloads={url: json.dumps(doc).encode()}, + ) + found = search_buildings(client, latitude=STOCKHOLM[0], longitude=STOCKHOLM[1]) + assert [b.building_id for b in found] == ["gj-1"] + + +def test_an_unreadable_building_tile_says_what_is_wrong(): + url = "https://api.lantmateriet.se/x/byggnad.gpkg" + client = FakeClient( + building_asset=Asset(url, MEDIA_GEOPACKAGE), + payloads={url: b"this is not a database"}, + ) + with pytest.raises(BuildingLookupError, match="could not be read"): + search_buildings(client, latitude=STOCKHOLM[0], longitude=STOCKHOLM[1]) + + +def test_an_item_with_no_usable_asset_is_skipped_not_fatal(): + client = FakeClient(building_asset=Asset("https://x/preview.png", "image/png")) + with pytest.raises(BuildingLookupError, match="no building footprints"): + search_buildings(client, latitude=STOCKHOLM[0], longitude=STOCKHOLM[1]) + + +# --- LiDAR: COPC windowing ------------------------------------------------- + + +@pytest.fixture +def lidar_scene(monkeypatch): + """A client whose LiDAR asset is COPC, with both read paths instrumented.""" + gpkg_url = "https://api.lantmateriet.se/x/byggnad.gpkg" + copc_url = "https://api.lantmateriet.se/x/tile.copc.laz" + client = FakeClient( + building_asset=Asset(gpkg_url, MEDIA_GEOPACKAGE), + lidar_asset=Asset(copc_url, MEDIA_COPC), + payloads={gpkg_url: a_geopackage_of_two_buildings(), copc_url: b"laz-bytes"}, + session=object(), + ) + calls: dict[str, object] = {} + + def fake_window(session, url, bounds, timeout=60.0): + calls["window"] = {"url": url, "bounds": bounds} + return HOUSE + + monkeypatch.setattr(pipeline.pointcloud, "read_copc_window", fake_window) + monkeypatch.setattr(pipeline, "load_points", lambda data: HOUSE) + return client, calls, copc_url + + +def test_a_picked_building_is_read_as_a_copc_window(lidar_scene): + """The payoff: a footprint costs a window, not a 2.5 km tile.""" + client, calls, copc_url = lidar_scene + model = derive( + latitude=STOCKHOLM[0], longitude=STOCKHOLM[1], + credentials=Credentials("u", "t"), client=client, building_id="house-1", + ) + + assert model["source"]["fetch"] == "copc-window" + assert calls["window"]["url"] == copc_url + min_x, min_y, max_x, max_y = calls["window"]["bounds"] + # The footprint is 12 m square around the site, plus the eaves buffer. + assert min_x == pytest.approx(E - 7, abs=0.5) + assert max_x == pytest.approx(E + 7, abs=0.5) + assert copc_url not in client.downloaded, "the tile was never downloaded whole" + + +def test_without_a_picked_building_the_tile_is_read_whole(lidar_scene): + """No footprint means no window to ask for.""" + client, calls, copc_url = lidar_scene + model = derive( + latitude=STOCKHOLM[0], longitude=STOCKHOLM[1], + credentials=Credentials("u", "t"), client=client, + ) + assert model["source"]["fetch"] == "whole-tile" + assert "window" not in calls + assert copc_url in client.downloaded + + +def test_a_host_without_range_support_falls_back_to_the_whole_tile(monkeypatch, lidar_scene): + """Best-effort: the operator still gets their roof, just slower.""" + client, _, copc_url = lidar_scene + + def refuses(session, url, bounds, timeout=60.0): + raise PointCloudError("the LiDAR host does not support range requests") + + monkeypatch.setattr(pipeline.pointcloud, "read_copc_window", refuses) + model = derive( + latitude=STOCKHOLM[0], longitude=STOCKHOLM[1], + credentials=Credentials("u", "t"), client=client, building_id="house-1", + ) + assert model["source"]["fetch"] == "whole-tile" + assert copc_url in client.downloaded + assert model["arrays"], "and the roof still comes out" + + +def test_a_plain_laz_asset_is_read_whole_even_with_a_building(monkeypatch): + """Only COPC can be windowed; plain LAZ has no index to seek with.""" + gpkg_url = "https://api.lantmateriet.se/x/byggnad.gpkg" + laz_url = "https://api.lantmateriet.se/x/tile.laz" + client = FakeClient( + building_asset=Asset(gpkg_url, MEDIA_GEOPACKAGE), + lidar_asset=Asset(laz_url, MEDIA_LAZ), + payloads={gpkg_url: a_geopackage_of_two_buildings(), laz_url: b"laz-bytes"}, + session=object(), + ) + monkeypatch.setattr(pipeline, "load_points", lambda data: HOUSE) + called = [] + monkeypatch.setattr(pipeline.pointcloud, "read_copc_window", + lambda *a, **k: called.append(1)) + + model = derive( + latitude=STOCKHOLM[0], longitude=STOCKHOLM[1], + credentials=Credentials("u", "t"), client=client, building_id="house-1", + ) + assert model["source"]["fetch"] == "whole-tile" + assert not called diff --git a/roofmodel/tests/test_sweref.py b/roofmodel/tests/test_sweref.py new file mode 100644 index 00000000..f6f20a4b --- /dev/null +++ b/roofmodel/tests/test_sweref.py @@ -0,0 +1,123 @@ +"""SWEREF 99 TM projection tests. + +The projection has exact analytic properties at the central meridian and the +equator, which pin the parameters without needing a published coordinate table. +Round-trip accuracy then pins the series expansion itself. +""" + +import math + +import pytest + +from ftw_roofmodel.sweref import ( + bbox_wgs84_to_sweref99tm, + metre_box_around, + sweref99tm_to_wgs84, + wgs84_to_sweref99tm, +) + + +@pytest.mark.parametrize("lat", [0.0, 55.0, 59.33, 63.0, 69.0]) +def test_central_meridian_maps_to_false_easting(lat): + """On the central meridian easting is exactly the false easting, by + definition. Any error in scale, ellipsoid or meridian shows up here.""" + _, easting = wgs84_to_sweref99tm(lat, 15.0) + assert easting == pytest.approx(500000.0, abs=1e-6) + + +def test_equator_on_central_meridian_is_the_projection_origin(): + northing, easting = wgs84_to_sweref99tm(0.0, 15.0) + assert northing == pytest.approx(0.0, abs=1e-6) + assert easting == pytest.approx(500000.0, abs=1e-6) + + +@pytest.mark.parametrize( + "lat,lon", + [ + (55.34, 13.15), # Smygehuk, southernmost Sweden + (59.33, 18.07), # Stockholm + (57.71, 11.97), # Gothenburg + (63.83, 20.26), # Umea + (67.86, 20.23), # Kiruna + (69.06, 20.55), # Treriksroeset, northernmost + ], +) +def test_round_trip_is_sub_millimetre(lat, lon): + """Project and unproject across the full extent of Sweden.""" + n, e = wgs84_to_sweref99tm(lat, lon) + back_lat, back_lon = sweref99tm_to_wgs84(n, e) + # 1e-8 degrees is about 1 mm of latitude. + assert back_lat == pytest.approx(lat, abs=1e-8) + assert back_lon == pytest.approx(lon, abs=1e-8) + + +def test_coordinates_land_in_the_expected_range_for_sweden(): + """Sanity-check magnitudes: Swedish SWEREF 99 TM eastings sit inside + 260-920 km and northings inside 6100-7700 km. A transposed or + wrongly-scaled result would fall far outside.""" + n, e = wgs84_to_sweref99tm(59.33, 18.07) + assert 260_000 < e < 920_000, e + assert 6_100_000 < n < 7_700_000, n + + +def test_east_of_the_meridian_increases_easting(): + _, west = wgs84_to_sweref99tm(59.33, 14.0) + _, east = wgs84_to_sweref99tm(59.33, 16.0) + assert west < 500000.0 < east + + +def test_north_increases_northing(): + south, _ = wgs84_to_sweref99tm(55.0, 15.0) + north, _ = wgs84_to_sweref99tm(65.0, 15.0) + assert north > south + + +def test_one_degree_of_latitude_is_about_111_km(): + a, _ = wgs84_to_sweref99tm(59.0, 15.0) + b, _ = wgs84_to_sweref99tm(60.0, 15.0) + # Scaled by k0 = 0.9996 on the central meridian. + assert 110_000 < (b - a) < 112_000 + + +def test_bbox_uses_all_four_corners(): + """The projection is not axis-aligned, so the projected box must be at + least as large as the one implied by the two diagonal corners.""" + min_lat, min_lon, max_lat, max_lon = 59.0, 17.0, 60.0, 19.0 + min_e, min_n, max_e, max_n = bbox_wgs84_to_sweref99tm( + min_lat, min_lon, max_lat, max_lon + ) + sw_n, sw_e = wgs84_to_sweref99tm(min_lat, min_lon) + ne_n, ne_e = wgs84_to_sweref99tm(max_lat, max_lon) + assert min_e <= sw_e and min_n <= sw_n + assert max_e >= ne_e and max_n >= ne_n + assert min_e < max_e and min_n < max_n + + +def test_metre_box_is_square_on_the_ground(): + """A 100 m box must measure 200 m on both axes regardless of latitude -- + the whole reason it is built in projected metres rather than in degrees.""" + for lat in (55.5, 59.33, 68.0): + south, west, north, east = metre_box_around(lat, 15.0, 100.0) + sn, se = wgs84_to_sweref99tm(south, west) + nn, ne = wgs84_to_sweref99tm(north, east) + assert (ne - se) == pytest.approx(200.0, abs=0.5) + assert (nn - sn) == pytest.approx(200.0, abs=0.5) + + +def test_metre_box_brackets_its_centre(): + south, west, north, east = metre_box_around(59.33, 18.07, 50.0) + assert south < 59.33 < north + assert west < 18.07 < east + + +def test_degree_box_would_have_been_wrong_at_high_latitude(): + """Guards the reason metre_box_around exists: a fixed degree offset gives + wildly different ground distances at Malmoe and Kiruna, so anyone tempted to + simplify it back to degrees has to defeat this test first.""" + span = [] + for lat in (55.5, 68.0): + # What a naive 0.001-degree longitude offset would span, in metres. + _, e0 = wgs84_to_sweref99tm(lat, 15.0) + _, e1 = wgs84_to_sweref99tm(lat, 15.001) + span.append(e1 - e0) + assert span[0] / span[1] > 1.5, span diff --git a/web/components/ftw-bar-chart.js b/web/components/ftw-bar-chart.js index d7465a6b..6f0144d8 100644 --- a/web/components/ftw-bar-chart.js +++ b/web/components/ftw-bar-chart.js @@ -188,6 +188,26 @@ class FtwBarChart extends FtwElement { opacity: 0.85; pointer-events: none; } + /* Optional dashed overlay line (e.g. STRÅNG-expected PV vs the + produced bars). An SVG stretched to fill .bar-area, plotted in a + 0..100 viewBox so it needs no pixel math against the CSS grid; + non-scaling-stroke keeps the dash crisp despite the stretch. */ + .overlay-line { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + pointer-events: none; + overflow: visible; + } + .overlay-line polyline { + fill: none; + stroke: var(--ftw-overlay-color, var(--amber, #f59e0b)); + stroke-width: 1.5; + stroke-dasharray: 4 3; + vector-effect: non-scaling-stroke; + opacity: 0.9; + } `; static get observedAttributes() { @@ -197,6 +217,7 @@ class FtwBarChart extends FtwElement { constructor() { super(); this._data = []; + this._overlay = null; } attributeChangedCallback() { this.update(); } @@ -209,6 +230,16 @@ class FtwBarChart extends FtwElement { } get data() { return this._data; } + // Optional dashed overlay line, index-aligned with .data. Shape: + // { values: (number|null)[], color?: string } + // values[i] is plotted above column i on the SAME axis as the bars + // (nulls / non-finite entries break the line). Set null to remove. + set overlay(o) { + this._overlay = o && Array.isArray(o.values) ? o : null; + this.update(); + } + get overlay() { return this._overlay; } + render() { const accent = this.getAttribute("accent"); const height = this.getAttribute("chart-height"); @@ -262,6 +293,17 @@ class FtwBarChart extends FtwElement { } const avg = count > 0 ? sum / count : 0; + // Fold overlay values into the axis max so the expected line and the + // produced bars share one scale (an expected level above the tallest + // bar must still fit in-frame). + const overlayVals = this._overlay ? this._overlay.values : null; + if (overlayVals) { + for (const ov of overlayVals) { + const n = Number(ov); + if (isFinite(n) && n > max) max = n; + } + } + const colsSvg = this._data.map((d) => { const v = Number(d.value) || 0; // 2% floor keeps tiny-but-nonzero values visible; gate on v>0 so @@ -300,10 +342,34 @@ class FtwBarChart extends FtwElement { `title="average ${display}">`; } + // Dashed overlay line (expected series). Plotted in a 0..100 viewBox + // stretched to fill .bar-area: x centers each column, y is the value + // as a percentage of the shared max (inverted — SVG y grows downward). + let overlayLine = ""; + if (overlayVals && max > 0) { + const n = this._data.length; + const pts = []; + for (let i = 0; i < n; i++) { + const val = Number(overlayVals[i]); + if (!isFinite(val)) continue; + const x = n > 1 ? (i + 0.5) / n * 100 : 50; + const y = Math.max(0, Math.min(100, 100 - (val / max) * 100)); + pts.push(`${x.toFixed(2)},${y.toFixed(2)}`); + } + if (pts.length >= 2) { + const color = this._overlay.color; + if (color) this.style.setProperty("--ftw-overlay-color", color); + overlayLine = + ``; + } + } + return `
-
${colsSvg}${avgOverlay}
+
${colsSvg}${avgOverlay}${overlayLine}
${lblsSvg}
diff --git a/web/components/ftw-history-card.js b/web/components/ftw-history-card.js index ed659092..1e1393c0 100644 --- a/web/components/ftw-history-card.js +++ b/web/components/ftw-history-card.js @@ -28,7 +28,7 @@ import { FtwElement, ftwDebugDelay } from "./ftw-element.js"; import { apiFetch } from "./api-fetch.js"; -import "./ftw-bar-chart.js"; +import "./ftw-bar-chart.js?v=strang1"; const FIELD_BY_METRIC = { import: "import_wh", @@ -82,6 +82,36 @@ function fetchDailyEnergy(days) { return promise; } +// STRÅNG-based PV performance (expected-vs-actual). Only the "Produced" +// (metric="pv") tile fetches this, to overlay the weather-expected line. +// Tolerant: a disabled/absent service or any error resolves to +// { enabled:false } so the bars still render without an overlay. +const pvPerfFetchCache = new Map(); // days -> { at, data?, promise? } + +function fetchPVPerformance(days) { + const now = Date.now(); + const cached = pvPerfFetchCache.get(days); + if (cached && cached.data && now - cached.at < DAILY_CACHE_TTL_MS) { + return Promise.resolve(cached.data); + } + if (cached && cached.promise && now - cached.at < DAILY_CACHE_TTL_MS) { + return cached.promise; + } + const promise = apiFetch("/api/pv/performance?days=" + days) + .then((r) => (r.ok ? r.json() : { enabled: false })) + .then((resp) => { + const data = resp || { enabled: false }; + pvPerfFetchCache.set(days, { at: Date.now(), data }); + return data; + }) + .catch(() => { + pvPerfFetchCache.delete(days); + return { enabled: false }; + }); + pvPerfFetchCache.set(days, { at: now, promise }); + return promise; +} + class FtwHistoryCard extends FtwElement { static styles = ` :host { display: block; } @@ -205,6 +235,27 @@ class FtwHistoryCard extends FtwElement { margin-left: 6px; letter-spacing: 0; } + /* STRÅNG expected-vs-actual caption under the Produced chart. Hidden + until the pv tile has a performance overlay to describe. The dashed + swatch mirrors the overlay line so the legend reads at a glance. */ + .strang-note { + margin-top: 6px; + font-size: 0.72rem; + color: var(--fg-muted); + font-family: var(--mono); + display: flex; + align-items: center; + gap: 6px; + } + .strang-note[hidden] { display: none; } + .strang-note .swatch { + display: inline-block; + width: 16px; + height: 0; + border-top: 2px dashed #fcd34d; + flex: 0 0 auto; + } + .strang-note .pr { color: var(--fg-label); font-weight: 600; } @media (max-width: 900px) { .card-inner { padding: var(--card-pad-tight, 12px 14px); } } @@ -302,6 +353,7 @@ class FtwHistoryCard extends FtwElement {
— kWh
+ `; } @@ -309,6 +361,7 @@ class FtwHistoryCard extends FtwElement { afterRender() { this._chart = this.shadowRoot.querySelector('[data-role="chart"]'); this._totalEl = this.shadowRoot.querySelector('[data-role="total"]'); + this._strangEl = this.shadowRoot.querySelector('[data-role="strang"]'); this._toggleEl = this.shadowRoot.querySelector('.toggle'); if (this._chart) this._chart.setAttribute("accent", this._accent()); if (this._toggleEl) { @@ -387,6 +440,12 @@ class FtwHistoryCard extends FtwElement { this._totalEl.textContent = "— kWh"; } this._chart.data = data; + // Produced tile: overlay the STRÅNG expected line. Fetched + // separately so a disabled/slow scoring service never holds up + // the bars. Aligned to the same day buckets by ISO date. + if (metric === "pv") { + this._loadOverlay(days, buckets.map((b) => b.day), seq); + } }; // `?delay=N` — hold in the skeleton state for N ms after the // fetch resolves, for inspecting the loading→loaded transition. @@ -402,6 +461,58 @@ class FtwHistoryCard extends FtwElement { this._totalEl.textContent = "failed to load"; }); } + + // Fetch STRÅNG performance scores and overlay the expected-production + // line onto the Produced bars, aligned to `dayKeys` (ISO dates, same + // order as the bars). Hides the overlay + caption when scoring is + // unavailable. Guarded by `seq` so a stale response can't paint over a + // newer Week/Month selection. + _loadOverlay(days, dayKeys, seq) { + fetchPVPerformance(days) + .then((perf) => { + if (seq !== this._reqSeq || !this._chart) return; + if (!perf || perf.enabled === false || !Array.isArray(perf.items) || !perf.items.length) { + this._chart.overlay = null; + if (this._strangEl) this._strangEl.setAttribute("hidden", ""); + return; + } + const expByDay = new Map(); + for (const it of perf.items) { + if (it && it.day != null) expByDay.set(it.day, Number(it.expected_wh) || 0); + } + // kWh, index-aligned to the bars; null where no score exists so + // the dashed line breaks rather than dropping to zero. + const values = dayKeys.map((d) => { + const wh = expByDay.get(d); + return wh == null ? null : wh / 1000; + }); + const hasAny = values.some((v) => v != null); + this._chart.overlay = hasAny ? { values, color: "#fcd34d" } : null; + + if (this._strangEl) { + const pr = typeof perf.performance_ratio === "number" ? perf.performance_ratio : null; + const prTxt = pr != null ? `${Math.round(pr * 100)}% of expected` : ""; + // Only announce the calibration once it is actually being applied to + // the forward forecast — reporting a factor we ignore would mislead. + const cal = perf.calibration; + const calTxt = + cal && cal.applied && typeof cal.factor === "number" + ? `calibrating forecast ×${cal.factor.toFixed(2)}` + : ""; + this._strangEl.innerHTML = + ` expected (STRÅNG)` + + (prTxt ? " · " + prTxt : "") + + (calTxt ? " · " + calTxt : ""); + if (hasAny) this._strangEl.removeAttribute("hidden"); + else this._strangEl.setAttribute("hidden", ""); + } + }) + .catch(() => { + if (seq !== this._reqSeq || !this._chart) return; + this._chart.overlay = null; + if (this._strangEl) this._strangEl.setAttribute("hidden", ""); + }); + } } function fmtKwh(wh) { diff --git a/web/components/index.js b/web/components/index.js index ee2f11fe..9a6a895b 100644 --- a/web/components/index.js +++ b/web/components/index.js @@ -16,8 +16,8 @@ import "./ftw-battery-control.js?v=apifetch1"; import "./ftw-pv-control.js?v=apifetch1"; import "./ftw-price-chart.js?v=apiread3"; import "./ftw-energy-cake.js"; -import "./ftw-bar-chart.js"; -import "./ftw-history-card.js?v=apiread2"; +import "./ftw-bar-chart.js?v=strang1"; +import "./ftw-history-card.js?v=strang2"; import "./ftw-savings-card.js?v=apiread1"; import "./ftw-update-check.js?v=apifetch1"; import "./ftw-notif-status.js?v=apifetch1"; diff --git a/web/index.html b/web/index.html index fdc42703..854c96fb 100644 --- a/web/index.html +++ b/web/index.html @@ -5,10 +5,10 @@ FTW - +