diff --git a/easysnowdata/stations/clients/README.md b/easysnowdata/stations/clients/README.md index 99b4e19d..bf4ce6ce 100644 --- a/easysnowdata/stations/clients/README.md +++ b/easysnowdata/stations/clients/README.md @@ -320,10 +320,13 @@ requests) to record `sensor_durations` (`{3: ["D", "H"], 18: ["M"], …}`). duration, and most of its ~560 snow stations are monthly courses. In November 2026, 145 had a daily snow sensor, ~340 only monthly ones and 74 only hourly or event ones. If a duration search fails, the flags fall back to sensor -presence (with a warning) and `sensor_durations` is `None`. SWE and snow depth -outside 0–15 m are returned as `None` at every duration (DESIGN.md §3.6): the -hourly and event series of CDEC's wireless sensor networks carry values such -as −799,899 cm (G01) and 21,049 cm (DPK). The snow course +presence (with a warning) and `sensor_durations` is `None`. In the hourly and +event series, SWE and snow depth outside 0–15 m are returned as `None` +(DESIGN.md §3.6): CDEC's wireless sensor networks carry values such as +−799,899 cm (G01) and 21,049 cm (DPK). Daily and monthly series are returned +as CDEC publishes them, small negative readings included (0.3.7 nulled those +too). A response that arrives cut off is retried once, then raises +`CDECError`. The snow course and pillow lists set `is_snow_course` and `is_snow_pillow`. ```python diff --git a/easysnowdata/stations/clients/cdec/cdec_client.py b/easysnowdata/stations/clients/cdec/cdec_client.py index 30083372..c7e41867 100644 --- a/easysnowdata/stations/clients/cdec/cdec_client.py +++ b/easysnowdata/stations/clients/cdec/cdec_client.py @@ -297,11 +297,16 @@ def _sensor( # (transform, emitted unit). Sensors not listed pass through unchanged # with their registry output_units (already metric). #: Plausibility bound for SWE and snow depth after conversion, cm (DESIGN.md -#: §3.6: 0–15 m). The hourly and event series of CDEC's wireless sensor -#: networks carry values far outside it (−799 899 cm at G01, 21 049 cm at -#: DPK, 5 894 cm at HEM; checked 2026-11-05). +#: §3.6: 0–15 m), applied to the hourly and event series only. Those, from +#: CDEC's wireless sensor networks, carry values far outside it (−799 899 cm +#: at G01, 21 049 cm at DPK, 5 894 cm at HEM; checked 2026-11-05). CDEC's +#: daily series is returned as published: its small negative readings +#: (sensor noise around zero, e.g. −7.6 cm depth at ADM, −18.7 cm SWE at +#: FLV) are part of archives built from it, and nulling them in 0.3.7 made +#: every refresh of those stations look like lost data. _SNOW_MAX_CM = 1500.0 _SNOW_SENSORS_CM = frozenset({3, 18, 82}) +_BOUNDED_DURATIONS = frozenset({"H", "E"}) _SENSOR_CONVERSIONS: dict[int, Any] = { 3: lambda v: v * 2.54, # in → cm @@ -825,6 +830,7 @@ def _get_data_cdec( # Batch by station to keep URL lengths reasonable. batch_size = 20 results_by_id: dict[str, dict] = {} + bounded = str(duration).upper() in _BOUNDED_DURATIONS for i in range(0, len(station_ids), batch_size): batch = station_ids[i : i + batch_size] @@ -868,6 +874,7 @@ def _get_data_cdec( value = fv if ( value is not None + and bounded and sensor_num in _SNOW_SENSORS_CM and not 0.0 <= value <= _SNOW_MAX_CM ): @@ -1118,8 +1125,25 @@ def _get_json( url: str, params: dict[str, str] | None = None, ) -> Any: - resp = self._request("GET", url, params=params) - return resp.json() + # A long full-record response sometimes arrives cut off (a 62 MB + # body ending mid-object, 2026-11-06). Retry once, then raise + # CDECError like any other failed request. + for attempt in (1, 2): + resp = self._request("GET", url, params=params) + try: + return resp.json() + except ValueError as exc: + if attempt == 2: + raise CDECError( + f"Unreadable JSON from {url} " + f"({len(resp.content):,} bytes): {exc}" + ) from exc + logger.warning( + "Unreadable JSON from %s (%d bytes); retrying once: %s", + url, + len(resp.content), + exc, + ) def _request( self, diff --git a/tests/stations/test_clients_offline.py b/tests/stations/test_clients_offline.py index ebbd81e7..c7cb42eb 100644 --- a/tests/stations/test_clients_offline.py +++ b/tests/stations/test_clients_offline.py @@ -1263,3 +1263,39 @@ def test_cdec_snow_values_outside_0_to_15_m_are_nulled(monkeypatch): assert [v["value"] for v in by_sensor[18]] == [None, None, None, 0.0, 152.4] # the limit is for snow only: -4 °F is a real temperature assert by_sensor[30][0]["value"] == pytest.approx(-20.0) + + +def test_cdec_daily_series_is_returned_as_published(monkeypatch): + """The bound is for the sub-daily series: CDEC's daily records carry + small negatives (-3 in at ADM) that archives built from them keep.""" + client = CDECClient() + rows = [ + {"stationId": "ADM", "SENSOR_NUM": 18, "date": "2024-1-1 0:00", "value": v} + for v in (-3.0, 0.0, 60.0) + ] + monkeypatch.setattr(client, "_get_json", lambda url, params=None: rows) + out = client._get_data_cdec(["ADM"], [18], "D", "2024-01-01", "2024-01-01") + assert [v["value"] for v in out[0]["data"][0]["values"]] == [-7.62, 0.0, 152.4] + + +def test_cdec_cut_off_json_is_retried_once_then_raises(monkeypatch): + client = CDECClient() + + class _Resp: + def __init__(self, body): + self.content = body + + def json(self): + import json + + return json.loads(self.content) + + bodies = [b'[{"a": 1}, {"b"', b'[{"a": 1}]'] + monkeypatch.setattr( + client, "_request", lambda m, u, params=None: _Resp(bodies.pop(0)) + ) + assert client._get_json("u") == [{"a": 1}] + + monkeypatch.setattr(client, "_request", lambda m, u, params=None: _Resp(b'[{"b"')) + with pytest.raises(CDECError, match="Unreadable JSON"): + client._get_json("u")