From b7fca3d11f876f819928fd8c9cc40819dfccfee2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 19:04:00 +0000 Subject: [PATCH 1/2] fix(cdec): bound only sub-daily snow values; retry cut-off JSON once 0.3.7 nulled SWE and snow depth outside 0-15 m at every duration. CDEC's daily series carries small negative readings (sensor noise around zero: 723 at ADM down to -7.6 cm, 231 SWE values at FLV down to -18.7 cm), so every refresh of those stations returned fewer values than archives built from the same series, and global_snow_networks' shrink guard refused the update for ~17 stations on 2026-11-06. The bound now applies to the hourly and event series it was meant for; daily and monthly series are returned as CDEC publishes them, as in 0.3.6. A full-record JSONDataServlet response also arrived cut off (62 MB, ending mid-object) and escaped as requests' JSONDecodeError. _get_json now retries once, then raises CDECError. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015LSmHDJN224tLPVAUrvQTE --- easysnowdata/stations/clients/README.md | 11 ++++--- .../stations/clients/cdec/cdec_client.py | 32 +++++++++++++++--- tests/stations/test_clients_offline.py | 33 +++++++++++++++++++ 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/easysnowdata/stations/clients/README.md b/easysnowdata/stations/clients/README.md index 99b4e19d..bf4ce6ce 100644 --- a/easysnowdata/stations/clients/README.md +++ b/easysnowdata/stations/clients/README.md @@ -320,10 +320,13 @@ requests) to record `sensor_durations` (`{3: ["D", "H"], 18: ["M"], …}`). duration, and most of its ~560 snow stations are monthly courses. In November 2026, 145 had a daily snow sensor, ~340 only monthly ones and 74 only hourly or event ones. If a duration search fails, the flags fall back to sensor -presence (with a warning) and `sensor_durations` is `None`. SWE and snow depth -outside 0–15 m are returned as `None` at every duration (DESIGN.md §3.6): the -hourly and event series of CDEC's wireless sensor networks carry values such -as −799,899 cm (G01) and 21,049 cm (DPK). The snow course +presence (with a warning) and `sensor_durations` is `None`. In the hourly and +event series, SWE and snow depth outside 0–15 m are returned as `None` +(DESIGN.md §3.6): CDEC's wireless sensor networks carry values such as +−799,899 cm (G01) and 21,049 cm (DPK). Daily and monthly series are returned +as CDEC publishes them, small negative readings included (0.3.7 nulled those +too). A response that arrives cut off is retried once, then raises +`CDECError`. The snow course and pillow lists set `is_snow_course` and `is_snow_pillow`. ```python diff --git a/easysnowdata/stations/clients/cdec/cdec_client.py b/easysnowdata/stations/clients/cdec/cdec_client.py index 30083372..dbd6608a 100644 --- a/easysnowdata/stations/clients/cdec/cdec_client.py +++ b/easysnowdata/stations/clients/cdec/cdec_client.py @@ -297,11 +297,16 @@ def _sensor( # (transform, emitted unit). Sensors not listed pass through unchanged # with their registry output_units (already metric). #: Plausibility bound for SWE and snow depth after conversion, cm (DESIGN.md -#: §3.6: 0–15 m). The hourly and event series of CDEC's wireless sensor -#: networks carry values far outside it (−799 899 cm at G01, 21 049 cm at -#: DPK, 5 894 cm at HEM; checked 2026-11-05). +#: §3.6: 0–15 m), applied to the hourly and event series only. Those, from +#: CDEC's wireless sensor networks, carry values far outside it (−799 899 cm +#: at G01, 21 049 cm at DPK, 5 894 cm at HEM; checked 2026-11-05). CDEC's +#: daily series is returned as published: its small negative readings +#: (sensor noise around zero, e.g. −7.6 cm depth at ADM, −18.7 cm SWE at +#: FLV) are part of archives built from it, and nulling them in 0.3.7 made +#: every refresh of those stations look like lost data. _SNOW_MAX_CM = 1500.0 _SNOW_SENSORS_CM = frozenset({3, 18, 82}) +_BOUNDED_DURATIONS = frozenset({"H", "E"}) _SENSOR_CONVERSIONS: dict[int, Any] = { 3: lambda v: v * 2.54, # in → cm @@ -825,6 +830,7 @@ def _get_data_cdec( # Batch by station to keep URL lengths reasonable. batch_size = 20 results_by_id: dict[str, dict] = {} + bounded = str(duration).upper() in _BOUNDED_DURATIONS for i in range(0, len(station_ids), batch_size): batch = station_ids[i : i + batch_size] @@ -868,6 +874,7 @@ def _get_data_cdec( value = fv if ( value is not None + and bounded and sensor_num in _SNOW_SENSORS_CM and not 0.0 <= value <= _SNOW_MAX_CM ): @@ -1118,8 +1125,23 @@ def _get_json( url: str, params: dict[str, str] | None = None, ) -> Any: - resp = self._request("GET", url, params=params) - return resp.json() + # A long full-record response sometimes arrives cut off (a 62 MB + # body ending mid-object, 2026-11-06). Retry once, then raise + # CDECError like any other failed request. + for attempt in (1, 2): + resp = self._request("GET", url, params=params) + try: + return resp.json() + except ValueError as exc: + if attempt == 2: + raise CDECError( + f"Unreadable JSON from {url} " + f"({len(resp.content):,} bytes): {exc}" + ) from exc + logger.warning( + "Unreadable JSON from %s (%d bytes); retrying once: %s", + url, len(resp.content), exc, + ) def _request( self, diff --git a/tests/stations/test_clients_offline.py b/tests/stations/test_clients_offline.py index ebbd81e7..dd189eaa 100644 --- a/tests/stations/test_clients_offline.py +++ b/tests/stations/test_clients_offline.py @@ -1263,3 +1263,36 @@ def test_cdec_snow_values_outside_0_to_15_m_are_nulled(monkeypatch): assert [v["value"] for v in by_sensor[18]] == [None, None, None, 0.0, 152.4] # the limit is for snow only: -4 °F is a real temperature assert by_sensor[30][0]["value"] == pytest.approx(-20.0) + + +def test_cdec_daily_series_is_returned_as_published(monkeypatch): + """The bound is for the sub-daily series: CDEC's daily records carry + small negatives (-3 in at ADM) that archives built from them keep.""" + client = CDECClient() + rows = [ + {"stationId": "ADM", "SENSOR_NUM": 18, "date": "2024-1-1 0:00", "value": v} + for v in (-3.0, 0.0, 60.0) + ] + monkeypatch.setattr(client, "_get_json", lambda url, params=None: rows) + out = client._get_data_cdec(["ADM"], [18], "D", "2024-01-01", "2024-01-01") + assert [v["value"] for v in out[0]["data"][0]["values"]] == [-7.62, 0.0, 152.4] + + +def test_cdec_cut_off_json_is_retried_once_then_raises(monkeypatch): + client = CDECClient() + + class _Resp: + def __init__(self, body): + self.content = body + + def json(self): + import json + return json.loads(self.content) + + bodies = [b'[{"a": 1}, {"b"', b'[{"a": 1}]'] + monkeypatch.setattr(client, "_request", lambda m, u, params=None: _Resp(bodies.pop(0))) + assert client._get_json("u") == [{"a": 1}] + + monkeypatch.setattr(client, "_request", lambda m, u, params=None: _Resp(b'[{"b"')) + with pytest.raises(CDECError, match="Unreadable JSON"): + client._get_json("u") From d2f5e85c39dcb3b3d23e59787b076001b42d71cc Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 19:09:17 +0000 Subject: [PATCH 2/2] style: ruff format Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015LSmHDJN224tLPVAUrvQTE --- easysnowdata/stations/clients/cdec/cdec_client.py | 4 +++- tests/stations/test_clients_offline.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/easysnowdata/stations/clients/cdec/cdec_client.py b/easysnowdata/stations/clients/cdec/cdec_client.py index dbd6608a..c7e41867 100644 --- a/easysnowdata/stations/clients/cdec/cdec_client.py +++ b/easysnowdata/stations/clients/cdec/cdec_client.py @@ -1140,7 +1140,9 @@ def _get_json( ) from exc logger.warning( "Unreadable JSON from %s (%d bytes); retrying once: %s", - url, len(resp.content), exc, + url, + len(resp.content), + exc, ) def _request( diff --git a/tests/stations/test_clients_offline.py b/tests/stations/test_clients_offline.py index dd189eaa..c7cb42eb 100644 --- a/tests/stations/test_clients_offline.py +++ b/tests/stations/test_clients_offline.py @@ -1287,10 +1287,13 @@ def __init__(self, body): def json(self): import json + return json.loads(self.content) bodies = [b'[{"a": 1}, {"b"', b'[{"a": 1}]'] - monkeypatch.setattr(client, "_request", lambda m, u, params=None: _Resp(bodies.pop(0))) + monkeypatch.setattr( + client, "_request", lambda m, u, params=None: _Resp(bodies.pop(0)) + ) assert client._get_json("u") == [{"a": 1}] monkeypatch.setattr(client, "_request", lambda m, u, params=None: _Resp(b'[{"b"'))