Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions easysnowdata/stations/clients/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -320,10 +320,13 @@ requests) to record `sensor_durations` (`{3: ["D", "H"], 18: ["M"], …}`).
duration, and most of its ~560 snow stations are monthly courses. In November
2026, 145 had a daily snow sensor, ~340 only monthly ones and 74 only hourly
or event ones. If a duration search fails, the flags fall back to sensor
presence (with a warning) and `sensor_durations` is `None`. SWE and snow depth
outside 0–15 m are returned as `None` at every duration (DESIGN.md §3.6): the
hourly and event series of CDEC's wireless sensor networks carry values such
as −799,899 cm (G01) and 21,049 cm (DPK). The snow course
presence (with a warning) and `sensor_durations` is `None`. In the hourly and
event series, SWE and snow depth outside 0–15 m are returned as `None`
(DESIGN.md §3.6): CDEC's wireless sensor networks carry values such as
−799,899 cm (G01) and 21,049 cm (DPK). Daily and monthly series are returned
as CDEC publishes them, small negative readings included (0.3.7 nulled those
too). A response that arrives cut off is retried once, then raises
`CDECError`. The snow course
and pillow lists set `is_snow_course` and `is_snow_pillow`.

```python
Expand Down
34 changes: 29 additions & 5 deletions easysnowdata/stations/clients/cdec/cdec_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -297,11 +297,16 @@ def _sensor(
# (transform, emitted unit). Sensors not listed pass through unchanged
# with their registry output_units (already metric).
#: Plausibility bound for SWE and snow depth after conversion, cm (DESIGN.md
#: §3.6: 0–15 m). The hourly and event series of CDEC's wireless sensor
#: networks carry values far outside it (−799 899 cm at G01, 21 049 cm at
#: DPK, 5 894 cm at HEM; checked 2026-11-05).
#: §3.6: 0–15 m), applied to the hourly and event series only. Those, from
#: CDEC's wireless sensor networks, carry values far outside it (−799 899 cm
#: at G01, 21 049 cm at DPK, 5 894 cm at HEM; checked 2026-11-05). CDEC's
#: daily series is returned as published: its small negative readings
#: (sensor noise around zero, e.g. −7.6 cm depth at ADM, −18.7 cm SWE at
#: FLV) are part of archives built from it, and nulling them in 0.3.7 made
#: every refresh of those stations look like lost data.
_SNOW_MAX_CM = 1500.0
_SNOW_SENSORS_CM = frozenset({3, 18, 82})
_BOUNDED_DURATIONS = frozenset({"H", "E"})

_SENSOR_CONVERSIONS: dict[int, Any] = {
3: lambda v: v * 2.54, # in → cm
Expand Down Expand Up @@ -825,6 +830,7 @@ def _get_data_cdec(
# Batch by station to keep URL lengths reasonable.
batch_size = 20
results_by_id: dict[str, dict] = {}
bounded = str(duration).upper() in _BOUNDED_DURATIONS

for i in range(0, len(station_ids), batch_size):
batch = station_ids[i : i + batch_size]
Expand Down Expand Up @@ -868,6 +874,7 @@ def _get_data_cdec(
value = fv
if (
value is not None
and bounded
and sensor_num in _SNOW_SENSORS_CM
and not 0.0 <= value <= _SNOW_MAX_CM
):
Expand Down Expand Up @@ -1118,8 +1125,25 @@ def _get_json(
url: str,
params: dict[str, str] | None = None,
) -> Any:
resp = self._request("GET", url, params=params)
return resp.json()
# A long full-record response sometimes arrives cut off (a 62 MB
# body ending mid-object, 2026-11-06). Retry once, then raise
# CDECError like any other failed request.
for attempt in (1, 2):
resp = self._request("GET", url, params=params)
try:
return resp.json()
except ValueError as exc:
if attempt == 2:
raise CDECError(
f"Unreadable JSON from {url} "
f"({len(resp.content):,} bytes): {exc}"
) from exc
logger.warning(
"Unreadable JSON from %s (%d bytes); retrying once: %s",
url,
len(resp.content),
exc,
)

def _request(
self,
Expand Down
36 changes: 36 additions & 0 deletions tests/stations/test_clients_offline.py
Original file line number Diff line number Diff line change
Expand Up @@ -1263,3 +1263,39 @@ def test_cdec_snow_values_outside_0_to_15_m_are_nulled(monkeypatch):
assert [v["value"] for v in by_sensor[18]] == [None, None, None, 0.0, 152.4]
# the limit is for snow only: -4 °F is a real temperature
assert by_sensor[30][0]["value"] == pytest.approx(-20.0)


def test_cdec_daily_series_is_returned_as_published(monkeypatch):
"""The bound is for the sub-daily series: CDEC's daily records carry
small negatives (-3 in at ADM) that archives built from them keep."""
client = CDECClient()
rows = [
{"stationId": "ADM", "SENSOR_NUM": 18, "date": "2024-1-1 0:00", "value": v}
for v in (-3.0, 0.0, 60.0)
]
monkeypatch.setattr(client, "_get_json", lambda url, params=None: rows)
out = client._get_data_cdec(["ADM"], [18], "D", "2024-01-01", "2024-01-01")
assert [v["value"] for v in out[0]["data"][0]["values"]] == [-7.62, 0.0, 152.4]


def test_cdec_cut_off_json_is_retried_once_then_raises(monkeypatch):
client = CDECClient()

class _Resp:
def __init__(self, body):
self.content = body

def json(self):
import json

return json.loads(self.content)

bodies = [b'[{"a": 1}, {"b"', b'[{"a": 1}]']
monkeypatch.setattr(
client, "_request", lambda m, u, params=None: _Resp(bodies.pop(0))
)
assert client._get_json("u") == [{"a": 1}]

monkeypatch.setattr(client, "_request", lambda m, u, params=None: _Resp(b'[{"b"'))
with pytest.raises(CDECError, match="Unreadable JSON"):
client._get_json("u")
Loading