From 117e5a2a610832fc0a48a5624c7993983e6a9ec5 Mon Sep 17 00:00:00 2001 From: Simon Osborne Date: Fri, 1 May 2026 09:34:49 +0100 Subject: [PATCH 01/11] Expand list of Cardington near-surface variables --- .../meta/observations/rose-meta.conf | 36 +++ .../cset_workflow/rose-suite.conf.example | 6 +- src/CSET/loaders/__init__.py | 2 + src/CSET/loaders/cardington.py | 236 ++++++++++++++++++ src/CSET/loaders/timeseries.py | 25 -- src/CSET/operators/_colorbar_definition.json | 6 +- src/CSET/operators/constraints.py | 46 +++- src/CSET/operators/misc.py | 181 ++++++++++++++ src/CSET/operators/read.py | 40 ++- src/CSET/operators/wind.py | 70 ++++++ ..._latent_heat_single_point_time_series.yaml | 60 +++++ ...ensible_heat_single_point_time_series.yaml | 43 ++++ .../cardington_single_point_time_series.yaml | 58 +++++ ...rdington_surface_pressure_time_series.yaml | 58 +++++ ...ngton_surface_temperature_time_series.yaml | 54 ++++ ..._temperature_single_point_time_series.yaml | 61 +++++ ...n_visibility_single_point_time_series.yaml | 52 ++++ ...on_wind_gust_single_point_time_series.yaml | 50 ++++ ...dington_wind_single_point_time_series.yaml | 86 +++++++ ...rdington_air_single_point_time_series.yaml | 61 +++++ ..._temperature_single_point_time_series.yaml | 2 +- ...ton_humidity_single_point_time_series.yaml | 58 +++++ ...ive_humidity_single_point_time_series.yaml | 0 23 files changed, 1253 insertions(+), 38 deletions(-) create mode 100644 src/CSET/loaders/cardington.py create mode 100644 src/CSET/recipes/surface_fields/cardington/cardington_latent_heat_single_point_time_series.yaml create mode 100644 src/CSET/recipes/surface_fields/cardington/cardington_sensible_heat_single_point_time_series.yaml create mode 100644 src/CSET/recipes/surface_fields/cardington/cardington_single_point_time_series.yaml create mode 100644 src/CSET/recipes/surface_fields/cardington/cardington_surface_pressure_time_series.yaml create mode 100644 src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml create mode 100644 src/CSET/recipes/surface_fields/cardington/cardington_temperature_single_point_time_series.yaml create mode 100644 src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml create mode 100644 src/CSET/recipes/surface_fields/cardington/cardington_wind_gust_single_point_time_series.yaml create mode 100644 src/CSET/recipes/surface_fields/cardington/cardington_wind_single_point_time_series.yaml create mode 100644 src/CSET/recipes/surface_fields/cardington_air_single_point_time_series.yaml rename src/CSET/recipes/surface_fields/{cardington => }/cardington_air_temperature_single_point_time_series.yaml (97%) create mode 100644 src/CSET/recipes/surface_fields/cardington_humidity_single_point_time_series.yaml rename src/CSET/recipes/surface_fields/{cardington => }/cardington_relative_humidity_single_point_time_series.yaml (100%) diff --git a/src/CSET/cset_workflow/meta/observations/rose-meta.conf b/src/CSET/cset_workflow/meta/observations/rose-meta.conf index 1bb00e125..6366bedda 100644 --- a/src/CSET/cset_workflow/meta/observations/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/observations/rose-meta.conf @@ -110,3 +110,39 @@ description=Create Cardington diagnostic variable timeseries type=python_boolean compulsory=true sort-key=0card2 + +# Temperature profile timeseries for air variables at various heights +[template variables=CARDINGTON_AIR_SINGLE_POINT_TIME_SERIES] +ns=Observations/Cardington +title=Cardington multi-height temperature air timeseries +description=Create Cardington diagnostic variable timeseries +type=python_boolean +compulsory=true +sort-key=0card3 + +# Generic timeseries for air variables at various heights +[template variables=CARDINGTON_SINGLE_POINT_TIME_SERIES] +ns=Observations/Cardington +title=Cardington multi-height timeseries +description=Create Cardington diagnostic variable timeseries +type=python_boolean +compulsory=true +sort-key=0card4 + +# Generic timeseries for surface (skin) temperature +[template variables=CARDINGTON_SURFACE_TEMPERATURE_TIME_SERIES] +ns=Observations/Cardington +title=Cardington surface temperature timeseries +description=Create Cardington skin temperature variable timeseries +type=python_boolean +compulsory=true +sort-key=0card5 + +# Generic timeseries for surface (skin) temperature +[template variables=CARDINGTON_WIND_SINGLE_POINT_TIME_SERIES] +ns=Observations/Cardington +title=Cardington wind timeseries +description=Create Cardington wind diagnostic variable timeseries +type=python_boolean +compulsory=true +sort-key=0card6 diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index 0cf6259e9..f133cb075 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -12,8 +12,10 @@ AVIATION_COLOUR_STATE_VISIBILITY=False AVIATION_FOG_PRESENCE_DOMAIN_MEAN_TIMESERIES=False AVIATION_FOG_PRESENCE_SPATIAL_DIFFERENCE=False AVIATION_FOG_PRESENCE_SPATIAL_PLOT=False -CARDINGTON_AIR_TEMPERATURE_SINGLE_POINT_TIME_SERIES=False -CARDINGTON_RELATIVE_HUMIDITY_SINGLE_POINT_TIME_SERIES=False +BASIC_QQ_PLOT=False +CARDINGTON_AIR_SINGLE_POINT_TIME_SERIES=False +!!CARDINGTON_AIR_TEMPERATURE_SINGLE_POINT_TIME_SERIES=False +!!CARDINGTON_RELATIVE_HUMIDITY_SINGLE_POINT_TIME_SERIES=False CLOUD_BASE_HEIGHT_LESS_THAN_50_M_DOMAIN_MEAN_TIMESERIES=False CLOUD_BASE_HEIGHT_LESS_THAN_50_M_SPATIAL_DIFFERENCE=False CLOUD_BASE_HEIGHT_LESS_THAN_50_M_SPATIAL_PLOT=False diff --git a/src/CSET/loaders/__init__.py b/src/CSET/loaders/__init__.py index 6b66395c2..5442a4e88 100644 --- a/src/CSET/loaders/__init__.py +++ b/src/CSET/loaders/__init__.py @@ -21,6 +21,7 @@ from CSET.loaders import ( aoa, + cardington, histograms, observations, power_spectrum, @@ -35,6 +36,7 @@ __all__ = [ "aoa", + "cardington", "histograms", "observations", "power_spectrum", diff --git a/src/CSET/loaders/cardington.py b/src/CSET/loaders/cardington.py new file mode 100644 index 000000000..dbe632690 --- /dev/null +++ b/src/CSET/loaders/cardington.py @@ -0,0 +1,236 @@ +# © Crown copyright, Met Office (2022-2025) and CSET contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Load cardington recipes.""" +# long_name='VIS AT 1.5M (incl precip) M' +#'SURFACE SENSIBLE HEAT FLUX W/M2' +#'SURFACE LATENT HEAT FLUX W/M2' +#'DEWPOINT AT 1.5M (K)' + +from CSET.recipes import Config, RawRecipe, get_models + + +def height_token(ht: float) -> str: + """Convert height to the token used in Cardington variable names.""" + # Cardington uses "1p2m" for 1.2 m + if abs(ht - 1.2) < 1e-9: + return "1p2m" + # Integers become "2m", "10m", etc. + if float(ht).is_integer(): + return f"{int(ht)}m" + # Fallback for any other non-integer heights (e.g. 0.5 -> "0p5m") + s = str(ht).replace(".", "p") + return f"{s}m" + + +def load(conf: Config): + """Yield recipes from the given workflow configuration.""" + # Load a list of model detail dictionaries. + models = get_models(conf.asdict()) + # base_model = models[0] + if conf.CARDINGTON_SURFACE_TEMPERATURE_TIME_SERIES: + base_model = models[0] + field = "surface_temperature" + card_label = "Cardington" + for model in models[1:]: + yield RawRecipe( + recipe="cardington_surface_temperature_time_series.yaml", + variables={ + "MODEL_NAME": model["name"], + "VARNAME": field, + "CARDINGTON_LABEL": card_label, + "UM_VARNAME": field, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) + fields = [("surface_air_pressure", "air_pressure")] + for model_field, card_field in fields: + for model in models[1:]: + yield RawRecipe( + recipe="cardington_surface_pressure_time_series.yaml", + variables={ + "MODEL_NAME": model["name"], + "VARNAME": card_field, + "CARDINGTON_LABEL": card_label, + "UM_VARNAME": model_field, + "PLOTNAME": model_field, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) + + if conf.CARDINGTON_WIND_SINGLE_POINT_TIME_SERIES: + base_model = models[0] + fields = [ + ("wind_speed", "wind_speed_vector", [2, 10, 25, 50]), + ("wind_direction", "wind_direction", [2, 10, 25, 50]), + ] + for plot_field, card_field, heights in fields: + for ht in heights: + card_label = f"Cardington ({ht} m)" + for model in models[1:]: + yield RawRecipe( + recipe="cardington_wind_single_point_time_series.yaml", + variables={ + "MODEL_NAME": model["name"], + "VARNAME": f"{card_field}_{height_token(ht)}", + "CARDINGTON_LABEL": card_label, + "UM_U_VARNAME": "x_wind", + "UM_V_VARNAME": "y_wind", + "PLOT_VARNAME": plot_field, + "HEIGHT": ht, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) + + if conf.CARDINGTON_SINGLE_POINT_TIME_SERIES: + base_model = models[0] + fields = [ + ( + "air temperature", + "air_temperature", + "air_temperature_rtd", + [1.2, 10, 25, 50], + ), + ( + "relative humidity", + "relative_humidity", + "relative_humidity", + [1.2, 10, 25, 50], + ), + ("dew point", "m01s03i250", "dewpoint_temperature", [1.2, 10, 25, 50]), + ( + "visibility", + "m01s03i281", + {"visibility_belfort", "visibility_vpf730"}, + [2], + ), + ("wind gust", "m01s03i463", "wind_speed_max", [2, 10, 25, 50]), + ("latent heat flux", "m01s03i234", "wq_covariance", [10]), + ( + "sensible heat flux", + "m01s03i217", + { + "wt_covariance": "HEIGHT", + "air_temperature_rtd": "HEIGHT_EXCEPT_2M", + "pressure_barometric": "FIXED", + }, + [2, 10, 25, 50], + ), + ] + for plot_field, model_field, card_field, heights in fields: + for ht in heights: + card_label = f"Cardington ({ht} m)" + varname = f"{card_field}_{height_token(ht)}" + for model in models[1:]: + if "latent" in plot_field: + yield RawRecipe( + recipe="cardington_latent_heat_single_point_time_series.yaml", + variables={ + "MODEL_NAME": model["name"], + "VARNAME": varname, + "CARDINGTON_LABEL": card_label, + "UM_VARNAME": model_field, + "HEIGHT": ht, + "PLOTNAME": plot_field, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) + elif "sensible" in plot_field and isinstance(card_field, dict): + card_varnames = [] + for var, rule in card_field.items(): + if rule == "HEIGHT": + card_varnames.append(f"{var}_{height_token(ht)}") + elif rule == "HEIGHT_EXCEPT_2M": + if ht == 2: + card_varnames.append(f"{var}_1p2m") + else: + card_varnames.append(f"{var}_{height_token(ht)}") + elif rule == "FIXED": + card_varnames.append(var) + else: + card_varnames.append(f"{var}_{rule}") + yield RawRecipe( + recipe="cardington_sensible_heat_single_point_time_series.yaml", + variables={ + "MODEL_NAME": model["name"], + "CARDINGTON_LABEL": f"Cardington ({ht} m)", + "UM_VARNAME": model_field, # m01s03i217 (unchanged) + "HEIGHT": ht, + "PLOTNAME": plot_field, + "CARDINGTON_VARNAMES": ",".join(card_varnames), + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) + elif "visibility" in card_field: + for vis_var in card_field: + yield RawRecipe( + recipe="cardington_visibility_single_point_time_series.yaml", + variables={ + "MODEL_NAME": model["name"], + "VARNAME": f"{vis_var}_{height_token(ht)}", + "CARDINGTON_LABEL": card_label, + "UM_VARNAME": model_field, + "HEIGHT": ht, + "PLOTNAME": plot_field, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) + elif "temperature" in model_field or "temperature" in card_field: + yield RawRecipe( + recipe="cardington_temperature_single_point_time_series.yaml", + variables={ + "MODEL_NAME": model["name"], + "VARNAME": varname, + "CARDINGTON_LABEL": card_label, + "UM_VARNAME": model_field, + "HEIGHT": ht, + "PLOTNAME": plot_field, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) + elif "gust" in plot_field: + yield RawRecipe( + recipe="cardington_wind_gust_single_point_time_series.yaml", + variables={ + "MODEL_NAME": model["name"], + "VARNAME": varname, + "CARDINGTON_LABEL": card_label, + "UM_VARNAME": model_field, + "HEIGHT": ht, + "PLOTNAME": plot_field, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) + else: + yield RawRecipe( + recipe="cardington_single_point_time_series.yaml", + variables={ + "MODEL_NAME": model["name"], + "VARNAME": varname, + "CARDINGTON_LABEL": card_label, + "UM_VARNAME": model_field, + "HEIGHT": ht, + "PLOTNAME": plot_field, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) diff --git a/src/CSET/loaders/timeseries.py b/src/CSET/loaders/timeseries.py index 567c99414..eced7fb7f 100644 --- a/src/CSET/loaders/timeseries.py +++ b/src/CSET/loaders/timeseries.py @@ -368,31 +368,6 @@ def load(conf: Config): aggregation=False, ) - # Cardington air temperature time series. - if conf.CARDINGTON_AIR_TEMPERATURE_SINGLE_POINT_TIME_SERIES: - base_model = models[0] - for model in models[1:]: - yield RawRecipe( - recipe="cardington_air_temperature_single_point_time_series.yaml", - variables={ - "MODEL_NAME": model["name"], - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) - - # Cardington relative humidity time series. - if conf.CARDINGTON_RELATIVE_HUMIDITY_SINGLE_POINT_TIME_SERIES: - base_model = models[0] - for model in models[1:]: - yield RawRecipe( - recipe="cardington_relative_humidity_single_point_time_series.yaml", - variables={ - "MODEL_NAME": model["name"], - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) # Surface wind gusts on Beaufort Scale if conf.SFC_WIND_GUSTS_BEAUFORT_SCALE_DOMAIN_MEAN_TIMESERIES: yield RawRecipe( diff --git a/src/CSET/operators/_colorbar_definition.json b/src/CSET/operators/_colorbar_definition.json index 757219d00..28c6515c7 100644 --- a/src/CSET/operators/_colorbar_definition.json +++ b/src/CSET/operators/_colorbar_definition.json @@ -273,8 +273,8 @@ "cmap": "bwr", "max": 250.0, "min": -250.0, - "ymax": 250.0, - "ymin": -50.0 + "ymax": "auto", + "ymin": "auto" }, "grid_surface_upward_sensible_heat_flux_difference": { "cmap": "PuOr", @@ -1030,7 +1030,7 @@ }, "visibility_in_air": { "cmap": "viridis", - "max": 30000.0, + "max": "auto", "min": 0.0 }, "visibility_in_air_difference": { diff --git a/src/CSET/operators/constraints.py b/src/CSET/operators/constraints.py index 905c660e4..aaa3a8649 100644 --- a/src/CSET/operators/constraints.py +++ b/src/CSET/operators/constraints.py @@ -61,14 +61,52 @@ def generate_var_constraint(varname: str, **kwargs) -> iris.Constraint: varname: str CF compliant name of variable, or a UM STASH code such as "m01s03i236". + Returns + ------- + An Iris constraint for either: + - a single UM STASH code + - a single variable name + - a list of variable names (Cardington multi-input case) + """ + _STASH_RE = re.compile(r"m\d{2}s\d{2}i\d{3}$") + + # ---- CASE 1: list of variable names (Cardington multi-variable) ---- + if isinstance(varname, (list, tuple)): + return iris.Constraint(cube_func=lambda cube: cube.name() in varname) + + # ---- CASE 2: single UM STASH code ---- + if _STASH_RE.match(varname): + return iris.AttributeConstraint(STASH=varname) + + # ---- CASE 3: single variable name ---- + return iris.Constraint(name=varname) + + +# if re.match(r"m[0-9]{2}s[0-9]{2}i[0-9]{3}$", varname): +# varname_constraint = iris.AttributeConstraint(STASH=varname) +# else: +# varname_constraint = iris.Constraint(name=varname) +# return varname_constraint + + +def generate_var_name_constraint(varname: str, **kwargs) -> iris.Constraint: + """Generate constraint from variable name (var_name). + + Operator that takes a CF compliant variable name string, and generates an + iris constraint to be passed into the read or filter operator. + + Arguments + --------- + varname: str + CF compliant name of variable using cube.var_name + Returns ------- varname_constraint: iris.Constraint """ - if re.match(r"m[0-9]{2}s[0-9]{2}i[0-9]{3}$", varname): - varname_constraint = iris.AttributeConstraint(STASH=varname) - else: - varname_constraint = iris.Constraint(name=varname) + varname_constraint = iris.Constraint( + var_name=lambda v: v is not None and varname in v + ) return varname_constraint diff --git a/src/CSET/operators/misc.py b/src/CSET/operators/misc.py index 9470dd5a1..ba02317b6 100644 --- a/src/CSET/operators/misc.py +++ b/src/CSET/operators/misc.py @@ -21,12 +21,15 @@ import iris import iris.analysis.calculus import numpy as np +from cf_units import Unit from iris.cube import Cube, CubeList from CSET._common import is_increasing, iter_maybe from CSET.operators._utils import fully_equalise_attributes, get_cube_yxcoordname from CSET.operators.regrid import regrid_onto_cube +LOG = logging.getLogger(__name__) + def noop(x, **kwargs): """Return its input without doing anything to it. @@ -180,6 +183,175 @@ def division(numerator, denominator): return numerator / denominator +def _ensure_temperature_K(temp_cube): + """ + Return a copy of temp_cube with units converted to Kelvin. + + If units are missing (no_unit/unknown), infer and set them. + """ + temp_K = temp_cube.copy() + + # Common Iris/cf_units representations of missing units + if ( + temp_K.units is None + or temp_K.units.is_unknown() + or str(temp_K.units) in ("no_unit", "unknown") + ): + # Heuristic: Cardington air_temperature is almost certainly in °C or K. + # Use value range to distinguish (works well for near-surface air temp). + data = temp_K.core_data() + # Pull a small sample without forcing full compute + try: + sample = float(data[0]) + except Exception: + # fallback: try first non-masked element if masked array + import numpy as np + + sample = float(np.asanyarray(data).ravel()[0]) + + # If values look like Kelvin (e.g. 200–330), assume K; else assume °C. + if 150.0 <= sample <= 350.0: + temp_K.units = Unit("K") + else: + temp_K.units = Unit( + "degC" + ) # or "deg_C" if you prefer; cf_units accepts degC + + # Now convert (will be a no-op if already K) + temp_K.convert_units("K") + return temp_K + + +def convert_visibility_to_km(cubes, **kwargs): + """Ensure visibility is converted to km if required.""" + if isinstance(cubes, iris.cube.Cube): + cubes = iris.cube.CubeList([cubes]) + else: + cubes = iris.cube.CubeList(cubes) + + for cube in cubes: + model = cube.attributes.get("model_name", "") + if model and "Cardington" not in model: + # UM visibility is in metres + cube.convert_units("km") + if model and "Cardington" in model: + # UM visibility is in metres + cube.convert_units("m") + return cubes if len(cubes) > 1 else cubes[0] + + +def sensible_heat_units(cubes, **kwargs): + """ + Compute surface upward sensible heat flux from Cardington inputs. + + EXPECTS (exactly one of each, preselected upstream): + - wt_covariance_2m + - air_temperature_rtd_1p2m + - barometric_pressure + UM cubes are passed through unchanged. + HEIGHT is treated as a *nominal* height (for UM selection / labelling), + not as a measurement height for Cardington. + """ + import iris + from cf_units import Unit + + Cp = 1004.67 # J kg-1 K-1 + Rd = 287.05 # J kg-1 K-1 + cubes = ( + iris.cube.CubeList(cubes) + if not isinstance(cubes, iris.cube.CubeList) + else cubes + ) + + # --- Extract Cardington inputs explicitly listed upstream --- + if "CARDINGTON_VARNAMES" not in kwargs: + raise ValueError("sensible_heat_units requires CARDINGTON_VARNAMES") + + wanted = set(kwargs["CARDINGTON_VARNAMES"].split(",")) + selected = {c.var_name: c for c in cubes if c.var_name in wanted} + missing = wanted - set(selected) + if missing: + raise ValueError( + f"sensible_heat_units missing Cardington inputs: {sorted(missing)}" + ) + + wT = next(v for k, v in selected.items() if k.startswith("wt_covariance_")) + temp = next(v for k, v in selected.items() if k.startswith("air_temperature_rtd_")) + pressure = selected["pressure_barometric"] + + # --- Unit handling --- + temp_K = temp.copy() + if temp_K.units is None or temp_K.units.is_unknown(): + temp_K.units = Unit("degC") + temp_K.convert_units("K") + + pres_Pa = pressure.copy() + if pres_Pa.units is None or pres_Pa.units.is_unknown(): + pres_Pa.units = Unit("hPa") + pres_Pa.convert_units("Pa") + + # --- Compute sensible heat flux --- + rho_air = pres_Pa.data / (Rd * temp_K.data) + shf = wT.copy() + shf.data = Cp * rho_air * wT.data + shf.units = "W m-2" + shf.rename("surface_upward_sensible_heat_flux_cardington") + shf.var_name = "surface_upward_sensible_heat_flux_cardington" + + # --- Metadata: be explicit about mixed heights --- + shf.attributes["model_name"] = wT.attributes.get("model_name") + shf.attributes["cardington_measurement_heights"] = { + "wt_covariance": wT.var_name.split("_")[-1], + "air_temperature": temp.var_name.split("_")[-1], + "air_pressure": "1.2 m", + } + if "HEIGHT" in kwargs: + shf.attributes["nominal_height"] = f"{kwargs['HEIGHT']} m" + + # --- Return: passthrough everything except Cardington inputs, plus SHF --- + out = iris.cube.CubeList(c for c in cubes if c.var_name not in wanted) + out.append(shf) + return out if len(out) > 1 else out[0] + + +def latent_heat_units( + cubes: Cube | CubeList, + **kwargs, +) -> Cube | CubeList: + """ + Convert w'q' covariance (e.g. from Cardington surface site netCDF files) to latent heat flux (W m-2). + + Note + ---- + Using fixed value of latent heat of vapourisation for now; varies by about 5% between -20 and +40degC. + Possible future improvement. + """ + REQUIRED_UNITS = Unit("kg m-2 s-1") + OUTPUT_UNITS = Unit("W m-2") + + Lc = 2.45e6 # J kg-1 + + out = iris.cube.CubeList() + + for cube in iter_maybe(cubes): + # ---- ONLY ACT ON MASS FLUXES ---- + if cube.units is None or cube.units.is_unknown(): + out.append(cube) + continue + if not cube.units.is_convertible(REQUIRED_UNITS): + # ✅ This is UM LE or some other diagnostic — leave untouched + out.append(cube) + continue + + cube_a = cube.copy() + cube_a = cube_a * Lc + cube_a.units = OUTPUT_UNITS + + out.append(cube_a) + + return out[0] if len(out) == 1 else out + + def multiplication( multiplicand: Cube | CubeList, multiplier: Cube | CubeList ) -> Cube | CubeList: @@ -489,6 +661,15 @@ def rename_cube(cubes: iris.cube.Cube | iris.cube.CubeList, name: str): -------- >>> light_rain_mask = misc.rename_cube(light_rain_mask,"mask_for_light_rainfall" """ + # LOG.warning( + # "RENAME INPUT: %s", + # [(c.var_name, c.attributes.get("model_name")) for c in cubes] + # ) + print( + "RENAME INPUT:", + [(c.var_name, c.attributes.get("model_name")) for c in cubes], + flush=True, + ) new_cubelist = iris.cube.CubeList([]) for cube in iter_maybe(cubes): cube.rename(name) diff --git a/src/CSET/operators/read.py b/src/CSET/operators/read.py index 412aac606..2c7f9b28f 100644 --- a/src/CSET/operators/read.py +++ b/src/CSET/operators/read.py @@ -23,7 +23,7 @@ from pathlib import Path from typing import Literal -import dask +import dask.array as da import iris import iris.coord_systems import iris.coords @@ -180,9 +180,11 @@ def read_cubes( # Split out first model's cubes and mark it as the base for comparisons. cubes = next(model_cubes) - for cube in cubes: + for i, cube in enumerate(cubes): # Use 1 to indicate True, as booleans can't be saved in NetCDF attributes. cube.attributes["cset_comparison_base"] = 1 + # set masked data and knowingly bad data to np.nan so they aren't plotted + cubes[i] = _mask_fill_value(cube) # returns a new cube into cubes # Load the rest of the models. cubes.extend(itertools.chain.from_iterable(model_cubes)) @@ -866,7 +868,39 @@ def _fix_lfric_cloud_base_altitude(cube: iris.cube.Cube): varnames = filter(None, [cube.long_name, cube.standard_name, cube.var_name]) if any("cloud_base_altitude" in name for name in varnames): # Mask cube where set > 144kft to catch default 144.35695538058164 - cube.data = dask.array.ma.masked_greater(cube.core_data(), 144.0) + cube.data = da.ma.masked_greater(cube.core_data(), 144.0) + + +def _mask_fill_value(cube: iris.cube.Cube, ulp_factor=10): + """Force masked data using fill_value to equal np.nan. + + Data previously flagged as bad and thereby ascribed as 1e11 or + 999999 are also set to np.nan. This ensures no such values are plotted. + + ulp_factor is used to scale up the float32 to float64 error to a + catch-all value + """ + x = cube.lazy_data() + fill_value = x._meta.fill_value + fill_values = (fill_value, 1e10, 1e11, 999999) + data = x.map_blocks(np.ma.getdata, dtype=x.dtype) + m0 = x.map_blocks(np.ma.getmaskarray, dtype=bool) + + data = data.astype(np.float32) + m_fill = da.zeros(data.shape, dtype=bool, chunks=data.chunks) + for fv in fill_values: + ulp = ulp_factor * np.spacing(np.float32(fv)).astype(np.float64) + m_fill = m_fill | da.isclose(data, np.float32(fv), rtol=0, atol=ulp) + + has_any_masked = da.any(m0).compute() + has_any_sentinel = da.any(m_fill).compute() + if (not has_any_masked) and (not has_any_sentinel): + return cube # nothing to clean, return cube unchanged + + m_all = m0 | m_fill + y = da.where(m_all, np.nan, data) + + return cube.copy(data=y) # returns modified cube def _fix_um_winds(cubes: iris.cube.CubeList): diff --git a/src/CSET/operators/wind.py b/src/CSET/operators/wind.py index 4ea767f63..6ac2ed960 100644 --- a/src/CSET/operators/wind.py +++ b/src/CSET/operators/wind.py @@ -21,6 +21,76 @@ from CSET._common import iter_maybe +def calculate_vector_wind( + cubes: iris.cube.Cube | iris.cube.CubeList, + *, + u_names: tuple[str, ...] = ("x_wind", "eastward_wind", "u", "u_wind"), + v_names: tuple[str, ...] = ("y_wind", "northward_wind", "v", "v_wind"), +) -> iris.cube.Cube | iris.cube.CubeList: + """ + Calculate wind speed and meteorological wind direction from U/V components. + + Notes + ----- + - Speed = sqrt(u^2 + v^2) + - Direction is meteorological "from" direction in degrees, 0..360: + 0 = from North, 90 = from East, 180 = from South, 270 = from West + computed as: (atan2(-u, -v) in degrees + 360) % 360 + + Returns + ------- + If input was a single Cube (not typical for U/V), returns a CubeList. + If input was a CubeList, returns a CubeList containing: + - wind_speed cube + - wind_direction cube + (and optionally any untouched cubes if you choose to keep them; here we return only derived cubes.) + Example + -------- + >>> vector_winds = wind.calculate_vector_wind(winds) + """ + # Normalize input to CubeList + in_list = ( + cubes if isinstance(cubes, iris.cube.CubeList) else iris.cube.CubeList([cubes]) + ) + + def _find_by_name( + cubelist: iris.cube.CubeList, names: tuple[str, ...] + ) -> iris.cube.Cube | None: + for nm in names: + matches = cubelist.extract(iris.Constraint(name=nm)) + if matches: + return matches[0] + return None + + u_cube = _find_by_name(in_list, u_names) + v_cube = _find_by_name(in_list, v_names) + + if u_cube is None or v_cube is None: + available = [c.name() for c in in_list] + raise ValueError( + "calculate_vector_wind needs both U and V component cubes. " + f"Looked for U names {u_names} and V names {v_names}. " + f"Available cube names: {available}" + ) + + u = u_cube.core_data() + v = v_cube.core_data() + direction = (np.degrees(np.arctan2(-u, -v)) + 360) % 360 + speed = np.sqrt(u**2 + v**2) + speed_cube = u_cube.copy(data=speed) + speed_cube.rename("wind_speed") + speed_cube.units = "m s-1" + direction_cube = u_cube.copy(data=direction) + direction_cube.units = "degrees" + direction_cube.rename("wind_direction") + + winds = iris.cube.CubeList([speed_cube, direction_cube]) + if len(winds) == 1: + return winds[0] + else: + return winds + + def convert_to_beaufort_scale( cubes: iris.cube.Cube | iris.cube.CubeList, ) -> iris.cube.Cube | iris.cube.CubeList: diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_latent_heat_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_latent_heat_single_point_time_series.yaml new file mode 100644 index 000000000..e60746ab2 --- /dev/null +++ b/src/CSET/recipes/surface_fields/cardington/cardington_latent_heat_single_point_time_series.yaml @@ -0,0 +1,60 @@ +category: Time series of air temperature at Cardington single point +title: "Time series of $PLOTNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." +description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoint." + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + + - operator: filters.filter_multiple_cubes + + Cardington: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + height_constraint: + operator: constraints.generate_attribute_constraint + attribute: 'height' + value: $HEIGHT + + $MODEL_NAME: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $UM_VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $UM_VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + levels: [ ] + coordinate: "pressure" + stash_constraint: + operator: constraints.generate_attribute_constraint + attribute: STASH + + - operator: misc.latent_heat_units + + - operator: misc.remove_attribute + attribute: [ 'time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created' ] + + - operator: regrid.regrid_to_single_point + lat_pt: 52.10438 + lon_pt: -0.42286 + latlon_in_type: "realworld" + method: "Nearest" + boundary_margin: 0 + + # Make a single NetCDF with all the data inside it. + - operator: write.write_cube_to_nc + overwrite: True + + # Plot the data. + - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_sensible_heat_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_sensible_heat_single_point_time_series.yaml new file mode 100644 index 000000000..a608b4483 --- /dev/null +++ b/src/CSET/recipes/surface_fields/cardington/cardington_sensible_heat_single_point_time_series.yaml @@ -0,0 +1,43 @@ +category: Time series of air temperature at Cardington single point +title: "Time series of $PLOTNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." +description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoint." +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: + - Cardington + - $MODEL_NAME + + - operator: misc.sensible_heat_units + HEIGHT: $HEIGHT + CARDINGTON_VARNAMES: $CARDINGTON_VARNAMES + + - operator: filters.filter_multiple_cubes + Cardington: + operator: constraints.generate_var_constraint + varname: surface_upward_sensible_heat_flux_cardington + + $MODEL_NAME: + operator: constraints.generate_var_constraint + varname: surface_upward_sensible_heat_flux + + - operator: misc.remove_attribute + attribute: + - time_coverage_end + - time_coverage_start + - valid_min + - valid_max + - date + - date_created + + - operator: regrid.regrid_to_single_point + lat_pt: 52.10438 + lon_pt: -0.42286 + latlon_in_type: "realworld" + method: "Nearest" + boundary_margin: 0 + + - operator: write.write_cube_to_nc + overwrite: True + + - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_single_point_time_series.yaml new file mode 100644 index 000000000..97e3d4643 --- /dev/null +++ b/src/CSET/recipes/surface_fields/cardington/cardington_single_point_time_series.yaml @@ -0,0 +1,58 @@ +category: Time series of variables at Cardington single point +title: "Time series of $PLOTNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." +description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoint." + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + + - operator: filters.filter_multiple_cubes + + Cardington: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + height_constraint: + operator: constraints.generate_attribute_constraint + attribute: 'height' + value: $HEIGHT + + $MODEL_NAME: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $UM_VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $UM_VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + levels: [] + coordinate: "pressure" + stash_constraint: + operator: constraints.generate_attribute_constraint + attribute: STASH + + - operator: misc.remove_attribute + attribute: [ 'time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created' ] + + - operator: regrid.regrid_to_single_point + lat_pt: 52.10438 + lon_pt: -0.42286 + latlon_in_type: "realworld" + method: "Nearest" + boundary_margin: 0 + + # Make a single NetCDF with all the data inside it. + - operator: write.write_cube_to_nc + overwrite: True + + # Plot the data. + - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_surface_pressure_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_surface_pressure_time_series.yaml new file mode 100644 index 000000000..08201bd64 --- /dev/null +++ b/src/CSET/recipes/surface_fields/cardington/cardington_surface_pressure_time_series.yaml @@ -0,0 +1,58 @@ +category: Time series of variables at Cardington single point +title: "Time series of $PLOTNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." +description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoint." + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + + - operator: filters.filter_multiple_cubes + + Cardington: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + + + $MODEL_NAME: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $UM_VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $UM_VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + levels: [] + coordinate: "pressure" + stash_constraint: + operator: constraints.generate_attribute_constraint + attribute: STASH + + - operator: misc.remove_attribute + attribute: [ 'time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created' ] + + - operator: misc.convert_units + units: 'hPa' + + - operator: regrid.regrid_to_single_point + lat_pt: 52.10438 + lon_pt: -0.42286 + latlon_in_type: "realworld" + method: "Nearest" + boundary_margin: 0 + + # Make a single NetCDF with all the data inside it. + - operator: write.write_cube_to_nc + overwrite: True + + # Plot the data. + - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml new file mode 100644 index 000000000..9a34fb92b --- /dev/null +++ b/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml @@ -0,0 +1,54 @@ +category: Time series of variables at Cardington single point +title: "Time series of $VARNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." +description: "Plots a time series of $VARNAME at a selected Cardington gridpoint." + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + + - operator: filters.filter_multiple_cubes + + Cardington: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + + $MODEL_NAME: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $UM_VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [ 'maximum' ] + coord: 'time' + interval: '1 hour' + stash_constraint: + operator: constraints.generate_attribute_constraint + attribute: STASH + + - operator: misc.remove_attribute + attribute: [ 'time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created' ] + + - operator: misc.convert_units + units: 'Celsius' + + - operator: regrid.regrid_to_single_point + lat_pt: 52.10438 + lon_pt: -0.42286 + latlon_in_type: "realworld" + method: "Nearest" + boundary_margin: 0 + + # Make a single NetCDF with all the data inside it. + - operator: write.write_cube_to_nc + overwrite: True + + # Plot the data. + - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_temperature_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_temperature_single_point_time_series.yaml new file mode 100644 index 000000000..2be1d6c62 --- /dev/null +++ b/src/CSET/recipes/surface_fields/cardington/cardington_temperature_single_point_time_series.yaml @@ -0,0 +1,61 @@ +category: Time series of air temperature at Cardington single point +title: "Time series of $PLOTNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." +description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoint." + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + + - operator: filters.filter_multiple_cubes + + Cardington: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + height_constraint: + operator: constraints.generate_attribute_constraint + attribute: 'height' + value: $HEIGHT + + $MODEL_NAME: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $UM_VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $UM_VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + levels: [] + coordinate: "pressure" + stash_constraint: + operator: constraints.generate_attribute_constraint + attribute: STASH + + - operator: misc.remove_attribute + attribute: [ 'time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created' ] + + - operator: misc.convert_units + units: 'celsius' + + - operator: regrid.regrid_to_single_point + lat_pt: 52.10438 + lon_pt: -0.42286 + latlon_in_type: "realworld" + method: "Nearest" + boundary_margin: 0 + + # Make a single NetCDF with all the data inside it. + - operator: write.write_cube_to_nc + overwrite: True + + # Plot the data. + - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml new file mode 100644 index 000000000..8a260d3ab --- /dev/null +++ b/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml @@ -0,0 +1,52 @@ +category: Time series of variables at Cardington single point +title: "Time series of $PLOTNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." +description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoint." +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + + - operator: misc.combine_cubes_into_cubelist + + first: # --- Cardington --- + operator: misc.rename_cube + name: $PLOTNAME + var_name: $PLOTNAME + cubes: + operator: filters.filter_multiple_cubes + $CARDINGTON_LABEL: + operator: constraints.generate_var_constraint + varname: $VARNAME + + second: # --- UM --- + operator: filters.filter_multiple_cubes + $MODEL_NAME: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $UM_VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + levels: [] + coordinate: "pressure" + stash_constraint: + operator: constraints.generate_attribute_constraint + attribute: STASH + + - operator: misc.remove_attribute + attribute: [ 'time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created' ] + + - operator: misc.convert_visibility_to_km + units: 'km' + + - operator: regrid.regrid_to_single_point + lat_pt: 52.10438 + lon_pt: -0.42286 + latlon_in_type: "realworld" + method: "Nearest" + boundary_margin: 0 + + - operator: write.write_cube_to_nc + overwrite: True + + - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_wind_gust_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_wind_gust_single_point_time_series.yaml new file mode 100644 index 000000000..7e4db1862 --- /dev/null +++ b/src/CSET/recipes/surface_fields/cardington/cardington_wind_gust_single_point_time_series.yaml @@ -0,0 +1,50 @@ +category: Time series of variables at Cardington single point +title: "Time series of $PLOTNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." +description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoint." + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + + - operator: filters.filter_multiple_cubes + + Cardington: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + height_constraint: + operator: constraints.generate_attribute_constraint + attribute: 'height' + value: $HEIGHT + + $MODEL_NAME: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $UM_VARNAME + stash_constraint: + operator: constraints.generate_attribute_constraint + attribute: STASH + + - operator: misc.remove_attribute + attribute: [ 'time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created' ] + + - operator: regrid.regrid_to_single_point + lat_pt: 52.10438 + lon_pt: -0.42286 + latlon_in_type: "realworld" + method: "Nearest" + boundary_margin: 0 + + # Make a single NetCDF with all the data inside it. + - operator: write.write_cube_to_nc + overwrite: True + + # Plot the data. + - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_wind_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_wind_single_point_time_series.yaml new file mode 100644 index 000000000..4995b18b9 --- /dev/null +++ b/src/CSET/recipes/surface_fields/cardington/cardington_wind_single_point_time_series.yaml @@ -0,0 +1,86 @@ +category: Time series of variables at Cardington single point +title: "Time series of $PLOT_VARNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." +description: "Plots a time series of $PLOT_VARNAME at a selected Cardington gridpoint." + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + + - operator: misc.combine_cubes_into_cubelist + first: + operator: misc.rename_cube + name: $PLOT_VARNAME + cubes: + operator: filters.filter_multiple_cubes + constraints: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + height_constraint: + operator: constraints.generate_attribute_constraint + attribute: height + value: $HEIGHT + + second: + # --- UM: calculate vector wind from both components --- + operator: wind.calculate_vector_wind + cubes: + operator: filters.filter_multiple_cubes + u_constraint: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $UM_U_VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $UM_U_VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + levels: [] + coordinate: "pressure" + stash_constraint: + operator: constraints.generate_attribute_constraint + attribute: STASH + + v_constraint: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $UM_V_VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $UM_V_VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + levels: [] + coordinate: "pressure" + stash_constraint: + operator: constraints.generate_attribute_constraint + attribute: STASH + +# PLOT_VARNAME is used to constrain to either speed or direction + - operator: filters.filter_multiple_cubes + to_plot: + operator: constraints.generate_var_constraint + varname: $PLOT_VARNAME + + - operator: misc.remove_attribute + attribute: [ 'time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created' ] + + - operator: regrid.regrid_to_single_point + lat_pt: 52.10438 + lon_pt: -0.42286 + latlon_in_type: "realworld" + method: "Nearest" + boundary_margin: 0 + + # Make a single NetCDF with all the data inside it. + - operator: write.write_cube_to_nc + overwrite: True + + # Plot the data. + - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington_air_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington_air_single_point_time_series.yaml new file mode 100644 index 000000000..5e71eb908 --- /dev/null +++ b/src/CSET/recipes/surface_fields/cardington_air_single_point_time_series.yaml @@ -0,0 +1,61 @@ +category: Time series of air temperature at Cardington single point +title: "Time series of $UM_VARNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." +description: "Plots a time series of $UM_VARNAME at a selected Cardington gridpoint." + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + + - operator: filters.filter_multiple_cubes + + Cardington: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + height_constraint: + operator: constraints.generate_attribute_constraint + attribute: 'height' + value: $HEIGHT + + $MODEL_NAME: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $UM_VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $UM_VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + levels: [] + coordinate: "pressure" + stash_constraint: + operator: constraints.generate_attribute_constraint + attribute: STASH + + - operator: misc.remove_attribute + attribute: [ 'time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created' ] + + - operator: misc.convert_units + units: 'Celsius' + + - operator: regrid.regrid_to_single_point + lat_pt: 52.10438 + lon_pt: -0.42286 + latlon_in_type: "realworld" + method: "Nearest" + boundary_margin: 0 + + # Make a single NetCDF with all the data inside it. + - operator: write.write_cube_to_nc + overwrite: True + + # Plot the data. + - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_air_temperature_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington_air_temperature_single_point_time_series.yaml similarity index 97% rename from src/CSET/recipes/surface_fields/cardington/cardington_air_temperature_single_point_time_series.yaml rename to src/CSET/recipes/surface_fields/cardington_air_temperature_single_point_time_series.yaml index 0657f9b2b..989a576c6 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_air_temperature_single_point_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington_air_temperature_single_point_time_series.yaml @@ -5,7 +5,7 @@ description: Plots a time series of the air temperature at a selected Cardington steps: - operator: read.read_cubes file_paths: $INPUT_PATHS - model_names: ['Cardington','$MODEL_NAME'] + model_names: ['Cardington', '$MODEL_NAME'] constraint: operator: constraints.combine_constraints varname_constraint: diff --git a/src/CSET/recipes/surface_fields/cardington_humidity_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington_humidity_single_point_time_series.yaml new file mode 100644 index 000000000..ac21e4541 --- /dev/null +++ b/src/CSET/recipes/surface_fields/cardington_humidity_single_point_time_series.yaml @@ -0,0 +1,58 @@ +category: Time series of variables at Cardington single point +title: "Time series of $UM_VARNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." +description: "Plots a time series of $UM_VARNAME at a selected Cardington gridpoint." + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + + - operator: filters.filter_multiple_cubes + + Cardington: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + height_constraint: + operator: constraints.generate_attribute_constraint + attribute: 'height' + value: $HEIGHT + + $MODEL_NAME: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $UM_VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $UM_VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + levels: [] + coordinate: "pressure" + stash_constraint: + operator: constraints.generate_attribute_constraint + attribute: STASH + + - operator: misc.remove_attribute + attribute: [ 'time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created' ] + + - operator: regrid.regrid_to_single_point + lat_pt: 52.10438 + lon_pt: -0.42286 + latlon_in_type: "realworld" + method: "Nearest" + boundary_margin: 0 + + # Make a single NetCDF with all the data inside it. + - operator: write.write_cube_to_nc + overwrite: True + + # Plot the data. + - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_relative_humidity_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington_relative_humidity_single_point_time_series.yaml similarity index 100% rename from src/CSET/recipes/surface_fields/cardington/cardington_relative_humidity_single_point_time_series.yaml rename to src/CSET/recipes/surface_fields/cardington_relative_humidity_single_point_time_series.yaml From 4aac7c9ee5a1252926a5c777483cb09edd6caa3c Mon Sep 17 00:00:00 2001 From: Simon Osborne Date: Mon, 11 May 2026 16:44:56 +0100 Subject: [PATCH 02/11] All working Cardington variables now invoked from single CARDINGTON_SINGLE_POINT_TIME_SERIES logical. Two input ovservation files can now be read in and plotted e.g. 30-min and 5-min time intervals. --- src/CSET/loaders/cardington.py | 327 ++++++++++-------- src/CSET/operators/constraints.py | 22 +- src/CSET/operators/misc.py | 273 ++++++++++++++- src/CSET/operators/plot.py | 14 +- src/CSET/operators/read.py | 60 ++-- ..._latent_heat_single_point_time_series.yaml | 4 +- ...ensible_heat_single_point_time_series.yaml | 4 +- ...rdington_surface_pressure_time_series.yaml | 7 +- ...ngton_surface_temperature_time_series.yaml | 75 ++-- ..._temperature_single_point_time_series.yaml | 7 +- ...n_visibility_single_point_time_series.yaml | 62 ++-- ...on_wind_gust_single_point_time_series.yaml | 7 +- ...dington_wind_single_point_time_series.yaml | 8 +- ...rdington_air_single_point_time_series.yaml | 61 ---- ..._temperature_single_point_time_series.yaml | 51 --- ...ton_humidity_single_point_time_series.yaml | 58 ---- ...ive_humidity_single_point_time_series.yaml | 48 --- 17 files changed, 596 insertions(+), 492 deletions(-) delete mode 100644 src/CSET/recipes/surface_fields/cardington_air_single_point_time_series.yaml delete mode 100644 src/CSET/recipes/surface_fields/cardington_air_temperature_single_point_time_series.yaml delete mode 100644 src/CSET/recipes/surface_fields/cardington_humidity_single_point_time_series.yaml delete mode 100644 src/CSET/recipes/surface_fields/cardington_relative_humidity_single_point_time_series.yaml diff --git a/src/CSET/loaders/cardington.py b/src/CSET/loaders/cardington.py index dbe632690..2c4ca5a21 100644 --- a/src/CSET/loaders/cardington.py +++ b/src/CSET/loaders/cardington.py @@ -38,66 +38,110 @@ def load(conf: Config): """Yield recipes from the given workflow configuration.""" # Load a list of model detail dictionaries. models = get_models(conf.asdict()) - # base_model = models[0] - if conf.CARDINGTON_SURFACE_TEMPERATURE_TIME_SERIES: - base_model = models[0] + T30 = "30min" + T05 = "05min" + if conf.CARDINGTON_SINGLE_POINT_TIME_SERIES: + cardington_models = [m for m in models if "Cardington" in m["name"]] + um_model = next(m for m in models if "Cardington" not in m["name"]) + + fields = [("visibility", "m01s03i281", "visibility_belfort", [2])] + for plot_field, model_field, card_field, heights in fields: + for ht in heights: + card_label_30 = f"Cardington {T30} ({ht} m)" + card_label_05 = f"Cardington {T05} ({ht} m)" + varname = None + if isinstance(card_field, str): + varname = f"{card_field}_{height_token(ht)}" + yield RawRecipe( + recipe="cardington_visibility_single_point_time_series.yaml", + variables={ + "MODEL_NAME": um_model["name"], + "VARNAME": varname, + "CARDINGTON_30_LABEL": card_label_30, + "CARDINGTON_05_LABEL": card_label_05, + "UM_VARNAME": model_field, + "HEIGHT": ht, + "PLOTNAME": plot_field, + }, + model_ids=[ + cardington_models[0]["id"], + cardington_models[1]["id"], + um_model["id"], + ], + aggregation=False, + ) + field = "surface_temperature" - card_label = "Cardington" - for model in models[1:]: + card_label_30 = f"Cardington {T30}" + card_label_05 = f"Cardington {T05}" + yield RawRecipe( + recipe="cardington_surface_temperature_time_series.yaml", + variables={ + "MODEL_NAME": um_model["name"], + "VARNAME": field, + "CARDINGTON_30_LABEL": card_label_30, + "CARDINGTON_05_LABEL": card_label_05, + "UM_VARNAME": "m01s00i024", + "PLOTNAME": field, + }, + model_ids=[ + cardington_models[0]["id"], + cardington_models[1]["id"], + um_model["id"], + ], + aggregation=False, + ) + + fields = [("surface_air_pressure", "air_pressure")] + for model_field, card_field in fields: + card_label_30 = f"Cardington {T30}" + card_label_05 = f"Cardington {T05}" yield RawRecipe( - recipe="cardington_surface_temperature_time_series.yaml", + recipe="cardington_surface_pressure_time_series.yaml", variables={ - "MODEL_NAME": model["name"], - "VARNAME": field, - "CARDINGTON_LABEL": card_label, - "UM_VARNAME": field, + "MODEL_NAME": um_model["name"], + "VARNAME": card_field, + "CARDINGTON_30_LABEL": card_label_30, + "CARDINGTON_05_LABEL": card_label_05, + "UM_VARNAME": model_field, + "PLOTNAME": card_field, }, - model_ids=[base_model["id"], model["id"]], + model_ids=[ + cardington_models[0]["id"], + cardington_models[1]["id"], + um_model["id"], + ], aggregation=False, ) - fields = [("surface_air_pressure", "air_pressure")] - for model_field, card_field in fields: - for model in models[1:]: - yield RawRecipe( - recipe="cardington_surface_pressure_time_series.yaml", - variables={ - "MODEL_NAME": model["name"], - "VARNAME": card_field, - "CARDINGTON_LABEL": card_label, - "UM_VARNAME": model_field, - "PLOTNAME": model_field, - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) - if conf.CARDINGTON_WIND_SINGLE_POINT_TIME_SERIES: - base_model = models[0] fields = [ ("wind_speed", "wind_speed_vector", [2, 10, 25, 50]), ("wind_direction", "wind_direction", [2, 10, 25, 50]), ] for plot_field, card_field, heights in fields: for ht in heights: - card_label = f"Cardington ({ht} m)" - for model in models[1:]: - yield RawRecipe( - recipe="cardington_wind_single_point_time_series.yaml", - variables={ - "MODEL_NAME": model["name"], - "VARNAME": f"{card_field}_{height_token(ht)}", - "CARDINGTON_LABEL": card_label, - "UM_U_VARNAME": "x_wind", - "UM_V_VARNAME": "y_wind", - "PLOT_VARNAME": plot_field, - "HEIGHT": ht, - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) + card_label_30 = f"Cardington {T30} ({ht} m)" + card_label_05 = f"Cardington {T05} ({ht} m)" + yield RawRecipe( + recipe="cardington_wind_single_point_time_series.yaml", + variables={ + "MODEL_NAME": um_model["name"], + "VARNAME": f"{card_field}_{height_token(ht)}", + "CARDINGTON_30_LABEL": card_label_30, + "CARDINGTON_05_LABEL": card_label_05, + "UM_U_VARNAME": "x_wind", + "UM_V_VARNAME": "y_wind", + "PLOT_VARNAME": plot_field, + "HEIGHT": ht, + }, + model_ids=[ + cardington_models[0]["id"], + cardington_models[1]["id"], + um_model["id"], + ], + aggregation=False, + ) - if conf.CARDINGTON_SINGLE_POINT_TIME_SERIES: - base_model = models[0] fields = [ ( "air temperature", @@ -112,12 +156,6 @@ def load(conf: Config): [1.2, 10, 25, 50], ), ("dew point", "m01s03i250", "dewpoint_temperature", [1.2, 10, 25, 50]), - ( - "visibility", - "m01s03i281", - {"visibility_belfort", "visibility_vpf730"}, - [2], - ), ("wind gust", "m01s03i463", "wind_speed_max", [2, 10, 25, 50]), ("latent heat flux", "m01s03i234", "wq_covariance", [10]), ( @@ -133,104 +171,89 @@ def load(conf: Config): ] for plot_field, model_field, card_field, heights in fields: for ht in heights: - card_label = f"Cardington ({ht} m)" - varname = f"{card_field}_{height_token(ht)}" - for model in models[1:]: - if "latent" in plot_field: - yield RawRecipe( - recipe="cardington_latent_heat_single_point_time_series.yaml", - variables={ - "MODEL_NAME": model["name"], - "VARNAME": varname, - "CARDINGTON_LABEL": card_label, - "UM_VARNAME": model_field, - "HEIGHT": ht, - "PLOTNAME": plot_field, - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) - elif "sensible" in plot_field and isinstance(card_field, dict): - card_varnames = [] - for var, rule in card_field.items(): - if rule == "HEIGHT": - card_varnames.append(f"{var}_{height_token(ht)}") - elif rule == "HEIGHT_EXCEPT_2M": - if ht == 2: - card_varnames.append(f"{var}_1p2m") - else: - card_varnames.append(f"{var}_{height_token(ht)}") - elif rule == "FIXED": - card_varnames.append(var) + card_label_30 = f"Cardington {T30} ({ht} m)" + card_label_05 = f"Cardington {T05} ({ht} m)" + varname = None + if isinstance(card_field, str): + varname = f"{card_field}_{height_token(ht)}" + + if "latent" in plot_field: + yield RawRecipe( + recipe="cardington_latent_heat_single_point_time_series.yaml", + variables={ + "MODEL_NAME": um_model["name"], + "VARNAME": varname, + "CARDINGTON_30_LABEL": card_label_30, + "UM_VARNAME": model_field, + "HEIGHT": ht, + "PLOTNAME": plot_field, + }, + model_ids=[cardington_models[1]["id"], um_model["id"]], + aggregation=False, + ) + elif "sensible" in plot_field and isinstance(card_field, dict): + card_varnames = [] + for var, rule in card_field.items(): + if rule == "HEIGHT": + card_varnames.append(f"{var}_{height_token(ht)}") + elif rule == "HEIGHT_EXCEPT_2M": + if ht == 2: + card_varnames.append(f"{var}_1p2m") else: - card_varnames.append(f"{var}_{rule}") - yield RawRecipe( - recipe="cardington_sensible_heat_single_point_time_series.yaml", - variables={ - "MODEL_NAME": model["name"], - "CARDINGTON_LABEL": f"Cardington ({ht} m)", - "UM_VARNAME": model_field, # m01s03i217 (unchanged) - "HEIGHT": ht, - "PLOTNAME": plot_field, - "CARDINGTON_VARNAMES": ",".join(card_varnames), - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) - elif "visibility" in card_field: - for vis_var in card_field: - yield RawRecipe( - recipe="cardington_visibility_single_point_time_series.yaml", - variables={ - "MODEL_NAME": model["name"], - "VARNAME": f"{vis_var}_{height_token(ht)}", - "CARDINGTON_LABEL": card_label, - "UM_VARNAME": model_field, - "HEIGHT": ht, - "PLOTNAME": plot_field, - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) - elif "temperature" in model_field or "temperature" in card_field: - yield RawRecipe( - recipe="cardington_temperature_single_point_time_series.yaml", - variables={ - "MODEL_NAME": model["name"], - "VARNAME": varname, - "CARDINGTON_LABEL": card_label, - "UM_VARNAME": model_field, - "HEIGHT": ht, - "PLOTNAME": plot_field, - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) - elif "gust" in plot_field: - yield RawRecipe( - recipe="cardington_wind_gust_single_point_time_series.yaml", - variables={ - "MODEL_NAME": model["name"], - "VARNAME": varname, - "CARDINGTON_LABEL": card_label, - "UM_VARNAME": model_field, - "HEIGHT": ht, - "PLOTNAME": plot_field, - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) - else: - yield RawRecipe( - recipe="cardington_single_point_time_series.yaml", - variables={ - "MODEL_NAME": model["name"], - "VARNAME": varname, - "CARDINGTON_LABEL": card_label, - "UM_VARNAME": model_field, - "HEIGHT": ht, - "PLOTNAME": plot_field, - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) + card_varnames.append(f"{var}_{height_token(ht)}") + elif rule == "FIXED": + card_varnames.append(var) + else: + card_varnames.append(f"{var}_{rule}") + + yield RawRecipe( + recipe="cardington_sensible_heat_single_point_time_series.yaml", + variables={ + "MODEL_NAME": um_model["name"], + "CARDINGTON_30_LABEL": card_label_30, + "UM_VARNAME": model_field, # m01s03i217 (unchanged) + "HEIGHT": ht, + "PLOTNAME": plot_field, + "CARDINGTON_VARNAMES": ",".join(card_varnames), + }, + model_ids=[cardington_models[1]["id"], um_model["id"]], + aggregation=False, + ) + elif "temperature" in model_field or "temperature" in card_field: + yield RawRecipe( + recipe="cardington_temperature_single_point_time_series.yaml", + variables={ + "MODEL_NAME": um_model["name"], + "VARNAME": varname, + "CARDINGTON_30_LABEL": card_label_30, + "CARDINGTON_05_LABEL": card_label_05, + "UM_VARNAME": model_field, + "HEIGHT": ht, + "PLOTNAME": plot_field, + }, + model_ids=[ + cardington_models[0]["id"], + cardington_models[1]["id"], + um_model["id"], + ], + aggregation=False, + ) + elif "gust" in plot_field: + yield RawRecipe( + recipe="cardington_wind_gust_single_point_time_series.yaml", + variables={ + "MODEL_NAME": um_model["name"], + "VARNAME": varname, + "CARDINGTON_30_LABEL": card_label_30, + "CARDINGTON_05_LABEL": card_label_05, + "UM_VARNAME": model_field, + "HEIGHT": ht, + "PLOTNAME": plot_field, + }, + model_ids=[ + cardington_models[0]["id"], + cardington_models[1]["id"], + um_model["id"], + ], + aggregation=False, + ) diff --git a/src/CSET/operators/constraints.py b/src/CSET/operators/constraints.py index aaa3a8649..d8cbf0adc 100644 --- a/src/CSET/operators/constraints.py +++ b/src/CSET/operators/constraints.py @@ -69,17 +69,31 @@ def generate_var_constraint(varname: str, **kwargs) -> iris.Constraint: - a list of variable names (Cardington multi-input case) """ _STASH_RE = re.compile(r"m\d{2}s\d{2}i\d{3}$") - - # ---- CASE 1: list of variable names (Cardington multi-variable) ---- + # ---- CASE 1: list of variable names (e.g. Cardington multi-variable) ---- if isinstance(varname, (list, tuple)): - return iris.Constraint(cube_func=lambda cube: cube.name() in varname) + return iris.Constraint( + cube_func=lambda cube: ( + cube.var_name in varname + or cube.standard_name in varname + or cube.name() in varname + ) + ) + # if isinstance(varname, (list, tuple)): + # return iris.Constraint(cube_func=lambda cube: cube.name() in varname) # ---- CASE 2: single UM STASH code ---- if _STASH_RE.match(varname): return iris.AttributeConstraint(STASH=varname) # ---- CASE 3: single variable name ---- - return iris.Constraint(name=varname) + # return iris.Constraint(name=varname) + return iris.Constraint( + cube_func=lambda cube: ( + cube.var_name == varname + or cube.standard_name == varname + or cube.name() == varname + ) + ) # if re.match(r"m[0-9]{2}s[0-9]{2}i[0-9]{3}$", varname): diff --git a/src/CSET/operators/misc.py b/src/CSET/operators/misc.py index ba02317b6..cfe89620a 100644 --- a/src/CSET/operators/misc.py +++ b/src/CSET/operators/misc.py @@ -78,12 +78,194 @@ def remove_attribute( # Combine things that can be merged due to remove removing the # attributes. - cubes = cubes.merge() + # cubes = cubes.merge() # combine items that can be merged after removing unwanted attributes cubes = cubes.concatenate() return cubes +def remove_scalar_coords(cubes, coords): + """Remove scalar coordinates. + + examples would be: realization, forecast_reference_time from model cubes. + """ + if not isinstance(cubes, CubeList): + cubes = CubeList([cubes]) + + for cube in cubes: + for coord in coords: + if cube.coords(coord): + cube.remove_coord(coord) + + return cubes + + +def _concat_over_time_safely(model_cubes: CubeList) -> iris.cube.Cube: + """Concatenate cubes over time. + + coping with: + - scalar time coords (promote to dim with new_axis) + - scalar realization / forecast_reference_time (remove) or singleton dims (squeeze) + - cases where time arrays are identical (skip concatenation). + """ + fixed = CubeList() + + for c in model_cubes: + # --- ensure time is a dimension coordinate --- + if c.coords("time"): + t = c.coord("time") + if c.coord_dims(t) == (): # scalar time on this cube + c = iris.util.new_axis(c, t) # IMPORTANT: reassign! + t = c.coord("time") + if t.has_bounds(): + t.bounds = None + + # --- remove/squeeze scalar coords that block concat --- + for name in ("realization", "forecast_reference_time"): + if c.coords(name): + dims = c.coord_dims(name) + if dims == (): # scalar -> remove + c.remove_coord(name) + else: + # singleton dimension -> squeeze away + dim = dims[0] + if c.shape[dim] == 1: + c = iris.util.squeeze(c) + + fixed.append(c) + + iris.util.equalise_attributes(fixed) + iris.util.unify_time_units(fixed) + + # --- If time point arrays are identical, there is no concat axis --- + if fixed and fixed[0].coords("time"): + sigs = [tuple(c.coord("time").points) for c in fixed] + if len(set(sigs)) == 1: + return fixed[0] + + # --- Try normal concatenation over time --- + fixed.sort(key=lambda c: c.coord("time").points.min() if c.coords("time") else 0) + + try: + return fixed.concatenate_cube() + except iris.exceptions.ConcatenateError: + # Fallback: split into single-time slices and merge (often clearer / more robust) [2](https://engage.cloud.microsoft/main/threads/eyJfdHlwZSI6IlRocmVhZCIsImlkIjoiMTE5MTY5OTYxNSJ9)[3](https://engage.cloud.microsoft/main/threads/eyJfdHlwZSI6IlRocmVhZCIsImlkIjoiMzY5MzE0NDYyMzk5NjkyOCJ9) + slices = CubeList() + for c in fixed: + if c.coords("time") and c.coord_dims("time") != (): + slices.extend(list(c.slices_over("time"))) + else: + slices.append(c) + + iris.util.equalise_attributes(slices) + iris.util.unify_time_units(slices) + return slices.merge_cube() + + +def merge_cardington(cubes, model_prefix="Cardington", height_coord="height"): + """Merge all Cardington cubes along time. + + but preserve other attributes. + """ + if isinstance(cubes, iris.cube.Cube): + cubes = CubeList([cubes]) + + card_cubes = CubeList( + c for c in cubes if c.attributes.get("model_name", "").startswith(model_prefix) + ) + + other_cubes = CubeList(c for c in cubes if c not in card_cubes) + + # If zero or one Cardington cube, nothing to do + if len(card_cubes) <= 1: + return cubes + + # Decide whether height_coord is consistently available + use_height = all(cube.coords(height_coord) for cube in card_cubes) + by_height = {} + for cube in card_cubes: + if use_height: + h = cube.coord(height_coord).points.item() + else: + h = "__no_height_split__" # single bucket + by_height.setdefault(h, CubeList()).append(cube) + + merged_card = CubeList() + + for _h, cubes_at_height in by_height.items(): + by_model = {} + for c in cubes_at_height: + model = c.attributes["model_name"] + by_model.setdefault(model, CubeList()).append(c) + + for model, model_cubes in by_model.items(): + names = {c.name() for c in model_cubes} + if len(names) > 1: + # Different physical variables (e.g. 10m vs 25m covariance) – do NOT concatenate + for c in model_cubes: + merged_card.append(c) + continue + + if len(model_cubes) == 1: + merged = model_cubes[0] + else: + iris.util.equalise_attributes(model_cubes) + iris.util.unify_time_units(model_cubes) + # ---- DE-DUPE IDENTICAL CUBES (same time series & processing) ---- + deduped = iris.cube.CubeList() + seen = set() + + for c in model_cubes: + # time signature (full time axis) + t_sig = tuple(c.coord("time").points) if c.coords("time") else None + + # processing signature (distinguish instantaneous vs max/mean/etc) + cm_sig = tuple( + (cm.method, cm.coord_names, cm.intervals, cm.comments) + for cm in c.cell_methods + ) + + # stash helps disambiguate, but may be absent for Cardington obs + stash = c.attributes.get("STASH", None) + + sig = (c.name(), c.var_name, stash, t_sig, cm_sig, c.units) + + if sig not in seen: + seen.add(sig) + deduped.append(c) + + model_cubes = deduped + model_cubes.sort(key=lambda c: c.coord("time").points.min()) + concatenated = model_cubes.concatenate() + if len(concatenated) == 1: + merged = concatenated[0] + else: + concatenated.sort(key=lambda c: c.coord("time").points.min()) + merged = concatenated.concatenate_cube() + if len(concatenated) > 1: + print( + "DIFF BETWEEN GROUP 0 and 1:\n", + iris.util.describe_diff( + concatenated[0], concatenated[1], output_file=None + ), + ) + + # Sort final cube by time + tdim = merged.coord_dims("time")[0] + order = np.argsort(merged.coord("time").points) + slc = [slice(None)] * merged.ndim + slc[tdim] = order + merged = merged[tuple(slc)] + # t = merged.coord("time").points + # merged = merged[np.argsort(t)] + + # 🔴 CRITICAL: enforce exactly one cube per model + merged.attributes["model_name"] = model + merged_card.append(merged) + + return merged_card + other_cubes + + def addition(addend_1, addend_2): """Addition of two fields. @@ -205,8 +387,6 @@ def _ensure_temperature_K(temp_cube): sample = float(data[0]) except Exception: # fallback: try first non-masked element if masked array - import numpy as np - sample = float(np.asanyarray(data).ravel()[0]) # If values look like Kelvin (e.g. 200–330), assume K; else assume °C. @@ -222,21 +402,89 @@ def _ensure_temperature_K(temp_cube): return temp_K +def _mask_fill_value(cube: iris.cube.Cube, ulp_factor=10): + """ + Avoid plotting data flagged as bad/missing. + + Force masked data and known sentinel values to np.nan + so they are not plotted. + + """ + import dask.array as da + + x = cube.lazy_data() + + # --- Collect possible fill values safely --- + fill_values = [] + + # 1. NetCDF-style fill value (if present) + try: + fv = getattr(x._meta, "fill_value", None) + if fv is not None: + fill_values.append(fv) + except AttributeError: + pass # x has no _meta (plain ndarray) + + # 2. Known Cardington sentinels + fill_values.extend([1e10, 1e11, 999999]) + + # --- Extract data and any existing mask --- + data = da.asarray(x, dtype=np.float32) + + if np.ma.isMaskedArray(x): + m0 = da.asarray(np.ma.getmaskarray(x), dtype=bool) + else: + m0 = da.zeros(data.shape, dtype=bool, chunks=data.chunks) + + # --- Build sentinel mask --- + m_fill = da.zeros(data.shape, dtype=bool, chunks=data.chunks) + for fv in fill_values: + ulp = ulp_factor * np.spacing(np.float32(fv)) + m_fill |= da.isclose(data, np.float32(fv), rtol=0, atol=ulp) + + if not da.any(m0 | m_fill).compute(): + return cube # nothing to clean + + y = da.where(m0 | m_fill, np.nan, data) + + return cube.copy(data=y) + + +def mask_fill_values(cubes, ulp_factor=10): + """ + Apply _mask_fill_value to every cube in the CubeList. + + This must be run AFTER any operator that recreates data + (e.g. vector wind calculation, regridding). + """ + if not isinstance(cubes, CubeList): + cubes = CubeList([cubes]) + + cleaned = CubeList() + for cube in cubes: + cleaned.append(_mask_fill_value(cube, ulp_factor=ulp_factor)) + + return cleaned + + def convert_visibility_to_km(cubes, **kwargs): - """Ensure visibility is converted to km if required.""" + """Ensure visibility is converted to km if required. UM data is always in metres.""" if isinstance(cubes, iris.cube.Cube): cubes = iris.cube.CubeList([cubes]) else: cubes = iris.cube.CubeList(cubes) for cube in cubes: - model = cube.attributes.get("model_name", "") - if model and "Cardington" not in model: - # UM visibility is in metres + model = cube.attributes.get("model_name", "") or "" + if "Cardington" in model: + cube *= 1000 + # data = cube.core_data() + # cube.data = np.ma.array(data * 1.0e3, copy=False) + cube.units = "km" + else: + # UM visibility is in metres – convert with scaling cube.convert_units("km") - if model and "Cardington" in model: - # UM visibility is in metres - cube.convert_units("m") + return cubes if len(cubes) > 1 else cubes[0] @@ -252,7 +500,6 @@ def sensible_heat_units(cubes, **kwargs): HEIGHT is treated as a *nominal* height (for UM selection / labelling), not as a measurement height for Cardington. """ - import iris from cf_units import Unit Cp = 1004.67 # J kg-1 K-1 @@ -661,10 +908,6 @@ def rename_cube(cubes: iris.cube.Cube | iris.cube.CubeList, name: str): -------- >>> light_rain_mask = misc.rename_cube(light_rain_mask,"mask_for_light_rainfall" """ - # LOG.warning( - # "RENAME INPUT: %s", - # [(c.var_name, c.attributes.get("model_name")) for c in cubes] - # ) print( "RENAME INPUT:", [(c.var_name, c.attributes.get("model_name")) for c in cubes], diff --git a/src/CSET/operators/plot.py b/src/CSET/operators/plot.py index c6a44e09b..fa433392b 100644 --- a/src/CSET/operators/plot.py +++ b/src/CSET/operators/plot.py @@ -1144,6 +1144,7 @@ def _plot_and_save_vertical_line_series( marker="o", ls="-", lw=3, + alpha=0.85, label=f"{label} (control)" if len(cube.coord(ensemble_coord).points) > 1 else label, @@ -1443,8 +1444,7 @@ def _plot_and_save_histogram_series( # Set default that histograms will produce probability density function # at each bin (integral over range sums to 1). density = True - - for cube in iter_maybe(cubes): + for cube in enumerate(iter_maybe(cubes)): # Easier to check title (where var name originates) # than seeing if long names exist etc. # Exception case, where distribution better fits log scales/bins. @@ -1473,7 +1473,6 @@ def _plot_and_save_histogram_series( np.min(bins), np.max(bins), ) - # Reshape cube data into a single array to allow for a single histogram. # Otherwise we plot xdim histograms stacked. cube_data_1d = (cube.data).flatten() @@ -1493,7 +1492,14 @@ def _plot_and_save_histogram_series( y = y[1:] ax.plot( - y[:-1], x, color=color, linewidth=3, marker="o", markersize=6, label=label + y[:-1], + x, + color=color, + linewidth=3, + linestyle="-", + marker="o", + markersize=6, + label=label, ) # Add some labels and tweak the style. diff --git a/src/CSET/operators/read.py b/src/CSET/operators/read.py index 2c7f9b28f..a573bdb59 100644 --- a/src/CSET/operators/read.py +++ b/src/CSET/operators/read.py @@ -180,11 +180,10 @@ def read_cubes( # Split out first model's cubes and mark it as the base for comparisons. cubes = next(model_cubes) - for i, cube in enumerate(cubes): + for _i, cube in enumerate(cubes): # Use 1 to indicate True, as booleans can't be saved in NetCDF attributes. cube.attributes["cset_comparison_base"] = 1 # set masked data and knowingly bad data to np.nan so they aren't plotted - cubes[i] = _mask_fill_value(cube) # returns a new cube into cubes # Load the rest of the models. cubes.extend(itertools.chain.from_iterable(model_cubes)) @@ -215,6 +214,23 @@ def read_cubes( return cubes +def _select_surface_temperature_variant(cubes): + st = [c for c in cubes if str(c.attributes.get("STASH")) == "m01s00i024"] + if len(st) <= 1: + return cubes + + # Prefer time-processed (max/mean/min) + processed = [ + c for c in st if any(cm.coord_names == ("time",) for cm in c.cell_methods) + ] + chosen = processed[:1] if processed else st[:1] + out = iris.cube.CubeList( + c for c in cubes if str(c.attributes.get("STASH")) != "m01s00i024" + ) + out.extend(chosen) + return out + + def _load_model( paths: str | list[str], model_name: str | None, @@ -225,6 +241,13 @@ def _load_model( # If unset, a constraint of None lets everything be loaded. logging.debug("Constraint: %s", constraint) cubes = iris.load(input_files, constraint, callback=_loading_callback) + + for c in cubes: + print(" *** SRO PRINT NAMES *** ", c.var_name, c.standard_name, c.units) + + # 🔴 DISAMBIGUATE surface_temperature variants (m01s00i024) + cubes = _select_surface_temperature_variant(cubes) + # Make the UM's winds consistent with LFRic. _fix_um_winds(cubes) @@ -233,6 +256,7 @@ def _load_model( if model_name is not None: for cube in cubes: cube.attributes["model_name"] = model_name + return cubes @@ -871,38 +895,6 @@ def _fix_lfric_cloud_base_altitude(cube: iris.cube.Cube): cube.data = da.ma.masked_greater(cube.core_data(), 144.0) -def _mask_fill_value(cube: iris.cube.Cube, ulp_factor=10): - """Force masked data using fill_value to equal np.nan. - - Data previously flagged as bad and thereby ascribed as 1e11 or - 999999 are also set to np.nan. This ensures no such values are plotted. - - ulp_factor is used to scale up the float32 to float64 error to a - catch-all value - """ - x = cube.lazy_data() - fill_value = x._meta.fill_value - fill_values = (fill_value, 1e10, 1e11, 999999) - data = x.map_blocks(np.ma.getdata, dtype=x.dtype) - m0 = x.map_blocks(np.ma.getmaskarray, dtype=bool) - - data = data.astype(np.float32) - m_fill = da.zeros(data.shape, dtype=bool, chunks=data.chunks) - for fv in fill_values: - ulp = ulp_factor * np.spacing(np.float32(fv)).astype(np.float64) - m_fill = m_fill | da.isclose(data, np.float32(fv), rtol=0, atol=ulp) - - has_any_masked = da.any(m0).compute() - has_any_sentinel = da.any(m_fill).compute() - if (not has_any_masked) and (not has_any_sentinel): - return cube # nothing to clean, return cube unchanged - - m_all = m0 | m_fill - y = da.where(m_all, np.nan, data) - - return cube.copy(data=y) # returns modified cube - - def _fix_um_winds(cubes: iris.cube.CubeList): """To make winds from the UM consistent with those from LFRic. diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_latent_heat_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_latent_heat_single_point_time_series.yaml index e60746ab2..dd49ad597 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_latent_heat_single_point_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_latent_heat_single_point_time_series.yaml @@ -5,7 +5,9 @@ description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoin steps: - operator: read.read_cubes file_paths: $INPUT_PATHS - model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + model_names: + - $CARDINGTON_30_LABEL + - $MODEL_NAME - operator: filters.filter_multiple_cubes diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_sensible_heat_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_sensible_heat_single_point_time_series.yaml index a608b4483..5393faf25 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_sensible_heat_single_point_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_sensible_heat_single_point_time_series.yaml @@ -5,9 +5,11 @@ steps: - operator: read.read_cubes file_paths: $INPUT_PATHS model_names: - - Cardington + - $CARDINGTON_30_LABEL - $MODEL_NAME + - operator: misc.mask_fill_values + - operator: misc.sensible_heat_units HEIGHT: $HEIGHT CARDINGTON_VARNAMES: $CARDINGTON_VARNAMES diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_surface_pressure_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_surface_pressure_time_series.yaml index 08201bd64..a7e700b90 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_surface_pressure_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_surface_pressure_time_series.yaml @@ -5,7 +5,10 @@ description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoin steps: - operator: read.read_cubes file_paths: $INPUT_PATHS - model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + model_names: + - $CARDINGTON_05_LABEL + - $CARDINGTON_30_LABEL + - $MODEL_NAME - operator: filters.filter_multiple_cubes @@ -50,6 +53,8 @@ steps: method: "Nearest" boundary_margin: 0 + - operator: misc.mask_fill_values + # Make a single NetCDF with all the data inside it. - operator: write.write_cube_to_nc overwrite: True diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml index 9a34fb92b..b2e3b6eaf 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml @@ -1,40 +1,57 @@ category: Time series of variables at Cardington single point title: "Time series of $VARNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." description: "Plots a time series of $VARNAME at a selected Cardington gridpoint." - steps: + # ------------------------------------------------------------ + # 1. Read data + # IMPORTANT: + # - Constrain the UM *at read time* so Iris only ever loads + # a single surface_temperature cube. + # ------------------------------------------------------------ + - operator: read.read_cubes file_paths: $INPUT_PATHS - model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + model_names: + - $CARDINGTON_05_LABEL + - $CARDINGTON_30_LABEL + - $MODEL_NAME + constraints: + $MODEL_NAME: + operator: constraints.combine_constraints + stash_constraint: + operator: constraints.generate_attribute_constraint + attribute: STASH + values: + - $UM_VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [ 'maximum' ] + coord: 'time' + interval: '1 hour' + preprocessors: + - operator: misc.remove_scalar_coords + coords: + - forecast_reference_time + - realization - - operator: filters.filter_multiple_cubes + - operator: misc.merge_cardington - Cardington: - operator: constraints.combine_constraints - varname_constraint: - operator: constraints.generate_var_constraint - varname: $VARNAME - cell_methods_constraint: - operator: constraints.generate_cell_methods_constraint - cell_methods: [] - varname: $VARNAME - - $MODEL_NAME: - operator: constraints.combine_constraints - varname_constraint: - operator: constraints.generate_var_constraint - varname: $UM_VARNAME - cell_methods_constraint: - operator: constraints.generate_cell_methods_constraint - cell_methods: [ 'maximum' ] - coord: 'time' - interval: '1 hour' - stash_constraint: - operator: constraints.generate_attribute_constraint - attribute: STASH +# IMPORTANT: Global temperature-only prune. +# filter_multiple_cubes must have exactly ONE key here, +# otherwise non-temperature cubes will survive and break convert_units. + - operator: filters.filter_multiple_cubes + temperature: + operator: constraints.generate_var_constraint + varname: $VARNAME - operator: misc.remove_attribute - attribute: [ 'time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created' ] + attribute: + - time_coverage_end + - time_coverage_start + - valid_min + - valid_max + - date + - date_created - operator: misc.convert_units units: 'Celsius' @@ -46,9 +63,9 @@ steps: method: "Nearest" boundary_margin: 0 - # Make a single NetCDF with all the data inside it. + - operator: misc.mask_fill_values + - operator: write.write_cube_to_nc overwrite: True - # Plot the data. - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_temperature_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_temperature_single_point_time_series.yaml index 2be1d6c62..073ae4242 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_temperature_single_point_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_temperature_single_point_time_series.yaml @@ -5,7 +5,10 @@ description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoin steps: - operator: read.read_cubes file_paths: $INPUT_PATHS - model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + model_names: + - $CARDINGTON_05_LABEL + - $CARDINGTON_30_LABEL + - $MODEL_NAME - operator: filters.filter_multiple_cubes @@ -53,6 +56,8 @@ steps: method: "Nearest" boundary_margin: 0 + - operator: misc.mask_fill_values + # Make a single NetCDF with all the data inside it. - operator: write.write_cube_to_nc overwrite: True diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml index 8a260d3ab..f92be3ab3 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml @@ -1,43 +1,45 @@ category: Time series of variables at Cardington single point title: "Time series of $PLOTNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoint." + steps: - operator: read.read_cubes file_paths: $INPUT_PATHS - model_names: [$CARDINGTON_LABEL, $MODEL_NAME] - - - operator: misc.combine_cubes_into_cubelist - - first: # --- Cardington --- - operator: misc.rename_cube - name: $PLOTNAME - var_name: $PLOTNAME - cubes: - operator: filters.filter_multiple_cubes - $CARDINGTON_LABEL: - operator: constraints.generate_var_constraint - varname: $VARNAME - - second: # --- UM --- - operator: filters.filter_multiple_cubes - $MODEL_NAME: - operator: constraints.combine_constraints - varname_constraint: - operator: constraints.generate_var_constraint - varname: $UM_VARNAME - pressure_level_constraint: - operator: constraints.generate_level_constraint - levels: [] - coordinate: "pressure" - stash_constraint: - operator: constraints.generate_attribute_constraint - attribute: STASH + model_names: + - $CARDINGTON_05_LABEL + - $CARDINGTON_30_LABEL + - $MODEL_NAME + + - operator: misc.merge_cardington + + - operator: filters.filter_multiple_cubes + Cardington: + operator: constraints.generate_var_constraint + varname: $VARNAME + + $MODEL_NAME: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $UM_VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + levels: [] + coordinate: "pressure" + stash_constraint: + operator: constraints.generate_attribute_constraint + attribute: STASH - operator: misc.remove_attribute - attribute: [ 'time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created' ] + attribute: + - time_coverage_end + - time_coverage_start + - valid_min + - valid_max + - date + - date_created - operator: misc.convert_visibility_to_km - units: 'km' - operator: regrid.regrid_to_single_point lat_pt: 52.10438 diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_wind_gust_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_wind_gust_single_point_time_series.yaml index 7e4db1862..44f0d4df7 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_wind_gust_single_point_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_wind_gust_single_point_time_series.yaml @@ -5,7 +5,10 @@ description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoin steps: - operator: read.read_cubes file_paths: $INPUT_PATHS - model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + model_names: + - $CARDINGTON_05_LABEL + - $CARDINGTON_30_LABEL + - $MODEL_NAME - operator: filters.filter_multiple_cubes @@ -42,6 +45,8 @@ steps: method: "Nearest" boundary_margin: 0 + - operator: misc.mask_fill_values + # Make a single NetCDF with all the data inside it. - operator: write.write_cube_to_nc overwrite: True diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_wind_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_wind_single_point_time_series.yaml index 4995b18b9..9086b3255 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_wind_single_point_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_wind_single_point_time_series.yaml @@ -5,7 +5,10 @@ description: "Plots a time series of $PLOT_VARNAME at a selected Cardington grid steps: - operator: read.read_cubes file_paths: $INPUT_PATHS - model_names: [$CARDINGTON_LABEL, $MODEL_NAME] + model_names: + - $CARDINGTON_05_LABEL + - $CARDINGTON_30_LABEL + - $MODEL_NAME - operator: misc.combine_cubes_into_cubelist first: @@ -78,6 +81,9 @@ steps: method: "Nearest" boundary_margin: 0 + # RE‑MASK fill values AFTER vector wind + regrid + - operator: misc.mask_fill_values + # Make a single NetCDF with all the data inside it. - operator: write.write_cube_to_nc overwrite: True diff --git a/src/CSET/recipes/surface_fields/cardington_air_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington_air_single_point_time_series.yaml deleted file mode 100644 index 5e71eb908..000000000 --- a/src/CSET/recipes/surface_fields/cardington_air_single_point_time_series.yaml +++ /dev/null @@ -1,61 +0,0 @@ -category: Time series of air temperature at Cardington single point -title: "Time series of $UM_VARNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." -description: "Plots a time series of $UM_VARNAME at a selected Cardington gridpoint." - -steps: - - operator: read.read_cubes - file_paths: $INPUT_PATHS - model_names: [$CARDINGTON_LABEL, $MODEL_NAME] - - - operator: filters.filter_multiple_cubes - - Cardington: - operator: constraints.combine_constraints - varname_constraint: - operator: constraints.generate_var_constraint - varname: $VARNAME - cell_methods_constraint: - operator: constraints.generate_cell_methods_constraint - cell_methods: [] - varname: $VARNAME - height_constraint: - operator: constraints.generate_attribute_constraint - attribute: 'height' - value: $HEIGHT - - $MODEL_NAME: - operator: constraints.combine_constraints - varname_constraint: - operator: constraints.generate_var_constraint - varname: $UM_VARNAME - cell_methods_constraint: - operator: constraints.generate_cell_methods_constraint - cell_methods: [] - varname: $UM_VARNAME - pressure_level_constraint: - operator: constraints.generate_level_constraint - levels: [] - coordinate: "pressure" - stash_constraint: - operator: constraints.generate_attribute_constraint - attribute: STASH - - - operator: misc.remove_attribute - attribute: [ 'time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created' ] - - - operator: misc.convert_units - units: 'Celsius' - - - operator: regrid.regrid_to_single_point - lat_pt: 52.10438 - lon_pt: -0.42286 - latlon_in_type: "realworld" - method: "Nearest" - boundary_margin: 0 - - # Make a single NetCDF with all the data inside it. - - operator: write.write_cube_to_nc - overwrite: True - - # Plot the data. - - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington_air_temperature_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington_air_temperature_single_point_time_series.yaml deleted file mode 100644 index 989a576c6..000000000 --- a/src/CSET/recipes/surface_fields/cardington_air_temperature_single_point_time_series.yaml +++ /dev/null @@ -1,51 +0,0 @@ -category: Time series of air temperature at Cardington single point -title: Time series of 'air_temperature' at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint. -description: Plots a time series of the air temperature at a selected Cardington gridpoint. - -steps: - - operator: read.read_cubes - file_paths: $INPUT_PATHS - model_names: ['Cardington', '$MODEL_NAME'] - constraint: - operator: constraints.combine_constraints - varname_constraint: - operator: constraints.generate_var_constraint - varname: 'air_temperature' - cell_methods_constraint: - operator: constraints.generate_cell_methods_constraint - cell_methods: [] - varname: 'air_temperature' - pressure_level_constraint: - operator: constraints.generate_level_constraint - levels: [] - coordinate: "pressure" - - - operator: misc.remove_attribute - attribute: ['time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created'] - - - operator: filters.filter_multiple_cubes - cardington: - operator: constraints.generate_attribute_constraint - attribute: 'height' - value: 2.0 - model: - operator: constraints.generate_attribute_constraint - # This currently only works for UM. Maybe we need another operator to make this nicer? - attribute: STASH - - - operator: misc.convert_units - units: 'Celsius' - - - operator: regrid.regrid_to_single_point - lat_pt: 52.10438 - lon_pt: -0.42286 - latlon_in_type: "realworld" - method: "Nearest" - boundary_margin: 0 - - # Make a single NetCDF with all the data inside it. - - operator: write.write_cube_to_nc - overwrite: True - - # Plot the data. - - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington_humidity_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington_humidity_single_point_time_series.yaml deleted file mode 100644 index ac21e4541..000000000 --- a/src/CSET/recipes/surface_fields/cardington_humidity_single_point_time_series.yaml +++ /dev/null @@ -1,58 +0,0 @@ -category: Time series of variables at Cardington single point -title: "Time series of $UM_VARNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." -description: "Plots a time series of $UM_VARNAME at a selected Cardington gridpoint." - -steps: - - operator: read.read_cubes - file_paths: $INPUT_PATHS - model_names: [$CARDINGTON_LABEL, $MODEL_NAME] - - - operator: filters.filter_multiple_cubes - - Cardington: - operator: constraints.combine_constraints - varname_constraint: - operator: constraints.generate_var_constraint - varname: $VARNAME - cell_methods_constraint: - operator: constraints.generate_cell_methods_constraint - cell_methods: [] - varname: $VARNAME - height_constraint: - operator: constraints.generate_attribute_constraint - attribute: 'height' - value: $HEIGHT - - $MODEL_NAME: - operator: constraints.combine_constraints - varname_constraint: - operator: constraints.generate_var_constraint - varname: $UM_VARNAME - cell_methods_constraint: - operator: constraints.generate_cell_methods_constraint - cell_methods: [] - varname: $UM_VARNAME - pressure_level_constraint: - operator: constraints.generate_level_constraint - levels: [] - coordinate: "pressure" - stash_constraint: - operator: constraints.generate_attribute_constraint - attribute: STASH - - - operator: misc.remove_attribute - attribute: [ 'time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created' ] - - - operator: regrid.regrid_to_single_point - lat_pt: 52.10438 - lon_pt: -0.42286 - latlon_in_type: "realworld" - method: "Nearest" - boundary_margin: 0 - - # Make a single NetCDF with all the data inside it. - - operator: write.write_cube_to_nc - overwrite: True - - # Plot the data. - - operator: plot.plot_line_series diff --git a/src/CSET/recipes/surface_fields/cardington_relative_humidity_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington_relative_humidity_single_point_time_series.yaml deleted file mode 100644 index dd0c47ead..000000000 --- a/src/CSET/recipes/surface_fields/cardington_relative_humidity_single_point_time_series.yaml +++ /dev/null @@ -1,48 +0,0 @@ -category: Time series of relative humidity at Cardington single point -title: Time series of 'relative humidity' at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint. -description: Plots a time series of the relative humidity at a selected Cardington gridpoint. - -steps: - - operator: read.read_cubes - file_paths: $INPUT_PATHS - model_names: ['Cardington','$MODEL_NAME'] - constraint: - operator: constraints.combine_constraints - varname_constraint: - operator: constraints.generate_var_constraint - varname: 'relative_humidity' - cell_methods_constraint: - operator: constraints.generate_cell_methods_constraint - cell_methods: [] - varname: 'relative_humidity' - pressure_level_constraint: - operator: constraints.generate_level_constraint - levels: [] - coordinate: "pressure" - - - operator: misc.remove_attribute - attribute: ['time_coverage_end', 'time_coverage_start', 'valid_min', 'valid_max', 'date', 'date_created'] - - - operator: filters.filter_multiple_cubes - cardington: - operator: constraints.generate_attribute_constraint - attribute: 'height' - value: 1.2 - model: - operator: constraints.generate_attribute_constraint - # This currently only works for UM. Maybe we need another operator to make this nicer? - attribute: STASH - - - operator: regrid.regrid_to_single_point - lat_pt: 52.10438 - lon_pt: -0.42286 - latlon_in_type: "realworld" - method: "Nearest" - boundary_margin: 0 - - # Make a single NetCDF with all the data inside it. - - operator: write.write_cube_to_nc - overwrite: True - - # Plot the data. - - operator: plot.plot_line_series From e05d37819790f4232c52bfa18041f5d07f62c096 Mon Sep 17 00:00:00 2001 From: Simon Osborne Date: Tue, 12 May 2026 06:43:11 +0100 Subject: [PATCH 03/11] Removed merge_cardington function from misc.py that was no longer required --- src/CSET/operators/misc.py | 104 ------------------ ...ngton_surface_temperature_time_series.yaml | 2 - 2 files changed, 106 deletions(-) diff --git a/src/CSET/operators/misc.py b/src/CSET/operators/misc.py index cfe89620a..5e2248b3f 100644 --- a/src/CSET/operators/misc.py +++ b/src/CSET/operators/misc.py @@ -162,110 +162,6 @@ def _concat_over_time_safely(model_cubes: CubeList) -> iris.cube.Cube: return slices.merge_cube() -def merge_cardington(cubes, model_prefix="Cardington", height_coord="height"): - """Merge all Cardington cubes along time. - - but preserve other attributes. - """ - if isinstance(cubes, iris.cube.Cube): - cubes = CubeList([cubes]) - - card_cubes = CubeList( - c for c in cubes if c.attributes.get("model_name", "").startswith(model_prefix) - ) - - other_cubes = CubeList(c for c in cubes if c not in card_cubes) - - # If zero or one Cardington cube, nothing to do - if len(card_cubes) <= 1: - return cubes - - # Decide whether height_coord is consistently available - use_height = all(cube.coords(height_coord) for cube in card_cubes) - by_height = {} - for cube in card_cubes: - if use_height: - h = cube.coord(height_coord).points.item() - else: - h = "__no_height_split__" # single bucket - by_height.setdefault(h, CubeList()).append(cube) - - merged_card = CubeList() - - for _h, cubes_at_height in by_height.items(): - by_model = {} - for c in cubes_at_height: - model = c.attributes["model_name"] - by_model.setdefault(model, CubeList()).append(c) - - for model, model_cubes in by_model.items(): - names = {c.name() for c in model_cubes} - if len(names) > 1: - # Different physical variables (e.g. 10m vs 25m covariance) – do NOT concatenate - for c in model_cubes: - merged_card.append(c) - continue - - if len(model_cubes) == 1: - merged = model_cubes[0] - else: - iris.util.equalise_attributes(model_cubes) - iris.util.unify_time_units(model_cubes) - # ---- DE-DUPE IDENTICAL CUBES (same time series & processing) ---- - deduped = iris.cube.CubeList() - seen = set() - - for c in model_cubes: - # time signature (full time axis) - t_sig = tuple(c.coord("time").points) if c.coords("time") else None - - # processing signature (distinguish instantaneous vs max/mean/etc) - cm_sig = tuple( - (cm.method, cm.coord_names, cm.intervals, cm.comments) - for cm in c.cell_methods - ) - - # stash helps disambiguate, but may be absent for Cardington obs - stash = c.attributes.get("STASH", None) - - sig = (c.name(), c.var_name, stash, t_sig, cm_sig, c.units) - - if sig not in seen: - seen.add(sig) - deduped.append(c) - - model_cubes = deduped - model_cubes.sort(key=lambda c: c.coord("time").points.min()) - concatenated = model_cubes.concatenate() - if len(concatenated) == 1: - merged = concatenated[0] - else: - concatenated.sort(key=lambda c: c.coord("time").points.min()) - merged = concatenated.concatenate_cube() - if len(concatenated) > 1: - print( - "DIFF BETWEEN GROUP 0 and 1:\n", - iris.util.describe_diff( - concatenated[0], concatenated[1], output_file=None - ), - ) - - # Sort final cube by time - tdim = merged.coord_dims("time")[0] - order = np.argsort(merged.coord("time").points) - slc = [slice(None)] * merged.ndim - slc[tdim] = order - merged = merged[tuple(slc)] - # t = merged.coord("time").points - # merged = merged[np.argsort(t)] - - # 🔴 CRITICAL: enforce exactly one cube per model - merged.attributes["model_name"] = model - merged_card.append(merged) - - return merged_card + other_cubes - - def addition(addend_1, addend_2): """Addition of two fields. diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml index b2e3b6eaf..62bf695f0 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml @@ -34,8 +34,6 @@ steps: - forecast_reference_time - realization - - operator: misc.merge_cardington - # IMPORTANT: Global temperature-only prune. # filter_multiple_cubes must have exactly ONE key here, # otherwise non-temperature cubes will survive and break convert_units. From 7b4300e03c75ead7ad64eb400377984124355c96 Mon Sep 17 00:00:00 2001 From: Simon Osborne Date: Tue, 12 May 2026 07:45:47 +0100 Subject: [PATCH 04/11] Simplified rose-meta.conf and removed unneeded reference to merge_cardington function in vis recipe --- .../meta/observations/rose-meta.conf | 51 +-------------- src/CSET/operators/misc.py | 62 ------------------- ...n_visibility_single_point_time_series.yaml | 2 - 3 files changed, 3 insertions(+), 112 deletions(-) diff --git a/src/CSET/cset_workflow/meta/observations/rose-meta.conf b/src/CSET/cset_workflow/meta/observations/rose-meta.conf index 6366bedda..386d2aeec 100644 --- a/src/CSET/cset_workflow/meta/observations/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/observations/rose-meta.conf @@ -93,56 +93,11 @@ ns=Observations/Cardington sort-key=sec-d4 title=Cardington UK observations -# Air temperature timeseries -[template variables=CARDINGTON_AIR_TEMPERATURE_SINGLE_POINT_TIME_SERIES] -ns=Observations/Cardington -title=Cardington air temperature timeseries -description=Create Cardington diagnostic variable timeseries -type=python_boolean -compulsory=true -sort-key=0card1 - -# Relative humidity timeseries -[template variables=CARDINGTON_RELATIVE_HUMIDITY_SINGLE_POINT_TIME_SERIES] -ns=Observations/Cardington -title=Cardington relative humidity timeseries -description=Create Cardington diagnostic variable timeseries -type=python_boolean -compulsory=true -sort-key=0card2 - -# Temperature profile timeseries for air variables at various heights -[template variables=CARDINGTON_AIR_SINGLE_POINT_TIME_SERIES] -ns=Observations/Cardington -title=Cardington multi-height temperature air timeseries -description=Create Cardington diagnostic variable timeseries -type=python_boolean -compulsory=true -sort-key=0card3 - -# Generic timeseries for air variables at various heights +# Generic timeseries for variables at various heights [template variables=CARDINGTON_SINGLE_POINT_TIME_SERIES] ns=Observations/Cardington -title=Cardington multi-height timeseries +title=Cardington multi-variable timeseries description=Create Cardington diagnostic variable timeseries type=python_boolean compulsory=true -sort-key=0card4 - -# Generic timeseries for surface (skin) temperature -[template variables=CARDINGTON_SURFACE_TEMPERATURE_TIME_SERIES] -ns=Observations/Cardington -title=Cardington surface temperature timeseries -description=Create Cardington skin temperature variable timeseries -type=python_boolean -compulsory=true -sort-key=0card5 - -# Generic timeseries for surface (skin) temperature -[template variables=CARDINGTON_WIND_SINGLE_POINT_TIME_SERIES] -ns=Observations/Cardington -title=Cardington wind timeseries -description=Create Cardington wind diagnostic variable timeseries -type=python_boolean -compulsory=true -sort-key=0card6 +sort-key=0card1 diff --git a/src/CSET/operators/misc.py b/src/CSET/operators/misc.py index 5e2248b3f..a74d30a06 100644 --- a/src/CSET/operators/misc.py +++ b/src/CSET/operators/misc.py @@ -100,68 +100,6 @@ def remove_scalar_coords(cubes, coords): return cubes -def _concat_over_time_safely(model_cubes: CubeList) -> iris.cube.Cube: - """Concatenate cubes over time. - - coping with: - - scalar time coords (promote to dim with new_axis) - - scalar realization / forecast_reference_time (remove) or singleton dims (squeeze) - - cases where time arrays are identical (skip concatenation). - """ - fixed = CubeList() - - for c in model_cubes: - # --- ensure time is a dimension coordinate --- - if c.coords("time"): - t = c.coord("time") - if c.coord_dims(t) == (): # scalar time on this cube - c = iris.util.new_axis(c, t) # IMPORTANT: reassign! - t = c.coord("time") - if t.has_bounds(): - t.bounds = None - - # --- remove/squeeze scalar coords that block concat --- - for name in ("realization", "forecast_reference_time"): - if c.coords(name): - dims = c.coord_dims(name) - if dims == (): # scalar -> remove - c.remove_coord(name) - else: - # singleton dimension -> squeeze away - dim = dims[0] - if c.shape[dim] == 1: - c = iris.util.squeeze(c) - - fixed.append(c) - - iris.util.equalise_attributes(fixed) - iris.util.unify_time_units(fixed) - - # --- If time point arrays are identical, there is no concat axis --- - if fixed and fixed[0].coords("time"): - sigs = [tuple(c.coord("time").points) for c in fixed] - if len(set(sigs)) == 1: - return fixed[0] - - # --- Try normal concatenation over time --- - fixed.sort(key=lambda c: c.coord("time").points.min() if c.coords("time") else 0) - - try: - return fixed.concatenate_cube() - except iris.exceptions.ConcatenateError: - # Fallback: split into single-time slices and merge (often clearer / more robust) [2](https://engage.cloud.microsoft/main/threads/eyJfdHlwZSI6IlRocmVhZCIsImlkIjoiMTE5MTY5OTYxNSJ9)[3](https://engage.cloud.microsoft/main/threads/eyJfdHlwZSI6IlRocmVhZCIsImlkIjoiMzY5MzE0NDYyMzk5NjkyOCJ9) - slices = CubeList() - for c in fixed: - if c.coords("time") and c.coord_dims("time") != (): - slices.extend(list(c.slices_over("time"))) - else: - slices.append(c) - - iris.util.equalise_attributes(slices) - iris.util.unify_time_units(slices) - return slices.merge_cube() - - def addition(addend_1, addend_2): """Addition of two fields. diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml index f92be3ab3..970c9b981 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml @@ -10,8 +10,6 @@ steps: - $CARDINGTON_30_LABEL - $MODEL_NAME - - operator: misc.merge_cardington - - operator: filters.filter_multiple_cubes Cardington: operator: constraints.generate_var_constraint From 8fdace68027401d6cf82766ea1a8c69c3fd4b6a6 Mon Sep 17 00:00:00 2001 From: Simon Osborne Date: Tue, 12 May 2026 09:29:05 +0100 Subject: [PATCH 05/11] Clean up constraints.py by removing unused comments Removed commented-out code related to variable name constraints. --- src/CSET/operators/constraints.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/CSET/operators/constraints.py b/src/CSET/operators/constraints.py index d8cbf0adc..708e85ce3 100644 --- a/src/CSET/operators/constraints.py +++ b/src/CSET/operators/constraints.py @@ -78,15 +78,12 @@ def generate_var_constraint(varname: str, **kwargs) -> iris.Constraint: or cube.name() in varname ) ) - # if isinstance(varname, (list, tuple)): - # return iris.Constraint(cube_func=lambda cube: cube.name() in varname) # ---- CASE 2: single UM STASH code ---- if _STASH_RE.match(varname): return iris.AttributeConstraint(STASH=varname) # ---- CASE 3: single variable name ---- - # return iris.Constraint(name=varname) return iris.Constraint( cube_func=lambda cube: ( cube.var_name == varname @@ -96,13 +93,6 @@ def generate_var_constraint(varname: str, **kwargs) -> iris.Constraint: ) -# if re.match(r"m[0-9]{2}s[0-9]{2}i[0-9]{3}$", varname): -# varname_constraint = iris.AttributeConstraint(STASH=varname) -# else: -# varname_constraint = iris.Constraint(name=varname) -# return varname_constraint - - def generate_var_name_constraint(varname: str, **kwargs) -> iris.Constraint: """Generate constraint from variable name (var_name). From 3de8541b70c6434e1e0b9b5b26e00d341a863c58 Mon Sep 17 00:00:00 2001 From: Simon Osborne Date: Tue, 12 May 2026 09:41:51 +0100 Subject: [PATCH 06/11] Remove unused logger from misc.py Removed unused logger initialization. --- src/CSET/operators/misc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/CSET/operators/misc.py b/src/CSET/operators/misc.py index a74d30a06..4f707ef97 100644 --- a/src/CSET/operators/misc.py +++ b/src/CSET/operators/misc.py @@ -28,8 +28,6 @@ from CSET.operators._utils import fully_equalise_attributes, get_cube_yxcoordname from CSET.operators.regrid import regrid_onto_cube -LOG = logging.getLogger(__name__) - def noop(x, **kwargs): """Return its input without doing anything to it. From da2325ffac8a56cd6d3fde79afad4d098404dd88 Mon Sep 17 00:00:00 2001 From: Simon Osborne Date: Tue, 12 May 2026 09:49:03 +0100 Subject: [PATCH 07/11] Uncomment cubes.merge() and adjust comments Uncomment merge operation for cubes and clean up code. --- src/CSET/operators/misc.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/CSET/operators/misc.py b/src/CSET/operators/misc.py index 4f707ef97..7507212d3 100644 --- a/src/CSET/operators/misc.py +++ b/src/CSET/operators/misc.py @@ -76,7 +76,7 @@ def remove_attribute( # Combine things that can be merged due to remove removing the # attributes. - # cubes = cubes.merge() + cubes = cubes.merge() # combine items that can be merged after removing unwanted attributes cubes = cubes.concatenate() return cubes @@ -310,8 +310,6 @@ def convert_visibility_to_km(cubes, **kwargs): model = cube.attributes.get("model_name", "") or "" if "Cardington" in model: cube *= 1000 - # data = cube.core_data() - # cube.data = np.ma.array(data * 1.0e3, copy=False) cube.units = "km" else: # UM visibility is in metres – convert with scaling From 94e33839808818125bbd12bd137f6991ee1da4c2 Mon Sep 17 00:00:00 2001 From: Simon Osborne Date: Tue, 12 May 2026 09:54:06 +0100 Subject: [PATCH 08/11] Remove debug print statement in rename_cube function Removed debug print statement for renaming input cubes. --- src/CSET/operators/misc.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/CSET/operators/misc.py b/src/CSET/operators/misc.py index 7507212d3..3490443ee 100644 --- a/src/CSET/operators/misc.py +++ b/src/CSET/operators/misc.py @@ -738,11 +738,6 @@ def rename_cube(cubes: iris.cube.Cube | iris.cube.CubeList, name: str): -------- >>> light_rain_mask = misc.rename_cube(light_rain_mask,"mask_for_light_rainfall" """ - print( - "RENAME INPUT:", - [(c.var_name, c.attributes.get("model_name")) for c in cubes], - flush=True, - ) new_cubelist = iris.cube.CubeList([]) for cube in iter_maybe(cubes): cube.rename(name) From b4b4514739ad9096e1bc8fd619fe174416614673 Mon Sep 17 00:00:00 2001 From: Simon Osborne Date: Tue, 12 May 2026 10:12:17 +0100 Subject: [PATCH 09/11] Simplify constraint creation for variable names --- src/CSET/operators/constraints.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/CSET/operators/constraints.py b/src/CSET/operators/constraints.py index 708e85ce3..ba7707523 100644 --- a/src/CSET/operators/constraints.py +++ b/src/CSET/operators/constraints.py @@ -84,13 +84,7 @@ def generate_var_constraint(varname: str, **kwargs) -> iris.Constraint: return iris.AttributeConstraint(STASH=varname) # ---- CASE 3: single variable name ---- - return iris.Constraint( - cube_func=lambda cube: ( - cube.var_name == varname - or cube.standard_name == varname - or cube.name() == varname - ) - ) + return iris.Constraint(name=varname) def generate_var_name_constraint(varname: str, **kwargs) -> iris.Constraint: From 528cb7381f5c991596f778f94e3a392c81549efa Mon Sep 17 00:00:00 2001 From: Simon Osborne Date: Tue, 12 May 2026 10:54:01 +0100 Subject: [PATCH 10/11] Remove unnecessary enumerate from cubes iteration --- src/CSET/operators/plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/operators/plot.py b/src/CSET/operators/plot.py index fa433392b..d50ea9279 100644 --- a/src/CSET/operators/plot.py +++ b/src/CSET/operators/plot.py @@ -1444,7 +1444,7 @@ def _plot_and_save_histogram_series( # Set default that histograms will produce probability density function # at each bin (integral over range sums to 1). density = True - for cube in enumerate(iter_maybe(cubes)): + for cube in iter_maybe(cubes): # Easier to check title (where var name originates) # than seeing if long names exist etc. # Exception case, where distribution better fits log scales/bins. From 0d5bdda0e4175fa97457e88e43c6fe9d285f8337 Mon Sep 17 00:00:00 2001 From: Simon Osborne Date: Tue, 12 May 2026 13:54:51 +0100 Subject: [PATCH 11/11] Corrections to recipe descriptions --- src/CSET/operators/constraints.py | 21 +++---- src/CSET/operators/misc.py | 56 ++++++++++++++++++- ..._latent_heat_single_point_time_series.yaml | 2 +- ...ensible_heat_single_point_time_series.yaml | 2 +- ...rdington_surface_pressure_time_series.yaml | 2 +- ...ngton_surface_temperature_time_series.yaml | 2 +- ...n_visibility_single_point_time_series.yaml | 2 +- 7 files changed, 67 insertions(+), 20 deletions(-) diff --git a/src/CSET/operators/constraints.py b/src/CSET/operators/constraints.py index d8cbf0adc..fa20e8386 100644 --- a/src/CSET/operators/constraints.py +++ b/src/CSET/operators/constraints.py @@ -78,8 +78,6 @@ def generate_var_constraint(varname: str, **kwargs) -> iris.Constraint: or cube.name() in varname ) ) - # if isinstance(varname, (list, tuple)): - # return iris.Constraint(cube_func=lambda cube: cube.name() in varname) # ---- CASE 2: single UM STASH code ---- if _STASH_RE.match(varname): @@ -87,20 +85,15 @@ def generate_var_constraint(varname: str, **kwargs) -> iris.Constraint: # ---- CASE 3: single variable name ---- # return iris.Constraint(name=varname) - return iris.Constraint( - cube_func=lambda cube: ( - cube.var_name == varname - or cube.standard_name == varname - or cube.name() == varname - ) - ) + return iris.Constraint(name=varname) -# if re.match(r"m[0-9]{2}s[0-9]{2}i[0-9]{3}$", varname): -# varname_constraint = iris.AttributeConstraint(STASH=varname) -# else: -# varname_constraint = iris.Constraint(name=varname) -# return varname_constraint +# cube_func=lambda cube: ( +# cube.var_name == varname +# or cube.standard_name == varname +# or cube.name() == varname +# ) +# ) def generate_var_name_constraint(varname: str, **kwargs) -> iris.Constraint: diff --git a/src/CSET/operators/misc.py b/src/CSET/operators/misc.py index a74d30a06..29e3228c2 100644 --- a/src/CSET/operators/misc.py +++ b/src/CSET/operators/misc.py @@ -78,7 +78,7 @@ def remove_attribute( # Combine things that can be merged due to remove removing the # attributes. - # cubes = cubes.merge() + cubes = cubes.merge() # combine items that can be merged after removing unwanted attributes cubes = cubes.concatenate() return cubes @@ -301,6 +301,60 @@ def mask_fill_values(cubes, ulp_factor=10): return cleaned +def convert_rainfall_amount_to_rate(cubes, **kwargs): + """ + Convert Cardington rainfall from depth per time interval (mm) + to rainfall rate (kg m-2 s-1). + + UM rainfall is already a rate and is left untouched. + """ + if isinstance(cubes, iris.cube.Cube): + cubes = iris.cube.CubeList([cubes]) + else: + cubes = iris.cube.CubeList(cubes) + + for cube in cubes: + model = cube.attributes.get("model_name", "") or "" + + # Only act on Cardington data + if "Cardington" not in model: + continue + + # --- sanity checks --- + if not cube.coords("time"): + raise ValueError( + "Cardington rainfall cube has no time coordinate; " + "cannot derive interval length." + ) + + time = cube.coord("time") + if time.bounds is None: + raise ValueError( + "Cardington rainfall cube has no time bounds; " + "cannot convert accumulated rainfall to a rate." + ) + + # --- derive interval duration in seconds --- + # bounds shape: (ntimes, 2) + bounds = time.bounds + duration_seconds = bounds[:, 1] - bounds[:, 0] + + if np.any(duration_seconds <= 0): + raise ValueError("Non-positive rainfall accumulation interval detected.") + + # reshape for broadcasting along data dimensions + reshape = [1] * cube.ndim + reshape[cube.coord_dims("time")[0]] = -1 + duration_seconds = duration_seconds.reshape(reshape) + + # --- convert depth -> rate --- + # mm / s == kg m-2 s-1 + cube.data = cube.core_data() / duration_seconds + cube.units = "kg m-2 s-1" + + return cubes if len(cubes) > 1 else cubes[0] + + def convert_visibility_to_km(cubes, **kwargs): """Ensure visibility is converted to km if required. UM data is always in metres.""" if isinstance(cubes, iris.cube.Cube): diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_latent_heat_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_latent_heat_single_point_time_series.yaml index dd49ad597..6733f3a79 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_latent_heat_single_point_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_latent_heat_single_point_time_series.yaml @@ -1,4 +1,4 @@ -category: Time series of air temperature at Cardington single point +category: Time series of latent heat flux at Cardington single point title: "Time series of $PLOTNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoint." diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_sensible_heat_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_sensible_heat_single_point_time_series.yaml index 5393faf25..77c7042b2 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_sensible_heat_single_point_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_sensible_heat_single_point_time_series.yaml @@ -1,4 +1,4 @@ -category: Time series of air temperature at Cardington single point +category: Time series of sensible heat flux at Cardington single point title: "Time series of $PLOTNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoint." steps: diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_surface_pressure_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_surface_pressure_time_series.yaml index a7e700b90..1c264f283 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_surface_pressure_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_surface_pressure_time_series.yaml @@ -1,4 +1,4 @@ -category: Time series of variables at Cardington single point +category: Time series of pressure at Cardington single point title: "Time series of $PLOTNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoint." diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml index 62bf695f0..53fb0e4ca 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_surface_temperature_time_series.yaml @@ -1,4 +1,4 @@ -category: Time series of variables at Cardington single point +category: Time series of surface temperature at Cardington single point title: "Time series of $VARNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." description: "Plots a time series of $VARNAME at a selected Cardington gridpoint." steps: diff --git a/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml b/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml index 970c9b981..bc39d0074 100644 --- a/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml +++ b/src/CSET/recipes/surface_fields/cardington/cardington_visibility_single_point_time_series.yaml @@ -1,4 +1,4 @@ -category: Time series of variables at Cardington single point +category: Time series of visibility at Cardington single point title: "Time series of $PLOTNAME at lat_pt:52.10438, lon_pt:-0.42286 specific to Cardington gridpoint." description: "Plots a time series of $PLOTNAME at a selected Cardington gridpoint."