Add side-by-side map comparisons of runs and baselines to the showcase

src/data_input/__init__.py

@@ -323,7 +323,7 @@ def _select_valid_times(ds, times: np.datetime64, strict: bool = False):
         )
         return ds.sel(time=times_np[times_included])
     else:
-        raise ValueError(
+        raise LookupError(
             "Valid times are not included in the dataset. "
             "Please check the valid times and the dataset."
         )

src/evalml/config.py

@@ -445,6 +445,55 @@ class AnimationsConfig(BaseModel):
     )
 
 
+class ComparisonsConfig(BaseModel):
+    """Configuration for side-by-side map comparisons of truth, runs and baselines."""
+
+    enabled: bool = Field(
+        default=False,
+        description=(
+            "Whether to generate side-by-side maps: one figure per init time, lead "
+            "time, parameter and domain, with a panel for the truth (if gridded), "
+            "each run and each selected baseline."
+        ),
+    )
+    truth: bool = Field(
+        default=True,
+        description=(
+            "Whether to include a truth panel. Ignored for station truth "
+            "(jretrievedwh), which cannot be drawn as a map."
+        ),
+    )
+    baselines: List[str] = Field(
+        default_factory=list,
+        description="Labels of the baselines to add as panels.",
+    )
+    domains: List[str | DomainConfig] = Field(
+        default=["alps"],
+        description=(
+            "Domains to plot. Same format as `animations.domains`: a named domain "
+            "defined in plotting.DOMAINS or a custom domain dict."
+        ),
+    )
+    lead_times: List[int] = Field(
+        default=[24],
+        description=(
+            "Lead times (hours) to plot. A run or baseline without data at a lead "
+            "time is shown as an empty panel."
+        ),
+    )
+
+    model_config = {"extra": "forbid"}
+
+    @model_validator(mode="after")
+    def _enabled_requires_a_reference(self):
+        if self.enabled and not (self.truth or self.baselines):
+            raise ValueError(
+                "comparisons need something to compare the runs with: enable "
+                "`truth` or list `baselines`."
+            )
+        return self
+
+
 class ScorecardConfig(BaseModel):
     """Configuration for a single named scorecard."""
 
@@ -558,6 +607,10 @@ class ShowcaseConfig(BaseModel):
         default_factory=AnimationsConfig,
         description="Configuration for animation generation.",
     )
+    comparisons: ComparisonsConfig = Field(
+        default_factory=ComparisonsConfig,
+        description="Configuration for side-by-side map comparisons.",
+    )
 
 
 class Locations(BaseModel):
@@ -1028,6 +1081,19 @@ def validate_scorecard_references(self) -> "ConfigModel":
                 )
         return self
 
+    @model_validator(mode="after")
+    def validate_comparison_baselines(self) -> "ConfigModel":
+        labels = [
+            item.baseline.label for item in self.runs if isinstance(item, BaselineItem)
+        ]
+        unknown = [b for b in self.showcase.comparisons.baselines if b not in labels]
+        if unknown:
+            raise ValueError(
+                f"showcase.comparisons.baselines {unknown} do not match the label of "
+                f"any baseline in `runs`: {labels}"
+            )
+        return self
+
     model_config = {
         "extra": "forbid",  # fail on misspelled keys
         "populate_by_name": True,

src/plotting/__init__.py

@@ -60,6 +60,32 @@ def get_projection(name: str) -> "ccrs.Projection":
 }
 
 
+def resolve_region(
+    region_name: str, region_cfg: dict, lead_time: int = 0
+) -> tuple["ccrs.Projection", list[float] | None]:
+    """Return (projection, extent) for a showcase region.
+
+    ``region_cfg`` is one entry of the dict built by ``parse_showcase_regions``.
+    A custom ``extent`` takes precedence; otherwise the named domain in
+    ``DOMAINS`` is used, rotated by ``lead_time`` when ``rotate`` is set.
+    """
+    if region_cfg.get("extent") is not None:
+        projection = get_projection(region_cfg.get("projection") or "orthographic")
+        return projection, region_cfg["extent"]
+    domain = DOMAINS[region_name]
+    if region_cfg.get("rotate"):
+        base = domain["projection"].proj4_params
+        central_longitude = (
+            base["lon_0"] + 360.0 * lead_time / region_cfg["hours_per_revolution"]
+        ) % 360
+        #  central_longitude=central_longitude, central_latitude=0.0 for zero-centered
+        projection = ccrs.Orthographic(
+            central_longitude=central_longitude, central_latitude=base["lat_0"]
+        )
+        return projection, domain["extent"]
+    return domain["projection"], domain["extent"]
+
+
 class StatePlotter:
     """A class to plot state fields on various DOMAINS."""
 
@@ -208,16 +234,19 @@ def plot_field(
         # Normalize style and color-related kwargs
         style_to_use, plot_kwargs = self._prepare_plot_kwargs(style, kwargs)
 
-        # Temporarily suppress earthkit-plots internal source-based kwargs
-        with self._temporary_plot_kwargs_override(subplot):
-            subplot.tricontourf(
-                x=x[finite],
-                y=y[finite],
-                z=field[finite],
-                style=style_to_use,
-                transform=proj,
-                **plot_kwargs,
-            )  # for earthkit.plots to work properly cmap and norm are needed here
+        # An all-NaN field (e.g. accumulations at lead 0) leaves nothing to
+        # contour: only the map layers below are drawn.
+        if finite.any():
+            # Temporarily suppress earthkit-plots internal source-based kwargs
+            with self._temporary_plot_kwargs_override(subplot):
+                subplot.tricontourf(
+                    x=x[finite],
+                    y=y[finite],
+                    z=field[finite],
+                    style=style_to_use,
+                    transform=proj,
+                    **plot_kwargs,
+                )  # for earthkit.plots to work properly cmap and norm are needed here
         # TODO: gridlines etc would be nicer to have in the init, but I didn't get
         # them to overlay the plot layer
 

src/plotting/styles.py

@@ -0,0 +1,116 @@
+"""Shared plotting styles and field preprocessing for map plots."""
+
+from earthkit.meteo.utils.convert import kelvin_to_celsius
+import earthkit.meteo.wind as ekm_wind
+import earthkit.plots as ekp
+from matplotlib.colors import Colormap
+import numpy as np
+
+from plotting.colormap_defaults import CMAP_DEFAULTS
+
+
+def _precip_lookup(accu):
+    """Key of the TOT_PREC palette for an accumulation window of ``accu`` hours.
+
+    Palettes exist only for some windows (1h, 6h); others use the longest
+    defined window not exceeding ``accu`` (e.g. 3h -> 1h, 24h -> 6h).
+    """
+    windows = sorted(
+        int(key[len("TOT_PREC_") : -1])
+        for key in CMAP_DEFAULTS
+        if key.startswith("TOT_PREC_") and key.endswith("H")
+    )
+    window = max((w for w in windows if w <= accu), default=windows[0])
+    return f"TOT_PREC_{window}H"
+
+
+def get_style(param, units_override=None, accu=1):
+    """Get style and colormap settings for the plot."""
+    lookup = _precip_lookup(accu) if param == "TOT_PREC" else param
+    cfg = CMAP_DEFAULTS[lookup]
+    units = units_override if units_override is not None else cfg.get("units", "")
+
+    bounds = cfg.get("bounds", cfg.get("levels", None))
+    prebuilt_cmap = cfg.get("cmap", None)
+
+    # When the config provides a pre-built matplotlib Colormap (e.g. a
+    # ListedColormap), we must use the earthkit Style's vmin/vmax path, which
+    # handles isinstance(colors, Colormap) correctly.  The levels path calls
+    # cmap_and_norm → len(Colormap) → TypeError.
+    #
+    # earthkit's configure_style intercepts _STYLE_KWARGS (cmap/colors/levels/vmin/vmax)
+    # from tricontourf kwargs before matplotlib sees them.  To work around this:
+    #  - embed the Colormap directly in the Style via colors=
+    #  - inject bounds as 'levels' into style._kwargs so they survive to matplotlib
+    #  - pass only norm= as a kwarg (not in _STYLE_KWARGS, so not intercepted)
+    extend = cfg.get("extend", "both")
+
+    if isinstance(prebuilt_cmap, Colormap) and bounds is not None:
+        style = ekp.styles.Style(
+            colors=prebuilt_cmap,
+            vmin=bounds[0],
+            vmax=bounds[-1],
+            extend=extend,
+            units=units,
+        )
+        style._kwargs["levels"] = list(bounds)
+        return {
+            "style": style,
+            "norm": cfg.get("norm", None),
+        }
+
+    return {
+        "style": ekp.styles.Style(
+            levels=bounds,
+            extend=extend,
+            units=units,
+            colors=cfg.get("colors", None),
+        ),
+        "norm": cfg.get("norm", None),
+        "cmap": prebuilt_cmap,
+        "vmin": cfg.get("vmin", None),
+        "vmax": cfg.get("vmax", None),
+    }
+
+
+def preprocess_field(param: str, state: dict):
+    """
+    - Wind speed: sqrt(u^2 + v^2)
+    - Everything else: see :func:`prepare_values`
+    Returns: (field_array, units_override or None)
+    """
+    fields = state["fields"]
+    if param == "SP_10M":
+        values = ekm_wind.speed(fields["U_10M"], fields["V_10M"])
+    elif param == "SP":
+        values = ekm_wind.speed(fields["U"], fields["V"])
+    else:
+        values = fields[param]
+    return prepare_values(param, values)
+
+
+def prepare_values(param: str, values):
+    """Unit conversion and clipping of an already derived ``param`` field.
+
+    - Temperatures: K -> °C
+    - Precipitation: m -> mm
+    Used directly by callers that load derived params (e.g. ``SP_10M``).
+    Returns: (field_array, units_override or None)
+    """
+    if param in ("T_2M", "TD_2M", "TW_2M", "SKT", "T", "TD"):
+        return kelvin_to_celsius(values), "°C"
+    if param in ("SP_10M", "SP"):
+        return values, "m/s"
+    if param == "TOT_PREC":
+        return np.maximum(values, 0), "mm"
+    if param in ("CLCT", "CLCL", "CLCM", "CLCH"):
+        # Cloud cover is in percent. Avoid exact 0/100 plateaus breaking
+        # tricontourf on orthographic projections (tmp/reproduce_clct_bug.py).
+        # Pair with extend="neither". Any new bounded field with silent-blank
+        # or GeometryCollection-crash globe frames likely needs the same
+        # clip-away-from-boundary fix.
+        return np.clip(values, 1e-4, 100 - 1e-4), None
+    if param == "SSRD":
+        # Same issue, bottom boundary only (night-side plateau).
+        return np.maximum(values, 1e-6), None
+    return values, None

tests/unit/test_config.py

@@ -90,6 +90,46 @@ def test_experiment_optional(example_config):
     assert cfg.experiment is None
 
 
+def test_comparisons_truth_only(example_config):
+    example_config["showcase"]["comparisons"] = {"enabled": True, "baselines": []}
+    cfg = ConfigModel.model_validate(example_config)
+    assert cfg.showcase.comparisons.truth is True
+
+
+def test_comparisons_need_a_reference(example_config):
+    example_config["showcase"]["comparisons"] = {
+        "enabled": True,
+        "truth": False,
+        "baselines": [],
+    }
+    with pytest.raises(ValueError, match="enable `truth` or list `baselines`"):
+        ConfigModel.model_validate(example_config)
+
+
+def test_comparison_baselines_must_be_baselines(example_config):
+    baseline_label = next(
+        item["baseline"]["label"]
+        for item in example_config["runs"]
+        if "baseline" in item
+    )
+    forecaster_label = next(
+        item[k]["label"]
+        for item in example_config["runs"]
+        for k in item
+        if k in ("forecaster", "temporal_downscaler")
+    )
+    example_config["showcase"]["comparisons"] = {
+        "enabled": True,
+        "baselines": [baseline_label],
+    }
+    ConfigModel.model_validate(example_config)
+
+    for label in ("no-such-label", forecaster_label):
+        example_config["showcase"]["comparisons"]["baselines"] = [label]
+        with pytest.raises(ValueError, match="do not match the label of any baseline"):
+            ConfigModel.model_validate(example_config)
+
+
 def test_score_on_common_points(example_config):
     """score_on_common_points accepts a subset of experiment.params only."""
 

tests/unit/test_plot_forecast_comparison.py

@@ -0,0 +1,68 @@
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parents[2] / "workflow" / "scripts"))
+import plot_forecast_comparison  # noqa: E402
+from plot_forecast_comparison import parse_source  # noqa: E402
+
+
+def test_parse_source():
+    assert parse_source("ICON-CH1=/store/x=y") == ("ICON-CH1", "/store/x=y")
+
+
+@pytest.mark.parametrize("text", ["no-separator", "=root", "label="])
+def test_parse_source_rejects(text):
+    with pytest.raises(ValueError):
+        parse_source(text)
+
+
+@pytest.fixture
+def run_main(monkeypatch, tmp_path):
+    """Run main() without plotting; loaders raise for roots listed in `errors`."""
+    loaded = []
+
+    def run(args, errors):
+        def loader(root, *_):
+            if root in errors:
+                raise errors[root]
+            loaded.append(root)
+            return np.zeros(1), np.zeros(1), np.zeros(1)
+
+        monkeypatch.setattr(plot_forecast_comparison, "load_source", loader)
+        monkeypatch.setattr(plot_forecast_comparison, "load_truth", loader)
+        monkeypatch.setattr(plot_forecast_comparison, "StatePlotter", lambda *_: None)
+        monkeypatch.setattr(
+            sys,
+            "argv",
+            ["plot_forecast_comparison.py", *args]
+            + ["--date", "202401010000", "--leadtime", "6", "--param", "T_2M"]
+            + ["--regions_json", "{}", "--outdir", str(tmp_path)],
+        )
+        plot_forecast_comparison.main()
+        return loaded
+
+    return run
+
+
+def test_missing_baseline_and_truth_are_skipped(run_main):
+    args = ["--source", "ML=run", "--baseline", "ICON=icon", "--truth", "truth"]
+    errors = {"icon": FileNotFoundError("icon"), "truth": LookupError("truth")}
+    assert run_main(args, errors) == ["run"]
+
+
+def test_candidate_failure_raises(run_main):
+    with pytest.raises(FileNotFoundError):
+        run_main(["--source", "ML=run"], {"run": FileNotFoundError("run")})
+
+
+def test_unexpected_baseline_failure_raises(run_main):
+    with pytest.raises(ValueError):
+        run_main(["--baseline", "ICON=icon"], {"icon": ValueError("corrupt")})
+
+
+def test_nothing_loaded_raises(run_main):
+    with pytest.raises(RuntimeError):
+        run_main(["--baseline", "ICON=icon"], {"icon": FileNotFoundError("icon")})