Enable fdb

.gitignore

@@ -57,3 +57,6 @@ uv.lock
 
 # evalml
 .evalml_snakemake_cmd.txt
+
+# Core dumps
+core_*

config/varda-single-1.0-inference-only.yaml

@@ -0,0 +1,43 @@
+# yaml-language-server: $schema=../workflow/tools/config.schema.json
+description: |
+  Run inference only for Varda-single-1.0 (no evaluation against observations).
+
+config_label: varda-single-1.0-inference-only
+
+dates:
+  start: 2025-03-01T00:00
+  end: 2025-03-03T00:00
+  frequency: 24h
+
+runs:
+  - temporal_downscaler:
+      checkpoint: https://service.meteoswiss.ch/mlstore#/models/sruc-m-2-interpolator/versions/3
+      label: Varda-single-1.0
+      steps: 0/120/1
+      config: resources/inference/configs/sgm-temporal-downscaler-global_trimedge_multi.yaml
+      output: fdb
+      extra_requirements:
+        - anemoi-datasets==0.5.35
+        - /scratch/mch/cosuna/evalml_fdb/anemoi-inference
+      forecaster:
+        checkpoint: https://service.meteoswiss.ch/mlstore#/models/sruc-m-1-forecaster/versions/4
+        config: resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper.yaml
+        steps: 0/120/6
+        extra_requirements:
+          - anemoi-inference==0.11.1
+
+locations:
+  output_root: output/
+
+profile:
+  executor: slurm
+  global_resources:
+    gpus: 16
+  default_resources:
+    slurm_partition: "postproc"
+    cpus_per_task: 1
+    mem_mb_per_cpu: 1800
+    runtime: "1h"
+    gpus: 0
+    slurm_account: s83
+  jobs: 50

pyproject.toml

@@ -26,6 +26,7 @@ dependencies = [
     "scores>=2.0.0",
     "eccodes>=2.44,<2.48",
     "eccodes-cosmo-resources-python==2.44.0.1",
+    "pyfdb", # for reading VARDA-SINGLE forecast output archived directly to FDB
     "earthkit-data>=1.0,<1.1",
     "earthkit-utils>=1.0,<1.1",
     "earthkit-plots>=1.0,<1.1",

resources/fdb/patch_metkit_language.py

@@ -0,0 +1,75 @@
+"""Patch metkitlib's bundled language.yaml to recognize VARDA-SINGLE's custom
+`model` values for FDB *retrieve* requests.
+
+`model` is not a standard MARS/metkit key -- it was added purely for FDB archival,
+via the `marsModel` concepts in eccodes-cosmo-mars's local.215.def/local.98.def
+(see resources/fdb/realtime-varda.schema and the VARDA-SINGLE FDB archival notes).
+`fdb.list()` doesn't validate key values against any vocabulary, so archival and
+completeness checks work fine without this patch. But `fdb.retrieve()` parses
+requests through metkit's MarsLanguage, which validates enum-typed keys like
+`model` against language.yaml's fixed value list -- and rejects anything not
+already registered there, even though the value is a perfectly valid archived key.
+
+Run once per venv build, after `metkitlib` is installed (it ships its own copy of
+language.yaml under its package data; there's no config-based override for this,
+unlike ECCODES_DEFINITION_PATH/FDB5_CONFIG_FILE, so the file has to be edited in place).
+"""
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+# Custom VARDA-SINGLE model names, per marsModel concepts in eccodes-cosmo-mars's
+# local.215.def (ICON, GPI=180) / local.98.def (IFS, GPI=180). "varda-single-g" is
+# the global-domain variant seen from the temporal downscaler's IFS-grid output.
+_CUSTOM_MODELS = ["varda-single", "varda-single-g"]
+
+# Structural anchor, not a specific value: metkitlib/share/metkit/language.yaml's
+# top-level `  model:` key ends with the default (no-context) `type: enum` block,
+# listing recognized model names -- its last entry (and thus the exact line to
+# anchor on) shifts between metkitlib releases as upstream adds new models, so
+# this matches the *end of the model: block* (a blank line before the next
+# top-level `  key:`) instead of any one entry.
+_BLOCK_END_RE = re.compile(r"(\n  model:\n(?:.*\n)*?)(\n  [a-zA-Z_]+:)")
+
+
+def patch(language_yaml: Path) -> None:
+    text = language_yaml.read_text()
+    if all(f"- [{m}]" in text for m in _CUSTOM_MODELS):
+        print(f"[patch_metkit_language] {language_yaml} already patched, skipping")
+        return
+    m = _BLOCK_END_RE.search(text)
+    if m is None:
+        raise RuntimeError(
+            f"Could not locate the end of the 'model:' block in {language_yaml} -- "
+            "metkitlib's language.yaml layout may have changed; update "
+            "_BLOCK_END_RE in this script."
+        )
+    addition = "\n".join(f"          - [{model}]" for model in _CUSTOM_MODELS)
+    text = text[: m.end(1)] + addition + text[m.end(1) :]
+    language_yaml.write_text(text)
+    print(f"[patch_metkit_language] added {_CUSTOM_MODELS} to {language_yaml}")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "venv",
+        type=Path,
+        help="Path to the venv root (contains lib/python*/site-packages)",
+    )
+    args = parser.parse_args()
+
+    matches = list(
+        args.venv.glob("lib/python*/site-packages/metkitlib/share/metkit/language.yaml")
+    )
+    if not matches:
+        raise RuntimeError(f"metkitlib language.yaml not found under {args.venv}")
+    for language_yaml in matches:
+        patch(language_yaml)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

resources/fdb/realtime-varda.schema

@@ -0,0 +1,107 @@
+
+# * Format of the rules is:
+
+# [a1, a2, a3 ...[b1, b2, b3... [c1, c2, c3...]]]
+
+# - The first level (a) defines which attributes are used to name the top level directory
+# - The second level (b) defines which attributes are used to name the data files
+# - The third level (c) defines which attributes are used as index keys
+
+# * Rules can be grouped
+
+# [a1, a2, a3 ...
+#   [b1, b2, b3... [c1, c2, c3...]]
+#   [B1, B2, B3... [C1, C2, C3...]]
+# ]
+
+# * A list of values can be given for an attribute
+# [ ..., stream=enfo/efov, ... ]
+# This will be used when matching rules.
+
+# * Attributes can be typed
+#   Globally, at the begining of this file:
+
+#   refdate: Date;
+
+#   or in the context of a rule:
+#   [type=cl, ... [date:ClimateMonth, ...]]
+
+# Typing attributes is done when the user's requests or the GRIB values need to be modified before directories, files and indexes are created. For example, ClimateMonth will transform 2010-04-01 to 'may' internally.
+
+# * Attributes can be optional
+# [ step, levelist?, param ]
+# They will be replaced internally by an empty value. It is also posiible to provide a default subtitution value: e.g. [domain?g] will consider the domain to be 'g' if missing.
+
+# * Attributes can be removed:
+# [grid-]
+# This is useful to remove attributes present in the GRIB that should not be ignored
+
+# * Rules are matched:
+
+# - If the attributes are present in the GRIB/Request, or marked optional or ignored
+# - If a list of possible value is provided, one of them must match, for example
+#   [ class, expver, stream=enfo/efov, date, time, domain ]
+#   will match either stream=enfo or stream=efov, all other attributes will be matched if they exist in the GRIB or user's request
+
+# * On archive:
+# - Attributes are extracted from the GRIB (namespace 'mars'), possibly modified by the attribute type
+# - Only the first rule is used, so order is important
+# - All GRIB attributes must be used by the rules, otherwise an error is raised
+
+# * On retrieve:
+# - Attributes are extracted from the user's request, possibly modified by the attribute type (e.g. for handling of U/V)
+# - All the matching rules are considered
+# - Only attributes listed in the rules are used to extract values from the user's request
+
+
+# Default types
+
+param:      Param;
+step:       Step;
+date:       Date;
+hdate:      Date;
+refdate:    Date;
+latitude:   Double;
+longitude:  Double;
+level:      Double;
+levelist:   Double;
+grid:       Grid;
+expver:     Expver;
+
+time:       Time;
+fcmonth:    Integer;
+
+number:     Integer;
+frequency:  Integer;
+direction:  Integer;
+channel:    Integer;
+
+instrument: Integer;
+ident:      Integer;
+
+diagnostic: Integer;
+iteration:  Integer;
+system:     Integer;
+method:     Integer;
+
+# ???????
+
+# reference:  Integer;
+# fcperiod:   Integer;
+
+# opttime:    Integer;
+# leadtime:   Integer;
+
+# quantile:   ??????
+# range:      ??????
+
+# band:       Integer;
+
+
+########################################################
+# These rules must be first, otherwise fields of These
+# classes will be index with the default rule for oper #expver?, model?, type?
+
+[ date, time, stream, class, expver, model, type, domain-
+    [ levtype, number?
+        [ step, param, levelist?, timespan?none ]]]

resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper.yaml

@@ -32,6 +32,7 @@ output:
         path: grib/{date}{time:04}_{step:03}.grib
         encoding:
           typeOfGeneratingProcess: 2
+          generatingProcessIdentifier: 180
           centre: lssw
         templates:
           samples: resources/templates_index_icon.yaml
@@ -84,7 +85,17 @@ output:
         path: grib/ifs-{date}{time:04}_{step:03}.grib
         encoding:
           typeOfGeneratingProcess: 2
+          generatingProcessIdentifier: 180
           centre: ecmf
+          # localNumberOfExperiment is not available in IFS/ECMWF GRIB templates;
+          # omitting it means marsClass=od (GPI=180 fallback) for the global grid
+          # Set native marsStream/marsType codetable keys so g2_mars_labeling returns
+          # stream=enfo and type=fc.  Cannot use a concept override in local.98.def
+          # for marsStream because it conflicts with the codetable key that
+          # g2_mars_labeling expects as input.  Setting them natively is portable
+          # across eccodes builds.
+          marsStream: enfo
+          marsType: fc
         templates:
           samples: resources/templates_index_ifs.yaml
         post_processors:

resources/inference/configs/sgm-temporal-downscaler-global_trimedge_multi.yaml

@@ -134,6 +134,7 @@ output:
         path: grib/{dateTime}_{step:03}.grib
         encoding:
           typeOfGeneratingProcess: 2
+          generatingProcessIdentifier: 180
         templates:
             samples: resources/templates_index_icon.yaml
         post_processors:
@@ -182,19 +183,33 @@ output:
                 offset: 0
                 param: CLCH
           # here, the trimedge mask can be specified when available
-          # at overlap steps (multiples of the forecaster stride) replace the
-          # re-predicted prognostics with the forecaster's boundary values
-          - forward_transform_filter:
-              copy-prognostic-from-forecaster:
-                forecaster_path: forecaster/20*
-                common_leadtime: 6h
-                params_to_keep: [tp] # NOTE, if more diagnostics are added, update accordingly.
-                namer: *namer
+          # DISABLED: copy-prognostic-from-forecaster is incompatible with an
+          # FDB-backed forecaster (it globs the local forecaster/ dir). Waiting
+          # on the design decision in ecmwf/anemoi-inference#468 for a native,
+          # backend-agnostic replacement before re-enabling. Until then,
+          # overlap steps (multiples of the forecaster stride) carry the
+          # downscaler's own re-predicted prognostics, not the forecaster's.
+          # - forward_transform_filter:
+          #     copy-prognostic-from-forecaster:
+          #       forecaster_path: forecaster/20*
+          #       common_leadtime: 6h
+          #       params_to_keep: [tp] # NOTE, if more diagnostics are added, update accordingly.
+          #       namer: *namer
 
     - grib:
         path: grib/ifs-{dateTime}_{step:03}.grib
         encoding:
           typeOfGeneratingProcess: 2
+          generatingProcessIdentifier: 180
+          # localNumberOfExperiment is not available in IFS/ECMWF GRIB templates;
+          # omitting it means marsClass=od (GPI=180 fallback) for the global grid
+          # Set native marsStream/marsType codetable keys so g2_mars_labeling returns
+          # stream=enfo and type=fc.  Cannot use a concept override in local.98.def
+          # for marsStream because it conflicts with the codetable key that
+          # g2_mars_labeling expects as input.  Setting them natively is portable
+          # across eccodes builds.
+          marsStream: enfo
+          marsType: fc
         templates:
           samples: resources/templates_index_ifs.yaml
         post_processors:
@@ -204,12 +219,13 @@ output:
               inverse: true
           - assign_mask: # fill local/global overlapping points with nan
               mask: "global/cutout_mask"
-          - forward_transform_filter:
-              copy-prognostic-from-forecaster:
-                forecaster_path: forecaster/ifs*
-                common_leadtime: 6h
-                params_to_keep: [tp] # NOTE, if more diagnostics are added, update accordingly.
-                namer: *namer
+          # DISABLED: see note above the lam_0 tee's copy-prognostic-from-forecaster block.
+          # - forward_transform_filter:
+          #     copy-prognostic-from-forecaster:
+          #       forecaster_path: forecaster/ifs*
+          #       common_leadtime: 6h
+          #       params_to_keep: [tp] # NOTE, if more diagnostics are added, update accordingly.
+          #       namer: *namer
         modifiers:
           - patches:
             - variable:

resources/inference/templates/templates.md

@@ -0,0 +1,27 @@
+# GRIB Template Notes
+
+## VARDA templates (`varda-*.grib`)
+
+Created for VARDA-SINGLE (generatingProcessIdentifier=180), a deterministic AI forecast model.
+Derived from the `icon-ch1-*.grib` templates with the following changes:
+
+| Key | icon-ch1 value | varda value | Reason |
+|-----|---------------|-------------|--------|
+| `productDefinitionTemplateNumber` | 1 (ensemble instantaneous) / 11 (ensemble accumulation) | 0 (deterministic instantaneous) / 8 (deterministic accumulation) | VARDA-SINGLE is deterministic, not an ensemble |
+| `typeOfGeneratingProcess` | 4 (ensemble forecast) | 2 (forecast) | Same reason |
+| `typeOfProcessedData` | 5 (control forecast member) | 1 (forecast products) | Same reason |
+
+### Files
+
+| File | levtype | PDT | Notes |
+|------|---------|-----|-------|
+| `varda-typeOfLevel=isobaricInhPa.grib` | pl | 0 | Pressure level instantaneous fields |
+| `varda-typeOfLevel=surface.grib` | sfc | 0 | Surface instantaneous fields (FR_LAND, PS, FIS, SKT, SSO_SIGMA, SSO_STDH, CLCT) |
+| `varda-typeOfLevel=heightAboveGround.grib` | sfc | 0 | Height-above-ground fields (T_2M, TD_2M, U_10M, V_10M) |
+| `varda-typeOfLevel=meanSea.grib` | sfc | 0 | Mean sea level fields (PMSL) |
+| `varda-shortName=TOT_PREC.grib` | sfc | 8 | Total precipitation (temporal accumulation) |
+| `varda-shortName=CLCH.grib` | pl | 0 | High cloud cover |
+| `varda-shortName=CLCM.grib` | pl | 0 | Medium cloud cover |
+
+Low cloud cover (CLCL) uses `icon-ch1-shortName=CLCL.grib` (not a `varda-*` variant) --
+merged in from main's own template batch, kept for consistency with its siblings there.