diff --git a/resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper-clouds.yaml b/resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper-clouds.yaml new file mode 100644 index 00000000..f1d0aed1 --- /dev/null +++ b/resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper-clouds.yaml @@ -0,0 +1,138 @@ +lead_time: 120h +write_initial_state: true +allow_nans: true + +env: + ANEMOI_INFERENCE_NUM_CHUNKS: 8 # OOM error if not set + +# inputs +input: + test: + use_original_paths: true + +post_processors: + - accumulate_from_start_of_forecast: # accumulate tp from start of forecast + accumulations: + - tp + # filters see the model name (tp) with anemoi-inference <= 0.11.1 and the GRIB name (TOT_PREC) from 0.11.2 + - forward_transform_filter: + rescale: + scale: 1000 # convert units from m to kg m-2 + offset: 0 + param: tp + - forward_transform_filter: + rescale: + scale: 1000 # convert units from m to kg m-2 + offset: 0 + param: TOT_PREC + +output: + tee: + - grib: + path: grib/{date}{time:04}_{step:03}.grib + encoding: + typeOfGeneratingProcess: 2 + centre: lssw + templates: + samples: resources/templates_index_icon.yaml + post_processors: + - extract_mask: # removes global points + mask: "lam_0/cutout_mask" + as_slice: true + # filters see the model name (tcc) with anemoi-inference <= 0.11.1 and the GRIB name (CLCT) from 0.11.2 + - forward_transform_filter: # cloud cover fraction -> percent, as in ICON + rescale: + scale: 100 + offset: 0 + param: tcc + - forward_transform_filter: # cloud cover fraction -> percent, as in ICON + rescale: + scale: 100 + offset: 0 + param: CLCT + - forward_transform_filter: # cloud cover fraction -> percent, as in ICON + rescale: + scale: 100 + offset: 0 + param: lcc + - forward_transform_filter: # cloud cover fraction -> percent, as in ICON + rescale: + scale: 100 + offset: 0 + param: CLCL + - forward_transform_filter: # cloud cover fraction -> percent, as in ICON + rescale: + scale: 100 + offset: 0 + param: mcc + - forward_transform_filter: # cloud cover fraction -> percent, as in ICON + rescale: + scale: 100 + offset: 0 + param: CLCM + - forward_transform_filter: # cloud cover fraction -> percent, as in ICON + rescale: + scale: 100 + offset: 0 + param: hcc + - forward_transform_filter: # cloud cover fraction -> percent, as in ICON + rescale: + scale: 100 + offset: 0 + param: CLCH + - grib: + path: grib/ifs-{date}{time:04}_{step:03}.grib + encoding: + typeOfGeneratingProcess: 2 + centre: ecmf + templates: + samples: resources/templates_index_ifs.yaml + post_processors: + - extract_mask: # removes lam points + mask: "lam_0/cutout_mask" + as_slice: true + inverse: true + - assign_mask: # fill local/global overlapping points with nan + mask: "global/cutout_mask" + modifiers: + - patches: + - variable: + U_10M: {"param": 165, "shortName": "10u"} + 10u: {"param": 165, "shortName": "10u"} + V_10M: {"param": 166, "shortName": "10v"} + 10v: {"param": 166, "shortName": "10v"} + TD_2M: {"param": 168, "shortName": "2d"} + 2d: {"param": 168, "shortName": "2d"} + T_2M: {"param": 167, "shortName": "2t"} + 2t: {"param": 167, "shortName": "2t"} + FR_LAND: {"param": 172, "shortName": "lsm"} + lsm: {"param": 172, "shortName": "lsm"} + PMSL: {"param": 151, "shortName": "msl"} + msl: {"param": 151, "shortName": "msl"} + PS: {"param": 134, "shortName": "sp"} + sp: {"param": 134, "shortName": "sp"} + SSO_SIGMA: {"param": 163, "shortName": "slor"} + slor: {"param": 163, "shortName": "slor"} + SSO_STDH: {"param": 160, "shortName": "sdor"} + sdor: {"param": 160, "shortName": "sdor"} + CLCT: {"param": 164, "shortName": "tcc"} + tcc: {"param": 164, "shortName": "tcc"} + CLCL: {"param": 186, "shortName": "lcc"} + lcc: {"param": 186, "shortName": "lcc"} + CLCM: {"param": 187, "shortName": "mcc"} + mcc: {"param": 187, "shortName": "mcc"} + CLCH: {"param": 188, "shortName": "hcc"} + hcc: {"param": 188, "shortName": "hcc"} + SKT: {"param": 235, "shortName": "skt"} + skt: {"param": 235, "shortName": "skt"} + TOT_PREC: {"param": 228, "shortName": "tp"} + tp: {"param": 228, "shortName": "tp"} + z: {"param": 129, "shortName": "z"} + "^q_(\\d+)$": {"param": 133, "shortName": "q"} + "^t_(\\d+)$": {"param": 130, "shortName": "t"} + "^u_(\\d+)$": {"param": 131, "shortName": "u"} + "^v_(\\d+)$": {"param": 132, "shortName": "v"} + "^w_(\\d+)$": {"param": 135, "shortName": "w"} + "^z_(\\d+)$": {"param": 129, "shortName": "z"} + +patch_metadata: resources/sgm-multidataset-ich1-oper-clouds-patch.yaml diff --git a/resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper.yaml b/resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper.yaml index b6a0c4cb..f5c79a4b 100644 --- a/resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper.yaml +++ b/resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper.yaml @@ -39,47 +39,6 @@ output: - extract_mask: # removes global points mask: "lam_0/cutout_mask" as_slice: true - # filters see the model name (tcc) with anemoi-inference <= 0.11.1 and the GRIB name (CLCT) from 0.11.2 - - forward_transform_filter: # cloud cover fraction -> percent, as in ICON - rescale: - scale: 100 - offset: 0 - param: tcc - - forward_transform_filter: # cloud cover fraction -> percent, as in ICON - rescale: - scale: 100 - offset: 0 - param: CLCT - - forward_transform_filter: # cloud cover fraction -> percent, as in ICON - rescale: - scale: 100 - offset: 0 - param: lcc - - forward_transform_filter: # cloud cover fraction -> percent, as in ICON - rescale: - scale: 100 - offset: 0 - param: CLCL - - forward_transform_filter: # cloud cover fraction -> percent, as in ICON - rescale: - scale: 100 - offset: 0 - param: mcc - - forward_transform_filter: # cloud cover fraction -> percent, as in ICON - rescale: - scale: 100 - offset: 0 - param: CLCM - - forward_transform_filter: # cloud cover fraction -> percent, as in ICON - rescale: - scale: 100 - offset: 0 - param: hcc - - forward_transform_filter: # cloud cover fraction -> percent, as in ICON - rescale: - scale: 100 - offset: 0 - param: CLCH - grib: path: grib/ifs-{date}{time:04}_{step:03}.grib encoding: @@ -115,14 +74,6 @@ output: slor: {"param": 163, "shortName": "slor"} SSO_STDH: {"param": 160, "shortName": "sdor"} sdor: {"param": 160, "shortName": "sdor"} - CLCT: {"param": 164, "shortName": "tcc"} - tcc: {"param": 164, "shortName": "tcc"} - CLCL: {"param": 186, "shortName": "lcc"} - lcc: {"param": 186, "shortName": "lcc"} - CLCM: {"param": 187, "shortName": "mcc"} - mcc: {"param": 187, "shortName": "mcc"} - CLCH: {"param": 188, "shortName": "hcc"} - hcc: {"param": 188, "shortName": "hcc"} SKT: {"param": 235, "shortName": "skt"} skt: {"param": 235, "shortName": "skt"} TOT_PREC: {"param": 228, "shortName": "tp"} diff --git a/resources/inference/configs/sgm-temporal-downscaler-global_trimedge_multi-clouds.yaml b/resources/inference/configs/sgm-temporal-downscaler-global_trimedge_multi-clouds.yaml new file mode 100644 index 00000000..6a3d8413 --- /dev/null +++ b/resources/inference/configs/sgm-temporal-downscaler-global_trimedge_multi-clouds.yaml @@ -0,0 +1,257 @@ +runner: temporal_downscaler + +input: + cutout: + - lam_0: + grib: + path: forecaster/20* + pre_processors: + # inputs read from GRIB carry the GRIB name (TOT_PREC), other sources may use the model name (tp) + - forward_transform_filter: + # does not have an effect if the temporal downscaler does not use tp as prognostic variable + rescale: + scale: 0.001 # convert from kg m-2 to m + offset: 0 + param: tp + - forward_transform_filter: + rescale: + scale: 0.001 # convert from kg m-2 to m + offset: 0 + param: TOT_PREC + # filters see the model name (tcc) with anemoi-inference <= 0.11.1 and the GRIB name (CLCT) from 0.11.2 + - forward_transform_filter: # cloud cover percent -> fraction, as in training + rescale: + scale: 0.01 + offset: 0 + param: tcc + - forward_transform_filter: # cloud cover percent -> fraction, as in training + rescale: + scale: 0.01 + offset: 0 + param: CLCT + - forward_transform_filter: # cloud cover percent -> fraction, as in training + rescale: + scale: 0.01 + offset: 0 + param: lcc + - forward_transform_filter: # cloud cover percent -> fraction, as in training + rescale: + scale: 0.01 + offset: 0 + param: CLCL + - forward_transform_filter: # cloud cover percent -> fraction, as in training + rescale: + scale: 0.01 + offset: 0 + param: mcc + - forward_transform_filter: # cloud cover percent -> fraction, as in training + rescale: + scale: 0.01 + offset: 0 + param: CLCM + - forward_transform_filter: # cloud cover percent -> fraction, as in training + rescale: + scale: 0.01 + offset: 0 + param: hcc + - forward_transform_filter: # cloud cover percent -> fraction, as in training + rescale: + scale: 0.01 + offset: 0 + param: CLCH + namer: &namer + rules: + - - shortName: T + - t_{level} + - - shortName: U + - u_{level} + - - shortName: V + - v_{level} + - - shortName: W + - w_{level} + - - shortName: QV + - q_{level} + - - shortName: FI + - z_{level} + - - shortName: PMSL + - msl + - - shortName: FIS + - z + - - shortName: PS + - sp + - - shortName: T_2M + - 2t + - - shortName: TD_2M + - 2d + - - shortName: SKT + - skt + - - shortName: CLCT + - tcc + - - shortName: CLCL + - lcc + - - shortName: CLCM + - mcc + - - shortName: CLCH + - hcc + - - shortName: U_10M + - 10u + - - shortName: V_10M + - 10v + - - shortName: FR_LAND + - lsm + - - shortName: TOT_PREC + - tp + - global: + grib: + path: forecaster/ifs* + namer: *namer + +constant_forcings: + test: + use_original_paths: true + +patch_metadata: resources/sgm-temporal-downscaler-ich1-oper-clouds-patch.yaml + +post_processors: + - accumulate_from_start_of_forecast: # accumulate tp from start of forecast + accumulations: + - tp + # filters see the model name (tp) with anemoi-inference <= 0.11.1 and the GRIB name (TOT_PREC) from 0.11.2 + - forward_transform_filter: + rescale: + scale: 1000 # convert units from m to kg m-2 + offset: 0 + param: tp + - forward_transform_filter: + rescale: + scale: 1000 # convert units from m to kg m-2 + offset: 0 + param: TOT_PREC + +output: + tee: + - grib: + path: grib/{dateTime}_{step:03}.grib + encoding: + typeOfGeneratingProcess: 2 + templates: + samples: resources/templates_index_icon.yaml + post_processors: + - extract_mask: # removes global points + mask: "lam_0/cutout_mask" + as_slice: true + # filters see the model name (tcc) with anemoi-inference <= 0.11.1 and the GRIB name (CLCT) from 0.11.2 + - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic + rescale: + scale: 100 + offset: 0 + param: tcc + - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic + rescale: + scale: 100 + offset: 0 + param: CLCT + - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic + rescale: + scale: 100 + offset: 0 + param: lcc + - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic + rescale: + scale: 100 + offset: 0 + param: CLCL + - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic + rescale: + scale: 100 + offset: 0 + param: mcc + - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic + rescale: + scale: 100 + offset: 0 + param: CLCM + - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic + rescale: + scale: 100 + offset: 0 + param: hcc + - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic + rescale: + scale: 100 + offset: 0 + param: CLCH + # here, the trimedge mask can be specified when available + # at overlap steps (multiples of the forecaster stride) replace the + # re-predicted prognostics with the forecaster's boundary values + - forward_transform_filter: + copy-prognostic-from-forecaster: + forecaster_path: forecaster/20* + common_leadtime: 6h + params_to_keep: [tp] # NOTE, if more diagnostics are added, update accordingly. + namer: *namer + + - grib: + path: grib/ifs-{dateTime}_{step:03}.grib + encoding: + typeOfGeneratingProcess: 2 + templates: + samples: resources/templates_index_ifs.yaml + post_processors: + - extract_mask: # removes lam points + mask: "lam_0/cutout_mask" + as_slice: true + inverse: true + - assign_mask: # fill local/global overlapping points with nan + mask: "global/cutout_mask" + - forward_transform_filter: + copy-prognostic-from-forecaster: + forecaster_path: forecaster/ifs* + common_leadtime: 6h + params_to_keep: [tp] # NOTE, if more diagnostics are added, update accordingly. + namer: *namer + modifiers: + - patches: + - variable: + U_10M: {"param": 165, "shortName": "10u"} + 10u: {"param": 165, "shortName": "10u"} + V_10M: {"param": 166, "shortName": "10v"} + 10v: {"param": 166, "shortName": "10v"} + TD_2M: {"param": 168, "shortName": "2d"} + 2d: {"param": 168, "shortName": "2d"} + T_2M: {"param": 167, "shortName": "2t"} + 2t: {"param": 167, "shortName": "2t"} + FR_LAND: {"param": 172, "shortName": "lsm"} + lsm: {"param": 172, "shortName": "lsm"} + PMSL: {"param": 151, "shortName": "msl"} + msl: {"param": 151, "shortName": "msl"} + PS: {"param": 134, "shortName": "sp"} + sp: {"param": 134, "shortName": "sp"} + SSO_SIGMA: {"param": 163, "shortName": "slor"} + slor: {"param": 163, "shortName": "slor"} + SSO_STDH: {"param": 160, "shortName": "sdor"} + sdor: {"param": 160, "shortName": "sdor"} + CLCT: {"param": 164, "shortName": "tcc"} + tcc: {"param": 164, "shortName": "tcc"} + CLCL: {"param": 186, "shortName": "lcc"} + lcc: {"param": 186, "shortName": "lcc"} + CLCM: {"param": 187, "shortName": "mcc"} + mcc: {"param": 187, "shortName": "mcc"} + CLCH: {"param": 188, "shortName": "hcc"} + hcc: {"param": 188, "shortName": "hcc"} + SKT: {"param": 235, "shortName": "skt"} + skt: {"param": 235, "shortName": "skt"} + TOT_PREC: {"param": 228, "shortName": "tp"} + tp: {"param": 228, "shortName": "tp"} + z: {"param": 129, "shortName": "z"} + "^q_(\\d+)$": {"param": 133, "shortName": "q"} + "^t_(\\d+)$": {"param": 130, "shortName": "t"} + "^u_(\\d+)$": {"param": 131, "shortName": "u"} + "^v_(\\d+)$": {"param": 132, "shortName": "v"} + "^w_(\\d+)$": {"param": 135, "shortName": "w"} + "^z_(\\d+)$": {"param": 129, "shortName": "z"} + +# silenced due to bug in anemoi-inference for multi-step temporal downscalers, can be removed when fixed +verbosity: 0 +allow_nans: true +output_frequency: "1h" diff --git a/resources/inference/configs/sgm-temporal-downscaler-global_trimedge_multi.yaml b/resources/inference/configs/sgm-temporal-downscaler-global_trimedge_multi.yaml index cc1be1bb..a1746739 100644 --- a/resources/inference/configs/sgm-temporal-downscaler-global_trimedge_multi.yaml +++ b/resources/inference/configs/sgm-temporal-downscaler-global_trimedge_multi.yaml @@ -18,47 +18,6 @@ input: scale: 0.001 # convert from kg m-2 to m offset: 0 param: TOT_PREC - # filters see the model name (tcc) with anemoi-inference <= 0.11.1 and the GRIB name (CLCT) from 0.11.2 - - forward_transform_filter: # cloud cover percent -> fraction, as in training - rescale: - scale: 0.01 - offset: 0 - param: tcc - - forward_transform_filter: # cloud cover percent -> fraction, as in training - rescale: - scale: 0.01 - offset: 0 - param: CLCT - - forward_transform_filter: # cloud cover percent -> fraction, as in training - rescale: - scale: 0.01 - offset: 0 - param: lcc - - forward_transform_filter: # cloud cover percent -> fraction, as in training - rescale: - scale: 0.01 - offset: 0 - param: CLCL - - forward_transform_filter: # cloud cover percent -> fraction, as in training - rescale: - scale: 0.01 - offset: 0 - param: mcc - - forward_transform_filter: # cloud cover percent -> fraction, as in training - rescale: - scale: 0.01 - offset: 0 - param: CLCM - - forward_transform_filter: # cloud cover percent -> fraction, as in training - rescale: - scale: 0.01 - offset: 0 - param: hcc - - forward_transform_filter: # cloud cover percent -> fraction, as in training - rescale: - scale: 0.01 - offset: 0 - param: CLCH namer: &namer rules: - - shortName: T @@ -85,14 +44,6 @@ input: - 2d - - shortName: SKT - skt - - - shortName: CLCT - - tcc - - - shortName: CLCL - - lcc - - - shortName: CLCM - - mcc - - - shortName: CLCH - - hcc - - shortName: U_10M - 10u - - shortName: V_10M @@ -140,47 +91,6 @@ output: - extract_mask: # removes global points mask: "lam_0/cutout_mask" as_slice: true - # filters see the model name (tcc) with anemoi-inference <= 0.11.1 and the GRIB name (CLCT) from 0.11.2 - - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic - rescale: - scale: 100 - offset: 0 - param: tcc - - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic - rescale: - scale: 100 - offset: 0 - param: CLCT - - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic - rescale: - scale: 100 - offset: 0 - param: lcc - - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic - rescale: - scale: 100 - offset: 0 - param: CLCL - - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic - rescale: - scale: 100 - offset: 0 - param: mcc - - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic - rescale: - scale: 100 - offset: 0 - param: CLCM - - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic - rescale: - scale: 100 - offset: 0 - param: hcc - - forward_transform_filter: # cloud cover fraction -> percent, before copy-prognostic - rescale: - scale: 100 - offset: 0 - param: CLCH # here, the trimedge mask can be specified when available # at overlap steps (multiples of the forecaster stride) replace the # re-predicted prognostics with the forecaster's boundary values @@ -231,14 +141,6 @@ output: slor: {"param": 163, "shortName": "slor"} SSO_STDH: {"param": 160, "shortName": "sdor"} sdor: {"param": 160, "shortName": "sdor"} - CLCT: {"param": 164, "shortName": "tcc"} - tcc: {"param": 164, "shortName": "tcc"} - CLCL: {"param": 186, "shortName": "lcc"} - lcc: {"param": 186, "shortName": "lcc"} - CLCM: {"param": 187, "shortName": "mcc"} - mcc: {"param": 187, "shortName": "mcc"} - CLCH: {"param": 188, "shortName": "hcc"} - hcc: {"param": 188, "shortName": "hcc"} SKT: {"param": 235, "shortName": "skt"} skt: {"param": 235, "shortName": "skt"} TOT_PREC: {"param": 228, "shortName": "tp"} diff --git a/resources/inference/metadata/sgm-multidataset-ich1-oper-clouds-patch.yaml b/resources/inference/metadata/sgm-multidataset-ich1-oper-clouds-patch.yaml new file mode 100644 index 00000000..6b1792b0 --- /dev/null +++ b/resources/inference/metadata/sgm-multidataset-ich1-oper-clouds-patch.yaml @@ -0,0 +1,791 @@ +config: + dataloader: + test: + datasets: + data: + dataset_config: + dataset: + cutout: + - dataset: /store_new/mch/msopr/ml/datasets/mch-ich1-1km-2024-2025-1h-pl13-ifsnames-v1.0.zarr + rescale: # cloud cover percent -> fraction, as in training + tcc: [0.01, 0.0] + lcc: [0.01, 0.0] + mcc: [0.01, 0.0] + hcc: [0.01, 0.0] + - dataset: /store_new/mch/msopr/ml/datasets/aifs-od-an-oper-0001-mars-n320-2016-2026-6h-v1-combined-land.zarr + start: null + end: null + +dataset: + data: + variables_metadata: + slor: + mars: + date: 20050101 + levtype: sfc + param: SSO_SIGMA + step: 12 + time: 0 + sdor: + mars: + date: 20050101 + levtype: sfc + param: SSO_STDH + step: 12 + time: 0 + 10u: + mars: + date: 20050101 + levtype: sfc + param: U_10M + step: 12 + time: 0 + 10v: + mars: + date: 20050101 + levtype: sfc + param: V_10M + step: 12 + time: 0 + 2d: + mars: + date: 20050101 + levtype: sfc + param: TD_2M + step: 12 + time: 0 + 2t: + mars: + date: 20050101 + levtype: sfc + param: T_2M + step: 12 + time: 0 + tcc: + mars: + date: 20050101 + levtype: sfc + param: CLCT + step: 12 + time: 0 + lcc: + mars: + date: 20050101 + levtype: sfc + param: CLCL + step: 12 + time: 0 + mcc: + mars: + date: 20050101 + levtype: sfc + param: CLCM + step: 12 + time: 0 + hcc: + mars: + date: 20050101 + levtype: sfc + param: CLCH + step: 12 + time: 0 + skt: + mars: + date: 20050101 + levtype: sfc + param: SKT + step: 12 + time: 0 + cos_julian_day: + computed_forcing: true + constant_in_time: false + cos_latitude: + computed_forcing: true + constant_in_time: true + cos_local_time: + computed_forcing: true + constant_in_time: false + cos_longitude: + computed_forcing: true + constant_in_time: true + insolation: + computed_forcing: true + constant_in_time: false + lsm: + constant_in_time: true + mars: + date: 20050101 + levtype: sfc + param: FR_LAND + step: 0 + time: 12 + msl: + mars: + date: 20050101 + levtype: sfc + param: PMSL + step: 12 + time: 0 + q_100: + mars: + date: 20050101 + levelist: 100 + levtype: pl + param: QV + step: 12 + time: 0 + q_1000: + mars: + date: 20050101 + levelist: 1000 + levtype: pl + param: QV + step: 12 + time: 0 + q_150: + mars: + date: 20050101 + levelist: 150 + levtype: pl + param: QV + step: 12 + time: 0 + q_200: + mars: + date: 20050101 + levelist: 200 + levtype: pl + param: QV + step: 12 + time: 0 + q_250: + mars: + date: 20050101 + levelist: 250 + levtype: pl + param: QV + step: 12 + time: 0 + q_300: + mars: + date: 20050101 + levelist: 300 + levtype: pl + param: QV + step: 12 + time: 0 + q_400: + mars: + date: 20050101 + levelist: 400 + levtype: pl + param: QV + step: 12 + time: 0 + q_50: + mars: + date: 20050101 + levelist: 50 + levtype: pl + param: QV + step: 12 + time: 0 + q_500: + mars: + date: 20050101 + levelist: 500 + levtype: pl + param: QV + step: 12 + time: 0 + q_600: + mars: + date: 20050101 + levelist: 600 + levtype: pl + param: QV + step: 12 + time: 0 + q_700: + mars: + date: 20050101 + levelist: 700 + levtype: pl + param: QV + step: 12 + time: 0 + q_850: + mars: + date: 20050101 + levelist: 850 + levtype: pl + param: QV + step: 12 + time: 0 + q_925: + mars: + date: 20050101 + levelist: 925 + levtype: pl + param: QV + step: 12 + time: 0 + sin_julian_day: + computed_forcing: true + constant_in_time: false + sin_latitude: + computed_forcing: true + constant_in_time: true + sin_local_time: + computed_forcing: true + constant_in_time: false + sin_longitude: + computed_forcing: true + constant_in_time: true + sp: + mars: + date: 20050101 + levtype: sfc + param: PS + step: 12 + time: 0 + t_100: + mars: + date: 20050101 + levelist: 100 + levtype: pl + param: T + step: 12 + time: 0 + t_1000: + mars: + date: 20050101 + levelist: 1000 + levtype: pl + param: T + step: 12 + time: 0 + t_150: + mars: + date: 20050101 + levelist: 150 + levtype: pl + param: T + step: 12 + time: 0 + t_200: + mars: + date: 20050101 + levelist: 200 + levtype: pl + param: T + step: 12 + time: 0 + t_250: + mars: + date: 20050101 + levelist: 250 + levtype: pl + param: T + step: 12 + time: 0 + t_300: + mars: + date: 20050101 + levelist: 300 + levtype: pl + param: T + step: 12 + time: 0 + t_400: + mars: + date: 20050101 + levelist: 400 + levtype: pl + param: T + step: 12 + time: 0 + t_50: + mars: + date: 20050101 + levelist: 50 + levtype: pl + param: T + step: 12 + time: 0 + t_500: + mars: + date: 20050101 + levelist: 500 + levtype: pl + param: T + step: 12 + time: 0 + t_600: + mars: + date: 20050101 + levelist: 600 + levtype: pl + param: T + step: 12 + time: 0 + t_700: + mars: + date: 20050101 + levelist: 700 + levtype: pl + param: T + step: 12 + time: 0 + t_850: + mars: + date: 20050101 + levelist: 850 + levtype: pl + param: T + step: 12 + time: 0 + t_925: + mars: + date: 20050101 + levelist: 925 + levtype: pl + param: T + step: 12 + time: 0 + tp: + mars: + date: 20050101 + levtype: sfc + param: TOT_PREC + step: 12 + time: 0 + period: + - 6h + - 12h + process: accumulation + u_100: + mars: + date: 20050101 + levelist: 100 + levtype: pl + param: U + step: 12 + time: 0 + u_1000: + mars: + date: 20050101 + levelist: 1000 + levtype: pl + param: U + step: 12 + time: 0 + u_150: + mars: + date: 20050101 + levelist: 150 + levtype: pl + param: U + step: 12 + time: 0 + u_200: + mars: + date: 20050101 + levelist: 200 + levtype: pl + param: U + step: 12 + time: 0 + u_250: + mars: + date: 20050101 + levelist: 250 + levtype: pl + param: U + step: 12 + time: 0 + u_300: + mars: + date: 20050101 + levelist: 300 + levtype: pl + param: U + step: 12 + time: 0 + u_400: + mars: + date: 20050101 + levelist: 400 + levtype: pl + param: U + step: 12 + time: 0 + u_50: + mars: + date: 20050101 + levelist: 50 + levtype: pl + param: U + step: 12 + time: 0 + u_500: + mars: + date: 20050101 + levelist: 500 + levtype: pl + param: U + step: 12 + time: 0 + u_600: + mars: + date: 20050101 + levelist: 600 + levtype: pl + param: U + step: 12 + time: 0 + u_700: + mars: + date: 20050101 + levelist: 700 + levtype: pl + param: U + step: 12 + time: 0 + u_850: + mars: + date: 20050101 + levelist: 850 + levtype: pl + param: U + step: 12 + time: 0 + u_925: + mars: + date: 20050101 + levelist: 925 + levtype: pl + param: U + step: 12 + time: 0 + v_100: + mars: + date: 20050101 + levelist: 100 + levtype: pl + param: V + step: 12 + time: 0 + v_1000: + mars: + date: 20050101 + levelist: 1000 + levtype: pl + param: V + step: 12 + time: 0 + v_150: + mars: + date: 20050101 + levelist: 150 + levtype: pl + param: V + step: 12 + time: 0 + v_200: + mars: + date: 20050101 + levelist: 200 + levtype: pl + param: V + step: 12 + time: 0 + v_250: + mars: + date: 20050101 + levelist: 250 + levtype: pl + param: V + step: 12 + time: 0 + v_300: + mars: + date: 20050101 + levelist: 300 + levtype: pl + param: V + step: 12 + time: 0 + v_400: + mars: + date: 20050101 + levelist: 400 + levtype: pl + param: V + step: 12 + time: 0 + v_50: + mars: + date: 20050101 + levelist: 50 + levtype: pl + param: V + step: 12 + time: 0 + v_500: + mars: + date: 20050101 + levelist: 500 + levtype: pl + param: V + step: 12 + time: 0 + v_600: + mars: + date: 20050101 + levelist: 600 + levtype: pl + param: V + step: 12 + time: 0 + v_700: + mars: + date: 20050101 + levelist: 700 + levtype: pl + param: V + step: 12 + time: 0 + v_850: + mars: + date: 20050101 + levelist: 850 + levtype: pl + param: V + step: 12 + time: 0 + v_925: + mars: + date: 20050101 + levelist: 925 + levtype: pl + param: V + step: 12 + time: 0 + w_100: + mars: + date: 20050101 + levelist: 100 + levtype: pl + param: OMEGA + step: 12 + time: 0 + w_1000: + mars: + date: 20050101 + levelist: 1000 + levtype: pl + param: OMEGA + step: 12 + time: 0 + w_150: + mars: + date: 20050101 + levelist: 150 + levtype: pl + param: OMEGA + step: 12 + time: 0 + w_200: + mars: + date: 20050101 + levelist: 200 + levtype: pl + param: OMEGA + step: 12 + time: 0 + w_250: + mars: + date: 20050101 + levelist: 250 + levtype: pl + param: OMEGA + step: 12 + time: 0 + w_300: + mars: + date: 20050101 + levelist: 300 + levtype: pl + param: OMEGA + step: 12 + time: 0 + w_400: + mars: + date: 20050101 + levelist: 400 + levtype: pl + param: OMEGA + step: 12 + time: 0 + w_50: + mars: + date: 20050101 + levelist: 50 + levtype: pl + param: OMEGA + step: 12 + time: 0 + w_500: + mars: + date: 20050101 + levelist: 500 + levtype: pl + param: OMEGA + step: 12 + time: 0 + w_600: + mars: + date: 20050101 + levelist: 600 + levtype: pl + param: OMEGA + step: 12 + time: 0 + w_700: + mars: + date: 20050101 + levelist: 700 + levtype: pl + param: OMEGA + step: 12 + time: 0 + w_850: + mars: + date: 20050101 + levelist: 850 + levtype: pl + param: OMEGA + step: 12 + time: 0 + w_925: + mars: + date: 20050101 + levelist: 925 + levtype: pl + param: OMEGA + step: 12 + time: 0 + z: + constant_in_time: true + mars: + date: 20050101 + levelist: null + levtype: sfc + param: FIS + step: 0 + time: 12 + z_100: + mars: + date: 20050101 + levelist: 100 + levtype: pl + param: FI + step: 12 + time: 0 + z_1000: + mars: + date: 20050101 + levelist: 1000 + levtype: pl + param: FI + step: 12 + time: 0 + z_150: + mars: + date: 20050101 + levelist: 150 + levtype: pl + param: FI + step: 12 + time: 0 + z_200: + mars: + date: 20050101 + levelist: 200 + levtype: pl + param: FI + step: 12 + time: 0 + z_250: + mars: + date: 20050101 + levelist: 250 + levtype: pl + param: FI + step: 12 + time: 0 + z_300: + mars: + date: 20050101 + levelist: 300 + levtype: pl + param: FI + step: 12 + time: 0 + z_400: + mars: + date: 20050101 + levelist: 400 + levtype: pl + param: FI + step: 12 + time: 0 + z_50: + mars: + date: 20050101 + levelist: 50 + levtype: pl + param: FI + step: 12 + time: 0 + z_500: + mars: + date: 20050101 + levelist: 500 + levtype: pl + param: FI + step: 12 + time: 0 + z_600: + mars: + date: 20050101 + levelist: 600 + levtype: pl + param: FI + step: 12 + time: 0 + z_700: + mars: + date: 20050101 + levelist: 700 + levtype: pl + param: FI + step: 12 + time: 0 + z_850: + mars: + date: 20050101 + levelist: 850 + levtype: pl + param: FI + step: 12 + time: 0 + z_925: + mars: + date: 20050101 + levelist: 925 + levtype: pl + param: FI + step: 12 + time: 0 diff --git a/resources/inference/metadata/sgm-multidataset-ich1-oper-patch.yaml b/resources/inference/metadata/sgm-multidataset-ich1-oper-patch.yaml index 6b1792b0..e54d0e7f 100644 --- a/resources/inference/metadata/sgm-multidataset-ich1-oper-patch.yaml +++ b/resources/inference/metadata/sgm-multidataset-ich1-oper-patch.yaml @@ -6,13 +6,12 @@ config: dataset_config: dataset: cutout: - - dataset: /store_new/mch/msopr/ml/datasets/mch-ich1-1km-2024-2025-1h-pl13-ifsnames-v1.0.zarr - rescale: # cloud cover percent -> fraction, as in training - tcc: [0.01, 0.0] - lcc: [0.01, 0.0] - mcc: [0.01, 0.0] - hcc: [0.01, 0.0] + - dataset: /store_new/mch/msopr/ml/datasets/mch-ich1-1km-2024-2026-1h-pl13-ifsnames-v2.0.zarr + start: "2024-05-01T00:00:00" + end: "2026-05-01T00:00:00" - dataset: /store_new/mch/msopr/ml/datasets/aifs-od-an-oper-0001-mars-n320-2016-2026-6h-v1-combined-land.zarr + start: "2024-05-01T00:00:00" + end: "2026-05-01T00:00:00" start: null end: null @@ -61,41 +60,6 @@ dataset: param: T_2M step: 12 time: 0 - tcc: - mars: - date: 20050101 - levtype: sfc - param: CLCT - step: 12 - time: 0 - lcc: - mars: - date: 20050101 - levtype: sfc - param: CLCL - step: 12 - time: 0 - mcc: - mars: - date: 20050101 - levtype: sfc - param: CLCM - step: 12 - time: 0 - hcc: - mars: - date: 20050101 - levtype: sfc - param: CLCH - step: 12 - time: 0 - skt: - mars: - date: 20050101 - levtype: sfc - param: SKT - step: 12 - time: 0 cos_julian_day: computed_forcing: true constant_in_time: false diff --git a/resources/inference/metadata/sgm-temporal-downscaler-ich1-oper-clouds-patch.yaml b/resources/inference/metadata/sgm-temporal-downscaler-ich1-oper-clouds-patch.yaml new file mode 100644 index 00000000..564070ae --- /dev/null +++ b/resources/inference/metadata/sgm-temporal-downscaler-ich1-oper-clouds-patch.yaml @@ -0,0 +1,274 @@ +config: + dataloader: + test: + datasets: + data: + dataset_config: + dataset: + cutout: + # training datasets not available on system due to training on different system + # and/or training period not in dataset + - dataset: /store_new/mch/msopr/ml/datasets/mch-ich1-1km-2024-2025-1h-pl13-ifsnames-v1.0.zarr + - dataset: /store_new/mch/msopr/ml/datasets/aifs-od-an-oper-0001-mars-n320-2016-2026-6h-v1-combined-land.zarr + # avoid errors from frequency mismatch between LAM (1h) and global (6h) datasets when aligning datasets + frequency: 6h + start: null + end: null + +dataset: + data: + constant_fields: [z, lsm] + # variables_metadata is necessary due to missing variables_metadata section in temporal downscaler patch_metdata + # else mapping with grib templates fails + variables_metadata: + 10u: + mars: + date: 20050101 + levtype: sfc + param: U_10M + step: 12 + time: 0 + 10v: + mars: + date: 20050101 + levtype: sfc + param: V_10M + step: 12 + time: 0 + 2d: + mars: + date: 20050101 + levtype: sfc + param: TD_2M + step: 12 + time: 0 + 2t: + mars: + date: 20050101 + levtype: sfc + param: T_2M + step: 12 + time: 0 + tcc: + mars: + date: 20050101 + levtype: sfc + param: CLCT + step: 12 + time: 0 + lcc: + mars: + date: 20050101 + levtype: sfc + param: CLCL + step: 12 + time: 0 + mcc: + mars: + date: 20050101 + levtype: sfc + param: CLCM + step: 12 + time: 0 + hcc: + mars: + date: 20050101 + levtype: sfc + param: CLCH + step: 12 + time: 0 + cos_julian_day: + computed_forcing: true + constant_in_time: false + cos_latitude: + computed_forcing: true + constant_in_time: true + cos_local_time: + computed_forcing: true + constant_in_time: false + cos_longitude: + computed_forcing: true + constant_in_time: true + insolation: + computed_forcing: true + constant_in_time: false + lsm: + constant_in_time: true + mars: + date: 20050101 + levtype: sfc + param: FR_LAND + step: 0 + time: 12 + msl: + mars: + date: 20050101 + levtype: sfc + param: PMSL + step: 12 + time: 0 + q_100: + mars: {date: 20050101, levelist: 100, levtype: pl, param: QV, step: 12, time: 0} + q_1000: + mars: {date: 20050101, levelist: 1000, levtype: pl, param: QV, step: 12, time: 0} + q_150: + mars: {date: 20050101, levelist: 150, levtype: pl, param: QV, step: 12, time: 0} + q_200: + mars: {date: 20050101, levelist: 200, levtype: pl, param: QV, step: 12, time: 0} + q_250: + mars: {date: 20050101, levelist: 250, levtype: pl, param: QV, step: 12, time: 0} + q_300: + mars: {date: 20050101, levelist: 300, levtype: pl, param: QV, step: 12, time: 0} + q_400: + mars: {date: 20050101, levelist: 400, levtype: pl, param: QV, step: 12, time: 0} + q_50: + mars: {date: 20050101, levelist: 50, levtype: pl, param: QV, step: 12, time: 0} + q_500: + mars: {date: 20050101, levelist: 500, levtype: pl, param: QV, step: 12, time: 0} + q_700: + mars: {date: 20050101, levelist: 700, levtype: pl, param: QV, step: 12, time: 0} + q_850: + mars: {date: 20050101, levelist: 850, levtype: pl, param: QV, step: 12, time: 0} + q_925: + mars: {date: 20050101, levelist: 925, levtype: pl, param: QV, step: 12, time: 0} + sin_julian_day: + computed_forcing: true + constant_in_time: false + sin_latitude: + computed_forcing: true + constant_in_time: true + sin_local_time: + computed_forcing: true + constant_in_time: false + sin_longitude: + computed_forcing: true + constant_in_time: true + sp: + mars: + date: 20050101 + levtype: sfc + param: PS + step: 12 + time: 0 + skt: + mars: + date: 20050101 + levtype: sfc + param: SKT + step: 12 + time: 0 + t_100: + mars: {date: 20050101, levelist: 100, levtype: pl, param: T, step: 12, time: 0} + t_1000: + mars: {date: 20050101, levelist: 1000, levtype: pl, param: T, step: 12, time: 0} + t_150: + mars: {date: 20050101, levelist: 150, levtype: pl, param: T, step: 12, time: 0} + t_200: + mars: {date: 20050101, levelist: 200, levtype: pl, param: T, step: 12, time: 0} + t_250: + mars: {date: 20050101, levelist: 250, levtype: pl, param: T, step: 12, time: 0} + t_300: + mars: {date: 20050101, levelist: 300, levtype: pl, param: T, step: 12, time: 0} + t_400: + mars: {date: 20050101, levelist: 400, levtype: pl, param: T, step: 12, time: 0} + t_50: + mars: {date: 20050101, levelist: 50, levtype: pl, param: T, step: 12, time: 0} + t_500: + mars: {date: 20050101, levelist: 500, levtype: pl, param: T, step: 12, time: 0} + t_700: + mars: {date: 20050101, levelist: 700, levtype: pl, param: T, step: 12, time: 0} + t_850: + mars: {date: 20050101, levelist: 850, levtype: pl, param: T, step: 12, time: 0} + t_925: + mars: {date: 20050101, levelist: 925, levtype: pl, param: T, step: 12, time: 0} + tp: + mars: + date: 20050101 + levtype: sfc + param: TOT_PREC + step: 12 + time: 0 + period: + - 1h + process: accumulation + u_100: + mars: {date: 20050101, levelist: 100, levtype: pl, param: U, step: 12, time: 0} + u_1000: + mars: {date: 20050101, levelist: 1000, levtype: pl, param: U, step: 12, time: 0} + u_150: + mars: {date: 20050101, levelist: 150, levtype: pl, param: U, step: 12, time: 0} + u_200: + mars: {date: 20050101, levelist: 200, levtype: pl, param: U, step: 12, time: 0} + u_250: + mars: {date: 20050101, levelist: 250, levtype: pl, param: U, step: 12, time: 0} + u_300: + mars: {date: 20050101, levelist: 300, levtype: pl, param: U, step: 12, time: 0} + u_400: + mars: {date: 20050101, levelist: 400, levtype: pl, param: U, step: 12, time: 0} + u_50: + mars: {date: 20050101, levelist: 50, levtype: pl, param: U, step: 12, time: 0} + u_500: + mars: {date: 20050101, levelist: 500, levtype: pl, param: U, step: 12, time: 0} + u_700: + mars: {date: 20050101, levelist: 700, levtype: pl, param: U, step: 12, time: 0} + u_850: + mars: {date: 20050101, levelist: 850, levtype: pl, param: U, step: 12, time: 0} + u_925: + mars: {date: 20050101, levelist: 925, levtype: pl, param: U, step: 12, time: 0} + v_100: + mars: {date: 20050101, levelist: 100, levtype: pl, param: V, step: 12, time: 0} + v_1000: + mars: {date: 20050101, levelist: 1000, levtype: pl, param: V, step: 12, time: 0} + v_150: + mars: {date: 20050101, levelist: 150, levtype: pl, param: V, step: 12, time: 0} + v_200: + mars: {date: 20050101, levelist: 200, levtype: pl, param: V, step: 12, time: 0} + v_250: + mars: {date: 20050101, levelist: 250, levtype: pl, param: V, step: 12, time: 0} + v_300: + mars: {date: 20050101, levelist: 300, levtype: pl, param: V, step: 12, time: 0} + v_400: + mars: {date: 20050101, levelist: 400, levtype: pl, param: V, step: 12, time: 0} + v_50: + mars: {date: 20050101, levelist: 50, levtype: pl, param: V, step: 12, time: 0} + v_500: + mars: {date: 20050101, levelist: 500, levtype: pl, param: V, step: 12, time: 0} + v_700: + mars: {date: 20050101, levelist: 700, levtype: pl, param: V, step: 12, time: 0} + v_850: + mars: {date: 20050101, levelist: 850, levtype: pl, param: V, step: 12, time: 0} + v_925: + mars: {date: 20050101, levelist: 925, levtype: pl, param: V, step: 12, time: 0} + z: + constant_in_time: true + mars: + date: 20050101 + levelist: null + levtype: sfc + param: FIS + step: 0 + time: 12 + z_100: + mars: {date: 20050101, levelist: 100, levtype: pl, param: FI, step: 12, time: 0} + z_1000: + mars: {date: 20050101, levelist: 1000, levtype: pl, param: FI, step: 12, time: 0} + z_150: + mars: {date: 20050101, levelist: 150, levtype: pl, param: FI, step: 12, time: 0} + z_200: + mars: {date: 20050101, levelist: 200, levtype: pl, param: FI, step: 12, time: 0} + z_250: + mars: {date: 20050101, levelist: 250, levtype: pl, param: FI, step: 12, time: 0} + z_300: + mars: {date: 20050101, levelist: 300, levtype: pl, param: FI, step: 12, time: 0} + z_400: + mars: {date: 20050101, levelist: 400, levtype: pl, param: FI, step: 12, time: 0} + z_50: + mars: {date: 20050101, levelist: 50, levtype: pl, param: FI, step: 12, time: 0} + z_500: + mars: {date: 20050101, levelist: 500, levtype: pl, param: FI, step: 12, time: 0} + z_700: + mars: {date: 20050101, levelist: 700, levtype: pl, param: FI, step: 12, time: 0} + z_850: + mars: {date: 20050101, levelist: 850, levtype: pl, param: FI, step: 12, time: 0} + z_925: + mars: {date: 20050101, levelist: 925, levtype: pl, param: FI, step: 12, time: 0} diff --git a/resources/inference/metadata/sgm-temporal-downscaler-ich1-oper-patch.yaml b/resources/inference/metadata/sgm-temporal-downscaler-ich1-oper-patch.yaml index 889d3f4c..ade0df7e 100644 --- a/resources/inference/metadata/sgm-temporal-downscaler-ich1-oper-patch.yaml +++ b/resources/inference/metadata/sgm-temporal-downscaler-ich1-oper-patch.yaml @@ -8,8 +8,12 @@ config: cutout: # training datasets not available on system due to training on different system # and/or training period not in dataset - - dataset: /store_new/mch/msopr/ml/datasets/mch-ich1-1km-2024-2025-1h-pl13-ifsnames-v1.0.zarr + - dataset: /store_new/mch/msopr/ml/datasets/mch-ich1-1km-2024-2026-1h-pl13-ifsnames-v2.0.zarr + start: "2024-05-01T00:00:00" + end: "2026-05-01T00:00:00" - dataset: /store_new/mch/msopr/ml/datasets/aifs-od-an-oper-0001-mars-n320-2016-2026-6h-v1-combined-land.zarr + start: "2024-05-01T00:00:00" + end: "2026-05-01T00:00:00" # avoid errors from frequency mismatch between LAM (1h) and global (6h) datasets when aligning datasets frequency: 6h start: null @@ -59,14 +63,6 @@ dataset: mars: {date: 20050101, levtype: sfc, param: TD_2M, step: 12, time: 0} 2t: mars: {date: 20050101, levtype: sfc, param: T_2M, step: 12, time: 0} - tcc: - mars: {date: 20050101, levtype: sfc, param: CLCT, step: 12, time: 0} - lcc: - mars: {date: 20050101, levtype: sfc, param: CLCL, step: 12, time: 0} - mcc: - mars: {date: 20050101, levtype: sfc, param: CLCM, step: 12, time: 0} - hcc: - mars: {date: 20050101, levtype: sfc, param: CLCH, step: 12, time: 0} cos_julian_day: computed_forcing: true constant_in_time: false