Update clip wrapper and standardise CRS handling

INTERFACE.yaml

@@ -16,7 +16,8 @@ pathvars:
     shapes:
       default: "<resources>/user/shapes/{shape}.parquet"
       description: >
-        Region geometries in parquet format.
+        Region geometries in GeoParquet format.
+        The module normalises these geometries to EPSG:4326 before processing.
         These should conform to the schema defined in https://github.com/calliope-project/module_geo_boundaries/blob/main/workflow/internal/shape.schema.yaml.
     wdpa:
       default: "<resources>/user/wdpa.gdb"
@@ -25,7 +26,7 @@ pathvars:
   results:
     area_potential:
       default: "<results>/{shape}/area_potential_{tech}.tif"
-      description: "Area potential GeoTIFF raster for the specified technology across all subunits."
+      description: "Area potential GeoTIFF raster in EPSG:4326 for the specified technology across all subunits."
 wildcards:
   shape: "Name of the shape to be processed, e.g., 'world', 'europe', 'MEX'."
   tech: "Name of the technology, e.g., 'pv_rooftop' or 'wind_offshore'. Available technologies are defined in the module configuration."

README.md

@@ -70,12 +70,12 @@ Here is a `wind_offshore` example. We start with the `pixel_area`, the total sur
 wind_onshore:
   initial_area: pixel_area
   continuous_layers:
-    slope:
-    min: 0
-    max: 20
+    slope_deg:
+      min: 0
+      max: 20
     settlement_share:
-    min: 0
-    max: 0.01
+      min: 0
+      max: 0.01
   binary_layers:
     regions_maritime: 0
     regions_land: 1

config/config.yaml

@@ -30,7 +30,7 @@ techs:
   pv_open_field:
     initial_area: pixel_area
     continuous_layers:
-      slope:
+      slope_deg:
         min: 0
         max: 3
       settlement_share:
@@ -49,7 +49,7 @@ techs:
   wind_onshore:
     initial_area: pixel_area
     continuous_layers:
-      slope:
+      slope_deg:
         min: 0
         max: 20
       settlement_share:

tests/integration/test_config.yaml

@@ -24,7 +24,7 @@ module_area_potentials:
     pv_open_field:
       initial_area: pixel_area
       continuous_layers:
-        slope:
+        slope_deg:
           min: 0
           max: 3
         settlement_share:
@@ -43,7 +43,7 @@ module_area_potentials:
     wind_onshore:
       initial_area: pixel_area
       continuous_layers:
-        slope:
+        slope_deg:
           min: 0
           max: 20
         settlement_share:

workflow/Snakefile

@@ -41,6 +41,7 @@ wildcard_constraints:
 
 # Add all your includes here.
 include: "rules/functions.smk"
+include: "rules/preprocess.smk"
 include: "rules/automatic.smk"
 include: "rules/process.smk"
 

workflow/internal/settings.yaml

@@ -11,8 +11,11 @@ resources:
     ghsl: "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/GHS_BUILT_S_GLOBE_R2023A/GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss/V1-0/GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss_V1_0.zip"
     ghsl_tif: "GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss_V1_0.tif"
 
+# NOTE: only works because GlobCover is already in this CRS!
+target_crs: "EPSG:4326"
+
 land_cover_types:
-  POST_FLOODING: FARM
+  POST_FLOODING_CROPLANDS: NOT_SUITABLE
   RAINFED_CROPLANDS: FARM
   MOSAIC_CROPLAND: FARM
   MOSAIC_VEGETATION: FARM
@@ -23,13 +26,13 @@ land_cover_types:
   OPEN_NEEDLELEAVED_FOREST: FOREST
   CLOSED_TO_OPEN_MIXED_FOREST: FOREST
   MOSAIC_FOREST: FOREST
-  CLOSED_TO_OPEN_REGULARLY_FLOODED_FOREST: FOREST
-  CLOSED_REGULARLY_FLOODED_FOREST: FOREST
+  CLOSED_TO_OPEN_REGULARLY_FLOODED_FOREST: NOT_SUITABLE
+  CLOSED_REGULARLY_FLOODED_FOREST: NOT_SUITABLE
   MOSAIC_GRASSLAND: OTHER
   CLOSED_TO_OPEN_SHRUBLAND: OTHER
   CLOSED_TO_OPEN_HERBS: OTHER
   SPARSE_VEGETATION: OTHER
-  CLOSED_TO_OPEN_REGULARLY_FLOODED_GRASSLAND: OTHER
+  CLOSED_TO_OPEN_REGULARLY_FLOODED_GRASSLAND: NOT_SUITABLE
   BARE_AREAS: OTHER
   ARTIFICIAL_SURFACES_AND_URBAN_AREAS: URBAN
   WATER_BODIES: WATER

workflow/rules/automatic.smk

@@ -8,7 +8,7 @@ if config.get("tiny_files", False):
 
     rule clip_slope:
         input:
-            vector="<shapes>",
+            like_vector=rules.normalise_shapes.output.shapes,
         output:
             path="<resources>/automatic/cutout/{shape}/slope.tif",
         log:
@@ -19,11 +19,11 @@ if config.get("tiny_files", False):
         message:
             "Download slope data covering the bounds of the input shapefile."
         wrapper:
-            "v7.2.0/geo/rasterio/clip-geotiff"
+            "v9.14.0/geo/rasterio/clip"
 
     rule clip_bathymetry:
         input:
-            vector="<shapes>",
+            like_vector=rules.normalise_shapes.output.shapes,
         output:
             path="<resources>/automatic/cutout/{shape}/bathymetry.tif",
         log:
@@ -34,7 +34,7 @@ if config.get("tiny_files", False):
         message:
             "Download bathymetry data covering the bounds of the input shapefile."
         wrapper:
-            "v7.2.0/geo/rasterio/clip-geotiff"
+            "v9.14.0/geo/rasterio/clip"
 
 else:
 
@@ -56,7 +56,7 @@ else:
             "Download global slope data."
         shell:
             """
-            curl -sSLo {output:q} {params.url:q}
+            curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1
             """
 
     rule download_bathymetry:
@@ -73,44 +73,34 @@ else:
             "Download global bathymetry data."
         shell:
             """
-            curl -sSLo {output:q} {params.url:q}
+            curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1
             """
 
     rule clip_slope:
         input:
-            script=workflow.source_path("../scripts/clip_raster.py"),
-            shapes="<shapes>",
-            slope=rules.download_slope.output,
+            like_vector=rules.normalise_shapes.output.shapes,
+            raster=rules.download_slope.output[0],
         output:
-            "<resources>/automatic/cutout/{shape}/slope.tif",
+            path="<resources>/automatic/cutout/{shape}/slope.tif",
         log:
             "<logs>/{shape}/clip_slope.log",
-        conda:
-            "../envs/module.yaml"
         message:
             "Cut slope data to the bounds of the input shapefile."
-        shell:
-            """
-            python {input.script:q} {input.slope:q} {input.shapes:q} {output:q} 2>{log:q}
-            """
+        wrapper:
+            "v9.14.0/geo/rasterio/clip"
 
     rule clip_bathymetry:
         input:
-            script=workflow.source_path("../scripts/clip_raster.py"),
-            shapes="<shapes>",
-            bathymetry=rules.download_bathymetry.output,
+            like_vector=rules.normalise_shapes.output.shapes,
+            raster=rules.download_bathymetry.output[0],
         output:
-            "<resources>/automatic/cutout/{shape}/bathymetry.tif",
+            path="<resources>/automatic/cutout/{shape}/bathymetry.tif",
         log:
             "<logs>/{shape}/clip_bathymetry.log",
-        conda:
-            "../envs/module.yaml"
         message:
             "Cut bathymetry data to the bounds of the input shapefile."
-        shell:
-            """
-            python {input.script:q} {input.bathymetry:q} {input.shapes:q} {output:q} 2>{log:q}
-            """
+        wrapper:
+            "v9.14.0/geo/rasterio/clip"
 
 
 ##
@@ -132,7 +122,7 @@ rule download_globcover:
         "Download the GlobCover land cover data (~380 MB)."
     shell:
         """
-        curl -sSLo {output:q} {params.url:q}
+        curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1
         """
 
 
@@ -152,27 +142,22 @@ rule unzip_globcover:
         "Unzip the relevant TIF files from the GlobCover zip file."
     shell:
         """
-        python {input.script:q} {input.zipfile:q} -f {params.target_file:q} -o {output:q} 2>{log:q}
+        python {input.script:q} {input.zipfile:q} -f {params.target_file:q} -o {output:q} >{log:q} 2>&1
         """
 
 
 rule clip_landcover:
     input:
-        script=workflow.source_path("../scripts/clip_raster.py"),
-        shapes="<shapes>",
-        landcover=rules.unzip_globcover.output,
+        like_vector=rules.normalise_shapes.output.shapes,
+        raster=rules.unzip_globcover.output[0],
     output:
-        "<resources>/automatic/cutout/{shape}/landcover.tif",
+        path="<resources>/automatic/cutout/{shape}/landcover.tif",
     log:
         "<logs>/{shape}/clip_landcover.log",
-    conda:
-        "../envs/module.yaml"
     message:
         "Cut land cover data to the bounds of the input shapefile."
-    shell:
-        """
-        python {input.script:q} {input.landcover:q} {input.shapes:q} {output:q} 2>{log:q}
-        """
+    wrapper:
+        "v9.14.0/geo/rasterio/clip"
 
 
 ##
@@ -194,7 +179,7 @@ rule download_ghsl:
         "Download the GHSL (Global Human Settlement Layer) built-up surface data."
     shell:
         """
-        curl -sSLo {output:q} {params.url:q}
+        curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1
         """
 
 
@@ -214,27 +199,22 @@ rule unzip_ghsl:
         "Unzip the relevant TIF file from the GHSL data."
     shell:
         """
-        python {input.script:q} {input.zipfile:q} -f {params.target_file:q} -o {output:q} 2>{log:q}
+        python {input.script:q} {input.zipfile:q} -f {params.target_file:q} -o {output:q} >{log:q} 2>&1
         """
 
 
 rule clip_settlement:
     input:
-        script=workflow.source_path("../scripts/clip_raster.py"),
-        shapes="<shapes>",
-        settlement=rules.unzip_ghsl.output,
+        like_vector=rules.normalise_shapes.output.shapes,
+        raster=rules.unzip_ghsl.output[0],
     output:
-        "<resources>/automatic/cutout/{shape}/settlement.tif",
+        path="<resources>/automatic/cutout/{shape}/settlement.tif",
     log:
         "<logs>/{shape}/clip_settlement.log",
-    conda:
-        "../envs/module.yaml"
     message:
         "Cut settlement data to the bounds of the input shapefile."
-    shell:
-        """
-        python {input.script:q} {input.settlement:q} {input.shapes:q} {output:q} 2>{log:q}
-        """
+    wrapper:
+        "v9.14.0/geo/rasterio/clip"
 
 
 ##
@@ -245,8 +225,8 @@ rule clip_settlement:
 rule rasterise_clip_wdpa:
     input:
         script=workflow.source_path("../scripts/clip_and_rasterise_polys.py"),
-        shapes="<shapes>",
-        reference_raster=rules.clip_landcover.output,
+        shapes=rules.normalise_shapes.output.shapes,
+        reference_raster=rules.clip_landcover.output[0],
         protected_areas="<wdpa>",
     output:
         "<resources>/automatic/cutout/{shape}/wdpa.tif",
@@ -258,5 +238,5 @@ rule rasterise_clip_wdpa:
         "Rasterise and cut WDPA data to the bounds of the input shapefile, using the landcover raster as reference for the rasterisation."
     shell:
         """
-        python {input.script:q} {input.shapes:q} {input.reference_raster:q} {input.protected_areas:q} {output:q} 2>{log:q}
+        python {input.script:q} {input.shapes:q} {input.reference_raster:q} {input.protected_areas:q} {output:q} >{log:q} 2>&1
         """

workflow/rules/preprocess.smk

@@ -0,0 +1,38 @@
+"""Rules related to data harmonisation."""
+
+
+rule normalise_shapes:
+    input:
+        shapes="<shapes>",
+    output:
+        shapes="<resources>/automatic/normalized_shapes/{shape}.parquet",
+    log:
+        "<logs>/{shape}/normalise_shapes.log",
+    conda:
+        "../envs/module.yaml"
+    params:
+        crs=internal["target_crs"],
+    message:
+        "Validate and normalize {wildcards.shape} to {params.crs}."
+    script:
+        "../scripts/normalise_shapes.py"
+
+
+checkpoint breakup_shape:
+    input:
+        script=workflow.source_path("../scripts/breakup_shape.py"),
+        shapes=rules.normalise_shapes.output.shapes,
+    output:
+        directory("<resources>/automatic/shapes/{shape}"),
+    log:
+        "<logs>/{shape}/breakup_shape.log",
+    conda:
+        "../envs/module.yaml"
+    params:
+        split_by=config["split_by"],
+    message:
+        "Break up {wildcards.shape} into the configured subunits."
+    shell:
+        """
+        python {input.script:q} {input.shapes:q} {params.split_by:q} {output:q} >{log:q} 2>&1
+        """