Add reproducible Yaounde boundary and 30 m grid pipeline

.gitignore

@@ -61,3 +61,42 @@ ENV/
 .cache/
 tmp/
 temp/
+
+# Virtual environments
+.venv/
+.venvt/
+venv/
+env/
+
+# Raw downloaded geospatial data
+data/raw/boundaries/**/*.zip
+data/raw/boundaries/**/*.geojson
+data/raw/boundaries/**/*.shp
+data/raw/boundaries/**/*.shx
+data/raw/boundaries/**/*.dbf
+data/raw/boundaries/**/*.prj
+data/raw/boundaries/**/*.gpkg
+data/raw/boundaries/hdx_cod_ab_cmr/extracted/
+data/metadata/landsat/_checkpoints/
+
+/data/raw/osm/*.zip
+/data/raw/osm/**/*.gpkg
+/data/processed/osm/*.gpkg
+
+# Generated geospatial outputs
+data/processed/**/*.tif
+data/processed/**/*.tiff
+data/processed/**/*.gpkg
+data/processed/**/*.parquet
+
+# Earth Engine resumable catalogue checkpoints
+**/_checkpoints/
+data/metadata/landsat/_checkpoints/
+
+# Temporary geospatial files
+*.aux.xml
+*.ovr
+
+
+
+

Makefile

@@ -0,0 +1,140 @@
+# =============================================================================
+#  Administrative boundaries and study grid
+# =============================================================================
+
+.PHONY: help install inspect-adm2 inspect-adm3 inspect-boundaries build-grid test-grid day1
+
+help:  ## Display this help with a description of all available commands
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
+
+install:  ## Install the package in development mode with dev dependencies
+	python -m pip install -e ".[dev]"
+
+inspect-adm2:  ## Inspect the ADM2 layer and generate the inspection report
+	python -m src.analysis.boundaries.inspect_boundaries \
+		--input data/raw/boundaries/hdx_cod_ab_cmr/extracted/cmr_admin2.geojson \
+		--report data/metadata/source_boundary_inspection_adm2.json
+
+inspect-adm3:  ## Inspect the ADM3 layer and generate the inspection report
+	python -m src.analysis.boundaries.inspect_boundaries \
+		--input data/raw/boundaries/hdx_cod_ab_cmr/extracted/cmr_admin3.geojson \
+		--report data/metadata/source_boundary_inspection_adm3.json
+
+inspect-boundaries: inspect-adm2 inspect-adm3  ## Run both ADM2 and ADM3 inspections
+
+build-grid:  ## Build the study grid from the YAML configuration
+	python -m src.analysis.boundaries.build_study_grid \
+		--config configs/study_area.yaml
+
+visualize-grid: ## Visualize the study grid and generate all diagnostic plots (overview, comparison, masks, and grid alignment) in reports/day1/
+	python -m src.analysis.boundaries.visualize_study_grid \
+		--config configs/study_area.yaml
+
+test-grid:  ## Run pytest on the study grid module
+	pytest tests/test_study_grid.py -v
+
+qa-day1: build-grid visualize-grid test-grid  ## Run the complete QA pipeline (build, visualize, test) without updating the reference baseline
+
+day1: inspect-boundaries build-grid test-grid  ## Run the complete Day 1 pipeline (inspection, grid build, tests)
+
+# =============================================================================
+#  — Landsat catalog and compositing windows
+# =============================================================================
+
+.PHONY: landsat-catalog landsat-visuals test-landsat day2
+
+landsat-catalog: ## Build the full Landsat scene manifest by querying Earth Engine, compute QA metrics, and select optimal compositing windows
+	python -m src.analysis.landsat.build_catalog \
+		--config configs/landsat_catalog.yaml
+
+landsat-visuals: ## Generate diagnostic plots for the Landsat catalog availability and selection strategy (reports/day2/)
+	python -m src.analysis.landsat.visualize_catalog \
+		--config configs/landsat_catalog.yaml
+
+test-landsat: ## Run the pytest suite for the Landsat catalog module with verbose output
+	pytest tests/test_landsat_catalog.py -v
+
+day2: landsat-catalog landsat-visuals test-landsat ## Execute the complete Day 2 pipeline: build the Landsat catalog, generate visualizations, and run tests
+
+# =============================================================================
+#  — Orchestration (Landsat composites, terrain, OSM)
+# =============================================================================
+
+.PHONY: \
+	orchestrate-preflight \
+	orchestrate-submit \
+	orchestrate-status \
+	orchestrate-osm \
+	orchestrate-finalize \
+	test-orchestrate \
+	freeze-orchestrate-v1
+
+orchestrate-preflight: ## Check all frozen Day 1 and Day 2 dependencies (grid, catalog, manifests) before submitting Day 3 exports
+	python -m src.analysis.orchestration.preflight \
+		--config configs/orchestrate_sources.yaml
+
+orchestrate-submit: ## Submit all Earth Engine export tasks for Landsat composites, valid-count layers, and the SRTM terrain product
+	python -m src.analysis.landsat.build_composites \
+		--config configs/orchestrate_sources.yaml \
+		--submit
+
+orchestrate-status:  ## Check the current running/completed status of all submitted Earth Engine export tasks
+	python -m src.analysis.orchestration.finalize \
+		--config configs/orchestrate_sources.yaml \
+		--status-only
+
+orchestrate-osm: ## Download the latest Geofabrik OpenStreetMap extract, clip it to the study area, and save it as a GeoPackage
+	python -m src.analysis.auxiliary.build_osm_extract \
+		--config configs/orchestrate_sources.yaml
+
+orchestrate-finalize: ## Validate the completed Earth Engine assets, generate QA manifests, summary statistics, and finalize Day 3 metadata
+	python -m src.analysis.orchestration.finalize \
+		--config configs/orchestrate_sources.yaml
+
+test-orchestrate: ## Run the pytest suite for the orchestration (Day 3) outputs and dependencies
+	pytest tests/test_orchestrate_sources.py -v
+
+freeze-orchestrate-v1: ## Copy the validated Day 3 version manifest to tests/reference as the frozen baseline for regression tests
+	mkdir -p tests/reference
+	cp data/metadata/orchestration/orchestrate_version.json \
+		tests/reference/orchestrate_sources_v1.json
+
+# =============================================================================
+#  Built-up candidates (spectral indices + Otsu thresholds)
+# =============================================================================
+
+.PHONY: \
+	built-up-preflight \
+	built-up-submit \
+	built-up-status \
+	built-up-finalize \
+	test-built-up \
+	freeze-built-up-v1
+
+built-up-preflight: ## Run preflight checks for Day 4: validate inputs, list completed epochs, and confirm expected index/candidate layers without submitting tasks
+	python -m src.analysis.built_up.build_candidates \
+		--config configs/built_up_candidates.yaml \
+		--preflight-only
+
+built-up-submit: ## Compute spectral indices, calculate Otsu thresholds, and submit all Day 4 Earth Engine exports (indices and candidate masks)
+	python -m src.analysis.built_up.build_candidates \
+		--config configs/built_up_candidates.yaml \
+		--submit
+
+built-up-status:  ## Check the current running/completed status of all submitted Day 4 Earth Engine export tasks
+	python -m src.analysis.built_up.finalize \
+		--config configs/built_up_candidates.yaml \
+		--status-only
+
+built-up-finalize: ## Validate the exported Day 4 assets, generate histograms, threshold tables, area summaries, and the final report
+	python -m src.analysis.built_up.finalize \
+		--config configs/built_up_candidates.yaml
+
+test-built-up: ## Run the pytest suite for the built-up candidates (Day 4) module
+	python -m pytest tests/test_built_up_candidates.py -v
+
+freeze-built-up-v1: ## Copy the validated Day 4 version manifest to tests/reference as the frozen baseline for regression tests
+	mkdir -p tests/reference
+	cp \
+		data/metadata/built_up_candidates/built_up_candidates_version.json \
+		tests/reference/built_up_candidates_v1.json

README.md

@@ -123,13 +123,82 @@ Feature Engineering
 
 ---
 
+
 ## Data Sources
 
+### Administrative Boundaries
+
+The primary boundary source is the Cameroon Common Operational Dataset for
+Administrative Boundaries (`cod-ab-cmr`), provided by OCHA and originally
+produced by the Institut National de Cartographie of Cameroon.
+
+The seven ADM3 units corresponding to Yaoundé I–VII were selected using their
+administrative P-codes and dissolved to construct the Yaoundé administrative
+core. Their union was validated against the ADM2 Mfoundi boundary.
+
+All processed geometries use `EPSG:32632`. A 5 km context buffer and a convex
+hull are retained separately. The authoritative Landsat grid has a 30 m
+resolution, a fixed `(0, 0)` anchor and stable global cell identifiers.
+
 ### Earth Observation
 
-- Landsat annual imagery (1990–2020)
+The project uses Landsat Collection 2 Tier 1 Level 2 Surface Reflectance
+imagery from:
+
+- Landsat 5 TM: `LANDSAT/LT05/C02/T1_L2`
+- Landsat 7 ETM+: `LANDSAT/LE07/C02/T1_L2`
+- Landsat 8 OLI/TIRS: `LANDSAT/LC08/C02/T1_L2`
+
+The target observation epochs are:
+
+`1990`, `1995`, `2000`, `2005`, `2010`, `2015` and `2020`.
+
+For each epoch, a three-year diagnostic period was queried to assess scene
+availability around the target year. Scene-level QA masking excludes fill,
+cloud, dilated cloud, cloud shadow, snow, cirrus where applicable and
+radiometric saturation. Water is retained as a valid observation.
+
+Generated output:
+
+- a complete Landsat scene manifest;
+- scene-level valid coverage over the administrative core and context area;
+- monthly availability summaries;
+- candidate compositing-window comparisons;
+- an epoch-level quality summary;
+- the exact selected Earth Engine scene identifiers;
+- a frozen compositing protocol and catalogue checksum.
+
+The selected scene set is recorded in:
+
+- `data/metadata/landsat/selected_scene_manifest.csv`
+- `data/metadata/landsat/compositing_protocol.yaml`
+- `data/metadata/landsat/catalog_version.json`
+
+Final Landsat composites have not yet been generated. They will be constructed
+during ... by loading the exact frozen scene identifiers rather than
+re-querying the collections dynamically.
+
+### Auxiliary and Spatial Predictors
+
+The following sources are planned for Day 3 and later stages but have not yet
+been integrated into the model-ready dataset:
+
+- SRTM elevation and derived slope;
+- OpenStreetMap roads and current infrastructure;
+- GHSL built-up surface and population products.
+
+GHSL will remain an auxiliary comparison and validation source, while current
+OpenStreetMap data will be documented carefully because its historical
+completeness varies.
+
+### Spectral Built-up Candidates
+
+Completed Landsat composites are converted into SAVI, MNDWI, NDBI, IBI, IBUI, VbSWIR1-BI and NDBSUI layers. Epoch-specific Otsu thresholds are estimated inside the Yaoundé administrative core and applied to the context grid.
+
+The resulting binary maps are unvalidated candidate pseudo-labels. Missing or numerically undefined pixels remain masked and are not treated as non-built-up.
+Final index selection, comparative validation and temporal correction are performed in later stages.
 
-### Spatial Predictors
+#### Spatial Predictors
 
 Potential predictors include:
 

checksums.txt

@@ -0,0 +1,3 @@
+f24a75c41ee2e107448ff1d2b8769ee970dad376d20cd7093da0c9106814ab5a  data/raw/boundaries/hdx_cod_ab_cmr/cmr_admin_boundaries.geojson.zip
+9d0ff998057c3afe7f3a6334d71e9d7c353c64fe6e7c35b01fe14d804c97f04b  data/raw/boundaries/geoboundaries/geoBoundaries-CMR-ADM3.geojson
+81b6159418f9b8d8341822d7bcfd9d42d31c893492ce336f5f629ca08bf939fb  data/raw/boundaries/geoboundaries/geoBoundaries-CMR-ADM2.geojson

configs/built_up_candidates.yaml

@@ -0,0 +1,95 @@
+version: 2
+
+project:
+  earth_engine_project: urban-sprawl-ssa
+
+inputs:
+  grid_specification: data/metadata/grid_specification.json
+  grid_reference: tests/reference/grid_v1.json
+  orchestration_version:
+    data/metadata/orchestration/orchestrate_version.json
+  composite_manifest:
+    data/metadata/orchestration/landsat_composite_manifest.csv
+  core_boundary:
+    data/processed/boundaries/yaounde_core.gpkg
+  context_boundary:
+    data/processed/boundaries/yaounde_context_buffer.gpkg
+
+epochs:
+  mode: completed_from_manifest
+  include: [2005, 2010, 2015, 2020, 2025]
+
+indices:
+  formula_version: 1
+  denominator_epsilon: 0.000001
+  savi_l: 0.5
+  intermediate: [savi, mndwi]
+  continuous: [ndbi, ibi, ibui, vbswir1_bi, ndbsui]
+  candidates: [ndbi, ibui, vbswir1_bi, ndbsui]
+
+  excluded_from_candidates:
+    ibi:
+      reason:
+        "Unstable ratio denominator and extreme histogram values."
+      decision:
+        "Retain as a continuous diagnostic band but exclude from Otsu classification and Day 5 method comparison."
+
+classification:
+  primary_strategy: epoch_specific_otsu
+  compute_pooled_common_thresholds: false
+  fail_all_exports_on_threshold_error: true
+
+  threshold_region: core
+  application_region: context
+
+  minimum_valid_observations_for_threshold: 1
+  minimum_valid_observations_for_candidate: 1
+
+  built_up_direction:
+    ndbi: greater_than
+    ibui: greater_than
+    vbswir1_bi: greater_than
+    ndbsui: greater_than
+
+histogram:
+  max_buckets: 256
+  min_bucket_width: null
+  max_raw: 100000
+  max_pixels: 10000000
+  tile_scale: 4
+
+exports:
+  asset_root: projects/urban-sprawl-ssa/assets/sprawlgp/v1
+  index_folder: indices
+  candidate_folder: built_up_candidates
+
+  # Change temporarily to true only when replacing existing assets.
+  overwrite_existing_assets: false
+
+  max_pixels: 10000000
+  shard_size: 256
+  priority: 100
+  task_manifest:
+    data/metadata/built_up_candidates/earth_engine_tasks.csv
+
+metadata:
+  directory: data/metadata/built_up_candidates
+  dependency_manifest: dependency_manifest.csv
+  histograms: histograms.json
+  threshold_table: threshold_table.csv
+  candidate_area_summary: candidate_area_summary.csv
+  output_manifest: output_manifest.csv
+  processing_recipe: processing_recipe.json
+  version: built_up_candidates_version.json
+
+reports:
+  directory: reports/built_up_candidates
+  index_gallery: 01_index_gallery.png
+  candidate_gallery: 02_candidate_map_gallery.png
+  histogram_gallery: 03_histogram_threshold_gallery.png
+  report: built_up_candidates_report.md
+  thumbnail_dimensions: 256
+  generate_map_galleries: true
+
+reference:
+  version: tests/reference/built_up_candidates_v2.json