fix(lineage): aligner les preuves du référentiel station

contracts/bronze-ref-station.odcs.yaml

@@ -2,7 +2,7 @@ apiVersion: v3.0.2
 kind: DataContract
 id: urn:infoclimat:contract:bronze-ref-station
 name: Bronze référentiel stations — source legacy immuable
-version: 0.1.0
+version: 0.2.0
 status: active
 domain: referentiel
 dataProduct: ref-station
@@ -44,6 +44,18 @@ schema:
       erronées de 67128/-19.467/55.483, qui restent des valeurs source
       immuables et auditables.
     properties:
+      - name: source_station_id
+        physicalType: TEXT
+        required: true
+        description: Copie textuelle de ic_id utilisée comme clé source stable du pipeline.
+      - name: source_catalog
+        physicalType: TEXT
+        required: true
+        description: Catalogue physique lu, actuellement warehouse.
+      - name: source_relation
+        physicalType: TEXT
+        required: true
+        description: Relation physique lue, actuellement dim.station.
       - name: source_snapshot_id
         physicalType: TEXT
         required: true
@@ -54,6 +66,10 @@ schema:
         physicalType: TEXT
         required: true
         description: Identifiant du run qui a écrit la capture Bronze.
+      - name: ingested_at
+        physicalType: TIMESTAMP
+        required: true
+        description: Instant UTC d'archivage de la capture.
       - name: ic_id
         physicalType: TEXT
         required: true
@@ -66,12 +82,6 @@ schema:
       - name: longitude
         physicalType: DOUBLE
         description: Longitude WGS84 brute, sans correction ; 67128 peut contenir sa mauvaise coordonnée source.
-      - name: station_name
-        physicalType: TEXT
-        description: Libellé station brut issu de dim.station.
-      - name: source_payload
-        physicalType: JSON
-        description: Enregistrement source complet, conservé pour audit et replay.
     quality:
       - rule: uniqueness
         description: Une capture ne contient qu'une ligne par couple (source_snapshot_id, ic_id).

contracts/gold-ref.odcs.yaml

@@ -2,7 +2,7 @@ apiVersion: v3.0.2
 kind: DataContract
 id: urn:infoclimat:contract:gold-ref
 name: Gold référentiel stations (annuaire géolocalisé + réseau + activité par paramètre)
-version: 0.2.0
+version: 0.2.1
 status: draft
 domain: referentiel
 dataProduct: gold-ref
@@ -121,9 +121,12 @@ schema:
     physicalName: gold_ref.station_alias
     physicalType: table
     description: >-
-      Relations d'identifiants historiques vers leur station Gold canonique.
-      Les alias permettent de résoudre les anciens codes sans diffuser les
-      lignes Silver quarantined.
+      Crosswalk interne d'identifiants historiques vers leur station Gold
+      canonique. Il alimente la résolution d'alias de /stations mais n'est ni un
+      dataset public ni un endpoint /ref/station-alias.
+    customProperties:
+      - property: served
+        value: false
     properties:
       - name: alias_ic_id
         physicalType: TEXT

contracts/silver-ref-station.odcs.yaml

@@ -2,7 +2,7 @@ apiVersion: v3.0.2
 kind: DataContract
 id: urn:infoclimat:contract:silver-ref-station
 name: Silver référentiel stations — canonisation et alias
-version: 0.1.0
+version: 0.2.0
 status: active
 domain: referentiel
 dataProduct: ref-station
@@ -49,6 +49,13 @@ schema:
         primaryKey: true
         primaryKeyPosition: 1
         description: Identifiant de cette ligne de référentiel station.
+      - name: source_station_id
+        physicalType: TEXT
+        required: true
+        description: Identifiant textuel de la ligne Bronze ayant produit cette décision.
+      - name: station_uid_source
+        physicalType: TEXT
+        description: UID interne de la ligne source, conservé pour les jointures historiques.
       - name: source_latitude
         physicalType: DOUBLE
         description: Latitude WGS84 observée dans Bronze, y compris une valeur erronée de 67128.
@@ -85,6 +92,12 @@ schema:
             mustBeGreaterOrEqualTo: -180
             mustBeLessOrEqualTo: 180
             severity: error
+      - name: canonical_ic_id
+        physicalType: TEXT
+        description: Identifiant canonique résolu ; nul uniquement pour une ligne quarantined.
+      - name: canonical_station_uid
+        physicalType: TEXT
+        description: UID interne de la station canonique résolue.
       - name: quality_status
         physicalType: TEXT
         required: true
@@ -103,6 +116,9 @@ schema:
       - name: quality_flags
         physicalType: ARRAY
         description: Drapeaux explicites motivant la décision qualité.
+      - name: aliases
+        physicalType: ARRAY
+        description: Identifiants obsolètes rattachés à cette ligne lorsqu'elle est canonique.
       - name: correction_rule_id
         physicalType: TEXT
         description: Identifiant versionné de la règle de correction ou de quarantaine appliquée.
@@ -120,6 +136,18 @@ schema:
         physicalType: TEXT
         required: true
         description: Run ayant produit ou réconcilié la ligne Silver.
+      - name: source_catalog
+        physicalType: TEXT
+        required: true
+        description: Catalogue Bronze d'origine.
+      - name: source_relation
+        physicalType: TEXT
+        required: true
+        description: Relation Bronze d'origine.
+      - name: ingested_at
+        physicalType: TIMESTAMP
+        required: true
+        description: Instant UTC d'archivage Bronze conservé en Silver.
 
   - name: station_alias
     physicalName: silver_ref.station_alias
@@ -134,10 +162,18 @@ schema:
         primaryKey: true
         primaryKeyPosition: 1
         description: Identifiant station alias, obsolète ou mis en quarantaine.
+      - name: alias_station_uid
+        physicalType: TEXT
+        required: true
+        description: UID interne historique de l'alias.
       - name: canonical_ic_id
         physicalType: TEXT
         required: true
         description: Identifiant station canonique à utiliser après résolution.
+      - name: canonical_station_uid
+        physicalType: TEXT
+        required: true
+        description: UID interne de la station canonique cible.
       - name: relation
         physicalType: TEXT
         required: true
@@ -156,9 +192,11 @@ schema:
         description: Identifiant versionné de la règle ayant créé la relation.
       - name: source_snapshot_id
         physicalType: TEXT
+        required: true
         description: Capture Bronze à l'origine de la relation.
       - name: ingest_run_id
         physicalType: TEXT
+        required: true
         description: Run ayant créé la relation d'alias.
 
 team:

contracts/station-reference.schema.json

@@ -0,0 +1,99 @@
+{
+  "contentSha256": "sha256:5910c785e5eab04313aa4c5dbc8e73124f2e8f44229372678a648c6253a8da66",
+  "manifestVersion": "station-reference.schema/v1",
+  "sourceContracts": {
+    "bronze-ref-station.odcs.yaml": "0.2.0",
+    "gold-ref.odcs.yaml": "0.2.1",
+    "silver-ref-station.odcs.yaml": "0.2.0"
+  },
+  "tables": {
+    "bronze_ref.station_source": {
+      "fields": {
+        "ic_id": {"physicalType": "TEXT", "required": true},
+        "ingest_run_id": {"physicalType": "TEXT", "required": true},
+        "ingested_at": {"physicalType": "TIMESTAMP", "required": true},
+        "latitude": {"physicalType": "DOUBLE", "required": false},
+        "longitude": {"physicalType": "DOUBLE", "required": false},
+        "source_catalog": {"physicalType": "TEXT", "required": true},
+        "source_relation": {"physicalType": "TEXT", "required": true},
+        "source_snapshot_id": {"physicalType": "TEXT", "required": true},
+        "source_station_id": {"physicalType": "TEXT", "required": true}
+      }
+    },
+    "gold_ref.station": {
+      "fields": {
+        "aliases": {"physicalType": "ARRAY", "required": false},
+        "altitude": {"physicalType": "INTEGER", "required": false},
+        "departement": {"physicalType": "TEXT", "required": false},
+        "derniere_activite": {"physicalType": "TIMESTAMP", "required": false},
+        "dh_ouverture": {"physicalType": "TIMESTAMP", "required": false},
+        "genre": {"physicalType": "TEXT", "required": false},
+        "ic_id": {"physicalType": "TEXT", "required": true},
+        "latitude": {"physicalType": "DOUBLE", "required": true},
+        "libelle": {"physicalType": "TEXT", "required": false},
+        "libelle_source": {"physicalType": "TEXT", "required": false},
+        "longitude": {"physicalType": "DOUBLE", "required": true},
+        "mfid": {"physicalType": "TEXT", "required": false},
+        "ouverte": {"physicalType": "BOOLEAN", "required": false},
+        "parametres_mesures": {"physicalType": "ARRAY", "required": false},
+        "pays": {"physicalType": "TEXT", "required": false},
+        "publiable": {"physicalType": "BOOLEAN", "required": true},
+        "station_uid": {"physicalType": "TEXT", "required": true},
+        "type_poste": {"physicalType": "TEXT", "required": false}
+      }
+    },
+    "gold_ref.station_alias": {
+      "fields": {
+        "alias_ic_id": {"physicalType": "TEXT", "required": true},
+        "canonical_ic_id": {"physicalType": "TEXT", "required": true},
+        "relation": {"physicalType": "TEXT", "required": true}
+      },
+      "served": false
+    },
+    "gold_ref.station_parametre": {
+      "fields": {
+        "derniere_obs": {"physicalType": "TIMESTAMP", "required": false},
+        "n_obs": {"physicalType": "BIGINT", "required": false},
+        "parametre": {"physicalType": "TEXT", "required": true},
+        "premiere_obs": {"physicalType": "TIMESTAMP", "required": false},
+        "station_uid": {"physicalType": "TEXT", "required": true}
+      }
+    },
+    "silver_ref.station": {
+      "fields": {
+        "aliases": {"physicalType": "ARRAY", "required": false},
+        "authority_checked_at": {"physicalType": "TIMESTAMP", "required": false},
+        "authority_uri": {"physicalType": "TEXT", "required": false},
+        "canonical_ic_id": {"physicalType": "TEXT", "required": false},
+        "canonical_latitude": {"physicalType": "DOUBLE", "required": false},
+        "canonical_longitude": {"physicalType": "DOUBLE", "required": false},
+        "canonical_station_uid": {"physicalType": "TEXT", "required": false},
+        "correction_rule_id": {"physicalType": "TEXT", "required": false},
+        "ic_id": {"physicalType": "TEXT", "required": true},
+        "ingest_run_id": {"physicalType": "TEXT", "required": true},
+        "ingested_at": {"physicalType": "TIMESTAMP", "required": true},
+        "quality_flags": {"physicalType": "ARRAY", "required": false},
+        "quality_status": {"physicalType": "TEXT", "required": true},
+        "source_catalog": {"physicalType": "TEXT", "required": true},
+        "source_latitude": {"physicalType": "DOUBLE", "required": false},
+        "source_longitude": {"physicalType": "DOUBLE", "required": false},
+        "source_relation": {"physicalType": "TEXT", "required": true},
+        "source_snapshot_id": {"physicalType": "TEXT", "required": true},
+        "source_station_id": {"physicalType": "TEXT", "required": true},
+        "station_uid_source": {"physicalType": "TEXT", "required": false}
+      }
+    },
+    "silver_ref.station_alias": {
+      "fields": {
+        "alias_ic_id": {"physicalType": "TEXT", "required": true},
+        "alias_station_uid": {"physicalType": "TEXT", "required": true},
+        "canonical_ic_id": {"physicalType": "TEXT", "required": true},
+        "canonical_station_uid": {"physicalType": "TEXT", "required": true},
+        "correction_rule_id": {"physicalType": "TEXT", "required": true},
+        "ingest_run_id": {"physicalType": "TEXT", "required": true},
+        "relation": {"physicalType": "TEXT", "required": true},
+        "source_snapshot_id": {"physicalType": "TEXT", "required": true}
+      }
+    }
+  }
+}

lineage/namespaces.md

@@ -51,6 +51,11 @@ jamais bloquante) + forwarder `tools/lineage_forward.py` → backend Marquez sel
 4. Le champ `producer` vaut `https://github.com/infoclimat/site-infoclimat` (versionné).
 5. Les pipelines contractualisés référencent leur job via la customProperty `lineageJob`
    du contrat ODCS.
+6. Les cinq jobs `station_ref` peuvent joindre au seul événement terminal le facet
+   custom `infoclimat_station_reference`. Son JSON `1.0`, borné à 64 KiB, contient
+   uniquement les snapshot IDs d'entrée/sortie, l'`ingest_run_id`, les comptes de
+   statuts/flags et la version content-addressed du registre. Les lignes brutes,
+   chemins de secrets et credentials y sont interdits.
 
 ## Pipelines prioritaires à instrumenter
 

tools/README.md

@@ -30,6 +30,13 @@ récurrent (cron / Kestra) qui publiera son résultat.
 | `check_dual_source.py [--check]` | Couverture/comptes MariaDB ↔ TimescaleDB par station/jour (cibles `audits/dual-source-targets.yaml`) | 2 agrégats bornés par fenêtre, colonnes indexées des deux côtés |
 | `volumetrie_audit.py` | Inventaire de volumétrie des bases | lecture `information_schema` |
 
+`lineage_run.py` accepte aussi `--evidence <fichier>` (ou
+`LINEAGE_EVIDENCE_PATH`). Après la commande, il valide un JSON borné à 64 KiB
+contenant uniquement les snapshot IDs, l'`ingest_run_id`, les comptes de statuts
+et flags et la version du registre. Cette preuve est jointe au facet terminal
+`infoclimat_station_reference`; une preuve absente ou invalide reste non
+bloquante et aucune ligne brute ou secret n'est admis.
+
 Prérequis Python : `pip install pymysql 'psycopg[binary]' pyyaml 'datacontract-cli[postgres,mysql]'`.
 
 ### Pièges connus

tools/export_station_reference_schema.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Exporte le manifeste JSON station-reference depuis les contrats ODCS."""
+
+import hashlib
+import json
+from pathlib import Path
+import sys
+
+import yaml
+
+
+CONTRACTS = (
+    "bronze-ref-station.odcs.yaml",
+    "silver-ref-station.odcs.yaml",
+    "gold-ref.odcs.yaml",
+)
+MANIFEST_VERSION = "station-reference.schema/v1"
+
+
+def custom_property(item: dict, name: str):
+    for prop in item.get("customProperties", []):
+        if prop.get("property") == name:
+            return prop.get("value")
+    return None
+
+
+def build_manifest(root: Path) -> dict:
+    source_contracts = {}
+    tables = {}
+    for name in CONTRACTS:
+        document = yaml.safe_load((root / "contracts" / name).read_text())
+        source_contracts[name] = document["version"]
+        for table in document["schema"]:
+            exported_table = {
+                "fields": {
+                    prop["name"]: {
+                        "physicalType": prop["physicalType"],
+                        "required": bool(prop.get("required", False)),
+                    }
+                    for prop in table["properties"]
+                }
+            }
+            served = custom_property(table, "served")
+            if served is not None:
+                exported_table["served"] = served
+            tables[table["physicalName"]] = exported_table
+    payload = {
+        "manifestVersion": MANIFEST_VERSION,
+        "sourceContracts": source_contracts,
+        "tables": tables,
+    }
+    digest = hashlib.sha256(
+        json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
+    ).hexdigest()
+    return {**payload, "contentSha256": f"sha256:{digest}"}
+
+
+def main() -> int:
+    root = Path(__file__).resolve().parents[1]
+    print(json.dumps(build_manifest(root), indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())