From be8c7a59f9819d49e2a08c9d995c83919bc2ec1e Mon Sep 17 00:00:00 2001 From: Zuhdil Herry Kurnia Date: Tue, 16 Jun 2026 17:50:37 +0700 Subject: [PATCH 1/5] Preserve raw 2025 pipeline-inclusion value through bronze-to-silver The WHO Priority page must count only candidates whose 2025 `new_includeinpipeline` is exactly Yes. The existing `includeinpipeline` flag forward-fills from earlier years during temporal expansion, so a blank 2025 value can still read as included. Capture the raw 2025 value into a candidate-grain `includeinpipeline_2025_raw` column before `_expand_temporal_rows` consumes and drops `new_includeinpipeline`, so the gold layer can derive a strict 2025-only flag. --- .../transformations/candidates.py | 10 ++++++++++ tests/unit/test_candidates.py | 20 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/igh_data_transform/transformations/candidates.py b/src/igh_data_transform/transformations/candidates.py index 637377b..242386c 100644 --- a/src/igh_data_transform/transformations/candidates.py +++ b/src/igh_data_transform/transformations/candidates.py @@ -307,6 +307,16 @@ def transform_candidates( # 1b. Normalize text-valued pipeline columns to integer codes df = _normalize_pipeline_cols(df) + # 1c. Capture the strict 2025 pipeline-inclusion value before temporal + # expansion consumes and drops `new_includeinpipeline`. The WHO + # Priority page needs the *actual* 2025 value (not the forward-filled + # flag), so we preserve it candidate-grain under a stable name. Absent + # column / NaN stays NaN; the gold CASE maps NaN and "No" to 0. + if "new_includeinpipeline" in df.columns: + df["includeinpipeline_2025_raw"] = df["new_includeinpipeline"] + else: + df["includeinpipeline_2025_raw"] = None + # 2. Temporal expansion (reads original bronze column names) df = _expand_temporal_rows(df) diff --git a/tests/unit/test_candidates.py b/tests/unit/test_candidates.py index e68eec5..15d71d8 100644 --- a/tests/unit/test_candidates.py +++ b/tests/unit/test_candidates.py @@ -1086,6 +1086,26 @@ def test_new_columns_not_dropped(self): # new_platform keeps its name (no prefix to strip) assert "new_platform" in result.columns + def test_includeinpipeline_2025_raw_preserved_at_candidate_grain(self): + """The raw 2025 inclusion value is captured into + `includeinpipeline_2025_raw` and copied onto every temporal row + for the candidate (not forward-filled, not dropped).""" + df = self._make_input_df() + lookup = self._make_lookup_tables() + result, _ = transform_candidates(df, lookup_tables=lookup) + + assert "includeinpipeline_2025_raw" in result.columns + + # Candidate A: bronze new_includeinpipeline == 100000000 (Yes) + rows_a = result[result["candidateid"] == "id-1"] + assert len(rows_a) >= 1 + assert (rows_a["includeinpipeline_2025_raw"] == 100000000.0).all() + + # Candidate C: bronze new_includeinpipeline == 100000001 (No) — + # carried verbatim, NOT coerced. + rows_c = result[result["candidateid"] == "id-3"] + assert (rows_c["includeinpipeline_2025_raw"] == 100000001.0).all() + def test_ctregistrylink_synthesis_applied(self): """CT registry link is cleaned during transform.""" df = self._make_input_df( From 799edc3374bd37e478999ca322333439f9c2ba42 Mon Sep 17 00:00:00 2001 From: Zuhdil Herry Kurnia Date: Tue, 16 Jun 2026 17:55:55 +0700 Subject: [PATCH 2/5] Preserve includeinpipeline_2025_raw when the 2025 value is entirely null drop_empty_columns would otherwise silently remove the new column when every candidate's 2025 inclusion is blank (or the source column is absent), leaving the gold layer to KeyError on a missing column. Add it to the preserve list so the schema stays consistent, and cover the all-null case with a test. Tighten the existing grain test's `len(rows_a) >= 1` to `> 1` to confirm the value fans out across all temporal rows for candidate id-1, not just one. --- .../transformations/candidates.py | 4 +++- tests/unit/test_candidates.py | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/igh_data_transform/transformations/candidates.py b/src/igh_data_transform/transformations/candidates.py index 242386c..5b0edd8 100644 --- a/src/igh_data_transform/transformations/candidates.py +++ b/src/igh_data_transform/transformations/candidates.py @@ -322,7 +322,9 @@ def transform_candidates( # 3. Drop columns (temporal sources already consumed by expansion) df = drop_columns_by_name(df, COLUMNS_TO_DROP) - df = drop_empty_columns(df, preserve=["valid_to", "valid_from"]) + df = drop_empty_columns( + df, preserve=["valid_to", "valid_from", "includeinpipeline_2025_raw"] + ) # 3b. Synthesize key clinical trial link if "new_ctregistrylink" in df.columns: diff --git a/tests/unit/test_candidates.py b/tests/unit/test_candidates.py index 15d71d8..96dc2aa 100644 --- a/tests/unit/test_candidates.py +++ b/tests/unit/test_candidates.py @@ -1098,7 +1098,7 @@ def test_includeinpipeline_2025_raw_preserved_at_candidate_grain(self): # Candidate A: bronze new_includeinpipeline == 100000000 (Yes) rows_a = result[result["candidateid"] == "id-1"] - assert len(rows_a) >= 1 + assert len(rows_a) > 1 assert (rows_a["includeinpipeline_2025_raw"] == 100000000.0).all() # Candidate C: bronze new_includeinpipeline == 100000001 (No) — @@ -1106,6 +1106,20 @@ def test_includeinpipeline_2025_raw_preserved_at_candidate_grain(self): rows_c = result[result["candidateid"] == "id-3"] assert (rows_c["includeinpipeline_2025_raw"] == 100000001.0).all() + def test_includeinpipeline_2025_raw_survives_when_all_null(self): + """includeinpipeline_2025_raw is never dropped by drop_empty_columns + even when every candidate's 2025 value is null (all-None override).""" + df = self._make_input_df( + overrides={"new_includeinpipeline": [None, None, None]} + ) + lookup = self._make_lookup_tables() + result, _ = transform_candidates(df, lookup_tables=lookup) + + # Column must exist — if drop_empty_columns swallowed it, gold KeyErrors. + assert "includeinpipeline_2025_raw" in result.columns + # Every value should be null (the source was all-None). + assert result["includeinpipeline_2025_raw"].isna().all() + def test_ctregistrylink_synthesis_applied(self): """CT registry link is cleaned during transform.""" df = self._make_input_df( From f0f2ce253634400efb23a78bab9f2ea088d3e2e5 Mon Sep 17 00:00:00 2001 From: Zuhdil Herry Kurnia Date: Tue, 16 Jun 2026 17:57:51 +0700 Subject: [PATCH 3/5] Add strict 2025 pipeline-inclusion flag to dim_candidate_core Map the captured `includeinpipeline_2025_raw` to a new `new_include_in_pipeline_2025` boolean that is 1 only when the 2025 value is exactly Yes. This is deliberately separate from the forward-filled `include_in_pipeline` fact flag so existing pages keep their behaviour while the WHO Priority page can filter strictly. --- .../silver_to_gold/config/schema_map.py | 5 +++++ tests/unit/test_silver_to_gold.py | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/igh_data_transform/transformations/silver_to_gold/config/schema_map.py b/src/igh_data_transform/transformations/silver_to_gold/config/schema_map.py index 971a4ef..2d0334a 100644 --- a/src/igh_data_transform/transformations/silver_to_gold/config/schema_map.py +++ b/src/igh_data_transform/transformations/silver_to_gold/config/schema_map.py @@ -53,6 +53,11 @@ "countries_approved_count": "numberofcountrieswithproductapproval", "countries_approved_agg": "countries_product_approved", "candidate_type": "CASE WHEN captype_value = 'c1746ad3-93d1-f011-bbd3-00224892cefa' THEN 'Candidate' WHEN captype_value = '545d63d9-93d1-f011-bbd3-00224892cefa' THEN 'Product' ELSE 'Other' END", + # Strict 2025 pipeline-inclusion flag for the WHO Priority page. + # 1 only when the raw 2025 `new_includeinpipeline` is exactly Yes; + # No, Pending, and blank all map to 0. Intentionally distinct from + # the forward-filled `fact_pipeline_snapshot.include_in_pipeline`. + "new_include_in_pipeline_2025": "CASE WHEN includeinpipeline_2025_raw = 100000000 THEN 1 ELSE 0 END", "indication": "indication", "indication_type": "OPTIONSET:indicationtype", "healthcare_facility_level": "OPTIONSET:healthcarefacilitylevel|vin_healthcarefacilitylevel", diff --git a/tests/unit/test_silver_to_gold.py b/tests/unit/test_silver_to_gold.py index a13872c..018b85f 100644 --- a/tests/unit/test_silver_to_gold.py +++ b/tests/unit/test_silver_to_gold.py @@ -90,3 +90,15 @@ def test_dedicated_to_women_or_children_uses_optionset_resolution( STAR_SCHEMA_MAP["dim_priority"]["dedicated_to_women_or_children"] == "OPTIONSET:crc8b_dedicatedtowomenorchildren" ) + + def test_new_include_in_pipeline_2025_is_strict_yes_only(self) -> None: + """`dim_candidate_core.new_include_in_pipeline_2025` is 1 only when + the raw 2025 value is exactly Yes (100000000); No/Pending/NULL → 0.""" + from igh_data_transform.transformations.silver_to_gold.config.schema_map import ( # noqa: E501 + STAR_SCHEMA_MAP, + ) + + assert ( + STAR_SCHEMA_MAP["dim_candidate_core"]["new_include_in_pipeline_2025"] + == "CASE WHEN includeinpipeline_2025_raw = 100000000 THEN 1 ELSE 0 END" + ) From 18191dd1a9f5557d593dbca6c08c4911def382d5 Mon Sep 17 00:00:00 2001 From: Zuhdil Herry Kurnia Date: Tue, 16 Jun 2026 18:02:18 +0700 Subject: [PATCH 4/5] Assert strict 2025 flag is present and boolean in gold dim_candidate_core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an e2e assertion that `new_include_in_pipeline_2025` exists in `dim_candidate_core` and holds only integer 0/1 values after the full bronze→silver→gold transform. Running the test against the real bronze DB revealed that the column was stored as TEXT ('0'/'1') rather than INTEGER. The DDL generator infers column affinity by name suffix, but `new_include_in_pipeline_2025` does not end in any of the known INTEGER suffixes (_key, _id, _flag, _count), so it defaulted to TEXT. Adding it to `INTEGER_EXACT_NAMES` in `ddl_generator.py` fixes the affinity, which also ensures the column reads back as integers wherever the gold schema is queried in production. --- .../transformations/silver_to_gold/core/ddl_generator.py | 4 ++++ tests/e2e/test_silver_to_gold_e2e.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/igh_data_transform/transformations/silver_to_gold/core/ddl_generator.py b/src/igh_data_transform/transformations/silver_to_gold/core/ddl_generator.py index c363838..d1b3b9d 100644 --- a/src/igh_data_transform/transformations/silver_to_gold/core/ddl_generator.py +++ b/src/igh_data_transform/transformations/silver_to_gold/core/ddl_generator.py @@ -26,6 +26,10 @@ "quarter", "enrollment_count", "option_code", + # Strict 2025 pipeline-inclusion flag: CASE WHEN evaluates to int 0/1, + # but the column name doesn't match any INTEGER_SUFFIXES pattern, so + # it must be listed here explicitly to get INTEGER affinity in SQLite. + "new_include_in_pipeline_2025", } diff --git a/tests/e2e/test_silver_to_gold_e2e.py b/tests/e2e/test_silver_to_gold_e2e.py index f8b2537..78fd6ff 100644 --- a/tests/e2e/test_silver_to_gold_e2e.py +++ b/tests/e2e/test_silver_to_gold_e2e.py @@ -176,6 +176,12 @@ def test_dim_candidate_core_test_format_is_label_not_code(self, gold_conn): f"{offenders.unique().tolist()[:5]}" ) + def test_dim_candidate_core_strict_2025_flag(self, gold_conn): + """The strict 2025 flag exists and is strictly 0/1.""" + df = _read_table(gold_conn, "dim_candidate_core") + assert "new_include_in_pipeline_2025" in df.columns + assert set(df["new_include_in_pipeline_2025"].dropna().unique()).issubset({0, 1}) + # -- dim_disease -- def test_dim_disease_no_null_diseaseid(self, gold_conn): From 47ac6529d9bf3c12f5ab18e599e605813d931ac4 Mon Sep 17 00:00:00 2001 From: Zuhdil Herry Kurnia Date: Wed, 17 Jun 2026 02:19:55 +0700 Subject: [PATCH 5/5] Format the strict-2025 e2e assertion to satisfy ruff format CI runs `ruff format --check src/ tests/`; the long assert line in the new dim_candidate_core strict-flag test exceeded the line length. Wrap it to match the formatter. --- tests/e2e/test_silver_to_gold_e2e.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/test_silver_to_gold_e2e.py b/tests/e2e/test_silver_to_gold_e2e.py index 78fd6ff..a948bef 100644 --- a/tests/e2e/test_silver_to_gold_e2e.py +++ b/tests/e2e/test_silver_to_gold_e2e.py @@ -180,7 +180,9 @@ def test_dim_candidate_core_strict_2025_flag(self, gold_conn): """The strict 2025 flag exists and is strictly 0/1.""" df = _read_table(gold_conn, "dim_candidate_core") assert "new_include_in_pipeline_2025" in df.columns - assert set(df["new_include_in_pipeline_2025"].dropna().unique()).issubset({0, 1}) + assert set(df["new_include_in_pipeline_2025"].dropna().unique()).issubset( + {0, 1} + ) # -- dim_disease --