feat(models): port production UMF model extensions from pulseflow

src/tablespec/__init__.py

@@ -18,6 +18,7 @@
     IngestionConfig,
     IngestionExclusionRule,
     JoinViaSpec,
+    MergeCondition,
     Nullable,
     OutgoingRelationship,
     OutputConfig,
@@ -28,6 +29,7 @@
     RelationshipSummary,
     Relationships,
     Survivorship,
+    TableReference,
     UMFColumn,
     UMFColumnDerivation,
     UMFMetadata,
@@ -147,6 +149,7 @@
     "IngestionConfig",
     "IngestionExclusionRule",
     "JoinViaSpec",
+    "MergeCondition",
     "Nullable",
     "OutgoingRelationship",
     "OutputConfig",
@@ -157,6 +160,7 @@
     "RelationshipSummary",
     "Relationships",
     "Survivorship",
+    "TableReference",
     "UMFColumnDerivation",
     "ValidationRule",
     "ValidationRules",

src/tablespec/compatibility.py

@@ -42,10 +42,16 @@ class CompatibilityReport:
 # ---------------------------------------------------------------------------
 
 
-def _nullable_contexts(n: Nullable | None) -> dict[str, bool]:
-    """Return a {context_key: bool} dict for a Nullable, treating None as empty."""
+def _nullable_contexts(n: bool | Nullable | None) -> dict[str, bool]:
+    """Return a {context_key: bool} dict for a Nullable, treating None as empty.
+
+    A plain boolean nullable applies to every context and is represented by
+    the synthetic "*" key so tightening (True -> False) is still detected.
+    """
     if n is None:
         return {}
+    if isinstance(n, bool):
+        return {"*": n}
     # Nullable uses model_config extra="allow", so extra fields are in __pydantic_extra__
     result: dict[str, bool] = {}
     for key in n.model_fields_set | set(n.__pydantic_extra__ or {}):

src/tablespec/excel_converter.py

@@ -490,8 +490,13 @@ def convert(self, umf: UMF) -> openpyxl.Workbook:
         self._create_readme_sheet(umf)
         self._create_schema_sheet(umf)
         self._create_columns_sheet(umf)
-        self._create_survivorship_sheet(umf)
-        self._create_derivations_sheet(umf)
+
+        # Derivation-related sheets only apply to generated/derived tables;
+        # source-layer specs without survivorship metadata get a lean workbook.
+        if any(col.derivation or col.provenance_policy for col in umf.columns):
+            self._create_survivorship_sheet(umf)
+        if any(col.derivation for col in umf.columns):
+            self._create_derivations_sheet(umf)
 
         if self._get_expectation_dicts_for_export(umf):
             self._create_validation_sheet(umf)
@@ -669,6 +674,7 @@ def _create_columns_sheet(self, umf: UMF) -> None:
             "Format",
             "Notes",
             "_Validation",
+            "Report Sheet",
         ]
         headers.extend(post_nullable_headers)
 
@@ -688,6 +694,7 @@ def col_letter(idx: int) -> str:
         reporting_idx = desc_idx + 5
         format_idx = desc_idx + 6
         notes_idx = desc_idx + 7
+        report_sheet_idx = desc_idx + 9  # desc_idx + 8 is the _Validation column
 
         # Data
         row = 2
@@ -722,9 +729,12 @@ def col_letter(idx: int) -> str:
             self._apply_font_to_cell(ws[f"G{row}"], default_font)
 
             # Nullable - write dynamic context columns
-            if col.nullable:
+            if col.nullable is not None:
                 nullable_data: dict[str, bool] = {}
-                if isinstance(col.nullable, Nullable):
+                if isinstance(col.nullable, bool):
+                    # Simple boolean applies to every context
+                    nullable_data = dict.fromkeys(context_keys, col.nullable)
+                elif isinstance(col.nullable, Nullable):
                     nullable_data = col.nullable.model_dump(exclude_none=True)
                 elif isinstance(col.nullable, dict):
                     nullable_data = col.nullable
@@ -770,12 +780,18 @@ def col_letter(idx: int) -> str:
                 ws[f"{col_letter(notes_idx)}{row}"] = ""
             self._apply_font_to_cell(ws[f"{col_letter(notes_idx)}{row}"], default_font)
 
+            # Report Sheet (workbook tab assignment for multi-sheet reports)
+            ws[f"{col_letter(report_sheet_idx)}{row}"] = col.report_sheet or ""
+            self._apply_font_to_cell(
+                ws[f"{col_letter(report_sheet_idx)}{row}"], default_font
+            )
+
             row += 1
 
         # Adjust column widths
         pre_nullable_widths = [15, 18, 20, 12, 10, 11, 8]
         nullable_widths = [12] * num_contexts
-        post_nullable_widths = [20, 20, 12, 18, 15, 12, 15, 30, 15]
+        post_nullable_widths = [20, 20, 12, 18, 15, 12, 15, 30, 15, 15]
         all_widths = pre_nullable_widths + nullable_widths + post_nullable_widths
         for i, width in enumerate(all_widths, 1):
             ws.column_dimensions[get_column_letter(i)].width = width
@@ -1938,6 +1954,7 @@ def _get(row: tuple, idx: int | None) -> Any:
         derivation_expression_idx = header_map.get("derivation expression")
         format_idx = header_map.get("format")
         notes_idx = header_map.get("notes")
+        report_sheet_idx = header_map.get("report sheet")
 
         columns = []
         for row in ws.iter_rows(min_row=2, values_only=False):
@@ -2066,6 +2083,11 @@ def _get(row: tuple, idx: int | None) -> Any:
                         line.strip() for line in notes_val.split("\n") if line.strip()
                     ]
 
+            # Report Sheet (workbook tab assignment for multi-sheet reports)
+            report_sheet_val = _get(row, report_sheet_idx)
+            if report_sheet_val:
+                col_dict["report_sheet"] = str(report_sheet_val).strip()
+
             columns.append(col_dict)
 
         return columns

src/tablespec/gx_baseline.py

@@ -226,9 +226,11 @@ def generate_baseline_expectations(
         if include_structural:
             expectations.extend(self._generate_structural_expectations(umf_data))
 
-        # Column-level baseline expectations
+        # Column-level baseline expectations (skip internal helper columns - not in output)
         context_column = umf_data.get("context_column")
         for column in umf_data.get("columns", []):
+            if column.get("internal", False):
+                continue
             expectations.extend(
                 self.generate_baseline_column_expectations(
                     column, context_column=context_column
@@ -262,7 +264,11 @@ def _generate_structural_expectations(
 
         """
         expectations = []
-        columns = umf_data.get("columns", [])
+        # Internal helper columns are excluded from the output table, so
+        # structural checks must count only the externally visible columns.
+        columns = [
+            col for col in umf_data.get("columns", []) if not col.get("internal", False)
+        ]
         column_names = [col["name"] for col in columns]
 
         # Expect specific column count
@@ -312,7 +318,9 @@ def _generate_cross_column_expectations(
 
         """
         expectations = []
-        columns = umf_data.get("columns", [])
+        columns = [
+            col for col in umf_data.get("columns", []) if not col.get("internal", False)
+        ]
 
         # Build lookup of date/datetime columns
         date_columns: dict[str, dict[str, Any]] = {}
@@ -391,7 +399,20 @@ def generate_baseline_column_expectations(
         # Note: expect_column_to_exist and expect_column_values_to_be_of_type
         # are no longer generated here as they are in REDUNDANT_VALIDATION_TYPES
         nullable = column.get("nullable", {})
-        if nullable:
+        if nullable is False:
+            # Simple boolean form: required in all contexts
+            expectations.append(
+                {
+                    "type": "expect_column_values_to_not_be_null",
+                    "kwargs": {"column": column_name},
+                    "meta": {
+                        "description": f"Column {column_name} is required (nullable=false)",
+                        "severity": "critical",
+                        "generated_from": "baseline",
+                    },
+                }
+            )
+        elif nullable:
             if isinstance(nullable, dict):
                 # Dict format: {context: is_nullable} e.g. {"MD": False, "MP": True}
                 required_contexts = [

src/tablespec/models/__init__.py

@@ -1,6 +1,12 @@
 """UMF data models for tablespec."""
 
+from tablespec.models.pipeline import (
+    PipelineDependency,
+    PipelineMetadata,
+    TableReference,
+)
 from tablespec.models.umf import (
+    DEFAULT_PRIMARY_KEY,
     INGESTED_QUALITY_CHECK_TYPES,
     RAW_VALIDATION_TYPES,
     REDUNDANT_VALIDATION_TYPES,
@@ -23,6 +29,7 @@
     JoinViaSpec,
     JsonProjection,
     JsonSource,
+    MergeCondition,
     Nullable,
     OutgoingRelationship,
     OutputConfig,
@@ -46,6 +53,7 @@
 )
 
 __all__ = [
+    "DEFAULT_PRIMARY_KEY",
     "INGESTED_QUALITY_CHECK_TYPES",
     "RAW_VALIDATION_TYPES",
     "REDUNDANT_VALIDATION_TYPES",
@@ -68,10 +76,13 @@
     "JoinViaSpec",
     "JsonProjection",
     "JsonSource",
+    "MergeCondition",
     "Nullable",
     "OutgoingRelationship",
     "OutputConfig",
     "ParquetSource",
+    "PipelineDependency",
+    "PipelineMetadata",
     "PostUpsertRule",
     "QualityCheck",
     "QualityChecks",
@@ -80,6 +91,7 @@
     "Relationships",
     "SourceSpec",
     "Survivorship",
+    "TableReference",
     "UMFColumn",
     "UMFColumnDerivation",
     "UMFMetadata",

src/tablespec/models/pipeline.py

@@ -36,4 +36,64 @@ class PipelineMetadata(BaseModel):
     )
 
 
-__all__ = ["PipelineDependency", "PipelineMetadata"]
+class TableReference(BaseModel):
+    """Parsed table reference with optional pipeline qualification.
+
+    Supports two formats:
+    - Bare: "table_name" (references table in same pipeline)
+    - Qualified: "pipeline_name.table_name" (references table in another pipeline)
+    """
+
+    pipeline: str | None = Field(
+        default=None,
+        description="Pipeline name (None = same pipeline as referencing table)",
+    )
+    table: str = Field(description="Table name within pipeline")
+
+    @classmethod
+    def parse(cls, ref: str) -> "TableReference":
+        """Parse table reference string.
+
+        Args:
+            ref: Table reference string ("table" or "pipeline.table")
+
+        Returns:
+            Parsed TableReference
+
+        Examples:
+            >>> TableReference.parse("outreach_list")
+            TableReference(pipeline=None, table='outreach_list')
+            >>> TableReference.parse("reference_data.icd_codes")
+            TableReference(pipeline='reference_data', table='icd_codes')
+        """
+        if "." in ref:
+            parts = ref.split(".", 1)
+            return cls(pipeline=parts[0], table=parts[1])
+        return cls(pipeline=None, table=ref)
+
+    def is_external(self) -> bool:
+        """Check if this references an external pipeline."""
+        return self.pipeline is not None
+
+    def to_qualified_name(self, current_pipeline: str | None = None) -> str:
+        """Convert to qualified name string.
+
+        Args:
+            current_pipeline: Optional current pipeline name (for resolving bare refs)
+
+        Returns:
+            Qualified name ("pipeline.table") when a pipeline is known, else bare name
+        """
+        target_pipeline = self.pipeline or current_pipeline
+        if target_pipeline:
+            return f"{target_pipeline}.{self.table}"
+        return self.table
+
+    def __str__(self) -> str:
+        """String representation (qualified or bare)."""
+        if self.pipeline:
+            return f"{self.pipeline}.{self.table}"
+        return self.table
+
+
+__all__ = ["PipelineDependency", "PipelineMetadata", "TableReference"]