feat: product completeness loop — KPIs, bootstrap, JDBC, e2e gate, FR-23 smoke

apps/data-profiling/profiler/smoke.py

@@ -0,0 +1,37 @@
+"""FR-23 agent-side smoke: resolve → provision → validate_config.
+
+No live workspace required when ``PROFILER_RUNTIME=mock`` and a fake SQL
+executor is supplied. Live deploy remains operational residual.
+"""
+
+from __future__ import annotations
+
+from typing import Mapping, Optional
+
+from profiler.config import AppConfig, resolve_config
+from profiler.diagnostics import validate_config
+from profiler.provision import SqlExecutor, provision
+
+
+def run_fr23_smoke(
+    *,
+    env: Optional[Mapping[str, str]] = None,
+    registry_path: str = "connections.yaml",
+    executor: Optional[SqlExecutor] = None,
+    grant_to: Optional[str] = None,
+) -> int:
+    """Return 0 if composition succeeds, 1 if startup faults on a non-mock runtime.
+
+    Always runs provision when an executor is provided. On mock runtime,
+    validate_config returns no faults (nothing to probe).
+    """
+    cfg: AppConfig = resolve_config(env=env, registry_path=registry_path)
+    if executor is not None:
+        provision(cfg, executor=executor, grant_to=grant_to)
+    faults = validate_config(cfg)
+    if faults and cfg.is_databricks:
+        for f in faults:
+            print(f.message())
+        return 1
+    print(f"FR-23 smoke OK: metadata home {cfg.describe()}")
+    return 0

apps/data-profiling/scripts/fr23_smoke.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+"""CLI smoke for FR-23 (unit path, no workspace required with mock runtime).
+
+    cd apps/data-profiling
+    PROFILER_RUNTIME=mock python scripts/fr23_smoke.py
+
+Exit 0 on success. For empty-environment provision, pass --provision with a
+Databricks warehouse (not the mock path).
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from profiler.smoke import run_fr23_smoke  # noqa: E402
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument(
+        "--registry",
+        default="connections.yaml",
+        help="Connection registry path (default connections.yaml)",
+    )
+    p.add_argument(
+        "--provision",
+        action="store_true",
+        help="Run provision against a live warehouse (requires DATABRICKS_*)",
+    )
+    args = p.parse_args()
+    executor = None
+    if args.provision:
+        from profiler.provision import DatabricksExecutor
+
+        executor = DatabricksExecutor()
+    return run_fr23_smoke(
+        env=os.environ,
+        registry_path=args.registry,
+        executor=executor,
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

apps/data-profiling/tests/test_fr23_stack.py

@@ -51,13 +51,30 @@ def test_fr23_resolve_provision_validate_compose(tmp_path: Path) -> None:
     assert cfg.metadata_catalog == "stack_cat"
     assert cfg.source_of("metadata_catalog") == "deployment"
 
-    report = provision(cfg, executor=_EmptyExecutor(), grant_to=None)
+    empty = _EmptyExecutor()
+    report = provision(cfg, executor=empty, grant_to=None)
     assert isinstance(report, ProvisionReport)
     assert report.changed is True
+    assert empty.statements  # DDL issued on empty environment
 
-    # Second provision against a fake that reports everything already present
-    # is covered in test_provision; here we only require composition succeeds.
+    # Smoke: mock runtime validation is list-shaped and non-raising.
     faults = validate_config(cfg)
-    # mock runtime without a live warehouse probe should not raise; faults may
-    # be empty or advisory depending on runtime probes — must be a list.
     assert isinstance(faults, list)
+    assert faults == []  # mock runtime skips warehouse probes
+
+
+def test_fr23_smoke_exit_zero_composition(tmp_path: Path) -> None:
+    """CLI-equivalent smoke: resolve + provision + validate returns success."""
+    from profiler.smoke import run_fr23_smoke
+
+    code = run_fr23_smoke(
+        env={
+            "PROFILER_METADATA_CATALOG": "smoke_cat",
+            "PROFILER_METADATA_SCHEMA": "smoke_schema",
+            "PROFILER_OUTPUT_VOLUME": "smoke_vol",
+            "PROFILER_RUNTIME": "mock",
+        },
+        registry_path=str(tmp_path / "missing.yaml"),
+        executor=_EmptyExecutor(),
+    )
+    assert code == 0

docs/guide/bootstrap.md

@@ -14,6 +14,8 @@ relationships, aliases, keys, raw-to-ingest SQL, validation suites, and manifest
 entries. Silver-layer work such as cross-source conformance, survivorship, entity
 resolution, enrichment, and dimensional modeling remains downstream.
 
+## Path A — from existing Spark tables
+
 ```python
 from tablespec import bootstrap_from_tables
 
@@ -29,7 +31,7 @@ print(artifacts.manifest_path)
 print(artifacts.table("member").suite_json)
 ```
 
-What the facade does:
+What Path A does:
 
 - reflects each table schema into UMF
 - when `profile=True`, profiles the table data natively and turns the profile
@@ -50,3 +52,33 @@ and friends), and the rest of the pinned tree that a production job installs and
 loads from disk.
 
 When you only want the schema-only baseline suite, pass `profile=False`.
+
+## Path B — from authored UMF specs (no Spark)
+
+```python
+from tablespec import bootstrap_from_specs
+
+artifacts = bootstrap_from_specs(
+    [
+        "tests/e2e/fixtures/member.umf.yaml",
+        "tests/e2e/fixtures/claims.umf.yaml",
+        "tests/e2e/fixtures/claim_enriched.umf.yaml",
+    ],
+    out_dir="/tmp/tablespec-bootstrap-specs",
+    dialect="duckdb",
+    gold_targets=["claim_enriched"],
+)
+
+print(artifacts.manifest_path)
+```
+
+Path B loads specs via `umfs_from_specs` and compiles the same artifact tree.
+Use it for CI, local DuckDB/Spark-less compile checks, and the onboarding
+benchmark (`docs/guide/onboarding-benchmark.md`).
+
+Demo scripts (also drive e2e tests):
+
+```bash
+uv run python scripts/bootstrap_from_specs.py --spec <a.yaml> --out <dir>
+uv run python scripts/bootstrap_from_tables.py ...   # requires Spark
+```

docs/guide/databricks-e2e.md

@@ -0,0 +1,47 @@
+# Databricks serverless e2e (opt-in)
+
+Vision KPI: multi-engine parity including Databricks serverless.
+
+Default CI and local `make test` **never** require a workspace. The real
+serverless lane is an **opt-in** marker:
+
+```bash
+# Without credentials: tests marked databricks_e2e SKIP with a precise reason
+uv run pytest -m databricks_e2e -q
+
+# With a configured workspace (export secrets first):
+export DATABRICKS_HOST=https://<workspace>
+export DATABRICKS_HTTP_PATH=/sql/1.0/warehouses/<id>
+export DATABRICKS_TOKEN=<pat>
+# plus dbt-databricks + databricks-sdk + databricks-sql-connector installed
+uv run pytest -m databricks_e2e -q
+```
+
+## Gate
+
+`tablespec.e2e.gating.databricks_e2e_availability()` returns:
+
+- a **skip reason string** when the tier must not run, or
+- `None` when credentials + adapters look complete
+
+Required env (all three):
+
+| Variable | Role |
+|----------|------|
+| `DATABRICKS_HOST` | Opt-in switch |
+| `DATABRICKS_HTTP_PATH` | SQL warehouse HTTP path |
+| `DATABRICKS_TOKEN` | PAT / token |
+
+Unit gate (no workspace):
+
+```bash
+uv run pytest tests/unit/test_databricks_e2e_gate.py -q
+```
+
+## What the tier proves
+
+When green against a real workspace: dbt/LDP deploy+run and read-back rows
+match the Spark-oracle corpus through the shared canonicalizer
+(`docs/helix/03-test/conformance-acceptance.md` §2.3).
+
+When skipped: **not a silent pass** — the skip reason names the missing piece.

docs/guide/happy-path.md

@@ -30,11 +30,12 @@ Current boundary notes:
 - Databricks-facing compile UX accepts `dialect="databricks"` for the
   Spark-family SQL emitted by tablespec; internal emitters may normalize to
   `spark` when the rendered SQL is identical.
-  The active bead trail is `tablespec-ed74497c` and child beads
-  `tablespec-0b146671`, `tablespec-171e409c`, and `tablespec-0fb0d1c2`.
 - Production runs consume the committed artifact tree and installed packages,
   not source-time orchestration. See the deployment checklist for the release
   boundary.
+- One-shot facades: `bootstrap_from_tables` (Path A, Spark) and
+  `bootstrap_from_specs` (Path B, no Spark) are public first-class entry
+  points — see [bootstrap.md](bootstrap.md).
 
 ## 1. Generate UMF from existing Spark or Databricks tables
 
@@ -63,17 +64,23 @@ If you do not need the intermediate `UMF` list, the convenience wrapper
 `bootstrap_from_tables(...)` reflects, profiles, compiles, and returns the
 manifest in one call.
 
-For authored specs, use the Path B loader instead:
+For authored specs, use the Path B loader or one-shot facade:
 
 ```python
 from tablespec.e2e import umfs_from_specs
+from tablespec import bootstrap_from_specs
 
 umfs = umfs_from_specs([
     "tables/member/table.yaml",
     "tables/claims/table.yaml",
 ])
+# or compile in one call (no Spark):
+# artifacts = bootstrap_from_specs([...], out_dir="...")
 ```
 
+JDBC databases use the same compile contract after discovery — see
+[jdbc-onboarding.md](jdbc-onboarding.md) and the Northwind demo/tests.
+
 ## 2. Generate sample data from the UMF or spec inputs
 
 Sample data is generated from UMF/spec inputs, not from the compiled artifact

docs/guide/jdbc-onboarding.md

@@ -0,0 +1,63 @@
+# JDBC onboarding (first-class source path)
+
+JDBC is a first-class FEAT-031 source kind: the same UMF → compile → runtime
+contract as delimited, with **typed raw** landing via Spark's JDBC connector
+(tablespec never opens a DB connection itself).
+
+## Composition
+
+```python
+from tablespec.profiling import JdbcToUmfMapper  # requires tablespec[spark]
+from tablespec import bootstrap_from_specs  # or compile_umfs after discovery
+from tablespec.ingestion import get_reader
+
+# 1) Discover UMF from INFORMATION_SCHEMA (Spark JDBC)
+mapper = JdbcToUmfMapper(spark, jdbc_url=..., options={...})
+umfs = mapper.discover(["dbo.customers", "dbo.orders"])  # shape may vary by mapper API
+
+# 2) Compile (no live DB required once UMF is in hand)
+from tablespec.e2e.compile import compile_umfs
+artifacts = compile_umfs(umfs, "/tmp/jdbc-out", source="tables", dialect="spark")
+
+# 3) Runtime land: Spark JDBC reader from the committed source: block
+reader = get_reader(umf.effective_source())
+df = reader.read(umf.effective_source(), spark)
+```
+
+### Public Path B when specs are already authored
+
+If UMFs already declare `source: {kind: jdbc, ...}`:
+
+```python
+from tablespec import bootstrap_from_specs
+
+artifacts = bootstrap_from_specs(
+    ["specs/customers", "specs/orders"],
+    out_dir="/tmp/jdbc-compile",
+    dialect="spark",
+)
+```
+
+`password_secret_ref` (or env-style secret name) is required; plaintext
+passwords are rejected by the model.
+
+## Demo / acceptance
+
+| Lane | What | Gate |
+|------|------|------|
+| Local | Docker SQL Server + Northwind fixture | `tests/integration/test_jdbc_discovery.py`, `test_northwind_e2e.py` (skip without Docker) |
+| Workspace | `notebooks/northwind-demo/` | Databricks job (operator residual) |
+
+## Backbone note
+
+The e2e **backbone** file loader supports `delimited` / `parquet` / `json`
+batch files for local multi-engine parity. JDBC tables are landed at runtime
+with `JdbcReader` + Spark — not by reading a CSV batch. That is intentional:
+JDBC onboarding is proven by discovery + compile + integration/Northwind, not
+by the CSV-shaped backbone corpus.
+
+## Secret safety
+
+- Never put credentials in UMF or compiled artifacts.
+- Use `password_secret_ref` naming a Databricks secret scope or env var.
+- Discovery and readers fail closed when the secret is missing.

docs/guide/onboarding-benchmark.md

@@ -0,0 +1,56 @@
+# Onboarding authoring-reduction benchmark
+
+PRD success metric: **at least 50% lower** transform/validation authoring time
+per onboarded table vs hand-authored baseline, measured on a **3-table**
+onboarding sample.
+
+## Automated sample (tablespec path)
+
+The in-repo sample is the Path B e2e fixture set:
+
+| Table | Spec |
+|-------|------|
+| `member` | `tests/e2e/fixtures/member.umf.yaml` |
+| `claims` | `tests/e2e/fixtures/claims.umf.yaml` |
+| `claim_enriched` | `tests/e2e/fixtures/claim_enriched.umf.yaml` |
+
+### Run
+
+```bash
+uv run python scripts/onboarding_benchmark.py --out /tmp/onboard-metrics
+```
+
+This records:
+
+- wall time for `umfs_from_specs` + `compile_umfs` (and optionally backbone)
+- per-table artifact presence (ingest SQL, DDL, suite, dbt ingest, …)
+- gold DAG / LDP project presence
+
+Output: `/tmp/onboard-metrics/onboarding_benchmark.json`.
+
+Unit gate (no Spark required):
+
+```bash
+uv run pytest tests/unit/test_onboarding_benchmark.py -q
+```
+
+## Manual baseline protocol
+
+To compute reduction, time a **manual** 3-table onboarding of the same
+semantics (member, claims, claim_enriched) *without* tablespec compile:
+
+1. Hand-write Spark DDL / dbt models / GX suite / LDP stubs for all three.
+2. Record elapsed wall-clock minutes \(t_{manual}\).
+3. Run the automated harness above; use `seconds.total_automated` as
+   \(t_{tablespec}\).
+4. Reduction = \(1 - t_{tablespec}/(t_{manual}\times 60)\).
+
+The automated harness is the **reproducible numerator**. The denominator is
+operator-measured once per release and stored under
+`docs/helix/06-iterate/metrics/` if desired (optional).
+
+## Relation to the happy path
+
+This sample is the same Path B composition as [happy-path.md](happy-path.md)
+and `scripts/bootstrap_from_specs.py`, scoped for metric capture rather than
+demo narration.