diff --git a/.ddx/beads.jsonl b/.ddx/beads.jsonl index 2dad982..a2d00c1 100644 --- a/.ddx/beads.jsonl +++ b/.ddx/beads.jsonl @@ -21,6 +21,7 @@ {"acceptance":"The dimension is either specified with SLAs or explicitly marked N/A with a reason; the self-attesting checklist line matches reality.","created_at":"2026-06-11T00:24:15.923612309Z","description":"Spec audit 2026-06-10: docs/helix/03-test/data-quality-expectations.md drops the template's Timeliness/Freshness dimension (Bronze freshness, Silver staleness SLAs) with no rationale, while its checklist self-attests 'SLA per layer'. Decide: add staleness expectations (the compiler could emit them) or record an explicit N/A-with-reason for a compile-time library.","id":"tablespec-01161962","issue_type":"task","labels":["helix","spec","decision"],"priority":4,"resolution":"DECIDED 2026-06-10 (operator): N/A-with-reason. Timeliness/Freshness row added to data-quality-expectations.md marked N/A (compile-time library); checklist line corrected; compiled freshness checks noted as future scope requiring a PRD requirement.","schema_version":1,"status":"closed","title":"DECISION: timeliness/freshness dimension in data-quality-expectations.md","updated_at":"2026-06-11T00:56:03.032101527Z"} {"acceptance":"tests/integration/test_northwind_e2e.py AC4 asserts source: and foreign_keys survive export-\u003eimport; existing excel tests unchanged.","created_at":"2026-06-11T13:55:45.647848727Z","description":"US-039 AC4 follow-up (2026-06-11): the Excel converter round-trips identity/order/types/length/precision/nullability/PK, but the source: block has no sheet and relationships.foreign_keys do not survive re-import (Relationships sheet only writes relationships.outgoing). Extend the workbook surface so discovered specs round-trip without loss.","id":"tablespec-036d3e9d","issue_type":"task","labels":["area:excel","feat-031","kind:feature"],"owner":"erik","priority":3,"schema_version":1,"status":"closed","title":"Excel round-trip: carry source: block and discovered foreign_keys","updated_at":"2026-06-12T16:53:20.498790999Z"} {"acceptance":"1. Child bead exists for full Excel export/import validity and represented-field round-trip tests.\n2. Child bead exists for steward-facing workbook guidance, helper/status behavior, stale command cleanup, and advisory sheet labeling.\n3. Child bead exists for auditing and closing advanced UMF field fidelity gaps in the workbook contract.\n4. Each child bead has named tests and verification commands.","created_at":"2026-06-08T04:42:45.443739645Z","description":"Parent epic for bringing the Excel representation of tablespecs into alignment with FEAT-009 and US-011. Review found that the specs were underspecified for the expected consumer persona and that implementation has drifted from the intended round-trip contract.\n\nContext now captured in specs:\n- Primary Excel consumer is a domain data steward / SME who reviews table specifications in Excel without needing YAML, JSON, Pydantic, or Great Expectations internals.\n- Data engineers own export/import, validation, and resolving technical errors.\n- Excel is a review/editing surface, not a production runtime contract. Split UMF remains canonical editable source; JSON/compiled artifacts remain machine/runtime contracts.\n- Workbooks are macro-free .xlsx files with README, Schema, Columns, _Instructions, Metadata, and optional Validation Rules, Relationships, File Format, and Survivorship sheets.\n- Editable versus generated/advisory fields must be visible.\n\nImplementation gaps found during review:\n1. Exported workbooks currently fail full import because ExcelToUMFConverter normalizes data types to Spark names such as IntegerType/StringType before UMF.model_validate(), while UMF requires INTEGER/VARCHAR.\n2. Workbook README uses stale convert-from-excel commands and the code docstring mentions VBA macros despite generating macro-free .xlsx.\n3. Helper/status and advisory/read-only boundaries are underspecified in the workbook experience.\n4. Full fidelity is not proven for represented advanced UMF fields such as unified expectations, unique constraints, indexes, structured metadata, relationships, derivation/survivorship, and file format.\n\nDo not implement this parent directly; execute the child beads.","events":[{"actor":"","body":"closed_because: all_children_terminal\ntotal_children: 1","created_at":"2026-06-08T15:18:53.977164732Z","kind":"epic_auto_close","source":"Store.Close.walkUpClosureCandidate","summary":"auto-closed: all children reached terminal state"}],"id":"tablespec-03cab8b1","issue_type":"epic","labels":["area:excel","area:docs","kind:feature"],"priority":1,"schema_version":1,"spec-id":"FEAT-009","status":"closed","title":"Align Excel workbook contract with domain-steward workflow","updated_at":"2026-06-08T15:18:53.981777711Z"} +{"acceptance":"1. Gate unit test. 2. docs/guide or helix deploy note for opt-in. 3. pytest -m databricks_e2e skips cleanly without credentials.","created_at":"2026-07-23T02:37:09.907174431Z","description":"Agent-side residual for serverless parity KPI: ensure databricks_e2e availability gate, skip reasons, and harness entrypoints are coherent and documented; add a dry-run or gate unit test that does not need credentials; document how CI opts in. Do not require a live workspace to close.","events_attachment":"tablespec-05a80589/events.jsonl","id":"tablespec-05a80589","issue_type":"task","labels":["helix","area:databricks","e2e"],"owner":"erik","parent":"tablespec-cc2cfafb","priority":2,"schema_version":1,"session_id":"product-completeness","status":"closed","title":"Databricks e2e opt-in residual: fail-closed gate + harness docs","updated_at":"2026-07-23T02:40:17.532925023Z"} {"acceptance":"US-039 ACs evidenced with command and exit status, or residual waived with recorded limitation.","created_at":"2026-07-22T21:41:08.254312981Z","description":"Close remaining US-039 ACs on real Databricks if still open beyond Docker lane.","id":"tablespec-0aa41072","issue_type":"task","labels":["helix","phase:test","area:data","FEAT-031"],"notes":"CLOSED 2026-07-22 queue surgery: US-039-AC1..6 all [x]; @covers in tests/integration/test_northwind_e2e.py; story status records Docker+workspace PASS. No residual.","parent":"tablespec-263a0248","priority":2,"schema_version":1,"status":"closed","title":"Northwind Databricks acceptance residual (US-039)","updated_at":"2026-07-22T21:47:08.923313908Z"} {"acceptance":"1. TestDatabricksDialectAliasAtCastLayer: in tests/unit/test_casting_utils.py, assert cast_column_sql(..., dialect=\"databricks\") emits byte-identical SQL to dialect=\"spark\" for INTEGER, DECIMAL, DATE with format, TIMESTAMP without format, BOOLEAN, EPOCH_MS_FORMAT, and EXCEL_SERIAL_FORMAT.\n2. TestBuildIngestSelectAcceptsDatabricksAlias: in tests/unit/test_ingest_generator.py, assert build_ingest_select(..., dialect=\"databricks\") succeeds, stores the accepted public dialect or documented normalized value, and emits the same Spark-family select_block as dialect=\"spark\".\n3. TestSharedDialectErrorMessages: add tests covering cast_column_sql, build_ingest_select, and dbt contract/profile helpers so unsupported values all report the same accepted values: spark, databricks, duckdb where the field is a cast dialect, and duckdb, spark, databricks where the field is a dbt profile target.\n4. Test gates: uv run pytest tests/unit/test_casting_utils.py tests/unit/test_ingest_generator.py tests/dbt_roadmap/test_contracts_functional.py; go test ./...; lefthook run pre-commit.","closing_commit_sha":"44fe34dc8ce6ed6a0a14a2db1258a9312e43af61","created_at":"2026-06-08T01:04:03.676003826Z","description":"PROBLEM: Databricks is advertised as a dialect in parts of the repo, but accepted dialect values and error messages are owned independently by casting, ingest, dbt contracts, and dbt profile code. That leaves agents and users vulnerable to stale or contradictory validators.\nROOT CAUSE: src/tablespec/casting_utils.py:627 accepts ('spark','databricks','duckdb') directly, src/tablespec/schemas/ingest_generator.py:90 and src/tablespec/schemas/ingest_generator.py:148 still document only spark/duckdb, src/tablespec/schemas/ingest_generator.py:162 and src/tablespec/schemas/ingest_generator.py:164 pass the raw value through without a shared normalizer, src/tablespec/dbt/contracts.py:88 has its own dialect guard, and src/tablespec/dbt/profiles.py:44 has a separate target guard.\nPROPOSED FIX: Add or use a shared dialect helper that defines accepted cast dialects, normalizes databricks to the Spark-family rendering path, and formats accepted-value error messages. Wire casting_utils, ingest_generator, and dbt contract/profile validation through it while preserving databricks as an explicit public spelling.\nNON-SCOPE: One-shot bootstrap facade implementation. Production pipeline JSON/wheel packaging contract. Inline UMF format cleanup. No change to cast SQL semantics beyond alias normalization and error text consistency.\nGOVERNING ARTIFACTS: docs/helix/02-design/adr/ADR-007-raw-to-ingest-sql-artifact.md; docs/helix/03-test/conformance-acceptance.md:109; .agents/skills/tablespec/SKILL.md Dialects section.\nDEPENDENCIES: none inherited from parent.\nVERIFICATION: Add focused unit tests and run uv/pytest, go test, and lefthook gates.","events_attachment":"tablespec-0b146671/events.jsonl","id":"tablespec-0b146671","issue_type":"task","labels":["area:databricks","area:dialects","area:docs","kind:bug","ac-quality:needs-refinement"],"parent":"tablespec-ed74497c","priority":0,"schema_version":1,"session_id":"eb-96f2578b","status":"closed","title":"Normalize Databricks as a shared Spark-family cast dialect alias","updated_at":"2026-06-08T06:46:35.676819963Z"} {"acceptance":"1. TestCliHelpExplainsDatabricksAlias: add a CLI help regression that verifies `--dialect` advertises `duckdb`, `spark`, and `databricks`, and explains that `databricks` is the Databricks-facing alias for Spark-family cast SQL.\\n2. TestCliValidationKeepsCanonicalValues: add a parser/validation regression that the accepted values remain stable and the help text does not imply a separate Databricks SQL dialect implementation.\\n3. go test ./...\\n4. lefthook run pre-commit.","closing_commit_sha":"4a0ab6555054ed52cb219761004efd1e60a7c47f","created_at":"2026-06-08T14:55:59.620585689Z","description":"PROBLEM: The CLI help text does not explain that Databricks uses the Spark-family dialect alias, so users cannot tell whether `databricks` is accepted or how it relates to `spark`. ROOT CAUSE: `src/tablespec/cli.py:525` lists `duckdb, spark, databricks` without describing the Spark-family normalization or alias semantics. PROPOSED FIX: Update the `--dialect` help/validation text to name the three accepted values, explicitly state that `databricks` is the Databricks-facing alias for Spark-family cast SQL, and keep the runtime behavior unchanged. NON-SCOPE: One-shot bootstrap facade implementation. Production pipeline JSON/wheel packaging contract. Inline UMF format cleanup. No broad documentation rewrite outside Databricks dialect guidance. GOVERNING ARTIFACTS: `docs/guide/bootstrap.md:10`; `README.md:139`; `docs/helix/03-test/conformance-acceptance.md:109`; `scripts/run_integration_tests_databricks.ipynb`. DEPENDENCIES: `tablespec-ed74497c`.","events_attachment":"tablespec-0d64cd5d/events.jsonl","id":"tablespec-0d64cd5d","issue_type":"task","labels":["area:cli","area:databricks","area:dialects","kind:bug","ac-quality:needs-refinement"],"parent":"tablespec-0fb0d1c2","priority":0,"schema_version":1,"session_id":"eb-ec05ad24","status":"closed","title":"Clarify Databricks alias in CLI dialect help","updated_at":"2026-06-08T19:24:05.557906118Z"} @@ -33,6 +34,7 @@ {"acceptance":"US-040 file exists with stable US-040-ACn IDs; each AC cites an existing test path or is marked covered by named test; FEAT-031 user-stories section links US-040; no new product code required unless a citation gap forces a tiny @covers add.","created_at":"2026-07-22T21:41:07.514181186Z","description":"Frame-only backfill. SRC seam and multi-kind source model are SHIPPED (models/umf.py + ingestion/). Author US-040 with US-040-ACn Given/When/Then mapped to existing unit tests (test_source_spec, test_ingestion_package). Do NOT re-implement the seam.","events_attachment":"tablespec-20513f4f/events.jsonl","id":"tablespec-20513f4f","issue_type":"task","labels":["helix","phase:frame","area:data","FEAT-031"],"notes":"Queue surgery 2026-07-22: re-scoped from build to story floor.","owner":"erik","parent":"tablespec-263a0248","priority":1,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"Author US-040: source model + ingestion seam AC backfill (story floor only)","updated_at":"2026-07-22T22:07:10.349061386Z"} {"acceptance":"B1 app FR-23 beads complete or actively blocked only on true deps; JSON residual closed or waived with evidence; story-floor beads closed; tests/docs green; no reopened closed-as-shipped beads without new failing evidence.","created_at":"2026-07-22T21:40:40.237053896Z","description":"Close code/ops gaps against governing HELIX specs after AR-2026-07-22. Specs describe desired future state. Queue surgery 2026-07-22 pruned false residuals (DUMP/PARQ/US-039/US-041/US-038/Trace Links/CORP example). Remaining primary build: B1 app FR-23 path + JSON backbone residual + story floor US-040/042/043/050 + US-045 residual evidence + US-046 @covers.","events":[{"actor":"","body":"closed_because: all_children_terminal\ntotal_children: 21","created_at":"2026-07-22T22:07:16.402671078Z","kind":"epic_auto_close","source":"Store.Close.walkUpClosureCandidate","summary":"auto-closed: all children reached terminal state"}],"id":"tablespec-263a0248","issue_type":"epic","labels":["helix","phase:align","area:specs"],"notes":"Queue surgery 2026-07-22: pruned 7 children as shipped/duplicate/cosmetic.","priority":1,"schema_version":1,"status":"closed","title":"HELIX desired-state alignment — implementation gaps","updated_at":"2026-07-22T22:07:16.406583784Z"} {"acceptance":"1. test_native_profiler_single_column_exact_key_candidate: a fixture with \u003e=100 rows, or an explicit key_min_rows override, produces exactly one verified_exact primary_key_candidate for non-null unique id, with every documented evidence field present and score in [0, 1].\n2. test_native_profiler_nullable_unique_is_not_pk: a nullable unique column produces no primary_key_candidate; it is represented as unique_constraint_candidate or advisory evidence with nullable=true.\n3. test_native_profiler_composite_minimal_key: member_id + effective_date exact uniqueness yields a composite candidate only when neither subset is exact unique.\n4. test_native_profiler_rejects_nonminimal_composite: if id is exact unique, id + date is rejected or marked nonminimal and is not emitted as a candidate.\n5. test_native_profiler_respects_candidate_and_pass_budget: a wide-table fixture exact-verifies no more than key_verification_pass_budget candidates and leaves skipped candidates verified_exact=false without suppressing legacy uniqueness expectations.\n6. test_profile_to_gx_no_duplicate_uniqueness_when_infer_keys_enabled: a verified exact single-column candidate prevents a duplicate approximate uniqueness expectation for that same column.\n7. test_profile_to_gx_legacy_uniqueness_unchanged_when_infer_keys_disabled: existing approximate uniqueness behavior remains byte/structurally unchanged when inference is disabled.\n8. Connect/classic parity test asserts serialized key_candidates structural equality on deterministic fixtures.\n9. uv run pytest tests/unit/test_native_profiler_key_candidates.py tests/unit/test_profiler_connect_sail.py tests/unit -k gx_expectation passes.","created_at":"2026-06-08T04:19:09.628002551Z","description":"PROBLEM: NativeSparkProfiler does not infer primary-key or unique-key candidates. ProfileToGxMapper currently emits approximate single-column uniqueness expectations from approximate distinct ratios, but that is not primary-key detection and can duplicate future inferred-key expectations. Users expect bootstrap/profile enrichment to surface likely keys while keeping authoritative UMF.primary_key and unique_constraints separate.\n\nROOT CAUSE: src/tablespec/profiling/types.py DataFrameProfile has only num_records and columns. src/tablespec/profiling/native_profiler.py computes per-column statistics but no key candidate model, scoring, exact verification, or minimality checks. src/tablespec/profiling/gx_expectation_builder.py has a separate approximate uniqueness threshold and no concept of verified key candidates.\n\nPROPOSED FIX: Add typed, JSON-serializable KeyCandidate and KeyCandidateEvidence models to the profiling layer. Add optional key-candidate inference to NativeSparkProfiler with defaults that preserve existing behavior unless enabled. Candidate inference must be advisory: it returns key_candidates on DataFrameProfile but does not mutate UMF. Inference must use cheap existing profile signals first, deterministically score/truncate candidates, then exact-verify only the bounded set with Connect-safe Spark-SQL/DataFrame operations.\n\nRequired behavior:\n- Candidate kinds: primary_key_candidate and unique_constraint_candidate.\n- Evidence fields: row_count, columns, null_count_by_column, exact_distinct_count, approximate_distinct_count_by_column, distinct_ratio, completeness_by_column, verified_exact, nullable, minimal, subset_unique, score, score_components, name_hints, type_hints, penalties, verification_pass_count, verification_query_count, reason.\n- Scoring is deterministic, score is in [0, 1], and exposed thresholds include key_promotion_min_score and key_promotion_min_gap for downstream use.\n- Ordering is total and stable: score descending, key width ascending, then column-name tuple ascending.\n- key_min_rows default prevents noisy inference on tiny tables; tests must either use at least 100 rows or override key_min_rows explicitly and assert candidates are non-empty.\n- Minimality is honest: minimal=True only when all proper subsets were exact-verified as non-unique; otherwise use false/unknown semantics in the evidence rather than guessing.\n- Composite search is bounded by key_max_width and key_max_candidates; no exhaustive all-column combinatorics.\n- Exact verification must not use RDDs or classic-only SparkContext APIs.\n- ProfileToGxMapper deduplicates legacy approximate uniqueness only for verified_exact=True single-column candidates covering the same column. Unverified or budget-skipped candidates must not suppress existing approximate uniqueness behavior.\n\nIn-scope files:\n- src/tablespec/profiling/types.py\n- src/tablespec/profiling/native_profiler.py\n- src/tablespec/profiling/gx_expectation_builder.py\n- tests/unit/test_native_profiler_key_candidates.py or equivalent focused tests\n- tests/unit/test_profiler_connect_sail.py if parity coverage belongs there\n\nOut-of-scope for this bead:\n- Writing sidecar files or extending manifest layout; tracked by FEAT-026 child.\n- Mutating UMF.primary_key or unique_constraints.\n- Making sample data, dbt, or LDP consume inferred candidates.\n- Implementing compound uniqueness execution in the validator; tracked by FEAT-007 child.","events_attachment":"tablespec-29002b40/events.jsonl","execution-eligible":false,"id":"tablespec-29002b40","issue_type":"task","labels":["area:profiling","area:validation","kind:feature","ac-quality:needs-refinement"],"parent":"tablespec-cc4f665b","priority":1,"schema_version":1,"spec-id":"FEAT-024","status":"closed","title":"Infer advisory key candidates in NativeSparkProfiler","updated_at":"2026-06-09T20:32:39.036602248Z"} +{"acceptance":"1. Smoke exits 0 with fakes. 2. deployment-checklist lists smoke command. 3. make test / pytest green.","created_at":"2026-07-23T02:37:09.997300175Z","description":"Close FR-23 product loop agent-side: smoke script or pytest that exercises resolve→provision→validate_config end-to-end with fakes; expand deployment-checklist app_deploy with recorded expected commands/exit codes. Live workspace deploy remains out of band.","events_attachment":"tablespec-292a6a59/events.jsonl","id":"tablespec-292a6a59","issue_type":"task","labels":["helix","area:app","FEAT-034"],"owner":"erik","parent":"tablespec-cc2cfafb","priority":1,"schema_version":1,"session_id":"product-completeness","status":"closed","title":"FR-23 app smoke: provision+startup composition + deploy checklist evidence","updated_at":"2026-07-23T02:41:13.011213343Z"} {"acceptance":"Unit tests cover each precedence tier; no production path reads ad-hoc env keys outside the resolver.","closing_commit_sha":"6857ad0da3e8839e3fb40c6fe806ee7f4060e562","created_at":"2026-07-22T21:41:06.841434626Z","description":"Implement one configuration object for apps/data-profiling resolving deployment env vars then connections.yaml then built-in defaults per ADR-019.","events_attachment":"tablespec-2a68a0ad/events.jsonl","id":"tablespec-2a68a0ad","issue_type":"task","labels":["helix","phase:build","area:app","FEAT-034"],"notes":"[2026-07-22T22:02:13Z] closure rejected: closure gate: insufficient evidence: no execution evidence (empty events and no closing_commit_sha)","owner":"erik","parent":"tablespec-263a0248","priority":1,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"App: single config precedence resolver (FR-23.1 / CFG-01)","updated_at":"2026-07-22T22:02:48.429677103Z"} {"acceptance":"1. test_bootstrap_candidates_write_sidecar_without_umf_mutation: infer_keys=\"candidates\" writes validation/\u003ctable\u003e.keycandidates.json, records the sidecar in manifest/provenance with a relative path, and the persisted UMF snapshot has no inferred primary_key or unique_constraints mutation.\n2. test_keycandidate_sidecar_json_schema: sidecar evidence validates against the documented JSON schema and contains deterministic sorted candidates.\n3. test_bootstrap_auto_conflict_does_not_promote: two exact plausible verified candidates whose scores tie or fail key_promotion_min_gap produce sidecar candidates but do not mutate UMF.primary_key.\n4. test_bootstrap_auto_single_clear_key_promotes_with_metadata: one exact, non-null, minimal candidate with score \u003e= key_promotion_min_score promotes only in infer_keys=\"auto\" and records promotion evidence plus downstream-effect metadata.\n5. test_candidate_sidecar_deterministic_across_runs: two runs over identical input produce byte-identical validation/\u003ctable\u003e.keycandidates.json and stable manifest entries.\n6. test_candidate_mode_does_not_change_generated_runtime_artifacts: candidate mode leaves sample/dbt/LDP-affecting UMF fields unchanged, so generated runtime artifacts match infer_keys=\"none\" except for the sidecar and manifest/provenance entries.\n7. uv run pytest tests/e2e/test_bootstrap_key_candidates.py tests/e2e/test_bootstrap_from_tables.py tests/e2e/test_bootstrap_from_specs.py passes.","closing_commit_sha":"409f4f1442747a2c0b0cd78a64c803e4be4553f3","created_at":"2026-06-08T04:19:32.437923415Z","dependencies":[{"issue_id":"tablespec-2bd32b44","depends_on_id":"tablespec-29002b40","type":"blocks","created_at":"2026-06-08T04:19:32Z"}],"description":"PROBLEM: Once profiler key candidates exist, bootstrap/compile needs a deterministic artifact contract for them. Candidate mode must be useful to users and agents without silently changing UMF snapshots, sample data, dbt, or LDP behavior. Auto-promotion must be explicit because writing UMF.primary_key changes downstream behavior.\n\nROOT CAUSE: FEAT-026 compile currently persists UMF snapshots, validation suites, runtime artifacts, and manifest.json. It has no validation/\u003ctable\u003e.keycandidates.json sidecar, no manifest slot for advisory key evidence, no bootstrap infer_keys mode, and no provenance for any inferred-key promotion.\n\nPROPOSED FIX: Extend the public bootstrap/compile path to support infer_keys modes: none, candidates, auto. Candidate mode writes advisory sidecar artifacts under validation/\u003ctable\u003e.keycandidates.json and records them in manifest/provenance without mutating persisted UMF snapshots. Auto mode may promote to UMF.primary_key only under a strict deterministic policy and must record promotion evidence.\n\nRequired behavior:\n- infer_keys defaults must preserve existing behavior unless product/spec explicitly chooses candidates by default.\n- Candidate sidecar JSON is deterministic and validates against the documented KeyCandidateEvidence schema from the profiler child.\n- Manifest/provenance records sidecar paths relative to root and remains relocatable.\n- Candidate mode never mutates UMF.primary_key or unique_constraints in persisted UMF snapshots.\n- Auto mode promotes only if exactly one candidate is verified_exact=true, non-null, minimal, score \u003e= key_promotion_min_score, and wins by at least key_promotion_min_gap under total deterministic ordering. Ties or conflicts produce sidecars only.\n- Promotion metadata must plainly state downstream effects: sample data uniqueness tracking may change, dbt incremental MERGE unique_key may change, and LDP APPLY CHANGES KEYS may change. For composite keys, metadata must also state any dbt unique-test asymmetry if composite uniqueness is not emitted as a dbt uniqueness test.\n- Candidate sidecars are advisory to sample data/dbt/LDP unless auto mode writes authoritative UMF fields.\n- Re-running compile/bootstrap over the same input writes byte-identical candidate sidecars and manifest entries.\n\nIn-scope files:\n- src/tablespec/bootstrap.py\n- src/tablespec/e2e/paths.py if table bootstrap owns profiler construction\n- src/tablespec/e2e/compile.py\n- src/tablespec/e2e/manifest.py\n- docs/guide/bootstrap.md or focused bootstrap docs if public mode documentation is needed\n- tests/e2e/test_bootstrap_key_candidates.py or equivalent focused tests\n\nOut-of-scope for this bead:\n- Implementing profiler candidate inference; depends on tablespec-29002b40.\n- Implementing native compound uniqueness validation execution; tracked by FEAT-007 child.\n- Making production runtime consume sidecars. Production still runs installed JSON pipeline artifacts and wheels, not authoring inference logic.","id":"tablespec-2bd32b44","issue_type":"task","labels":["area:bootstrap","area:compile","area:manifest","kind:feature"],"owner":"erik","parent":"tablespec-cc4f665b","priority":1,"schema_version":1,"spec-id":"FEAT-026","status":"closed","title":"Persist key-candidate sidecars and gated bootstrap auto-promotion","updated_at":"2026-06-09T20:38:50.448962098Z"} {"acceptance":"Demo job green on DBR 17.3 LTS (Spark 4) with mssql-server-2025; still green on 16.4; README documents both pairings.","created_at":"2026-06-11T15:45:56.545479913Z","description":"2026-06-11: notebooks/northwind-demo pins DBR 16.4 LTS because mssql-server-2022 packages support Ubuntu \u003c=22.04 while DBR 17+ is Ubuntu 24.04. Make 01-provision detect the host Ubuntu version and select the matching mssql-server release (22.04 -\u003e 2022, 24.04 -\u003e 2025) so the demo runs on DBR 17.3 LTS — which is Spark 4.0, matching tablespec's local engine matrix (Spark 4.0/JDK 21). Update README cluster requirements accordingly; validate with a real run on 17.3.","id":"tablespec-2daf281d","issue_type":"task","labels":["area:demo","feat-031","databricks"],"owner":"erik","priority":3,"resolution":"DONE 2026-06-11: notebook 01 is version-aware (Ubuntu 22.04-\u003eSQL 2022, 24.04-\u003eSQL 2025). Proven green on workspace dbw-dev-eus2 BOTH pairings: DBR 17.3 LTS/Spark 4 + SQL Server 2025 (run 419590905072679) and DBR 16.4 LTS + SQL Server 2022 (run 1039374257789724); demo exit PASS/13 tables on both.","schema_version":1,"status":"closed","title":"Northwind demo: auto-select mssql release by host Ubuntu; unlock DBR 17.3 LTS / Spark 4","updated_at":"2026-06-11T16:30:49.547241906Z"} @@ -51,6 +53,7 @@ {"acceptance":"Unit/integration tests for PARQ residual pass; US-043 ACs satisfied.","created_at":"2026-07-22T21:41:08.072147747Z","dependencies":[{"issue_id":"tablespec-502c6126","depends_on_id":"tablespec-e9c21567","type":"blocks","created_at":"2026-07-22T21:41:08Z"}],"description":"Close FR-21.3 residual: typed raw never string-parses DATE/TIMESTAMP.","id":"tablespec-502c6126","issue_type":"task","labels":["helix","phase:build","area:data","FEAT-031"],"notes":"CLOSED 2026-07-22 queue surgery: PARQ typed_raw cast shipped — casting_utils + test_casting_utils typed_raw DATE/TIMESTAMP + ingest generator parquet native typed. FEAT-031/PRD FR-21.3 marked Shipped.","parent":"tablespec-263a0248","priority":1,"schema_version":1,"status":"closed","title":"Implement parquet identity/safe-narrowing residual (PARQ)","updated_at":"2026-07-22T21:47:10.158184288Z"} {"acceptance":"Declared inputs listed; two environments differ only by inputs (US-047-AC2).","closing_commit_sha":"6857ad0da3e8839e3fb40c6fe806ee7f4060e562","created_at":"2026-07-22T21:41:07.136915736Z","description":"Expose catalog, schema, volume, warehouse, and optional links as declared inputs on app.yaml or asset bundle.","events_attachment":"tablespec-5173ff2e/events.jsonl","id":"tablespec-5173ff2e","issue_type":"task","labels":["helix","phase:build","area:app","FEAT-034"],"notes":"[2026-07-22T22:02:14Z] closure rejected: closure gate: insufficient evidence: no execution evidence (empty events and no closing_commit_sha)","owner":"erik","parent":"tablespec-263a0248","priority":1,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"App: parameterized deployment manifest inputs (FR-23.4)","updated_at":"2026-07-22T22:03:00.419363072Z"} {"acceptance":"Plan has the 5 required template sections; module inventory matches src/tablespec; date/version fields consistent; v1/v2 naming disambiguated or merged.","created_at":"2026-06-11T00:24:15.559170406Z","description":"Spec audit 2026-06-10: docs/helix/04-build/implementation-plan.md misses 4 of 5 required sections (scope, implementation_slices, issue_decomposition, risks_and_rollbacks); header says Version 2.0 / 2026-03-16 but content references FEAT-026-era artifacts; module inventory (lines ~52-88) omits every v3.0 compiler-era module (core/, e2e/, dbt/, ldp/, authoring/, compatibility.py, bootstrap.py). Also resolve the naming inversion: the live plan is implementation-plan.md while the file named -v2 is the superseded tombstone.","id":"tablespec-52386c13","issue_type":"task","labels":["helix","spec","backfill"],"owner":"erik","priority":2,"schema_version":1,"status":"closed","title":"Restructure implementation-plan.md to template; refresh stale module inventory","updated_at":"2026-06-12T16:46:15.981169173Z"} +{"acceptance":"1. Public bootstrap_from_tables / umfs_from_specs + compile path documented in docs/guide or README. 2. Test proves end-to-end local path green. 3. No workspace required.","created_at":"2026-07-23T02:37:09.724475595Z","description":"Polish public one-shot bootstrap so Path A/B are equal first-class: public API, CLI or script entry if missing, unit/integration tests, guide docs. Acceptance uses only local/uv tests.","events_attachment":"tablespec-540b1605/events.jsonl","id":"tablespec-540b1605","issue_type":"task","labels":["helix","area:bootstrap","FEAT-026"],"owner":"erik","parent":"tablespec-cc2cfafb","priority":1,"schema_version":1,"session_id":"product-completeness","status":"closed","title":"First-class bootstrap happy path (tables + specs)","updated_at":"2026-07-23T02:38:58.533968809Z"} {"acceptance":"US-050 file exists with US-050-ACn; distinguishes shipped model/reader from residual compile/backbone; FEAT-031 and PRD FR-21.7 cross-link it.","created_at":"2026-07-22T21:41:07.882777607Z","description":"Author US-050 for FR-21.7 JSON residual. Model + JsonReader shipped; backbone still rejects json (delimited/parquet only). Story must state residual vs shipped clearly so implement bead tablespec-9f98cf03 is executable.","events_attachment":"tablespec-557f8a24/events.jsonl","id":"tablespec-557f8a24","issue_type":"task","labels":["helix","phase:frame","area:data","FEAT-031"],"notes":"Queue surgery 2026-07-22: kept; clarify shipped vs residual in ACs.","owner":"erik","parent":"tablespec-263a0248","priority":2,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"Author US-050: JSON/JSONL source kind residual (FR-21.7)","updated_at":"2026-07-22T22:07:12.600304764Z"} {"created_at":"2026-07-23T01:38:49.710437377Z","description":"Notebooks already under notebooks/kaggle-demo/. Update US-044 status, check ACs against notebooks, link FEAT-031; add minimal @covers or documented evidence.","events_attachment":"tablespec-56427779/events.jsonl","id":"tablespec-56427779","issue_type":"task","labels":["helix","demo","FEAT-031"],"owner":"erik","parent":"tablespec-7fe9f279","priority":1,"schema_version":1,"session_id":"tranche-abc-2026-07-22","status":"closed","title":"Tranche B: US-044 Kaggle demo status and evidence","updated_at":"2026-07-23T01:40:23.436331676Z"} {"acceptance":"The two constants agree (or their difference is justified in code comments + spec); FEAT-017 no longer records an open contradiction.","created_at":"2026-06-11T00:24:15.852217623Z","description":"Spec audit 2026-06-10: FEAT-017-validation-pipeline.md line ~44 records an unresolved contradiction: REQUIRED_BASELINE_EXPECTATION_TYPES (gx_baseline.py) includes expect_column_to_exist while REDUNDANT_VALIDATION_TYPES (models/umf.py) lists it as redundant. Disposition: pick one semantics, fix the constant or the spec, and remove the recorded contradiction.","id":"tablespec-58fb4abf","issue_type":"task","labels":["helix","spec","decision","code"],"priority":3,"schema_version":1,"status":"closed","title":"DECISION: resolve expect_column_to_exist redundancy contradiction (FEAT-017)","updated_at":"2026-06-11T00:45:53.250783888Z"} @@ -74,6 +77,7 @@ {"acceptance":"1. TestNativeProfilerCompositeKeys::test_native_profiler_composite_minimal_key verifies member_id + effective_date exact uniqueness yields a composite candidate only when neither subset is exact unique.\n2. TestNativeProfilerCompositeKeys::test_native_profiler_rejects_nonminimal_composite verifies if id is exact unique, id + date is rejected or marked nonminimal and is not emitted as a candidate.\n3. TestNativeProfilerCompositeKeys::test_native_profiler_respects_candidate_and_pass_budget verifies a wide-table fixture exact-verifies no more than key_verification_pass_budget candidates and leaves skipped candidates verified_exact=false.\n4. TestNativeProfilerCompositeKeys::test_composite_search_respects_max_width_and_max_candidates verifies no exhaustive all-column combinatorics occur and emitted/preverified candidates are bounded by key_max_width and key_max_candidates.\n5. TestNativeProfilerCompositeKeys::test_minimality_is_honest_when_subset_verification_is_skipped verifies minimal=True is used only after every proper subset was exact-verified as non-unique; otherwise evidence records false/unknown minimality semantics and reason.\n6. TestProfilerConnectSailKeyCandidates::test_connect_classic_key_candidates_structural_equality verifies serialized key_candidates structural equality on deterministic classic and Spark Connect/Sail fixtures.\n7. TestQualityGates::test_composite_go_test_gate documents and passes go test ./... or records operator_required when no Go module/toolchain exists.\n8. TestQualityGates::test_composite_lefthook_gate documents and passes lefthook run pre-commit or records operator_required when lefthook is unavailable.\n9. Verification command passes: uv run pytest tests/unit/test_native_profiler_key_candidates.py tests/unit/test_profiler_connect_sail.py tests/unit -k gx_expectation.","closing_commit_sha":"e4533c64a2c40225c17763e449269a532cc4969c","created_at":"2026-06-09T04:49:35.174959604Z","description":"PROBLEM: NativeSparkProfiler cannot detect composite advisory keys and has no bounded exact-verification strategy. A naive implementation would either miss useful member_id + effective_date-style keys or exhaustively verify all column combinations, which is too expensive and risks dishonest minimality claims.\n\nROOT CAUSE: src/tablespec/profiling/native_profiler.py:157 counts rows and src/tablespec/profiling/native_profiler.py:186 collects profile signals in one batch, but there is no key candidate model, bounded composite candidate generator, exact distinct verification for column tuples, subset verification, verification pass budget, or skipped-candidate semantics.\n\nPROPOSED FIX: Add bounded composite inference behind infer_key_candidates=True using cheap profile signals first. Generate at most key_max_candidates pre-verification candidates up to key_max_width, prefer promising columns deterministically, and exact-verify only within key_verification_pass_budget using Connect-safe DataFrame operations. Mark minimal=True only when all proper subsets were exact-verified as non-unique; otherwise use false/unknown semantics in evidence rather than guessing. Reject or do not emit nonminimal composites when an exact unique subset exists. Budget-skipped candidates must remain verified_exact=false and must not suppress legacy approximate uniqueness behavior downstream.\n\nGOVERNING ARTIFACT: parent bead tablespec-cc4f665b.\n\nDEPENDENCY ID: tablespec-cc4f665b.\n\nNON-SCOPE: Single-column model serialization except what is required to reuse KeyCandidate/KeyCandidateEvidence. Writing sidecar files or extending manifest layout; tracked by FEAT-026 child. Mutating UMF.primary_key or unique_constraints. Making sample data, dbt, or LDP consume inferred candidates. Implementing compound uniqueness execution in the validator; tracked by FEAT-007 child.","id":"tablespec-87e12737","issue_type":"task","labels":["area:profiling","kind:feature","ac-quality:refined","parent:tablespec-cc4f665b"],"parent":"tablespec-29002b40","priority":0,"schema_version":1,"status":"closed","title":"Infer bounded composite advisory keys with honest minimality and budgets","updated_at":"2026-06-09T20:17:34.343973218Z"} {"acceptance":"All US-039 ACs (AC1-AC6) pass on the Docker lane; databricks_e2e lane green when credentials configured; evidence recorded (commands + exit status) in the bead.","closing_commit_sha":"224a8af9789d52195ccec8d91c7009fd82ba027b","created_at":"2026-06-11T02:34:36.619401849Z","dependencies":[{"issue_id":"tablespec-8980c812","depends_on_id":"tablespec-4b65c810","type":"blocks","created_at":"2026-06-11T02:34:36Z"}],"description":"Drive US-039 ACs: Northwind in SQL Server -\u003e discover one UMF per table (FKs incl. orders.customer_id-\u003ecustomers.customer_id; 'Order Details'-\u003eorder_details sanitization) -\u003e tablespec validate all pass -\u003e export-excel workbook round-trips -\u003e FK-aware sample data -\u003e staged validation report per table with real results and zero silent NULL-out of typed columns. Local lane: Docker SQL Server + Northwind, SKIP without Docker. Databricks lane: opt-in databricks_e2e tier. Consumer integration (entropy-exchange 03_etl.py replacement) follows in that repo.","id":"tablespec-8980c812","issue_type":"task","labels":["helix","area:ingestion","feat-031","phase:3","acceptance-goal"],"owner":"erik","parent":"tablespec-ef91646f","priority":1,"schema_version":1,"status":"closed","title":"US-039 Northwind end-to-end acceptance (Docker lane + databricks_e2e)","updated_at":"2026-06-11T13:55:45.336976785Z"} {"acceptance":"Checklist has the required sections with measurable decision points and is execution-ready for the next release.","created_at":"2026-06-11T00:24:15.631071468Z","description":"Spec audit 2026-06-10: docs/helix/05-deploy/deployment-checklist.md is a descriptive backfill of release.yml; missing release_scope, rollout_plan, rollback_triggers (has steps, no thresholds), go_or_no_go_decision; blocking measurable_decision_points check unmet; not execution-ready (no checkboxes/owners/window).","id":"tablespec-8a432681","issue_type":"task","labels":["helix","spec","backfill"],"owner":"erik","priority":3,"schema_version":1,"status":"closed","title":"Restructure deployment-checklist.md to template (rollout, rollback triggers, go/no-go)","updated_at":"2026-06-12T16:49:44.346035199Z"} +{"acceptance":"1. Script or pytest under scripts/ or tests/ runs 3-table bootstrap+compile path and writes a metrics JSON/markdown. 2. Docs cite the sample and how to re-run. 3. uv run pytest or script exits 0.","created_at":"2026-07-23T02:37:09.631948504Z","description":"Implement a documented 3-table onboarding sample + measurement harness that times/counts steps for hand-authored baseline vs bootstrap+compile+review workflow. Records results to a committed report template so PRD 50% authoring reduction metric has in-repo evidence path. Agent-executable: pure repo scripts/tests.","events_attachment":"tablespec-8df106fd/events.jsonl","id":"tablespec-8df106fd","issue_type":"task","labels":["helix","area:product","metric"],"owner":"erik","parent":"tablespec-cc2cfafb","priority":1,"schema_version":1,"session_id":"product-completeness","status":"closed","title":"3-table onboarding authoring-reduction benchmark harness","updated_at":"2026-07-23T02:38:13.272958078Z"} {"acceptance":"1. TestCompileUmfsAcceptsDatabricksDialect: add a compile-path test proving compile_umfs(..., dialect=\"databricks\") completes without ValueError and persists ingest, dbt_ingest, optional dbt_gold, LDP, and manifest artifacts under the pinned layout.\n2. TestCompileUmfsPreservesPublicDialectInManifest: add a regression test proving the generated metadata records the public databricks dialect while any Spark-family internal normalization remains invisible to downstream artifact consumers.\n3. TestPytestGate: `uv run pytest tests/unit/test_bootstrap.py tests/e2e/test_bootstrap_from_specs.py`\n4. TestGoGate: `go test ./...`\n5. TestLefthookGate: `lefthook run pre-commit`","closing_commit_sha":"73a4d3cab2ccb57de6daf5527f7d3c495232c6dc","created_at":"2026-06-08T14:50:25.00456272Z","description":"PROBLEM: The compile/bootstrap path can receive dialect=databricks, but the artifact pipeline still mixes public and internal dialect handling, so the emitted ingest/dbt_ingest/LDP/manifest set can reject databricks or record the wrong dialect.\nROOT CAUSE: src/tablespec/e2e/compile.py:129 forwards the caller dialect to generate_dbt_dag_project, src/tablespec/e2e/compile.py:141 hard-codes generate_ldp_project(..., dialect=\"spark\"), src/tablespec/e2e/compile.py:193 calls generate_ingest_sql without a dialect, and src/tablespec/schemas/ingest_generator.py:214 hard-codes build_ingest_select(..., dialect=\"spark\").\nPROPOSED FIX: Thread the public databricks dialect through compile_umfs, normalize Spark-family emitters internally only where SQL is identical, keep dbt target selection as databricks, and preserve the public dialect in generated metadata while still emitting the pinned artifact layout.\nNON-SCOPE: One-shot bootstrap facade implementation. Production pipeline JSON/wheel packaging contract. Inline UMF format cleanup. No real Databricks workspace execution beyond existing opt-in tiers. GOVERNING ARTIFACTS: docs/helix/01-frame/features/FEAT-026-compile-orchestrator-bootstrap.md:84; docs/helix/03-test/conformance-acceptance.md:49; docs/helix/03-test/conformance-acceptance.md:85; docs/helix/03-test/conformance-acceptance.md:109. DEPENDENCIES: parent bead tablespec-ed74497c; no additional dependencies.","events_attachment":"tablespec-924c7384/events.jsonl","id":"tablespec-924c7384","issue_type":"task","labels":["area:databricks","area:dialects","kind:bug","ac-quality:needs-refinement"],"parent":"tablespec-171e409c","priority":0,"schema_version":1,"session_id":"eb-e961169d","status":"closed","title":"Thread databricks through compile-path normalization","updated_at":"2026-06-08T15:41:34.831875417Z"} {"acceptance":"1. Normalization decision documented in compile.py + conformance-acceptance.md.\n2. Docs regression + normalize_cast_dialect tests.\n3. Pytest + pre-commit. (No Go gate — pure Python repo.)","created_at":"2026-06-08T14:50:25.086649929Z","description":"PROBLEM: The public contract still reads like Spark/duckdb-only support in places, so even after code paths accept databricks, downstream reviewers and validators can still infer the wrong supported dialect set.\nROOT CAUSE: docs/helix/01-frame/features/FEAT-026-compile-orchestrator-bootstrap.md:84 and docs/helix/03-test/conformance-acceptance.md:49, docs/helix/03-test/conformance-acceptance.md:85, and docs/helix/03-test/conformance-acceptance.md:109 are the governing acceptance artifacts, but the implementation notes and emitted metadata/comments do not clearly state that public dialect=\"databricks\" is accepted while Spark-family emitters may normalize internally.\nPROPOSED FIX: Update implementation notes, comments, or generated metadata to spell out the dialect contract, including the Spark-family normalization decision and the reason it is safe because the cast SQL is identical.\nNON-SCOPE: One-shot bootstrap facade implementation. Production pipeline JSON/wheel packaging contract. Inline UMF format cleanup. No real Databricks workspace execution beyond existing opt-in tiers. GOVERNING ARTIFACTS: docs/helix/01-frame/features/FEAT-026-compile-orchestrator-bootstrap.md:84; docs/helix/03-test/conformance-acceptance.md:49; docs/helix/03-test/conformance-acceptance.md:85; docs/helix/03-test/conformance-acceptance.md:109. DEPENDENCIES: parent bead tablespec-ed74497c; no additional dependencies.","events_attachment":"tablespec-943b21a5/events.jsonl","execution-eligible":false,"id":"tablespec-943b21a5","issue_type":"task","labels":["area:databricks","area:dialects","area:docs","kind:bug","ac-quality:needs-refinement"],"parent":"tablespec-171e409c","priority":0,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"Document the databricks dialect contract and normalization rule","updated_at":"2026-07-23T00:26:00.229083238Z"} {"acceptance":"Each US-046-AC has a named test with @covers US-046-ACn or a documented non-automatable exception.","created_at":"2026-07-22T21:41:08.54317106Z","description":"Guidebook generation tests exist (test_guidebook_generate.py etc.) but lack @covers US-046-ACn. Add citations or document exceptions; do not re-implement guidebook.","events_attachment":"tablespec-9c79765e/events.jsonl","id":"tablespec-9c79765e","issue_type":"task","labels":["helix","phase:test","area:docs","FEAT-033"],"notes":"Queue surgery 2026-07-22: confirmed hygiene-only; kept.","owner":"erik","parent":"tablespec-263a0248","priority":3,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"Link US-046 guidebook ACs to tests","updated_at":"2026-07-22T22:07:14.196384917Z"} @@ -99,6 +103,7 @@ {"acceptance":"pytest green incl. new unit tests per consumer; a spec with EMBEDDING(1024) round-trips, compiles to ARRAY\u003cFLOAT\u003e DDL + ArrayType pyspark + JSON schema; GX suite contains the dimensionality expectation; dimension mismatch fails validation; existing corpus byte-identical.","created_at":"2026-06-12T16:49:59.863931435Z","description":"ADR-016 (Accepted): dimensioned EMBEDDING logical type -\u003e ARRAY\u003cFLOAT\u003e (Spark SQL/Delta), ArrayType(FloatType()) (PySpark), array-of-number (JSON Schema). Touch: models/umf.py:509-511 type pattern + dimension field, type_mappings.py, schemas/generators.py, gx_baseline.py (dimensionality expectation; %16 Vector-Search advisory non-blocking; exclude from string-shape checks), sample_data (seeded unit vectors), compatibility.py/type_lattice.py (EMBEDDING compatible only with same-dimension EMBEDDING), profiling passthrough, umf.schema.json. Per FEAT-032 EMB-01..09.","id":"tablespec-c5f57c11","issue_type":"task","labels":["feat-032","area:models","area:types","kind:feature"],"owner":"erik","priority":1,"schema_version":1,"status":"closed","title":"Implement EMBEDDING(dim) type across the type system (FEAT-032 EMB-)","updated_at":"2026-06-12T17:46:28.040478934Z"} {"acceptance":"1. TestDatabricksCompileArtifactsStateNormalizationDecision: compile.py + dialects.normalize_cast_dialect document/prove public dialect=databricks normalizes to spark for identical cast SQL.\n2. TestPytestGate: `uv run pytest tests/docs/test_dialect_contract.py tests/unit/test_bootstrap.py`\n3. TestPreCommitGate: pre-commit / CI pre-commit job.\nNote: `go test ./...` is not a gate for this pure-Python repository.","created_at":"2026-06-08T18:22:28.923233689Z","description":"PROBLEM: The implementation notes, comments, or generated metadata do not clearly explain why public dialect=\"databricks\" may be normalized internally by Spark-family SQL emitters, leaving the normalization behavior easy to misread as unsupported dialect handling.\nROOT CAUSE: docs/helix/01-frame/features/FEAT-026-compile-orchestrator-bootstrap.md:84 is the governing compile-orchestrator artifact, but the implementation notes and emitted metadata/comments do not state that public dialect=\"databricks\" is accepted while Spark-family emitters may normalize to spark internally because cast SQL is identical.\nPROPOSED FIX: Update implementation notes, comments, or generated metadata and bootstrap regression coverage to explicitly state the normalization decision: databricks remains accepted at the public contract boundary, and Spark-family SQL emitters may normalize internally to spark because the generated cast SQL is identical.\nNON-SCOPE: One-shot bootstrap facade implementation. Production pipeline JSON/wheel packaging contract. Inline UMF format cleanup. No real Databricks workspace execution beyond existing opt-in tiers. GOVERNING ARTIFACTS: docs/helix/01-frame/features/FEAT-026-compile-orchestrator-bootstrap.md:84; docs/helix/03-test/conformance-acceptance.md:49; docs/helix/03-test/conformance-acceptance.md:85; docs/helix/03-test/conformance-acceptance.md:109. DEPENDENCIES: parent bead tablespec-ed74497c; no additional dependencies.","events_attachment":"tablespec-c65a49de/events.jsonl","id":"tablespec-c65a49de","issue_type":"task","labels":["area:databricks","area:dialects","area:docs","kind:bug","ac-quality:refined"],"owner":"erik","parent":"tablespec-943b21a5","priority":0,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"Document Spark-family normalization for databricks emitters","updated_at":"2026-07-23T00:25:57.310822858Z","work-last-detail":"The acceptance gate requires `go test ./...`, but this worktree contains no Go module or Go packages, so the command cannot execute here without repository changes outside the bead scope.","work-last-status":"no_changes_operator_required","work-no-changes-count":1,"work-suggested-action":"review and accept, split, block, or cancel this proposed work"} {"acceptance":"Each ADR has the template sections; no in-place post-acceptance tracker edits remain (history preserved via supersession/status annotations).","created_at":"2026-06-11T00:23:44.419722289Z","description":"Spec audit 2026-06-10: ADR-001..008 predate the catalog template: prose Status instead of header table, no Alternatives section (blocking alternatives_evaluated check), missing Risks/Validation/Supersession/Concern Impact/References. ADR-006 has alternatives content inline needing only heading restructure. ADR-007/008 contain post-acceptance status edits that should move to supersession notes. Exemplars: ADR-009..013.","id":"tablespec-cb798660","issue_type":"task","labels":["helix","spec","backfill"],"owner":"erik","priority":3,"schema_version":1,"status":"closed","title":"Backfill ADR-001..008 to current ADR template (Alternatives, Risks, Validation)","updated_at":"2026-06-12T17:02:16.419944093Z"} +{"created_at":"2026-07-23T02:37:09.544118814Z","description":"Close the loop on vision realization: measured onboarding authoring reduction, polished bootstrap/JDBC happy paths, Databricks e2e opt-in residual (agent-side harness), and FR-23 app smoke (unit/script residual). No beads that require the operator to run a workspace by hand as the only acceptance path.","events_attachment":"tablespec-cc2cfafb/events.jsonl","id":"tablespec-cc2cfafb","issue_type":"epic","labels":["helix","product","completeness"],"priority":1,"schema_version":1,"session_id":"product-completeness","status":"closed","title":"Product completeness loop: vision KPIs + first-class happy paths","updated_at":"2026-07-23T02:41:13.693581346Z"} {"acceptance":"1. Child bead for FEAT-024 profiler candidate inference exists and is linked under this parent.\n2. Child bead for FEAT-026 bootstrap/compile sidecar and auto-promotion policy exists and depends on the profiler child.\n3. Child bead for FEAT-007 compound uniqueness validation support/quarantine exists and depends on the profiler child.\n4. Each child bead includes command-based acceptance criteria with named tests and scope boundaries.","created_at":"2026-06-08T04:18:43.35752831Z","description":"Parent epic for candidate key inference from native Spark profiles. Current behavior: NativeSparkProfiler computes row count, completeness, cardinality, numeric/string stats, quantiles, values, and patterns, but it does not detect primary keys. ProfileToGxMapper may emit approximate single-column uniqueness expectations based on cardinality ratio; that is validation-rule enrichment, not authoritative primary-key inference.\n\nGoal: add advisory key-candidate inference without treating guesses as authoritative UMF keys by default. Split implementation across the owning specs: FEAT-024 owns profiler models/inference and mapper dedup, FEAT-026 owns bootstrap/compile sidecar artifacts and auto-promotion provenance, and FEAT-007 owns compound uniqueness execution or explicit non-emission when unsupported.\n\nDesign constraints accepted from review:\n- Candidate mode is advisory and must not mutate persisted UMF snapshots.\n- Auto-promotion is explicitly configured and is a behavior-changing write to UMF.primary_key.\n- Compound uniqueness is only considered validation-supported when executable by the native executor, not merely schema-allowed.\n- Candidate ordering, scoring, and serialized sidecars must be deterministic.\n- Exact verification must use Spark-SQL/DataFrame operations that are Connect/serverless safe.\n\nChild beads should be executable independently and cold-readable. Do not implement this parent directly.","events":[{"actor":"","body":"closed_because: all_children_terminal\ntotal_children: 3","created_at":"2026-06-09T20:47:18.126777759Z","kind":"epic_auto_close","source":"Store.Close.walkUpClosureCandidate","summary":"auto-closed: all children reached terminal state"}],"id":"tablespec-cc4f665b","issue_type":"epic","labels":["area:profiling","area:bootstrap","area:validation","kind:feature"],"priority":1,"schema_version":1,"spec-id":"FEAT-024","status":"closed","title":"Add native-profiler candidate key inference","updated_at":"2026-06-09T20:47:18.129298422Z"} {"acceptance":"1. TestKeyCandidateModels::test_key_candidate_models_are_json_serializable verifies KeyCandidate and KeyCandidateEvidence serialize with every documented evidence field present, candidate kind constrained to primary_key_candidate or unique_constraint_candidate, score bounded in [0, 1], and DataFrameProfile exposes key_candidates without breaking existing columns/num_records construction.\n2. TestKeyCandidateModels::test_dataframe_profile_default_key_candidates_isolated verifies DataFrameProfile.key_candidates defaults to an empty list and does not share mutable list state between profile instances.\n3. TestQualityGates::test_key_candidate_models_go_test_gate documents and passes go test ./... or records operator_required when no Go module/toolchain exists.\n4. TestQualityGates::test_key_candidate_models_lefthook_gate documents and passes lefthook run pre-commit or records operator_required when lefthook is unavailable.\n5. TestVerificationCommands::test_key_candidate_models_pytest_gate documents and passes uv run pytest tests/unit/test_native_profiler_key_candidates.py tests/unit/test_profiler_connect_sail.py tests/unit -k gx_expectation.","closing_commit_sha":"d16c665ef63409828d9179ed5376fe8bce6b5eee","created_at":"2026-06-09T04:53:43.812978901Z","description":"PROBLEM: DataFrameProfile has no typed profile-level representation for advisory primary-key or unique-key candidates, so later inference work has nowhere JSON-serializable to put evidence without conflating advisory candidates with authoritative UMF.primary_key or unique_constraints.\n\nROOT CAUSE: src/tablespec/profiling/types.py:53 defines DataFrameProfile with only num_records and columns; src/tablespec/profiling/types.py has no KeyCandidate or KeyCandidateEvidence model.\n\nPROPOSED FIX: Add typed dataclass models KeyCandidate and KeyCandidateEvidence in src/tablespec/profiling/types.py. Add key_candidates: list[KeyCandidate] to DataFrameProfile with default_factory=list while preserving existing DataFrameProfile(num_records=..., columns=...) construction. Models must serialize through dataclasses.asdict/json-compatible primitives. Candidate kind must be constrained to primary_key_candidate or unique_constraint_candidate. Evidence fields must include row_count, columns, null_count_by_column, exact_distinct_count, approximate_distinct_count_by_column, distinct_ratio, completeness_by_column, verified_exact, nullable, minimal, subset_unique, score, score_components, name_hints, type_hints, penalties, verification_pass_count, verification_query_count, and reason. Score must be bounded in [0, 1].\n\nGOVERNING ARTIFACT: parent bead tablespec-cc4f665b.\n\nDEPENDENCY ID: tablespec-cc4f665b.\n\nNON-SCOPE: Writing sidecar files or extending manifest layout; tracked by FEAT-026 child. Mutating UMF.primary_key or unique_constraints. Making sample data, dbt, or LDP consume inferred candidates. Implementing compound uniqueness execution in the validator; tracked by FEAT-007 child.\n\nVERIFICATION REQUIREMENT: uv run pytest tests/unit/test_native_profiler_key_candidates.py tests/unit/test_profiler_connect_sail.py tests/unit -k gx_expectation. Also run go test ./... or record operator_required when no Go module/toolchain exists, and lefthook run pre-commit or record operator_required when lefthook is unavailable.","id":"tablespec-ce4a8ebc","issue_type":"task","labels":["area:profiling","kind:feature","ac-quality:refined","parent:tablespec-cc4f665b","component:profile-models"],"owner":"erik","parent":"tablespec-a727e762","priority":0,"schema_version":1,"status":"closed","title":"Add JSON-serializable advisory key-candidate profile models","updated_at":"2026-06-09T20:29:54.71566449Z"} {"acceptance":"uv run pytest tests/test_core_encapsulation.py passes in full on a clean tree.","created_at":"2026-06-11T13:30:54.285175084Z","description":"Found during FEAT-031 work (2026-06-11): tests/test_core_encapsulation.py::test_ldp_feeds_only_on_core_seam fails on a clean tree because src/tablespec/ldp/expectations.py imports tablespec.expectation_utils, violating the LDP-feeds-only-on-core-seam guard (ADR-013). Either move the needed helper into the core seam or adjust the guard's allowlist with justification.","id":"tablespec-d27c334f","issue_type":"bug","labels":["kind:bug","area:ldp","encapsulation"],"owner":"erik","priority":2,"schema_version":1,"status":"closed","title":"Fix pre-existing encapsulation failure: ldp imports expectation_utils","updated_at":"2026-06-12T16:43:23.279877922Z"} @@ -111,6 +116,7 @@ {"acceptance":"Dump fixture tier green in tests/conformance; skip_rows/footer/\\N covered by fixtures; readers derive every option from the source spec.","created_at":"2026-06-11T02:34:36.697697453Z","dependencies":[{"issue_id":"tablespec-df8bc351","depends_on_id":"tablespec-4bea5c6c","type":"blocks","created_at":"2026-06-11T02:34:36Z"}],"description":"FEAT-031 DUMP-01..04: multi-char line terminators, \\N-style null escapes, footer handling, skip_rows honored end-to-end through the reader seam; conformance corpus gains a dump fixture tier.","id":"tablespec-df8bc351","issue_type":"task","labels":["helix","area:ingestion","feat-031","phase:4"],"owner":"erik","parent":"tablespec-ef91646f","priority":2,"schema_version":1,"status":"closed","title":"Dump-dialect delimited options end-to-end","updated_at":"2026-06-12T17:17:31.962452089Z"} {"acceptance":"US-038 ACs evidenced green or residual filed with pointer.","created_at":"2026-07-22T21:41:08.635350717Z","description":"Verify Playwright/Hugo evidence covers US-038-AC1..5.","id":"tablespec-e1956759","issue_type":"task","labels":["helix","phase:test","area:docs","FEAT-030"],"notes":"CLOSED 2026-07-22 queue surgery: website/e2e Playwright covers Getting Started, /simple/ install URL, raw-ingested-silver boundary, nav — US-038 ACs evidenced. No residual.","parent":"tablespec-263a0248","priority":3,"schema_version":1,"status":"closed","title":"Close residual US-038 microsite ACs if any open","updated_at":"2026-07-22T21:47:12.04065791Z"} {"acceptance":"US-042 file exists with ACs mapping DUMP-01..04; each AC cites existing test evidence; FEAT-031 links US-042.","created_at":"2026-07-22T21:41:07.69343755Z","description":"DUMP-01..04 are SHIPPED (delimited dump options + dump reader + tests/unit/test_ingestion_package.py). Author US-042 ACs that cite those tests. No implement child remains (tablespec-7ec86390 closed).","events_attachment":"tablespec-e322b612/events.jsonl","id":"tablespec-e322b612","issue_type":"task","labels":["helix","phase:frame","area:data","FEAT-031"],"notes":"Queue surgery 2026-07-22: re-scoped; DUMP implement closed as shipped.","owner":"erik","parent":"tablespec-263a0248","priority":1,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"Author US-042: dump-dialect AC backfill (story floor only)","updated_at":"2026-07-22T22:07:11.111822114Z"} +{"acceptance":"1. Documented happy path command/composition. 2. Docker-gated or unit tests green. 3. backbone/compile does not fail closed on kind=jdbc without a supported residual story.","created_at":"2026-07-23T02:37:09.811567073Z","description":"Make JDBC discovery→UMF→compile as first-class as delimited: ensure public docs and docker-gated tests cover the path; fill any compile residual for jdbc kind if still fail-closed incorrectly.","events_attachment":"tablespec-e62cd6b8/events.jsonl","id":"tablespec-e62cd6b8","issue_type":"task","labels":["helix","area:data","FEAT-031","JDBC"],"owner":"erik","parent":"tablespec-cc2cfafb","priority":1,"schema_version":1,"session_id":"product-completeness","status":"closed","title":"JDBC onboarding path parity with delimited demos","updated_at":"2026-07-23T02:39:57.34065859Z"} {"acceptance":"Each of the three FEATs has a real PRD anchor or a recorded convention explains the exception.","created_at":"2026-06-11T00:24:15.773623381Z","description":"Spec audit 2026-06-10: three FEATs have only nominal PRD anchors: FEAT-015 cites FR-1.1 (Pydantic models), FEAT-016 cites FR-20.3, FEAT-022 cites FR-11.1/11.2 (which the PRD assigns to FEAT-010 scope). The PRD has no requirement for API documentation, test infrastructure as a product concern, or schema-compatibility checking. Either add FRs to the PRD for these capabilities or record that infrastructure FEATs may anchor to vision/principles instead.","id":"tablespec-e728ca79","issue_type":"task","labels":["helix","spec","decision"],"priority":3,"resolution":"DECIDED 2026-06-10 (operator): split. FR-11.6 added to PRD (FEAT-022 anchors to it); FEAT-015/016 recorded as meta-features anchored to Vision/Principles per principles.md §Tension Resolution; FR-20.3 platform posture reassigned to FEAT-029 with FEAT-016 as evidence tier.","schema_version":1,"status":"closed","title":"DECISION: PRD anchors for FEAT-015 (api-docs), FEAT-016 (testing infra), FEAT-022 (compatibility)","updated_at":"2026-06-11T00:56:02.68797967Z"} {"acceptance":"US-043 file exists with ACs for identity/safe-narrowing and negative DATE path if already covered by tests; FEAT-031 links US-043; no greenfield cast work.","created_at":"2026-07-22T21:41:07.78305571Z","description":"PARQ typed-raw cast path is SHIPPED (casting_utils typed_raw + test_casting_utils typed_raw DATE/TIMESTAMP + ingest generator parquet native typed). Author US-043 ACs citing those tests. No implement child remains (tablespec-502c6126 closed).","events_attachment":"tablespec-e9c21567/events.jsonl","id":"tablespec-e9c21567","issue_type":"task","labels":["helix","phase:frame","area:data","FEAT-031"],"notes":"Queue surgery 2026-07-22: re-scoped; PARQ implement closed as shipped.","owner":"erik","parent":"tablespec-263a0248","priority":1,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"Author US-043: parquet typed-raw AC backfill (story floor only)","updated_at":"2026-07-22T22:07:11.85870052Z"} {"acceptance":"1. A user can pass dialect=databricks through the documented Databricks compile path without ValueError, or all public docs/examples reject that spelling consistently and require dialect=spark. The implementation notes must state which decision was made.\n2. Error messages across casting, ingest, dbt, LDP, compile, and CLI agree on the accepted dialect values.\n3. Tests cover Databricks dialect handling at the lowest cast layer and at least one compile/emitter path.\n4. Docs and notebook examples explain the relationship between Databricks target/runtime and Spark-family cast SQL in one consistent way.\n5. A verification command such as uv run pytest tests/unit/test_casting_utils.py plus the added dialect tests passes.","created_at":"2026-06-08T00:21:05.838311929Z","description":"Resolve the current inconsistent Databricks dialect surface. Users compiling for Databricks can be told to pass dialect=spark because Databricks SQL is Spark-family for our cast expressions, but parts of the repo also expose dialect=databricks as a valid option. Current evidence: casting_utils.cast_column_sql accepts spark, databricks, and duckdb; compile_umfs forwards dialect to dbt DAG generation but emits LDP with spark; docs and CLI help mention databricks in some places while other comments/docs say only spark or duckdb. A Databricks notebook saw ValueError: Unsupported dialect: databricks expected spark or duckdb, which means either stale installed code or a downstream validator still rejects databricks. Canonicalize this so users and agents have one correct instruction.\n\nDecision to encode:\n- Public Databricks compile UX should accept dialect=databricks as an explicit alias for Spark-family cast SQL, or public docs should consistently require dialect=spark and never advertise databricks. Prefer accepting databricks everywhere as an alias because the CLI already advertises it and users naturally choose it for Databricks.\n- Internal rendering may normalize databricks to spark when emitters share identical SQL.\n- Error messages and docs must name the same accepted values.\n\nIn-scope files:\n- src/tablespec/casting_utils.py\n- src/tablespec/schemas/ingest_generator.py\n- src/tablespec/ldp/project.py\n- src/tablespec/e2e/compile.py\n- src/tablespec/cli.py\n- docs/helix and Databricks bootstrap/try-it-out docs\n- tests covering Databricks dialect acceptance through compile/emitter paths\n\nOut-of-scope for this bead:\n- One-shot bootstrap facade implementation\n- Production pipeline JSON/wheel packaging contract\n- Inline UMF format cleanup","events_attachment":"tablespec-ed74497c/events.jsonl","execution-eligible":false,"id":"tablespec-ed74497c","issue_type":"task","labels":["area:databricks","area:dialects","area:docs","kind:bug","ac-quality:needs-refinement"],"priority":0,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","spec-id":"FEAT-026","status":"closed","title":"Canonicalize Databricks dialect handling across code and docs","updated_at":"2026-07-23T00:26:02.172377035Z"} diff --git a/apps/data-profiling/profiler/smoke.py b/apps/data-profiling/profiler/smoke.py new file mode 100644 index 0000000..5a56e9c --- /dev/null +++ b/apps/data-profiling/profiler/smoke.py @@ -0,0 +1,37 @@ +"""FR-23 agent-side smoke: resolve → provision → validate_config. + +No live workspace required when ``PROFILER_RUNTIME=mock`` and a fake SQL +executor is supplied. Live deploy remains operational residual. +""" + +from __future__ import annotations + +from typing import Mapping, Optional + +from profiler.config import AppConfig, resolve_config +from profiler.diagnostics import validate_config +from profiler.provision import SqlExecutor, provision + + +def run_fr23_smoke( + *, + env: Optional[Mapping[str, str]] = None, + registry_path: str = "connections.yaml", + executor: Optional[SqlExecutor] = None, + grant_to: Optional[str] = None, +) -> int: + """Return 0 if composition succeeds, 1 if startup faults on a non-mock runtime. + + Always runs provision when an executor is provided. On mock runtime, + validate_config returns no faults (nothing to probe). + """ + cfg: AppConfig = resolve_config(env=env, registry_path=registry_path) + if executor is not None: + provision(cfg, executor=executor, grant_to=grant_to) + faults = validate_config(cfg) + if faults and cfg.is_databricks: + for f in faults: + print(f.message()) + return 1 + print(f"FR-23 smoke OK: metadata home {cfg.describe()}") + return 0 diff --git a/apps/data-profiling/scripts/fr23_smoke.py b/apps/data-profiling/scripts/fr23_smoke.py new file mode 100644 index 0000000..f018565 --- /dev/null +++ b/apps/data-profiling/scripts/fr23_smoke.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +"""CLI smoke for FR-23 (unit path, no workspace required with mock runtime). + + cd apps/data-profiling + PROFILER_RUNTIME=mock python scripts/fr23_smoke.py + +Exit 0 on success. For empty-environment provision, pass --provision with a +Databricks warehouse (not the mock path). +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from profiler.smoke import run_fr23_smoke # noqa: E402 + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument( + "--registry", + default="connections.yaml", + help="Connection registry path (default connections.yaml)", + ) + p.add_argument( + "--provision", + action="store_true", + help="Run provision against a live warehouse (requires DATABRICKS_*)", + ) + args = p.parse_args() + executor = None + if args.provision: + from profiler.provision import DatabricksExecutor + + executor = DatabricksExecutor() + return run_fr23_smoke( + env=os.environ, + registry_path=args.registry, + executor=executor, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/data-profiling/tests/test_fr23_stack.py b/apps/data-profiling/tests/test_fr23_stack.py index 618eb9c..ef6102c 100644 --- a/apps/data-profiling/tests/test_fr23_stack.py +++ b/apps/data-profiling/tests/test_fr23_stack.py @@ -51,13 +51,30 @@ def test_fr23_resolve_provision_validate_compose(tmp_path: Path) -> None: assert cfg.metadata_catalog == "stack_cat" assert cfg.source_of("metadata_catalog") == "deployment" - report = provision(cfg, executor=_EmptyExecutor(), grant_to=None) + empty = _EmptyExecutor() + report = provision(cfg, executor=empty, grant_to=None) assert isinstance(report, ProvisionReport) assert report.changed is True + assert empty.statements # DDL issued on empty environment - # Second provision against a fake that reports everything already present - # is covered in test_provision; here we only require composition succeeds. + # Smoke: mock runtime validation is list-shaped and non-raising. faults = validate_config(cfg) - # mock runtime without a live warehouse probe should not raise; faults may - # be empty or advisory depending on runtime probes — must be a list. assert isinstance(faults, list) + assert faults == [] # mock runtime skips warehouse probes + + +def test_fr23_smoke_exit_zero_composition(tmp_path: Path) -> None: + """CLI-equivalent smoke: resolve + provision + validate returns success.""" + from profiler.smoke import run_fr23_smoke + + code = run_fr23_smoke( + env={ + "PROFILER_METADATA_CATALOG": "smoke_cat", + "PROFILER_METADATA_SCHEMA": "smoke_schema", + "PROFILER_OUTPUT_VOLUME": "smoke_vol", + "PROFILER_RUNTIME": "mock", + }, + registry_path=str(tmp_path / "missing.yaml"), + executor=_EmptyExecutor(), + ) + assert code == 0 diff --git a/docs/guide/bootstrap.md b/docs/guide/bootstrap.md index 4a7d1fe..44254a4 100644 --- a/docs/guide/bootstrap.md +++ b/docs/guide/bootstrap.md @@ -14,6 +14,8 @@ relationships, aliases, keys, raw-to-ingest SQL, validation suites, and manifest entries. Silver-layer work such as cross-source conformance, survivorship, entity resolution, enrichment, and dimensional modeling remains downstream. +## Path A — from existing Spark tables + ```python from tablespec import bootstrap_from_tables @@ -29,7 +31,7 @@ print(artifacts.manifest_path) print(artifacts.table("member").suite_json) ``` -What the facade does: +What Path A does: - reflects each table schema into UMF - when `profile=True`, profiles the table data natively and turns the profile @@ -50,3 +52,33 @@ and friends), and the rest of the pinned tree that a production job installs and loads from disk. When you only want the schema-only baseline suite, pass `profile=False`. + +## Path B — from authored UMF specs (no Spark) + +```python +from tablespec import bootstrap_from_specs + +artifacts = bootstrap_from_specs( + [ + "tests/e2e/fixtures/member.umf.yaml", + "tests/e2e/fixtures/claims.umf.yaml", + "tests/e2e/fixtures/claim_enriched.umf.yaml", + ], + out_dir="/tmp/tablespec-bootstrap-specs", + dialect="duckdb", + gold_targets=["claim_enriched"], +) + +print(artifacts.manifest_path) +``` + +Path B loads specs via `umfs_from_specs` and compiles the same artifact tree. +Use it for CI, local DuckDB/Spark-less compile checks, and the onboarding +benchmark (`docs/guide/onboarding-benchmark.md`). + +Demo scripts (also drive e2e tests): + +```bash +uv run python scripts/bootstrap_from_specs.py --spec --out +uv run python scripts/bootstrap_from_tables.py ... # requires Spark +``` diff --git a/docs/guide/databricks-e2e.md b/docs/guide/databricks-e2e.md new file mode 100644 index 0000000..1504153 --- /dev/null +++ b/docs/guide/databricks-e2e.md @@ -0,0 +1,47 @@ +# Databricks serverless e2e (opt-in) + +Vision KPI: multi-engine parity including Databricks serverless. + +Default CI and local `make test` **never** require a workspace. The real +serverless lane is an **opt-in** marker: + +```bash +# Without credentials: tests marked databricks_e2e SKIP with a precise reason +uv run pytest -m databricks_e2e -q + +# With a configured workspace (export secrets first): +export DATABRICKS_HOST=https:// +export DATABRICKS_HTTP_PATH=/sql/1.0/warehouses/ +export DATABRICKS_TOKEN= +# plus dbt-databricks + databricks-sdk + databricks-sql-connector installed +uv run pytest -m databricks_e2e -q +``` + +## Gate + +`tablespec.e2e.gating.databricks_e2e_availability()` returns: + +- a **skip reason string** when the tier must not run, or +- `None` when credentials + adapters look complete + +Required env (all three): + +| Variable | Role | +|----------|------| +| `DATABRICKS_HOST` | Opt-in switch | +| `DATABRICKS_HTTP_PATH` | SQL warehouse HTTP path | +| `DATABRICKS_TOKEN` | PAT / token | + +Unit gate (no workspace): + +```bash +uv run pytest tests/unit/test_databricks_e2e_gate.py -q +``` + +## What the tier proves + +When green against a real workspace: dbt/LDP deploy+run and read-back rows +match the Spark-oracle corpus through the shared canonicalizer +(`docs/helix/03-test/conformance-acceptance.md` §2.3). + +When skipped: **not a silent pass** — the skip reason names the missing piece. diff --git a/docs/guide/happy-path.md b/docs/guide/happy-path.md index 0a3b4f9..181f4a5 100644 --- a/docs/guide/happy-path.md +++ b/docs/guide/happy-path.md @@ -30,11 +30,12 @@ Current boundary notes: - Databricks-facing compile UX accepts `dialect="databricks"` for the Spark-family SQL emitted by tablespec; internal emitters may normalize to `spark` when the rendered SQL is identical. - The active bead trail is `tablespec-ed74497c` and child beads - `tablespec-0b146671`, `tablespec-171e409c`, and `tablespec-0fb0d1c2`. - Production runs consume the committed artifact tree and installed packages, not source-time orchestration. See the deployment checklist for the release boundary. +- One-shot facades: `bootstrap_from_tables` (Path A, Spark) and + `bootstrap_from_specs` (Path B, no Spark) are public first-class entry + points — see [bootstrap.md](bootstrap.md). ## 1. Generate UMF from existing Spark or Databricks tables @@ -63,17 +64,23 @@ If you do not need the intermediate `UMF` list, the convenience wrapper `bootstrap_from_tables(...)` reflects, profiles, compiles, and returns the manifest in one call. -For authored specs, use the Path B loader instead: +For authored specs, use the Path B loader or one-shot facade: ```python from tablespec.e2e import umfs_from_specs +from tablespec import bootstrap_from_specs umfs = umfs_from_specs([ "tables/member/table.yaml", "tables/claims/table.yaml", ]) +# or compile in one call (no Spark): +# artifacts = bootstrap_from_specs([...], out_dir="...") ``` +JDBC databases use the same compile contract after discovery — see +[jdbc-onboarding.md](jdbc-onboarding.md) and the Northwind demo/tests. + ## 2. Generate sample data from the UMF or spec inputs Sample data is generated from UMF/spec inputs, not from the compiled artifact diff --git a/docs/guide/jdbc-onboarding.md b/docs/guide/jdbc-onboarding.md new file mode 100644 index 0000000..28e8163 --- /dev/null +++ b/docs/guide/jdbc-onboarding.md @@ -0,0 +1,63 @@ +# JDBC onboarding (first-class source path) + +JDBC is a first-class FEAT-031 source kind: the same UMF → compile → runtime +contract as delimited, with **typed raw** landing via Spark's JDBC connector +(tablespec never opens a DB connection itself). + +## Composition + +```python +from tablespec.profiling import JdbcToUmfMapper # requires tablespec[spark] +from tablespec import bootstrap_from_specs # or compile_umfs after discovery +from tablespec.ingestion import get_reader + +# 1) Discover UMF from INFORMATION_SCHEMA (Spark JDBC) +mapper = JdbcToUmfMapper(spark, jdbc_url=..., options={...}) +umfs = mapper.discover(["dbo.customers", "dbo.orders"]) # shape may vary by mapper API + +# 2) Compile (no live DB required once UMF is in hand) +from tablespec.e2e.compile import compile_umfs +artifacts = compile_umfs(umfs, "/tmp/jdbc-out", source="tables", dialect="spark") + +# 3) Runtime land: Spark JDBC reader from the committed source: block +reader = get_reader(umf.effective_source()) +df = reader.read(umf.effective_source(), spark) +``` + +### Public Path B when specs are already authored + +If UMFs already declare `source: {kind: jdbc, ...}`: + +```python +from tablespec import bootstrap_from_specs + +artifacts = bootstrap_from_specs( + ["specs/customers", "specs/orders"], + out_dir="/tmp/jdbc-compile", + dialect="spark", +) +``` + +`password_secret_ref` (or env-style secret name) is required; plaintext +passwords are rejected by the model. + +## Demo / acceptance + +| Lane | What | Gate | +|------|------|------| +| Local | Docker SQL Server + Northwind fixture | `tests/integration/test_jdbc_discovery.py`, `test_northwind_e2e.py` (skip without Docker) | +| Workspace | `notebooks/northwind-demo/` | Databricks job (operator residual) | + +## Backbone note + +The e2e **backbone** file loader supports `delimited` / `parquet` / `json` +batch files for local multi-engine parity. JDBC tables are landed at runtime +with `JdbcReader` + Spark — not by reading a CSV batch. That is intentional: +JDBC onboarding is proven by discovery + compile + integration/Northwind, not +by the CSV-shaped backbone corpus. + +## Secret safety + +- Never put credentials in UMF or compiled artifacts. +- Use `password_secret_ref` naming a Databricks secret scope or env var. +- Discovery and readers fail closed when the secret is missing. diff --git a/docs/guide/onboarding-benchmark.md b/docs/guide/onboarding-benchmark.md new file mode 100644 index 0000000..d8d3ac5 --- /dev/null +++ b/docs/guide/onboarding-benchmark.md @@ -0,0 +1,56 @@ +# Onboarding authoring-reduction benchmark + +PRD success metric: **at least 50% lower** transform/validation authoring time +per onboarded table vs hand-authored baseline, measured on a **3-table** +onboarding sample. + +## Automated sample (tablespec path) + +The in-repo sample is the Path B e2e fixture set: + +| Table | Spec | +|-------|------| +| `member` | `tests/e2e/fixtures/member.umf.yaml` | +| `claims` | `tests/e2e/fixtures/claims.umf.yaml` | +| `claim_enriched` | `tests/e2e/fixtures/claim_enriched.umf.yaml` | + +### Run + +```bash +uv run python scripts/onboarding_benchmark.py --out /tmp/onboard-metrics +``` + +This records: + +- wall time for `umfs_from_specs` + `compile_umfs` (and optionally backbone) +- per-table artifact presence (ingest SQL, DDL, suite, dbt ingest, …) +- gold DAG / LDP project presence + +Output: `/tmp/onboard-metrics/onboarding_benchmark.json`. + +Unit gate (no Spark required): + +```bash +uv run pytest tests/unit/test_onboarding_benchmark.py -q +``` + +## Manual baseline protocol + +To compute reduction, time a **manual** 3-table onboarding of the same +semantics (member, claims, claim_enriched) *without* tablespec compile: + +1. Hand-write Spark DDL / dbt models / GX suite / LDP stubs for all three. +2. Record elapsed wall-clock minutes \(t_{manual}\). +3. Run the automated harness above; use `seconds.total_automated` as + \(t_{tablespec}\). +4. Reduction = \(1 - t_{tablespec}/(t_{manual}\times 60)\). + +The automated harness is the **reproducible numerator**. The denominator is +operator-measured once per release and stored under +`docs/helix/06-iterate/metrics/` if desired (optional). + +## Relation to the happy path + +This sample is the same Path B composition as [happy-path.md](happy-path.md) +and `scripts/bootstrap_from_specs.py`, scoped for metric capture rather than +demo narration. diff --git a/docs/helix/05-deploy/deployment-checklist.md b/docs/helix/05-deploy/deployment-checklist.md index b3d0b4e..eb015e3 100644 --- a/docs/helix/05-deploy/deployment-checklist.md +++ b/docs/helix/05-deploy/deployment-checklist.md @@ -111,10 +111,11 @@ in alignment beads (do not shrink the procedure to match incomplete tooling). ## app_deploy (FR-23 / FEAT-034) -Desired operating procedure for standing up `apps/data-profiling/` in a target -Databricks environment. Full automation of provision + startup fail-fast is the -implementation target; until beads close, operators may execute steps manually -but **must not** edit tracked application source for environment identity. +Operating procedure for standing up `apps/data-profiling/` in a target +Databricks environment. Config, provision, and startup fail-fast are +implemented; **agent-side smoke** (no workspace) is the CI gate. Live +deploy-and-drive remains operational residual. Operators **must not** edit +tracked application source for environment identity. ### App deploy inputs @@ -128,17 +129,26 @@ but **must not** edit tracked application source for environment identity. 1. Declare the metadata home and compute for the target environment as deployment inputs (never as literals in tracked app source). -2. Run (or re-run) the idempotent provision step: create/verify schema, volume, - and governance tables; report what was created; second run is a no-op. -3. Deploy the Databricks App artifact from `apps/data-profiling/` (or the - subdirectory package that satisfies the Apps file-count limit), installing +2. **Agent smoke (no workspace)** — must exit 0 before claiming FR-23 unit path: + ```bash + cd apps/data-profiling + PROFILER_RUNTIME=mock \ + PROFILER_METADATA_CATALOG=main \ + PROFILER_METADATA_SCHEMA=tablespec_profiler \ + uv run pytest tests/test_fr23_stack.py tests/test_config.py tests/test_provision.py tests/test_diagnostics.py -q + # or: PROFILER_RUNTIME=mock python scripts/fr23_smoke.py + ``` +3. Run (or re-run) the idempotent provision step against the target warehouse: + `python scripts/provision.py` (second run is a no-op). +4. Deploy the Databricks App artifact from `apps/data-profiling/`, installing `tablespec` as a dependency rather than shipping the full monorepo. -4. Start the app and confirm startup validation: missing warehouse, unreachable +5. Start the app and confirm startup validation: missing warehouse, unreachable location, or missing grant surfaces one actionable error naming the setting and the grant required. -5. Confirm the UI displays the resolved metadata location without opening source +6. Confirm the UI displays the resolved metadata location without opening source files. -6. Smoke: open guidebook/profile surfaces against the declared location. +7. Live smoke: open guidebook/profile surfaces against the declared location + (operational residual — not default CI). ### App success thresholds diff --git a/scripts/onboarding_benchmark.py b/scripts/onboarding_benchmark.py new file mode 100644 index 0000000..e5eb016 --- /dev/null +++ b/scripts/onboarding_benchmark.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python +"""3-table onboarding authoring-reduction benchmark (PRD success metric). + +Times the *automated* Path B workflow used as the tablespec side of the +50% manual-authoring reduction claim: + + load 3 UMF specs → compile_umfs → (optional) backbone green stages + +A hand-authored baseline is *not* executed here (that requires human +timing). This script records the automated path so the reduction formula: + + reduction = 1 - (t_tablespec / t_manual) + +has a reproducible automated numerator. The denominator protocol is +documented in ``docs/guide/onboarding-benchmark.md``. + +Usage:: + + uv run python scripts/onboarding_benchmark.py --out /tmp/onboard-metrics + +Exit 0 on success; writes ``onboarding_benchmark.json`` under ``--out``. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[1] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +# Default 3-table onboarding sample (same fixtures as Path B e2e). +DEFAULT_SPECS = [ + _REPO_ROOT / "tests/e2e/fixtures/member.umf.yaml", + _REPO_ROOT / "tests/e2e/fixtures/claims.umf.yaml", + _REPO_ROOT / "tests/e2e/fixtures/claim_enriched.umf.yaml", +] + + +def run_benchmark( + *, + specs: list[Path], + out_dir: Path, + dialect: str = "duckdb", + run_backbone: bool = False, +) -> dict: + """Run the automated onboarding path and return a metrics dict.""" + from tablespec.e2e.compile import compile_umfs + from tablespec.e2e.paths import umfs_from_specs + + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + t0 = time.perf_counter() + umfs = umfs_from_specs(specs) + t_load = time.perf_counter() - t0 + + t1 = time.perf_counter() + artifacts = compile_umfs( + umfs, + out_dir / "artifacts", + source="specs", + dialect=dialect, + gold_targets=["claim_enriched"], + ) + t_compile = time.perf_counter() - t1 + + tables = sorted(artifacts.tables.keys()) + per_table: dict[str, dict[str, bool]] = {} + for name in tables: + ta = artifacts.table(name) + per_table[name] = { + "umf_snapshot": ta.umf_snapshot.exists(), + "ingest_sql": ta.ingest_sql.exists(), + "ddl_sql": ta.ddl_sql.exists(), + "pyspark_schema": ta.pyspark_schema.exists(), + "json_schema": ta.json_schema.exists(), + "suite_json": ta.suite_json.exists(), + "dbt_ingest": ta.dbt_ingest_project is not None + and (ta.dbt_ingest_project / "dbt_project.yml").exists(), + } + + backbone_s: float | None = None + backbone_ok: bool | None = None + if run_backbone: + from tablespec.e2e.backbone import run_backbone + + t2 = time.perf_counter() + # Path B e2e supplies batches via fixture convention; optional here. + result = run_backbone(artifacts, backend="duckdb", batches={}) + backbone_s = time.perf_counter() - t2 + backbone_ok = result.ok if hasattr(result, "ok") else True + + total = t_load + t_compile + (backbone_s or 0.0) + metrics = { + "sample": "member + claims + claim_enriched (tests/e2e/fixtures)", + "table_count": len(tables), + "tables": tables, + "dialect": dialect, + "seconds": { + "load_specs": round(t_load, 4), + "compile_umfs": round(t_compile, 4), + "backbone": None if backbone_s is None else round(backbone_s, 4), + "total_automated": round(total, 4), + }, + "artifacts_present": per_table, + "manifest": str(artifacts.manifest_path), + "dbt_gold_project": artifacts.dbt_gold_project is not None, + "ldp_project": artifacts.ldp_project is not None, + "backbone_ok": backbone_ok, + "manual_baseline_protocol": ( + "docs/guide/onboarding-benchmark.md#manual-baseline-protocol" + ), + "reduction_formula": "1 - (seconds.total_automated / t_manual_minutes*60)", + } + + out_path = out_dir / "onboarding_benchmark.json" + out_path.write_text(json.dumps(metrics, indent=2) + "\n", encoding="utf-8") + metrics["metrics_path"] = str(out_path) + return metrics + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument( + "--out", + type=Path, + default=Path("/tmp/tablespec-onboarding-benchmark"), + help="Output directory for metrics + compiled artifacts", + ) + p.add_argument( + "--dialect", + default="duckdb", + help="Compile dialect (default duckdb for local runs)", + ) + p.add_argument( + "--backbone", + action="store_true", + help="Also time backbone (requires batches; experimental)", + ) + p.add_argument( + "--spec", + action="append", + type=Path, + dest="specs", + help="UMF spec path (repeatable); default is the 3-table e2e sample", + ) + args = p.parse_args(argv) + specs = args.specs or DEFAULT_SPECS + for s in specs: + if not s.exists(): + print(f"missing spec: {s}", file=sys.stderr) + return 2 + metrics = run_benchmark( + specs=list(specs), + out_dir=args.out, + dialect=args.dialect, + run_backbone=args.backbone, + ) + print(json.dumps(metrics, indent=2)) + print(f"\nwrote {metrics['metrics_path']}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/tablespec/__init__.py b/src/tablespec/__init__.py index d45b397..cb52584 100644 --- a/src/tablespec/__init__.py +++ b/src/tablespec/__init__.py @@ -75,7 +75,7 @@ CompatibilityReport, check_compatibility, ) -from tablespec.bootstrap import bootstrap_from_tables +from tablespec.bootstrap import bootstrap_from_specs, bootstrap_from_tables from tablespec.guidebook import generate as generate_guidebook from tablespec.excel_converter import ExcelToUMFConverter, UMFToExcelConverter from tablespec.inference.domain_types import DomainTypeInference, DomainTypeRegistry @@ -180,6 +180,7 @@ "CompatibilityReport", "CompatibilityIssue", "bootstrap_from_tables", + "bootstrap_from_specs", ] # SparkToUmfMapper and TableValidator are available only if pyspark is installed (via tablespec[spark]) diff --git a/src/tablespec/bootstrap.py b/src/tablespec/bootstrap.py index 80ac268..1fc3046 100644 --- a/src/tablespec/bootstrap.py +++ b/src/tablespec/bootstrap.py @@ -1,9 +1,11 @@ """Public one-shot bootstrap helpers. -These helpers reflect existing Spark tables into UMF, optionally profile the -data to derive GX expectations, and compile the full artifact tree in one call. +Path A — existing Spark tables → reflect (+ optional profile) → compile. +Path B — authored UMF specs → load → compile. -The native profiler enriches validation; it does not create UMF. +Both return a :class:`~tablespec.e2e.manifest.CompiledArtifacts` tree suitable +for review and for ``run_backbone``. The native profiler enriches validation; it +does not create UMF. """ from __future__ import annotations @@ -14,9 +16,9 @@ from tablespec.e2e.compile import compile_umfs from tablespec.e2e.manifest import CompiledArtifacts -from tablespec.e2e.paths import umfs_from_tables +from tablespec.e2e.paths import umfs_from_specs, umfs_from_tables -__all__ = ["bootstrap_from_tables"] +__all__ = ["bootstrap_from_tables", "bootstrap_from_specs"] InferKeysMode = Literal["none", "candidates", "auto"] @@ -33,7 +35,7 @@ def bootstrap_from_tables( key_promotion_min_score: float = 0.9, key_promotion_min_gap: float = 0.05, ) -> CompiledArtifacts: - """Bootstrap one or more existing Spark tables into compiled artifacts.""" + """Path A: bootstrap one or more existing Spark tables into compiled artifacts.""" tables = [table_names] if isinstance(table_names, str) else list(table_names) key_candidates: dict[str, list[dict[str, Any]]] = {} umfs, suites = umfs_from_tables( @@ -56,3 +58,30 @@ def bootstrap_from_tables( key_promotion_min_score=key_promotion_min_score, key_promotion_min_gap=key_promotion_min_gap, ) + + +def bootstrap_from_specs( + spec_paths: str | Path | Sequence[str | Path], + out_dir: str | Path, + *, + dialect: str = "duckdb", + gold_targets: Sequence[str] | None = None, +) -> CompiledArtifacts: + """Path B: bootstrap authored UMF specs into compiled artifacts (no Spark). + + Accepts split table directories, ``*.umf.yaml``, or JSON interchange paths — + anything :func:`~tablespec.e2e.paths.umfs_from_specs` loads. + """ + if isinstance(spec_paths, (str, Path)): + paths: list[str | Path] = [spec_paths] + else: + paths = list(spec_paths) + umfs = umfs_from_specs(paths) + return compile_umfs( + umfs, + out_dir, + source="specs", + profile_enriched=False, + dialect=dialect, + gold_targets=list(gold_targets) if gold_targets is not None else None, + ) diff --git a/tests/unit/test_bootstrap_from_specs_public.py b/tests/unit/test_bootstrap_from_specs_public.py new file mode 100644 index 0000000..66333d0 --- /dev/null +++ b/tests/unit/test_bootstrap_from_specs_public.py @@ -0,0 +1,27 @@ +"""Public Path B bootstrap facade (no Spark).""" + +from __future__ import annotations + +from pathlib import Path + +from tablespec import bootstrap_from_specs + +FIXTURES = Path(__file__).resolve().parents[1] / "e2e" / "fixtures" +SPECS = [ + FIXTURES / "member.umf.yaml", + FIXTURES / "claims.umf.yaml", + FIXTURES / "claim_enriched.umf.yaml", +] + + +def test_bootstrap_from_specs_public_api(tmp_path: Path) -> None: + artifacts = bootstrap_from_specs( + SPECS, + tmp_path / "out", + dialect="duckdb", + gold_targets=["claim_enriched"], + ) + assert artifacts.manifest_path.exists() + assert artifacts.source == "specs" + assert artifacts.table("member").ingest_sql.exists() + assert artifacts.table("claim_enriched").gold_plan_sql is not None diff --git a/tests/unit/test_databricks_e2e_gate.py b/tests/unit/test_databricks_e2e_gate.py new file mode 100644 index 0000000..517663e --- /dev/null +++ b/tests/unit/test_databricks_e2e_gate.py @@ -0,0 +1,33 @@ +"""Fail-closed opt-in gate for the real Databricks e2e tier.""" + +from __future__ import annotations + + +from tablespec.e2e.gating import ( + DATABRICKS_E2E_REQUIRED_ENV, + databricks_e2e_availability, +) + + +def test_databricks_e2e_gate_skips_without_host(monkeypatch) -> None: + monkeypatch.delenv("DATABRICKS_HOST", raising=False) + reason = databricks_e2e_availability() + assert reason is not None + assert "DATABRICKS_HOST not set" in reason + assert "opt-in" in reason + + +def test_databricks_e2e_gate_partial_config(monkeypatch) -> None: + monkeypatch.setenv("DATABRICKS_HOST", "https://example.cloud.databricks.com") + monkeypatch.delenv("DATABRICKS_HTTP_PATH", raising=False) + monkeypatch.delenv("DATABRICKS_TOKEN", raising=False) + reason = databricks_e2e_availability() + assert reason is not None + assert "partially configured" in reason + assert "DATABRICKS_HTTP_PATH" in reason or "DATABRICKS_TOKEN" in reason + + +def test_databricks_e2e_required_env_is_documented() -> None: + assert "DATABRICKS_HOST" in DATABRICKS_E2E_REQUIRED_ENV + assert "DATABRICKS_TOKEN" in DATABRICKS_E2E_REQUIRED_ENV + assert len(DATABRICKS_E2E_REQUIRED_ENV) >= 3 diff --git a/tests/unit/test_happy_path_guide.py b/tests/unit/test_happy_path_guide.py index 082aa0e..2b90e45 100644 --- a/tests/unit/test_happy_path_guide.py +++ b/tests/unit/test_happy_path_guide.py @@ -37,6 +37,7 @@ def test_happy_path_guide_is_ordered_and_indexed() -> None: required_strings = [ "bootstrap_from_tables", + "bootstrap_from_specs", "umfs_from_tables", "umfs_from_specs", "SampleDataGenerator", @@ -51,10 +52,7 @@ def test_happy_path_guide_is_ordered_and_indexed() -> None: "generate_dbt_dag_project", "generate_ldp_project", "run_backbone", - "tablespec-ed74497c", - "tablespec-0b146671", - "tablespec-171e409c", - "tablespec-0fb0d1c2", + 'dialect="databricks"', "pytest.main", "uv run pytest", ] diff --git a/tests/unit/test_jdbc_compile_path.py b/tests/unit/test_jdbc_compile_path.py new file mode 100644 index 0000000..ec981a4 --- /dev/null +++ b/tests/unit/test_jdbc_compile_path.py @@ -0,0 +1,80 @@ +"""JDBC source kind compiles first-class (no live DB required).""" + +from __future__ import annotations + +from pathlib import Path + +from tablespec.e2e.compile import compile_umfs +from tablespec.models.umf import UMF + + +def _jdbc_umf() -> UMF: + return UMF.model_validate( + { + "version": "1.0", + "table_name": "customers", + "source": { + "kind": "jdbc", + "url": "jdbc:sqlserver://localhost:1433;databaseName=Northwind", + "dbtable": "dbo.customers", + "password_secret_ref": "secrets/jdbc-password", + }, + "columns": [ + { + "name": "customer_id", + "data_type": "CHAR", + "max_length": 5, + "nullable": {"default": False}, + }, + { + "name": "company_name", + "data_type": "VARCHAR", + "max_length": 40, + "nullable": {"default": True}, + }, + ], + "primary_key": ["customer_id"], + } + ) + + +def test_jdbc_source_compiles_artifact_tree(tmp_path: Path) -> None: + arts = compile_umfs([_jdbc_umf()], tmp_path, source="specs", dialect="spark") + ta = arts.table("customers") + assert ta.ingest_sql.exists() + assert ta.suite_json.exists() + assert ta.umf_snapshot.exists() + snap = ta.umf_snapshot.read_text(encoding="utf-8") + assert "kind: jdbc" in snap or "kind:jdbc" in snap.replace(" ", "") + assert "password_secret_ref" in snap + assert "plaintext" not in snap.lower() + # Ingest SQL is generated (typed path); no embedded password. + sql = ta.ingest_sql.read_text(encoding="utf-8") + assert "ingested_customers" in sql or "raw_customers" in sql + assert "secrets/jdbc-password" not in sql # secret stays in UMF/source only + + +def test_jdbc_plaintext_password_rejected() -> None: + import pytest + from pydantic import ValidationError + + with pytest.raises(ValidationError): + UMF.model_validate( + { + "version": "1.0", + "table_name": "t", + "source": { + "kind": "jdbc", + "url": "jdbc:x", + "dbtable": "dbo.t", + "password": "s3cret", # forbidden + }, + "columns": [ + { + "name": "id", + "data_type": "INTEGER", + "nullable": {"default": False}, + } + ], + } + ) diff --git a/tests/unit/test_onboarding_benchmark.py b/tests/unit/test_onboarding_benchmark.py new file mode 100644 index 0000000..e9cb8dc --- /dev/null +++ b/tests/unit/test_onboarding_benchmark.py @@ -0,0 +1,34 @@ +"""Unit gate for the 3-table onboarding authoring-reduction benchmark.""" + +# @covers product-completeness onboarding metric harness + +from __future__ import annotations + +import json +from pathlib import Path + +from scripts.onboarding_benchmark import DEFAULT_SPECS, run_benchmark + + +def test_onboarding_benchmark_writes_metrics(tmp_path: Path) -> None: + metrics = run_benchmark( + specs=list(DEFAULT_SPECS), + out_dir=tmp_path, + dialect="duckdb", + run_backbone=False, + ) + + assert metrics["table_count"] == 3 + assert set(metrics["tables"]) == {"member", "claims", "claim_enriched"} + assert metrics["seconds"]["compile_umfs"] > 0 + assert metrics["seconds"]["total_automated"] > 0 + assert metrics["dbt_gold_project"] is True or metrics["ldp_project"] is not None + + path = Path(metrics["metrics_path"]) + assert path.exists() + loaded = json.loads(path.read_text(encoding="utf-8")) + assert loaded["sample"].startswith("member") + for name in ("member", "claims", "claim_enriched"): + arts = loaded["artifacts_present"][name] + assert arts["ingest_sql"] + assert arts["suite_json"]