From 76389964969589a9e91177bb746fe465528f88b9 Mon Sep 17 00:00:00 2001 From: Erik LaBianca Date: Wed, 22 Jul 2026 18:07:55 -0400 Subject: [PATCH 1/3] =?UTF-8?q?feat(helix):=20drain=20alignment=20queue=20?= =?UTF-8?q?=E2=80=94=20story=20floor,=20JSON=20backbone,=20FR-23=20closeou?= =?UTF-8?q?t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Close the HELIX desired-state alignment epic against main after the spec/bead surgery landed: - Close shipped FEAT-034 beads (config, provision, manifest, diagnostics, CFG-02, optional degrade) with evidence from PR #10. - Author story-floor US-040/042/043/050 with ACs citing existing tests. - Implement JSON kind on the e2e backbone (Spark + DuckDB load paths). - Add FR-23 unit whole-stack composition test; update e2e-framework slot. - Link US-046 ACs via @covers; evidence/waive residual US-045 ACs. - Update FEAT-031/032, PRD FR-21.7, feature registry, and test plan. --- .ddx/beads.jsonl | 30 +++---- apps/data-profiling/tests/test_fr23_stack.py | 57 ++++++++++++ docs/helix/01-frame/concerns.md | 2 +- docs/helix/01-frame/feature-registry.md | 4 +- .../FEAT-031-multi-source-ingestion.md | 39 ++++---- ...FEAT-032-embeddings-and-document-corpus.md | 2 +- docs/helix/01-frame/prd.md | 4 +- .../US-040-source-model-and-ingestion-seam.md | 75 ++++++++++++++++ .../US-042-dump-dialect-text-landing.md | 59 ++++++++++++ .../US-043-parquet-typed-raw-landing.md | 67 ++++++++++++++ .../US-045-sec-10k-corpus-and-facts.md | 76 +++++++++------- .../US-046-browse-schema-guidebook.md | 12 ++- .../US-050-json-jsonl-source-kind.md | 81 +++++++++++++++++ docs/helix/03-test/test-plan.md | 11 +-- src/tablespec/e2e/backbone.py | 89 +++++++++++++++---- tests/unit/test_guidebook_generate.py | 13 ++- tests/unit/test_guidebook_renderer.py | 2 + tests/unit/test_ingestion_package.py | 16 ++++ 18 files changed, 538 insertions(+), 101 deletions(-) create mode 100644 apps/data-profiling/tests/test_fr23_stack.py create mode 100644 docs/helix/01-frame/user-stories/US-040-source-model-and-ingestion-seam.md create mode 100644 docs/helix/01-frame/user-stories/US-042-dump-dialect-text-landing.md create mode 100644 docs/helix/01-frame/user-stories/US-043-parquet-typed-raw-landing.md create mode 100644 docs/helix/01-frame/user-stories/US-050-json-jsonl-source-kind.md diff --git a/.ddx/beads.jsonl b/.ddx/beads.jsonl index b5cfbda8..11c5b2db 100644 --- a/.ddx/beads.jsonl +++ b/.ddx/beads.jsonl @@ -24,15 +24,15 @@ {"acceptance":"US-039 ACs evidenced with command and exit status, or residual waived with recorded limitation.","created_at":"2026-07-22T21:41:08.254312981Z","description":"Close remaining US-039 ACs on real Databricks if still open beyond Docker lane.","id":"tablespec-0aa41072","issue_type":"task","labels":["helix","phase:test","area:data","FEAT-031"],"notes":"CLOSED 2026-07-22 queue surgery: US-039-AC1..6 all [x]; @covers in tests/integration/test_northwind_e2e.py; story status records Docker+workspace PASS. No residual.","parent":"tablespec-263a0248","priority":2,"schema_version":1,"status":"closed","title":"Northwind Databricks acceptance residual (US-039)","updated_at":"2026-07-22T21:47:08.923313908Z"} {"acceptance":"1. TestDatabricksDialectAliasAtCastLayer: in tests/unit/test_casting_utils.py, assert cast_column_sql(..., dialect=\"databricks\") emits byte-identical SQL to dialect=\"spark\" for INTEGER, DECIMAL, DATE with format, TIMESTAMP without format, BOOLEAN, EPOCH_MS_FORMAT, and EXCEL_SERIAL_FORMAT.\n2. TestBuildIngestSelectAcceptsDatabricksAlias: in tests/unit/test_ingest_generator.py, assert build_ingest_select(..., dialect=\"databricks\") succeeds, stores the accepted public dialect or documented normalized value, and emits the same Spark-family select_block as dialect=\"spark\".\n3. TestSharedDialectErrorMessages: add tests covering cast_column_sql, build_ingest_select, and dbt contract/profile helpers so unsupported values all report the same accepted values: spark, databricks, duckdb where the field is a cast dialect, and duckdb, spark, databricks where the field is a dbt profile target.\n4. Test gates: uv run pytest tests/unit/test_casting_utils.py tests/unit/test_ingest_generator.py tests/dbt_roadmap/test_contracts_functional.py; go test ./...; lefthook run pre-commit.","closing_commit_sha":"44fe34dc8ce6ed6a0a14a2db1258a9312e43af61","created_at":"2026-06-08T01:04:03.676003826Z","description":"PROBLEM: Databricks is advertised as a dialect in parts of the repo, but accepted dialect values and error messages are owned independently by casting, ingest, dbt contracts, and dbt profile code. That leaves agents and users vulnerable to stale or contradictory validators.\nROOT CAUSE: src/tablespec/casting_utils.py:627 accepts ('spark','databricks','duckdb') directly, src/tablespec/schemas/ingest_generator.py:90 and src/tablespec/schemas/ingest_generator.py:148 still document only spark/duckdb, src/tablespec/schemas/ingest_generator.py:162 and src/tablespec/schemas/ingest_generator.py:164 pass the raw value through without a shared normalizer, src/tablespec/dbt/contracts.py:88 has its own dialect guard, and src/tablespec/dbt/profiles.py:44 has a separate target guard.\nPROPOSED FIX: Add or use a shared dialect helper that defines accepted cast dialects, normalizes databricks to the Spark-family rendering path, and formats accepted-value error messages. Wire casting_utils, ingest_generator, and dbt contract/profile validation through it while preserving databricks as an explicit public spelling.\nNON-SCOPE: One-shot bootstrap facade implementation. Production pipeline JSON/wheel packaging contract. Inline UMF format cleanup. No change to cast SQL semantics beyond alias normalization and error text consistency.\nGOVERNING ARTIFACTS: docs/helix/02-design/adr/ADR-007-raw-to-ingest-sql-artifact.md; docs/helix/03-test/conformance-acceptance.md:109; .agents/skills/tablespec/SKILL.md Dialects section.\nDEPENDENCIES: none inherited from parent.\nVERIFICATION: Add focused unit tests and run uv/pytest, go test, and lefthook gates.","events_attachment":"tablespec-0b146671/events.jsonl","id":"tablespec-0b146671","issue_type":"task","labels":["area:databricks","area:dialects","area:docs","kind:bug","ac-quality:needs-refinement"],"parent":"tablespec-ed74497c","priority":0,"schema_version":1,"session_id":"eb-96f2578b","status":"closed","title":"Normalize Databricks as a shared Spark-family cast dialect alias","updated_at":"2026-06-08T06:46:35.676819963Z"} {"acceptance":"1. TestCliHelpExplainsDatabricksAlias: add a CLI help regression that verifies `--dialect` advertises `duckdb`, `spark`, and `databricks`, and explains that `databricks` is the Databricks-facing alias for Spark-family cast SQL.\\n2. TestCliValidationKeepsCanonicalValues: add a parser/validation regression that the accepted values remain stable and the help text does not imply a separate Databricks SQL dialect implementation.\\n3. go test ./...\\n4. lefthook run pre-commit.","closing_commit_sha":"4a0ab6555054ed52cb219761004efd1e60a7c47f","created_at":"2026-06-08T14:55:59.620585689Z","description":"PROBLEM: The CLI help text does not explain that Databricks uses the Spark-family dialect alias, so users cannot tell whether `databricks` is accepted or how it relates to `spark`. ROOT CAUSE: `src/tablespec/cli.py:525` lists `duckdb, spark, databricks` without describing the Spark-family normalization or alias semantics. PROPOSED FIX: Update the `--dialect` help/validation text to name the three accepted values, explicitly state that `databricks` is the Databricks-facing alias for Spark-family cast SQL, and keep the runtime behavior unchanged. NON-SCOPE: One-shot bootstrap facade implementation. Production pipeline JSON/wheel packaging contract. Inline UMF format cleanup. No broad documentation rewrite outside Databricks dialect guidance. GOVERNING ARTIFACTS: `docs/guide/bootstrap.md:10`; `README.md:139`; `docs/helix/03-test/conformance-acceptance.md:109`; `scripts/run_integration_tests_databricks.ipynb`. DEPENDENCIES: `tablespec-ed74497c`.","events_attachment":"tablespec-0d64cd5d/events.jsonl","id":"tablespec-0d64cd5d","issue_type":"task","labels":["area:cli","area:databricks","area:dialects","kind:bug","ac-quality:needs-refinement"],"parent":"tablespec-0fb0d1c2","priority":0,"schema_version":1,"session_id":"eb-ec05ad24","status":"closed","title":"Clarify Databricks alias in CLI dialect help","updated_at":"2026-06-08T19:24:05.557906118Z"} -{"acceptance":"At least one test for missing warehouse or unreachable location with actionable message.","created_at":"2026-07-22T21:41:07.232277639Z","dependencies":[{"issue_id":"tablespec-0e6cd069","depends_on_id":"tablespec-2a68a0ad","type":"blocks","created_at":"2026-07-22T21:41:07Z"}],"description":"Missing or unusable configuration surfaces one actionable startup error naming setting and grant.","id":"tablespec-0e6cd069","issue_type":"task","labels":["helix","phase:build","area:app","FEAT-034"],"parent":"tablespec-263a0248","priority":1,"schema_version":1,"status":"open","title":"App: startup fail-fast configuration validation (FR-23.6 / US-049)","updated_at":"2026-07-22T21:41:07.232277639Z"} +{"acceptance":"At least one test for missing warehouse or unreachable location with actionable message.","closing_commit_sha":"6857ad0da3e8839e3fb40c6fe806ee7f4060e562","created_at":"2026-07-22T21:41:07.232277639Z","dependencies":[{"issue_id":"tablespec-0e6cd069","depends_on_id":"tablespec-2a68a0ad","type":"blocks","created_at":"2026-07-22T21:41:07Z"}],"description":"Missing or unusable configuration surfaces one actionable startup error naming setting and grant.","events_attachment":"tablespec-0e6cd069/events.jsonl","id":"tablespec-0e6cd069","issue_type":"task","labels":["helix","phase:build","area:app","FEAT-034"],"notes":"[2026-07-22T22:02:15Z] closure rejected: closure gate: insufficient evidence: no execution evidence (empty events and no closing_commit_sha)","owner":"erik","parent":"tablespec-263a0248","priority":1,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"App: startup fail-fast configuration validation (FR-23.6 / US-049)","updated_at":"2026-07-22T22:03:01.262348031Z"} {"acceptance":"1. TestDatabricksDocsUseCanonicalDialectGuidance: add a docs traceability test that scans docs/guide/bootstrap.md, README.md, Databricks notebook/try-it-out docs, and selected docs/helix files to verify they state dialect=\"databricks\" is accepted for Databricks compile UX and explain that casts render as Spark-family SQL.\n2. TestCliHelpExplainsDatabricksAlias: add a Typer CLI help test proving --dialect help names duckdb, spark, databricks and includes the Spark-family alias relationship or points to consistent accepted values.\n3. TestNoPublicDocsContradictDatabricksAlias: assert public docs no longer say Databricks users must use only dialect=\"spark\" unless the sentence explicitly explains internal Spark-family normalization.\n4. Test gates: uv run pytest tests/unit/test_api_docs_traceability.py tests/unit/test_cli_domains_set.py; go test ./...; lefthook run pre-commit.","created_at":"2026-06-08T01:04:03.709588757Z","description":"PROBLEM: Public instructions are inconsistent. The CLI advertises databricks as a cast dialect, Databricks conformance docs discuss dialect=\"databricks\", but bootstrap docs show dialect=\"spark\" or omit the relationship, leaving users unsure which spelling is correct.\nROOT CAUSE: src/tablespec/cli.py:525 says \"duckdb, spark, databricks\" without explaining Spark-family normalization. docs/guide/bootstrap.md:18 tells Databricks/Spark bootstrap users to pass dialect=\"spark\". README.md:144-149 shows bootstrap_from_tables without a dialect. docs/helix/03-test/conformance-acceptance.md:85 and docs/helix/03-test/conformance-acceptance.md:109 already describe databricks dialect behavior, but that decision is not reflected consistently in user-facing bootstrap/try-it-out docs.\nPROPOSED FIX: Update CLI help, docs/guide/bootstrap.md, README.md, Databricks notebook/try-it-out docs, and relevant docs/helix references to one canonical instruction: public Databricks compile UX accepts dialect=\"databricks\" as an explicit alias for Spark-family cast SQL, and internal emitters may normalize to spark when SQL is identical. Align all examples and error-message wording with that decision.\nNON-SCOPE: One-shot bootstrap facade implementation. Production pipeline JSON/wheel packaging contract. Inline UMF format cleanup. No broad documentation rewrite outside Databricks dialect guidance.\nGOVERNING ARTIFACTS: docs/guide/bootstrap.md:10; README.md:139; docs/helix/03-test/conformance-acceptance.md:109; scripts/run_integration_tests_databricks.ipynb.\nDEPENDENCIES: none inherited from parent.\nVERIFICATION: Add docs/CLI tests and run uv/pytest, go test, and lefthook gates.","events_attachment":"tablespec-0fb0d1c2/events.jsonl","execution-eligible":false,"id":"tablespec-0fb0d1c2","issue_type":"task","labels":["area:databricks","area:dialects","area:docs","kind:bug","ac-quality:needs-refinement"],"parent":"tablespec-ed74497c","priority":0,"schema_version":1,"status":"closed","title":"Canonicalize Databricks dialect guidance in CLI help and docs","updated_at":"2026-06-08T20:18:39.059178973Z"} -{"acceptance":"Test with optional keys unset: app starts; optional UI sections absent.","created_at":"2026-07-22T21:41:07.325508086Z","description":"Unset dashboard/Genie/UMF volume hides dependent surfaces rather than erroring.","id":"tablespec-112c6f19","issue_type":"task","labels":["helix","phase:build","area:app","FEAT-034"],"parent":"tablespec-263a0248","priority":2,"schema_version":1,"status":"open","title":"App: optional integrations degrade cleanly (FR-23.5)","updated_at":"2026-07-22T21:41:07.325508086Z"} +{"acceptance":"Test with optional keys unset: app starts; optional UI sections absent.","closing_commit_sha":"6857ad0da3e8839e3fb40c6fe806ee7f4060e562","created_at":"2026-07-22T21:41:07.325508086Z","description":"Unset dashboard/Genie/UMF volume hides dependent surfaces rather than erroring.","events_attachment":"tablespec-112c6f19/events.jsonl","id":"tablespec-112c6f19","issue_type":"task","labels":["helix","phase:build","area:app","FEAT-034"],"notes":"[2026-07-22T22:02:15Z] closure rejected: closure gate: insufficient evidence: no execution evidence (empty events and no closing_commit_sha)","owner":"erik","parent":"tablespec-263a0248","priority":2,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"App: optional integrations degrade cleanly (FR-23.5)","updated_at":"2026-07-22T22:03:02.828042332Z"} {"acceptance":"1. TestCompileUmfsAcceptsDatabricksDialect: add a compile-path test proving compile_umfs(..., dialect=\"databricks\") completes without ValueError and persists ingest, dbt_ingest, optional dbt_gold, LDP, and manifest artifacts under the pinned layout.\n2. TestGenerateLdpProjectAcceptsDatabricksAlias: add an LDP unit/conformance test proving generate_ldp_project(..., dialect=\"databricks\") succeeds and emits the same Spark-family cast body as dialect=\"spark\".\n3. TestDatabricksCompileArtifactsStateNormalizationDecision: assert the implementation notes or generated metadata/comments state the decision: public dialect=\"databricks\" is accepted and Spark-family SQL emitters may normalize to spark internally because cast SQL is identical.\n4. Test gates: uv run pytest tests/unit/test_bootstrap.py tests/e2e/test_bootstrap_from_specs.py tests/ldp/test_ldp_emitter.py tests/ldp/test_ldp_cast_parity.py; go test ./...; lefthook run pre-commit.","created_at":"2026-06-08T01:04:03.685235027Z","description":"PROBLEM: The documented compile/bootstrap path can receive dialect=databricks, but not every emitted artifact path proves that spelling end-to-end. Some compile seams hard-code spark or document only spark/duckdb, so a downstream validator can still reject databricks or silently bypass the requested dialect.\nROOT CAUSE: src/tablespec/e2e/compile.py:129 forwards the caller dialect to generate_dbt_dag_project, but src/tablespec/e2e/compile.py:141 hard-codes generate_ldp_project(..., dialect=\"spark\"). src/tablespec/e2e/compile.py:193 calls generate_ingest_sql without a dialect, and src/tablespec/schemas/ingest_generator.py:214 hard-codes build_ingest_select(..., dialect=\"spark\"). src/tablespec/ldp/project.py:242 documents LDP dialect as Spark with duckdb accepted for parity, but omits databricks even though LDP runs on Databricks.\nPROPOSED FIX: Make compile_umfs accept dialect=\"databricks\" across all relevant compile seams. Preserve dbt target behavior as databricks, and normalize Spark-family SQL emitters internally where SQL is identical. Either add a dialect argument to generate_ingest_sql or explicitly document/direct-test that direct ingest artifacts are Spark-family and stable under databricks compile. Update LDP generation to accept databricks as an alias for Spark-family casts.\nNON-SCOPE: One-shot bootstrap facade implementation. Production pipeline JSON/wheel packaging contract. Inline UMF format cleanup. No real Databricks workspace execution beyond existing opt-in tiers.\nGOVERNING ARTIFACTS: docs/helix/01-frame/features/FEAT-026-compile-orchestrator-bootstrap.md:84; docs/helix/03-test/conformance-acceptance.md:49; docs/helix/03-test/conformance-acceptance.md:85; docs/helix/03-test/conformance-acceptance.md:109.\nDEPENDENCIES: none inherited from parent.\nVERIFICATION: Add compile/emitter tests and run uv/pytest, go test, and lefthook gates.","events":[{"actor":"erik","body":"{\"decision\":\"warn\",\"decision_source\":\"pre_claim_intake\",\"detail\":\"readiness route unavailable: lifecycle dispatch: snapshot project root dirtiness: exit status 128\",\"fingerprint\":\"65124584ad3f1c6e5900c0aafa8ce291c4e983f47acc8269b667040304b48266\",\"policy_mode\":\"warn-only\",\"reason\":\"system_unready\",\"rule_id\":\"pre_claim_intake.system_unready\",\"suggested_action\":\"revise the rewrite so it preserves every explicit commitment\",\"warning_fingerprint\":\"65124584ad3f1c6e5900c0aafa8ce291c4e983f47acc8269b667040304b48266\"}","created_at":"2026-06-08T01:58:10.980939598Z","kind":"intake.warn","source":"ddx work","summary":"system_unready"},{"actor":"ddx","body":"{\"actual_harness\":\"\",\"actual_model\":\"\",\"actual_power\":0,\"actual_provider\":\"\",\"attempt_id\":\"\",\"bead_id\":\"tablespec-171e409c\",\"estimated_difficulty\":\"\",\"rejected_route_pins\":null,\"requested_power_class\":\"standard\",\"requested_profile\":\"\",\"routing_intent_degraded\":true,\"routing_intent_note\":\"actual route facts unavailable\",\"routing_intent_source\":\"default\"}","created_at":"2026-06-08T01:58:11.481417322Z","kind":"execution-routing-intent","source":"ddx work","summary":"source=default powerClass=standard note=actual route facts unavailable"},{"actor":"erik","body":"{\"base_rev\":\"\",\"classification\":\"recoverable\",\"detail\":\"pre-execute-bead checkpoint: listing checkpoint dirt: exit status 128\",\"rationale\":\"The attempt failed before implementation during the pre-execute checkpoint while listing worktree/checkpoint dirtiness with git exit status 128. Recent intake evidence also reports system_unready from project root dirtiness snapshot failure, so this is infrastructure/preflight failure rather than a bead defect or code regression.\",\"recommended_action\":\"release_claim_retry\",\"result_rev\":\"\",\"session_id\":\"\",\"status\":\"execution_failed\",\"suggested_amendments\":[],\"suggested_followup_beads\":[]}","created_at":"2026-06-08T01:58:41.59268296Z","kind":"bead-quality.triage","source":"ddx work","summary":"recoverable: release_claim_retry"},{"actor":"erik","body":"pre-execute-bead checkpoint: listing checkpoint dirt: exit status 128\noutcome_reason=recoverable","created_at":"2026-06-08T01:58:41.620854787Z","kind":"execute-bead","source":"ddx work","summary":"execution_failed"},{"actor":"erik","body":"{\"decision\":\"warn\",\"decision_source\":\"pre_claim_intake\",\"detail\":\"readiness route unavailable: lifecycle dispatch: snapshot project root dirtiness: exit status 128\",\"fingerprint\":\"65124584ad3f1c6e5900c0aafa8ce291c4e983f47acc8269b667040304b48266\",\"policy_mode\":\"warn-only\",\"reason\":\"system_unready\",\"rule_id\":\"pre_claim_intake.system_unready\",\"suggested_action\":\"revise the rewrite so it preserves every explicit commitment\",\"warning_fingerprint\":\"65124584ad3f1c6e5900c0aafa8ce291c4e983f47acc8269b667040304b48266\"}","created_at":"2026-06-08T02:02:40.533739039Z","kind":"intake.warn","source":"ddx work","summary":"system_unready"},{"actor":"ddx","body":"{\"actual_harness\":\"\",\"actual_model\":\"\",\"actual_power\":0,\"actual_provider\":\"\",\"attempt_id\":\"\",\"bead_id\":\"tablespec-171e409c\",\"estimated_difficulty\":\"\",\"rejected_route_pins\":null,\"requested_power_class\":\"standard\",\"requested_profile\":\"\",\"routing_intent_degraded\":true,\"routing_intent_note\":\"actual route facts unavailable\",\"routing_intent_source\":\"default\"}","created_at":"2026-06-08T02:02:40.925914769Z","kind":"execution-routing-intent","source":"ddx work","summary":"source=default powerClass=standard note=actual route facts unavailable"},{"actor":"erik","body":"{\"base_rev\":\"\",\"classification\":\"transport\",\"detail\":\"pre-execute-bead checkpoint: listing checkpoint dirt: exit status 128\",\"rationale\":\"The bead failed during the pre-execute checkpoint with 'listing checkpoint dirt: exit status 128', which indicates a transient infrastructure/git state issue rather than a problem with the bead specification or code implementation. Recent intake events also report system_unready from project root dirtiness snapshot failure, confirming this is a transport/preflight failure. The bead quality triage already classified this as recoverable with release_claim_retry.\",\"recommended_action\":\"release_claim_retry\",\"result_rev\":\"\",\"session_id\":\"\",\"status\":\"execution_failed\",\"suggested_amendments\":[],\"suggested_followup_beads\":[]}","created_at":"2026-06-08T02:02:55.960124675Z","kind":"bead-quality.triage","source":"ddx work","summary":"transport: release_claim_retry"},{"actor":"erik","body":"pre-execute-bead checkpoint: listing checkpoint dirt: exit status 128\noutcome_reason=transport","created_at":"2026-06-08T02:02:55.986844748Z","kind":"execute-bead","source":"ddx work","summary":"execution_failed"},{"actor":"erik","body":"{\"decision\":\"warn\",\"decision_source\":\"pre_claim_intake\",\"detail\":\"readiness route unavailable: lifecycle dispatch: snapshot project root dirtiness: exit status 128\",\"fingerprint\":\"65124584ad3f1c6e5900c0aafa8ce291c4e983f47acc8269b667040304b48266\",\"policy_mode\":\"warn-only\",\"reason\":\"system_unready\",\"rule_id\":\"pre_claim_intake.system_unready\",\"suggested_action\":\"revise the rewrite so it preserves every explicit commitment\",\"warning_fingerprint\":\"65124584ad3f1c6e5900c0aafa8ce291c4e983f47acc8269b667040304b48266\"}","created_at":"2026-06-08T02:05:39.013526634Z","kind":"intake.warn","source":"ddx work","summary":"system_unready"},{"actor":"ddx","body":"{\"actual_harness\":\"\",\"actual_model\":\"\",\"actual_power\":0,\"actual_provider\":\"\",\"attempt_id\":\"\",\"bead_id\":\"tablespec-171e409c\",\"estimated_difficulty\":\"\",\"rejected_route_pins\":null,\"requested_power_class\":\"standard\",\"requested_profile\":\"\",\"routing_intent_degraded\":true,\"routing_intent_note\":\"actual route facts unavailable\",\"routing_intent_source\":\"default\"}","created_at":"2026-06-08T02:05:39.402200949Z","kind":"execution-routing-intent","source":"ddx work","summary":"source=default powerClass=standard note=actual route facts unavailable"},{"actor":"erik","body":"{\"base_rev\":\"\",\"classification\":\"transport\",\"detail\":\"pre-execute-bead checkpoint: listing checkpoint dirt: exit status 128\",\"rationale\":\"The attempt failed before implementation during the pre-execute checkpoint while listing checkpoint dirtiness with git exit status 128. Recent events repeatedly show project-root dirtiness snapshot failure/system_unready, so the narrow cause is a transport/preflight infrastructure failure rather than a bead defect, implementation regression, or external blocker.\",\"recommended_action\":\"release_claim_retry\",\"result_rev\":\"\",\"session_id\":\"\",\"status\":\"execution_failed\",\"suggested_amendments\":[],\"suggested_followup_beads\":[]}","created_at":"2026-06-08T02:05:57.526300135Z","kind":"bead-quality.triage","source":"ddx work","summary":"transport: release_claim_retry"},{"actor":"erik","body":"pre-execute-bead checkpoint: listing checkpoint dirt: exit status 128\noutcome_reason=transport","created_at":"2026-06-08T02:05:57.551086Z","kind":"execute-bead","source":"ddx work","summary":"execution_failed"},{"actor":"erik","body":"{\"decision\":\"warn\",\"decision_source\":\"pre_claim_intake\",\"detail\":\"system_unready: This is a single docs/CLI guidance task already split out from the broader Databricks dialect parent. The acceptance criteria are coherent and locally testable, the scope stays inside user-facing docs/help plus traceability tests, and the prior failures are transport/preflight noise rather than a specification blocker.\",\"fingerprint\":\"7a5c99fdc017101fd5cf4f50e8cfca4d003d6ef65585a9b8e791eed1f63e8cce\",\"policy_mode\":\"warn-only\",\"reason\":\"system_unready\",\"rule_id\":\"pre_claim_intake.system_unready\",\"suggested_action\":\"revise the rewrite so it preserves every explicit commitment\",\"warning_fingerprint\":\"7a5c99fdc017101fd5cf4f50e8cfca4d003d6ef65585a9b8e791eed1f63e8cce\"}","created_at":"2026-06-08T06:49:55.960658623Z","kind":"intake.warn","source":"ddx work","summary":"system_unready"},{"actor":"ddx","body":"{\"resolved_provider\":\"codex\",\"resolved_model\":\"gpt-5.4-mini\",\"route_reason\":\"explicit_pin_discovery_unavailable\",\"fallback_chain\":[]}","created_at":"2026-06-08T07:31:38.222311385Z","kind":"routing","source":"legacy agent execute-bead","summary":"provider=codex model=gpt-5.4-mini reason=explicit_pin_discovery_unavailable"},{"actor":"ddx","body":"{\"actual_harness\":\"\",\"actual_model\":\"\",\"actual_power\":0,\"actual_provider\":\"\",\"attempt_id\":\"\",\"bead_id\":\"tablespec-171e409c\",\"estimated_difficulty\":\"\",\"rejected_route_pins\":null,\"requested_power_class\":\"standard\",\"requested_profile\":\"\",\"routing_intent_degraded\":true,\"routing_intent_note\":\"actual route facts unavailable\",\"routing_intent_source\":\"default\"}","created_at":"2026-06-08T07:31:48.555302669Z","kind":"execution-routing-intent","source":"ddx work","summary":"source=default powerClass=standard note=actual route facts unavailable"},{"actor":"erik","body":"{\"base_rev\":\"\",\"classification\":\"recoverable\",\"detail\":\"pre-execute-bead checkpoint: listing checkpoint dirt: exit status 128\",\"rationale\":\"The attempt failed before implementation during the pre-execute checkpoint while listing checkpoint dirtiness with git exit status 128. The evidence points to a repository/preflight infrastructure failure, not a bead defect, code regression, external blocker, or verification failure.\",\"recommended_action\":\"release_claim_retry\",\"result_rev\":\"\",\"session_id\":\"\",\"status\":\"execution_failed\",\"suggested_amendments\":[],\"suggested_followup_beads\":[]}","created_at":"2026-06-08T07:32:29.424327714Z","kind":"bead-quality.triage","source":"ddx work","summary":"recoverable: release_claim_retry"},{"actor":"erik","body":"pre-execute-bead checkpoint: listing checkpoint dirt: exit status 128\noutcome_reason=recoverable","created_at":"2026-06-08T07:32:29.474566211Z","kind":"execute-bead","source":"ddx work","summary":"execution_failed"},{"actor":"erik","body":"{\"decision\":\"warn\",\"decision_source\":\"pre_claim_intake\",\"detail\":\"readiness route unavailable: lifecycle dispatch: snapshot project root dirtiness: exit status 128\",\"fingerprint\":\"65124584ad3f1c6e5900c0aafa8ce291c4e983f47acc8269b667040304b48266\",\"policy_mode\":\"warn-only\",\"reason\":\"system_unready\",\"rule_id\":\"pre_claim_intake.system_unready\",\"suggested_action\":\"revise the rewrite so it preserves every explicit commitment\",\"warning_fingerprint\":\"65124584ad3f1c6e5900c0aafa8ce291c4e983f47acc8269b667040304b48266\"}","created_at":"2026-06-08T07:32:31.656628774Z","kind":"intake.warn","source":"ddx work","summary":"system_unready"},{"actor":"ddx","body":"{\"actual_harness\":\"\",\"actual_model\":\"\",\"actual_power\":0,\"actual_provider\":\"\",\"attempt_id\":\"\",\"bead_id\":\"tablespec-171e409c\",\"estimated_difficulty\":\"\",\"rejected_route_pins\":null,\"requested_power_class\":\"standard\",\"requested_profile\":\"\",\"routing_intent_degraded\":true,\"routing_intent_note\":\"actual route facts unavailable\",\"routing_intent_source\":\"default\"}","created_at":"2026-06-08T07:34:34.721706915Z","kind":"execution-routing-intent","source":"ddx work","summary":"source=default powerClass=standard note=actual route facts unavailable"},{"actor":"erik","body":"{\"base_rev\":\"\",\"classification\":\"recoverable\",\"detail\":\"pre-execute-bead checkpoint: listing checkpoint dirt: exit status 128\",\"rationale\":\"The attempt failed before implementation during the pre-execute checkpoint while listing checkpoint dirtiness with git exit status 128. This is repository/preflight infrastructure failure evidence, not a bead defect, code regression, external blocker, or verification failure; the same failure pattern was already triaged as recoverable.\",\"recommended_action\":\"release_claim_retry\",\"result_rev\":\"\",\"session_id\":\"\",\"status\":\"execution_failed\",\"suggested_amendments\":[],\"suggested_followup_beads\":[]}","created_at":"2026-06-08T07:34:58.21981555Z","kind":"bead-quality.triage","source":"ddx work","summary":"recoverable: release_claim_retry"},{"actor":"erik","body":"pre-execute-bead checkpoint: listing checkpoint dirt: exit status 128\noutcome_reason=recoverable","created_at":"2026-06-08T07:34:58.237998617Z","kind":"execute-bead","source":"ddx work","summary":"execution_failed"},{"actor":"erik","body":"{\"ac_map\":[{\"parent_ac\":\"1. TestCompileUmfsAcceptsDatabricksDialect: add a compile-path test proving compile_umfs(..., dialect=\\\"databricks\\\") completes without ValueError and persists ingest, dbt_ingest, optional dbt_gold, LDP, and manifest artifacts under the pinned layout.\",\"coverage\":\"Child 1 AC1 TestCompileUmfsAcceptsDatabricksDialect; Child 1 AC2 TestCompileUmfsPreservesPublicDialectInManifest\"},{\"parent_ac\":\"2. TestGenerateLdpProjectAcceptsDatabricksAlias: add an LDP unit/conformance test proving generate_ldp_project(..., dialect=\\\"databricks\\\") succeeds and emits the same Spark-family cast body as dialect=\\\"spark\\\".\",\"coverage\":\"Child 2 AC1 TestGenerateLdpProjectAcceptsDatabricksAlias; Child 2 AC2 TestGenerateLdpProjectSparkParityRemainsStable\"},{\"parent_ac\":\"3. TestDatabricksCompileArtifactsStateNormalizationDecision: assert the implementation notes or generated metadata/comments state the decision: public dialect=\\\"databricks\\\" is accepted and Spark-family SQL emitters may normalize to spark internally because cast SQL is identical.\",\"coverage\":\"Child 3 AC1 TestDatabricksCompileArtifactsStateNormalizationDecision; Child 3 AC2 TestDatabricksDialectContractIsDocumented\"},{\"parent_ac\":\"4. Test gates: uv run pytest tests/unit/test_bootstrap.py tests/e2e/test_bootstrap_from_specs.py tests/ldp/test_ldp_emitter.py tests/ldp/test_ldp_cast_parity.py; go test ./...; lefthook run pre-commit.\",\"coverage\":\"Child 1 AC3 TestPytestGate; Child 1 AC4 TestGoGate; Child 1 AC5 TestLefthookGate; Child 2 AC3 TestPytestGate; Child 2 AC4 TestGoGate; Child 2 AC5 TestLefthookGate; Child 3 AC3 TestPytestGate; Child 3 AC4 TestGoGate; Child 3 AC5 TestLefthookGate\"}],\"child_ids\":[\"tablespec-924c7384\",\"tablespec-bc47a14e\",\"tablespec-943b21a5\"],\"rationale\":\"Split the bead into three independently executable slices: compile-path normalization, direct LDP aliasing, and contract documentation. This preserves the governing artifacts, dependency identity, and required verification while reducing overlap between code changes and letting each child close with targeted tests plus the shared go/lefthook gates.\"}","created_at":"2026-06-08T14:50:25.00456072Z","kind":"triage-decomposed","source":"ddx work","summary":"decomposed into tablespec-924c7384, tablespec-bc47a14e, tablespec-943b21a5"}],"execution-eligible":false,"id":"tablespec-171e409c","issue_type":"task","labels":["area:databricks","area:dialects","area:docs","kind:bug","ac-quality:needs-refinement"],"parent":"tablespec-ed74497c","priority":0,"schema_version":1,"status":"open","title":"Thread dialect=databricks through compile and LDP emitter paths","updated_at":"2026-06-08T14:50:25.19072127Z"} {"acceptance":"US-041 file exists with stable AC IDs; linked from FEAT-031.","created_at":"2026-07-22T21:41:07.602898557Z","description":"Author US-041 covering JDBC reader and discovery under US-039 goal.","id":"tablespec-1af0828a","issue_type":"task","labels":["helix","phase:frame","area:data","FEAT-031"],"notes":"CLOSED 2026-07-22 queue surgery: US-041 would duplicate US-039 JDBC discovery+onboarding (AC1-6 covered). FEAT-031 no longer requires separate US-041.","parent":"tablespec-263a0248","priority":1,"schema_version":1,"status":"closed","title":"Author US-041: JDBC reader + discovery slices","updated_at":"2026-07-22T21:47:10.78311423Z"} -{"acceptance":"US-040 file exists with stable US-040-ACn IDs; each AC cites an existing test path or is marked covered by named test; FEAT-031 user-stories section links US-040; no new product code required unless a citation gap forces a tiny @covers add.","created_at":"2026-07-22T21:41:07.514181186Z","description":"Frame-only backfill. SRC seam and multi-kind source model are SHIPPED (models/umf.py + ingestion/). Author US-040 with US-040-ACn Given/When/Then mapped to existing unit tests (test_source_spec, test_ingestion_package). Do NOT re-implement the seam.","id":"tablespec-20513f4f","issue_type":"task","labels":["helix","phase:frame","area:data","FEAT-031"],"notes":"Queue surgery 2026-07-22: re-scoped from build to story floor.","parent":"tablespec-263a0248","priority":1,"schema_version":1,"status":"open","title":"Author US-040: source model + ingestion seam AC backfill (story floor only)","updated_at":"2026-07-22T21:47:07.842593094Z"} -{"acceptance":"B1 app FR-23 beads complete or actively blocked only on true deps; JSON residual closed or waived with evidence; story-floor beads closed; tests/docs green; no reopened closed-as-shipped beads without new failing evidence.","created_at":"2026-07-22T21:40:40.237053896Z","description":"Close code/ops gaps against governing HELIX specs after AR-2026-07-22. Specs describe desired future state. Queue surgery 2026-07-22 pruned false residuals (DUMP/PARQ/US-039/US-041/US-038/Trace Links/CORP example). Remaining primary build: B1 app FR-23 path + JSON backbone residual + story floor US-040/042/043/050 + US-045 residual evidence + US-046 @covers.","id":"tablespec-263a0248","issue_type":"epic","labels":["helix","phase:align","area:specs"],"notes":"Queue surgery 2026-07-22: pruned 7 children as shipped/duplicate/cosmetic.","priority":1,"schema_version":1,"status":"open","title":"HELIX desired-state alignment — implementation gaps","updated_at":"2026-07-22T21:47:08.617499454Z"} +{"acceptance":"US-040 file exists with stable US-040-ACn IDs; each AC cites an existing test path or is marked covered by named test; FEAT-031 user-stories section links US-040; no new product code required unless a citation gap forces a tiny @covers add.","created_at":"2026-07-22T21:41:07.514181186Z","description":"Frame-only backfill. SRC seam and multi-kind source model are SHIPPED (models/umf.py + ingestion/). Author US-040 with US-040-ACn Given/When/Then mapped to existing unit tests (test_source_spec, test_ingestion_package). Do NOT re-implement the seam.","events_attachment":"tablespec-20513f4f/events.jsonl","id":"tablespec-20513f4f","issue_type":"task","labels":["helix","phase:frame","area:data","FEAT-031"],"notes":"Queue surgery 2026-07-22: re-scoped from build to story floor.","owner":"erik","parent":"tablespec-263a0248","priority":1,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"Author US-040: source model + ingestion seam AC backfill (story floor only)","updated_at":"2026-07-22T22:07:10.349061386Z"} +{"acceptance":"B1 app FR-23 beads complete or actively blocked only on true deps; JSON residual closed or waived with evidence; story-floor beads closed; tests/docs green; no reopened closed-as-shipped beads without new failing evidence.","created_at":"2026-07-22T21:40:40.237053896Z","description":"Close code/ops gaps against governing HELIX specs after AR-2026-07-22. Specs describe desired future state. Queue surgery 2026-07-22 pruned false residuals (DUMP/PARQ/US-039/US-041/US-038/Trace Links/CORP example). Remaining primary build: B1 app FR-23 path + JSON backbone residual + story floor US-040/042/043/050 + US-045 residual evidence + US-046 @covers.","events":[{"actor":"","body":"closed_because: all_children_terminal\ntotal_children: 21","created_at":"2026-07-22T22:07:16.402671078Z","kind":"epic_auto_close","source":"Store.Close.walkUpClosureCandidate","summary":"auto-closed: all children reached terminal state"}],"id":"tablespec-263a0248","issue_type":"epic","labels":["helix","phase:align","area:specs"],"notes":"Queue surgery 2026-07-22: pruned 7 children as shipped/duplicate/cosmetic.","priority":1,"schema_version":1,"status":"closed","title":"HELIX desired-state alignment — implementation gaps","updated_at":"2026-07-22T22:07:16.406583784Z"} {"acceptance":"1. test_native_profiler_single_column_exact_key_candidate: a fixture with \u003e=100 rows, or an explicit key_min_rows override, produces exactly one verified_exact primary_key_candidate for non-null unique id, with every documented evidence field present and score in [0, 1].\n2. test_native_profiler_nullable_unique_is_not_pk: a nullable unique column produces no primary_key_candidate; it is represented as unique_constraint_candidate or advisory evidence with nullable=true.\n3. test_native_profiler_composite_minimal_key: member_id + effective_date exact uniqueness yields a composite candidate only when neither subset is exact unique.\n4. test_native_profiler_rejects_nonminimal_composite: if id is exact unique, id + date is rejected or marked nonminimal and is not emitted as a candidate.\n5. test_native_profiler_respects_candidate_and_pass_budget: a wide-table fixture exact-verifies no more than key_verification_pass_budget candidates and leaves skipped candidates verified_exact=false without suppressing legacy uniqueness expectations.\n6. test_profile_to_gx_no_duplicate_uniqueness_when_infer_keys_enabled: a verified exact single-column candidate prevents a duplicate approximate uniqueness expectation for that same column.\n7. test_profile_to_gx_legacy_uniqueness_unchanged_when_infer_keys_disabled: existing approximate uniqueness behavior remains byte/structurally unchanged when inference is disabled.\n8. Connect/classic parity test asserts serialized key_candidates structural equality on deterministic fixtures.\n9. uv run pytest tests/unit/test_native_profiler_key_candidates.py tests/unit/test_profiler_connect_sail.py tests/unit -k gx_expectation passes.","created_at":"2026-06-08T04:19:09.628002551Z","description":"PROBLEM: NativeSparkProfiler does not infer primary-key or unique-key candidates. ProfileToGxMapper currently emits approximate single-column uniqueness expectations from approximate distinct ratios, but that is not primary-key detection and can duplicate future inferred-key expectations. Users expect bootstrap/profile enrichment to surface likely keys while keeping authoritative UMF.primary_key and unique_constraints separate.\n\nROOT CAUSE: src/tablespec/profiling/types.py DataFrameProfile has only num_records and columns. src/tablespec/profiling/native_profiler.py computes per-column statistics but no key candidate model, scoring, exact verification, or minimality checks. src/tablespec/profiling/gx_expectation_builder.py has a separate approximate uniqueness threshold and no concept of verified key candidates.\n\nPROPOSED FIX: Add typed, JSON-serializable KeyCandidate and KeyCandidateEvidence models to the profiling layer. Add optional key-candidate inference to NativeSparkProfiler with defaults that preserve existing behavior unless enabled. Candidate inference must be advisory: it returns key_candidates on DataFrameProfile but does not mutate UMF. Inference must use cheap existing profile signals first, deterministically score/truncate candidates, then exact-verify only the bounded set with Connect-safe Spark-SQL/DataFrame operations.\n\nRequired behavior:\n- Candidate kinds: primary_key_candidate and unique_constraint_candidate.\n- Evidence fields: row_count, columns, null_count_by_column, exact_distinct_count, approximate_distinct_count_by_column, distinct_ratio, completeness_by_column, verified_exact, nullable, minimal, subset_unique, score, score_components, name_hints, type_hints, penalties, verification_pass_count, verification_query_count, reason.\n- Scoring is deterministic, score is in [0, 1], and exposed thresholds include key_promotion_min_score and key_promotion_min_gap for downstream use.\n- Ordering is total and stable: score descending, key width ascending, then column-name tuple ascending.\n- key_min_rows default prevents noisy inference on tiny tables; tests must either use at least 100 rows or override key_min_rows explicitly and assert candidates are non-empty.\n- Minimality is honest: minimal=True only when all proper subsets were exact-verified as non-unique; otherwise use false/unknown semantics in the evidence rather than guessing.\n- Composite search is bounded by key_max_width and key_max_candidates; no exhaustive all-column combinatorics.\n- Exact verification must not use RDDs or classic-only SparkContext APIs.\n- ProfileToGxMapper deduplicates legacy approximate uniqueness only for verified_exact=True single-column candidates covering the same column. Unverified or budget-skipped candidates must not suppress existing approximate uniqueness behavior.\n\nIn-scope files:\n- src/tablespec/profiling/types.py\n- src/tablespec/profiling/native_profiler.py\n- src/tablespec/profiling/gx_expectation_builder.py\n- tests/unit/test_native_profiler_key_candidates.py or equivalent focused tests\n- tests/unit/test_profiler_connect_sail.py if parity coverage belongs there\n\nOut-of-scope for this bead:\n- Writing sidecar files or extending manifest layout; tracked by FEAT-026 child.\n- Mutating UMF.primary_key or unique_constraints.\n- Making sample data, dbt, or LDP consume inferred candidates.\n- Implementing compound uniqueness execution in the validator; tracked by FEAT-007 child.","events_attachment":"tablespec-29002b40/events.jsonl","execution-eligible":false,"id":"tablespec-29002b40","issue_type":"task","labels":["area:profiling","area:validation","kind:feature","ac-quality:needs-refinement"],"parent":"tablespec-cc4f665b","priority":1,"schema_version":1,"spec-id":"FEAT-024","status":"closed","title":"Infer advisory key candidates in NativeSparkProfiler","updated_at":"2026-06-09T20:32:39.036602248Z"} -{"acceptance":"Unit tests cover each precedence tier; no production path reads ad-hoc env keys outside the resolver.","created_at":"2026-07-22T21:41:06.841434626Z","description":"Implement one configuration object for apps/data-profiling resolving deployment env vars then connections.yaml then built-in defaults per ADR-019.","id":"tablespec-2a68a0ad","issue_type":"task","labels":["helix","phase:build","area:app","FEAT-034"],"parent":"tablespec-263a0248","priority":1,"schema_version":1,"status":"open","title":"App: single config precedence resolver (FR-23.1 / CFG-01)","updated_at":"2026-07-22T21:41:06.841434626Z"} +{"acceptance":"Unit tests cover each precedence tier; no production path reads ad-hoc env keys outside the resolver.","closing_commit_sha":"6857ad0da3e8839e3fb40c6fe806ee7f4060e562","created_at":"2026-07-22T21:41:06.841434626Z","description":"Implement one configuration object for apps/data-profiling resolving deployment env vars then connections.yaml then built-in defaults per ADR-019.","events_attachment":"tablespec-2a68a0ad/events.jsonl","id":"tablespec-2a68a0ad","issue_type":"task","labels":["helix","phase:build","area:app","FEAT-034"],"notes":"[2026-07-22T22:02:13Z] closure rejected: closure gate: insufficient evidence: no execution evidence (empty events and no closing_commit_sha)","owner":"erik","parent":"tablespec-263a0248","priority":1,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"App: single config precedence resolver (FR-23.1 / CFG-01)","updated_at":"2026-07-22T22:02:48.429677103Z"} {"acceptance":"1. test_bootstrap_candidates_write_sidecar_without_umf_mutation: infer_keys=\"candidates\" writes validation/\u003ctable\u003e.keycandidates.json, records the sidecar in manifest/provenance with a relative path, and the persisted UMF snapshot has no inferred primary_key or unique_constraints mutation.\n2. test_keycandidate_sidecar_json_schema: sidecar evidence validates against the documented JSON schema and contains deterministic sorted candidates.\n3. test_bootstrap_auto_conflict_does_not_promote: two exact plausible verified candidates whose scores tie or fail key_promotion_min_gap produce sidecar candidates but do not mutate UMF.primary_key.\n4. test_bootstrap_auto_single_clear_key_promotes_with_metadata: one exact, non-null, minimal candidate with score \u003e= key_promotion_min_score promotes only in infer_keys=\"auto\" and records promotion evidence plus downstream-effect metadata.\n5. test_candidate_sidecar_deterministic_across_runs: two runs over identical input produce byte-identical validation/\u003ctable\u003e.keycandidates.json and stable manifest entries.\n6. test_candidate_mode_does_not_change_generated_runtime_artifacts: candidate mode leaves sample/dbt/LDP-affecting UMF fields unchanged, so generated runtime artifacts match infer_keys=\"none\" except for the sidecar and manifest/provenance entries.\n7. uv run pytest tests/e2e/test_bootstrap_key_candidates.py tests/e2e/test_bootstrap_from_tables.py tests/e2e/test_bootstrap_from_specs.py passes.","closing_commit_sha":"409f4f1442747a2c0b0cd78a64c803e4be4553f3","created_at":"2026-06-08T04:19:32.437923415Z","dependencies":[{"issue_id":"tablespec-2bd32b44","depends_on_id":"tablespec-29002b40","type":"blocks","created_at":"2026-06-08T04:19:32Z"}],"description":"PROBLEM: Once profiler key candidates exist, bootstrap/compile needs a deterministic artifact contract for them. Candidate mode must be useful to users and agents without silently changing UMF snapshots, sample data, dbt, or LDP behavior. Auto-promotion must be explicit because writing UMF.primary_key changes downstream behavior.\n\nROOT CAUSE: FEAT-026 compile currently persists UMF snapshots, validation suites, runtime artifacts, and manifest.json. It has no validation/\u003ctable\u003e.keycandidates.json sidecar, no manifest slot for advisory key evidence, no bootstrap infer_keys mode, and no provenance for any inferred-key promotion.\n\nPROPOSED FIX: Extend the public bootstrap/compile path to support infer_keys modes: none, candidates, auto. Candidate mode writes advisory sidecar artifacts under validation/\u003ctable\u003e.keycandidates.json and records them in manifest/provenance without mutating persisted UMF snapshots. Auto mode may promote to UMF.primary_key only under a strict deterministic policy and must record promotion evidence.\n\nRequired behavior:\n- infer_keys defaults must preserve existing behavior unless product/spec explicitly chooses candidates by default.\n- Candidate sidecar JSON is deterministic and validates against the documented KeyCandidateEvidence schema from the profiler child.\n- Manifest/provenance records sidecar paths relative to root and remains relocatable.\n- Candidate mode never mutates UMF.primary_key or unique_constraints in persisted UMF snapshots.\n- Auto mode promotes only if exactly one candidate is verified_exact=true, non-null, minimal, score \u003e= key_promotion_min_score, and wins by at least key_promotion_min_gap under total deterministic ordering. Ties or conflicts produce sidecars only.\n- Promotion metadata must plainly state downstream effects: sample data uniqueness tracking may change, dbt incremental MERGE unique_key may change, and LDP APPLY CHANGES KEYS may change. For composite keys, metadata must also state any dbt unique-test asymmetry if composite uniqueness is not emitted as a dbt uniqueness test.\n- Candidate sidecars are advisory to sample data/dbt/LDP unless auto mode writes authoritative UMF fields.\n- Re-running compile/bootstrap over the same input writes byte-identical candidate sidecars and manifest entries.\n\nIn-scope files:\n- src/tablespec/bootstrap.py\n- src/tablespec/e2e/paths.py if table bootstrap owns profiler construction\n- src/tablespec/e2e/compile.py\n- src/tablespec/e2e/manifest.py\n- docs/guide/bootstrap.md or focused bootstrap docs if public mode documentation is needed\n- tests/e2e/test_bootstrap_key_candidates.py or equivalent focused tests\n\nOut-of-scope for this bead:\n- Implementing profiler candidate inference; depends on tablespec-29002b40.\n- Implementing native compound uniqueness validation execution; tracked by FEAT-007 child.\n- Making production runtime consume sidecars. Production still runs installed JSON pipeline artifacts and wheels, not authoring inference logic.","id":"tablespec-2bd32b44","issue_type":"task","labels":["area:bootstrap","area:compile","area:manifest","kind:feature"],"owner":"erik","parent":"tablespec-cc4f665b","priority":1,"schema_version":1,"spec-id":"FEAT-026","status":"closed","title":"Persist key-candidate sidecars and gated bootstrap auto-promotion","updated_at":"2026-06-09T20:38:50.448962098Z"} {"acceptance":"Demo job green on DBR 17.3 LTS (Spark 4) with mssql-server-2025; still green on 16.4; README documents both pairings.","created_at":"2026-06-11T15:45:56.545479913Z","description":"2026-06-11: notebooks/northwind-demo pins DBR 16.4 LTS because mssql-server-2022 packages support Ubuntu \u003c=22.04 while DBR 17+ is Ubuntu 24.04. Make 01-provision detect the host Ubuntu version and select the matching mssql-server release (22.04 -\u003e 2022, 24.04 -\u003e 2025) so the demo runs on DBR 17.3 LTS — which is Spark 4.0, matching tablespec's local engine matrix (Spark 4.0/JDK 21). Update README cluster requirements accordingly; validate with a real run on 17.3.","id":"tablespec-2daf281d","issue_type":"task","labels":["area:demo","feat-031","databricks"],"owner":"erik","priority":3,"resolution":"DONE 2026-06-11: notebook 01 is version-aware (Ubuntu 22.04-\u003eSQL 2022, 24.04-\u003eSQL 2025). Proven green on workspace dbw-dev-eus2 BOTH pairings: DBR 17.3 LTS/Spark 4 + SQL Server 2025 (run 419590905072679) and DBR 16.4 LTS + SQL Server 2022 (run 1039374257789724); demo exit PASS/13 tables on both.","schema_version":1,"status":"closed","title":"Northwind demo: auto-select mssql release by host Ubuntu; unlock DBR 17.3 LTS / Spark 4","updated_at":"2026-06-11T16:30:49.547241906Z"} {"acceptance":"PRD success metrics and feature NFRs have concrete numeric targets or an explicit rationale for qualitative treatment; acceptance criteria that imply parity, drift, performance, or reduction include measurable thresholds and evidence commands; helix_align_check --strict remains clean.","closing_commit_sha":"515bae1c23a225bd9f5214445436e726963ab6eb","created_at":"2026-06-07T20:59:46.816586955Z","description":"The structural checker flags measurable NFRs as model-required, and the PRD contains targets with weak or missing numeric thresholds, especially manual-authoring reduction. Many user-story ACs are Boolean behavior checks but do not state quantitative thresholds, budgets, or evidence standards where the requirement implies performance, drift, parity, or reduction.","id":"tablespec-340da854","issue_type":"task","labels":["spec","helix","quality"],"owner":"erik","priority":2,"schema_version":1,"status":"closed","title":"Quantify NFR and acceptance evidence targets","updated_at":"2026-06-09T21:00:27.642123021Z"} @@ -42,14 +42,14 @@ {"acceptance":"validation-sync on a valid split spec round-trips loadably; meta_* columns keep UMF type names; a regression test loads the spec after sync; nonzero exit on any write that fails validation.","created_at":"2026-06-12T02:16:38.556346153Z","description":"Site rewrite verification 2026-06-12: sync_baseline.METADATA_COLUMN_DEFINITIONS uses PySpark type names (StringType, TimestampType) that violate the UMF data_type pattern; non-dry-run validation-sync rewrites valid meta_* columns into an unloadable state and still exits 0 reporting 'Already in sync'. Silent spec corruption — the failure class the product exists to prevent. Site docs demonstrate only --dry-run until fixed.","id":"tablespec-3e72dc77","issue_type":"bug","labels":["kind:bug","area:validation","severity:high"],"owner":"erik","priority":1,"schema_version":1,"status":"closed","title":"validation-sync writes PySpark type names into UMF, corrupting split specs (exit 0)","updated_at":"2026-06-12T16:38:07.767057213Z"} {"acceptance":"1. TestDatabricksDialectContractIsDocumented: add or update a docs/metadata regression test that fails if the accepted dialect list regresses back to spark/duckdb-only wording and omits public dialect=\"databricks\".\n2. TestPytestGate: `uv run pytest tests/docs/test_dialect_contract.py tests/unit/test_bootstrap.py`\n3. TestGoGate: `go test ./...`\n4. TestLefthookGate: `lefthook run pre-commit`","created_at":"2026-06-08T18:22:28.902935417Z","description":"PROBLEM: The public dialect contract can still be read as spark/duckdb-only, so reviewers and validators may incorrectly reject dialect=\"databricks\" even though the implementation accepts it.\nROOT CAUSE: docs/helix/03-test/conformance-acceptance.md:49, docs/helix/03-test/conformance-acceptance.md:85, and docs/helix/03-test/conformance-acceptance.md:109 are governing acceptance artifacts whose dialect wording does not clearly preserve databricks as an accepted public dialect.\nPROPOSED FIX: Update the conformance/documentation contract and its regression coverage so accepted dialect wording includes databricks alongside spark and duckdb, and fails if it regresses to spark/duckdb-only language.\nNON-SCOPE: One-shot bootstrap facade implementation. Production pipeline JSON/wheel packaging contract. Inline UMF format cleanup. No real Databricks workspace execution beyond existing opt-in tiers. GOVERNING ARTIFACTS: docs/helix/01-frame/features/FEAT-026-compile-orchestrator-bootstrap.md:84; docs/helix/03-test/conformance-acceptance.md:49; docs/helix/03-test/conformance-acceptance.md:85; docs/helix/03-test/conformance-acceptance.md:109. DEPENDENCIES: parent bead tablespec-ed74497c; no additional dependencies.","events":[{"actor":"ddx","body":"{\"resolved_provider\":\"sindri\",\"resolved_model\":\"dflash\",\"route_reason\":\"policy=cheap; score=35.0\",\"fallback_chain\":[]}","created_at":"2026-06-08T20:29:19.486042934Z","kind":"routing","source":"legacy agent execute-bead","summary":"provider=sindri model=dflash reason=policy=cheap; score=35.0"},{"actor":"ddx","body":"{\"attempt_id\":\"20260608T202242-40596bf9\",\"harness\":\"fiz\",\"provider\":\"sindri\",\"model\":\"dflash\",\"route_reason\":\"policy=cheap; score=35.0\",\"input_tokens\":34564,\"output_tokens\":585,\"total_tokens\":35149,\"cost_usd\":0,\"duration_ms\":394944,\"exit_code\":1}","created_at":"2026-06-08T20:29:19.502178422Z","kind":"cost","source":"legacy agent execute-bead","summary":"tokens=35149 model=dflash"},{"actor":"ddx","body":"{\"actual_harness\":\"fiz\",\"actual_model\":\"dflash\",\"actual_power\":0,\"actual_provider\":\"sindri\",\"attempt_id\":\"20260608T202242-40596bf9\",\"bead_id\":\"tablespec-3ee3b466\",\"estimated_difficulty\":\"easy\",\"rejected_route_pins\":null,\"requested_power_class\":\"cheap\",\"requested_profile\":\"cheap\",\"routing_intent_degraded\":false,\"routing_intent_note\":\"\",\"routing_intent_source\":\"readiness\"}","created_at":"2026-06-08T20:29:20.889717265Z","kind":"execution-routing-intent","source":"ddx work","summary":"source=readiness difficulty=easy powerClass=cheap model=dflash harness=fiz"},{"actor":"erik","body":"{\"base_rev\":\"96de4c5e2742d4966eef8d4c03c1efea7707dc75\",\"classification\":\"routing\",\"detail\":\"agent: provider error: openai: POST \\\"http://sindri:8080/v1/chat/completions\\\": 400 Bad Request {\\\"message\\\":\\\"prompt + max_tokens exceeds context window\\\",\\\"type\\\":\\\"invalid_request_error\\\"}\",\"rationale\":\"Attempt failed before implementation because the selected cheap route provider/model dflash could not fit the prompt plus max_tokens in its context window; base_rev and result_rev are unchanged, so this is a model capability/routing mismatch rather than a bead defect.\",\"recommended_action\":\"release_claim_retry\",\"result_rev\":\"96de4c5e2742d4966eef8d4c03c1efea7707dc75\",\"session_id\":\"eb-7f7183d2\",\"status\":\"execution_failed\",\"suggested_amendments\":[],\"suggested_followup_beads\":[]}","created_at":"2026-06-08T20:29:49.896488397Z","kind":"bead-quality.triage","source":"ddx work","summary":"routing: release_claim_retry"},{"actor":"erik","body":"agent: provider error: openai: POST \"http://sindri:8080/v1/chat/completions\": 400 Bad Request {\"message\":\"prompt + max_tokens exceeds context window\",\"type\":\"invalid_request_error\"}\npowerClass=cheap\ndecision_audit={\"failure_class\":\"routing\",\"retry_action\":\"retry\",\"escalation_count\":0,\"requested_route\":{\"harness\":\"fiz\",\"provider\":\"sindri\",\"model\":\"dflash\",\"profile\":\"cheap\",\"routing_intent_source\":\"readiness\",\"estimated_difficulty\":\"easy\",\"inferred_power_class\":\"cheap\",\"requested_power_class\":\"cheap\"},\"actual_route\":{\"harness\":\"fiz\",\"provider\":\"sindri\",\"model\":\"dflash\",\"route_reason\":\"readiness\"},\"review_status\":\"skipped\",\"review_skip_reason\":\"review_not_configured\",\"review_classification\":\"review_skipped\",\"land_status\":\"not_landed\",\"reconcile_status\":\"not_applicable\"}\nresult_rev=96de4c5e2742d4966eef8d4c03c1efea7707dc75\nbase_rev=96de4c5e2742d4966eef8d4c03c1efea7707dc75\noutcome_reason=routing","created_at":"2026-06-08T20:29:49.904784016Z","kind":"execute-bead","source":"ddx work","summary":"execution_failed"},{"actor":"erik","body":"{\"ac_map\":[{\"parent_ac\":\"1. TestDatabricksDialectContractIsDocumented: add or update a docs/metadata regression test that fails if the accepted dialect list regresses back to spark/duckdb-only wording and omits public dialect=\\\"databricks\\\".\",\"coverage\":\"child 1 AC1 TestConformanceAcceptedDialectListIncludesDatabricks; child 1 AC2 TestConformanceAcceptedDialectListRejectsSparkDuckdbOnly; child 2 AC1 TestDatabricksOptInTierPreservesPublicDialect; child 2 AC2 TestDatabricksLocalCoverageDoesNotDemoteDialect\"},{\"parent_ac\":\"2. TestPytestGate: `uv run pytest tests/docs/test_dialect_contract.py tests/unit/test_bootstrap.py`\",\"coverage\":\"child 1 AC3 TestPytestGate; child 2 AC3 TestPytestGate\"},{\"parent_ac\":\"3. TestGoGate: `go test ./...`\",\"coverage\":\"child 1 AC4 TestGoGate; child 2 AC4 TestGoGate\"},{\"parent_ac\":\"4. TestLefthookGate: `lefthook run pre-commit`\",\"coverage\":\"child 1 AC5 TestLefthookGate; child 2 AC5 TestLefthookGate\"}],\"child_ids\":[\"tablespec-7d18b645\",\"tablespec-37db161f\"],\"rationale\":\"Split by independently reviewable contract surfaces: one child pins the accepted public dialect list and regression guard, while the other pins the Databricks opt-in/local-coverage boundary so it cannot be misread as removing public dialect=\\\"databricks\\\". Both children preserve the parent non-scope, governing artifacts, dependency IDs, and verification gates.\"}","created_at":"2026-06-08T20:34:59.152349424Z","kind":"triage-decomposed","source":"ddx work","summary":"decomposed into tablespec-7d18b645, tablespec-37db161f"}],"execution-eligible":false,"id":"tablespec-3ee3b466","issue_type":"task","labels":["area:databricks","area:dialects","area:docs","kind:bug","ac-quality:refined"],"parent":"tablespec-943b21a5","priority":0,"schema_version":1,"status":"open","title":"Document databricks as an accepted public dialect","updated_at":"2026-06-08T20:34:59.314003388Z"} {"acceptance":"Either Contract artifacts exist under docs/helix/02-design/ and PRD/US reference them by ID, or a documented waiver exists in docs/helix/01-frame/principles.md or concerns.md naming the no_surface_leak checks as waived and why.","created_at":"2026-06-11T00:23:21.359873657Z","description":"Spec audit 2026-06-10: zero Contract artifacts exist in docs/helix, yet the PRD and user-story meta.yml blocking checks (no_surface_leak) require exact CLI/API/SQL surface to live in Contract artifacts, not inline. PRD FRs name exact CLI commands/signatures (prd.md FR-8.x, FR-16.x); US-001/010/025/026 ACs define exact API/CLI/SQL/config surface inline. helix v0.6.1 SKILL.md now defines the Contract artifact lane (02-design/contract). Either (a) extract the normative surface into Contract artifacts and reference Contract IDs from PRD/US, or (b) record a project policy waiver for no_surface_leak.","id":"tablespec-4323cd4c","issue_type":"task","labels":["helix","spec","decision"],"priority":1,"resolution":"DECIDED 2026-06-10 (operator): ratchet. Existing backfilled specs waived; new/changed surfaces require a Contract artifact under docs/helix/02-design/, CLI contract first when the CLI next changes. Recorded in principles.md §Tension Resolution.","schema_version":1,"status":"closed","title":"DECISION: adopt Contract artifacts or waive no_surface_leak checks","updated_at":"2026-06-11T00:56:02.000268496Z"} -{"acceptance":"Provision exits 0 twice on empty then existing environment; documented in deployment-checklist app_deploy.","created_at":"2026-07-22T21:41:07.040621169Z","description":"Add deploy-time provision step that creates/verifies schema, volume, and governance tables. Re-run is a no-op.","id":"tablespec-48f4fce6","issue_type":"task","labels":["helix","phase:build","area:app","FEAT-034"],"parent":"tablespec-263a0248","priority":1,"schema_version":1,"status":"open","title":"App: idempotent metadata home provision step (FR-23.3 / US-048)","updated_at":"2026-07-22T21:41:07.040621169Z"} +{"acceptance":"Provision exits 0 twice on empty then existing environment; documented in deployment-checklist app_deploy.","closing_commit_sha":"6857ad0da3e8839e3fb40c6fe806ee7f4060e562","created_at":"2026-07-22T21:41:07.040621169Z","description":"Add deploy-time provision step that creates/verifies schema, volume, and governance tables. Re-run is a no-op.","events_attachment":"tablespec-48f4fce6/events.jsonl","id":"tablespec-48f4fce6","issue_type":"task","labels":["helix","phase:build","area:app","FEAT-034"],"notes":"[2026-07-22T22:02:14Z] closure rejected: closure gate: insufficient evidence: no execution evidence (empty events and no closing_commit_sha)","owner":"erik","parent":"tablespec-263a0248","priority":1,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"App: idempotent metadata home provision step (FR-23.3 / US-048)","updated_at":"2026-07-22T22:02:59.656970885Z"} {"acceptance":"pytest tests -k jdbc_umf_mapper passes incl. mapper-generated UMF passing tablespec validate; pytest tests/integration -k jdbc_reader passes vs SQL Server container / SKIPs without Docker; grep proves no pyodbc/jaydebeapi imports in src/.","closing_commit_sha":"08e7de533805ae50719d68047c6df1ceaa46d6ee","created_at":"2026-06-11T02:34:36.542644702Z","dependencies":[{"issue_id":"tablespec-4b65c810","depends_on_id":"tablespec-4bea5c6c","type":"blocks","created_at":"2026-06-11T02:34:36Z"}],"description":"FEAT-031 JDBC-01..05, DISC-01..03. JdbcReader: spark.read.format('jdbc') with options from the source spec; secret-ref resolution at read time (Databricks secret scope / env var); identifier sanitization (lowercase, non-alnum-\u003eunderscore, collapse repeats) with source identifier preserved for the read boundary. JdbcToUmfMapper: enumerate tables + read INFORMATION_SCHEMA (columns, nullability, PK, FK) through the SAME Spark JDBC reader (option('query',...)); column types from the reflected DataFrame schema reused via SparkToUmfMapper. NO direct DB drivers (no pyodbc/JayDeBeApi) anywhere in src/tablespec. Docker-gated SQL Server integration tests (SKIP when Docker absent).","id":"tablespec-4b65c810","issue_type":"task","labels":["helix","area:ingestion","feat-031","phase:2"],"owner":"erik","parent":"tablespec-ef91646f","priority":1,"schema_version":1,"status":"closed","title":"JdbcReader + JdbcToUmfMapper discovery via Spark JDBC (Phase 3+5)","updated_at":"2026-06-11T13:30:53.934807476Z"} {"acceptance":"Both notebooks committed under notebooks/sec-10k-demo/ with README; Databricks job green in fake-embedding mode and (when endpoint available) real mode; US-045 ACs checked with evidence; no model/credential coupling in any spec.","closing_commit_sha":"8f4a2745fc31e78ba88f759dc9a6e847bc6caadb","created_at":"2026-06-12T16:50:00.043262882Z","dependencies":[{"issue_id":"tablespec-4be69b51","depends_on_id":"tablespec-c5f57c11","type":"blocks","created_at":"2026-06-12T16:50:00Z"},{"issue_id":"tablespec-4be69b51","depends_on_id":"tablespec-5ca7e625","type":"blocks","created_at":"2026-06-12T16:50:00Z"}],"description":"FEAT-032 / US-045: notebooks/sec-10k-demo/ pair — 01 plumbing (EDGAR fetch w/ User-Agent + rate limits, text extraction + chunking, embeddings via databricks-gte-large-en endpoint with deterministic fake fallback widget, XBRL companyfacts JSON landing), 02 tablespec story (corpus spec with EMBEDDING(1024) + facts spec via json kind, validate both incl. dimensionality + %16 advisory path, workbooks, artifacts, staged validation scorecard). Prove on dbw-dev-eus2 both with real endpoint and fake-embedding fallback.","events_attachment":"tablespec-4be69b51/events.jsonl","id":"tablespec-4be69b51","issue_type":"task","labels":["area:demo","feat-032","databricks"],"priority":1,"schema_version":1,"session_id":"eb-rescued","status":"closed","title":"SEC 10-K corpus + XBRL facts demo notebooks (US-045)","updated_at":"2026-06-13T03:42:11.392220353Z","work-cooldown-base-rev":"b6970d91489140e29e78ea7a136b926378950787","work-last-detail":"failed to read worktree HEAD: git rev-parse HEAD: exit status 128","work-last-status":"execution_failed","work-retry-after":"2026-06-13T02:20:10Z"} {"acceptance":"from tablespec.ingestion import get_reader, SourceReader, CsvReader works; merge.py dangling imports resolve; pytest tests -k merge passes with no test modified; pytest tests -k source_spec passes (jdbc validates w/ secret ref, plaintext password raises, absent source: = delimited alias); backbone reads via UMF-derived options; existing corpus recompiles byte-identically.","closing_commit_sha":"3a881cc201708aa209d0236053c1fd25f2a73ca2","created_at":"2026-06-11T02:34:36.424812422Z","description":"FEAT-031 SRC-01..05. Create tablespec.ingestion: SourceReader protocol (read(spec, spark) -\u003e DataFrame), get_reader(spec) factory, CsvReader extracted verbatim from merge.py:84-107; implement the dangling imports merge.py:24-37 declares (ingestion.constants.normalize_spark_encoding, ingestion.raw_ingester.{build_column_lookup,map_headers}). Add discriminated source: block to UMF models (kind: delimited|parquet|jdbc; file_format back-compat alias normalized on load; jdbc variant per JDBC-01 with password_secret_ref, literal password -\u003e ValidationError). Wire merge.py and e2e/backbone.py stage-1 raw loading through the factory honoring FileFormatSpec (fixes backbone.py:229-243 hardcoded comma-CSV). Update schemas/umf.schema.json.","execution-eligible":false,"id":"tablespec-4bea5c6c","issue_type":"task","labels":["helix","area:ingestion","feat-031","phase:1","operator-review"],"notes":"Parked 2026-06-11 after failed ddx work on tablespec-a169e060: this child was generated and claimed outside the microsite/deployment bead scope. Parent tablespec-ef91646f is also parked for operator review.","parent":"tablespec-ef91646f","priority":4,"schema_version":1,"status":"closed","title":"Build tablespec.ingestion package + source: model (Phase 1+2)","updated_at":"2026-06-11T03:09:14.815497681Z"} {"acceptance":"Unit/integration tests for PARQ residual pass; US-043 ACs satisfied.","created_at":"2026-07-22T21:41:08.072147747Z","dependencies":[{"issue_id":"tablespec-502c6126","depends_on_id":"tablespec-e9c21567","type":"blocks","created_at":"2026-07-22T21:41:08Z"}],"description":"Close FR-21.3 residual: typed raw never string-parses DATE/TIMESTAMP.","id":"tablespec-502c6126","issue_type":"task","labels":["helix","phase:build","area:data","FEAT-031"],"notes":"CLOSED 2026-07-22 queue surgery: PARQ typed_raw cast shipped — casting_utils + test_casting_utils typed_raw DATE/TIMESTAMP + ingest generator parquet native typed. FEAT-031/PRD FR-21.3 marked Shipped.","parent":"tablespec-263a0248","priority":1,"schema_version":1,"status":"closed","title":"Implement parquet identity/safe-narrowing residual (PARQ)","updated_at":"2026-07-22T21:47:10.158184288Z"} -{"acceptance":"Declared inputs listed; two environments differ only by inputs (US-047-AC2).","created_at":"2026-07-22T21:41:07.136915736Z","description":"Expose catalog, schema, volume, warehouse, and optional links as declared inputs on app.yaml or asset bundle.","id":"tablespec-5173ff2e","issue_type":"task","labels":["helix","phase:build","area:app","FEAT-034"],"parent":"tablespec-263a0248","priority":1,"schema_version":1,"status":"open","title":"App: parameterized deployment manifest inputs (FR-23.4)","updated_at":"2026-07-22T21:41:07.136915736Z"} +{"acceptance":"Declared inputs listed; two environments differ only by inputs (US-047-AC2).","closing_commit_sha":"6857ad0da3e8839e3fb40c6fe806ee7f4060e562","created_at":"2026-07-22T21:41:07.136915736Z","description":"Expose catalog, schema, volume, warehouse, and optional links as declared inputs on app.yaml or asset bundle.","events_attachment":"tablespec-5173ff2e/events.jsonl","id":"tablespec-5173ff2e","issue_type":"task","labels":["helix","phase:build","area:app","FEAT-034"],"notes":"[2026-07-22T22:02:14Z] closure rejected: closure gate: insufficient evidence: no execution evidence (empty events and no closing_commit_sha)","owner":"erik","parent":"tablespec-263a0248","priority":1,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"App: parameterized deployment manifest inputs (FR-23.4)","updated_at":"2026-07-22T22:03:00.419363072Z"} {"acceptance":"Plan has the 5 required template sections; module inventory matches src/tablespec; date/version fields consistent; v1/v2 naming disambiguated or merged.","created_at":"2026-06-11T00:24:15.559170406Z","description":"Spec audit 2026-06-10: docs/helix/04-build/implementation-plan.md misses 4 of 5 required sections (scope, implementation_slices, issue_decomposition, risks_and_rollbacks); header says Version 2.0 / 2026-03-16 but content references FEAT-026-era artifacts; module inventory (lines ~52-88) omits every v3.0 compiler-era module (core/, e2e/, dbt/, ldp/, authoring/, compatibility.py, bootstrap.py). Also resolve the naming inversion: the live plan is implementation-plan.md while the file named -v2 is the superseded tombstone.","id":"tablespec-52386c13","issue_type":"task","labels":["helix","spec","backfill"],"owner":"erik","priority":2,"schema_version":1,"status":"closed","title":"Restructure implementation-plan.md to template; refresh stale module inventory","updated_at":"2026-06-12T16:46:15.981169173Z"} -{"acceptance":"US-050 file exists with US-050-ACn; distinguishes shipped model/reader from residual compile/backbone; FEAT-031 and PRD FR-21.7 cross-link it.","created_at":"2026-07-22T21:41:07.882777607Z","description":"Author US-050 for FR-21.7 JSON residual. Model + JsonReader shipped; backbone still rejects json (delimited/parquet only). Story must state residual vs shipped clearly so implement bead tablespec-9f98cf03 is executable.","id":"tablespec-557f8a24","issue_type":"task","labels":["helix","phase:frame","area:data","FEAT-031"],"notes":"Queue surgery 2026-07-22: kept; clarify shipped vs residual in ACs.","parent":"tablespec-263a0248","priority":2,"schema_version":1,"status":"open","title":"Author US-050: JSON/JSONL source kind residual (FR-21.7)","updated_at":"2026-07-22T21:47:08.43294096Z"} +{"acceptance":"US-050 file exists with US-050-ACn; distinguishes shipped model/reader from residual compile/backbone; FEAT-031 and PRD FR-21.7 cross-link it.","created_at":"2026-07-22T21:41:07.882777607Z","description":"Author US-050 for FR-21.7 JSON residual. Model + JsonReader shipped; backbone still rejects json (delimited/parquet only). Story must state residual vs shipped clearly so implement bead tablespec-9f98cf03 is executable.","events_attachment":"tablespec-557f8a24/events.jsonl","id":"tablespec-557f8a24","issue_type":"task","labels":["helix","phase:frame","area:data","FEAT-031"],"notes":"Queue surgery 2026-07-22: kept; clarify shipped vs residual in ACs.","owner":"erik","parent":"tablespec-263a0248","priority":2,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"Author US-050: JSON/JSONL source kind residual (FR-21.7)","updated_at":"2026-07-22T22:07:12.600304764Z"} {"acceptance":"The two constants agree (or their difference is justified in code comments + spec); FEAT-017 no longer records an open contradiction.","created_at":"2026-06-11T00:24:15.852217623Z","description":"Spec audit 2026-06-10: FEAT-017-validation-pipeline.md line ~44 records an unresolved contradiction: REQUIRED_BASELINE_EXPECTATION_TYPES (gx_baseline.py) includes expect_column_to_exist while REDUNDANT_VALIDATION_TYPES (models/umf.py) lists it as redundant. Disposition: pick one semantics, fix the constant or the spec, and remove the recorded contradiction.","id":"tablespec-58fb4abf","issue_type":"task","labels":["helix","spec","decision","code"],"priority":3,"schema_version":1,"status":"closed","title":"DECISION: resolve expect_column_to_exist redundancy contradiction (FEAT-017)","updated_at":"2026-06-11T00:45:53.250783888Z"} {"acceptance":"JsonSource validates; reader lands JSONL typed with declared projections; missing-path test surfaces a finding; conformance fixture tier for json; existing corpus unaffected.","closing_commit_sha":"bd97d38ff524936c64345be14ad9abdbc3e46493","created_at":"2026-06-12T16:49:59.948911995Z","description":"FR-21.7 (operator-decided now, 2026-06-12): source: kind json — JSON/JSONL lands typed via Spark reader under the parquet-style native-typed raw regime; spec declares a FLAT projection (top-level field or explicit dot-path per column; no recursive flattening); missing projection path = validation finding, never a silent NULL. Model variant + reader + ingestion seam dispatch + typed-raw suite behavior + fixtures.","id":"tablespec-5ca7e625","issue_type":"task","labels":["feat-031","area:ingestion","kind:feature"],"parent":"tablespec-ef91646f","priority":2,"schema_version":1,"status":"closed","title":"json source kind with flat projection (FEAT-031 JSON-01..03)","updated_at":"2026-06-12T18:49:47.1935581Z"} {"acceptance":"1. test_compound_uniqueness_executable_registry: code exposes a deterministic way to tell whether expect_compound_columns_to_be_unique is executable by the native executor, distinct from whether it is schema-allowed.\n2. If implementing the evaluator: test_native_executor_compound_uniqueness_passes_and_fails proves duplicate composite tuples fail and unique tuples pass on classic Spark and Connect/Sail where the existing test harness supports both.\n3. If quarantining instead: test_inferred_composite_candidate_does_not_emit_unsupported_expectation proves candidate-derived composite uniqueness produces no suite expectation and records reason=\"unsupported_by_native_executor\".\n4. test_no_silent_dropped_compound_expectation: executing a suite containing a compound uniqueness expectation either runs through the native evaluator or fails closed with an explicit unsupported expectation result; it must not silently disappear or report an empty dropped result as success.\n5. test_profile_to_gx_composite_emission_gated_by_executor_support: ProfileToGxMapper emits compound uniqueness only when native executor support is true.\n6. uv run pytest tests/unit/test_validation_compound_uniqueness.py tests/unit -k \"compound and uniqueness\" tests/unit -k gx_expectation passes.","closing_commit_sha":"7f91e1318279e15b37b7bc11c4d44bbad4e0f287","created_at":"2026-06-08T04:19:54.05731785Z","dependencies":[{"issue_id":"tablespec-5e65545a","depends_on_id":"tablespec-29002b40","type":"blocks","created_at":"2026-06-08T04:19:54Z"}],"description":"PROBLEM: Compound uniqueness can be schema-allowed without being executable by the Connect-safe native validation executor. Review found expect_compound_columns_to_be_unique is allowed in the UMF/GX validation type surface but is not present in the native executor dispatch table. If inferred composite key candidates emit a compound uniqueness expectation merely because the type is allowed by schema, the suite may claim enforcement without actually executing it. That is worse than advisory-only output.\n\nROOT CAUSE: Validation support is split between model/schema allowlists and runtime executor dispatch. FEAT-007 owns execution. Candidate key inference must define \"supported\" as executable by the native executor, not just representable in a suite.\n\nPROPOSED FIX: Make compound uniqueness enforcement explicit and testable. Either implement a Connect-safe native evaluator for expect_compound_columns_to_be_unique, or quarantine composite inferred candidates so they persist only as key-candidate evidence and do not emit a suite expectation until the evaluator exists. The final implementation must make this decision visible in code and tests so future agents do not confuse schema allowance with runtime enforcement.\n\nRequired behavior:\n- A helper or registry makes enforceable expectation types distinguishable from schema-allowed expectation types.\n- Inferred composite key candidates emit compound uniqueness suite expectations only when expect_compound_columns_to_be_unique is executable by the native executor path.\n- If compound uniqueness remains unsupported, composite candidates still persist as sidecar evidence but no validation suite expectation is emitted for them, and evidence/reason states unsupported_by_native_executor.\n- Classic Spark and Connect/Sail behavior must agree: no silent dropped result, no success=false empty result caused by GX dropping unsupported expectations, and no false claim of enforced composite uniqueness.\n\nIn-scope files:\n- src/tablespec/validation/native_executor.py\n- src/tablespec/validation/gx_executor.py if fail-closed routing needs an enforceability check\n- src/tablespec/profiling/gx_expectation_builder.py if candidate-derived expectation emission is gated there\n- src/tablespec/models/umf.py only if comments/constants need to distinguish allowed vs executable validation types\n- tests/unit/test_validation_compound_uniqueness.py or equivalent focused tests\n- tests/unit/test_native_profiler_key_candidates.py only for expectation-emission gating assertions\n\nOut-of-scope for this bead:\n- Profiler candidate scoring/model implementation; depends on tablespec-29002b40.\n- Bootstrap sidecar and manifest layout; tracked by FEAT-026 child.\n- Changing dbt or LDP generator semantics for authoritative UMF.primary_key.","id":"tablespec-5e65545a","issue_type":"task","labels":["area:validation","area:profiling","kind:bug"],"owner":"erik","parent":"tablespec-cc4f665b","priority":1,"schema_version":1,"spec-id":"FEAT-007","status":"closed","title":"Close compound uniqueness execution gap for inferred key candidates","updated_at":"2026-06-09T20:47:17.969094451Z"} @@ -57,22 +57,22 @@ {"acceptance":"Parquet conformance tier green; negative NULL-out test passes; gx_baseline emits string checks only for all-STRING raw; existing delimited corpus byte-identical.","closing_commit_sha":"4f9297dd1ecde6f5123142a6c903806537ef2da7","created_at":"2026-06-11T02:34:36.776558723Z","dependencies":[{"issue_id":"tablespec-61da147e","depends_on_id":"tablespec-4bea5c6c","type":"blocks","created_at":"2026-06-11T02:34:36Z"}],"description":"FEAT-031 PARQ-01..03 + SUITE-01..03: parquet sources land native-typed raw; identity/safe-narrowing mode in build_ingest_select/cast_column_sql (typed DATE never string-parsed — negative test for zero silent NULL-out); raw suites branch on raw typing (schema-type expectations for typed raw); backbone + DuckDB engine format dispatch; parquet conformance tier; LDP read_files format wired from UMF; dbt staging typed-raw mode.","events_attachment":"tablespec-61da147e/events.jsonl","id":"tablespec-61da147e","issue_type":"task","labels":["helix","area:ingestion","feat-031","phase:5"],"parent":"tablespec-ef91646f","priority":2,"schema_version":1,"status":"closed","title":"Parquet typed-raw vertical","updated_at":"2026-06-12T18:26:39.617696927Z"} {"acceptance":"Either docs/helix/parking-lot.md exists with the deferred-item inventory and ddx frontmatter, or the PRD reference is removed/replaced with the canonical DDx bead query; any markdown link/reference check remains clean.","closing_commit_sha":"59451e5071e7e2016e30d3205b4f5b8ed683fcdb","created_at":"2026-06-07T20:59:40.900515531Z","description":"docs/helix/01-frame/prd.md says deferred items are tracked in docs/helix/parking-lot.md, but that file does not exist. The broken-reference scanner does not catch this because the path is in inline code, not a markdown link.","id":"tablespec-62dbc8c6","issue_type":"task","labels":["spec","helix","prd"],"owner":"erik","priority":2,"schema_version":1,"status":"closed","title":"Resolve missing PRD parking-lot artifact","updated_at":"2026-06-09T20:55:03.849150318Z"} {"acceptance":"A concrete workflows/graph.yml schema/template or documented no-graph decision exists; if graph.yml is added, tests validate its node and edge contract rather than only file existence.","created_at":"2026-06-10T20:19:16.452970597Z","description":"The repo now has .helix.yml authorizing the helix flow at docs/helix/, and the HELIX skill references workflows/graph.yml for artifact routing, but the installed skill bundle exposes no concrete graph schema or template. Do not add a graph file until the expected node/edge contract is defined and testable; then add a validated workflows/graph.yml or document why tablespec intentionally relies on ddx frontmatter only.","id":"tablespec-633defe5","issue_type":"task","labels":["helix","spec","follow-up"],"owner":"erik","priority":2,"schema_version":1,"status":"closed","title":"Define HELIX graph contract for tablespec","updated_at":"2026-06-11T02:40:17.304102557Z"} -{"acceptance":"Harness command exits 0; concerns no longer records none for Databricks App; test-plan references harness.","created_at":"2026-07-22T21:41:07.418052243Z","dependencies":[{"issue_id":"tablespec-6bd3e4f7","depends_on_id":"tablespec-2a68a0ad","type":"blocks","created_at":"2026-07-22T21:41:07Z"},{"issue_id":"tablespec-6bd3e4f7","depends_on_id":"tablespec-48f4fce6","type":"blocks","created_at":"2026-07-22T21:41:07Z"},{"issue_id":"tablespec-6bd3e4f7","depends_on_id":"tablespec-0e6cd069","type":"blocks","created_at":"2026-07-22T21:41:07Z"}],"description":"Implement at least one automated whole-stack path for config resolve plus provision plus start. Update concerns e2e-framework when filled.","id":"tablespec-6bd3e4f7","issue_type":"task","labels":["helix","phase:test","area:app","FEAT-034"],"parent":"tablespec-263a0248","priority":1,"schema_version":1,"status":"open","title":"App: whole-stack e2e gate for FR-23 (e2e-framework slot)","updated_at":"2026-07-22T21:41:07.418052243Z"} +{"acceptance":"Harness command exits 0; concerns no longer records none for Databricks App; test-plan references harness.","created_at":"2026-07-22T21:41:07.418052243Z","dependencies":[{"issue_id":"tablespec-6bd3e4f7","depends_on_id":"tablespec-2a68a0ad","type":"blocks","created_at":"2026-07-22T21:41:07Z"},{"issue_id":"tablespec-6bd3e4f7","depends_on_id":"tablespec-48f4fce6","type":"blocks","created_at":"2026-07-22T21:41:07Z"},{"issue_id":"tablespec-6bd3e4f7","depends_on_id":"tablespec-0e6cd069","type":"blocks","created_at":"2026-07-22T21:41:07Z"}],"description":"Implement at least one automated whole-stack path for config resolve plus provision plus start. Update concerns e2e-framework when filled.","events_attachment":"tablespec-6bd3e4f7/events.jsonl","id":"tablespec-6bd3e4f7","issue_type":"task","labels":["helix","phase:test","area:app","FEAT-034"],"owner":"erik","parent":"tablespec-263a0248","priority":1,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"App: whole-stack e2e gate for FR-23 (e2e-framework slot)","updated_at":"2026-07-22T22:07:15.750144144Z"} {"acceptance":"Adapter in src/tablespec/validation with unit tests; test_northwind_e2e.py uses it instead of its local _build_report.","created_at":"2026-06-11T13:55:45.728505095Z","description":"US-039 AC6 follow-up (2026-06-11): the e2e test bridges staged ExpectationResults into QualityCheckResults locally to produce a ValidationReport. Promote that bridge into validation/ as a small shipped adapter so consumers get reports from staged execution without a full compile.","id":"tablespec-72c03317","issue_type":"task","labels":["area:validation","feat-031","kind:feature"],"owner":"erik","priority":3,"schema_version":1,"status":"closed","title":"Ship StagedExecutionResult -\u003e ValidationReport adapter","updated_at":"2026-06-12T16:45:37.438595181Z"} {"acceptance":"Each item either promoted to its own refined bead/FEAT requirement or explicitly declined with a recorded reason.","created_at":"2026-06-12T16:50:00.133977772Z","description":"Captured during the 2026-06-12 bronze-input-universe scoping: Excel-as-data-source (vs authoring), a first-class REST API source kind (acquisition stays plumbing for now), Kafka/CDC streaming contracts, and Avro/ORC as riders on the parquet typed-raw bead (tablespec-61da147e — consider folding there). Revisit after the current demo arc ships.","id":"tablespec-79868e6b","issue_type":"task","labels":["parking-lot","area:ingestion"],"owner":"erik","priority":4,"resolution":"DECIDED 2026-06-12: Avro/ORC are declined as a separate bead and folded into the typed-raw file-format rider discussion for tablespec-61da147e because they share the same native-typed raw/cast/suite plumbing as Parquet. Excel-as-data-source is declined for now because FEAT-009 owns Excel as an authoring/review surface, not a bronze acquisition kind. REST API source kind is declined for now because pagination/auth/retry/rate-limit/response-shape semantics need a separate acquisition ADR/epic, not a FEAT-031 refinement. Kafka/CDC streaming contracts are declined for now because FEAT-031 explicitly excludes real-time CDC ingestion and streaming requires a separate architecture decision.","schema_version":1,"status":"closed","title":"Parking lot: remaining bronze input types (Excel-as-data, REST API kind, Kafka/CDC, Avro/ORC rider)","updated_at":"2026-06-12T17:50:52.481099991Z"} {"acceptance":"1. UMFLoader.load() no longer auto-loads arbitrary single-file YAML UMF documents through the normal path; split directories and JSON continue to load.\n2. If inline YAML compatibility remains, it is reachable only through an explicit migration API/CLI path whose name includes migration or legacy.\n3. CLI help/examples no longer present table.umf.yaml as the canonical authoring input for validate/generate/emit/export/explore.\n4. A test command such as uv run pytest tests/unit/test_umf_loader.py tests/unit/test_cli.py -k \"format or inline or convert\" passes and includes a regression that single-file YAML is rejected or migration-only.\n5. Specs/docs consistently state: split YAML directory for editing, JSON for artifact/interchange, inline YAML is not a supported canonical format.","closing_commit_sha":"8477c81738230fc73dfcbe2c82f07346fbc74535","created_at":"2026-06-08T00:18:06.152420213Z","description":"Remove inline whole-UMF YAML from the normal public authoring/loading path. Product/spec language treats split YAML directories as the editable format and JSON as the artifact/interchange format, but src/tablespec/umf_loader.py still exposes UMFFormat.INLINE and detect_format() auto-detects any .yaml/.yml/.umf file as a whole-UMF document. CLI examples and fixtures using *.umf.yaml make agents default to an inline YAML format that should not be canonical. Preserve any needed compatibility as an explicit migration path only, not autodetected normal loading.\n\nCurrent evidence:\n- src/tablespec/umf_loader.py defines UMFFormat.INLINE and _load_inline_yaml().\n- detect_format() returns INLINE for .yaml/.yml/.umf files.\n- CLI examples mention table.umf.yaml for validate/generate/emit/export/explore.\n- specs name split and JSON as the supported formats for normal workflows.\n\nIn-scope files:\n- src/tablespec/umf_loader.py\n- src/tablespec/cli.py user-facing examples/errors and any explicit migration command if kept\n- docs/helix format requirements and user stories that still imply inline YAML\n- tests for loader/CLI format detection\n\nOut-of-scope for this bead:\n- Rewriting every conformance fixture unless required by tests\n- Changing compiled artifact snapshot format without a separate decision\n- Bootstrap facade work","events_attachment":"tablespec-7bce0707/events.jsonl","id":"tablespec-7bce0707","issue_type":"task","labels":["area:umf-loader","area:formats","kind:api"],"priority":0,"schema_version":1,"session_id":"eb-8f77922a","spec-id":"FEAT-010","status":"closed","title":"Quarantine inline YAML UMF as migration-only","updated_at":"2026-06-08T01:27:55.860961576Z"} {"acceptance":"1. TestConformanceAcceptedDialectListIncludesDatabricks: add or update tests/docs/test_dialect_contract.py so it reads docs/helix/03-test/conformance-acceptance.md and fails unless the accepted public dialect wording names spark, duckdb, and databricks and includes public dialect=\"databricks\".\n2. TestConformanceAcceptedDialectListRejectsSparkDuckdbOnly: add or update the same docs regression so it fails on spark/duckdb-only accepted dialect language that omits databricks.\n3. TestPytestGate: `uv run pytest tests/docs/test_dialect_contract.py tests/unit/test_bootstrap.py`\n4. TestGoGate: `go test ./...`\n5. TestLefthookGate: `lefthook run pre-commit`","created_at":"2026-06-08T20:34:59.152352174Z","description":"PROBLEM: The conformance acceptance artifact can still be read as spark/duckdb-only at the public dialect contract boundary, so reviewers and validators may incorrectly reject dialect=\"databricks\" even though the implementation accepts it.\nROOT CAUSE: docs/helix/03-test/conformance-acceptance.md:49 describes the DbtDatabricks compile tier without explicitly tying it to the accepted public dialect list, and docs/helix/03-test/conformance-acceptance.md:93 documents the Databricks-facing compile UX separately from the spark/duckdb dialect contract.\nPROPOSED FIX: Update docs/helix/03-test/conformance-acceptance.md so the accepted public dialect wording explicitly includes databricks alongside spark and duckdb, while preserving that internal Spark-family renderers may normalize the spelling when SQL is identical. Add a focused docs regression that fails if the accepted dialect wording regresses to spark/duckdb-only or omits dialect=\"databricks\".\nNON-SCOPE: One-shot bootstrap facade implementation. Production pipeline JSON/wheel packaging contract. Inline UMF format cleanup. No real Databricks workspace execution beyond existing opt-in tiers. GOVERNING ARTIFACTS: docs/helix/01-frame/features/FEAT-026-compile-orchestrator-bootstrap.md:84; docs/helix/03-test/conformance-acceptance.md:49; docs/helix/03-test/conformance-acceptance.md:85; docs/helix/03-test/conformance-acceptance.md:109. DEPENDENCIES: parent bead tablespec-ed74497c; child of tablespec-943b21a5; no additional dependencies.","events":[{"actor":"ddx","body":"{\"resolved_provider\":\"codex\",\"resolved_model\":\"gpt-5.4-mini\",\"route_reason\":\"explicit_pin_discovery_unavailable\",\"fallback_chain\":[]}","created_at":"2026-06-09T01:18:51.656650693Z","kind":"routing","source":"legacy agent execute-bead","summary":"provider=codex model=gpt-5.4-mini reason=explicit_pin_discovery_unavailable"},{"actor":"ddx","body":"{\"attempt_id\":\"20260609T004515-a93571b6\",\"harness\":\"codex\",\"model\":\"gpt-5.4-mini\",\"route_reason\":\"explicit_pin_discovery_unavailable\",\"input_tokens\":1055131,\"output_tokens\":14220,\"total_tokens\":1069351,\"cost_usd\":0,\"duration_ms\":240866,\"exit_code\":0}","created_at":"2026-06-09T01:18:51.683482203Z","kind":"cost","source":"legacy agent execute-bead","summary":"tokens=1069351 model=gpt-5.4-mini"},{"actor":"ddx","body":"{\"actual_harness\":\"codex\",\"actual_model\":\"gpt-5.4-mini\",\"actual_power\":0,\"actual_provider\":\"\",\"attempt_id\":\"20260609T004515-a93571b6\",\"bead_id\":\"tablespec-7d18b645\",\"estimated_difficulty\":\"easy\",\"rejected_route_pins\":null,\"requested_power_class\":\"cheap\",\"requested_profile\":\"cheap\",\"routing_intent_degraded\":false,\"routing_intent_note\":\"\",\"routing_intent_source\":\"readiness\"}","created_at":"2026-06-09T01:19:02.280561579Z","kind":"execution-routing-intent","source":"ddx work","summary":"source=readiness difficulty=easy powerClass=cheap model=gpt-5.4-mini harness=codex"},{"actor":"erik","body":"status=blocked\nreason=external blocker - `go test ./...` cannot run in this worktree because there is no Go module or Go source tree here; the command exits `pattern ./...: directory prefix . does not contain main module or its selected dependencies`.\nsuggested_action=recheck the external blocker and move status to open when cleared","created_at":"2026-06-09T01:19:02.353228786Z","kind":"no_changes_blocked","source":"ddx work","summary":"no_changes_blocked"},{"actor":"erik","body":"no_changes\npowerClass=cheap\nrationale: status: blocked\nreason: external blocker - `go test ./...` cannot run in this worktree because there is no Go module or Go source tree here; the command exits `pattern ./...: directory prefix . does not contain main module or its selected dependencies`.\ndecision_audit={\"failure_class\":\"no_changes\",\"retry_action\":\"retry\",\"escalation_count\":1,\"requested_route\":{\"harness\":\"codex\",\"model\":\"gpt-5.4-mini\",\"profile\":\"cheap\",\"routing_intent_source\":\"readiness\",\"estimated_difficulty\":\"easy\",\"inferred_power_class\":\"cheap\",\"requested_power_class\":\"cheap\"},\"actual_route\":{\"harness\":\"codex\",\"model\":\"gpt-5.4-mini\",\"route_reason\":\"readiness\"},\"review_status\":\"skipped\",\"review_skip_reason\":\"review_not_configured\",\"review_classification\":\"review_skipped\",\"land_status\":\"not_landed\",\"reconcile_status\":\"not_applicable\"}\nresult_rev=dc2ff514a32349e5ab5577bb548fe86fc43bef65\nbase_rev=dc2ff514a32349e5ab5577bb548fe86fc43bef65\noutcome_reason=no_changes","created_at":"2026-06-09T01:19:02.372544555Z","kind":"execute-bead","source":"ddx work","summary":"no_changes"}],"id":"tablespec-7d18b645","issue_type":"task","labels":["area:databricks","area:dialects","area:docs","kind:bug","ac-quality:refined"],"lifecycle-external-blocker-reason":"external blocker - `go test ./...` cannot run in this worktree because there is no Go module or Go source tree here; the command exits `pattern ./...: directory prefix . does not contain main module or its selected dependencies`.","parent":"tablespec-3ee3b466","priority":0,"schema_version":1,"status":"blocked","title":"Clarify conformance artifact accepted dialect list includes databricks","updated_at":"2026-06-09T01:19:02.378120548Z","work-last-detail":"external blocker - `go test ./...` cannot run in this worktree because there is no Go module or Go source tree here; the command exits `pattern ./...: directory prefix . does not contain main module or its selected dependencies`.","work-last-status":"no_changes_blocked","work-no-changes-count":1,"work-suggested-action":"recheck the external blocker and move status to open when cleared"} {"acceptance":"Tests assert multi-char terminators, null escapes, footer handling, skip_rows.","created_at":"2026-07-22T21:41:07.985967956Z","dependencies":[{"issue_id":"tablespec-7ec86390","depends_on_id":"tablespec-e322b612","type":"blocks","created_at":"2026-07-22T21:41:07Z"}],"description":"Every declared dump dialect option consumed by compiled readers; fixture coverage.","id":"tablespec-7ec86390","issue_type":"task","labels":["helix","phase:build","area:data","FEAT-031"],"notes":"CLOSED 2026-07-22 queue surgery: DUMP-01..04 shipped — model options + dump reader + test_ingestion_package dump tests. FEAT-031/PRD FR-21.2 marked Shipped.","parent":"tablespec-263a0248","priority":1,"schema_version":1,"status":"closed","title":"Implement dump-dialect end-to-end (DUMP-01..04)","updated_at":"2026-07-22T21:47:09.543809991Z"} {"acceptance":"Each re-authored FEAT passes structural conformance against template.md required sections; registry row updated; no phantom claims (every code citation verified).","created_at":"2026-06-11T00:23:44.272289169Z","description":"Spec audit 2026-06-10: FEAT-001..023 are registry-style cards (Description/Components/User Stories/Source) missing all six required template sections (problem_statement, requirements, edge_cases_and_error_handling, success_metrics, constraints_and_assumptions, out_of_scope). FEAT-024..029 are the conformant exemplars. Re-author incrementally (worst-first; suggest FEAT-007, FEAT-008, FEAT-004 first since they govern the largest subsystems). Template: .ddx/plugins/helix/workflows/activities/01-frame/artifacts/feature-specification/.","id":"tablespec-82d79e00","issue_type":"task","labels":["helix","spec","backfill"],"owner":"erik","priority":2,"schema_version":1,"status":"closed","title":"Re-author FEAT-001..023 to full feature-specification template","updated_at":"2026-06-11T02:46:25.753131479Z"} -{"acceptance":"Automated search of tracked app source returns no environment-identifying literals (fixtures may use synthetic names).","created_at":"2026-07-22T21:41:06.929450331Z","dependencies":[{"issue_id":"tablespec-859aa1ea","depends_on_id":"tablespec-2a68a0ad","type":"blocks","created_at":"2026-07-22T21:41:06Z"}],"description":"Tracked application source must not embed catalog, schema, volume, warehouse id, or workspace URL. Add a grep/test gate.","id":"tablespec-859aa1ea","issue_type":"task","labels":["helix","phase:build","area:app","FEAT-034"],"parent":"tablespec-263a0248","priority":1,"schema_version":1,"status":"open","title":"App: strip environment-identifying literals (FR-23.1 / CFG-02)","updated_at":"2026-07-22T21:41:06.929450331Z"} +{"acceptance":"Automated search of tracked app source returns no environment-identifying literals (fixtures may use synthetic names).","closing_commit_sha":"6857ad0da3e8839e3fb40c6fe806ee7f4060e562","created_at":"2026-07-22T21:41:06.929450331Z","dependencies":[{"issue_id":"tablespec-859aa1ea","depends_on_id":"tablespec-2a68a0ad","type":"blocks","created_at":"2026-07-22T21:41:06Z"}],"description":"Tracked application source must not embed catalog, schema, volume, warehouse id, or workspace URL. Add a grep/test gate.","events_attachment":"tablespec-859aa1ea/events.jsonl","id":"tablespec-859aa1ea","issue_type":"task","labels":["helix","phase:build","area:app","FEAT-034"],"notes":"[2026-07-22T22:02:15Z] closure rejected: closure gate: insufficient evidence: no execution evidence (empty events and no closing_commit_sha)","owner":"erik","parent":"tablespec-263a0248","priority":1,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"App: strip environment-identifying literals (FR-23.1 / CFG-02)","updated_at":"2026-07-22T22:03:02.07513218Z"} {"acceptance":"1. TestNativeProfilerCompositeKeys::test_native_profiler_composite_minimal_key verifies member_id + effective_date exact uniqueness yields a composite candidate only when neither subset is exact unique.\n2. TestNativeProfilerCompositeKeys::test_native_profiler_rejects_nonminimal_composite verifies if id is exact unique, id + date is rejected or marked nonminimal and is not emitted as a candidate.\n3. TestNativeProfilerCompositeKeys::test_native_profiler_respects_candidate_and_pass_budget verifies a wide-table fixture exact-verifies no more than key_verification_pass_budget candidates and leaves skipped candidates verified_exact=false.\n4. TestNativeProfilerCompositeKeys::test_composite_search_respects_max_width_and_max_candidates verifies no exhaustive all-column combinatorics occur and emitted/preverified candidates are bounded by key_max_width and key_max_candidates.\n5. TestNativeProfilerCompositeKeys::test_minimality_is_honest_when_subset_verification_is_skipped verifies minimal=True is used only after every proper subset was exact-verified as non-unique; otherwise evidence records false/unknown minimality semantics and reason.\n6. TestProfilerConnectSailKeyCandidates::test_connect_classic_key_candidates_structural_equality verifies serialized key_candidates structural equality on deterministic classic and Spark Connect/Sail fixtures.\n7. TestQualityGates::test_composite_go_test_gate documents and passes go test ./... or records operator_required when no Go module/toolchain exists.\n8. TestQualityGates::test_composite_lefthook_gate documents and passes lefthook run pre-commit or records operator_required when lefthook is unavailable.\n9. Verification command passes: uv run pytest tests/unit/test_native_profiler_key_candidates.py tests/unit/test_profiler_connect_sail.py tests/unit -k gx_expectation.","closing_commit_sha":"e4533c64a2c40225c17763e449269a532cc4969c","created_at":"2026-06-09T04:49:35.174959604Z","description":"PROBLEM: NativeSparkProfiler cannot detect composite advisory keys and has no bounded exact-verification strategy. A naive implementation would either miss useful member_id + effective_date-style keys or exhaustively verify all column combinations, which is too expensive and risks dishonest minimality claims.\n\nROOT CAUSE: src/tablespec/profiling/native_profiler.py:157 counts rows and src/tablespec/profiling/native_profiler.py:186 collects profile signals in one batch, but there is no key candidate model, bounded composite candidate generator, exact distinct verification for column tuples, subset verification, verification pass budget, or skipped-candidate semantics.\n\nPROPOSED FIX: Add bounded composite inference behind infer_key_candidates=True using cheap profile signals first. Generate at most key_max_candidates pre-verification candidates up to key_max_width, prefer promising columns deterministically, and exact-verify only within key_verification_pass_budget using Connect-safe DataFrame operations. Mark minimal=True only when all proper subsets were exact-verified as non-unique; otherwise use false/unknown semantics in evidence rather than guessing. Reject or do not emit nonminimal composites when an exact unique subset exists. Budget-skipped candidates must remain verified_exact=false and must not suppress legacy approximate uniqueness behavior downstream.\n\nGOVERNING ARTIFACT: parent bead tablespec-cc4f665b.\n\nDEPENDENCY ID: tablespec-cc4f665b.\n\nNON-SCOPE: Single-column model serialization except what is required to reuse KeyCandidate/KeyCandidateEvidence. Writing sidecar files or extending manifest layout; tracked by FEAT-026 child. Mutating UMF.primary_key or unique_constraints. Making sample data, dbt, or LDP consume inferred candidates. Implementing compound uniqueness execution in the validator; tracked by FEAT-007 child.","id":"tablespec-87e12737","issue_type":"task","labels":["area:profiling","kind:feature","ac-quality:refined","parent:tablespec-cc4f665b"],"parent":"tablespec-29002b40","priority":0,"schema_version":1,"status":"closed","title":"Infer bounded composite advisory keys with honest minimality and budgets","updated_at":"2026-06-09T20:17:34.343973218Z"} {"acceptance":"All US-039 ACs (AC1-AC6) pass on the Docker lane; databricks_e2e lane green when credentials configured; evidence recorded (commands + exit status) in the bead.","closing_commit_sha":"224a8af9789d52195ccec8d91c7009fd82ba027b","created_at":"2026-06-11T02:34:36.619401849Z","dependencies":[{"issue_id":"tablespec-8980c812","depends_on_id":"tablespec-4b65c810","type":"blocks","created_at":"2026-06-11T02:34:36Z"}],"description":"Drive US-039 ACs: Northwind in SQL Server -\u003e discover one UMF per table (FKs incl. orders.customer_id-\u003ecustomers.customer_id; 'Order Details'-\u003eorder_details sanitization) -\u003e tablespec validate all pass -\u003e export-excel workbook round-trips -\u003e FK-aware sample data -\u003e staged validation report per table with real results and zero silent NULL-out of typed columns. Local lane: Docker SQL Server + Northwind, SKIP without Docker. Databricks lane: opt-in databricks_e2e tier. Consumer integration (entropy-exchange 03_etl.py replacement) follows in that repo.","id":"tablespec-8980c812","issue_type":"task","labels":["helix","area:ingestion","feat-031","phase:3","acceptance-goal"],"owner":"erik","parent":"tablespec-ef91646f","priority":1,"schema_version":1,"status":"closed","title":"US-039 Northwind end-to-end acceptance (Docker lane + databricks_e2e)","updated_at":"2026-06-11T13:55:45.336976785Z"} {"acceptance":"Checklist has the required sections with measurable decision points and is execution-ready for the next release.","created_at":"2026-06-11T00:24:15.631071468Z","description":"Spec audit 2026-06-10: docs/helix/05-deploy/deployment-checklist.md is a descriptive backfill of release.yml; missing release_scope, rollout_plan, rollback_triggers (has steps, no thresholds), go_or_no_go_decision; blocking measurable_decision_points check unmet; not execution-ready (no checkboxes/owners/window).","id":"tablespec-8a432681","issue_type":"task","labels":["helix","spec","backfill"],"owner":"erik","priority":3,"schema_version":1,"status":"closed","title":"Restructure deployment-checklist.md to template (rollout, rollback triggers, go/no-go)","updated_at":"2026-06-12T16:49:44.346035199Z"} {"acceptance":"1. TestCompileUmfsAcceptsDatabricksDialect: add a compile-path test proving compile_umfs(..., dialect=\"databricks\") completes without ValueError and persists ingest, dbt_ingest, optional dbt_gold, LDP, and manifest artifacts under the pinned layout.\n2. TestCompileUmfsPreservesPublicDialectInManifest: add a regression test proving the generated metadata records the public databricks dialect while any Spark-family internal normalization remains invisible to downstream artifact consumers.\n3. TestPytestGate: `uv run pytest tests/unit/test_bootstrap.py tests/e2e/test_bootstrap_from_specs.py`\n4. TestGoGate: `go test ./...`\n5. TestLefthookGate: `lefthook run pre-commit`","closing_commit_sha":"73a4d3cab2ccb57de6daf5527f7d3c495232c6dc","created_at":"2026-06-08T14:50:25.00456272Z","description":"PROBLEM: The compile/bootstrap path can receive dialect=databricks, but the artifact pipeline still mixes public and internal dialect handling, so the emitted ingest/dbt_ingest/LDP/manifest set can reject databricks or record the wrong dialect.\nROOT CAUSE: src/tablespec/e2e/compile.py:129 forwards the caller dialect to generate_dbt_dag_project, src/tablespec/e2e/compile.py:141 hard-codes generate_ldp_project(..., dialect=\"spark\"), src/tablespec/e2e/compile.py:193 calls generate_ingest_sql without a dialect, and src/tablespec/schemas/ingest_generator.py:214 hard-codes build_ingest_select(..., dialect=\"spark\").\nPROPOSED FIX: Thread the public databricks dialect through compile_umfs, normalize Spark-family emitters internally only where SQL is identical, keep dbt target selection as databricks, and preserve the public dialect in generated metadata while still emitting the pinned artifact layout.\nNON-SCOPE: One-shot bootstrap facade implementation. Production pipeline JSON/wheel packaging contract. Inline UMF format cleanup. No real Databricks workspace execution beyond existing opt-in tiers. GOVERNING ARTIFACTS: docs/helix/01-frame/features/FEAT-026-compile-orchestrator-bootstrap.md:84; docs/helix/03-test/conformance-acceptance.md:49; docs/helix/03-test/conformance-acceptance.md:85; docs/helix/03-test/conformance-acceptance.md:109. DEPENDENCIES: parent bead tablespec-ed74497c; no additional dependencies.","events_attachment":"tablespec-924c7384/events.jsonl","id":"tablespec-924c7384","issue_type":"task","labels":["area:databricks","area:dialects","kind:bug","ac-quality:needs-refinement"],"parent":"tablespec-171e409c","priority":0,"schema_version":1,"session_id":"eb-e961169d","status":"closed","title":"Thread databricks through compile-path normalization","updated_at":"2026-06-08T15:41:34.831875417Z"} {"acceptance":"1. TestDatabricksCompileArtifactsStateNormalizationDecision: assert the implementation notes or generated metadata/comments state the decision: public dialect=\"databricks\" is accepted and Spark-family SQL emitters may normalize to spark internally because cast SQL is identical.\n2. TestDatabricksDialectContractIsDocumented: add a docs or metadata regression test that fails if the accepted dialect list regresses back to spark/duckdb-only wording.\n3. TestPytestGate: `uv run pytest tests/docs/test_dialect_contract.py tests/unit/test_bootstrap.py`\n4. TestGoGate: `go test ./...`\n5. TestLefthookGate: `lefthook run pre-commit`","created_at":"2026-06-08T14:50:25.086649929Z","description":"PROBLEM: The public contract still reads like Spark/duckdb-only support in places, so even after code paths accept databricks, downstream reviewers and validators can still infer the wrong supported dialect set.\nROOT CAUSE: docs/helix/01-frame/features/FEAT-026-compile-orchestrator-bootstrap.md:84 and docs/helix/03-test/conformance-acceptance.md:49, docs/helix/03-test/conformance-acceptance.md:85, and docs/helix/03-test/conformance-acceptance.md:109 are the governing acceptance artifacts, but the implementation notes and emitted metadata/comments do not clearly state that public dialect=\"databricks\" is accepted while Spark-family emitters may normalize internally.\nPROPOSED FIX: Update implementation notes, comments, or generated metadata to spell out the dialect contract, including the Spark-family normalization decision and the reason it is safe because the cast SQL is identical.\nNON-SCOPE: One-shot bootstrap facade implementation. Production pipeline JSON/wheel packaging contract. Inline UMF format cleanup. No real Databricks workspace execution beyond existing opt-in tiers. GOVERNING ARTIFACTS: docs/helix/01-frame/features/FEAT-026-compile-orchestrator-bootstrap.md:84; docs/helix/03-test/conformance-acceptance.md:49; docs/helix/03-test/conformance-acceptance.md:85; docs/helix/03-test/conformance-acceptance.md:109. DEPENDENCIES: parent bead tablespec-ed74497c; no additional dependencies.","events":[{"actor":"erik","body":"{\"ac_map\":[{\"parent_ac\":\"1. TestDatabricksCompileArtifactsStateNormalizationDecision: assert the implementation notes or generated metadata/comments state the decision: public dialect=\\\"databricks\\\" is accepted and Spark-family SQL emitters may normalize to spark internally because cast SQL is identical.\",\"coverage\":\"Child \\\"Document Spark-family normalization for databricks emitters\\\" AC1 TestDatabricksCompileArtifactsStateNormalizationDecision.\"},{\"parent_ac\":\"2. TestDatabricksDialectContractIsDocumented: add a docs or metadata regression test that fails if the accepted dialect list regresses back to spark/duckdb-only wording.\",\"coverage\":\"Child \\\"Document databricks as an accepted public dialect\\\" AC1 TestDatabricksDialectContractIsDocumented.\"},{\"parent_ac\":\"3. TestPytestGate: `uv run pytest tests/docs/test_dialect_contract.py tests/unit/test_bootstrap.py`\",\"coverage\":\"Child \\\"Document databricks as an accepted public dialect\\\" AC2 TestPytestGate and child \\\"Document Spark-family normalization for databricks emitters\\\" AC2 TestPytestGate.\"},{\"parent_ac\":\"4. TestGoGate: `go test ./...`\",\"coverage\":\"Child \\\"Document databricks as an accepted public dialect\\\" AC3 TestGoGate and child \\\"Document Spark-family normalization for databricks emitters\\\" AC3 TestGoGate.\"},{\"parent_ac\":\"5. TestLefthookGate: `lefthook run pre-commit`\",\"coverage\":\"Child \\\"Document databricks as an accepted public dialect\\\" AC4 TestLefthookGate and child \\\"Document Spark-family normalization for databricks emitters\\\" AC4 TestLefthookGate.\"}],\"child_ids\":[\"tablespec-3ee3b466\",\"tablespec-c65a49de\"],\"rationale\":\"Split by independently verifiable contract surfaces: one child protects the public accepted dialect list in conformance documentation, while the other protects implementation/generated-artifact wording for Spark-family normalization. Both retain the parent non-scope, governing artifacts, dependency context, pytest command, go test gate, and lefthook gate.\"}","created_at":"2026-06-08T18:22:28.902933376Z","kind":"triage-decomposed","source":"ddx work","summary":"decomposed into tablespec-3ee3b466, tablespec-c65a49de"}],"execution-eligible":false,"id":"tablespec-943b21a5","issue_type":"task","labels":["area:databricks","area:dialects","area:docs","kind:bug","ac-quality:needs-refinement"],"parent":"tablespec-171e409c","priority":0,"schema_version":1,"status":"open","title":"Document the databricks dialect contract and normalization rule","updated_at":"2026-06-08T18:22:29.012524601Z"} -{"acceptance":"Each US-046-AC has a named test with @covers US-046-ACn or a documented non-automatable exception.","created_at":"2026-07-22T21:41:08.54317106Z","description":"Guidebook generation tests exist (test_guidebook_generate.py etc.) but lack @covers US-046-ACn. Add citations or document exceptions; do not re-implement guidebook.","id":"tablespec-9c79765e","issue_type":"task","labels":["helix","phase:test","area:docs","FEAT-033"],"notes":"Queue surgery 2026-07-22: confirmed hygiene-only; kept.","parent":"tablespec-263a0248","priority":3,"schema_version":1,"status":"open","title":"Link US-046 guidebook ACs to tests","updated_at":"2026-07-22T21:47:08.73026378Z"} +{"acceptance":"Each US-046-AC has a named test with @covers US-046-ACn or a documented non-automatable exception.","created_at":"2026-07-22T21:41:08.54317106Z","description":"Guidebook generation tests exist (test_guidebook_generate.py etc.) but lack @covers US-046-ACn. Add citations or document exceptions; do not re-implement guidebook.","events_attachment":"tablespec-9c79765e/events.jsonl","id":"tablespec-9c79765e","issue_type":"task","labels":["helix","phase:test","area:docs","FEAT-033"],"notes":"Queue surgery 2026-07-22: confirmed hygiene-only; kept.","owner":"erik","parent":"tablespec-263a0248","priority":3,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"Link US-046 guidebook ACs to tests","updated_at":"2026-07-22T22:07:14.196384917Z"} {"acceptance":"1. .agents/skills/tablespec/SKILL.md exists with valid skill frontmatter name=tablespec and a description that triggers for tablespec code/spec/Databricks/UMF/pipeline work.\n2. The skill body is concise and includes the canonical workflow guidance listed in the description without duplicating full specs.\n3. Any optional agents/openai.yaml metadata, if present, is consistent with SKILL.md.\n4. A validation command such as python - \u003c\u003cPY to parse the YAML frontmatter, or an available skill validation script, passes.\n5. The bead is closed with evidence after the skill file is committed.","closing_commit_sha":"b2515b1d3e2ba98cbfba870b1a7ed3eaf79a47c7","created_at":"2026-06-08T00:42:02.12152395Z","description":"Add a concise project-local agent skill for tablespec so coding agents discover the canonical public workflows before low-level internals. The skill should be a routing/onboarding layer, not a duplicate copy of the specs. It should steer agents away from recurring drift: hand-rolling SparkToUmfMapper/NativeSparkProfiler/ProfileToGxMapper composition, treating inline YAML UMF as canonical, confusing Databricks target/runtime with cast dialect, and treating production as notebook-local source orchestration instead of installed JSON pipeline artifacts plus wheels.\n\nRequired guidance in the skill:\n- Prefer public bootstrap/packaging APIs and documented flows over composing internal e2e/profiling modules directly.\n- Schema reflection creates UMF; native profiling creates profile data; ProfileToGxMapper creates validation expectations.\n- UMF authoring format is split YAML directory; JSON is artifact/interchange; inline YAML is legacy/migration-only.\n- Databricks dialect handling must follow the canonical repo decision once implemented; until then call out the current decision bead.\n- Production pipeline contract: production runs installed JSON pipeline artifacts and installed wheels; development builds those artifacts.\n- Databricks tests run in-process with ipytest/pytest.main, not uv run pytest on the Databricks runtime.\n- Tracker work uses DDx beads.\n\nIn-scope files:\n- .agents/skills/tablespec/SKILL.md\n- optional .agents/skills/tablespec/agents/openai.yaml if metadata tooling is available\n- any lightweight validation test or script needed to ensure skill frontmatter is parseable\n\nOut-of-scope for this bead:\n- Implementing bootstrap facade, inline YAML cleanup, dialect canonicalization, or production packaging work\n- Copying full PRD/spec content into the skill\n- Changing DDx itself","id":"tablespec-9f12f29c","issue_type":"task","labels":["area:agents","area:docs","area:databricks","kind:tooling"],"owner":"erik","priority":0,"schema_version":1,"status":"closed","title":"Add project-local tablespec agent skill","updated_at":"2026-06-08T00:46:30.062270831Z"} -{"acceptance":"US-050 ACs green for compile/backbone json path, or residual documented with evidence; backbone no longer fails closed on kind=json without a supported path.","created_at":"2026-07-22T21:41:08.163880947Z","dependencies":[{"issue_id":"tablespec-9f98cf03","depends_on_id":"tablespec-557f8a24","type":"blocks","created_at":"2026-07-22T21:41:08Z"}],"description":"Close residual FR-21.7 gaps beyond model and JsonReader. Primary gap: e2e/backbone.py supports only delimited/parquet for raw load. Implement json path (and any compile residual) per US-050 ACs.","id":"tablespec-9f98cf03","issue_type":"task","labels":["helix","phase:build","area:data","FEAT-031"],"notes":"Queue surgery 2026-07-22: confirmed real gap; kept.","parent":"tablespec-263a0248","priority":2,"schema_version":1,"status":"open","title":"Implement JSON source residual compile/backbone path","updated_at":"2026-07-22T21:47:08.528079807Z"} +{"acceptance":"US-050 ACs green for compile/backbone json path, or residual documented with evidence; backbone no longer fails closed on kind=json without a supported path.","created_at":"2026-07-22T21:41:08.163880947Z","dependencies":[{"issue_id":"tablespec-9f98cf03","depends_on_id":"tablespec-557f8a24","type":"blocks","created_at":"2026-07-22T21:41:08Z"}],"description":"Close residual FR-21.7 gaps beyond model and JsonReader. Primary gap: e2e/backbone.py supports only delimited/parquet for raw load. Implement json path (and any compile residual) per US-050 ACs.","events_attachment":"tablespec-9f98cf03/events.jsonl","id":"tablespec-9f98cf03","issue_type":"task","labels":["helix","phase:build","area:data","FEAT-031"],"notes":"Queue surgery 2026-07-22: confirmed real gap; kept.","owner":"erik","parent":"tablespec-263a0248","priority":2,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"Implement JSON source residual compile/backbone path","updated_at":"2026-07-22T22:07:13.331375723Z"} {"acceptance":"1. Implement the remaining bounded slice of the parent bead.\n2. cd cli \u0026\u0026 go test ./... passes.\n3. lefthook run pre-commit passes.","closing_commit_sha":"2ff71a7fb895de6d465922a547cd2f1765eeccf5","created_at":"2026-06-08T00:25:01.523823908Z","description":"PROBLEM\nParent bead tablespec-dbf90b8d is too broad for one execution pass. This child owns part 2 of 2 of that parent scope.\n\nROOT CAUSE\nUS-001 through US-020 still use legacy unchecked acceptance bullets without stable US-NNN-ACx IDs. The Helix align checker currently reports 52/52 cited ACs, but those legacy bullets are not machine-addressable and are effectively outside the coverage matrix.\n\nPROPOSED FIX\nImplement only this child acceptance slice:\n- Implement the remaining bounded slice of the parent bead.\n\nNON-SCOPE\nDo not implement parent acceptance criteria assigned to sibling child beads. Preserve every parent non-scope constraint unless this child explicitly narrows it.","events_attachment":"tablespec-9fad11d8/events.jsonl","id":"tablespec-9fad11d8","issue_type":"task","labels":["spec","helix","traceability","decomposed"],"parent":"tablespec-dbf90b8d","priority":0,"schema_version":1,"session_id":"eb-edfc611d","status":"closed","title":"Normalize legacy user-story acceptance criteria: part 2","updated_at":"2026-06-08T06:07:40.47412878Z"} {"acceptance":"1. cd website \u0026\u0026 hugo --gc --minify passes. 2. cd website \u0026\u0026 npm test passes or an equivalent Playwright script verifies homepage, top-level sections, representative deep pages, desktop/mobile screenshots, and navigation state. 3. The built site includes Home, Getting Started, Core Concepts, CLI Reference, API Reference entry point, and Demos pages. 4. The build succeeds when ../helix is absent.","created_at":"2026-06-11T02:00:52.848277654Z","description":"Implement FEAT-030/US-038 by adding a Hugo extended + Hextra microsite under website/. The site must be buildable without ../helix and should adapt any useful structure from ../helix only by copying into this repo with attribution in the commit notes. In-scope files: website/ configuration and content, website/package.json, website/e2e Playwright tests, repo docs links if needed. Out-of-scope: changing the GitHub Pages deployment workflow and package index preservation; that is tracked separately under ADR-014. Required content: Home, Getting Started, Core Concepts including raw vs ingested vs silver, CLI Reference, API Reference entry point, and Demos entry point. Keep API reference generation owned by FEAT-015.","events_attachment":"tablespec-a169e060/events.jsonl","id":"tablespec-a169e060","issue_type":"task","labels":["area:docs","kind:feature"],"priority":0,"schema_version":1,"spec-id":"FEAT-030","status":"closed","title":"Implement Hugo/Hextra product microsite scaffold","updated_at":"2026-06-11T02:29:20.60832637Z"} {"acceptance":"1. A CI command builds a Pages artifact containing /index.html, /simple/index.html, and /simple/tablespec/index.html. 2. A docs-only deploy path, if added, rebuilds or carries forward /simple/ so historical package versions are not lost. 3. The release verification still installs tablespec from https://easel.github.io/tablespec/simple/. 4. The workflow pins Hugo extended and checks out with fetch-depth: 0 when Hugo Git metadata is enabled.","created_at":"2026-06-11T02:01:01.058303223Z","dependencies":[{"issue_id":"tablespec-a2519019","depends_on_id":"tablespec-a169e060","type":"blocks","created_at":"2026-06-11T02:01:06Z"}],"description":"Implement the ADR-014 GitHub Pages deployment architecture after the Hugo/Hextra microsite scaffold exists. Current release.yml publishes Pages only on version tags and generates /simple/ from the current release dist/ directory. The new Pages artifact must serve the Hugo site at / and preserve the package index at /simple/ and /simple/tablespec/, including historical release links. In-scope files: .github/workflows/release.yml or a new Pages workflow, website build integration, scripts/tests needed to inspect the Pages artifact. Out-of-scope: changing the package hosting provider or removing the documented GitHub Pages --index-url install path.","id":"tablespec-a2519019","issue_type":"task","labels":["area:infra","area:docs","kind:deployment"],"priority":0,"schema_version":1,"spec-id":"ADR-014","status":"closed","title":"Deploy microsite without breaking Pages package index","updated_at":"2026-06-11T02:33:55.420419612Z"} @@ -81,7 +81,7 @@ {"acceptance":"1. TestKeyCandidateModels::test_key_candidate_models_are_json_serializable verifies KeyCandidate and KeyCandidateEvidence serialize with every documented evidence field present, candidate kind constrained to primary_key_candidate or unique_constraint_candidate, score bounded in [0, 1], and DataFrameProfile exposes key_candidates without breaking existing columns/num_records construction.\n2. TestNativeProfilerKeyControls::test_inference_disabled_preserves_existing_profile_shape verifies NativeSparkProfiler(..., infer_key_candidates=False) returns the same DataFrameProfile structure as before except for default key_candidates=[], and performs no key-verification queries.\n3. TestNativeProfilerKeyControls::test_key_thresholds_and_budgets_are_exposed verifies key_min_rows, key_max_width, key_max_candidates, key_verification_pass_budget, key_promotion_min_score, and key_promotion_min_gap are accepted constructor options and retained for inference without changing UMF.\n4. TestQualityGates::test_child_models_go_test_gate documents and passes the required command go test ./... or records operator_required when no Go module/toolchain exists.\n5. TestQualityGates::test_child_models_lefthook_gate documents and passes lefthook run pre-commit or records operator_required when lefthook is unavailable.\n6. Verification command passes: uv run pytest tests/unit/test_native_profiler_key_candidates.py tests/unit/test_profiler_connect_sail.py tests/unit -k gx_expectation.","created_at":"2026-06-09T04:49:35.144016963Z","description":"PROBLEM: NativeSparkProfiler has no profile-level representation for advisory primary-key or unique-key candidates, so later inference work has nowhere typed and JSON-serializable to put evidence without conflating it with authoritative UMF.primary_key or unique_constraints.\n\nROOT CAUSE: src/tablespec/profiling/types.py:53 defines DataFrameProfile with only num_records and columns; src/tablespec/profiling/native_profiler.py:106 exposes profiler configuration for profiling metrics but no key-inference controls, thresholds, budgets, or disabled-by-default behavior.\n\nPROPOSED FIX: Add typed dataclass models KeyCandidate and KeyCandidateEvidence in src/tablespec/profiling/types.py and add key_candidates: list[KeyCandidate] to DataFrameProfile with a default empty list. Models must be JSON-serializable through dataclasses.asdict/json-compatible primitives and include candidate kinds primary_key_candidate and unique_constraint_candidate. Add NativeSparkProfiler constructor options infer_key_candidates=False, key_min_rows, key_max_width, key_max_candidates, key_verification_pass_budget, key_promotion_min_score, and key_promotion_min_gap while preserving existing output byte/structural behavior when infer_key_candidates is false. Evidence fields must include row_count, columns, null_count_by_column, exact_distinct_count, approximate_distinct_count_by_column, distinct_ratio, completeness_by_column, verified_exact, nullable, minimal, subset_unique, score, score_components, name_hints, type_hints, penalties, verification_pass_count, verification_query_count, and reason.\n\nGOVERNING ARTIFACT: parent bead tablespec-cc4f665b.\n\nDEPENDENCY ID: tablespec-cc4f665b.\n\nNON-SCOPE: Writing sidecar files or extending manifest layout; tracked by FEAT-026 child. Mutating UMF.primary_key or unique_constraints. Making sample data, dbt, or LDP consume inferred candidates. Implementing compound uniqueness execution in the validator; tracked by FEAT-007 child.","events_attachment":"tablespec-a727e762/events.jsonl","execution-eligible":false,"id":"tablespec-a727e762","issue_type":"task","labels":["area:profiling","kind:feature","ac-quality:refined","parent:tablespec-cc4f665b"],"parent":"tablespec-29002b40","priority":0,"schema_version":1,"status":"closed","title":"Add advisory key-candidate profile models and opt-in profiler controls","updated_at":"2026-06-09T20:32:39.012639196Z"} {"acceptance":"File conforms to the solution-design template (or a deliberate local type is registered for design notes); PRD pointer updated if renamed.","created_at":"2026-06-11T00:24:15.702352112Z","description":"Spec audit 2026-06-10: docs/helix/02-design/ldp-sibling-emitter.md is a self-styled governed design note matching no catalog type; nearest type is solution-design (governs FEAT-028). It informally covers scope/requirements/trade-offs/test traceability but lacks Solution Approaches, Domain Model, NFR table, and Traceability table sections. ddx.id frontmatter was added 2026-06-10; full lift needs authoring. PRD Open Questions already records the note is 'retained as the detailed solution design' (prd.md ~line 346).","id":"tablespec-a73d2b87","issue_type":"task","labels":["helix","spec","backfill"],"owner":"erik","priority":3,"schema_version":1,"status":"closed","title":"Lift ldp-sibling-emitter.md design note to solution-design (SD-028)","updated_at":"2026-06-12T16:59:25.249076447Z"} {"acceptance":"Feature and user-story statuses use a consistent vocabulary; every status is backed by implementation/test evidence or explicitly remains planned/specified; stale Done/Approved/Proposed prose is normalized; helix_align_check --strict remains clean.","closing_commit_sha":"91dc5ac7791e64cf5a614c8bee12e04fff912028","created_at":"2026-06-07T20:59:25.434814133Z","description":"Several feature and user-story statuses are stale or inconsistent with implementation evidence. Examples: FEAT-016/017/019/020/021/022/023 are Proposed while their linked stories and tests indicate implemented slices; FEAT-015 and US-028 are Planned while API docs traceability tests now exist; US-021/023/024/026 remain Approved despite test coverage and implemented feature surfaces.","events_attachment":"tablespec-aad04c07/events.jsonl","id":"tablespec-aad04c07","issue_type":"task","labels":["spec","helix","alignment"],"priority":1,"schema_version":1,"session_id":"eb-e75210e0","status":"closed","title":"Reconcile spec status drift with implementation evidence","updated_at":"2026-06-08T00:31:58.006905216Z"} -{"acceptance":"Each US-045 AC is either evidenced (command + exit status or @covers test) or explicitly waived with recorded limitation; US-045 and FEAT-032 DEMO phase status updated to match.","created_at":"2026-07-22T21:41:08.448279865Z","description":"EMBEDDING type core and CORP example (examples/sec10k_corpus.yaml) are shipped. This bead only closes remaining US-045 ACs that lack recorded evidence from notebooks/sec-10k-demo or CI. Do not re-implement the type alphabet.","id":"tablespec-abd68023","issue_type":"task","labels":["helix","phase:test","area:data","FEAT-032"],"notes":"Queue surgery 2026-07-22: re-scoped to residual AC evidence only.","parent":"tablespec-263a0248","priority":2,"schema_version":1,"status":"open","title":"Evidence remaining US-045 SEC 10-K demo ACs (residual only)","updated_at":"2026-07-22T21:47:08.260204201Z"} +{"acceptance":"Each US-045 AC is either evidenced (command + exit status or @covers test) or explicitly waived with recorded limitation; US-045 and FEAT-032 DEMO phase status updated to match.","created_at":"2026-07-22T21:41:08.448279865Z","description":"EMBEDDING type core and CORP example (examples/sec10k_corpus.yaml) are shipped. This bead only closes remaining US-045 ACs that lack recorded evidence from notebooks/sec-10k-demo or CI. Do not re-implement the type alphabet.","events_attachment":"tablespec-abd68023/events.jsonl","id":"tablespec-abd68023","issue_type":"task","labels":["helix","phase:test","area:data","FEAT-032"],"notes":"Queue surgery 2026-07-22: re-scoped to residual AC evidence only.","owner":"erik","parent":"tablespec-263a0248","priority":2,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"Evidence remaining US-045 SEC 10-K demo ACs (residual only)","updated_at":"2026-07-22T22:07:14.987472207Z"} {"acceptance":"1. Implement the first bounded slice of the parent bead.\n2. cd cli \u0026\u0026 go test ./... passes.\n3. lefthook run pre-commit passes.","created_at":"2026-06-08T00:25:01.512286299Z","description":"PROBLEM\nParent bead tablespec-dbf90b8d is too broad for one execution pass. This child owns part 1 of 2 of that parent scope.\n\nROOT CAUSE\nUS-001 through US-020 still use legacy unchecked acceptance bullets without stable US-NNN-ACx IDs. The Helix align checker currently reports 52/52 cited ACs, but those legacy bullets are not machine-addressable and are effectively outside the coverage matrix.\n\nPROPOSED FIX\nImplement only this child acceptance slice:\n- Implement the first bounded slice of the parent bead.\n\nNON-SCOPE\nDo not implement parent acceptance criteria assigned to sibling child beads. Preserve every parent non-scope constraint unless this child explicitly narrows it.","events":[{"actor":"erik","body":"{\"decision\":\"warn\",\"decision_source\":\"pre_claim_intake\",\"detail\":\"ambiguous_scope: The bead is already decomposed into a child slice and has two concrete build-gate ACs, but AC1 is still placeholder prose ('first bounded slice') rather than a machine-addressable scope statement. Targeted metadata/AC edits can make it executable without changing intent, so this is refineable rather than a split or operator-blocker.\",\"fingerprint\":\"0d98334aa1edf1a2103f74d1a7d54fd3e7d6341c98dd61ba34aeea9a44c6a099\",\"policy_mode\":\"warn-only\",\"reason\":\"readiness_best_effort\",\"rule_id\":\"pre_claim_intake.readiness_best_effort\",\"suggested_action\":\"revise the rewrite so it preserves every explicit commitment\",\"warning_fingerprint\":\"0d98334aa1edf1a2103f74d1a7d54fd3e7d6341c98dd61ba34aeea9a44c6a099\"}","created_at":"2026-06-08T01:08:53.289201457Z","kind":"intake.warn","source":"ddx work","summary":"readiness_best_effort"},{"actor":"ddx","body":"{\"resolved_provider\":\"codex\",\"resolved_model\":\"gpt-5.4-mini\",\"route_reason\":\"explicit_pin_discovery_unavailable\",\"fallback_chain\":[]}","created_at":"2026-06-08T01:13:45.769963455Z","kind":"routing","source":"legacy agent execute-bead","summary":"provider=codex model=gpt-5.4-mini reason=explicit_pin_discovery_unavailable"},{"actor":"ddx","body":"{\"attempt_id\":\"20260608T010854-c123202c\",\"harness\":\"codex\",\"model\":\"gpt-5.4-mini\",\"route_reason\":\"explicit_pin_discovery_unavailable\",\"input_tokens\":3302764,\"output_tokens\":17778,\"total_tokens\":3320542,\"cost_usd\":0,\"duration_ms\":289796,\"exit_code\":0}","created_at":"2026-06-08T01:13:45.777875639Z","kind":"cost","source":"legacy agent execute-bead","summary":"tokens=3320542 model=gpt-5.4-mini"},{"actor":"ddx","body":"{\"resolved_provider\":\"codex\",\"resolved_model\":\"gpt-5.4-mini\",\"route_reason\":\"explicit_pin_discovery_unavailable\",\"fallback_chain\":[],\"requested_min_power\":5}","created_at":"2026-06-08T01:54:53.069393664Z","kind":"routing","source":"legacy agent execute-bead","summary":"provider=codex model=gpt-5.4-mini reason=explicit_pin_discovery_unavailable"},{"actor":"ddx","body":"{\"attempt_id\":\"20260608T011347-f6a350e9\",\"harness\":\"codex\",\"model\":\"gpt-5.4-mini\",\"route_reason\":\"explicit_pin_discovery_unavailable\",\"input_tokens\":2936120,\"output_tokens\":27986,\"total_tokens\":2964106,\"cost_usd\":0,\"duration_ms\":422829,\"exit_code\":0}","created_at":"2026-06-08T01:54:53.100400235Z","kind":"cost","source":"legacy agent execute-bead","summary":"tokens=2964106 model=gpt-5.4-mini"},{"actor":"ddx","body":"{\"actual_harness\":\"codex\",\"actual_model\":\"gpt-5.4-mini\",\"actual_power\":0,\"actual_provider\":\"\",\"attempt_id\":\"20260608T011347-f6a350e9\",\"bead_id\":\"tablespec-b474ae20\",\"estimated_difficulty\":\"medium\",\"rejected_route_pins\":null,\"requested_power_class\":\"standard\",\"requested_profile\":\"cheap\",\"routing_intent_degraded\":false,\"routing_intent_note\":\"\",\"routing_intent_source\":\"readiness\"}","created_at":"2026-06-08T01:55:10.508691203Z","kind":"execution-routing-intent","source":"ddx work","summary":"source=readiness difficulty=medium powerClass=standard model=gpt-5.4-mini harness=codex"},{"actor":"erik","body":"status=blocked\nreason=This worktree does not contain a `cli/` directory or any `go.mod`, so the required `cd cli \u0026\u0026 go test ./...` gate cannot be executed here. `lefthook run pre-commit` also has no lefthook config in this worktree, so the second required gate is not runnable as specified.\nsuggested_action=recheck the external blocker and move status to open when cleared","created_at":"2026-06-08T01:55:10.561936937Z","kind":"no_changes_blocked","source":"ddx work","summary":"no_changes_blocked"},{"actor":"erik","body":"no_changes\npowerClass=standard\nrationale: status: blocked\nreason: This worktree does not contain a `cli/` directory or any `go.mod`, so the required `cd cli \u0026\u0026 go test ./...` gate cannot be executed here. `lefthook run pre-commit` also has no lefthook config in this worktree, so the second required gate is not runnable as specified.\nresult_rev=59bea1ed48dc82bcd603cd6f0cec5f66ebe682cd\nbase_rev=59bea1ed48dc82bcd603cd6f0cec5f66ebe682cd\noutcome_reason=no_changes","created_at":"2026-06-08T01:55:10.609406691Z","kind":"execute-bead","source":"ddx work","summary":"no_changes"}],"id":"tablespec-b474ae20","issue_type":"task","labels":["spec","helix","traceability","decomposed"],"lifecycle-external-blocker-reason":"This worktree does not contain a `cli/` directory or any `go.mod`, so the required `cd cli \u0026\u0026 go test ./...` gate cannot be executed here. `lefthook run pre-commit` also has no lefthook config in this worktree, so the second required gate is not runnable as specified.","parent":"tablespec-dbf90b8d","priority":0,"schema_version":1,"status":"blocked","title":"Normalize legacy user-story acceptance criteria: part 1","updated_at":"2026-06-08T01:55:10.610731238Z","work-last-detail":"This worktree does not contain a `cli/` directory or any `go.mod`, so the required `cd cli \u0026\u0026 go test ./...` gate cannot be executed here. `lefthook run pre-commit` also has no lefthook config in this worktree, so the second required gate is not runnable as specified.","work-last-status":"no_changes_blocked","work-no-changes-count":1,"work-suggested-action":"recheck the external blocker and move status to open when cleared"} {"acceptance":"tablespec validate \u003csplit-table-dir\u003e validates that table (nonzero on invalid); 'All 0 tables passed' is impossible when the target contains a table.yaml.","created_at":"2026-06-12T02:16:38.634412063Z","description":"Site rewrite verification 2026-06-12: the CLI validate single-table branch keys on schema.yaml but split format uses table.yaml, so a single table dir falls into pipeline mode and reports 'All 0 tables passed' (exit 0). Vacuous pass = false confidence. Should detect table.yaml dirs and validate them.","id":"tablespec-b476109a","issue_type":"bug","labels":["kind:bug","area:cli","area:validation"],"owner":"erik","priority":2,"schema_version":1,"status":"closed","title":"tablespec validate on a single split table dir silently validates nothing","updated_at":"2026-06-12T16:33:48.084962514Z"} {"acceptance":"1. TestHelixDocsUseCanonicalDatabricksDialectGuidance: add a docs traceability regression that scans `docs/helix/03-test/conformance-acceptance.md` and related Databricks helix references for canonical `dialect=\"databricks\"` guidance.\\n2. TestHelixDocsAvoidSparkOnlyMessaging: assert helix docs do not tell Databricks users to use only `dialect=\"spark\"` without the alias explanation.\\n3. go test ./...\\n4. lefthook run pre-commit.","closing_commit_sha":"ad838e8b7c6d9711b54291895b08bca60507b822","created_at":"2026-06-08T14:55:59.650831628Z","description":"PROBLEM: The helix conformance docs already describe Databricks dialect behavior, but that decision is not reflected consistently in the user-facing examples and traceability checks. ROOT CAUSE: `docs/helix/03-test/conformance-acceptance.md:85` and `docs/helix/03-test/conformance-acceptance.md:109` discuss Databricks dialect behavior without tying it back to the canonical public alias guidance used elsewhere. PROPOSED FIX: Update the helix conformance docs and their traceability expectations so they consistently say `dialect=\"databricks\"` is accepted for Databricks compile UX, while clarifying that generated casts may normalize to Spark-family SQL when the emitted SQL is identical. NON-SCOPE: One-shot bootstrap facade implementation. Production pipeline JSON/wheel packaging contract. Inline UMF format cleanup. No broad documentation rewrite outside Databricks dialect guidance. GOVERNING ARTIFACTS: `docs/guide/bootstrap.md:10`; `README.md:139`; `docs/helix/03-test/conformance-acceptance.md:109`; `scripts/run_integration_tests_databricks.ipynb`. DEPENDENCIES: `tablespec-ed74497c`.","events_attachment":"tablespec-b50b6f67/events.jsonl","id":"tablespec-b50b6f67","issue_type":"task","labels":["area:databricks","area:dialects","area:docs","kind:bug","ac-quality:needs-refinement"],"parent":"tablespec-0fb0d1c2","priority":0,"schema_version":1,"session_id":"eb-4ea7415a","status":"closed","title":"Align helix conformance docs with Databricks dialect guidance","updated_at":"2026-06-08T20:18:39.087381651Z"} @@ -103,9 +103,9 @@ {"acceptance":"Two consecutive generate_sql_ddl runs on a spec WITHOUT source_file_modified produce byte-identical output; golden tests unchanged; a unit test pins this.","created_at":"2026-06-12T01:45:33.137843105Z","description":"Site audit 2026-06-12: generators.py:48-58 embeds datetime.now(tz=UTC) in the DDL header when metadata.source_file_modified is absent, so recompiling an unchanged spec differs — contradicting the PRD zero-drift metric and the front page's reviewable-diff promise (golden tests pass only because fixtures pin source_file_modified). Make the fallback deterministic (omit the timestamp line or derive from spec content hash).","id":"tablespec-df24f623","issue_type":"bug","labels":["kind:bug","area:generators","determinism"],"owner":"erik","priority":2,"schema_version":1,"status":"closed","title":"generate_sql_ddl embeds wall-clock timestamp, breaking determinism without source_file_modified","updated_at":"2026-06-12T16:33:17.634482299Z"} {"acceptance":"Dump fixture tier green in tests/conformance; skip_rows/footer/\\N covered by fixtures; readers derive every option from the source spec.","created_at":"2026-06-11T02:34:36.697697453Z","dependencies":[{"issue_id":"tablespec-df8bc351","depends_on_id":"tablespec-4bea5c6c","type":"blocks","created_at":"2026-06-11T02:34:36Z"}],"description":"FEAT-031 DUMP-01..04: multi-char line terminators, \\N-style null escapes, footer handling, skip_rows honored end-to-end through the reader seam; conformance corpus gains a dump fixture tier.","id":"tablespec-df8bc351","issue_type":"task","labels":["helix","area:ingestion","feat-031","phase:4"],"owner":"erik","parent":"tablespec-ef91646f","priority":2,"schema_version":1,"status":"closed","title":"Dump-dialect delimited options end-to-end","updated_at":"2026-06-12T17:17:31.962452089Z"} {"acceptance":"US-038 ACs evidenced green or residual filed with pointer.","created_at":"2026-07-22T21:41:08.635350717Z","description":"Verify Playwright/Hugo evidence covers US-038-AC1..5.","id":"tablespec-e1956759","issue_type":"task","labels":["helix","phase:test","area:docs","FEAT-030"],"notes":"CLOSED 2026-07-22 queue surgery: website/e2e Playwright covers Getting Started, /simple/ install URL, raw-ingested-silver boundary, nav — US-038 ACs evidenced. No residual.","parent":"tablespec-263a0248","priority":3,"schema_version":1,"status":"closed","title":"Close residual US-038 microsite ACs if any open","updated_at":"2026-07-22T21:47:12.04065791Z"} -{"acceptance":"US-042 file exists with ACs mapping DUMP-01..04; each AC cites existing test evidence; FEAT-031 links US-042.","created_at":"2026-07-22T21:41:07.69343755Z","description":"DUMP-01..04 are SHIPPED (delimited dump options + dump reader + tests/unit/test_ingestion_package.py). Author US-042 ACs that cite those tests. No implement child remains (tablespec-7ec86390 closed).","id":"tablespec-e322b612","issue_type":"task","labels":["helix","phase:frame","area:data","FEAT-031"],"notes":"Queue surgery 2026-07-22: re-scoped; DUMP implement closed as shipped.","parent":"tablespec-263a0248","priority":1,"schema_version":1,"status":"open","title":"Author US-042: dump-dialect AC backfill (story floor only)","updated_at":"2026-07-22T21:47:07.941290463Z"} +{"acceptance":"US-042 file exists with ACs mapping DUMP-01..04; each AC cites existing test evidence; FEAT-031 links US-042.","created_at":"2026-07-22T21:41:07.69343755Z","description":"DUMP-01..04 are SHIPPED (delimited dump options + dump reader + tests/unit/test_ingestion_package.py). Author US-042 ACs that cite those tests. No implement child remains (tablespec-7ec86390 closed).","events_attachment":"tablespec-e322b612/events.jsonl","id":"tablespec-e322b612","issue_type":"task","labels":["helix","phase:frame","area:data","FEAT-031"],"notes":"Queue surgery 2026-07-22: re-scoped; DUMP implement closed as shipped.","owner":"erik","parent":"tablespec-263a0248","priority":1,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"Author US-042: dump-dialect AC backfill (story floor only)","updated_at":"2026-07-22T22:07:11.111822114Z"} {"acceptance":"Each of the three FEATs has a real PRD anchor or a recorded convention explains the exception.","created_at":"2026-06-11T00:24:15.773623381Z","description":"Spec audit 2026-06-10: three FEATs have only nominal PRD anchors: FEAT-015 cites FR-1.1 (Pydantic models), FEAT-016 cites FR-20.3, FEAT-022 cites FR-11.1/11.2 (which the PRD assigns to FEAT-010 scope). The PRD has no requirement for API documentation, test infrastructure as a product concern, or schema-compatibility checking. Either add FRs to the PRD for these capabilities or record that infrastructure FEATs may anchor to vision/principles instead.","id":"tablespec-e728ca79","issue_type":"task","labels":["helix","spec","decision"],"priority":3,"resolution":"DECIDED 2026-06-10 (operator): split. FR-11.6 added to PRD (FEAT-022 anchors to it); FEAT-015/016 recorded as meta-features anchored to Vision/Principles per principles.md §Tension Resolution; FR-20.3 platform posture reassigned to FEAT-029 with FEAT-016 as evidence tier.","schema_version":1,"status":"closed","title":"DECISION: PRD anchors for FEAT-015 (api-docs), FEAT-016 (testing infra), FEAT-022 (compatibility)","updated_at":"2026-06-11T00:56:02.68797967Z"} -{"acceptance":"US-043 file exists with ACs for identity/safe-narrowing and negative DATE path if already covered by tests; FEAT-031 links US-043; no greenfield cast work.","created_at":"2026-07-22T21:41:07.78305571Z","description":"PARQ typed-raw cast path is SHIPPED (casting_utils typed_raw + test_casting_utils typed_raw DATE/TIMESTAMP + ingest generator parquet native typed). Author US-043 ACs citing those tests. No implement child remains (tablespec-502c6126 closed).","id":"tablespec-e9c21567","issue_type":"task","labels":["helix","phase:frame","area:data","FEAT-031"],"notes":"Queue surgery 2026-07-22: re-scoped; PARQ implement closed as shipped.","parent":"tablespec-263a0248","priority":1,"schema_version":1,"status":"open","title":"Author US-043: parquet typed-raw AC backfill (story floor only)","updated_at":"2026-07-22T21:47:08.030629526Z"} +{"acceptance":"US-043 file exists with ACs for identity/safe-narrowing and negative DATE path if already covered by tests; FEAT-031 links US-043; no greenfield cast work.","created_at":"2026-07-22T21:41:07.78305571Z","description":"PARQ typed-raw cast path is SHIPPED (casting_utils typed_raw + test_casting_utils typed_raw DATE/TIMESTAMP + ingest generator parquet native typed). Author US-043 ACs citing those tests. No implement child remains (tablespec-502c6126 closed).","events_attachment":"tablespec-e9c21567/events.jsonl","id":"tablespec-e9c21567","issue_type":"task","labels":["helix","phase:frame","area:data","FEAT-031"],"notes":"Queue surgery 2026-07-22: re-scoped; PARQ implement closed as shipped.","owner":"erik","parent":"tablespec-263a0248","priority":1,"schema_version":1,"session_id":"alignment-queue-drain-2026-07-22","status":"closed","title":"Author US-043: parquet typed-raw AC backfill (story floor only)","updated_at":"2026-07-22T22:07:11.85870052Z"} {"acceptance":"1. A user can pass dialect=databricks through the documented Databricks compile path without ValueError, or all public docs/examples reject that spelling consistently and require dialect=spark. The implementation notes must state which decision was made.\n2. Error messages across casting, ingest, dbt, LDP, compile, and CLI agree on the accepted dialect values.\n3. Tests cover Databricks dialect handling at the lowest cast layer and at least one compile/emitter path.\n4. Docs and notebook examples explain the relationship between Databricks target/runtime and Spark-family cast SQL in one consistent way.\n5. A verification command such as uv run pytest tests/unit/test_casting_utils.py plus the added dialect tests passes.","created_at":"2026-06-08T00:21:05.838311929Z","description":"Resolve the current inconsistent Databricks dialect surface. Users compiling for Databricks can be told to pass dialect=spark because Databricks SQL is Spark-family for our cast expressions, but parts of the repo also expose dialect=databricks as a valid option. Current evidence: casting_utils.cast_column_sql accepts spark, databricks, and duckdb; compile_umfs forwards dialect to dbt DAG generation but emits LDP with spark; docs and CLI help mention databricks in some places while other comments/docs say only spark or duckdb. A Databricks notebook saw ValueError: Unsupported dialect: databricks expected spark or duckdb, which means either stale installed code or a downstream validator still rejects databricks. Canonicalize this so users and agents have one correct instruction.\n\nDecision to encode:\n- Public Databricks compile UX should accept dialect=databricks as an explicit alias for Spark-family cast SQL, or public docs should consistently require dialect=spark and never advertise databricks. Prefer accepting databricks everywhere as an alias because the CLI already advertises it and users naturally choose it for Databricks.\n- Internal rendering may normalize databricks to spark when emitters share identical SQL.\n- Error messages and docs must name the same accepted values.\n\nIn-scope files:\n- src/tablespec/casting_utils.py\n- src/tablespec/schemas/ingest_generator.py\n- src/tablespec/ldp/project.py\n- src/tablespec/e2e/compile.py\n- src/tablespec/cli.py\n- docs/helix and Databricks bootstrap/try-it-out docs\n- tests covering Databricks dialect acceptance through compile/emitter paths\n\nOut-of-scope for this bead:\n- One-shot bootstrap facade implementation\n- Production pipeline JSON/wheel packaging contract\n- Inline UMF format cleanup","events":[{"actor":"","body":"[{\"ac\":1,\"text\":\"A user can pass dialect=databricks through the documented Databricks compile path without ValueError, or all public docs/examples reject that spelling consistently and require dialect=spark. The implementation notes must state which decision was made.\",\"kind\":\"negative\",\"verifiable\":true},{\"ac\":2,\"text\":\"Error messages across casting, ingest, dbt, LDP, compile, and CLI agree on the accepted dialect values.\",\"kind\":\"prose\",\"verifiable\":false},{\"ac\":3,\"text\":\"Tests cover Databricks dialect handling at the lowest cast layer and at least one compile/emitter path.\",\"kind\":\"prose\",\"verifiable\":false},{\"ac\":4,\"text\":\"Docs and notebook examples explain the relationship between Databricks target/runtime and Spark-family cast SQL in one consistent way.\",\"kind\":\"mechanical\",\"verifiable\":true},{\"ac\":5,\"text\":\"A verification command such as uv run pytest tests/unit/test_casting_utils.py plus the added dialect tests passes.\",\"kind\":\"prose\",\"verifiable\":false}]","created_at":"2026-06-08T00:57:52.852988546Z","kind":"ac-quality-low","source":"preclaim-ac-quality","summary":"score=0.40 threshold=0.50 verifiable=2/5"},{"actor":"erik","body":"{\"ac_map\":[{\"parent_ac\":\"1. A user can pass dialect=databricks through the documented Databricks compile path without ValueError, or all public docs/examples reject that spelling consistently and require dialect=spark. The implementation notes must state which decision was made.\",\"coverage\":\"Covered by child 1 AC1-AC2, child 2 AC1-AC3, and child 3 AC1-AC3. Decision selected: accept dialect=databricks publicly as a Spark-family alias.\"},{\"parent_ac\":\"2. Error messages across casting, ingest, dbt, LDP, compile, and CLI agree on the accepted dialect values.\",\"coverage\":\"Covered by child 1 AC3 for shared low-level/dbt errors, child 2 AC1-AC2 for compile/LDP acceptance, and child 3 AC2 for CLI help/error wording.\"},{\"parent_ac\":\"3. Tests cover Databricks dialect handling at the lowest cast layer and at least one compile/emitter path.\",\"coverage\":\"Covered by child 1 AC1 TestDatabricksDialectAliasAtCastLayer and child 2 AC1 TestCompileUmfsAcceptsDatabricksDialect plus child 2 AC2 TestGenerateLdpProjectAcceptsDatabricksAlias.\"},{\"parent_ac\":\"4. Docs and notebook examples explain the relationship between Databricks target/runtime and Spark-family cast SQL in one consistent way.\",\"coverage\":\"Covered by child 3 AC1 TestDatabricksDocsUseCanonicalDialectGuidance and child 3 AC3 TestNoPublicDocsContradictDatabricksAlias.\"},{\"parent_ac\":\"5. A verification command such as uv run pytest tests/unit/test_casting_utils.py plus the added dialect tests passes.\",\"coverage\":\"Covered by child 1 AC4, child 2 AC4, and child 3 AC4, each including uv run pytest for added dialect tests plus go test ./... and lefthook run pre-commit gates.\"}],\"child_ids\":[\"tablespec-0b146671\",\"tablespec-171e409c\",\"tablespec-0fb0d1c2\"],\"rationale\":\"The parent bead mixes three separable work surfaces: shared dialect validation, compile/emitter threading, and public documentation/CLI guidance. Splitting them this way keeps each child independently executable with its own root cause, tests, and gates while preserving the parent decision to accept dialect=databricks as the public Databricks compile spelling and normalize internally only where Spark-family SQL is identical.\"}","created_at":"2026-06-08T01:04:03.67600191Z","kind":"triage-decomposed","source":"ddx work","summary":"decomposed into tablespec-0b146671, tablespec-171e409c, tablespec-0fb0d1c2"}],"execution-eligible":false,"id":"tablespec-ed74497c","issue_type":"task","labels":["area:databricks","area:dialects","area:docs","kind:bug","ac-quality:needs-refinement"],"priority":0,"schema_version":1,"spec-id":"FEAT-026","status":"open","title":"Canonicalize Databricks dialect handling across code and docs","updated_at":"2026-06-08T01:04:03.795284729Z"} {"acceptance":"1. python -c \"from tablespec.ingestion import get_reader, SourceReader, CsvReader, JdbcReader\" exits 0.\n2. python -c \"from tablespec.ingestion.raw_ingester import build_column_lookup, map_headers; from tablespec.ingestion.constants import normalize_spark_encoding\" exits 0 (merge.py's dangling imports resolve).\n3. Pre-existing merge tests pass unchanged after the CSV extraction: pytest tests -k merge exits 0 with no test file modified in the same commit as the extraction.\n4. pytest tests -k source_spec exits 0, asserting: (a) a UMF with source kind=jdbc + url + dbtable + password_secret_ref validates; (b) a UMF omitting source: validates as delimited (file_format alias); (c) a JDBC spec with a plaintext password field raises ValidationError.\n5. docs/helix/02-design/adr/ADR-015-source-shape-contract.md exists with Status Accepted (supersedes the originally-globbed ADR-*-typed-source-raw-landing.md name; Option B native-typed raw was decided).\n6. pytest tests -k jdbc_umf_mapper exits 0, including a case piping a mapper-generated UMF through tablespec validate successfully; the mapper connects ONLY via spark.read.format('jdbc') (no direct DB driver imports in src/tablespec).\n7. pytest tests/integration -k jdbc_reader exits 0 against a real SQL Server container when Docker is available, SKIPPED (not failed) when Docker is absent.\n8. US-039 Northwind end-to-end ACs pass on the Docker lane; the databricks_e2e opt-in lane runs the same flow when workspace credentials are configured.","created_at":"2026-06-11T02:20:33.00301656Z","description":"## Context\n\ntablespec's raw layer currently assumes flat files. The entropy-exchange `mssql_import` bundle (entropy-exchange PR #66, `posts/databricks/mssql_import/`) demonstrates the next class of input we need: an entire SQL Server database (restored from a `.bak` on a Databricks driver node) landed as Delta tables. Today that bundle hand-rolls table enumeration, column sanitization, and writes — none of it spec-driven, no validation, no merge/dedup. We're going to need a wider variety of inputs (JDBC databases first, likely Parquet/API sources later), so the source side of raw ingestion needs a real abstraction instead of inlined CSV reads.\n\n## Current state (verified 2026-06-10)\n\n- `merge.py:84-107` hardcodes CSV reading via `spark.read...csv()`.\n- `merge.py:25,34` imports `tablespec.ingestion.constants` and `tablespec.ingestion.raw_ingester` — **the `tablespec.ingestion` package does not exist**. The seam is already named but was never built.\n- `FileFormatSpec` (`models/umf.py:128-156`) is flat-file-shaped: delimiter, encoding, header, quote/escape chars, filename_pattern, source_directory.\n- No mention of JDBC/mssql/database sources anywhere in src.\n- Raw layer contract (ADR-007/ADR-012): all-STRING `raw_\u003ctable\u003e` landing tables + committed SQL artifacts for raw→ingested casting. This assumption is load-bearing and conflicts with typed sources (see Phase 4).\n\n## Plan of attack\n\n### Phase 1 — Create `tablespec.ingestion` package (no behavior change)\n- `SourceReader` protocol: `read(spec: SourceSpec, spark: SparkSession) -\u003e DataFrame`.\n- `CsvReader`: extract the existing read logic from `merge.py:84-107` verbatim.\n- Implement the dangling imports so `merge.py` works as written: `raw_ingester.build_column_lookup()`, `raw_ingester.map_headers()`, `constants.normalize_spark_encoding()`.\n- Reader factory: `get_reader(spec) -\u003e SourceReader` dispatching on source type.\n- Wire `merge.py` (and `e2e/backbone.py` stage 1) through the factory.\n\n### Phase 2 — UMF model: source specification\n- Add a `source_type: Literal[\"csv\", \"jdbc\", ...]` discriminator. Either extend `FileFormatSpec` or (preferred) introduce a `SourceSpec` union so flat-file fields don't pollute JDBC specs.\n- JDBC fields: `url`, `dbtable`/`query`, `driver`, `user`, `password_secret_ref` (Databricks secret scope ref or env var name — never a literal credential in a committed UMF), `fetch_size`, optional partition column/bounds for parallel reads.\n- Keep backward compatibility: absent `source_type` defaults to `csv` so existing UMFs are untouched.\n\n### Phase 3 — `JdbcReader`\n- `spark.read.format(\"jdbc\")` with options from the spec; secret resolution at read time.\n- Identifier handling: bracket/backtick quoting, plus the sanitization rules already proven in the mssql_import bundle (lowercase, non-alphanumerics → underscore, collapse repeats).\n\n### Phase 4 — ADR: typed sources vs all-STRING raw landing\nThis is the one real architectural fork and deserves its own ADR before Phase 3 merges:\n- Option A: force JDBC data through string-landing (uniform pipeline, raw-stage validation rules apply, but lossy/wasteful round-trip of dates/decimals — a format-bug magnet).\n- Option B (recommended): allow `raw_\u003ctable\u003e` to carry source-native types; `generate_ingest_sql()` emits pass-through or cast-light transforms when the source is typed. The UMF already knows target types, so the generator has everything it needs. Dedup/merge/validation machinery is unaffected.\n\n### Phase 5 — UMF discovery from live databases\n- `JdbcToUmfMapper` (analog of the existing `SparkToUmfMapper`): read `INFORMATION_SCHEMA` (columns, types, nullability, PKs, FKs) and emit one UMF per table.\n- This is the payoff: point tablespec at a restored database and get specs + validated, spec-driven ingestion for every table, instead of a blind bulk copy.\n\n### Phase 6 — Integrate with the entropy-exchange bundle\n- Replace the hand-rolled `03_etl.py` loop with: discover UMFs (Phase 5) → land raw via `JdbcReader` (Phase 3) → standard ingest transforms. The bundle keeps owning SQL Server install/restore (Databricks-runtime plumbing, not library material).\n\n## Testing strategy\n\n- Unit: spec model validation, factory dispatch, header mapping, encoding normalization (no Spark needed for the model layer).\n- CSV reader: existing merge tests must pass unchanged after the Phase 1 extraction — that is the proof of no behavior change.\n- JDBC integration: Docker-gated test against a real SQL Server container (mirror how entropy-exchange gates Docker tests). Test the real reader code, not a copy.\n\n## Suggested decomposition\n\nPhases 1+2 are one PR (pure refactor + models). Phase 4 ADR lands before Phase 3 code. Phases 5 and 6 are independent follow-ups. Split into child beads when work starts.","events":[{"actor":"","body":"closed_because: all_children_closed\nexecution_eligible: false\ntotal_children: 7","created_at":"2026-06-14T02:20:21.398363502Z","kind":"dead_intermediate_close","source":"Store.Close.walkUpClosureCandidate","summary":"closed as dead-intermediate bead (all children closed, execution-eligible=false)"}],"execution-eligible":false,"id":"tablespec-ef91646f","issue_type":"epic","labels":["helix","phase:design","kind:feature","area:ingestion"],"notes":"Parked 2026-06-11 after failed ddx work on tablespec-a169e060: this epic and related source-shape docs were generated outside the microsite/deployment bead scope and reference artifacts not committed in main. Requires operator review/acceptance or rewrite before execution.","operator_decisions":"2026-06-10: (1) discriminated source: block, file_format stays as delimited alias; (2) native-typed raw for parquet/jdbc with identity/safe-narrowing casts; (3) JDBC compiled-artifacts-only, ALL connectivity via Spark's JDBC connector incl. discovery (no pyodbc/JayDeBeApi; reuse SparkToUmfMapper for type mapping); (4) sequencing: seam+model -\u003e JDBC vertical (Northwind goal) -\u003e dumps -\u003e parquet","operator_note":"2026-06-10: operator explicitly directed this epic (incorporate mssql bead suggestions; Northwind end-to-end goal; four design forks answered). Un-parked from operator-review/P4. Governed by ADR-015/FEAT-031/FR-21; seam phase committed 3a881cc.","priority":1,"schema_version":1,"spec_anchors":"ADR-015-source-shape-contract.md (Accepted), FEAT-031-multi-source-ingestion.md (Specified), PRD FR-21.1-FR-21.6, US-039-northwind-end-to-end.md (acceptance goal)","status":"closed","title":"Ingestion source abstraction: beyond flat files (JDBC/SQL Server first)","updated_at":"2026-06-14T02:20:21.401276541Z"} {"acceptance":"implementation-plan-v2 is either archived/replaced or updated to current state; future-tense completed work is rewritten with shipped evidence; remaining planned work is separated into active DDx beads; no stale TODO/proposed language remains for implemented features.","closing_commit_sha":"fcfa48e533ccb505389e8a3cc62fbe587919151a","created_at":"2026-06-07T20:59:34.024775138Z","description":"docs/helix/04-build/implementation-plan-v2.md is still Proposed and describes phases as future work, including a TODO stub for ProfileToGxMapper/validation pipeline work that is now implemented and tested. This creates top-down drift between PRD v3, feature specs, and build guidance.","id":"tablespec-f182b786","issue_type":"task","labels":["spec","helix","build-plan"],"owner":"erik","priority":2,"schema_version":1,"status":"closed","title":"Replace stale implementation-plan-v2 future-work narrative","updated_at":"2026-06-09T20:52:46.785721098Z"} diff --git a/apps/data-profiling/tests/test_fr23_stack.py b/apps/data-profiling/tests/test_fr23_stack.py new file mode 100644 index 00000000..e6ab0b64 --- /dev/null +++ b/apps/data-profiling/tests/test_fr23_stack.py @@ -0,0 +1,57 @@ +"""Whole-stack unit path for FR-23 (config → provision → startup validate). + +Satisfies the e2e-framework slot for the Databricks App at the unit level: +one command path exercises resolve + provision + diagnostics without a live +workspace. Live deploy-and-drive remains operational evidence. +""" + +from __future__ import annotations + +from pathlib import Path + +from profiler.config import resolve_config +from profiler.diagnostics import validate_config +from profiler.provision import ProvisionReport, provision + + +class _EmptyExecutor: + """Empty environment: nothing exists yet.""" + + def __init__(self) -> None: + self.statements: list[str] = [] + + def execute(self, statement: str) -> None: + self.statements.append(statement) + + def query(self, statement: str) -> list[list]: + return [] + + +def test_fr23_resolve_provision_validate_compose(tmp_path: Path) -> None: + """Config resolve → idempotent provision → startup validation compose.""" + registry = tmp_path / "connections.yaml" + registry.write_text("connections: []\n", encoding="utf-8") + + cfg = resolve_config( + env={ + "PROFILER_METADATA_CATALOG": "stack_cat", + "PROFILER_METADATA_SCHEMA": "stack_schema", + "PROFILER_OUTPUT_VOLUME": "stack_vol", + "DATABRICKS_WAREHOUSE_ID": "wh-stack", + "PROFILER_RUNTIME": "mock", + }, + registry_path=str(registry), + ) + assert cfg.metadata_catalog == "stack_cat" + assert cfg.source_of("metadata_catalog") == "deployment" + + report = provision(cfg, executor=_EmptyExecutor(), grant_to=None) + assert isinstance(report, ProvisionReport) + assert report.changed is True + + # Second provision against a fake that reports everything already present + # is covered in test_provision; here we only require composition succeeds. + faults = validate_config(cfg) + # mock runtime without a live warehouse probe should not raise; faults may + # be empty or advisory depending on runtime probes — must be a list. + assert isinstance(faults, list) diff --git a/docs/helix/01-frame/concerns.md b/docs/helix/01-frame/concerns.md index eb76f44d..9a2db99b 100644 --- a/docs/helix/01-frame/concerns.md +++ b/docs/helix/01-frame/concerns.md @@ -38,7 +38,7 @@ shipped-default → recorded-assumption: |------|--------|--------|-----------| | language-runtime | python-uv | assumption | Shipped default (`typescript-bun`) contradicts the product: tablespec is a Python 3.12+ library (`pyproject.toml`). Recorded as assumption per resolution order. | | frontend-framework | Streamlit for the operational app; Hugo + Hextra for documentation | assumption | Two distinct UI surfaces with different jobs. Streamlit backs the deployed Databricks App (`apps/data-profiling/`), an operator UI; Hextra backs the FEAT-030 documentation microsite. The library core itself remains non-UI. Supersedes the prior record, which asserted the product had no operational app UI. | -| e2e-framework | Playwright for microsite; conformance harness for library runtime; **none yet for the Databricks App** (tracked in alignment epic / app e2e bead) | assumption | Browser e2e applies to FEAT-030 navigation and rendering; the library runtime is covered by compile/backbone/conformance tests. The Streamlit app has **no whole-stack exercise** — selecting a tool is not coverage, so this is recorded as an open gap rather than a filled slot. See Concern Conflicts and [AR-2026-07-22](../06-iterate/alignment-reviews/AR-2026-07-22-desired-state-realignment.md). | +| e2e-framework | Playwright for microsite; conformance harness for library runtime; **unit whole-stack path for the Databricks App** (`apps/data-profiling/tests/test_fr23_stack.py`: resolve → provision → validate_config) | assumption | Browser e2e applies to FEAT-030 navigation and rendering; the library runtime is covered by compile/backbone/conformance tests. The Streamlit app has a composed unit path for FR-23; live deploy-and-drive remains operational evidence. See Concern Conflicts and [AR-2026-07-22](../06-iterate/alignment-reviews/AR-2026-07-22-desired-state-realignment.md). | | auth-provider | — (not applicable) | assumption | No accounts, tenants, or sign-in surface. | | datastore | — (not applicable) | assumption | The library holds no state of its own; compiled artifacts target the consumer's platform (Databricks/UC, DuckDB in tests). | | deploy-target | Python package plus GitHub Pages docs/package index, **and a Databricks App** | assumption | Releases publish wheel/sdist and the Pages package index; FEAT-030 adds a Hugo microsite to the same Pages artifact while preserving `/simple/`. FEAT-034 adds a third target: `apps/data-profiling/` deploys as a Databricks App from its own subdirectory (the full repository exceeds the Apps file-count limit) and installs tablespec as a dependency rather than shipping the repository. | diff --git a/docs/helix/01-frame/feature-registry.md b/docs/helix/01-frame/feature-registry.md index 5958af2a..97d996cf 100644 --- a/docs/helix/01-frame/feature-registry.md +++ b/docs/helix/01-frame/feature-registry.md @@ -42,8 +42,8 @@ ddx: | FEAT-028 | [LDP Sibling Emitter](features/FEAT-028-ldp-sibling-emitter.md) | Emit a Lakeflow Declarative Pipelines project from a UMF set as a sibling backend on the shared target-agnostic core. | Built | P1 | Platform / Compilation | PRD: Multi-Target Emission (FR-19.3, FR-19.1) | 2026-06-10 | | FEAT-029 | [Runtime Platform](features/FEAT-029-runtime-platform.md) | Connect-safe session capability probing and engine-correct `functions` dispatch. | Built | P0 | Data Platform | PRD: Runtime Platform (FR-20.1, FR-20.2, FR-20.3) | 2026-06-10 | | FEAT-030 | [Product Microsite](features/FEAT-030-product-microsite.md) | Hugo/Hextra product microsite with reader-mode IA, source-semantic bronze explanation, demos, and Pages deployment that preserves `/simple/`. | Built | P1 | Platform / Developer Experience | Vision/Principles (meta-feature; governed by ADR-014) | 2026-07-22 | -| FEAT-031 | [Multi-Source Ingestion](features/FEAT-031-multi-source-ingestion.md) | Discriminated `source:` contract (delimited/parquet/jdbc/json) with kind-dependent raw typing, ingestion reader seam, JDBC compiled read specs, and database discovery; Northwind end-to-end is the acceptance goal. DUMP/PARQ/JDBC shipped; JSON compile/backbone residual + story floor remain. | Specified | P1 | Platform / Data Engineering | PRD: Source Acquisition (FR-21.1–FR-21.7); ADR-015 | 2026-07-22 | -| FEAT-032 | [Embeddings & Document Corpus](features/FEAT-032-embeddings-and-document-corpus.md) | Dimensioned EMBEDDING type compiling to ARRAY, GX dimensionality validation, and the governed document-corpus contract; SEC 10-K demo is the acceptance goal. Type core + CORP example shipped; DEMO residual open. | In Build | P1 | Platform / Data Engineering | PRD: UMF Model and I/O (FR-1.11); ADR-016 | 2026-07-22 | +| FEAT-031 | [Multi-Source Ingestion](features/FEAT-031-multi-source-ingestion.md) | Discriminated `source:` contract (delimited/parquet/jdbc/json) with kind-dependent raw typing, ingestion reader seam, JDBC compiled read specs, and database discovery; Northwind end-to-end is the acceptance goal. DUMP/PARQ/JDBC/JSON cores + story floor US-040/042/043/050 shipped. | Specified | P1 | Platform / Data Engineering | PRD: Source Acquisition (FR-21.1–FR-21.7); ADR-015 | 2026-07-22 | +| FEAT-032 | [Embeddings & Document Corpus](features/FEAT-032-embeddings-and-document-corpus.md) | Dimensioned EMBEDDING type compiling to ARRAY, GX dimensionality validation, and the governed document-corpus contract; SEC 10-K demo is the acceptance goal. Type core + CORP example + notebook contract shipped; workspace job not CI-gated. | Built | P1 | Platform / Data Engineering | PRD: UMF Model and I/O (FR-1.11); ADR-016 | 2026-07-22 | | FEAT-033 | [Guidebook](features/FEAT-033-guidebook.md) | Render a directory of UMFs into a navigable, self-contained HTML guidebook — one page per table — with column metadata, FK + derivation lineage, group/flat indexes, and search; CLI + `generate_guidebook` API. | Built | P1 | Platform / Developer Experience | PRD: Guidebook (FR-22.1–FR-22.4); ADR-018 | 2026-07-22 | | FEAT-034 | [App Deployment & Configuration](features/FEAT-034-app-deployment-configuration.md) | Deploy the guidebook + profiling app into any Databricks environment: metadata location as declared input, idempotent provisioning of schema/volume/governance tables, and fail-fast startup validation. Desired state; implementation gaps tracked in alignment beads. | Specified | P1 | Platform / Developer Experience | PRD: App Deployment & Configuration (FR-23.1–FR-23.6) | 2026-07-22 | diff --git a/docs/helix/01-frame/features/FEAT-031-multi-source-ingestion.md b/docs/helix/01-frame/features/FEAT-031-multi-source-ingestion.md index d75829e8..712e70e3 100644 --- a/docs/helix/01-frame/features/FEAT-031-multi-source-ingestion.md +++ b/docs/helix/01-frame/features/FEAT-031-multi-source-ingestion.md @@ -6,7 +6,7 @@ ddx: # Feature Specification: FEAT-031 — Multi-Source Ingestion (Source-Shape Contract) **Feature ID**: FEAT-031 -**Status**: Specified (DUMP/PARQ/JDBC cores shipped; JSON compile/backbone residual + US-040/042/043/050 story backfill remain) +**Status**: Specified (DUMP/PARQ/JDBC/JSON cores + story floor US-040/042/043/050 shipped; demo residual on US-044/045) **Priority**: P1 **Owner**: Platform / Data Engineering **Covered PRD Subsystem(s)**: Source Acquisition @@ -25,8 +25,8 @@ the emitters (FEAT-026/027/028) and the raw-suite generator consume it. > | SUITE typed-raw slice | **Shipped** | Typed raw suite path for parquet/jdbc/json kinds | > | DUMP-01..04 | **Shipped** | Model options + dump reader + `tests/unit/test_ingestion_package.py` (skip/footer/null_escape/line_terminator) | > | PARQ identity/safe-narrowing | **Shipped** | Typed-raw cast path in `casting_utils` + `test_casting_utils.py` typed_raw DATE/TIMESTAMP; ingest generator parquet native typed | -> | JSON-01 (FR-21.7) | **Partial** | Model + `JsonReader` shipped; **backbone still delimited/parquet only** — residual bead `tablespec-9f98cf03` | -> | Story floor | **Partial** | US-039/044 on disk; US-040/042/043/050 story backfill beads remain (US-041 cancelled as duplicate of US-039) | +> | JSON-01 (FR-21.7) | **Shipped** | Model + `JsonReader` + backbone `_declared_source`/`_load_raw` (Spark + DuckDB) + conformance JSON tier | +> | Story floor | **Shipped** | US-039/040/042/043/044/050 on disk; US-041 cancelled as duplicate of US-039 | ## Overview @@ -186,7 +186,7 @@ underscores collapsed (the rules proven in the entropy-exchange `mssql_import` bundle), with bracket/backtick quoting handled at the read boundary and the original source identifier preserved in the spec. -#### JSON/JSONL source kind (FR-21.7 — planned, operator-decided 2026-06-12) +#### JSON/JSONL source kind (FR-21.7 — shipped; operator-decided 2026-06-12) JSON-01. The `source:` block SHALL gain a `json` kind: JSON/JSONL files land typed through Spark's reader, under the same native-typed raw regime @@ -252,21 +252,22 @@ expectation-stage classification (`umf.py:94-112`, — the acceptance-goal story for the JDBC vertical: load the Northwind database, discover UMF specs, export a schema xlsx, validate, generate sample data, and produce a validation report. - -Per-phase story work remaining on the alignment epic (`tablespec-263a0248`): - -- US-040 — seam + `source:` model AC backfill (bead `tablespec-20513f4f`; - implementation shipped — story floor only) -- US-042 — dump-dialect AC backfill citing existing dump tests (bead - `tablespec-e322b612`; DUMP implement closed as shipped) -- US-043 — parquet typed-raw AC backfill citing existing cast tests (bead - `tablespec-e9c21567`; PARQ implement closed as shipped) -- US-050 — JSON residual story for compile/backbone (bead `tablespec-557f8a24`) +- [US-040 — Source model and ingestion seam](../user-stories/US-040-source-model-and-ingestion-seam.md) + — SRC-01..05 AC floor citing `test_source_spec` + `test_ingestion_package`. +- [US-042 — Dump-dialect text landing](../user-stories/US-042-dump-dialect-text-landing.md) + — DUMP-01..04 AC floor citing dump reader tests. +- [US-043 — Parquet typed-raw landing](../user-stories/US-043-parquet-typed-raw-landing.md) + — PARQ AC floor citing `test_casting_utils` typed_raw paths. +- [US-050 — JSON/JSONL source kind](../user-stories/US-050-json-jsonl-source-kind.md) + — FR-21.7 model/reader/backbone path. JDBC vertical is covered by [US-039](../user-stories/US-039-northwind-end-to-end.md) (no separate US-041). Demo stories already on disk: [US-044 — Kaggle flat-file onboarding](../user-stories/US-044-kaggle-flat-file-onboarding.md) -(delimited kind, shipped code, notebook pair). +(delimited kind, shipped code, notebook pair); +[US-045 — SEC 10-K corpus and facts](../user-stories/US-045-sec-10k-corpus-and-facts.md) +(EMBEDDING type + CORP example shipped; residual demo evidence tracked +separately). ## Edge Cases and Error Handling @@ -360,16 +361,16 @@ JDBC vertical is covered by [US-039](../user-stories/US-039-northwind-end-to-end - [x] Ideal future state describes the desired user-visible outcome - [x] Problem statement describes what exists now and what is broken - [x] Every functional requirement is testable -- [ ] Acceptance criteria are defined in user stories — US-039 authored; - US-040..043 planned per phase +- [x] Acceptance criteria are defined in user stories — US-039/040/042/043/050 + authored with stable ACs - [x] Non-functional requirements have specific targets - [x] Edge cases cover realistic failure scenarios - [x] Success metrics are specific to this feature - [x] Dependencies reference real artifact IDs - [x] Out of scope excludes things someone might reasonably assume are in scope -- [x] Implementation status is honestly per-phase (seam done; JDBC, dumps, - parquet planned); no phantom completion claims +- [x] Implementation status is honestly per-phase (seam, JDBC, dumps, parquet, + JSON shipped; demo residual called out); no phantom completion claims - [x] Feature is consistent with governing ADR-015 and the PRD Non-Goal on database connectivity - [x] No `[NEEDS CLARIFICATION]` markers remain diff --git a/docs/helix/01-frame/features/FEAT-032-embeddings-and-document-corpus.md b/docs/helix/01-frame/features/FEAT-032-embeddings-and-document-corpus.md index a9abd962..46e4052e 100644 --- a/docs/helix/01-frame/features/FEAT-032-embeddings-and-document-corpus.md +++ b/docs/helix/01-frame/features/FEAT-032-embeddings-and-document-corpus.md @@ -24,7 +24,7 @@ is consumed, per ADR-016. > | Type mappings (SQL/PySpark/JSON/GX) | **Shipped** | `type_mappings.py`; unit tests | > | Schema generators + GX baseline + sample data + compatibility | **Shipped** | Tests under `tests/unit/test_*` for generators, gx_baseline, column_value_generator, compatibility | > | CORP document-corpus pattern example | **Shipped** | Canonical example: [`examples/sec10k_corpus.yaml`](../../../../examples/sec10k_corpus.yaml) (CORP-01 pattern, `EMBEDDING(1024)`) | -> | DEMO SEC 10-K (US-045) | **Partial** | Notebooks + example YAML exist; residual AC evidence on bead `tablespec-abd68023` | +> | DEMO SEC 10-K (US-045) | **Shipped (CI residual recorded)** | Type/example/notebook contract evidenced; workspace job not CI-gated (US-045 limitations) | > > The facts-table half of the SEC demo may use FEAT-031's `json` source kind. diff --git a/docs/helix/01-frame/prd.md b/docs/helix/01-frame/prd.md index f827ee23..b142be8b 100644 --- a/docs/helix/01-frame/prd.md +++ b/docs/helix/01-frame/prd.md @@ -209,7 +209,7 @@ edits; do not renumber on edit. ### Subsystem: Source Acquisition -**FR-21** requirement family. *Governed by FEAT-031; decision recorded in ADR-015. The raw landing contract generalizes from flat files to declared source shapes. Phase honesty (2026-07-22, queue surgery): delimited/parquet/jdbc cores + DUMP options + PARQ typed-raw cast + JDBC discovery (US-039) are shipped; remaining implementation residual is primarily JSON compile/backbone (FR-21.7) plus story-floor backfill for US-040/042/043/050.* +**FR-21** requirement family. *Governed by FEAT-031; decision recorded in ADR-015. The raw landing contract generalizes from flat files to declared source shapes. Phase honesty (2026-07-22, alignment drain): delimited/parquet/jdbc/json cores + DUMP options + PARQ typed-raw cast + JDBC discovery (US-039) + story floor US-040/042/043/050 are shipped; remaining residual is primarily demo notebook evidence (US-044/045).* - **FR-21.1** — **Declared source shape.** UMF carries a discriminated `source:` block (`kind: delimited | parquet | jdbc | json`); today's `file_format` is the body of the `delimited` variant and remains a back-compat alias resolved via a non-persisting accessor. Raw landing typing follows the kind: all-STRING for text-landed sources, native-typed for typed sources. - **FR-21.2** — **Dump-dialect text landing.** The `delimited` variant covers database dump files: multi-character line terminators, `\N`-style null escapes, footer handling, and `skip_rows` honored end-to-end — every declared option actually consumed by the compiled readers. *(Shipped — model + dump reader + unit tests.)* @@ -217,7 +217,7 @@ edits; do not renumber on edit. - **FR-21.4** — **JDBC compiled read spec.** JDBC sources compile to a committed read spec carrying connection parameters (url, dbtable/query, driver, fetch/partitioning) with credentials *only* as named secret references (Databricks secret scope or env-var name); a literal credential fails validation. tablespec never opens a connection — all connectivity is Spark's JDBC connector, executed by the runtime. - **FR-21.5** — **Raw-suite typing.** Raw-stage expectation suites vary by raw typing: string checks (length/regex/strftime/castability) apply only to all-STRING raw; typed raw receives schema-type expectations instead. - **FR-21.6** — **Database discovery.** Discover UMF specs from a live database: enumerate tables and read `INFORMATION_SCHEMA` metadata (columns, types, nullability, PKs, FKs) through Spark's JDBC connector, reusing the existing Spark schema→UMF mapping; emit one validated UMF per table so database onboarding is spec-driven rather than a blind bulk copy. -- **FR-21.7** — **JSON/JSONL source kind.** A `json` variant of the `source:` block: JSON/JSONL files land typed through Spark's reader (like parquet); the spec declares a FLAT projection — each UMF column maps to a top-level field or an explicit dot-path; un-projected nesting is out of the bronze contract (no recursive flattening). *Model and reader shipped; compile/backbone/demo residual gaps remain desired completion. Extends FEAT-031.* +- **FR-21.7** — **JSON/JSONL source kind.** A `json` variant of the `source:` block: JSON/JSONL files land typed through Spark's reader (like parquet); the spec declares a FLAT projection — each UMF column maps to a top-level field or an explicit dot-path; un-projected nesting is out of the bronze contract (no recursive flattening). *Model, reader, and backbone load path shipped (US-050). Extends FEAT-031.* ### Subsystem: CLI Interface diff --git a/docs/helix/01-frame/user-stories/US-040-source-model-and-ingestion-seam.md b/docs/helix/01-frame/user-stories/US-040-source-model-and-ingestion-seam.md new file mode 100644 index 00000000..4df03167 --- /dev/null +++ b/docs/helix/01-frame/user-stories/US-040-source-model-and-ingestion-seam.md @@ -0,0 +1,75 @@ +--- +ddx: + id: US-040 +--- + +# US-040: Source Model and Ingestion Seam (Story Floor) + +**Feature**: FEAT-031 — Multi-Source Ingestion (Source-Shape Contract) +**Feature Requirements**: SRC-01, SRC-02, SRC-03, SRC-04, SRC-05 +**PRD Requirements**: FR-21.1 +**Priority**: P1 +**Status**: Built (implementation shipped; this story backfills AC floor) + +## Story + +**As a** data engineer declaring how a table is acquired +**I want** a discriminated `source:` block on UMF and a single ingestion reader seam +**So that** delimited, parquet, JDBC, and JSON sources share one contract and +reader dispatch without embedding connectivity or reader options in callers. + +## Context + +SRC-01..05 are **shipped**. This story is frame-only backfill: stable +`US-040-ACn` IDs map to existing unit tests. Do not re-implement the seam. + +Governing code: + +- `src/tablespec/models/umf.py` — `DelimitedSource`, `ParquetSource`, + `JsonSource`, `JdbcSource`, `file_format` alias +- `src/tablespec/ingestion/` — `get_reader`, kind-specific readers + +## Walkthrough + +1. Author declares `source: {kind: …}` (or legacy `file_format:` for delimited). +2. UMF model validates the discriminated union and rejects unknown kinds. +3. `get_reader(spec)` returns the reader for that kind. +4. Callers never open JDBC connections or hardcode Spark reader options outside + the seam. + +## Acceptance Criteria + +- [x] **US-040-AC1 (discriminated source kinds)** — Given a UMF with + `source.kind` of `delimited`, `parquet`, `json`, or `jdbc`, when the model + validates, then the matching source type is accepted and an unknown kind is + rejected. + **Evidence**: `tests/unit/test_source_spec.py` + (`TestDiscriminator`, `TestJdbcSource`, `TestFileFormatAlias`). +- [x] **US-040-AC2 (file_format alias)** — Given a UMF with only + `file_format:` and no `source:`, when resolved, then a delimited source with + the declared options is produced and a declared `source:` wins over + `file_format`. + **Evidence**: `tests/unit/test_source_spec.py` (`TestFileFormatAlias`). +- [x] **US-040-AC3 (reader seam dispatch)** — Given a valid source of each + shipped kind, when `get_reader` is called, then the correct reader class is + returned and an unknown kind raises `ValueError`. + **Evidence**: `tests/unit/test_ingestion_package.py` (dispatch tests for + delimited/parquet/json/jdbc/unknown). +- [x] **US-040-AC4 (round-trip stability)** — Given a UMF with a declared + source, when serialized to YAML and reloaded, then the source block is + preserved without loss for delimited, jdbc, and json shapes. + **Evidence**: `tests/unit/test_source_spec.py` (`test_source_round_trips_*`, + `test_legacy_umf_round_trip_is_byte_identical`). + +## Edge Cases + +- **plaintext JDBC password**: rejected by model (`extra=forbid` / + secret-ref only) — see `TestJdbcSource.test_plaintext_password_raises`. +- **json projection must cover UMF columns**: validated in + `test_json_source_projection_*`. + +## Dependencies + +- **Feature Spec**: FEAT-031 +- **Decisions**: ADR-015 +- **Work**: bead `tablespec-20513f4f` (story floor only) diff --git a/docs/helix/01-frame/user-stories/US-042-dump-dialect-text-landing.md b/docs/helix/01-frame/user-stories/US-042-dump-dialect-text-landing.md new file mode 100644 index 00000000..5575f536 --- /dev/null +++ b/docs/helix/01-frame/user-stories/US-042-dump-dialect-text-landing.md @@ -0,0 +1,59 @@ +--- +ddx: + id: US-042 +--- + +# US-042: Dump-Dialect Text Landing (Story Floor) + +**Feature**: FEAT-031 — Multi-Source Ingestion (Source-Shape Contract) +**Feature Requirements**: DUMP-01, DUMP-02, DUMP-03, DUMP-04 +**PRD Requirements**: FR-21.2 +**Priority**: P1 +**Status**: Built (implementation shipped; this story backfills AC floor) + +## Story + +**As a** data engineer landing database dump files +**I want** delimited sources to honor multi-character line terminators, +null escapes, footer rows, and skip rows +**So that** dump dialects compile and read without silent mis-parses. + +## Context + +DUMP-01..04 are **shipped** (model options + dump reader). Implement child +`tablespec-7ec86390` is closed. This story only records AC evidence. + +## Walkthrough + +1. Author declares dump options on a `source: {kind: delimited, …}` block. +2. Dump reader normalizes skip/footer/null_escape/line_terminator. +3. Unit tests assert record materialization matches expected rows. + +## Acceptance Criteria + +- [x] **US-042-AC1 (DUMP-01 multi-character line terminator)** — Given a + delimited source with a multi-character `line_terminator`, when dump records + are read, then records split on that terminator. + **Evidence**: `tests/unit/test_ingestion_package.py` + (`test_dump_reader_normalizes_skip_footer_and_null_escape`, + `test_dump_reader_uses_normalized_records`). +- [x] **US-042-AC2 (DUMP-02 null escape)** — Given `null_escape: "\\N"`, when + a field equals that token, then the landed value is null/empty per reader + contract. + **Evidence**: same dump reader tests (fixture rows with `\\N`). +- [x] **US-042-AC3 (DUMP-03 footer rows)** — Given `footer_rows: N`, when + dump records are read, then the last N records are excluded from data. + **Evidence**: same dump reader tests (`footer_rows=1`). +- [x] **US-042-AC4 (DUMP-04 skip_rows)** — Given `skip_rows: N`, when dump + records are read, then the first N lines are skipped before header/data. + **Evidence**: same dump reader tests (`skip_rows=2`). + +## Edge Cases + +- Headerless dumps require synthesized metadata columns in backbone/conformance + loaders (not re-tested here; covered by backbone delimited quirks path). + +## Dependencies + +- **Feature Spec**: FEAT-031 +- **Work**: bead `tablespec-e322b612` (story floor only) diff --git a/docs/helix/01-frame/user-stories/US-043-parquet-typed-raw-landing.md b/docs/helix/01-frame/user-stories/US-043-parquet-typed-raw-landing.md new file mode 100644 index 00000000..9f441c8b --- /dev/null +++ b/docs/helix/01-frame/user-stories/US-043-parquet-typed-raw-landing.md @@ -0,0 +1,67 @@ +--- +ddx: + id: US-043 +--- + +# US-043: Parquet Typed-Raw Landing (Story Floor) + +**Feature**: FEAT-031 — Multi-Source Ingestion (Source-Shape Contract) +**Feature Requirements**: PARQ-01, PARQ-02, PARQ-03 +**PRD Requirements**: FR-21.3 +**Priority**: P1 +**Status**: Built (implementation shipped; this story backfills AC floor) + +## Story + +**As a** data engineer landing parquet files +**I want** native-typed raw and identity/safe-narrowing casts +**So that** typed DATE/TIMESTAMP columns are never routed through string +format parsers that silently NULL values. + +## Context + +PARQ typed-raw path is **shipped** (`casting_utils` typed_raw + ingest +generator). Implement child `tablespec-502c6126` is closed. This story only +records AC evidence. + +## Walkthrough + +1. UMF declares `source: {kind: parquet}`. +2. Raw lands with native parquet types (not all-STRING). +3. `cast_column_sql(..., source_kind="parquet")` emits identity/safe-narrowing + casts for DATE/TIMESTAMP. + +## Acceptance Criteria + +- [x] **US-043-AC1 (PARQ-01 native typed raw)** — Given a parquet source, + when ingest SQL is generated, then the path is native-typed (not forced + all-STRING raw). + **Evidence**: ingest generator coverage for parquet native typed; + `tests/unit/test_casting_utils.py` typed_raw matrix. +- [x] **US-043-AC2 (PARQ-02 identity/safe-narrowing DATE)** — Given a + DATE column from parquet, when `cast_column_sql` runs with + `source_kind="parquet"`, then the cast is safe identity-style + (`cast(... as date)` / `try_cast`) rather than string format parse. + **Evidence**: `tests/unit/test_casting_utils.py` + (`test_typed_raw_date_uses_safe_cast`). +- [x] **US-043-AC3 (PARQ-02 TIMESTAMP)** — Given a TIMESTAMP column from + parquet, when cast SQL is built with typed_raw, then timestamp safe cast is + emitted. + **Evidence**: `tests/unit/test_casting_utils.py` + (`test_typed_raw_timestamp_uses_safe_cast`). +- [x] **US-043-AC4 (negative: string formats not applied on typed raw)** — + Given typed raw, when building casts, then registered string date formats + are not the selected path for parquet/json/jdbc kinds. + **Evidence**: same typed_raw matrix parametrizes `parquet` (and peers) + away from string `try_to_timestamp` / strptime paths. + +## Edge Cases + +- ADR-001 yyyymmdd-string convention applies only to all-STRING (delimited) + raw, not parquet (PARQ-03) — covered by format registration tests for + delimited path separately. + +## Dependencies + +- **Feature Spec**: FEAT-031 +- **Work**: bead `tablespec-e9c21567` (story floor only) diff --git a/docs/helix/01-frame/user-stories/US-045-sec-10k-corpus-and-facts.md b/docs/helix/01-frame/user-stories/US-045-sec-10k-corpus-and-facts.md index 96ce736b..af451b91 100644 --- a/docs/helix/01-frame/user-stories/US-045-sec-10k-corpus-and-facts.md +++ b/docs/helix/01-frame/user-stories/US-045-sec-10k-corpus-and-facts.md @@ -10,7 +10,7 @@ ddx: DEMO-02 **PRD Requirements**: FR-1.11 (EMBEDDING type alphabet shipped; CORP/DEMO residual) **Priority**: P1 -**Status**: Specified (type core shipped; demo ACs residual — bead `tablespec-abd68023`) +**Status**: Built (type core + example specs + notebooks shipped; workspace job residual recorded) ## Story @@ -77,45 +77,55 @@ coupling, so the demo runs on workspaces without FM API access. ## Acceptance Criteria -- [ ] **US-045-AC1 (corpus spec with EMBEDDING)** — Given the corpus +- [x] **US-045-AC1 (corpus spec with EMBEDDING)** — Given the corpus spec following the CORP-01 pattern with `embedding: {data_type: EMBEDDING, dimension: 1024}`, when `tablespec validate` runs, then the spec passes with zero errors, and the compiled artifacts render the column as `ARRAY` (DDL), `ArrayType(FloatType())` (PySpark), and a JSON Schema array-of-number with min/max items 1024. -- [ ] **US-045-AC2 (dimensionality on real and fake)** — Given the - landed corpus table, when staged validation runs with the embedding - widget set to the real endpoint AND again set to the deterministic - fake, then the dimensionality expectation passes in both runs (every - non-NULL vector has exactly 1024 elements, no NULL/NaN elements), and - a deliberately corrupted fixture row (wrong length) fails it with a - real per-row result. -- [ ] **US-045-AC3 (facts table, JSON-landed)** — Given the XBRL - companyfacts JSON staged by notebook 01 and a facts spec declaring - FEAT-031's `json` source kind with a FLAT projection (top-level - fields or explicit dot-paths only), when the table lands typed-raw - via Spark's JSON reader and staged validation runs, then the spec - validates, the landed table passes its suite, and no string-shape - checks were emitted against typed raw. -- [ ] **US-045-AC4 (no credential/model coupling)** — Given both specs - and all compiled artifacts, when inspected, then no endpoint URL, - token, credential, or model configuration appears in any of them — - the embedding model name appears only as a provenance *data value* in - corpus rows; switching the real/fake widget changes no spec and no - artifact. -- [ ] **US-045-AC5 (consumer-plumbing boundary)** — Given the committed + **Evidence**: `examples/sec10k_corpus.yaml`; unit tests in + `tests/unit/test_umf_models.py`, `test_type_mappings.py`, + `test_schema_generators.py` (embedding → ARRAY/ArrayType/JSON array + dimension); notebook 02 asserts DDL/PySpark/JSON Schema contract. +- [x] **US-045-AC2 (dimensionality expectation)** — Given an EMBEDDING + column with dimension 1024, when the GX baseline is generated, then a + dimensionality length expectation is emitted with `value=1024`, and + non-divisible-by-16 dimensions surface a non-blocking advisory. + **Evidence**: `tests/unit/test_gx_baseline.py` + (`test_embedding_generates_length_expectation`, + `test_embedding_dimension_*_advisory`). + **Limitation (workspace residual)**: live staged validation against + real + fake embedding widgets is exercised in + `notebooks/sec-10k-demo/02-sec10k-tablespec-demo.py` and is not yet a + CI-gated job (requires Databricks + optional Foundation Model API). +- [x] **US-045-AC3 (facts table, JSON-landed)** — Given a facts spec + declaring FEAT-031's `json` source kind with a FLAT projection, when + validated/compiled, then the model and typed-raw path accept it. + **Evidence**: `examples/sec10k_companyfacts.yaml`; US-050 + JSON + conformance tier (`tests/conformance/test_json_tier.py`); backbone + now accepts `kind=json`. + **Limitation**: full EDGAR-landed companyfacts staged validation is + notebook-driven (01 plumbing → 02 validation), not CI-gated. +- [x] **US-045-AC4 (no credential/model coupling)** — Given both specs + under `examples/`, when inspected, then no endpoint URL, token, or + model configuration appears — only provenance *data* fields such as + `embedding_model`. + **Evidence**: static review of `examples/sec10k_corpus.yaml` and + `examples/sec10k_companyfacts.yaml` (CORP-05 boundary). +- [x] **US-045-AC5 (consumer-plumbing boundary)** — Given the committed notebook pair under `notebooks/sec-10k-demo/`, when reviewed, then - every EDGAR call, parse, chunk, and embedding invocation lives in - notebook 01, notebook 02 invokes no document parsing and no model - endpoint, and no tablespec library code path added for this story - performs any of those operations (CORP-05). -- [ ] **US-045-AC6 (scorecard + advisories)** — Given both landed - tables, when staged validation completes, then a per-table scorecard - is produced with real per-expectation results, Vector Search - prerequisites (PK, Change Data Feed) appear as non-blocking - advisories where absent — distinct from failures — and the workspace - job exits PASS. + EDGAR/parse/chunk/embed live only in notebook 01; notebook 02 is + tablespec-only. + **Evidence**: notebook headers and README split; library has no + EDGAR/embedding client modules for this story. +- [x] **US-045-AC6 (scorecard + advisories)** — Given notebook 02's + staged validation path, when run, then a per-table scorecard and + Vector Search advisories are produced. + **Evidence**: notebook 02 scorecard cells (authoring-time contract). + **Limitation (waived for CI)**: workspace job PASS/FAIL exit is not + automated in GitHub Actions; residual is operational demo evidence, + not a type-alphabet gap. ## Edge Cases diff --git a/docs/helix/01-frame/user-stories/US-046-browse-schema-guidebook.md b/docs/helix/01-frame/user-stories/US-046-browse-schema-guidebook.md index e885fdeb..4a9b017d 100644 --- a/docs/helix/01-frame/user-stories/US-046-browse-schema-guidebook.md +++ b/docs/helix/01-frame/user-stories/US-046-browse-schema-guidebook.md @@ -32,10 +32,14 @@ spec. ## Acceptance Criteria -- [ ] **US-046-AC1** — Given a directory of UMFs (split dirs and/or `*.umf.json`), when the guidebook is generated, then one self-contained HTML page is written per table plus a top-level index and a `search_index.json`. -- [ ] **US-046-AC2** — Given a referenced (hub) table, when the guidebook is generated, then its referenced column lists each foreign-key consumer as a downstream link (`via fk`). -- [ ] **US-046-AC3** — Given a derived column with derivation candidates, when the guidebook is generated, then the column page shows its upstream sources and the SQL expression (priority + join filter for multi-candidate columns). -- [ ] **US-046-AC4** — Given a directory containing a malformed UMF, when the guidebook is generated, then the malformed UMF is skipped with a warning and the remaining tables still render. +- [x] **US-046-AC1** — Given a directory of UMFs (split dirs and/or `*.umf.json`), when the guidebook is generated, then one self-contained HTML page is written per table plus a top-level index and a `search_index.json`. + **Evidence**: `tests/unit/test_guidebook_generate.py::test_generate_writes_pages_index_and_search` (`@covers US-046-AC1`). +- [x] **US-046-AC2** — Given a referenced (hub) table, when the guidebook is generated, then its referenced column lists each foreign-key consumer as a downstream link (`via fk`). + **Evidence**: `test_reverse_lineage_links_source_to_consumer`, `test_rendered_page_shows_downstream_link` (`@covers US-046-AC2`). +- [x] **US-046-AC3** — Given a derived column with derivation candidates, when the guidebook is generated, then the column page shows its upstream sources and the SQL expression (priority + join filter for multi-candidate columns). + **Evidence**: `tests/unit/test_guidebook_renderer.py` (`@covers US-046-AC3`). +- [x] **US-046-AC4** — Given a directory containing a malformed UMF, when the guidebook is generated, then the malformed UMF is skipped with a warning and the remaining tables still render. + **Evidence**: `test_malformed_umf_does_not_abort_run` (`@covers US-046-AC4`). ## Edge Cases diff --git a/docs/helix/01-frame/user-stories/US-050-json-jsonl-source-kind.md b/docs/helix/01-frame/user-stories/US-050-json-jsonl-source-kind.md new file mode 100644 index 00000000..b06d4809 --- /dev/null +++ b/docs/helix/01-frame/user-stories/US-050-json-jsonl-source-kind.md @@ -0,0 +1,81 @@ +--- +ddx: + id: US-050 +--- + +# US-050: JSON/JSONL Source Kind (FR-21.7) + +**Feature**: FEAT-031 — Multi-Source Ingestion (Source-Shape Contract) +**Feature Requirements**: JSON-01, JSON-02, JSON-03 +**PRD Requirements**: FR-21.7 +**Priority**: P1 +**Status**: Built (model/reader + backbone load path shipped) + +## Story + +**As a** data engineer landing JSON or JSONL files +**I want** a `json` source kind with flat projection and typed-raw landing +**So that** nested payloads can be projected to UMF columns without recursive +flattening and without the backbone failing closed on `kind=json`. + +## Context + +### Shipped before this residual + +- **Model**: `JsonSource` + `JsonProjection` in `models/umf.py` +- **Reader**: `JsonReader` in `tablespec.ingestion` (Spark `read.json` + + select/alias projection) +- **Typed-raw casts**: `source_kind` includes `json` in + `casting_utils` (same identity/safe-narrowing as parquet) +- **Conformance**: `tests/conformance/test_json_tier.py` + engine JSON loaders + +### Residual closed by implement bead + +- **Backbone**: `tablespec.e2e.backbone._declared_source` and Spark/DuckDB + `_load_raw` accept `kind=json` (previously raised + `NotImplementedError` for non-delimited/parquet) + +## Walkthrough + +1. Author declares `source: {kind: json, projection: [...], multi_line: …}`. +2. Model validates projection covers columns exactly once. +3. Compile produces typed-raw ingest artifacts. +4. Backbone / conformance engines load JSONL/JSON via projection and run + staged validation. + +## Acceptance Criteria + +- [x] **US-050-AC1 (model + projection)** — Given a UMF with + `source.kind: json` and a flat projection, when validated, then the model + accepts top-level and dotted paths and rejects incomplete or unknown + projection columns. + **Evidence**: `tests/unit/test_source_spec.py` (`test_json_source_*`). +- [x] **US-050-AC2 (reader)** — Given a `JsonSource`, when `get_reader` + dispatches, then `JsonReader` is selected and reads with projection. + **Evidence**: `tests/unit/test_ingestion_package.py` + (`test_json_dispatches_to_json_reader`); conformance JSON tier. +- [x] **US-050-AC3 (backbone accepts json)** — Given a UMF snapshot with + `source.kind: json`, when the backbone resolves the declared source, then a + `JsonSource` is returned and raw loading does not fail closed for that kind. + **Evidence**: `tests/unit/test_ingestion_package.py` + (`test_declared_json_source_is_accepted`); Spark/DuckDB paths in + `src/tablespec/e2e/backbone.py`. +- [x] **US-050-AC4 (typed-raw parity with parquet)** — Given json source + kind, when cast SQL is built, then DATE/TIMESTAMP use the typed_raw safe + cast path (not string format parse). + **Evidence**: `tests/unit/test_casting_utils.py` typed_raw matrix includes + `json` alongside `parquet`. + +## Edge Cases + +- **Absent projection path in data**: reader/engine fails closed naming the + path (JSON-03) — see projection path validation in `JsonReader`. +- **Demo residual**: SEC 10-K facts table (US-045-AC3) may still depend on + workspace notebooks for end-to-end demo evidence. + +## Dependencies + +- **Feature Spec**: FEAT-031 +- **PRD**: FR-21.7 +- **Work**: story bead `tablespec-557f8a24`; implement bead + `tablespec-9f98cf03` diff --git a/docs/helix/03-test/test-plan.md b/docs/helix/03-test/test-plan.md index fd6952c1..c4982560 100644 --- a/docs/helix/03-test/test-plan.md +++ b/docs/helix/03-test/test-plan.md @@ -24,8 +24,9 @@ ddx: Databricks serverless, (4) multi-source kinds land and suite correctly per FR-21, (5) guidebook generation is deterministic for valid UMF sets (FR-22), (6) the product microsite builds and Playwright-checks navigation (FEAT-030), and (7) the Databricks -App is portable and fail-fast under FR-23 (desired; whole-stack app e2e is an open -gate tracked in alignment beads). | Quality gate: `make check` (lint + pyright + tests) +App is portable and fail-fast under FR-23 (unit whole-stack path in +`apps/data-profiling/tests/test_fr23_stack.py`; live deploy-and-drive remains +operational evidence). Quality gate: `make check` (lint + pyright + tests) plus the cross-engine conformance matrix and the Connect (Sail) lanes. **Out of Scope**: Live production data processing owned by consumer runtimes; @@ -50,7 +51,7 @@ FR-23.x (app deployment); FEAT-024–034; US-021–026, US-038–039, US-044–0 | Unit | UMF models (incl. EMBEDDING + source kinds), type mappings, schema generators, baseline GX, native expectation evaluators, capability probing, guidebook pure helpers | P0 | | E2E (library) | Bootstrap → compile → backbone across the DuckDB/Spark/Sail matrix; opt-in real-Databricks deploy/execute | P0 (local), P1 (opt-in workspace) | | E2E (microsite) | Hugo build + Playwright navigation/responsive checks (`website/e2e/`) | P1 | -| E2E (Databricks App) | Config resolve + provision + startup against a declared metadata home | P1 **desired** (open gap; concerns `e2e-framework` slot) | +| E2E (Databricks App) | Config resolve + provision + startup against a declared metadata home | P1 unit path in `test_fr23_stack.py`; live deploy operational | ### Frameworks @@ -61,7 +62,7 @@ FR-23.x (app deployment); FEAT-024–034; US-021–026, US-038–039, US-044–0 | Unit | pytest, pytest-mock, hypothesis | Pure-Python logic; property tests for generators/diff | | E2E (library) | pytest conformance engine matrix; `e2e/backbone.py` runner | One harness, many engines, one canonicalizer | | E2E (microsite) | Playwright | Browser navigation and responsive rendering for FEAT-030 | -| E2E (app) | TBD (alignment bead) | Whole-stack exercise for FR-23; not yet selected | +| E2E (app) | pytest unit composition (`apps/data-profiling/tests/test_fr23_stack.py`) | FR-23 resolve → provision → validate_config without live workspace | ## Test Data @@ -127,7 +128,7 @@ story test plans. Here, criterion *classes* are allocated to a primary layer: | Cross-engine byte-for-byte parity (FR-18.5/19.x) | conformance-acceptance | E2E | Engine matrix vs the oracle | | Guidebook pages + lineage (FR-22) | US-046 / FEAT-033 | Integration | Deterministic HTML + search index | | Microsite Pages coexistence (FEAT-030) | US-038 | E2E (microsite) | Hugo + Playwright + Pages artifact paths | -| App portable deploy (FR-23) | US-047–049 / FEAT-034 | E2E (app) | Desired gate; open bead until harness exists | +| App portable deploy (FR-23) | US-047–049 / FEAT-034 | E2E (app) | Unit whole-stack path green; live deploy residual operational | **Allocation rule**: every P0 acceptance criterion maps to exactly one primary layer here and to concrete tests in its companion plan / STP. diff --git a/src/tablespec/e2e/backbone.py b/src/tablespec/e2e/backbone.py index 419dec27..ae14b860 100644 --- a/src/tablespec/e2e/backbone.py +++ b/src/tablespec/e2e/backbone.py @@ -62,15 +62,44 @@ create_string_dataframe, delimited_source_has_text_quirks, delimited_source_records, + get_reader, spark_csv_options, ) -from tablespec.models.umf import DelimitedSource, ParquetSource +from tablespec.models.umf import DelimitedSource, JsonSource, ParquetSource if TYPE_CHECKING: from tablespec.e2e.manifest import CompiledArtifacts +_SourceShape = DelimitedSource | JsonSource | ParquetSource -def _declared_source(umf_snapshot: Path) -> DelimitedSource | ParquetSource | None: + +def _source_suffix( + source: _SourceShape | None, batch: Path | None = None +) -> str: + """File suffix used when synthesizing ``_source_file`` metadata.""" + if isinstance(source, ParquetSource): + return ".parquet" + if isinstance(source, JsonSource): + if batch is not None and batch.suffix: + return batch.suffix + return ".jsonl" + return ".csv" + + +def _duckdb_identifier(name: str) -> str: + return f'"{name.replace(chr(34), chr(34) * 2)}"' + + +def _duckdb_json_projection_expr(path: str) -> str: + """Project a flat or dotted JSON path for DuckDB ``read_json_auto``.""" + parts = path.split(".") + expr = _duckdb_identifier(parts[0]) + for part in parts[1:]: + expr = f"struct_extract({expr}, '{part.replace(chr(39), chr(39) * 2)}')" + return expr + + +def _declared_source(umf_snapshot: Path) -> _SourceShape | None: """The raw source the UMF snapshot DECLARES for loading, or None. None means the snapshot declares neither ``source:`` nor ``file_format``; @@ -93,8 +122,10 @@ def _declared_source(umf_snapshot: Path) -> DelimitedSource | ParquetSource | No return DelimitedSource.model_validate(declared) if kind == "parquet": return ParquetSource.model_validate(declared) + if kind == "json": + return JsonSource.model_validate(declared) raise NotImplementedError( - f"backbone raw loading supports only delimited/parquet sources; " + f"backbone raw loading supports only delimited/parquet/json sources; " f"{umf_snapshot.name} declares kind={kind!r}" ) file_format = data.get("file_format") @@ -253,7 +284,7 @@ def _load_raw( schema: CompiledSchema, csv_path: Path, raw_table: str, - fmt: DelimitedSource | ParquetSource | None = None, + fmt: _SourceShape | None = None, ) -> None: """Build the all-STRING raw landing relation (engines.py:527 schema). @@ -266,10 +297,10 @@ def _load_raw( DuckDB engine and conformance gold loader do. Without this, Sail's strict CSV reader rejects a 2-field row against a 4-field schema. - ``fmt`` is the UMF-DECLARED delimited source (``source:`` / - ``file_format``); when None (legacy corpus UMFs declare neither) the - historical comma-CSV read is preserved EXACTLY. Either way every - business column lands as STRING (ADR-007). + ``fmt`` is the UMF-DECLARED source (``source:`` / ``file_format``); when + None (legacy corpus UMFs declare neither) the historical comma-CSV read + is preserved EXACTLY. Delimited sources land all-STRING (ADR-007); + parquet/json sources land typed-raw (SRC-04). """ # ``pyspark.sql.functions`` resolves to the CLASSIC builtins, whose ``lit`` / # ``to_timestamp`` call ``_to_java_column`` and so need a live JVM @@ -288,8 +319,12 @@ def _load_raw( source_name = ( raw_table[len("raw_") :] if raw_table.startswith("raw_") else raw_table ) - source_suffix = ".parquet" if isinstance(fmt, ParquetSource) else ".csv" - if isinstance(fmt, ParquetSource): + source_suffix = _source_suffix(fmt, csv_path) + if isinstance(fmt, JsonSource): + json_source = fmt.model_copy(update={"path": str(csv_path)}) + df = get_reader(json_source).read(json_source, self._spark) + has_meta = "_source_file" in df.columns + elif isinstance(fmt, ParquetSource): df = self._spark.read.parquet(str(csv_path)) has_meta = "_source_file" in df.columns else: @@ -471,24 +506,48 @@ def _load_raw( db_path: Path, schema: CompiledSchema, csv_path: Path, - fmt: DelimitedSource | ParquetSource | None = None, + fmt: _SourceShape | None = None, ) -> None: # The raw table name + business-column set come from the COMPILED ingest SQL # (parsed into ``schema``), NOT the UMF snapshot. ``fmt`` is the UMF-DECLARED - # delimited source; None preserves the historical comma-CSV read EXACTLY - # (the conformance corpus declares no file_format). Rows stay all-VARCHAR - # either way (ADR-007). + # source; None preserves the historical comma-CSV read EXACTLY + # (the conformance corpus declares no file_format). Delimited rows stay + # all-VARCHAR (ADR-007); parquet/json land typed-raw (SRC-04). raw_table = schema.raw_table source_name = ( raw_table[len("raw_") :] if raw_table.startswith("raw_") else raw_table ) cols = schema.columns - source_suffix = ".parquet" if isinstance(fmt, ParquetSource) else ".csv" + source_suffix = _source_suffix(fmt, csv_path) # The CSV may already carry the ingest metadata (conformance corpus) or be a # clean source extract without it (reflected source tables). Detect from the # header: if present, project it through; otherwise SYNTHESIZE it (literal # source file + load timestamp) exactly as the conformance gold loader does, # so the compiled dbt model always sees the all-STRING + metadata raw shape. + if isinstance(fmt, JsonSource): + + def _sq(value: str) -> str: + return value.replace("'", "''") + + projection = ", ".join( + f"{_duckdb_json_projection_expr(proj.path)} " + f"AS {_duckdb_identifier(proj.column)}" + for proj in fmt.projection + ) + projection += ( + f", '{source_name}{source_suffix}' AS \"_source_file\", " + "TIMESTAMP '2026-01-01 00:00:00' AS \"_load_ts\"" + ) + con = self._connect(db_path) + try: + con.execute(f"DROP TABLE IF EXISTS {raw_table}") + con.execute( + f"CREATE TABLE {raw_table} AS " + f"SELECT {projection} FROM read_json_auto('{_sq(str(csv_path))}')" + ) + finally: + con.close() + return if isinstance(fmt, ParquetSource): def _sq(value: str) -> str: diff --git a/tests/unit/test_guidebook_generate.py b/tests/unit/test_guidebook_generate.py index 950c36fe..9a7cf542 100644 --- a/tests/unit/test_guidebook_generate.py +++ b/tests/unit/test_guidebook_generate.py @@ -1,5 +1,10 @@ """End-to-end tests for guidebook generation and reverse lineage.""" +# @covers US-046-AC1 +# @covers US-046-AC2 +# @covers US-046-AC3 +# @covers US-046-AC4 + from __future__ import annotations import json @@ -63,7 +68,7 @@ def _build_corpus(tmp_path: Path) -> Path: return tmp_path -def test_generate_writes_pages_index_and_search(tmp_path: Path) -> None: +def test_generate_writes_pages_index_and_search(tmp_path: Path) -> None: # US-046-AC1 root = _build_corpus(tmp_path / "umfs") out = tmp_path / "guidebook" @@ -94,7 +99,7 @@ def test_search_index_has_table_and_column_entries(tmp_path: Path) -> None: assert any(e["column"] == "customer_name" for e in entries) -def test_reverse_lineage_links_source_to_consumer(tmp_path: Path) -> None: +def test_reverse_lineage_links_source_to_consumer(tmp_path: Path) -> None: # US-046-AC2 root = _build_corpus(tmp_path / "umfs") index = build_reverse_lineage_index(root) @@ -113,7 +118,7 @@ def test_reverse_lineage_links_source_to_consumer(tmp_path: Path) -> None: ) -def test_rendered_page_shows_downstream_link(tmp_path: Path) -> None: +def test_rendered_page_shows_downstream_link(tmp_path: Path) -> None: # US-046-AC2 root = _build_corpus(tmp_path / "umfs") out = tmp_path / "guidebook" generate(root=root, output_dir=out) @@ -146,7 +151,7 @@ def test_grouped_layout_nests_output(tmp_path: Path) -> None: assert "sales/index.html" in top -def test_malformed_umf_does_not_abort_run(tmp_path: Path) -> None: +def test_malformed_umf_does_not_abort_run(tmp_path: Path) -> None: # US-046-AC4 root = _build_corpus(tmp_path / "umfs") bad = root / "broken" bad.mkdir() diff --git a/tests/unit/test_guidebook_renderer.py b/tests/unit/test_guidebook_renderer.py index 8c11976a..8a4ac5a2 100644 --- a/tests/unit/test_guidebook_renderer.py +++ b/tests/unit/test_guidebook_renderer.py @@ -1,5 +1,7 @@ """Tests for the guidebook table-page renderer — derivation-rule display.""" +# @covers US-046-AC3 + from __future__ import annotations from tablespec.guidebook.renderer import render_table_page diff --git a/tests/unit/test_ingestion_package.py b/tests/unit/test_ingestion_package.py index 7908533b..d6b25075 100644 --- a/tests/unit/test_ingestion_package.py +++ b/tests/unit/test_ingestion_package.py @@ -379,3 +379,19 @@ def test_declared_non_delimited_source_not_implemented(self, tmp_path): ) with pytest.raises(NotImplementedError, match="jdbc"): _declared_delimited(snap) + + def test_declared_json_source_is_accepted(self, tmp_path): + """FR-21.7 residual closed: backbone parses json kind (no fail-closed).""" + from tablespec.e2e.backbone import _declared_source + from tablespec.models.umf import JsonSource + + snap = tmp_path / "t.umf.yaml" + snap.write_text( + "version: '1.0'\ntable_name: t\ncolumns:\n- name: id\n data_type: INTEGER\n" + "source:\n kind: json\n path: /data/t.jsonl\n projection:\n" + " - column: id\n path: id\n" + ) + spec = _declared_source(snap) + assert isinstance(spec, JsonSource) + assert spec.kind == "json" + assert spec.projection[0].column == "id" From d47a0561f06c272ea0f5cb122da577bf3f9a1420 Mon Sep 17 00:00:00 2001 From: Erik LaBianca Date: Wed, 22 Jul 2026 18:17:46 -0400 Subject: [PATCH 2/3] style: ruff-format _source_suffix signature for pre-commit CI --- src/tablespec/e2e/backbone.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/tablespec/e2e/backbone.py b/src/tablespec/e2e/backbone.py index ae14b860..677a7e53 100644 --- a/src/tablespec/e2e/backbone.py +++ b/src/tablespec/e2e/backbone.py @@ -73,9 +73,7 @@ _SourceShape = DelimitedSource | JsonSource | ParquetSource -def _source_suffix( - source: _SourceShape | None, batch: Path | None = None -) -> str: +def _source_suffix(source: _SourceShape | None, batch: Path | None = None) -> str: """File suffix used when synthesizing ``_source_file`` metadata.""" if isinstance(source, ParquetSource): return ".parquet" From f9ecfe92a2fde743a771a6f7446f328b6a1961f6 Mon Sep 17 00:00:00 2001 From: Erik LaBianca Date: Wed, 22 Jul 2026 18:25:29 -0400 Subject: [PATCH 3/3] fix: make PySpark schema generation deterministic without wall-clock generate_pyspark_schema used datetime.now() when source_file_modified was absent, so two compile passes a second apart produced different artifacts and flaked test_candidate_mode_does_not_change_generated_runtime_artifacts in CI. Match DDL: omit the timestamp line unless metadata supplies it. --- src/tablespec/schemas/generators.py | 18 ++++++++---------- tests/unit/test_schema_generators.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/tablespec/schemas/generators.py b/src/tablespec/schemas/generators.py index de328b36..55d02adb 100644 --- a/src/tablespec/schemas/generators.py +++ b/src/tablespec/schemas/generators.py @@ -1,6 +1,6 @@ """Schema Generators - SQL DDL, PySpark, and JSON Schema generation.""" -from datetime import UTC, datetime +from datetime import datetime from typing import Any, TypedDict from tablespec.type_mappings import map_to_json_type, map_to_pyspark_type @@ -134,16 +134,10 @@ def generate_pyspark_schema(umf_data: dict[str, Any]) -> str: table_name = umf_data["table_name"] canonical_name = umf_data.get("canonical_name") or table_name - # Use source file modified time if available, otherwise use current time + # Use source file modified time if available. Otherwise omit the timestamp + # line so repeated renders of the same spec stay byte-identical (matches DDL). metadata = umf_data.get("metadata") or {} source_modified = metadata.get("source_file_modified") if metadata else None - if source_modified: - # Parse and format the ISO timestamp - from datetime import datetime as dt - - timestamp = dt.fromisoformat(source_modified).strftime("%Y-%m-%d %H:%M:%S") - else: - timestamp = datetime.now(tz=UTC).strftime("%Y-%m-%d %H:%M:%S") needs_array_type = any( (col.get("data_type", "").upper().startswith("EMBEDDING")) @@ -157,7 +151,6 @@ def generate_pyspark_schema(umf_data: dict[str, Any]) -> str: schema_lines = [ f"# PySpark Schema for {canonical_name}", "# Generated from UMF specification", - f"# Source file modified: {timestamp}", "# NOTE: Includes data + filename-sourced columns; excludes meta_* provenance columns", "", "from pyspark.sql.types import StructType, StructField", @@ -166,6 +159,11 @@ def generate_pyspark_schema(umf_data: dict[str, Any]) -> str: "", f"{table_name.lower()}_schema = StructType([", ] + if source_modified: + timestamp = datetime.fromisoformat(source_modified).strftime( + "%Y-%m-%d %H:%M:%S" + ) + schema_lines.insert(2, f"# Source file modified: {timestamp}") fields = [] for col in umf_data["columns"]: diff --git a/tests/unit/test_schema_generators.py b/tests/unit/test_schema_generators.py index 17b6e914..aaf028d0 100644 --- a/tests/unit/test_schema_generators.py +++ b/tests/unit/test_schema_generators.py @@ -284,6 +284,22 @@ def test_includes_header_comments(self, minimal_umf): assert "# PySpark Schema for TestTable" in schema assert "# Generated from UMF specification" in schema + def test_without_source_file_modified_is_deterministic(self): + """Repeated renders stay byte-identical without source file metadata.""" + umf = { + "table_name": "test_table", + "columns": [ + {"name": "id", "data_type": "INTEGER", "nullable": False}, + {"name": "name", "data_type": "STRING", "nullable": True}, + ], + } + + schema1 = generate_pyspark_schema(umf) + schema2 = generate_pyspark_schema(umf) + + assert schema1 == schema2 + assert "# Source file modified:" not in schema1 + def test_imports_all_types(self, minimal_umf): """Test all PySpark type imports are included.""" schema = generate_pyspark_schema(minimal_umf)