From 5eaac2ca0b4c73c7a3f690663f67f8ef479ec8ad Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 30 Aug 2026 16:23:11 +1000 Subject: [PATCH 01/11] feat(databricks): add table discovery workflow --- semapact/platforms/databricks/discovery.py | 64 ++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 semapact/platforms/databricks/discovery.py diff --git a/semapact/platforms/databricks/discovery.py b/semapact/platforms/databricks/discovery.py new file mode 100644 index 0000000..87f33ff --- /dev/null +++ b/semapact/platforms/databricks/discovery.py @@ -0,0 +1,64 @@ +"""Read-only Databricks table discovery. + +Discovery answers which assets exist in a Unity Catalog schema. It does not +observe table structure, create an ODCS contract, evaluate governance, or +resolve authentication. Callers supply an initialized ``WorkspaceClient``. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from databricks.sdk import WorkspaceClient + + +class _TableInfoLike(Protocol): + full_name: str | None + + +class _TablesApiLike(Protocol): + def list( + self, + *, + catalog_name: str, + schema_name: str, + ) -> Iterable[_TableInfoLike]: ... + + +class _WorkspaceClientLike(Protocol): + tables: _TablesApiLike + + +def discover_databricks_tables( + *, + client: WorkspaceClient | _WorkspaceClientLike, + catalog_name: str, + schema_name: str, +) -> tuple[str, ...]: + """Return stable fully qualified names for tables visible in one UC schema.""" + catalog = _required_name(catalog_name, field="catalog_name") + schema = _required_name(schema_name, field="schema_name") + + discovered: set[str] = set() + for table in client.tables.list(catalog_name=catalog, schema_name=schema): + full_name = _text(table.full_name) + if full_name: + discovered.add(full_name) + + return tuple(sorted(discovered, key=lambda value: (value.casefold(), value))) + + +def _required_name(value: str, *, field: str) -> str: + cleaned = _text(value) + if not cleaned: + raise ValueError(f"{field} is required for Databricks discovery") + return cleaned + + +def _text(value: object | None) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None From 97746d776a3a1b8aee0068e06328dbabc13dceb9 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 30 Aug 2026 16:23:28 +1000 Subject: [PATCH 02/11] test(databricks): cover discovery workflow --- tests/test_databricks_discovery.py | 104 +++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 tests/test_databricks_discovery.py diff --git a/tests/test_databricks_discovery.py b/tests/test_databricks_discovery.py new file mode 100644 index 0000000..0eab781 --- /dev/null +++ b/tests/test_databricks_discovery.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import pytest + +from semapact.platforms.databricks.discovery import discover_databricks_tables + + +@dataclass +class _FakeTable: + full_name: str | None + + +class _FakeTablesApi: + def __init__(self, tables: list[_FakeTable]) -> None: + self._tables = tables + self.calls: list[dict[str, str]] = [] + + def list(self, *, catalog_name: str, schema_name: str) -> list[_FakeTable]: + self.calls.append( + { + "catalog_name": catalog_name, + "schema_name": schema_name, + } + ) + return self._tables + + +class _FakeWorkspaceClient: + def __init__(self, tables: list[_FakeTable]) -> None: + self.tables = _FakeTablesApi(tables) + + +def test_discover_databricks_tables_lists_without_observing_or_importing() -> None: + client = _FakeWorkspaceClient( + [ + _FakeTable("main.sales.Z_orders"), + _FakeTable("main.sales.accounts"), + _FakeTable("main.sales.accounts"), + _FakeTable(None), + ] + ) + + discovered = discover_databricks_tables( + client=client, + catalog_name=" main ", + schema_name=" sales ", + ) + + assert discovered == ( + "main.sales.accounts", + "main.sales.Z_orders", + ) + assert client.tables.calls == [ + { + "catalog_name": "main", + "schema_name": "sales", + } + ] + + +@pytest.mark.parametrize( + ("catalog_name", "schema_name", "message"), + [ + ("", "sales", "catalog_name is required for Databricks discovery"), + (" ", "sales", "catalog_name is required for Databricks discovery"), + ("main", "", "schema_name is required for Databricks discovery"), + ("main", " ", "schema_name is required for Databricks discovery"), + ], +) +def test_discover_databricks_tables_validates_scope_before_sdk_call( + catalog_name: str, + schema_name: str, + message: str, +) -> None: + client = _FakeWorkspaceClient([]) + + with pytest.raises(ValueError, match=message): + discover_databricks_tables( + client=client, + catalog_name=catalog_name, + schema_name=schema_name, + ) + + assert client.tables.calls == [] + + +def test_databricks_discovery_module_has_no_contract_or_governance_dependency() -> None: + from pathlib import Path + + source = Path("semapact/platforms/databricks/discovery.py").read_text( + encoding="utf-8" + ) + + forbidden = ( + "datacontract", + "open_data_contract_standard", + "semapact.importers", + "semapact.observation", + "semapact.lifecycle", + "semapact.governance", + ) + assert not any(name in source for name in forbidden) From 68898ebaf350449f4b7a82531c6a122dc8ab465a Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 30 Aug 2026 16:23:36 +1000 Subject: [PATCH 03/11] feat(databricks): expose discovery workflow --- semapact/platforms/databricks/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/semapact/platforms/databricks/__init__.py b/semapact/platforms/databricks/__init__.py index 2df926e..9a44ddb 100644 --- a/semapact/platforms/databricks/__init__.py +++ b/semapact/platforms/databricks/__init__.py @@ -1,5 +1,9 @@ """Databricks platform-access helpers.""" from semapact.platforms.databricks.client import create_databricks_workspace_client +from semapact.platforms.databricks.discovery import discover_databricks_tables -__all__ = ["create_databricks_workspace_client"] +__all__ = [ + "create_databricks_workspace_client", + "discover_databricks_tables", +] From 63a47f464d7d5c9bf4c4ef3b04bedc4ffbf61eae Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 30 Aug 2026 16:24:09 +1000 Subject: [PATCH 04/11] ci(databricks): verify discovery boundary --- .github/workflows/ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1592bd9..5cfe796 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,13 +66,17 @@ jobs: - name: Install SemaPact Databricks extra from project metadata run: uv pip install --python .venv/bin/python -e ".[databricks]" pytest - - name: Verify Databricks SDK, client factory, and existing Unity import boundaries + - name: Verify Databricks SDK and platform workflow boundaries run: >- .venv/bin/python -c - "import inspect; from databricks.sdk import WorkspaceClient; from databricks.sdk.service.catalog import TableInfo; from semapact.importers.unity_importer import import_unity_contract; from semapact.platforms.databricks import create_databricks_workspace_client; params = inspect.signature(WorkspaceClient).parameters; assert {'host', 'token', 'profile'} <= set(params); print(WorkspaceClient.__name__, TableInfo.__name__, import_unity_contract.__name__, create_databricks_workspace_client.__name__)" + "import inspect; from databricks.sdk import WorkspaceClient; from databricks.sdk.service.catalog import TableInfo; from semapact.importers.unity_importer import import_unity_contract; from semapact.platforms.databricks import create_databricks_workspace_client, discover_databricks_tables; client_params = inspect.signature(WorkspaceClient).parameters; list_params = inspect.signature(WorkspaceClient.tables.fget).parameters if False else None; assert {'host', 'token', 'profile'} <= set(client_params); print(WorkspaceClient.__name__, TableInfo.__name__, import_unity_contract.__name__, create_databricks_workspace_client.__name__, discover_databricks_tables.__name__)" - name: Run Databricks platform-boundary tests with official SDK installed - run: .venv/bin/python -m pytest tests/test_databricks_client.py tests/test_observation_databricks.py + run: >- + .venv/bin/python -m pytest + tests/test_databricks_client.py + tests/test_databricks_discovery.py + tests/test_observation_databricks.py coverage: runs-on: ubuntu-latest From dda403ea52461ffc316d5c229f7c432fe7a15d79 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 30 Aug 2026 16:24:30 +1000 Subject: [PATCH 05/11] ci(databricks): verify tables discovery SDK surface --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cfe796..29d31da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,7 @@ jobs: - name: Verify Databricks SDK and platform workflow boundaries run: >- .venv/bin/python -c - "import inspect; from databricks.sdk import WorkspaceClient; from databricks.sdk.service.catalog import TableInfo; from semapact.importers.unity_importer import import_unity_contract; from semapact.platforms.databricks import create_databricks_workspace_client, discover_databricks_tables; client_params = inspect.signature(WorkspaceClient).parameters; list_params = inspect.signature(WorkspaceClient.tables.fget).parameters if False else None; assert {'host', 'token', 'profile'} <= set(client_params); print(WorkspaceClient.__name__, TableInfo.__name__, import_unity_contract.__name__, create_databricks_workspace_client.__name__, discover_databricks_tables.__name__)" + "import inspect; from databricks.sdk import WorkspaceClient; from databricks.sdk.service.catalog import TableInfo, TablesAPI; from semapact.importers.unity_importer import import_unity_contract; from semapact.platforms.databricks import create_databricks_workspace_client, discover_databricks_tables; client_params = inspect.signature(WorkspaceClient).parameters; list_params = inspect.signature(TablesAPI.list).parameters; assert {'host', 'token', 'profile'} <= set(client_params); assert {'catalog_name', 'schema_name'} <= set(list_params); print(WorkspaceClient.__name__, TableInfo.__name__, import_unity_contract.__name__, create_databricks_workspace_client.__name__, discover_databricks_tables.__name__)" - name: Run Databricks platform-boundary tests with official SDK installed run: >- From 0bba9f856298646c939e6014bb0eb5b3d113b2ff Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 30 Aug 2026 16:24:51 +1000 Subject: [PATCH 06/11] test(databricks): verify import remains independent --- tests/test_databricks_workflow_separation.py | 48 ++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tests/test_databricks_workflow_separation.py diff --git a/tests/test_databricks_workflow_separation.py b/tests/test_databricks_workflow_separation.py new file mode 100644 index 0000000..e3573e0 --- /dev/null +++ b/tests/test_databricks_workflow_separation.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from semapact.importers import unity_importer + + +def test_unity_contract_import_remains_independently_callable(monkeypatch) -> None: + imported_contract = object() + calls: list[dict[str, object]] = [] + + class _FakeDataContract: + @staticmethod + def import_from_source(**kwargs: object) -> object: + calls.append(kwargs) + return imported_contract + + from semapact.core.config import config_manager + + monkeypatch.setattr(unity_importer, "DataContract", _FakeDataContract) + monkeypatch.setattr(config_manager, "get", lambda _key: None) + monkeypatch.setattr( + unity_importer, + "enrich_unity_contract_relationships", + lambda contract, **_kwargs: contract, + ) + + result = unity_importer.import_unity_contract( + table_fqn="main.sales.orders", + workspace_url="https://adb.example", + token="secret-token", + ) + + assert result is imported_contract + assert calls == [ + { + "format": "unity", + "source": None, + "unity_table_full_name": ["main.sales.orders"], + } + ] + + +def test_unity_contract_import_does_not_depend_on_observation_or_discovery() -> None: + from pathlib import Path + + source = Path("semapact/importers/unity_importer.py").read_text(encoding="utf-8") + + assert "semapact.observation" not in source + assert "semapact.platforms.databricks.discovery" not in source From 80caaa67e15bed19b758ea044d726718c53f5562 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 30 Aug 2026 16:25:56 +1000 Subject: [PATCH 07/11] ci(databricks): cover workflow separation --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29d31da..6ab5f13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,6 +76,7 @@ jobs: .venv/bin/python -m pytest tests/test_databricks_client.py tests/test_databricks_discovery.py + tests/test_databricks_workflow_separation.py tests/test_observation_databricks.py coverage: From 14cdfa844fe2d620f0f6f84c4d87aee70fd4fdde Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 30 Aug 2026 16:27:03 +1000 Subject: [PATCH 08/11] fix(databricks): fail closed on incomplete discovery identity --- semapact/platforms/databricks/discovery.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/semapact/platforms/databricks/discovery.py b/semapact/platforms/databricks/discovery.py index 87f33ff..49de3ae 100644 --- a/semapact/platforms/databricks/discovery.py +++ b/semapact/platforms/databricks/discovery.py @@ -16,6 +16,7 @@ class _TableInfoLike(Protocol): full_name: str | None + name: str | None class _TablesApiLike(Protocol): @@ -44,8 +45,12 @@ def discover_databricks_tables( discovered: set[str] = set() for table in client.tables.list(catalog_name=catalog, schema_name=schema): full_name = _text(table.full_name) - if full_name: - discovered.add(full_name) + if not full_name: + name = _text(table.name) + if not name: + raise ValueError("Databricks discovery returned a table without an identity") + full_name = f"{catalog}.{schema}.{name}" + discovered.add(full_name) return tuple(sorted(discovered, key=lambda value: (value.casefold(), value))) From 5311e42939421268ff84c066fb24bf88a8084224 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 30 Aug 2026 16:27:20 +1000 Subject: [PATCH 09/11] test(databricks): cover discovery identity fallback --- tests/test_databricks_discovery.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/test_databricks_discovery.py b/tests/test_databricks_discovery.py index 0eab781..6f43d2d 100644 --- a/tests/test_databricks_discovery.py +++ b/tests/test_databricks_discovery.py @@ -10,6 +10,7 @@ @dataclass class _FakeTable: full_name: str | None + name: str | None = None class _FakeTablesApi: @@ -38,7 +39,7 @@ def test_discover_databricks_tables_lists_without_observing_or_importing() -> No _FakeTable("main.sales.Z_orders"), _FakeTable("main.sales.accounts"), _FakeTable("main.sales.accounts"), - _FakeTable(None), + _FakeTable(None, name="customers"), ] ) @@ -50,6 +51,7 @@ def test_discover_databricks_tables_lists_without_observing_or_importing() -> No assert discovered == ( "main.sales.accounts", + "main.sales.customers", "main.sales.Z_orders", ) assert client.tables.calls == [ @@ -60,6 +62,20 @@ def test_discover_databricks_tables_lists_without_observing_or_importing() -> No ] +def test_discover_databricks_tables_fails_on_missing_table_identity() -> None: + client = _FakeWorkspaceClient([_FakeTable(None)]) + + with pytest.raises( + ValueError, + match="Databricks discovery returned a table without an identity", + ): + discover_databricks_tables( + client=client, + catalog_name="main", + schema_name="sales", + ) + + @pytest.mark.parametrize( ("catalog_name", "schema_name", "message"), [ From 866b46a34c0713850f14e62d3fb15cf96f400122 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 30 Aug 2026 20:36:53 +1000 Subject: [PATCH 10/11] test(databricks): keep issue 11 scoped to read side --- tests/test_databricks_workflow_separation.py | 48 -------------------- 1 file changed, 48 deletions(-) delete mode 100644 tests/test_databricks_workflow_separation.py diff --git a/tests/test_databricks_workflow_separation.py b/tests/test_databricks_workflow_separation.py deleted file mode 100644 index e3573e0..0000000 --- a/tests/test_databricks_workflow_separation.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import annotations - -from semapact.importers import unity_importer - - -def test_unity_contract_import_remains_independently_callable(monkeypatch) -> None: - imported_contract = object() - calls: list[dict[str, object]] = [] - - class _FakeDataContract: - @staticmethod - def import_from_source(**kwargs: object) -> object: - calls.append(kwargs) - return imported_contract - - from semapact.core.config import config_manager - - monkeypatch.setattr(unity_importer, "DataContract", _FakeDataContract) - monkeypatch.setattr(config_manager, "get", lambda _key: None) - monkeypatch.setattr( - unity_importer, - "enrich_unity_contract_relationships", - lambda contract, **_kwargs: contract, - ) - - result = unity_importer.import_unity_contract( - table_fqn="main.sales.orders", - workspace_url="https://adb.example", - token="secret-token", - ) - - assert result is imported_contract - assert calls == [ - { - "format": "unity", - "source": None, - "unity_table_full_name": ["main.sales.orders"], - } - ] - - -def test_unity_contract_import_does_not_depend_on_observation_or_discovery() -> None: - from pathlib import Path - - source = Path("semapact/importers/unity_importer.py").read_text(encoding="utf-8") - - assert "semapact.observation" not in source - assert "semapact.platforms.databricks.discovery" not in source From d3c66ee89faf14601ae815a75b971c21d5791abb Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 30 Aug 2026 20:37:11 +1000 Subject: [PATCH 11/11] ci(databricks): keep issue 11 on discovery and observation --- .github/workflows/ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ab5f13..c2aedb1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,17 +66,16 @@ jobs: - name: Install SemaPact Databricks extra from project metadata run: uv pip install --python .venv/bin/python -e ".[databricks]" pytest - - name: Verify Databricks SDK and platform workflow boundaries + - name: Verify Databricks SDK read-side boundaries run: >- .venv/bin/python -c - "import inspect; from databricks.sdk import WorkspaceClient; from databricks.sdk.service.catalog import TableInfo, TablesAPI; from semapact.importers.unity_importer import import_unity_contract; from semapact.platforms.databricks import create_databricks_workspace_client, discover_databricks_tables; client_params = inspect.signature(WorkspaceClient).parameters; list_params = inspect.signature(TablesAPI.list).parameters; assert {'host', 'token', 'profile'} <= set(client_params); assert {'catalog_name', 'schema_name'} <= set(list_params); print(WorkspaceClient.__name__, TableInfo.__name__, import_unity_contract.__name__, create_databricks_workspace_client.__name__, discover_databricks_tables.__name__)" + "import inspect; from databricks.sdk import WorkspaceClient; from databricks.sdk.service.catalog import TableInfo, TablesAPI; from semapact.platforms.databricks import create_databricks_workspace_client, discover_databricks_tables; client_params = inspect.signature(WorkspaceClient).parameters; list_params = inspect.signature(TablesAPI.list).parameters; assert {'host', 'token', 'profile'} <= set(client_params); assert {'catalog_name', 'schema_name'} <= set(list_params); print(WorkspaceClient.__name__, TableInfo.__name__, create_databricks_workspace_client.__name__, discover_databricks_tables.__name__)" - - name: Run Databricks platform-boundary tests with official SDK installed + - name: Run Databricks read-side tests with official SDK installed run: >- .venv/bin/python -m pytest tests/test_databricks_client.py tests/test_databricks_discovery.py - tests/test_databricks_workflow_separation.py tests/test_observation_databricks.py coverage: