diff --git a/README.md b/README.md index 29d7b4f..e901ca1 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ Load from a parent directory containing multiple dataset subdirectories: uv run python scripts/admin.py Bigpicture load /path/to/datasets/ --multi-dir --load ``` -Omit `--load` parse XMLs without loading them to the database. +Omit `--load` to parse XMLs without loading them to the database. To also sync to OpenSearch immediately after loading, add `--sync`: @@ -270,6 +270,34 @@ uv run python scripts/admin.py Bigpicture generate-index An unit test fails if this file is different from a freshy generated one. +#### Create the OpenSearch index in a new environment + +`generate-index` only writes the mapping to a local file — it does not create the index in +OpenSearch. **A new OpenSearch instance needs the index created from that mapping before the +first `--sync`.** If documents are synced into an index that doesn't exist yet, OpenSearch +silently auto-creates it with a dynamic mapping (e.g. `keyword` fields become `text`, and +`nested` fields become plain objects), which breaks aggregations and nested queries in ways +that only surface later, disconnected from the actual cause. + +Create the index explicitly: + +```bash +uv run python scripts/admin.py --env-file Bigpicture create-index +``` + +This fails loudly if the index already exists, rather than silently leaving a stale mapping in +place. If an index was already auto-created with the wrong mapping, OpenSearch cannot change an +existing field's type in place, so it must be deleted and recreated, and previously-synced +documents must be resynced: + +```bash +curl -X DELETE https://:9200/bp-image-index -u : +uv run python scripts/admin.py --env-file Bigpicture create-index +# Reset sync state so the next --sync repopulates the recreated index: +# UPDATE document SET synced_at = NULL; +uv run python scripts/admin.py --env-file Bigpicture load --load --sync +``` + ## LLM search The experimental Bigpicture LLM search endpoint uses a small local [Ollama](https://ollama.com) model. Install and diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index f3f2ec2..4dbc10a 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -12,6 +12,11 @@ COPY pyproject.toml uv.lock .python-version ./ COPY README.md . COPY search_api ./search_api -RUN uv sync --frozen +RUN uv sync --frozen && \ + chgrp -R 0 /app && \ + chmod -R g=u /app + +ENV HOME=/app \ + UV_CACHE_DIR=/app/.cache/uv ENTRYPOINT ["uv", "run", "sd_search_api"] diff --git a/scripts/admin.py b/scripts/admin.py index ac55e4e..c235c28 100644 --- a/scripts/admin.py +++ b/scripts/admin.py @@ -13,6 +13,7 @@ from search_api.api.deployments import DOMAINS from search_api.api.domain import Domain from search_api.api.opensearch.index_generator import OpenSearchIndexGeneratorService +from search_api.api.opensearch.services import create_index, create_search from search_api.services.load import LoadService from search_api.services.sync import SyncService from search_api.database.repository import get_cursor @@ -81,6 +82,16 @@ def _generate_index(domain: Domain) -> None: logging.info("Wrote OpenSearch index to %s.", path) +async def _create_index(domain: Domain) -> None: + body = OpenSearchIndexGeneratorService(domain.opensearch_fields).generate() + search = create_search() + try: + await create_index(search, domain.opensearch_index, body) + logging.info("Created OpenSearch index %s.", domain.opensearch_index) + finally: + await search.close() + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Search admin CLI.") parser.add_argument( @@ -128,6 +139,16 @@ def _generate_index(domain: Domain) -> None: "generate-index", help="Generate the OpenSearch JSON index." ) + # create-index + commands.add_parser( + "create-index", + help=( + "Create the OpenSearch index in the cluster from the generated " + "mapping. Required once per environment before the first --sync; " + "fails if the index already exists." + ), + ) + # snomed (deployment-independent) snomed_commands = groups.add_parser( "snomed", help="Manage the shared SNOMED CT preferred terms cache." @@ -152,3 +173,5 @@ def _generate_index(domain: Domain) -> None: asyncio.run(_load(domain, args)) elif args.command == "generate-index": _generate_index(domain) + elif args.command == "create-index": + asyncio.run(_create_index(domain)) diff --git a/search_api/api/opensearch/services.py b/search_api/api/opensearch/services.py index d55d93b..c88e7c0 100644 --- a/search_api/api/opensearch/services.py +++ b/search_api/api/opensearch/services.py @@ -28,6 +28,29 @@ def create_search() -> AsyncOpenSearch: ) +async def create_index( + search: AsyncOpenSearch, index: str, body: dict[str, Any] +) -> None: + """Create an OpenSearch index with the given settings and mappings. + + Raises SystemException if the index already exists. OpenSearch cannot change + an existing field's type, so an index created with the wrong (e.g. dynamic) + mapping must be deleted and recreated deliberately rather than silently left + in place. + + :param search: The OpenSearch client. + :param index: The OpenSearch index name. + :param body: The index body (settings and mappings) to create it with. + """ + if await search.indices.exists(index=index): + raise SystemException( + f"Index '{index}' already exists. Delete it explicitly first if you " + "intend to recreate it (e.g. `curl -X DELETE .../`), then rerun " + "this command and resync." + ) + await search.indices.create(index=index, body=body) + + async def index_document( search: AsyncOpenSearch, index: str, diff --git a/search_api/bigpicture/services/extract.py b/search_api/bigpicture/services/extract.py index a0bd204..18c7597 100644 --- a/search_api/bigpicture/services/extract.py +++ b/search_api/bigpicture/services/extract.py @@ -540,6 +540,13 @@ def _extract_staining_fields(xml: ElementTree) -> list[BigpictureStainingFields] return fields +_XSI_NIL = "{http://www.w3.org/2001/XMLSchema-instance}nil" + + +def _is_nil(elem: Any) -> bool: + return elem.get(_XSI_NIL) == "true" + + def _extract_code_attribute_value( elem: ElementTree, tag: str, *, is_attributes=True ) -> BigpictureCodeAttributeValue | None: @@ -548,7 +555,7 @@ def _extract_code_attribute_value( else: values = elem.xpath(f"CODE_ATTRIBUTE[TAG='{tag}']/VALUE") - if not values: + if not values or _is_nil(values[0]): return None value = values[0] @@ -576,6 +583,7 @@ def _extract_code_attribute_values( scheme_version=v.findtext("SCHEME_VERSION"), ) for v in values + if not _is_nil(v) )