Address issues found with deploying the API and loading data with the admin.py script

README.md

@@ -239,7 +239,7 @@ Load from a parent directory containing multiple dataset subdirectories:
 uv run python scripts/admin.py Bigpicture load /path/to/datasets/ --multi-dir --load
 ```
 
-Omit `--load` parse XMLs without loading them to the database.
+Omit `--load` to parse XMLs without loading them to the database.
 
 To also sync to OpenSearch immediately after loading, add `--sync`:
 
@@ -270,6 +270,34 @@ uv run python scripts/admin.py Bigpicture generate-index
 
 An unit test fails if this file is different from a freshy generated one.
 
+#### Create the OpenSearch index in a new environment
+
+`generate-index` only writes the mapping to a local file — it does not create the index in
+OpenSearch. **A new OpenSearch instance needs the index created from that mapping before the
+first `--sync`.** If documents are synced into an index that doesn't exist yet, OpenSearch
+silently auto-creates it with a dynamic mapping (e.g. `keyword` fields become `text`, and
+`nested` fields become plain objects), which breaks aggregations and nested queries in ways
+that only surface later, disconnected from the actual cause.
+
+Create the index explicitly:
+
+```bash
+uv run python scripts/admin.py --env-file <env> Bigpicture create-index
+```
+
+This fails loudly if the index already exists, rather than silently leaving a stale mapping in
+place. If an index was already auto-created with the wrong mapping, OpenSearch cannot change an
+existing field's type in place, so it must be deleted and recreated, and previously-synced
+documents must be resynced:
+
+```bash
+curl -X DELETE https://<opensearch-host>:9200/bp-image-index -u <user>:<password>
+uv run python scripts/admin.py --env-file <env> Bigpicture create-index
+# Reset sync state so the next --sync repopulates the recreated index:
+#   UPDATE document SET synced_at = NULL;
+uv run python scripts/admin.py --env-file <env> Bigpicture load <dir> --load --sync
+```
+
 ## LLM search
 
 The experimental Bigpicture LLM search endpoint uses a small local [Ollama](https://ollama.com) model. Install and

dockerfiles/Dockerfile

@@ -12,6 +12,11 @@ COPY pyproject.toml uv.lock .python-version ./
 COPY README.md .
 COPY search_api ./search_api
 
-RUN uv sync --frozen
+RUN uv sync --frozen && \
+    chgrp -R 0 /app && \
+    chmod -R g=u /app
+
+ENV HOME=/app \
+    UV_CACHE_DIR=/app/.cache/uv
 
 ENTRYPOINT ["uv", "run", "sd_search_api"]

scripts/admin.py

@@ -13,6 +13,7 @@
 from search_api.api.deployments import DOMAINS
 from search_api.api.domain import Domain
 from search_api.api.opensearch.index_generator import OpenSearchIndexGeneratorService
+from search_api.api.opensearch.services import create_index, create_search
 from search_api.services.load import LoadService
 from search_api.services.sync import SyncService
 from search_api.database.repository import get_cursor
@@ -81,6 +82,16 @@ def _generate_index(domain: Domain) -> None:
     logging.info("Wrote OpenSearch index to %s.", path)
 
 
+async def _create_index(domain: Domain) -> None:
+    body = OpenSearchIndexGeneratorService(domain.opensearch_fields).generate()
+    search = create_search()
+    try:
+        await create_index(search, domain.opensearch_index, body)
+        logging.info("Created OpenSearch index %s.", domain.opensearch_index)
+    finally:
+        await search.close()
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Search admin CLI.")
     parser.add_argument(
@@ -128,6 +139,16 @@ def _generate_index(domain: Domain) -> None:
             "generate-index", help="Generate the OpenSearch JSON index."
         )
 
+        # create-index
+        commands.add_parser(
+            "create-index",
+            help=(
+                "Create the OpenSearch index in the cluster from the generated "
+                "mapping. Required once per environment before the first --sync; "
+                "fails if the index already exists."
+            ),
+        )
+
     # snomed (deployment-independent)
     snomed_commands = groups.add_parser(
         "snomed", help="Manage the shared SNOMED CT preferred terms cache."
@@ -152,3 +173,5 @@ def _generate_index(domain: Domain) -> None:
             asyncio.run(_load(domain, args))
         elif args.command == "generate-index":
             _generate_index(domain)
+        elif args.command == "create-index":
+            asyncio.run(_create_index(domain))

search_api/api/opensearch/services.py

@@ -28,6 +28,29 @@ def create_search() -> AsyncOpenSearch:
     )
 
 
+async def create_index(
+    search: AsyncOpenSearch, index: str, body: dict[str, Any]
+) -> None:
+    """Create an OpenSearch index with the given settings and mappings.
+
+    Raises SystemException if the index already exists. OpenSearch cannot change
+    an existing field's type, so an index created with the wrong (e.g. dynamic)
+    mapping must be deleted and recreated deliberately rather than silently left
+    in place.
+
+    :param search: The OpenSearch client.
+    :param index: The OpenSearch index name.
+    :param body: The index body (settings and mappings) to create it with.
+    """
+    if await search.indices.exists(index=index):
+        raise SystemException(
+            f"Index '{index}' already exists. Delete it explicitly first if you "
+            "intend to recreate it (e.g. `curl -X DELETE .../<index>`), then rerun "
+            "this command and resync."
+        )
+    await search.indices.create(index=index, body=body)
+
+
 async def index_document(
     search: AsyncOpenSearch,
     index: str,

search_api/bigpicture/services/extract.py

@@ -540,6 +540,13 @@ def _extract_staining_fields(xml: ElementTree) -> list[BigpictureStainingFields]
     return fields
 
 
+_XSI_NIL = "{http://www.w3.org/2001/XMLSchema-instance}nil"
+
+
+def _is_nil(elem: Any) -> bool:
+    return elem.get(_XSI_NIL) == "true"
+
+
 def _extract_code_attribute_value(
     elem: ElementTree, tag: str, *, is_attributes=True
 ) -> BigpictureCodeAttributeValue | None:
@@ -548,7 +555,7 @@ def _extract_code_attribute_value(
     else:
         values = elem.xpath(f"CODE_ATTRIBUTE[TAG='{tag}']/VALUE")
 
-    if not values:
+    if not values or _is_nil(values[0]):
         return None
     value = values[0]
 
@@ -576,6 +583,7 @@ def _extract_code_attribute_values(
             scheme_version=v.findtext("SCHEME_VERSION"),
         )
         for v in values
+        if not _is_nil(v)
     )