fix: populate index-pattern field lists

charts/observability-stack/files/init-opensearch-dashboards.py

@@ -1835,6 +1835,22 @@ def main():
 
     print("📊 Created index patterns for spans, logs, and service map")
 
+    # Warm the field list immediately after creation. Patterns created via the
+    # raw saved-objects API land with an empty `fields` attribute (OSD only
+    # fetches fields lazily on first Discover/Explore visit, and the on-read
+    # auto-fetch safety net was removed upstream). Without this, pre-generated
+    # datasets show "Fields (0)" and charts error with "Could not locate that
+    # index-pattern-field" until a manual refresh. This mirrors what the
+    # dataset-creation wizard does (pre-fetch _fields_for_wildcard at create
+    # time). delayed_field_refresh() below remains the backstop for indices
+    # that are still empty at init time.
+    for pid, title in (
+        (logs_pattern_id, "logs-otel-v1*"),
+        (traces_pattern_id, "otel-v1-apm-span*"),
+        (service_map_pattern_id, "otel-v2-apm-service-map*"),
+    ):
+        refresh_index_pattern_fields(workspace_id, pid, title)
+
     # Set logs as the default index pattern
     if logs_pattern_id:
         set_default_index_pattern(workspace_id, logs_pattern_id)
@@ -1904,14 +1920,19 @@ def refresh_index_pattern_fields(workspace_id, pattern_id, title):
         return False
 
     if workspace_id and workspace_id != "default":
-        url = f"{BASE_URL}/w/{workspace_id}/api/index_patterns/_fields_for_wildcard?pattern={title}&meta_fields=_source&meta_fields=_id&meta_fields=_type&meta_fields=_index&meta_fields=_score"
+        url = f"{BASE_URL}/w/{workspace_id}/api/index_patterns/_fields_for_wildcard"
     else:
-        url = f"{BASE_URL}/api/index_patterns/_fields_for_wildcard?pattern={title}&meta_fields=_source&meta_fields=_id&meta_fields=_type&meta_fields=_index&meta_fields=_score"
+        url = f"{BASE_URL}/api/index_patterns/_fields_for_wildcard"
 
+    # Only `pattern` is passed. Do NOT send `meta_fields` — this OSD version's
+    # route validation rejects the repeated meta_fields query params with a 400
+    # ("[request query.params]: definition for this key is missing"). requests
+    # URL-encodes the pattern's wildcard/special chars via the params dict.
     try:
         resp = requests.get(
             url, auth=(USERNAME, PASSWORD),
             headers={"Content-Type": "application/json", "osd-xsrf": "true"},
+            params={"pattern": title},
             verify=False, timeout=30,
         )
         if resp.status_code != 200:
@@ -1948,35 +1969,84 @@ def refresh_index_pattern_fields(workspace_id, pattern_id, title):
         return False
 
 
-def delayed_field_refresh(workspace_id, patterns):
-    """Wait for data to land in indices, then refresh field lists.
-
-    Called after the main init completes. Waits 10 minutes for the otel-demo
-    and agent examples to populate indices with representative documents so
-    the field refresh picks up all mapped fields.
+def delayed_field_refresh(workspace_id, titles, max_attempts=12, interval_minutes=5):
+    """Periodically refresh field lists until every pattern is populated.
+
+    main() warms the field lists immediately, but indices that are still empty
+    at init time (otel-demo and agent examples take a while to send their first
+    documents) return no fields on that first pass. Rather than a single blind
+    wait, poll on an interval and retry only the patterns that haven't been
+    populated yet, stopping early once they all succeed. With the defaults this
+    covers a one-hour window (12 attempts × 5 minutes).
+
+    Patterns are resolved by title on every attempt rather than once up front,
+    so this self-heals regardless of start order: if the backstop starts before
+    main() has created the index patterns, the early attempts simply find
+    nothing and retry until the patterns exist. This is what lets the compose
+    service / k8s Job run without a hard `depends_on` the init step — a
+    dependency that otherwise breaks `docker compose up --wait` (a
+    `service_completed_successfully` target exiting mid-startup makes --wait
+    abort with a non-zero code).
     """
-    delay_minutes = 10
-    print(f"\n⏳ Waiting {delay_minutes} minutes for indices to populate before refreshing fields...")
-    time.sleep(delay_minutes * 60)
+    pending = list(titles)
+
+    print(
+        f"\n⏳ Will refresh field lists as indices populate "
+        f"(up to {max_attempts} attempts every {interval_minutes} min)..."
+    )
+    for attempt in range(1, max_attempts + 1):
+        print(f"🔄 Field refresh attempt {attempt}/{max_attempts} ({len(pending)} pending)...")
+        still_pending = []
+        for title in pending:
+            pattern_id = get_existing_index_pattern(workspace_id, title)
+            if not pattern_id:
+                # Pattern not created yet (backstop outran init) — retry later.
+                still_pending.append(title)
+                continue
+            if not refresh_index_pattern_fields(workspace_id, pattern_id, title):
+                still_pending.append(title)
+        pending = still_pending
+        if not pending:
+            print("✅ Field refresh complete — all patterns populated")
+            return
+        # Sleep between attempts (not before the first / after the last), so a
+        # stack whose indices are already populated exits promptly.
+        if attempt < max_attempts:
+            time.sleep(interval_minutes * 60)
+
+    print(
+        f"⚠️  Field refresh finished with {len(pending)} pattern(s) still empty: "
+        f"{', '.join(pending)}"
+    )
 
-    print("🔄 Refreshing index pattern field lists...")
-    for pattern_id, title in patterns:
-        refresh_index_pattern_fields(workspace_id, pattern_id, title)
-    print("✅ Field refresh complete")
 
+def run_field_refresh_backstop():
+    """Run the delayed field-refresh loop as a standalone process.
 
-if __name__ == "__main__":
-    main()
+    Split out from main() so it can run as a separate, long-running process
+    (its own compose service / Kubernetes Job). main() is the fast, blocking
+    setup that a Helm post-install/upgrade hook waits on; this backstop can run
+    for up to an hour and must never block install/upgrade completion.
 
-    # Re-read workspace and pattern IDs for the delayed refresh.
+    Intentionally has no hard ordering dependency on the init step: it waits for
+    dashboards, then delayed_field_refresh() resolves pattern IDs by title on
+    each attempt, so it works whether it starts before or after init.
+    """
+    wait_for_dashboards()
     workspace_id = get_existing_workspace()
-    logs_id = get_existing_index_pattern(workspace_id, "logs-otel-v1*")
-    traces_id = get_existing_index_pattern(workspace_id, "otel-v1-apm-span*")
-    svc_map_id = get_existing_index_pattern(workspace_id, "otel-v2-apm-service-map*")
-
-    patterns = [
-        (logs_id, "logs-otel-v1*"),
-        (traces_id, "otel-v1-apm-span*"),
-        (svc_map_id, "otel-v2-apm-service-map*"),
-    ]
-    delayed_field_refresh(workspace_id, patterns)
+    titles = ["logs-otel-v1*", "otel-v1-apm-span*", "otel-v2-apm-service-map*"]
+    delayed_field_refresh(workspace_id, titles)
+
+
+if __name__ == "__main__":
+    import sys
+
+    # Two modes, run as separate processes so the long-running field-refresh
+    # backstop never blocks init completion (or a Helm hook):
+    #   (default)      → main(): fast one-shot setup + immediate field warm
+    #   refresh-loop   → periodic field refresh until indices populate
+    mode = sys.argv[1] if len(sys.argv) > 1 else "init"
+    if mode == "refresh-loop":
+        run_field_refresh_backstop()
+    else:
+        main()

charts/observability-stack/templates/field-refresh-configmap.yaml

@@ -0,0 +1,16 @@
+{{- if index .Values "opensearch-dashboards" "enabled" }}
+# Dedicated, non-hook copy of the init script for the field-refresh backstop
+# Job. Kept separate from the hook-managed init-script ConfigMap so the
+# non-hook Job has no ordering/delete-recreate coupling with the post-install
+# hook phase. The refresh-loop mode only calls the OpenSearch Dashboards API,
+# so only the Python script is needed here (no /config assets).
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "observability-stack.fullname" . }}-field-refresh-script
+  labels:
+    {{- include "observability-stack.labels" . | nindent 4 }}
+data:
+  init-opensearch-dashboards.py: |
+{{ .Files.Get "files/init-opensearch-dashboards.py" | indent 4 }}
+{{- end }}

charts/observability-stack/templates/field-refresh-job.yaml

@@ -0,0 +1,59 @@
+{{- if index .Values "opensearch-dashboards" "enabled" }}
+# Field-refresh backstop - populates index-pattern field lists as indices fill
+# with data. Deliberately NOT a Helm hook: it can run for up to ~1 hour and must
+# never block install/upgrade completion. The fast one-shot setup lives in the
+# init-dashboards hook Job, which Helm waits on; this Job runs independently and
+# refreshes field lists once telemetry lands in otherwise-empty indices.
+#
+# Job spec.template is immutable, so a plain `helm upgrade` cannot patch an
+# existing Job. The name is suffixed with the release revision so each upgrade
+# creates a fresh Job instead; ttlSecondsAfterFinished reaps completed ones.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "observability-stack.fullname" . }}-field-refresh-{{ .Release.Revision }}
+  labels:
+    {{- include "observability-stack.labels" . | nindent 4 }}
+spec:
+  backoffLimit: 3
+  # Cap total runtime so a stuck Job cannot linger indefinitely. Matches the
+  # script's default loop window (12 attempts x 5 min) plus setup slack.
+  activeDeadlineSeconds: 4200
+  ttlSecondsAfterFinished: 600
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: field-refresh
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: field-refresh
+          image: python:3.12-slim
+          command:
+            - /bin/sh
+            - -c
+            - |
+              pip install -q requests pyyaml && python -u /scripts/init-opensearch-dashboards.py refresh-loop
+          env:
+            - name: OPENSEARCH_USER
+              valueFrom:
+                secretKeyRef:
+                  name: opensearch-credentials
+                  key: username
+            - name: OPENSEARCH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: opensearch-credentials
+                  key: password
+            - name: BASE_URL
+              value: "http://{{ .Release.Name }}-opensearch-dashboards:5601"
+            - name: OPENSEARCH_ENDPOINT
+              value: "https://{{ .Values.opensearchServiceName | default "opensearch-cluster-master" }}:9200"
+          volumeMounts:
+            - name: field-refresh-script
+              mountPath: /scripts
+      volumes:
+        - name: field-refresh-script
+          configMap:
+            name: {{ include "observability-stack.fullname" . }}-field-refresh-script
+{{- end }}

charts/observability-stack/templates/init-dashboards-job.yaml

@@ -24,7 +24,7 @@ spec:
             - /bin/sh
             - -c
             - |
-              pip install -q requests pyyaml && python /scripts/init-opensearch-dashboards.py
+              pip install -q requests pyyaml && python -u /scripts/init-opensearch-dashboards.py
           env:
             - name: OPENSEARCH_USER
               valueFrom:

docker-compose.yml

@@ -272,7 +272,7 @@ services:
   opensearch-dashboards-init:
     image: python:3.11-alpine
     container_name: opensearch-dashboards-init
-    command: sh -c "pip install requests pyyaml && python /init.py"
+    command: sh -c "pip install requests pyyaml && python -u /init.py"
     environment:
       - OPENSEARCH_USER=${OPENSEARCH_USER}
       - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
@@ -311,3 +311,31 @@ services:
     restart: "no"
     logging: *logging
 
+  # Field-refresh backstop - populates index-pattern field lists as indices
+  # fill with data. Runs as a separate long-lived process (up to ~1 hour) so it
+  # never blocks the one-shot init above. Index patterns created on a fresh
+  # stack start empty because their indices have no documents yet; this loop
+  # refreshes them once telemetry lands.
+  opensearch-dashboards-field-refresh:
+    image: python:3.11-alpine
+    container_name: opensearch-dashboards-field-refresh
+    command: sh -c "pip install requests pyyaml && python -u /init.py refresh-loop"
+    environment:
+      - OPENSEARCH_USER=${OPENSEARCH_USER}
+      - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
+      - OPENSEARCH_DASHBOARDS_HOST=${OPENSEARCH_DASHBOARDS_HOST}
+      - OPENSEARCH_DASHBOARDS_PORT=${OPENSEARCH_DASHBOARDS_PORT}
+      - OPENSEARCH_DASHBOARDS_PROTOCOL=${OPENSEARCH_DASHBOARDS_PROTOCOL}
+    volumes:
+      - ./docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py:/init.py
+    # No `depends_on` the init service on purpose. A
+    # `service_completed_successfully` dependency on a one-shot that exits
+    # mid-startup makes `docker compose up --wait` (test/e2e.sh) abort with a
+    # non-zero code. The refresh loop instead waits for dashboards itself and
+    # resolves index-pattern IDs by title on each attempt, so it self-heals
+    # whether it starts before or after init.
+    networks:
+      - observability-stack-network
+    restart: "no"
+    logging: *logging
+