diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index 5fc342ed..a620ac70 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -1835,6 +1835,22 @@ def main(): print("šŸ“Š Created index patterns for spans, logs, and service map") + # Warm the field list immediately after creation. Patterns created via the + # raw saved-objects API land with an empty `fields` attribute (OSD only + # fetches fields lazily on first Discover/Explore visit, and the on-read + # auto-fetch safety net was removed upstream). Without this, pre-generated + # datasets show "Fields (0)" and charts error with "Could not locate that + # index-pattern-field" until a manual refresh. This mirrors what the + # dataset-creation wizard does (pre-fetch _fields_for_wildcard at create + # time). delayed_field_refresh() below remains the backstop for indices + # that are still empty at init time. + for pid, title in ( + (logs_pattern_id, "logs-otel-v1*"), + (traces_pattern_id, "otel-v1-apm-span*"), + (service_map_pattern_id, "otel-v2-apm-service-map*"), + ): + refresh_index_pattern_fields(workspace_id, pid, title) + # Set logs as the default index pattern if logs_pattern_id: set_default_index_pattern(workspace_id, logs_pattern_id) @@ -1904,14 +1920,19 @@ def refresh_index_pattern_fields(workspace_id, pattern_id, title): return False if workspace_id and workspace_id != "default": - url = f"{BASE_URL}/w/{workspace_id}/api/index_patterns/_fields_for_wildcard?pattern={title}&meta_fields=_source&meta_fields=_id&meta_fields=_type&meta_fields=_index&meta_fields=_score" + url = f"{BASE_URL}/w/{workspace_id}/api/index_patterns/_fields_for_wildcard" else: - url = f"{BASE_URL}/api/index_patterns/_fields_for_wildcard?pattern={title}&meta_fields=_source&meta_fields=_id&meta_fields=_type&meta_fields=_index&meta_fields=_score" + url = f"{BASE_URL}/api/index_patterns/_fields_for_wildcard" + # Only `pattern` is passed. Do NOT send `meta_fields` ā€” this OSD version's + # route validation rejects the repeated meta_fields query params with a 400 + # ("[request query.params]: definition for this key is missing"). requests + # URL-encodes the pattern's wildcard/special chars via the params dict. try: resp = requests.get( url, auth=(USERNAME, PASSWORD), headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + params={"pattern": title}, verify=False, timeout=30, ) if resp.status_code != 200: @@ -1948,35 +1969,84 @@ def refresh_index_pattern_fields(workspace_id, pattern_id, title): return False -def delayed_field_refresh(workspace_id, patterns): - """Wait for data to land in indices, then refresh field lists. - - Called after the main init completes. Waits 10 minutes for the otel-demo - and agent examples to populate indices with representative documents so - the field refresh picks up all mapped fields. +def delayed_field_refresh(workspace_id, titles, max_attempts=12, interval_minutes=5): + """Periodically refresh field lists until every pattern is populated. + + main() warms the field lists immediately, but indices that are still empty + at init time (otel-demo and agent examples take a while to send their first + documents) return no fields on that first pass. Rather than a single blind + wait, poll on an interval and retry only the patterns that haven't been + populated yet, stopping early once they all succeed. With the defaults this + covers a one-hour window (12 attempts Ɨ 5 minutes). + + Patterns are resolved by title on every attempt rather than once up front, + so this self-heals regardless of start order: if the backstop starts before + main() has created the index patterns, the early attempts simply find + nothing and retry until the patterns exist. This is what lets the compose + service / k8s Job run without a hard `depends_on` the init step ā€” a + dependency that otherwise breaks `docker compose up --wait` (a + `service_completed_successfully` target exiting mid-startup makes --wait + abort with a non-zero code). """ - delay_minutes = 10 - print(f"\nā³ Waiting {delay_minutes} minutes for indices to populate before refreshing fields...") - time.sleep(delay_minutes * 60) + pending = list(titles) + + print( + f"\nā³ Will refresh field lists as indices populate " + f"(up to {max_attempts} attempts every {interval_minutes} min)..." + ) + for attempt in range(1, max_attempts + 1): + print(f"šŸ”„ Field refresh attempt {attempt}/{max_attempts} ({len(pending)} pending)...") + still_pending = [] + for title in pending: + pattern_id = get_existing_index_pattern(workspace_id, title) + if not pattern_id: + # Pattern not created yet (backstop outran init) ā€” retry later. + still_pending.append(title) + continue + if not refresh_index_pattern_fields(workspace_id, pattern_id, title): + still_pending.append(title) + pending = still_pending + if not pending: + print("āœ… Field refresh complete ā€” all patterns populated") + return + # Sleep between attempts (not before the first / after the last), so a + # stack whose indices are already populated exits promptly. + if attempt < max_attempts: + time.sleep(interval_minutes * 60) + + print( + f"āš ļø Field refresh finished with {len(pending)} pattern(s) still empty: " + f"{', '.join(pending)}" + ) - print("šŸ”„ Refreshing index pattern field lists...") - for pattern_id, title in patterns: - refresh_index_pattern_fields(workspace_id, pattern_id, title) - print("āœ… Field refresh complete") +def run_field_refresh_backstop(): + """Run the delayed field-refresh loop as a standalone process. -if __name__ == "__main__": - main() + Split out from main() so it can run as a separate, long-running process + (its own compose service / Kubernetes Job). main() is the fast, blocking + setup that a Helm post-install/upgrade hook waits on; this backstop can run + for up to an hour and must never block install/upgrade completion. - # Re-read workspace and pattern IDs for the delayed refresh. + Intentionally has no hard ordering dependency on the init step: it waits for + dashboards, then delayed_field_refresh() resolves pattern IDs by title on + each attempt, so it works whether it starts before or after init. + """ + wait_for_dashboards() workspace_id = get_existing_workspace() - logs_id = get_existing_index_pattern(workspace_id, "logs-otel-v1*") - traces_id = get_existing_index_pattern(workspace_id, "otel-v1-apm-span*") - svc_map_id = get_existing_index_pattern(workspace_id, "otel-v2-apm-service-map*") - - patterns = [ - (logs_id, "logs-otel-v1*"), - (traces_id, "otel-v1-apm-span*"), - (svc_map_id, "otel-v2-apm-service-map*"), - ] - delayed_field_refresh(workspace_id, patterns) + titles = ["logs-otel-v1*", "otel-v1-apm-span*", "otel-v2-apm-service-map*"] + delayed_field_refresh(workspace_id, titles) + + +if __name__ == "__main__": + import sys + + # Two modes, run as separate processes so the long-running field-refresh + # backstop never blocks init completion (or a Helm hook): + # (default) ā†’ main(): fast one-shot setup + immediate field warm + # refresh-loop ā†’ periodic field refresh until indices populate + mode = sys.argv[1] if len(sys.argv) > 1 else "init" + if mode == "refresh-loop": + run_field_refresh_backstop() + else: + main() diff --git a/charts/observability-stack/templates/field-refresh-configmap.yaml b/charts/observability-stack/templates/field-refresh-configmap.yaml new file mode 100644 index 00000000..706cfd54 --- /dev/null +++ b/charts/observability-stack/templates/field-refresh-configmap.yaml @@ -0,0 +1,16 @@ +{{- if index .Values "opensearch-dashboards" "enabled" }} +# Dedicated, non-hook copy of the init script for the field-refresh backstop +# Job. Kept separate from the hook-managed init-script ConfigMap so the +# non-hook Job has no ordering/delete-recreate coupling with the post-install +# hook phase. The refresh-loop mode only calls the OpenSearch Dashboards API, +# so only the Python script is needed here (no /config assets). +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "observability-stack.fullname" . }}-field-refresh-script + labels: + {{- include "observability-stack.labels" . | nindent 4 }} +data: + init-opensearch-dashboards.py: | +{{ .Files.Get "files/init-opensearch-dashboards.py" | indent 4 }} +{{- end }} diff --git a/charts/observability-stack/templates/field-refresh-job.yaml b/charts/observability-stack/templates/field-refresh-job.yaml new file mode 100644 index 00000000..763c9236 --- /dev/null +++ b/charts/observability-stack/templates/field-refresh-job.yaml @@ -0,0 +1,59 @@ +{{- if index .Values "opensearch-dashboards" "enabled" }} +# Field-refresh backstop - populates index-pattern field lists as indices fill +# with data. Deliberately NOT a Helm hook: it can run for up to ~1 hour and must +# never block install/upgrade completion. The fast one-shot setup lives in the +# init-dashboards hook Job, which Helm waits on; this Job runs independently and +# refreshes field lists once telemetry lands in otherwise-empty indices. +# +# Job spec.template is immutable, so a plain `helm upgrade` cannot patch an +# existing Job. The name is suffixed with the release revision so each upgrade +# creates a fresh Job instead; ttlSecondsAfterFinished reaps completed ones. +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "observability-stack.fullname" . }}-field-refresh-{{ .Release.Revision }} + labels: + {{- include "observability-stack.labels" . | nindent 4 }} +spec: + backoffLimit: 3 + # Cap total runtime so a stuck Job cannot linger indefinitely. Matches the + # script's default loop window (12 attempts x 5 min) plus setup slack. + activeDeadlineSeconds: 4200 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: + app.kubernetes.io/name: field-refresh + spec: + restartPolicy: OnFailure + containers: + - name: field-refresh + image: python:3.12-slim + command: + - /bin/sh + - -c + - | + pip install -q requests pyyaml && python -u /scripts/init-opensearch-dashboards.py refresh-loop + env: + - name: OPENSEARCH_USER + valueFrom: + secretKeyRef: + name: opensearch-credentials + key: username + - name: OPENSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: opensearch-credentials + key: password + - name: BASE_URL + value: "http://{{ .Release.Name }}-opensearch-dashboards:5601" + - name: OPENSEARCH_ENDPOINT + value: "https://{{ .Values.opensearchServiceName | default "opensearch-cluster-master" }}:9200" + volumeMounts: + - name: field-refresh-script + mountPath: /scripts + volumes: + - name: field-refresh-script + configMap: + name: {{ include "observability-stack.fullname" . }}-field-refresh-script +{{- end }} diff --git a/charts/observability-stack/templates/init-dashboards-job.yaml b/charts/observability-stack/templates/init-dashboards-job.yaml index c3143e08..19d73e1f 100644 --- a/charts/observability-stack/templates/init-dashboards-job.yaml +++ b/charts/observability-stack/templates/init-dashboards-job.yaml @@ -24,7 +24,7 @@ spec: - /bin/sh - -c - | - pip install -q requests pyyaml && python /scripts/init-opensearch-dashboards.py + pip install -q requests pyyaml && python -u /scripts/init-opensearch-dashboards.py env: - name: OPENSEARCH_USER valueFrom: diff --git a/docker-compose.yml b/docker-compose.yml index de8c213f..1cbf1ca3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -272,7 +272,7 @@ services: opensearch-dashboards-init: image: python:3.11-alpine container_name: opensearch-dashboards-init - command: sh -c "pip install requests pyyaml && python /init.py" + command: sh -c "pip install requests pyyaml && python -u /init.py" environment: - OPENSEARCH_USER=${OPENSEARCH_USER} - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} @@ -311,3 +311,31 @@ services: restart: "no" logging: *logging + # Field-refresh backstop - populates index-pattern field lists as indices + # fill with data. Runs as a separate long-lived process (up to ~1 hour) so it + # never blocks the one-shot init above. Index patterns created on a fresh + # stack start empty because their indices have no documents yet; this loop + # refreshes them once telemetry lands. + opensearch-dashboards-field-refresh: + image: python:3.11-alpine + container_name: opensearch-dashboards-field-refresh + command: sh -c "pip install requests pyyaml && python -u /init.py refresh-loop" + environment: + - OPENSEARCH_USER=${OPENSEARCH_USER} + - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} + - OPENSEARCH_DASHBOARDS_HOST=${OPENSEARCH_DASHBOARDS_HOST} + - OPENSEARCH_DASHBOARDS_PORT=${OPENSEARCH_DASHBOARDS_PORT} + - OPENSEARCH_DASHBOARDS_PROTOCOL=${OPENSEARCH_DASHBOARDS_PROTOCOL} + volumes: + - ./docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py:/init.py + # No `depends_on` the init service on purpose. A + # `service_completed_successfully` dependency on a one-shot that exits + # mid-startup makes `docker compose up --wait` (test/e2e.sh) abort with a + # non-zero code. The refresh loop instead waits for dashboards itself and + # resolves index-pattern IDs by title on each attempt, so it self-heals + # whether it starts before or after init. + networks: + - observability-stack-network + restart: "no" + logging: *logging + diff --git a/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py b/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py index f1980d90..a8ef2298 100644 --- a/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py +++ b/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py @@ -1761,6 +1761,22 @@ def main(): print("šŸ“Š Created index patterns for spans, logs, and service map") + # Warm the field list immediately after creation. Patterns created via the + # raw saved-objects API land with an empty `fields` attribute (OSD only + # fetches fields lazily on first Discover/Explore visit, and the on-read + # auto-fetch safety net was removed upstream). Without this, pre-generated + # datasets show "Fields (0)" and charts error with "Could not locate that + # index-pattern-field" until a manual refresh. This mirrors what the + # dataset-creation wizard does (pre-fetch _fields_for_wildcard at create + # time). delayed_field_refresh() below remains the backstop for indices + # that are still empty at init time. + for pid, title in ( + (logs_pattern_id, "logs-otel-v1*"), + (traces_pattern_id, "otel-v1-apm-span*"), + (service_map_pattern_id, "otel-v2-apm-service-map*"), + ): + refresh_index_pattern_fields(workspace_id, pid, title) + # Set logs as the default index pattern if logs_pattern_id: set_default_index_pattern(workspace_id, logs_pattern_id) @@ -1831,14 +1847,19 @@ def refresh_index_pattern_fields(workspace_id, pattern_id, title): return False if workspace_id and workspace_id != "default": - url = f"{BASE_URL}/w/{workspace_id}/api/index_patterns/_fields_for_wildcard?pattern={title}&meta_fields=_source&meta_fields=_id&meta_fields=_type&meta_fields=_index&meta_fields=_score" + url = f"{BASE_URL}/w/{workspace_id}/api/index_patterns/_fields_for_wildcard" else: - url = f"{BASE_URL}/api/index_patterns/_fields_for_wildcard?pattern={title}&meta_fields=_source&meta_fields=_id&meta_fields=_type&meta_fields=_index&meta_fields=_score" + url = f"{BASE_URL}/api/index_patterns/_fields_for_wildcard" + # Only `pattern` is passed. Do NOT send `meta_fields` ā€” this OSD version's + # route validation rejects the repeated meta_fields query params with a 400 + # ("[request query.params]: definition for this key is missing"). requests + # URL-encodes the pattern's wildcard/special chars via the params dict. try: resp = requests.get( url, auth=(USERNAME, PASSWORD), headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + params={"pattern": title}, verify=False, timeout=30, ) if resp.status_code != 200: @@ -1875,36 +1896,84 @@ def refresh_index_pattern_fields(workspace_id, pattern_id, title): return False -def delayed_field_refresh(workspace_id, patterns): - """Wait for data to land in indices, then refresh field lists. - - Called after the main init completes. Waits 10 minutes for the otel-demo - and agent examples to populate indices with representative documents so - the field refresh picks up all mapped fields. +def delayed_field_refresh(workspace_id, titles, max_attempts=12, interval_minutes=5): + """Periodically refresh field lists until every pattern is populated. + + main() warms the field lists immediately, but indices that are still empty + at init time (otel-demo and agent examples take a while to send their first + documents) return no fields on that first pass. Rather than a single blind + wait, poll on an interval and retry only the patterns that haven't been + populated yet, stopping early once they all succeed. With the defaults this + covers a one-hour window (12 attempts Ɨ 5 minutes). + + Patterns are resolved by title on every attempt rather than once up front, + so this self-heals regardless of start order: if the backstop starts before + main() has created the index patterns, the early attempts simply find + nothing and retry until the patterns exist. This is what lets the compose + service / k8s Job run without a hard `depends_on` the init step ā€” a + dependency that otherwise breaks `docker compose up --wait` (a + `service_completed_successfully` target exiting mid-startup makes --wait + abort with a non-zero code). """ - delay_minutes = 10 - print(f"\nā³ Waiting {delay_minutes} minutes for indices to populate before refreshing fields...") - time.sleep(delay_minutes * 60) + pending = list(titles) + + print( + f"\nā³ Will refresh field lists as indices populate " + f"(up to {max_attempts} attempts every {interval_minutes} min)..." + ) + for attempt in range(1, max_attempts + 1): + print(f"šŸ”„ Field refresh attempt {attempt}/{max_attempts} ({len(pending)} pending)...") + still_pending = [] + for title in pending: + pattern_id = get_existing_index_pattern(workspace_id, title) + if not pattern_id: + # Pattern not created yet (backstop outran init) ā€” retry later. + still_pending.append(title) + continue + if not refresh_index_pattern_fields(workspace_id, pattern_id, title): + still_pending.append(title) + pending = still_pending + if not pending: + print("āœ… Field refresh complete ā€” all patterns populated") + return + # Sleep between attempts (not before the first / after the last), so a + # stack whose indices are already populated exits promptly. + if attempt < max_attempts: + time.sleep(interval_minutes * 60) + + print( + f"āš ļø Field refresh finished with {len(pending)} pattern(s) still empty: " + f"{', '.join(pending)}" + ) - print("šŸ”„ Refreshing index pattern field lists...") - for pattern_id, title in patterns: - refresh_index_pattern_fields(workspace_id, pattern_id, title) - print("āœ… Field refresh complete") +def run_field_refresh_backstop(): + """Run the delayed field-refresh loop as a standalone process. -if __name__ == "__main__": - main() + Split out from main() so it can run as a separate, long-running process + (its own compose service / Kubernetes Job). main() is the fast, blocking + setup that a Helm post-install/upgrade hook waits on; this backstop can run + for up to an hour and must never block install/upgrade completion. - # Re-read workspace and pattern IDs for the delayed refresh. - # main() already printed success, so this is a background follow-up. + Intentionally has no hard ordering dependency on the init step: it waits for + dashboards, then delayed_field_refresh() resolves pattern IDs by title on + each attempt, so it works whether it starts before or after init. + """ + wait_for_dashboards() workspace_id = get_existing_workspace() - logs_id = get_existing_index_pattern(workspace_id, "logs-otel-v1*") - traces_id = get_existing_index_pattern(workspace_id, "otel-v1-apm-span*") - svc_map_id = get_existing_index_pattern(workspace_id, "otel-v2-apm-service-map*") - - patterns = [ - (logs_id, "logs-otel-v1*"), - (traces_id, "otel-v1-apm-span*"), - (svc_map_id, "otel-v2-apm-service-map*"), - ] - delayed_field_refresh(workspace_id, patterns) + titles = ["logs-otel-v1*", "otel-v1-apm-span*", "otel-v2-apm-service-map*"] + delayed_field_refresh(workspace_id, titles) + + +if __name__ == "__main__": + import sys + + # Two modes, run as separate processes so the long-running field-refresh + # backstop never blocks init completion (or a Helm hook): + # (default) ā†’ main(): fast one-shot setup + immediate field warm + # refresh-loop ā†’ periodic field refresh until indices populate + mode = sys.argv[1] if len(sys.argv) > 1 else "init" + if mode == "refresh-loop": + run_field_refresh_backstop() + else: + main()