From 03de6926cd7f2aa6449a0b505dfdbb1742e749d4 Mon Sep 17 00:00:00 2001 From: Adam Tackett Date: Mon, 3 Aug 2026 14:41:53 -0700 Subject: [PATCH 1/3] init fields fix Signed-off-by: Adam Tackett --- .../files/init-opensearch-dashboards.py | 88 ++++++++++++++---- .../templates/field-refresh-configmap.yaml | 16 ++++ .../templates/field-refresh-job.yaml | 59 ++++++++++++ .../templates/init-dashboards-job.yaml | 2 +- docker-compose.yml | 27 +++++- .../init/init-opensearch-dashboards.py | 89 +++++++++++++++---- 6 files changed, 248 insertions(+), 33 deletions(-) create mode 100644 charts/observability-stack/templates/field-refresh-configmap.yaml create mode 100644 charts/observability-stack/templates/field-refresh-job.yaml diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index 5fc342ed..96cf618d 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -1835,6 +1835,22 @@ def main(): print("šŸ“Š Created index patterns for spans, logs, and service map") + # Warm the field list immediately after creation. Patterns created via the + # raw saved-objects API land with an empty `fields` attribute (OSD only + # fetches fields lazily on first Discover/Explore visit, and the on-read + # auto-fetch safety net was removed upstream). Without this, pre-generated + # datasets show "Fields (0)" and charts error with "Could not locate that + # index-pattern-field" until a manual refresh. This mirrors what the + # dataset-creation wizard does (pre-fetch _fields_for_wildcard at create + # time). delayed_field_refresh() below remains the backstop for indices + # that are still empty at init time. + for pid, title in ( + (logs_pattern_id, "logs-otel-v1*"), + (traces_pattern_id, "otel-v1-apm-span*"), + (service_map_pattern_id, "otel-v2-apm-service-map*"), + ): + refresh_index_pattern_fields(workspace_id, pid, title) + # Set logs as the default index pattern if logs_pattern_id: set_default_index_pattern(workspace_id, logs_pattern_id) @@ -1948,27 +1964,55 @@ def refresh_index_pattern_fields(workspace_id, pattern_id, title): return False -def delayed_field_refresh(workspace_id, patterns): - """Wait for data to land in indices, then refresh field lists. +def delayed_field_refresh(workspace_id, patterns, max_attempts=12, interval_minutes=5): + """Periodically refresh field lists until every pattern is populated. - Called after the main init completes. Waits 10 minutes for the otel-demo - and agent examples to populate indices with representative documents so - the field refresh picks up all mapped fields. + main() warms the field lists immediately, but indices that are still empty + at init time (otel-demo and agent examples take a while to send their first + documents) return no fields on that first pass. Rather than a single blind + wait, poll on an interval and retry only the patterns that haven't been + populated yet, stopping early once they all succeed. With the defaults this + covers a one-hour window (12 attempts Ɨ 5 minutes). """ - delay_minutes = 10 - print(f"\nā³ Waiting {delay_minutes} minutes for indices to populate before refreshing fields...") - time.sleep(delay_minutes * 60) + pending = [(pid, title) for pid, title in patterns if pid] + if not pending: + print("ā­ļø No index patterns to refresh") + return - print("šŸ”„ Refreshing index pattern field lists...") - for pattern_id, title in patterns: - refresh_index_pattern_fields(workspace_id, pattern_id, title) - print("āœ… Field refresh complete") + print( + f"\nā³ Will refresh field lists as indices populate " + f"(up to {max_attempts} attempts every {interval_minutes} min)..." + ) + for attempt in range(1, max_attempts + 1): + print(f"šŸ”„ Field refresh attempt {attempt}/{max_attempts} ({len(pending)} pending)...") + still_pending = [] + for pattern_id, title in pending: + if not refresh_index_pattern_fields(workspace_id, pattern_id, title): + still_pending.append((pattern_id, title)) + pending = still_pending + if not pending: + print("āœ… Field refresh complete ā€” all patterns populated") + return + # Sleep between attempts (not before the first / after the last), so a + # stack whose indices are already populated exits promptly. + if attempt < max_attempts: + time.sleep(interval_minutes * 60) + + print( + f"āš ļø Field refresh finished with {len(pending)} pattern(s) still empty: " + f"{', '.join(title for _, title in pending)}" + ) -if __name__ == "__main__": - main() +def run_field_refresh_backstop(): + """Re-read workspace + pattern IDs and run the delayed field-refresh loop. - # Re-read workspace and pattern IDs for the delayed refresh. + Split out from main() so it can run as a separate, long-running process + (its own compose service / Kubernetes Job). main() is the fast, blocking + setup that a Helm post-install/upgrade hook waits on; this backstop can run + for up to an hour and must never block install/upgrade completion. + """ + wait_for_dashboards() workspace_id = get_existing_workspace() logs_id = get_existing_index_pattern(workspace_id, "logs-otel-v1*") traces_id = get_existing_index_pattern(workspace_id, "otel-v1-apm-span*") @@ -1980,3 +2024,17 @@ def delayed_field_refresh(workspace_id, patterns): (svc_map_id, "otel-v2-apm-service-map*"), ] delayed_field_refresh(workspace_id, patterns) + + +if __name__ == "__main__": + import sys + + # Two modes, run as separate processes so the long-running field-refresh + # backstop never blocks init completion (or a Helm hook): + # (default) ā†’ main(): fast one-shot setup + immediate field warm + # refresh-loop ā†’ periodic field refresh until indices populate + mode = sys.argv[1] if len(sys.argv) > 1 else "init" + if mode == "refresh-loop": + run_field_refresh_backstop() + else: + main() diff --git a/charts/observability-stack/templates/field-refresh-configmap.yaml b/charts/observability-stack/templates/field-refresh-configmap.yaml new file mode 100644 index 00000000..706cfd54 --- /dev/null +++ b/charts/observability-stack/templates/field-refresh-configmap.yaml @@ -0,0 +1,16 @@ +{{- if index .Values "opensearch-dashboards" "enabled" }} +# Dedicated, non-hook copy of the init script for the field-refresh backstop +# Job. Kept separate from the hook-managed init-script ConfigMap so the +# non-hook Job has no ordering/delete-recreate coupling with the post-install +# hook phase. The refresh-loop mode only calls the OpenSearch Dashboards API, +# so only the Python script is needed here (no /config assets). +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "observability-stack.fullname" . }}-field-refresh-script + labels: + {{- include "observability-stack.labels" . | nindent 4 }} +data: + init-opensearch-dashboards.py: | +{{ .Files.Get "files/init-opensearch-dashboards.py" | indent 4 }} +{{- end }} diff --git a/charts/observability-stack/templates/field-refresh-job.yaml b/charts/observability-stack/templates/field-refresh-job.yaml new file mode 100644 index 00000000..763c9236 --- /dev/null +++ b/charts/observability-stack/templates/field-refresh-job.yaml @@ -0,0 +1,59 @@ +{{- if index .Values "opensearch-dashboards" "enabled" }} +# Field-refresh backstop - populates index-pattern field lists as indices fill +# with data. Deliberately NOT a Helm hook: it can run for up to ~1 hour and must +# never block install/upgrade completion. The fast one-shot setup lives in the +# init-dashboards hook Job, which Helm waits on; this Job runs independently and +# refreshes field lists once telemetry lands in otherwise-empty indices. +# +# Job spec.template is immutable, so a plain `helm upgrade` cannot patch an +# existing Job. The name is suffixed with the release revision so each upgrade +# creates a fresh Job instead; ttlSecondsAfterFinished reaps completed ones. +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "observability-stack.fullname" . }}-field-refresh-{{ .Release.Revision }} + labels: + {{- include "observability-stack.labels" . | nindent 4 }} +spec: + backoffLimit: 3 + # Cap total runtime so a stuck Job cannot linger indefinitely. Matches the + # script's default loop window (12 attempts x 5 min) plus setup slack. + activeDeadlineSeconds: 4200 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: + app.kubernetes.io/name: field-refresh + spec: + restartPolicy: OnFailure + containers: + - name: field-refresh + image: python:3.12-slim + command: + - /bin/sh + - -c + - | + pip install -q requests pyyaml && python -u /scripts/init-opensearch-dashboards.py refresh-loop + env: + - name: OPENSEARCH_USER + valueFrom: + secretKeyRef: + name: opensearch-credentials + key: username + - name: OPENSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: opensearch-credentials + key: password + - name: BASE_URL + value: "http://{{ .Release.Name }}-opensearch-dashboards:5601" + - name: OPENSEARCH_ENDPOINT + value: "https://{{ .Values.opensearchServiceName | default "opensearch-cluster-master" }}:9200" + volumeMounts: + - name: field-refresh-script + mountPath: /scripts + volumes: + - name: field-refresh-script + configMap: + name: {{ include "observability-stack.fullname" . }}-field-refresh-script +{{- end }} diff --git a/charts/observability-stack/templates/init-dashboards-job.yaml b/charts/observability-stack/templates/init-dashboards-job.yaml index c3143e08..19d73e1f 100644 --- a/charts/observability-stack/templates/init-dashboards-job.yaml +++ b/charts/observability-stack/templates/init-dashboards-job.yaml @@ -24,7 +24,7 @@ spec: - /bin/sh - -c - | - pip install -q requests pyyaml && python /scripts/init-opensearch-dashboards.py + pip install -q requests pyyaml && python -u /scripts/init-opensearch-dashboards.py env: - name: OPENSEARCH_USER valueFrom: diff --git a/docker-compose.yml b/docker-compose.yml index de8c213f..d5925076 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -272,7 +272,7 @@ services: opensearch-dashboards-init: image: python:3.11-alpine container_name: opensearch-dashboards-init - command: sh -c "pip install requests pyyaml && python /init.py" + command: sh -c "pip install requests pyyaml && python -u /init.py" environment: - OPENSEARCH_USER=${OPENSEARCH_USER} - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} @@ -311,3 +311,28 @@ services: restart: "no" logging: *logging + # Field-refresh backstop - populates index-pattern field lists as indices + # fill with data. Runs as a separate long-lived process (up to ~1 hour) so it + # never blocks the one-shot init above. Index patterns created on a fresh + # stack start empty because their indices have no documents yet; this loop + # refreshes them once telemetry lands. + opensearch-dashboards-field-refresh: + image: python:3.11-alpine + container_name: opensearch-dashboards-field-refresh + command: sh -c "pip install requests pyyaml && python -u /init.py refresh-loop" + environment: + - OPENSEARCH_USER=${OPENSEARCH_USER} + - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} + - OPENSEARCH_DASHBOARDS_HOST=${OPENSEARCH_DASHBOARDS_HOST} + - OPENSEARCH_DASHBOARDS_PORT=${OPENSEARCH_DASHBOARDS_PORT} + - OPENSEARCH_DASHBOARDS_PROTOCOL=${OPENSEARCH_DASHBOARDS_PROTOCOL} + volumes: + - ./docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py:/init.py + depends_on: + opensearch-dashboards-init: + condition: service_completed_successfully + networks: + - observability-stack-network + restart: "no" + logging: *logging + diff --git a/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py b/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py index f1980d90..de26c91f 100644 --- a/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py +++ b/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py @@ -1761,6 +1761,22 @@ def main(): print("šŸ“Š Created index patterns for spans, logs, and service map") + # Warm the field list immediately after creation. Patterns created via the + # raw saved-objects API land with an empty `fields` attribute (OSD only + # fetches fields lazily on first Discover/Explore visit, and the on-read + # auto-fetch safety net was removed upstream). Without this, pre-generated + # datasets show "Fields (0)" and charts error with "Could not locate that + # index-pattern-field" until a manual refresh. This mirrors what the + # dataset-creation wizard does (pre-fetch _fields_for_wildcard at create + # time). delayed_field_refresh() below remains the backstop for indices + # that are still empty at init time. + for pid, title in ( + (logs_pattern_id, "logs-otel-v1*"), + (traces_pattern_id, "otel-v1-apm-span*"), + (service_map_pattern_id, "otel-v2-apm-service-map*"), + ): + refresh_index_pattern_fields(workspace_id, pid, title) + # Set logs as the default index pattern if logs_pattern_id: set_default_index_pattern(workspace_id, logs_pattern_id) @@ -1875,28 +1891,55 @@ def refresh_index_pattern_fields(workspace_id, pattern_id, title): return False -def delayed_field_refresh(workspace_id, patterns): - """Wait for data to land in indices, then refresh field lists. +def delayed_field_refresh(workspace_id, patterns, max_attempts=12, interval_minutes=5): + """Periodically refresh field lists until every pattern is populated. - Called after the main init completes. Waits 10 minutes for the otel-demo - and agent examples to populate indices with representative documents so - the field refresh picks up all mapped fields. + main() warms the field lists immediately, but indices that are still empty + at init time (otel-demo and agent examples take a while to send their first + documents) return no fields on that first pass. Rather than a single blind + wait, poll on an interval and retry only the patterns that haven't been + populated yet, stopping early once they all succeed. With the defaults this + covers a one-hour window (12 attempts Ɨ 5 minutes). """ - delay_minutes = 10 - print(f"\nā³ Waiting {delay_minutes} minutes for indices to populate before refreshing fields...") - time.sleep(delay_minutes * 60) + pending = [(pid, title) for pid, title in patterns if pid] + if not pending: + print("ā­ļø No index patterns to refresh") + return - print("šŸ”„ Refreshing index pattern field lists...") - for pattern_id, title in patterns: - refresh_index_pattern_fields(workspace_id, pattern_id, title) - print("āœ… Field refresh complete") + print( + f"\nā³ Will refresh field lists as indices populate " + f"(up to {max_attempts} attempts every {interval_minutes} min)..." + ) + for attempt in range(1, max_attempts + 1): + print(f"šŸ”„ Field refresh attempt {attempt}/{max_attempts} ({len(pending)} pending)...") + still_pending = [] + for pattern_id, title in pending: + if not refresh_index_pattern_fields(workspace_id, pattern_id, title): + still_pending.append((pattern_id, title)) + pending = still_pending + if not pending: + print("āœ… Field refresh complete ā€” all patterns populated") + return + # Sleep between attempts (not before the first / after the last), so a + # stack whose indices are already populated exits promptly. + if attempt < max_attempts: + time.sleep(interval_minutes * 60) + + print( + f"āš ļø Field refresh finished with {len(pending)} pattern(s) still empty: " + f"{', '.join(title for _, title in pending)}" + ) -if __name__ == "__main__": - main() +def run_field_refresh_backstop(): + """Re-read workspace + pattern IDs and run the delayed field-refresh loop. - # Re-read workspace and pattern IDs for the delayed refresh. - # main() already printed success, so this is a background follow-up. + Split out from main() so it can run as a separate, long-running process + (its own compose service / Kubernetes Job). main() is the fast, blocking + setup that a Helm post-install/upgrade hook waits on; this backstop can run + for up to an hour and must never block install/upgrade completion. + """ + wait_for_dashboards() workspace_id = get_existing_workspace() logs_id = get_existing_index_pattern(workspace_id, "logs-otel-v1*") traces_id = get_existing_index_pattern(workspace_id, "otel-v1-apm-span*") @@ -1908,3 +1951,17 @@ def delayed_field_refresh(workspace_id, patterns): (svc_map_id, "otel-v2-apm-service-map*"), ] delayed_field_refresh(workspace_id, patterns) + + +if __name__ == "__main__": + import sys + + # Two modes, run as separate processes so the long-running field-refresh + # backstop never blocks init completion (or a Helm hook): + # (default) ā†’ main(): fast one-shot setup + immediate field warm + # refresh-loop ā†’ periodic field refresh until indices populate + mode = sys.argv[1] if len(sys.argv) > 1 else "init" + if mode == "refresh-loop": + run_field_refresh_backstop() + else: + main() From 29d6deb9fe62e924b880e4020d2549a2a8289439 Mon Sep 17 00:00:00 2001 From: Adam Tackett Date: Mon, 3 Aug 2026 16:46:44 -0700 Subject: [PATCH 2/3] fix flaky test2 Signed-off-by: Adam Tackett --- .../files/init-opensearch-dashboards.py | 45 +++++++++++-------- docker-compose.yml | 9 ++-- .../init/init-opensearch-dashboards.py | 45 +++++++++++-------- 3 files changed, 58 insertions(+), 41 deletions(-) diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index 96cf618d..08ead836 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -1964,7 +1964,7 @@ def refresh_index_pattern_fields(workspace_id, pattern_id, title): return False -def delayed_field_refresh(workspace_id, patterns, max_attempts=12, interval_minutes=5): +def delayed_field_refresh(workspace_id, titles, max_attempts=12, interval_minutes=5): """Periodically refresh field lists until every pattern is populated. main() warms the field lists immediately, but indices that are still empty @@ -1973,11 +1973,17 @@ def delayed_field_refresh(workspace_id, patterns, max_attempts=12, interval_minu wait, poll on an interval and retry only the patterns that haven't been populated yet, stopping early once they all succeed. With the defaults this covers a one-hour window (12 attempts Ɨ 5 minutes). + + Patterns are resolved by title on every attempt rather than once up front, + so this self-heals regardless of start order: if the backstop starts before + main() has created the index patterns, the early attempts simply find + nothing and retry until the patterns exist. This is what lets the compose + service / k8s Job run without a hard `depends_on` the init step ā€” a + dependency that otherwise breaks `docker compose up --wait` (a + `service_completed_successfully` target exiting mid-startup makes --wait + abort with a non-zero code). """ - pending = [(pid, title) for pid, title in patterns if pid] - if not pending: - print("ā­ļø No index patterns to refresh") - return + pending = list(titles) print( f"\nā³ Will refresh field lists as indices populate " @@ -1986,9 +1992,14 @@ def delayed_field_refresh(workspace_id, patterns, max_attempts=12, interval_minu for attempt in range(1, max_attempts + 1): print(f"šŸ”„ Field refresh attempt {attempt}/{max_attempts} ({len(pending)} pending)...") still_pending = [] - for pattern_id, title in pending: + for title in pending: + pattern_id = get_existing_index_pattern(workspace_id, title) + if not pattern_id: + # Pattern not created yet (backstop outran init) ā€” retry later. + still_pending.append(title) + continue if not refresh_index_pattern_fields(workspace_id, pattern_id, title): - still_pending.append((pattern_id, title)) + still_pending.append(title) pending = still_pending if not pending: print("āœ… Field refresh complete ā€” all patterns populated") @@ -2000,30 +2011,26 @@ def delayed_field_refresh(workspace_id, patterns, max_attempts=12, interval_minu print( f"āš ļø Field refresh finished with {len(pending)} pattern(s) still empty: " - f"{', '.join(title for _, title in pending)}" + f"{', '.join(pending)}" ) def run_field_refresh_backstop(): - """Re-read workspace + pattern IDs and run the delayed field-refresh loop. + """Run the delayed field-refresh loop as a standalone process. Split out from main() so it can run as a separate, long-running process (its own compose service / Kubernetes Job). main() is the fast, blocking setup that a Helm post-install/upgrade hook waits on; this backstop can run for up to an hour and must never block install/upgrade completion. + + Intentionally has no hard ordering dependency on the init step: it waits for + dashboards, then delayed_field_refresh() resolves pattern IDs by title on + each attempt, so it works whether it starts before or after init. """ wait_for_dashboards() workspace_id = get_existing_workspace() - logs_id = get_existing_index_pattern(workspace_id, "logs-otel-v1*") - traces_id = get_existing_index_pattern(workspace_id, "otel-v1-apm-span*") - svc_map_id = get_existing_index_pattern(workspace_id, "otel-v2-apm-service-map*") - - patterns = [ - (logs_id, "logs-otel-v1*"), - (traces_id, "otel-v1-apm-span*"), - (svc_map_id, "otel-v2-apm-service-map*"), - ] - delayed_field_refresh(workspace_id, patterns) + titles = ["logs-otel-v1*", "otel-v1-apm-span*", "otel-v2-apm-service-map*"] + delayed_field_refresh(workspace_id, titles) if __name__ == "__main__": diff --git a/docker-compose.yml b/docker-compose.yml index d5925076..1cbf1ca3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -328,9 +328,12 @@ services: - OPENSEARCH_DASHBOARDS_PROTOCOL=${OPENSEARCH_DASHBOARDS_PROTOCOL} volumes: - ./docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py:/init.py - depends_on: - opensearch-dashboards-init: - condition: service_completed_successfully + # No `depends_on` the init service on purpose. A + # `service_completed_successfully` dependency on a one-shot that exits + # mid-startup makes `docker compose up --wait` (test/e2e.sh) abort with a + # non-zero code. The refresh loop instead waits for dashboards itself and + # resolves index-pattern IDs by title on each attempt, so it self-heals + # whether it starts before or after init. networks: - observability-stack-network restart: "no" diff --git a/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py b/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py index de26c91f..2eb05007 100644 --- a/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py +++ b/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py @@ -1891,7 +1891,7 @@ def refresh_index_pattern_fields(workspace_id, pattern_id, title): return False -def delayed_field_refresh(workspace_id, patterns, max_attempts=12, interval_minutes=5): +def delayed_field_refresh(workspace_id, titles, max_attempts=12, interval_minutes=5): """Periodically refresh field lists until every pattern is populated. main() warms the field lists immediately, but indices that are still empty @@ -1900,11 +1900,17 @@ def delayed_field_refresh(workspace_id, patterns, max_attempts=12, interval_minu wait, poll on an interval and retry only the patterns that haven't been populated yet, stopping early once they all succeed. With the defaults this covers a one-hour window (12 attempts Ɨ 5 minutes). + + Patterns are resolved by title on every attempt rather than once up front, + so this self-heals regardless of start order: if the backstop starts before + main() has created the index patterns, the early attempts simply find + nothing and retry until the patterns exist. This is what lets the compose + service / k8s Job run without a hard `depends_on` the init step ā€” a + dependency that otherwise breaks `docker compose up --wait` (a + `service_completed_successfully` target exiting mid-startup makes --wait + abort with a non-zero code). """ - pending = [(pid, title) for pid, title in patterns if pid] - if not pending: - print("ā­ļø No index patterns to refresh") - return + pending = list(titles) print( f"\nā³ Will refresh field lists as indices populate " @@ -1913,9 +1919,14 @@ def delayed_field_refresh(workspace_id, patterns, max_attempts=12, interval_minu for attempt in range(1, max_attempts + 1): print(f"šŸ”„ Field refresh attempt {attempt}/{max_attempts} ({len(pending)} pending)...") still_pending = [] - for pattern_id, title in pending: + for title in pending: + pattern_id = get_existing_index_pattern(workspace_id, title) + if not pattern_id: + # Pattern not created yet (backstop outran init) ā€” retry later. + still_pending.append(title) + continue if not refresh_index_pattern_fields(workspace_id, pattern_id, title): - still_pending.append((pattern_id, title)) + still_pending.append(title) pending = still_pending if not pending: print("āœ… Field refresh complete ā€” all patterns populated") @@ -1927,30 +1938,26 @@ def delayed_field_refresh(workspace_id, patterns, max_attempts=12, interval_minu print( f"āš ļø Field refresh finished with {len(pending)} pattern(s) still empty: " - f"{', '.join(title for _, title in pending)}" + f"{', '.join(pending)}" ) def run_field_refresh_backstop(): - """Re-read workspace + pattern IDs and run the delayed field-refresh loop. + """Run the delayed field-refresh loop as a standalone process. Split out from main() so it can run as a separate, long-running process (its own compose service / Kubernetes Job). main() is the fast, blocking setup that a Helm post-install/upgrade hook waits on; this backstop can run for up to an hour and must never block install/upgrade completion. + + Intentionally has no hard ordering dependency on the init step: it waits for + dashboards, then delayed_field_refresh() resolves pattern IDs by title on + each attempt, so it works whether it starts before or after init. """ wait_for_dashboards() workspace_id = get_existing_workspace() - logs_id = get_existing_index_pattern(workspace_id, "logs-otel-v1*") - traces_id = get_existing_index_pattern(workspace_id, "otel-v1-apm-span*") - svc_map_id = get_existing_index_pattern(workspace_id, "otel-v2-apm-service-map*") - - patterns = [ - (logs_id, "logs-otel-v1*"), - (traces_id, "otel-v1-apm-span*"), - (svc_map_id, "otel-v2-apm-service-map*"), - ] - delayed_field_refresh(workspace_id, patterns) + titles = ["logs-otel-v1*", "otel-v1-apm-span*", "otel-v2-apm-service-map*"] + delayed_field_refresh(workspace_id, titles) if __name__ == "__main__": From 81b08f0889ac5cfd46ddcdc6dee194d6e1dcc9b6 Mon Sep 17 00:00:00 2001 From: Adam Tackett Date: Tue, 4 Aug 2026 10:18:18 -0700 Subject: [PATCH 3/3] fix 3 Signed-off-by: Adam Tackett --- .../files/init-opensearch-dashboards.py | 9 +++++++-- .../init/init-opensearch-dashboards.py | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/charts/observability-stack/files/init-opensearch-dashboards.py b/charts/observability-stack/files/init-opensearch-dashboards.py index 08ead836..a620ac70 100644 --- a/charts/observability-stack/files/init-opensearch-dashboards.py +++ b/charts/observability-stack/files/init-opensearch-dashboards.py @@ -1920,14 +1920,19 @@ def refresh_index_pattern_fields(workspace_id, pattern_id, title): return False if workspace_id and workspace_id != "default": - url = f"{BASE_URL}/w/{workspace_id}/api/index_patterns/_fields_for_wildcard?pattern={title}&meta_fields=_source&meta_fields=_id&meta_fields=_type&meta_fields=_index&meta_fields=_score" + url = f"{BASE_URL}/w/{workspace_id}/api/index_patterns/_fields_for_wildcard" else: - url = f"{BASE_URL}/api/index_patterns/_fields_for_wildcard?pattern={title}&meta_fields=_source&meta_fields=_id&meta_fields=_type&meta_fields=_index&meta_fields=_score" + url = f"{BASE_URL}/api/index_patterns/_fields_for_wildcard" + # Only `pattern` is passed. Do NOT send `meta_fields` ā€” this OSD version's + # route validation rejects the repeated meta_fields query params with a 400 + # ("[request query.params]: definition for this key is missing"). requests + # URL-encodes the pattern's wildcard/special chars via the params dict. try: resp = requests.get( url, auth=(USERNAME, PASSWORD), headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + params={"pattern": title}, verify=False, timeout=30, ) if resp.status_code != 200: diff --git a/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py b/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py index 2eb05007..a8ef2298 100644 --- a/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py +++ b/docker-compose/opensearch-dashboards/init/init-opensearch-dashboards.py @@ -1847,14 +1847,19 @@ def refresh_index_pattern_fields(workspace_id, pattern_id, title): return False if workspace_id and workspace_id != "default": - url = f"{BASE_URL}/w/{workspace_id}/api/index_patterns/_fields_for_wildcard?pattern={title}&meta_fields=_source&meta_fields=_id&meta_fields=_type&meta_fields=_index&meta_fields=_score" + url = f"{BASE_URL}/w/{workspace_id}/api/index_patterns/_fields_for_wildcard" else: - url = f"{BASE_URL}/api/index_patterns/_fields_for_wildcard?pattern={title}&meta_fields=_source&meta_fields=_id&meta_fields=_type&meta_fields=_index&meta_fields=_score" + url = f"{BASE_URL}/api/index_patterns/_fields_for_wildcard" + # Only `pattern` is passed. Do NOT send `meta_fields` ā€” this OSD version's + # route validation rejects the repeated meta_fields query params with a 400 + # ("[request query.params]: definition for this key is missing"). requests + # URL-encodes the pattern's wildcard/special chars via the params dict. try: resp = requests.get( url, auth=(USERNAME, PASSWORD), headers={"Content-Type": "application/json", "osd-xsrf": "true"}, + params={"pattern": title}, verify=False, timeout=30, ) if resp.status_code != 200: