diff --git a/docs/guides/add-app.md b/docs/guides/add-app.md index 3e6f79d..f223d1a 100644 --- a/docs/guides/add-app.md +++ b/docs/guides/add-app.md @@ -243,6 +243,19 @@ CI の build job では `scripts/build_tool_wrappers/` を `PATH` の先頭に この actual build snapshot は、将来の build cache key や、同じ source から異なる binary が生じた場合の原因確認に使う前提の記録です。 +### run placement / node status snapshot の方針 + +CI の共通 job は、benchmark 本体の `run_start` より前に `results/node_status_snapshot_run.json` を記録します。 +app の `run.sh` からこの snapshot 用 helper を呼ぶ必要はありません。 + +この snapshot は scheduler-neutral な診断情報で、scheduler kind、scheduler が示す node list、観測できた host 数、CPU 数、memory、load average、GPU の軽量状態、run 前に見える GPU compute process の件数と memory 合計を記録します。 +process ID、process name、user name は記録しません。 +SLURM では可能なら allocation 内の各 node で軽い worker を動かします。 +PBS / PJM / unknown scheduler では、取れる範囲の nodefile / environment / local host 情報だけを partial または unsupported として残します。 + +Result JSON には hash、summary、Measurement Artifact への参照だけを入れ、詳細な node list や GPU 状態は `results/node_status_snapshot_run.json` 側に残します。 +Portal の public surface ではこの Measurement Artifact は表示しません。 + ### build cache の方針 cross build job と native `build_run` job の build phase では、共通 wrapper `scripts/build_with_cache.sh` が `build.sh` の前後で build artifact cache を扱います。 diff --git a/result_server/routes/results_detail_routes.py b/result_server/routes/results_detail_routes.py index 84dd90b..552f11f 100644 --- a/result_server/routes/results_detail_routes.py +++ b/result_server/routes/results_detail_routes.py @@ -383,6 +383,17 @@ def _list_result_measurement_artifact_filenames(result, artifact_dir, *, include seen.add(filename) if os.path.isfile(os.path.join(artifact_dir, filename)): filenames.append(filename) + for artifact_path in _iter_result_node_status_artifact_paths(result): + filename = stored_measurement_artifact_filename_from_path( + timestamp, + result_uuid, + artifact_path, + ) + if not filename or filename in seen: + continue + seen.add(filename) + if os.path.isfile(os.path.join(artifact_dir, filename)): + filenames.append(filename) return filenames @@ -417,5 +428,17 @@ def _iter_result_timing_artifact_paths(result): yield path +def _iter_result_node_status_artifact_paths(result): + node_status_snapshot = result.get("node_status_snapshot") + if not isinstance(node_status_snapshot, dict): + return + artifact = node_status_snapshot.get("artifact") + if not isinstance(artifact, dict) or artifact.get("type") != "file_reference": + return + path = _clean_result_value(artifact.get("path")) + if path: + yield path + + def _clean_result_value(value): return str(value or "").strip() diff --git a/result_server/templates/result_detail.html b/result_server/templates/result_detail.html index 396666a..a554a26 100644 --- a/result_server/templates/result_detail.html +++ b/result_server/templates/result_detail.html @@ -73,6 +73,10 @@ {{ render_titled_key_value_table("PA Data Summary", profile_rows, "meta-table") }} {% endif %} +{% if node_status_rows %} +{{ render_titled_key_value_table("Node Status Snapshot", node_status_rows, "meta-table") }} +{% endif %} + {% if measurement_artifact_rows %}

Measurement Artifacts

diff --git a/result_server/tests/test_measurement_artifacts.py b/result_server/tests/test_measurement_artifacts.py index f7a26a8..6f7a1a7 100644 --- a/result_server/tests/test_measurement_artifacts.py +++ b/result_server/tests/test_measurement_artifacts.py @@ -62,6 +62,11 @@ def test_builds_canonical_stored_names_for_legacy_profiles_and_timing_json(): UUID, "results/qws_timing_CASE0.json", ) == f"measurement_artifact_{TIMESTAMP}_{UUID}_qws_timing_CASE0.json" + assert stored_measurement_artifact_filename_from_path( + TIMESTAMP, + UUID, + "results/node_status_snapshot_run.json", + ) == f"measurement_artifact_{TIMESTAMP}_{UUID}_node_status_snapshot_run.json" def test_profile_archive_candidates_keep_legacy_and_generic_names(): @@ -81,6 +86,7 @@ def test_profile_archive_candidates_keep_legacy_and_generic_names(): f"padata_{TIMESTAMP}_{UUID}.tgz", f"padata_{TIMESTAMP}_{UUID}_padata_pairlist.tgz", f"measurement_artifact_{TIMESTAMP}_{UUID}_qws_timing_CASE0.json", + f"measurement_artifact_{TIMESTAMP}_{UUID}_node_status_snapshot_run.json", ], ) def test_recognizes_served_measurement_artifact_filenames(filename): diff --git a/result_server/tests/test_result_detail_template.py b/result_server/tests/test_result_detail_template.py index e34ceaa..57baf9a 100644 --- a/result_server/tests/test_result_detail_template.py +++ b/result_server/tests/test_result_detail_template.py @@ -141,6 +141,31 @@ def app(): } ], }, + "node_status_snapshot": { + "schema_version": 1, + "kind": "node_status_snapshot", + "collection_status": "ok", + "collection_warnings": [], + "scheduler_kind": "slurm", + "summary": { + "scheduler_host_count": 2, + "observed_host_count": 2, + "cpu_logical_counts": [64], + "memory_total_mib_min": 262144, + "memory_total_mib_max": 262144, + "memory_available_mib_min": 131072, + "load_average_1m_max": 0.5, + "load_average_5m_max": 0.8, + "observed_gpu_count": 4, + "gpu_memory_used_total_mib": 0, + "gpu_compute_process_count": 0, + "gpu_compute_memory_used_mib": 0, + }, + "artifact": { + "type": "file_reference", + "path": "results/node_status_snapshot_run.json", + }, + }, } FULL_QUALITY = { @@ -204,6 +229,20 @@ def test_meta_info_section(self, app): assert "build inputs hash matched" in html assert "rccs-cloud" in html assert "slurm" in html + assert "Node Status Snapshot" in html + assert "Hosts Observed" in html + assert "2/2" in html + assert "CPU Counts Observed" in html + assert "64" in html + assert "Host Memory Total" in html + assert "262144.000 MiB" in html + assert "Min Memory Available Before Run" in html + assert "131072.000 MiB" in html + assert "Max Load Average Before Run" in html + assert "1m=0.500; 5m=0.800" in html + assert "Compute Processes Before Run" in html + assert "0 process(es); 0.000 MiB" in html + assert "results/node_status_snapshot_run.json" in html assert "Timing Observations" in html assert "qws-case0-timers" in html assert "producer=qws" in html @@ -233,6 +272,8 @@ def test_public_surface_meta_omits_operator_fields(self, app): assert "Cached Binary Created At" not in html assert "Allocation Project ID" not in html assert "Runner" not in html + assert "Node Status Snapshot" not in html + assert "node_status_snapshot_run" not in html assert "Timing Observations" not in html assert "qws_timing_CASE0" not in html assert "rccs-cloud" not in html @@ -398,6 +439,48 @@ def test_timing_observation_artifact_is_hidden_on_public_surface(self, app): assert "Timing observation" not in html assert filename not in html + def test_node_status_snapshot_artifact_is_linked_on_console_surface(self, app): + result = { + **FULL_RESULT, + "_server_uuid": "12345678-1234-1234-1234-123456789abc", + "_server_timestamp": "20260819_161329", + } + filename = ( + "measurement_artifact_20260819_161329_" + "12345678-1234-1234-1234-123456789abc_node_status_snapshot_run.json" + ) + + with app.test_request_context(): + html = _render_result_detail(result, FULL_QUALITY, [filename]) + + assert "Measurement Artifacts" in html + assert "Node status snapshot" in html + assert "Run placement" in html + assert "results/node_status_snapshot_run.json" in html + assert f'href="/results/{filename}"' in html + + def test_node_status_snapshot_artifact_is_hidden_on_public_surface(self, app): + result = { + **FULL_RESULT, + "_server_uuid": "12345678-1234-1234-1234-123456789abc", + "_server_timestamp": "20260819_161329", + } + filename = ( + "measurement_artifact_20260819_161329_" + "12345678-1234-1234-1234-123456789abc_node_status_snapshot_run.json" + ) + + with app.test_request_context(): + html = _render_result_detail( + result, + FULL_QUALITY, + [filename], + public_surface=True, + ) + + assert "Node status snapshot" not in html + assert filename not in html + def test_vector_data_table(self, app): with app.test_request_context(): html = _render_result_detail(FULL_RESULT, FULL_QUALITY) diff --git a/result_server/utils/result_detail_view.py b/result_server/utils/result_detail_view.py index 90ae6c3..8142204 100644 --- a/result_server/utils/result_detail_view.py +++ b/result_server/utils/result_detail_view.py @@ -115,6 +115,9 @@ def build_result_detail_context( measurement_artifact_filenames or [], include_timing=not public_surface, ), + "node_status_rows": ( + [] if public_surface else _build_node_status_rows(result.get("node_status_snapshot")) + ), "timing_observation_rows": ( [] if public_surface else _build_timing_observation_rows(result.get("timing_observations")) ), @@ -230,6 +233,11 @@ def _build_measurement_artifact_rows( rows.extend( _build_timing_measurement_artifact_rows(result, timestamp, result_uuid, uploaded) ) + rows.extend( + _build_node_status_measurement_artifact_rows( + result, timestamp, result_uuid, uploaded + ) + ) return rows @@ -310,6 +318,186 @@ def _build_timing_measurement_artifact_rows(result, timestamp, result_uuid, uplo return rows +def _build_node_status_measurement_artifact_rows(result, timestamp, result_uuid, uploaded): + node_status_snapshot = result.get("node_status_snapshot") + if not isinstance(node_status_snapshot, dict): + return [] + + artifact = node_status_snapshot.get("artifact") + artifact = artifact if isinstance(artifact, dict) else {} + if artifact.get("type") != "file_reference": + return [] + + artifact_path = str(artifact.get("path") or "").strip() + filename = stored_measurement_artifact_filename_from_path( + timestamp, + result_uuid, + artifact_path, + ) + if not filename: + return [] + + return [ + { + "kind": "Node status snapshot", + "source": "Run placement", + "artifact_path": artifact_path, + "filename": filename, + "link": ( + url_for("results.show_result", filename=filename) + if filename in uploaded + else None + ), + } + ] + + +def _build_node_status_rows(node_status_snapshot): + if not isinstance(node_status_snapshot, dict): + return [] + + summary = node_status_snapshot.get("summary") + summary = summary if isinstance(summary, dict) else {} + artifact = node_status_snapshot.get("artifact") + artifact = artifact if isinstance(artifact, dict) else {} + warnings = _unique_strings( + list(_list_values(node_status_snapshot.get("collection_warnings"))) + + list(_list_values(summary.get("warnings"))) + ) + + rows = build_labeled_value_rows( + [ + ("Status", node_status_snapshot.get("collection_status") or "unknown"), + ("Scheduler", node_status_snapshot.get("scheduler_kind") or "unknown"), + ( + "Hosts Observed", + _format_observed_total( + summary.get("observed_host_count"), + summary.get("scheduler_host_count"), + ), + ), + ( + "CPU Counts Observed", + _format_value_list(summary.get("cpu_logical_counts")), + ), + ( + "Host Memory Total", + _format_mib_range( + summary.get("memory_total_mib_min"), + summary.get("memory_total_mib_max"), + ), + ), + ( + "Min Memory Available Before Run", + _format_mib(summary.get("memory_available_mib_min")), + ), + ( + "Max Load Average Before Run", + _format_load_summary( + summary.get("load_average_1m_max"), + summary.get("load_average_5m_max"), + ), + ), + ("GPUs Observed", summary.get("observed_gpu_count")), + ( + "GPU Memory Used Before Run", + _format_mib(summary.get("gpu_memory_used_total_mib")), + ), + ( + "Compute Processes Before Run", + _format_process_summary( + summary.get("gpu_compute_process_count"), + summary.get("gpu_compute_memory_used_mib"), + ), + ), + ("Snapshot Hash", node_status_snapshot.get("hash")), + ] + ) + if warnings: + rows.append({"label": "Warnings", "value": ", ".join(str(item) for item in warnings)}) + if artifact.get("path"): + rows.append({"label": "Artifact", "value": artifact["path"]}) + return rows + + +def _list_values(value): + if isinstance(value, list): + return value + if value in (None, ""): + return [] + return [value] + + +def _unique_strings(values): + unique_values = [] + seen = set() + for value in values: + text = str(value) + if text in seen: + continue + seen.add(text) + unique_values.append(text) + return unique_values + + +def _format_observed_total(observed, total): + if observed in (None, "") and total in (None, ""): + return "not recorded" + if total in (None, ""): + return str(observed) + return f"{observed or 0}/{total}" + + +def _format_mib(value): + if value in (None, ""): + return "not recorded" + return f"{format_numeric_value(value)} MiB" + + +def _format_mib_range(min_value, max_value): + if min_value in (None, "") and max_value in (None, ""): + return "not recorded" + if min_value == max_value or max_value in (None, ""): + return _format_mib(min_value) + if min_value in (None, ""): + return _format_mib(max_value) + return f"{format_numeric_value(min_value)}-{format_numeric_value(max_value)} MiB" + + +def _format_value_list(values): + if not isinstance(values, list) or not values: + return "not recorded" + return ", ".join(_format_count_value(value) for value in values) + + +def _format_load_summary(one_minute, five_minutes): + if one_minute in (None, "") and five_minutes in (None, ""): + return "not recorded" + values = [] + if one_minute not in (None, ""): + values.append(f"1m={format_numeric_value(one_minute)}") + if five_minutes not in (None, ""): + values.append(f"5m={format_numeric_value(five_minutes)}") + return "; ".join(values) + + +def _format_count_value(value): + try: + numeric_value = float(value) + except (TypeError, ValueError): + return str(value) + if numeric_value.is_integer(): + return str(int(numeric_value)) + return format_numeric_value(value) + + +def _format_process_summary(count, memory_mib): + if count in (None, ""): + return "not recorded" + memory_text = _format_mib(memory_mib) + return f"{count} process(es); {memory_text}" + + def _choose_uploaded_filename(candidates, uploaded): for filename in candidates: if filename in uploaded: diff --git a/scripts/collect_node_status_snapshot.sh b/scripts/collect_node_status_snapshot.sh new file mode 100644 index 0000000..12a2dfb --- /dev/null +++ b/scripts/collect_node_status_snapshot.sh @@ -0,0 +1,422 @@ +#!/bin/bash +set -euo pipefail + +out_file="${1:-results/node_status_snapshot_run.json}" +snapshot_stage="${BK_NODE_STATUS_SNAPSHOT_STAGE:-run}" +script_path="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" + +json_string_array() { + jq -R -s -c 'split("\n") | map(select(length > 0))' +} + +json_lines_to_array() { + jq -s -c 'unique_by(.hostname // "")' +} + +detect_scheduler_kind() { + if [ -n "${SLURM_JOB_ID:-}" ] || [ -n "${SLURM_JOBID:-}" ]; then + printf '%s\n' "slurm" + elif [ -n "${PJM_JOBID:-}" ] || [ -n "${PJM_JOB_ID:-}" ]; then + printf '%s\n' "pjm" + elif [ -n "${PBS_JOBID:-}" ]; then + printf '%s\n' "pbs" + else + printf '%s\n' "unknown" + fi +} + +collect_scheduler_hosts() { + local kind="$1" + local nodelist="" + local nodefile="" + + case "$kind" in + slurm) + nodelist="${SLURM_JOB_NODELIST:-${SLURM_NODELIST:-}}" + if [ -n "$nodelist" ] && command -v scontrol >/dev/null 2>&1; then + scontrol show hostnames "$nodelist" 2>/dev/null || true + elif [ -n "$nodelist" ]; then + printf '%s\n' "$nodelist" + fi + ;; + pbs) + nodefile="${PBS_NODEFILE:-}" + if [ -n "$nodefile" ] && [ -r "$nodefile" ]; then + awk 'NF {print $1}' "$nodefile" | sort -u + fi + ;; + pjm) + nodefile="${PJM_NODEINF:-${PJM_O_NODEINF:-}}" + if [ -n "$nodefile" ] && [ -r "$nodefile" ]; then + awk 'NF {print $1}' "$nodefile" | sort -u + elif [ -n "${PJM_NODE:-}" ]; then + printf '%s\n' "${PJM_NODE}" | tr ',' '\n' + fi + ;; + esac +} + +trimmed_number_or_zero() { + awk ' + { + value = $0 + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + if (value ~ /^[0-9]+$/) { + total += value + } + } + END { print total + 0 } + ' +} + +count_compute_app_rows() { + awk -F, ' + { + pid = $1 + gsub(/^[[:space:]]+|[[:space:]]+$/, "", pid) + if (pid ~ /^[0-9]+$/) { + count += 1 + } + } + END {print count + 0} + ' +} + +collect_gpu_state_json() { + local gpu_csv="" + local gpus_json="[]" + local gpu_query_status="unavailable" + local compute_apps_csv="" + local compute_process_count=0 + local compute_memory_used_mib=0 + + if command -v nvidia-smi >/dev/null 2>&1; then + if gpu_csv=$(nvidia-smi \ + --query-gpu=index,name,memory.used,memory.total,utilization.gpu,temperature.gpu,clocks.sm,clocks.mem,pstate \ + --format=csv,noheader,nounits 2>/dev/null); then + gpu_query_status="ok" + gpus_json=$(printf '%s\n' "$gpu_csv" | jq -R -s -c ' + def trim: sub("^[[:space:]]+"; "") | sub("[[:space:]]+$"; ""); + def maybe_number: tonumber? // null; + split("\n") + | map(select(length > 0)) + | map( + split(",") + | map(trim) + | { + index: (.[0] // ""), + name: (.[1] // ""), + memory_used_mib: ((.[2] // "") | maybe_number), + memory_total_mib: ((.[3] // "") | maybe_number), + utilization_gpu_percent: ((.[4] // "") | maybe_number), + temperature_c: ((.[5] // "") | maybe_number), + clocks_sm_mhz: ((.[6] // "") | maybe_number), + clocks_mem_mhz: ((.[7] // "") | maybe_number), + pstate: (.[8] // "") + } + ) + ') + else + gpu_query_status="failed" + fi + + compute_apps_csv=$(nvidia-smi \ + --query-compute-apps=pid,used_memory \ + --format=csv,noheader,nounits 2>/dev/null || true) + compute_process_count=$(printf '%s\n' "$compute_apps_csv" | count_compute_app_rows) + compute_memory_used_mib=$( + printf '%s\n' "$compute_apps_csv" \ + | awk -F, '{print $2}' \ + | trimmed_number_or_zero + ) + fi + + jq -n -c \ + --arg query_status "$gpu_query_status" \ + --argjson gpus "$gpus_json" \ + --argjson compute_process_count "$compute_process_count" \ + --argjson compute_memory_used_mib "$compute_memory_used_mib" \ + '{ + available: ($query_status == "ok"), + query_status: $query_status, + gpus: $gpus, + process_summary: { + compute_process_count: $compute_process_count, + memory_used_mib: $compute_memory_used_mib + } + }' +} + +collect_host_state_json() { + local cpu_logical_count="" + local memory_total_kib="" + local memory_available_kib="" + local load_average_1m="" + local load_average_5m="" + local load_average_15m="" + + cpu_logical_count=$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || true) + if [ -r /proc/meminfo ]; then + memory_total_kib=$(awk '/^MemTotal:/ {print $2; exit}' /proc/meminfo) + memory_available_kib=$(awk '/^MemAvailable:/ {print $2; exit}' /proc/meminfo) + fi + if [ -r /proc/loadavg ]; then + read -r load_average_1m load_average_5m load_average_15m _ < /proc/loadavg || true + fi + + jq -n -c \ + --arg cpu_logical_count "$cpu_logical_count" \ + --arg memory_total_kib "$memory_total_kib" \ + --arg memory_available_kib "$memory_available_kib" \ + --arg load_average_1m "$load_average_1m" \ + --arg load_average_5m "$load_average_5m" \ + --arg load_average_15m "$load_average_15m" \ + '{ + cpu_logical_count: ($cpu_logical_count | tonumber?), + memory_total_mib: (($memory_total_kib | tonumber?) as $v | if $v == null then null else $v / 1024 end), + memory_available_mib: (($memory_available_kib | tonumber?) as $v | if $v == null then null else $v / 1024 end), + load_average: { + one_minute: ($load_average_1m | tonumber?), + five_minutes: ($load_average_5m | tonumber?), + fifteen_minutes: ($load_average_15m | tonumber?) + } + }' +} + +collect_worker_json() { + local hostname_value="" + local host_state_json="{}" + local gpu_state_json="{}" + + hostname_value=$(hostname 2>/dev/null || printf '%s' "") + host_state_json=$(collect_host_state_json) + gpu_state_json=$(collect_gpu_state_json) + + jq -n -c \ + --arg hostname "$hostname_value" \ + --arg scheduler_kind "$(detect_scheduler_kind)" \ + --arg slurm_procid "${SLURM_PROCID:-}" \ + --arg slurm_localid "${SLURM_LOCALID:-}" \ + --arg ompi_rank "${OMPI_COMM_WORLD_RANK:-}" \ + --arg ompi_local_rank "${OMPI_COMM_WORLD_LOCAL_RANK:-}" \ + --arg cuda_visible_devices "${CUDA_VISIBLE_DEVICES:-}" \ + --argjson host_state "$host_state_json" \ + --argjson gpu_state "$gpu_state_json" \ + '{ + hostname: $hostname, + scheduler_kind: $scheduler_kind, + host_state: $host_state, + rank_environment: { + slurm_procid: $slurm_procid, + slurm_localid: $slurm_localid, + ompi_rank: $ompi_rank, + ompi_local_rank: $ompi_local_rank, + cuda_visible_devices: $cuda_visible_devices + }, + gpu_state: $gpu_state + }' +} + +remote_snapshot_enabled() { + case "${BK_NODE_STATUS_SNAPSHOT_REMOTE:-auto}" in + 0|false|False|FALSE|no|No|NO|off|Off|OFF) + return 1 + ;; + *) + return 0 + ;; + esac +} + +collect_slurm_remote_workers() { + local scheduler_host_count="$1" + local timeout_seconds="${BK_NODE_STATUS_SNAPSHOT_TIMEOUT_SECONDS:-20}" + local stdout_file="" + local stderr_file="" + local status=0 + + if ! remote_snapshot_enabled; then + return 1 + fi + if [ "$scheduler_host_count" -le 1 ]; then + return 1 + fi + if ! command -v srun >/dev/null 2>&1; then + return 1 + fi + + stdout_file=$(mktemp) + stderr_file=$(mktemp) + if command -v timeout >/dev/null 2>&1; then + timeout "${timeout_seconds}s" srun -N "$scheduler_host_count" -n "$scheduler_host_count" \ + --ntasks-per-node=1 bash "$script_path" --worker >"$stdout_file" 2>"$stderr_file" || status=$? + else + srun -N "$scheduler_host_count" -n "$scheduler_host_count" \ + --ntasks-per-node=1 bash "$script_path" --worker >"$stdout_file" 2>"$stderr_file" || status=$? + fi + + rm -f "$stderr_file" + if [ "$status" -ne 0 ]; then + rm -f "$stdout_file" + return 1 + fi + + awk '/^[[:space:]]*\{/ {print}' "$stdout_file" + rm -f "$stdout_file" +} + +build_summary_json() { + local scheduler_hosts_json="$1" + local hosts_json="$2" + + jq -n -c \ + --argjson scheduler_hosts "$scheduler_hosts_json" \ + --argjson hosts "$hosts_json" \ + ' + ($hosts | map(.gpu_state.gpus // []) | add // []) as $gpus + | ($hosts | map(.gpu_state.process_summary.compute_process_count // 0) | add // 0) as $process_count + | ($hosts | map(.gpu_state.process_summary.memory_used_mib // 0) | add // 0) as $process_memory + | ($gpus | map(.memory_used_mib // 0) | add // 0) as $gpu_memory + | ($hosts | map(.host_state.cpu_logical_count) | map(select(. != null)) | unique) as $cpu_counts + | ($hosts | map(.host_state.memory_total_mib) | map(select(. != null)) | unique) as $memory_totals + | ($hosts | map(.host_state.memory_available_mib) | map(select(. != null))) as $memory_available + | ($hosts | map(.host_state.load_average.one_minute) | map(select(. != null))) as $load_1m + | ($hosts | map(.host_state.load_average.five_minutes) | map(select(. != null))) as $load_5m + | { + scheduler_host_count: ($scheduler_hosts | length), + observed_host_count: ($hosts | length), + cpu_logical_counts: $cpu_counts, + memory_total_mib_min: ($memory_totals | min), + memory_total_mib_max: ($memory_totals | max), + memory_available_mib_min: ($memory_available | min), + load_average_1m_max: ($load_1m | max), + load_average_5m_max: ($load_5m | max), + observed_gpu_count: ($gpus | length), + gpu_memory_used_total_mib: $gpu_memory, + gpu_compute_process_count: $process_count, + gpu_compute_memory_used_mib: $process_memory, + gpu_query_statuses: ($hosts | map(.gpu_state.query_status // "unknown") | unique), + warnings: ( + [] + + (if (($scheduler_hosts | length) > 0 and ($hosts | length) != ($scheduler_hosts | length)) + then ["observed_host_count_differs_from_scheduler_host_count"] else [] end) + + (if ($cpu_counts | length) > 1 + then ["cpu_count_differs_across_observed_hosts"] else [] end) + + (if ($memory_totals | length) > 1 + then ["memory_total_differs_across_observed_hosts"] else [] end) + + (if $process_count > 0 + then ["gpu_compute_processes_present_before_run"] else [] end) + ) + }' +} + +if [ "${1:-}" = "--worker" ]; then + collect_worker_json + exit 0 +fi + +mkdir -p "$(dirname "$out_file")" + +scheduler_kind=$(detect_scheduler_kind) +scheduler_hosts_json=$(collect_scheduler_hosts "$scheduler_kind" | json_string_array) +scheduler_host_count=$(printf '%s' "$scheduler_hosts_json" | jq 'length') +collection_status="ok" +remote_collection_status="not_attempted" +collection_warnings_json="[]" + +worker_lines="" +if [ "$scheduler_kind" = "slurm" ]; then + if worker_lines=$(collect_slurm_remote_workers "$scheduler_host_count"); then + remote_collection_status="ok" + elif [ "$scheduler_host_count" -gt 1 ] && remote_snapshot_enabled; then + remote_collection_status="failed" + collection_status="partial" + collection_warnings_json='["slurm_remote_collection_failed"]' + fi +fi + +if [ -z "$worker_lines" ]; then + worker_lines=$(collect_worker_json) +fi + +if ! hosts_json=$(printf '%s\n' "$worker_lines" | json_lines_to_array 2>/dev/null); then + hosts_json=$(collect_worker_json | json_lines_to_array) + collection_status="partial" + collection_warnings_json='["worker_output_parse_failed"]' +fi + +if [ "$scheduler_kind" = "unknown" ]; then + collection_status="unsupported" +fi + +summary_json=$(build_summary_json "$scheduler_hosts_json" "$hosts_json") + +jq -n \ + --argjson schema_version 1 \ + --arg kind "node_status_snapshot" \ + --arg stage "$snapshot_stage" \ + --arg collected_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg collection_status "$collection_status" \ + --arg remote_collection_status "$remote_collection_status" \ + --arg scheduler_kind "$scheduler_kind" \ + --arg slurm_job_id "${SLURM_JOB_ID:-${SLURM_JOBID:-}}" \ + --arg slurm_partition "${SLURM_JOB_PARTITION:-}" \ + --arg slurm_job_nodelist "${SLURM_JOB_NODELIST:-${SLURM_NODELIST:-}}" \ + --arg slurm_job_num_nodes "${SLURM_JOB_NUM_NODES:-${SLURM_NNODES:-}}" \ + --arg slurm_ntasks "${SLURM_NTASKS:-}" \ + --arg slurm_tasks_per_node "${SLURM_TASKS_PER_NODE:-}" \ + --arg slurm_cpus_per_task "${SLURM_CPUS_PER_TASK:-}" \ + --arg slurm_job_gpus "${SLURM_JOB_GPUS:-}" \ + --arg slurm_gpus_on_node "${SLURM_GPUS_ON_NODE:-}" \ + --arg slurm_gpus_per_node "${SLURM_GPUS_PER_NODE:-}" \ + --arg slurm_gpus_per_task "${SLURM_GPUS_PER_TASK:-}" \ + --arg pbs_jobid "${PBS_JOBID:-}" \ + --arg pbs_nodefile "${PBS_NODEFILE:-}" \ + --arg pjm_jobid "${PJM_JOBID:-${PJM_JOB_ID:-}}" \ + --arg pjm_nodeinf "${PJM_NODEINF:-${PJM_O_NODEINF:-}}" \ + --argjson scheduler_hosts "$scheduler_hosts_json" \ + --argjson observed_hosts "$hosts_json" \ + --argjson summary "$summary_json" \ + --argjson collection_warnings "$collection_warnings_json" \ + '{ + schema_version: $schema_version, + kind: $kind, + stage: $stage, + collected_at: $collected_at, + collection_status: $collection_status, + collection_warnings: $collection_warnings, + scheduler: { + kind: $scheduler_kind, + slurm: { + job_id: $slurm_job_id, + partition: $slurm_partition, + job_nodelist: $slurm_job_nodelist, + job_num_nodes: $slurm_job_num_nodes, + ntasks: $slurm_ntasks, + tasks_per_node: $slurm_tasks_per_node, + cpus_per_task: $slurm_cpus_per_task, + job_gpus: $slurm_job_gpus, + gpus_on_node: $slurm_gpus_on_node, + gpus_per_node: $slurm_gpus_per_node, + gpus_per_task: $slurm_gpus_per_task, + remote_collection_status: $remote_collection_status + }, + pbs: { + job_id: $pbs_jobid, + nodefile: $pbs_nodefile + }, + pjm: { + job_id: $pjm_jobid, + nodeinf: $pjm_nodeinf + } + }, + allocation: { + scheduler_hosts: $scheduler_hosts + }, + observed: { + hosts: $observed_hosts + }, + summary: $summary + }' > "$out_file" + +echo "Wrote node status snapshot: $out_file" diff --git a/scripts/matrix_generate.sh b/scripts/matrix_generate.sh index 829475c..3459cdb 100644 --- a/scripts/matrix_generate.sh +++ b/scripts/matrix_generate.sh @@ -161,6 +161,7 @@ ${job_prefix}_run: - bash scripts/record_ci_timing_context.sh run - ls -la $program_path/ - BK_SYSTEM=\"$system\" BK_SNAPSHOT_STAGE=run bash scripts/collect_environment_snapshot.sh results/environment_snapshot_run.json + - BK_SYSTEM=\"$system\" BK_NODE_STATUS_SNAPSHOT_STAGE=run bash scripts/collect_node_status_snapshot.sh results/node_status_snapshot_run.json - bash scripts/record_timestamp.sh results/run_start - bash $program_path/run.sh $system $nodes ${numproc_node} ${nthreads} - bash scripts/record_timestamp.sh results/run_end @@ -215,6 +216,7 @@ ${job_prefix}_build_run: - bash scripts/record_timestamp.sh results/build_start - bash scripts/build_with_cache.sh $program $system $program_path - bash scripts/record_timestamp.sh results/build_end + - BK_SYSTEM=\"$system\" BK_NODE_STATUS_SNAPSHOT_STAGE=run bash scripts/collect_node_status_snapshot.sh results/node_status_snapshot_run.json - bash scripts/record_timestamp.sh results/run_start - bash $program_path/run.sh $system $nodes ${numproc_node} ${nthreads} - bash scripts/record_timestamp.sh results/run_end diff --git a/scripts/result.sh b/scripts/result.sh index 2690c8e..3d3173c 100644 --- a/scripts/result.sh +++ b/scripts/result.sh @@ -730,6 +730,49 @@ build_environment_snapshot_block() { environment_snapshot_block=$(build_environment_snapshot_block) +build_node_status_snapshot_block() { + local snapshot_file="results/node_status_snapshot_run.json" + + if [ ! -f "$snapshot_file" ]; then + printf '%s' "" + return 0 + fi + + local snapshot_json + if ! snapshot_json=$(jq -cS 'if type == "object" then . else error("node snapshot must be an object") end' "$snapshot_file" 2>/dev/null); then + echo "WARNING: results/node_status_snapshot_run.json is invalid; omitting node status snapshot summary" >&2 + printf '%s' "" + return 0 + fi + + local snapshot_hash + snapshot_hash=$(printf '%s' "$snapshot_json" | sha256_text 2>/dev/null || true) + if [ -z "$snapshot_hash" ]; then + printf '%s' "" + return 0 + fi + + jq -n -c \ + --arg hash "sha256:${snapshot_hash}" \ + --arg artifact_path "$snapshot_file" \ + --argjson snapshot "$snapshot_json" \ + '{ + schema_version: ($snapshot.schema_version // 1), + kind: ($snapshot.kind // "node_status_snapshot"), + hash: $hash, + collection_status: ($snapshot.collection_status // "unknown"), + collection_warnings: ($snapshot.collection_warnings // []), + scheduler_kind: ($snapshot.scheduler.kind // "unknown"), + summary: ($snapshot.summary // {}), + artifact: { + type: "file_reference", + path: $artifact_path + } + }' +} + +node_status_snapshot_block=$(build_node_status_snapshot_block) + build_input_info_block() { local input_info_file="results/input_info.json" @@ -988,6 +1031,12 @@ write_result_json() { \"build_cache\": ${build_cache_block}" fi + local node_status_snapshot_json_block="" + if [ -n "$node_status_snapshot_block" ]; then + node_status_snapshot_json_block=", + \"node_status_snapshot\": ${node_status_snapshot_block}" + fi + local input_info_json_block="" local result_input_info_block="" result_input_info_block=$(filter_input_info_block_for_result "$exp") @@ -1055,7 +1104,7 @@ write_result_json() { "nthreads": "$nthreads", "description": "$description", "confidential": "$confidential", - "source_info": $source_info_block${input_info_json_block}${timing_observations_json_block}${profile_data_block}${fom_breakdown_block}${timing_block}${mode_block}${trigger_block}${build_job_block}${run_job_block}${pipeline_id_block}${parent_pipeline_id_block}${execution_trigger_block}${environment_snapshot_json_block}${build_cache_json_block} + "source_info": $source_info_block${input_info_json_block}${timing_observations_json_block}${profile_data_block}${fom_breakdown_block}${timing_block}${mode_block}${trigger_block}${build_job_block}${run_job_block}${pipeline_id_block}${parent_pipeline_id_block}${execution_trigger_block}${environment_snapshot_json_block}${build_cache_json_block}${node_status_snapshot_json_block} } EOF diff --git a/scripts/result_server/send_results.sh b/scripts/result_server/send_results.sh index 57dbedd..e83633f 100644 --- a/scripts/result_server/send_results.sh +++ b/scripts/result_server/send_results.sh @@ -163,7 +163,8 @@ collect_measurement_artifacts_for_result() { def file_reference_paths: ((.fom_breakdown.sections // [])[]? | (.artifacts // [])[]? | select(.type == "file_reference") | .path // empty), ((.fom_breakdown.overlaps // [])[]? | (.artifacts // [])[]? | select(.type == "file_reference") | .path // empty), - ((.timing_observations.observations // [])[]? | .artifact? | select(.type == "file_reference") | .path // empty); + ((.timing_observations.observations // [])[]? | .artifact? | select(.type == "file_reference") | .path // empty), + (.node_status_snapshot.artifact? | select(.type == "file_reference") | .path // empty); file_reference_paths ' "$json_file" 2>/dev/null || true) } diff --git a/scripts/tests/run_profile_data_shell_tests.sh b/scripts/tests/run_profile_data_shell_tests.sh index 4dd9a27..4eefbd2 100644 --- a/scripts/tests/run_profile_data_shell_tests.sh +++ b/scripts/tests/run_profile_data_shell_tests.sh @@ -45,6 +45,7 @@ scripts/tests/test_bk_input_info.sh scripts/tests/test_bk_timing_observations.sh scripts/tests/test_qws_timing_artifact.sh scripts/tests/test_build_environment_snapshot.sh +scripts/tests/test_node_status_snapshot.sh scripts/tests/test_ci_timing_context.sh scripts/tests/test_ncu_plan_generation.sh scripts/tests/test_scheduler_extra_args.sh diff --git a/scripts/tests/test_node_status_snapshot.sh b/scripts/tests/test_node_status_snapshot.sh new file mode 100644 index 0000000..be99ccb --- /dev/null +++ b/scripts/tests/test_node_status_snapshot.sh @@ -0,0 +1,137 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_DIR=$(cd "${SCRIPT_DIR}/../.." && pwd) + +if ! command -v jq >/dev/null 2>&1; then + echo "jq not found; skipping node status snapshot test" + exit 0 +fi + +TMP_DIR=$(mktemp -d) +trap 'rm -rf "${TMP_DIR}"' EXIT + +mkdir -p "${TMP_DIR}/unknown/results" +pushd "${TMP_DIR}/unknown" >/dev/null +BK_NODE_STATUS_SNAPSHOT_REMOTE=0 \ + bash "${REPO_DIR}/scripts/collect_node_status_snapshot.sh" results/node_status_snapshot_run.json >/dev/null +jq -e ' + .schema_version == 1 and + .kind == "node_status_snapshot" and + .collection_status == "unsupported" and + .scheduler.kind == "unknown" and + .summary.observed_host_count == 1 and + (.summary.gpu_query_statuses | type) == "array" +' results/node_status_snapshot_run.json >/dev/null +popd >/dev/null + +mkdir -p "${TMP_DIR}/slurm/results" "${TMP_DIR}/bin" + +cat > "${TMP_DIR}/bin/hostname" <<'EOF' +#!/bin/bash +printf '%s\n' "${BK_TEST_HOSTNAME:-node-a}" +EOF + +cat > "${TMP_DIR}/bin/scontrol" <<'EOF' +#!/bin/bash +set -euo pipefail +if [ "${1:-}" = "show" ] && [ "${2:-}" = "hostnames" ]; then + printf '%s\n' node-a node-b + exit 0 +fi +exit 1 +EOF + +cat > "${TMP_DIR}/bin/srun" <<'EOF' +#!/bin/bash +set -euo pipefail +while [ "$#" -gt 0 ]; do + case "$1" in + -N|-n|--nodes|--ntasks|--ntasks-per-node) + shift 2 + ;; + --nodes=*|--ntasks=*|--ntasks-per-node=*) + shift + ;; + --) + shift + break + ;; + -*) + shift + ;; + *) + break + ;; + esac +done +BK_TEST_HOSTNAME=node-a "$@" +BK_TEST_HOSTNAME=node-b "$@" +EOF + +cat > "${TMP_DIR}/bin/nvidia-smi" <<'EOF' +#!/bin/bash +set -euo pipefail +case "$*" in + *--query-gpu=*) + if [ "${BK_TEST_HOSTNAME:-node-a}" = "node-b" ]; then + printf '%s\n' \ + "0, NVIDIA B200, 128, 183000, 0, 35, 1230, 2000, P0" \ + "1, NVIDIA B200, 256, 183000, 0, 36, 1230, 2000, P0" + else + printf '%s\n' \ + "0, NVIDIA B200, 0, 183000, 0, 33, 1230, 2000, P0" \ + "1, NVIDIA B200, 0, 183000, 0, 34, 1230, 2000, P0" + fi + ;; + *--query-compute-apps=*) + if [ "${BK_TEST_HOSTNAME:-node-a}" = "node-b" ]; then + printf '%s\n' "1234, 512" + fi + ;; + *) + exit 1 + ;; +esac +EOF + +chmod +x "${TMP_DIR}/bin/hostname" "${TMP_DIR}/bin/scontrol" "${TMP_DIR}/bin/srun" "${TMP_DIR}/bin/nvidia-smi" + +pushd "${TMP_DIR}/slurm" >/dev/null +PATH="${TMP_DIR}/bin:${PATH}" \ +SLURM_JOB_ID=123 \ +SLURM_JOB_PARTITION=gpu \ +SLURM_JOB_NODELIST="node-[a-b]" \ +SLURM_JOB_NUM_NODES=2 \ +SLURM_NTASKS=4 \ +SLURM_TASKS_PER_NODE="2(x2)" \ +SLURM_CPUS_PER_TASK=32 \ +SLURM_JOB_GPUS="0,1,2,3" \ +BK_NODE_STATUS_SNAPSHOT_TIMEOUT_SECONDS=5 \ + bash "${REPO_DIR}/scripts/collect_node_status_snapshot.sh" results/node_status_snapshot_run.json >/dev/null + +jq -e ' + .schema_version == 1 and + .kind == "node_status_snapshot" and + .collection_status == "ok" and + .scheduler.kind == "slurm" and + .scheduler.slurm.remote_collection_status == "ok" and + .allocation.scheduler_hosts == ["node-a", "node-b"] and + (.observed.hosts | length) == 2 and + .summary.scheduler_host_count == 2 and + .summary.observed_host_count == 2 and + .summary.observed_gpu_count == 4 and + .summary.gpu_memory_used_total_mib == 384 and + .summary.gpu_compute_process_count == 1 and + .summary.gpu_compute_memory_used_mib == 512 and + (.summary.warnings | index("gpu_compute_processes_present_before_run") != null) +' results/node_status_snapshot_run.json >/dev/null + +if jq -e 'tostring | contains("1234")' results/node_status_snapshot_run.json >/dev/null; then + echo "node status snapshot should not record process identifiers" >&2 + exit 1 +fi +popd >/dev/null + +echo "node status snapshot test passed" diff --git a/scripts/tests/test_process_and_send_results.sh b/scripts/tests/test_process_and_send_results.sh index 8e55678..843e8aa 100644 --- a/scripts/tests/test_process_and_send_results.sh +++ b/scripts/tests/test_process_and_send_results.sh @@ -61,6 +61,29 @@ cat > "${TMP_DIR}/project/results/environment_snapshot_run.json" <<'EOF' } EOF +cat > "${TMP_DIR}/project/results/node_status_snapshot_run.json" <<'EOF' +{ + "schema_version": 1, + "kind": "node_status_snapshot", + "stage": "run", + "collected_at": "2026-09-14T00:01:30Z", + "collection_status": "ok", + "scheduler": { + "kind": "slurm" + }, + "summary": { + "scheduler_host_count": 2, + "observed_host_count": 2, + "observed_gpu_count": 4, + "gpu_memory_used_total_mib": 0, + "gpu_compute_process_count": 0, + "gpu_compute_memory_used_mib": 0, + "gpu_query_statuses": ["ok"], + "warnings": [] + } +} +EOF + cat > "${TMP_DIR}/project/results/input_info.json" <<'EOF' { "schema_version": 1, @@ -95,6 +118,9 @@ EOF cat > "${TMP_DIR}/bin/curl" <<'EOF' #!/bin/bash set -euo pipefail +if [ -n "${CURL_LOG:-}" ]; then + printf '%s\n' "$*" >> "$CURL_LOG" +fi if printf '%s\n' "$*" | grep -q '/api/ingest/result'; then printf '%s\n' '{"id":"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee","timestamp":"20260728_070000"}' exit 0 @@ -115,6 +141,7 @@ export BK_TRIGGER_ID="qws-fugaku-1400" export BK_TRIGGER_TYPE="scheduled" export BK_TRIGGER_REASON="cron:0 14 * * *@2026-08-07T14:00+09:00" export PARENT_PIPELINE_ID="54321" +export CURL_LOG="${TMP_DIR}/curl.log" pushd "${TMP_DIR}/project" >/dev/null bash scripts/result_server/process_and_send_results.sh qws Fugaku cross qws_Fugaku_build qws_Fugaku_N1_P2_T3_run 12345 > "${TMP_DIR}/process.log" @@ -165,7 +192,11 @@ jq -e ' (.pipeline_timing.run_time | type) == "number" and .pipeline_timing.run_time_scope == "job" and (.pipeline_timing | has("profiled_run_included") | not) and - (.execution_trigger | type) == "object" + (.execution_trigger | type) == "object" and + .node_status_snapshot.collection_status == "ok" and + .node_status_snapshot.scheduler_kind == "slurm" and + .node_status_snapshot.summary.observed_gpu_count == 4 and + .node_status_snapshot.artifact.path == "results/node_status_snapshot_run.json" ' "${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null jq -e ' .input_info.schema_version == 1 and @@ -184,6 +215,8 @@ jq -e ' ' "${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null jq -e '."result0.json".uuid == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"' \ "${TMP_DIR}/project/send_results_workspace/results/server_result_meta.json" >/dev/null +grep -q '/api/ingest/measurement-artifact' "${TMP_DIR}/curl.log" +grep -q 'file=@results/node_status_snapshot_run.json' "${TMP_DIR}/curl.log" grep -q "Result summary: code=qws system=Fugaku mode=cross exp=CASE0 fom=1.25 pipeline=12345" "${TMP_DIR}/process.log" if grep -q '"environment_snapshot"' "${TMP_DIR}/process.log"; then echo "process log should not include full result JSON" >&2 diff --git a/scripts/tests/test_result_common_json_contract.sh b/scripts/tests/test_result_common_json_contract.sh index 4bcabbf..9247bcb 100644 --- a/scripts/tests/test_result_common_json_contract.sh +++ b/scripts/tests/test_result_common_json_contract.sh @@ -159,6 +159,55 @@ cat > "${TMP_DIR}/results/environment_snapshot_run.json" <<'EOF' } EOF +cat > "${TMP_DIR}/results/node_status_snapshot_run.json" <<'EOF' +{ + "schema_version": 1, + "kind": "node_status_snapshot", + "stage": "run", + "collected_at": "2026-09-14T00:01:30Z", + "collection_status": "ok", + "collection_warnings": ["remote_collection_not_used"], + "scheduler": { + "kind": "slurm", + "slurm": { + "job_id": "123", + "partition": "gpu", + "remote_collection_status": "ok" + } + }, + "allocation": { + "scheduler_hosts": ["node-a", "node-b"] + }, + "observed": { + "hosts": [ + { + "hostname": "node-a", + "gpu_state": { + "query_status": "ok", + "gpus": [ + {"index": "0", "memory_used_mib": 0} + ], + "process_summary": { + "compute_process_count": 0, + "memory_used_mib": 0 + } + } + } + ] + }, + "summary": { + "scheduler_host_count": 2, + "observed_host_count": 1, + "observed_gpu_count": 1, + "gpu_memory_used_total_mib": 0, + "gpu_compute_process_count": 0, + "gpu_compute_memory_used_mib": 0, + "gpu_query_statuses": ["ok"], + "warnings": ["observed_host_count_differs_from_scheduler_host_count"] + } +} +EOF + cat > "${TMP_DIR}/results/build_cache.env" <<'EOF' BK_BUILD_CACHE_STATUS=hit BK_BUILD_CACHE_REASON=restored cached build artifacts @@ -230,7 +279,19 @@ jq -e ' .pipeline_timing.scheduler_queue_time_source == "runner_metadata" and .pipeline_timing.run_time == 34 and .pipeline_timing.run_time_scope == "job" and - (.pipeline_timing | has("profiled_run_included") | not) + (.pipeline_timing | has("profiled_run_included") | not) and + .node_status_snapshot.schema_version == 1 and + .node_status_snapshot.kind == "node_status_snapshot" and + (.node_status_snapshot.hash | startswith("sha256:")) and + .node_status_snapshot.collection_status == "ok" and + .node_status_snapshot.collection_warnings == ["remote_collection_not_used"] and + .node_status_snapshot.scheduler_kind == "slurm" and + .node_status_snapshot.summary.scheduler_host_count == 2 and + .node_status_snapshot.summary.observed_host_count == 1 and + .node_status_snapshot.artifact.type == "file_reference" and + .node_status_snapshot.artifact.path == "results/node_status_snapshot_run.json" and + (.node_status_snapshot | has("observed") | not) and + (.node_status_snapshot | has("allocation") | not) ' "${RESULT_JSON}" >/dev/null jq -e ' diff --git a/scripts/tests/test_scheduler_extra_args.sh b/scripts/tests/test_scheduler_extra_args.sh index 9fa79d5..ff1fb79 100644 --- a/scripts/tests/test_scheduler_extra_args.sh +++ b/scripts/tests/test_scheduler_extra_args.sh @@ -34,7 +34,7 @@ test "$(get_scheduler_extra_args Fugaku)" = "--account=global" tmpdir="" estimate_tmpdir=$(mktemp -d) -trap 'rm -rf "$tmpdir" "$estimate_tmpdir"' EXIT +trap 'rm -rf "$tmpdir" "$estimate_tmpdir"; rm -f "${REPO_DIR}/script.sh"' EXIT mkdir -p "$estimate_tmpdir/app" if has_estimate_script "$estimate_tmpdir/app"; then