From fffa2bbd78fb9a9d5e83cc72b0d8104278874599 Mon Sep 17 00:00:00 2001 From: xinyuej Date: Mon, 31 Aug 2026 16:15:45 +0800 Subject: [PATCH 1/2] sync develop from internal repo 14da8ab --- docs-site/scripts/prepare-versioned-docs.js | 32 +- .../scripts/prepare-versioned-docs.test.js | 20 - docs-site/static/sflow_intro.html | 258 +--- docs/release_notes/RELEASE_NOTES_v0.3.1.md | 255 ---- docs/user/architecture.md | 3 +- docs/user/backends.md | 39 +- docs/user/cli.md | 2 - docs/user/configuration.md | 2 +- docs/user/outputs.md | 10 - docs/user/quick-reference.md | 13 +- docs/user/resources.md | 9 - docs/user/run-output.md | 14 +- docs/user/samples.md | 6 +- .../inference_x_v2/benchmark_aiperf.yaml | 1 + .../modular/inference_x_v2/bulk_input.csv | 4 +- .../inference_x_v2/common_workflow.yaml | 2 +- .../sglang_agg_benchmark_aiperf_2n_008.yaml | 18 +- .../sglang_prefill_decode_benchmar_003.yaml | 4 +- .../sglang_prefill_decode_benchmar_004.yaml | 6 +- .../trtllm_agg_benchmark_aiperf_1n_007.yaml | 6 +- .../trtllm_prefill_decode_benchmar_001.yaml | 12 +- .../trtllm_prefill_decode_benchmar_002.yaml | 12 +- .../vllm_agg_benchmark_aiperf_1n_2_009.yaml | 2 +- .../modular/inference_x_v2/sglang/agg.yaml | 2 +- .../modular/inference_x_v2/sglang/decode.yaml | 2 +- .../inference_x_v2/sglang/prefill.yaml | 2 +- .../modular/inference_x_v2/trtllm/agg.yaml | 2 +- .../modular/inference_x_v2/trtllm/decode.yaml | 4 + .../inference_x_v2/trtllm/prefill.yaml | 4 + .../kubernetes/dynamo_sglang_agg.yaml | 1 + .../kubernetes/dynamo_sglang_disagg.yaml | 1 + .../kubernetes/dynamo_sglang_disagg_dsr1.yaml | 1 + .../kubernetes/dynamo_trtllm_agg.yaml | 1 + .../kubernetes/dynamo_trtllm_disagg.yaml | 1 + .../kubernetes/dynamo_vllm_agg.yaml | 1 + .../kubernetes/dynamo_vllm_disagg.yaml | 1 + .../self_contained/slurm/aiperf_template.yaml | 1 + .../slurm/dynamo_sglang_agg.yaml | 3 +- .../slurm/dynamo_sglang_disagg.yaml | 3 +- .../slurm/dynamo_trtllm_agg.yaml | 3 +- .../slurm/dynamo_trtllm_disagg.yaml | 11 +- .../self_contained/slurm/dynamo_vllm_agg.yaml | 1 + .../slurm/dynamo_vllm_disagg.yaml | 21 +- .../slurm/gpu_placement_matrix.yaml | 366 ----- .../self_contained/slurm/infmax_v1_ds_r1.yaml | 3 +- .../self_contained/slurm/monitor_mixed.yaml | 167 +-- .../slurm/sglang_server_client.yaml | 1 + .../slurm/trtllm_serve_disagg.yaml | 18 - pyproject.toml | 5 - scripts/full_sample_tests.sh | 268 ++-- scripts/use_under_dev_sflow.sh | 87 +- src/sflow/app/assembly.py | 13 +- src/sflow/cli/batch.py | 7 - src/sflow/cli/sample.py | 3 + src/sflow/config/loader.py | 22 - src/sflow/config/schema.py | 8 - src/sflow/core/compute_node.py | 10 - src/sflow/core/execution_summary.py | 17 +- src/sflow/core/launcher.py | 191 +++ src/sflow/core/task_graph.py | 31 + src/sflow/exceptions.py | 10 + .../postprocess_monitor_timeline.py | 175 +-- src/sflow/plugins/backends/slurm.py | 226 +-- src/sflow/plugins/operators/gpu_placement.sh | 260 ---- src/sflow/plugins/operators/srun.py | 176 +-- .../inference_x_v2/benchmark_aiperf.yaml | 1 + .../modular/inference_x_v2/bulk_input.csv | 4 +- .../inference_x_v2/common_workflow.yaml | 2 +- .../sglang_agg_benchmark_aiperf_2n_008.yaml | 18 +- .../sglang_prefill_decode_benchmar_003.yaml | 4 +- .../sglang_prefill_decode_benchmar_004.yaml | 6 +- .../trtllm_agg_benchmark_aiperf_1n_007.yaml | 6 +- .../trtllm_prefill_decode_benchmar_001.yaml | 12 +- .../trtllm_prefill_decode_benchmar_002.yaml | 12 +- .../vllm_agg_benchmark_aiperf_1n_2_009.yaml | 2 +- .../modular/inference_x_v2/sglang/agg.yaml | 2 +- .../modular/inference_x_v2/sglang/decode.yaml | 2 +- .../inference_x_v2/sglang/prefill.yaml | 2 +- .../modular/inference_x_v2/trtllm/agg.yaml | 2 +- .../modular/inference_x_v2/trtllm/decode.yaml | 4 + .../inference_x_v2/trtllm/prefill.yaml | 4 + .../kubernetes/dynamo_sglang_agg.yaml | 1 + .../kubernetes/dynamo_sglang_disagg.yaml | 1 + .../kubernetes/dynamo_sglang_disagg_dsr1.yaml | 1 + .../kubernetes/dynamo_trtllm_agg.yaml | 1 + .../kubernetes/dynamo_trtllm_disagg.yaml | 1 + .../kubernetes/dynamo_vllm_agg.yaml | 1 + .../kubernetes/dynamo_vllm_disagg.yaml | 1 + .../self_contained/slurm/aiperf_template.yaml | 1 + .../slurm/dynamo_sglang_agg.yaml | 3 +- .../slurm/dynamo_sglang_disagg.yaml | 3 +- .../slurm/dynamo_trtllm_agg.yaml | 3 +- .../slurm/dynamo_trtllm_disagg.yaml | 11 +- .../self_contained/slurm/dynamo_vllm_agg.yaml | 1 + .../slurm/dynamo_vllm_disagg.yaml | 21 +- .../slurm/gpu_placement_matrix.yaml | 366 ----- .../self_contained/slurm/infmax_v1_ds_r1.yaml | 3 +- .../self_contained/slurm/monitor_mixed.yaml | 167 +-- .../slurm/sglang_server_client.yaml | 1 + .../slurm/trtllm_serve_disagg.yaml | 18 - .../writing-sflow-yaml/schema-reference.md | 3 - src/sflow/utils/gpu.py | 54 +- tests/conftest.py | 5 + tests/e2e_tests/sample_test.sh | 908 +----------- tests/integration/guide/sflow_dynamo.yaml | 1 + .../k8s_recipes/dynamo_sglang_disagg.yaml | 9 +- .../k8s_recipes/dynamo_trtllm_disagg.yaml | 9 +- .../golden/k8s_recipes/dynamo_vllm_agg.yaml | 9 +- .../test_app_assembly_resolve_variables.py | 21 - tests/unit/test_cli_batch.py | 9 +- tests/unit/test_config_loader.py | 53 - tests/unit/test_core_execution_summary.py | 57 - tests/unit/test_monitoring_postprocess.py | 183 --- .../test_plugin_backends_slurm_backend.py | 226 +-- tests/unit/test_srun_gpu_placement.py | 1310 +---------------- tests/unit/test_srun_operator.py | 37 - tests/unit/test_utils_gpu.py | 84 -- 117 files changed, 814 insertions(+), 5712 deletions(-) delete mode 100644 docs/release_notes/RELEASE_NOTES_v0.3.1.md delete mode 100644 examples/self_contained/slurm/gpu_placement_matrix.yaml create mode 100644 src/sflow/exceptions.py delete mode 100644 src/sflow/plugins/operators/gpu_placement.sh delete mode 100644 src/sflow/samples/self_contained/slurm/gpu_placement_matrix.yaml diff --git a/docs-site/scripts/prepare-versioned-docs.js b/docs-site/scripts/prepare-versioned-docs.js index c66c1c2..833b931 100644 --- a/docs-site/scripts/prepare-versioned-docs.js +++ b/docs-site/scripts/prepare-versioned-docs.js @@ -205,33 +205,6 @@ function writeJson(filePath, value) { fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`); } -// In-page anchors that were already broken in a published snapshot. A released tag -// cannot be re-cut, so for frozen versions the only place left to repair them is -// here, at extraction time. Current docs are deliberately NOT covered: those are -// fixed at source in docs/. -// -// Each entry carries the heading it assumes, and the repair only fires when that -// heading is actually present. `main` is a branch, not a frozen tag -- once the -// source fix lands there its snapshot already has the corrected heading, and an -// unconditional rewrite would break the link a second time in the other direction. -const FROZEN_ANCHOR_REPAIRS = [ - { - from: "#modular-inference-recipe-inference_x_v2", - to: "#inference_x_v2", - onlyWhenHeading: /^### inference_x_v2\s*$/m, - }, -]; - -function repairFrozenAnchors(text) { - let out = text; - for (const repair of FROZEN_ANCHOR_REPAIRS) { - if (out.includes(repair.from) && repair.onlyWhenHeading.test(out)) { - out = out.replaceAll(repair.from, repair.to); - } - } - return out; -} - function rewriteVersionedDocsLinks(rootDir, versionLabel) { const entries = fs.readdirSync(rootDir, { withFileTypes: true }); for (const entry of entries) { @@ -240,9 +213,7 @@ function rewriteVersionedDocsLinks(rootDir, versionLabel) { rewriteVersionedDocsLinks(entryPath, versionLabel); } else if (entry.isFile() && entry.name.endsWith(".md")) { const text = fs.readFileSync(entryPath, "utf8"); - const rewritten = repairFrozenAnchors( - text.replaceAll("](/docs/", `](/docs/${versionLabel}/`), - ); + const rewritten = text.replaceAll("](/docs/", `](/docs/${versionLabel}/`); if (rewritten !== text) { fs.writeFileSync(entryPath, rewritten); } @@ -313,7 +284,6 @@ if (require.main === module) { module.exports = { buildDocVersionPlan, - repairFrozenAnchors, currentDocsSource, docsPaths, isReleaseTag, diff --git a/docs-site/scripts/prepare-versioned-docs.test.js b/docs-site/scripts/prepare-versioned-docs.test.js index 1d7e6fb..f8d82bf 100644 --- a/docs-site/scripts/prepare-versioned-docs.test.js +++ b/docs-site/scripts/prepare-versioned-docs.test.js @@ -11,7 +11,6 @@ const { safeVersionDirName, currentDocsSource, prepareVersionedDocs, - repairFrozenAnchors, } = require("./prepare-versioned-docs"); function git(cwd, args) { @@ -165,22 +164,3 @@ test("prepareVersionedDocs materializes current docs, versioned docs, sidebars, "See [SPP](/docs/v1.2.3/sflow_spp).\n", ); }); - -test("repairFrozenAnchors fixes a stale anchor only while the old heading is present", () => { - const stale = [ - "See [Modular inference recipe](#modular-inference-recipe-inference_x_v2).", - "", - "### inference_x_v2", - ].join("\n"); - assert.match(repairFrozenAnchors(stale), /\(#inference_x_v2\)/); - - // The guard that matters: `main` is a branch, so once the source fix lands its - // snapshot already carries the long heading and the link is correct. Rewriting - // unconditionally would break it a second time, in the other direction. - const fixed = [ - "See [Modular inference recipe](#modular-inference-recipe-inference_x_v2).", - "", - "### Modular inference recipe (inference_x_v2)", - ].join("\n"); - assert.equal(repairFrozenAnchors(fixed), fixed); -}); diff --git a/docs-site/static/sflow_intro.html b/docs-site/static/sflow_intro.html index b7fcdca..54a0db7 100644 --- a/docs-site/static/sflow_intro.html +++ b/docs-site/static/sflow_intro.html @@ -3,7 +3,7 @@ -sflow — Declarative Workflow Descriptor for Massive GPU Clusters +sflow — Declarative Workflow Descriptor @@ -549,21 +352,17 @@
01 / 13 -
-
NV-SFLOW
-

Workflow orchestrator for large-scale GPU clusters of any kind

-

Describe once. Run anywhere._

-
-vLLMSGLangTensorRT-LLMDynamoPyTorch… -sflow -KubernetesSlurmDocker +
+ +

NV-sflow

+

Declarative Workflow Descriptor

+

Describe once. Run anywhere._

+
+View Documentation → +Explore Feature Map → +GitHub
-
-View Documentation → -Feature Map → -GitHub
-
Scroll @@ -597,12 +396,12 @@

The workload stays the same. The plumbing multiplies.

03 / 13
-Backend Agnostic by Design -

Keep what. Swap where. Rewrite nothing.

+Cross-Backend Transplanting +

Keep what. Swap where. Change only a few lines.

-
✎

One Semantic

A deployment's logic never changes — only the infrastructure glue does.

-
⇄

Swap the Backend Block

The same sflow.yaml runs on Docker, Slurm and Kubernetes.

-
⚙

Delegate, Don't Reimplement

Backends defer to each platform's native ecosystem instead of rebuilding it.

+
✎

Describe Once

Portable YAML — tasks, deps, resources, launch methods

+
⇄

Swap the Substrate

Change the backend/operator block; keep DAG, scripts, probes, and results.

+
⚙

Pluggable Plugins

Probes, artifacts, replicas — no platform coupling.

@@ -683,7 +482,7 @@

Dependencies activate the next stage

05 / 13
-Cluster-Level Orchestration at Scale +Resource Planning

Topology-aware GPU Allocation

sflow topology plannerGPU capacity · node affinity · task dependencies
▼ backend-native assignment
@@ -710,7 +509,6 @@

Topology-aware GPU Allocation

-
Node and GPU placement, multi-node replicas and sweeps, readiness probes, and batch submission — so one descriptor drives hundreds of GPUs instead of a pile of hand-written bash.
@@ -756,7 +554,7 @@

Understand

Backend Adapters

One intent becomes four native execution plans

Portable workflow contract   DAG · resources · probes · artifacts · results
-
▼ sflow delegates to each platform's native ecosystem
+
▼ sflow adapts the execution substrate
>_

Local

bash process
synthetic node/GPU map

▣

Docker

docker run
container GPU devices

@@ -780,9 +578,9 @@

Co-located GPU tasks, one fast intra-node fabric

Separate pods · isolated views

-
worker AGPU 0-1 visible
+
worker AGPU 0 visible
IB / TCP
↔
-
worker BGPU 2-3 visible
+
worker BGPU 1 visible
sflow merge
➔
@@ -938,7 +736,7 @@

Structured Error Analysis

identifiers that the TCP router can't parse. Fix: Verify DYN_REQUEST_PLANE and frontend -networking config match Dynamo disagg routing.
+networking config match SGLang disagg routing. @@ -953,13 +751,12 @@

CLI at a Glance

- - - - + + + + - - +
CommandPurposeKey Flags
sflow runExecute a workflow--dry-run --tui --set -f (multi-file)
sflow batchGenerate sbatch scripts--submit --bulk-input --row
sflow composeMerge multiple YAMLs--resolve --missable-tasks -o
sflow visualizeRender DAG graph--format png/svg/mermaid
sflow runExecute a workflow--dry-run --tui --set
sflow batchGenerate sbatch scripts--submit --bulk-input
sflow composeMerge multiple YAMLs--resolve --validate
sflow visualizeRender DAG image--format png/svg/mermaid
sflow sampleList / copy examples--list -o
sflow skillInstall AI agent skills--list -o
sflow upgradeReinstall in place (alias: sflow update)--branch --dry-run --force
sflow skillExport AI agent skills--list -o
@@ -1055,7 +852,6 @@

Install. Explore. Run.

Start locally, validate before allocating hardware, then use the docs and repository as your source of truth.

# install from the repository -$ uv venv --python python3 && source .venv/bin/activate $ uv pip install "sflow @ git+https://github.com/NVIDIA/nv-sflow.git@main" # verify and try a local workflow diff --git a/docs/release_notes/RELEASE_NOTES_v0.3.1.md b/docs/release_notes/RELEASE_NOTES_v0.3.1.md deleted file mode 100644 index 761cdef..0000000 --- a/docs/release_notes/RELEASE_NOTES_v0.3.1.md +++ /dev/null @@ -1,255 +0,0 @@ -# sflow v0.3.1 Release Notes - -**Release date:** August 2026 -**Previous release:** [v0.3.0](https://github.com/NVIDIA/nv-sflow/releases/tag/v0.3.0) (July 2026) - ---- - -## Highlights - -v0.3.0 made a recipe portable across Local, Docker, Slurm, and Kubernetes. v0.3.1 is about trusting the run it produces: **knowing which physical GPU each task actually landed on**, and **not losing the run to a silent hang or a truncated log**. - -Three things drive the release. **GPU placement** became explicit — you can pin device indices, sflow re-applies its plan *inside* the Slurm step where `slurmstepd` used to overwrite it, and concurrent runs on one box no longer collide on GPU 0. **The driver stopped hanging** — every `kubectl` call is bounded, console output is length-capped, an event-loop watchdog captures stalls that previously left no trace at all, and the post-run log re-fetch that silently truncated long-running pods is gone. **Monitoring became aimable** — reports are on by default, a `window:` clips them to the phase you actually measured, and GPU charts draw one line per device instead of an average. - -| Area | v0.3.0 | v0.3.1 | -| --- | --- | --- | -| **GPU selection** | `gpus.count` only, index-agnostic | **+ `gpus.indices`** to pin device ids; `count` must divide the node count | -| **Slurm GPU placement** | `slurmstepd` overwrote `CUDA_VISIBLE_DEVICES` → every rank on GPU 0 | in-step re-select **by UUID** + a per-task `sflow_gpus.log` audit record | -| **Concurrent runs on one host** | each run packed from device 0, unaware of the others | machine-local **UUID reservation registry** (Docker) | -| **Kubernetes task logs** | post-run re-fetch, truncated to the last rotation window | exactly what `kubectl logs -f` delivered | -| **Kubernetes control plane** | one `get pod` per pod per tick; no call bounded | batched per context; **every call timeout-bounded** | -| **Monitoring** | reports opt-in, one averaged GPU line | reports **on by default**, `window:` markers, one line per device | -| **Merge-pod + intra-group deps** | rejected outright | **gated in-pod**, the run proceeds | -| **Upgrading sflow** | hand-written `uv pip install` incantation | **`sflow upgrade`** | - -> **Read the [Breaking Changes](#breaking-changes) before upgrading.** Despite the patch version, this release changes several defaults that affect existing recipes — most notably monitor report output, GPU count validation, and resource release timing. The [Upgrade Guide](#upgrade-guide) has a five-minute checklist. - ---- - -## New Features - -### 1. Pin GPUs by device index — `resources.gpus.indices` - -`resources.gpus` used to take only a `count`, which is index-agnostic: the planner packs the task into any contiguous idle run on one node. That is the right default, but it cannot express NUMA/NVLink affinity, reproduce a vendor benchmark's exact topology, or steer around a known-bad device. - -```yaml -resources: - gpus: - indices: [2, 3] # pin these device ids -``` - -Three modes, all on Local, Docker, and Slurm: - -| You write | You get | -| --- | --- | -| `count: 4` | any contiguous idle run of 4 on a single node (never straddles a node boundary) | -| `indices: [2, 3]` | the first node where **both** are free; the node scan restarts at node 0, so later tasks backfill | -| `count: 8` + `indices: [0, 1]` | fan-out — `count` is the **total across nodes**, `indices` the per-node slice, so the task spans `8 / 2 = 4` nodes | - -`CUDA_VISIBLE_DEVICES` preserves the order you wrote. Indices must be non-negative, unique, and non-empty — deliberately unlike `nodes.indices`, which allows `-1`. - -**Kubernetes rejects `indices` at plan time.** The cluster's device plugin or DRA assigns the physical devices, so sflow will not pretend otherwise; use `count` to size the request. - -Samples: `sflow sample self_contained/slurm/gpu_indices` (all three modes on a 4×4 board, with the expected placement map annotated) and `self_contained/slurm/gpu_placement_matrix` (a regression matrix that asserts placement by UUID, not by GPU count). - -### 2. Slurm: the GPU plan is re-applied inside the step - -On a GRES partition, `slurmstepd` overwrites the `CUDA_VISIBLE_DEVICES` sflow exports. Every step then saw the whole allocation and every rank picked device 0 — tasks planned onto different GPUs all piled onto the same one, and the run's numbers were quietly wrong rather than loudly broken. - -sflow no longer trusts the inherited value. A prelude sourced inside each Slurm step probes the devices the step can really see, looks up the physical UUIDs the driver resolved this task's plan to, and re-exports the indices those same cards have *there*. Matching by UUID rather than by index is also what fixes pyxis/enroot containers, which renumber devices from 0. - -On by default for any Slurm task with a `resources.gpus` slice; srun `gpus_per_task` opts out, since Slurm already carves per rank. - -- **`/sflow_gpus.log`** — a per-task audit record: planned indices and UUIDs, the inherited environment, the visible index→UUID map, and the final selection. Multi-node tasks write one per node. -- **`exit 97`** — a planned card not visible at all, a step holding fewer GPUs than planned, or a planned slot out of range now aborts the step instead of silently running on the wrong device. When *Slurm* chose the devices rather than sflow, this degrades to index arithmetic instead of failing. - -See [Backends → GPU placement inside the step](https://nvidia.github.io/nv-sflow/docs/user/backends). - -### 3. Concurrent runs on one host stop colliding (Docker) - -Several `sflow run` processes on one machine each packed GPUs from device 0 independently — the in-process planner had no idea another run existed. Two runs on one workstation fought over the same cards. - -A file-locked registry in machine-local temp now makes concurrent runs, and concurrent tasks within a run, claim **disjoint physical GPUs by nvidia-smi UUID**. It refuses to take a GPU a foreign workload is already on. The container is launched with `--gpus "device="` naming those exact cards. - -**On by default for the Docker backend**, with no YAML opt-in. It is skipped — each case logging which one you hit — for remote `hosts:` pools, any `nodes > 1`, and hosts where `nvidia-smi` will not run. It requires POSIX `fcntl`, so it is inert on Windows. - -| Environment variable | Default | Effect | -| --- | --- | --- | -| `SFLOW_GPU_RESERVATION` | `1` | `0` disables the registry entirely | -| `SFLOW_WAIT_FOR_GPUS` | unset | any value turns waiting on; unset restores fail-fast | -| `SFLOW_GPU_RESERVATION_DIR` | `$TMPDIR/sflow-gpu-reservations` | must stay machine-local — on NFS another host's records read as local | -| `SFLOW_GPU_BUSY_MEM_MIB` | `512` | how much foreign memory counts as "busy" | -| `SFLOW_GPU_IGNORE_FOREIGN` | unset | `1` when sflow owns the box | - -Reservations are held per task, not per run, and released in a `finally` bounded at 10s so `Ctrl-C` can never hang on the lock. Records whose owning PID is gone are reaped on the next claim, guarded against PID recycling; another user's records are never reaped. - -**New: `--wait-for-gpus `** (Docker) — when too few GPUs are free at reserve time, wait instead of failing fast. `0` waits forever, `N` bounds the wait, omitting it fails fast. Also settable per-recipe as the backend field `wait_for_gpus`. - -### 4. Kubernetes: the driver stops hanging, and logs stop lying - -None of this needs a config change. - -- **Task logs are now the streamed log.** The post-run one-shot `kubectl logs` re-fetch that replaced the streamed file is gone. Because the kubelet **rotates container logs**, that re-fetch returned only the last window — one hour of server output was persisted as its final ~11 seconds. -- **Every `kubectl` call is bounded** (30s poll, 300s delete) and retried on the next tick. A silently-dead TCP connection to the API server used to wedge the driver for 15–20 minutes with nothing logged. -- **Pod-status polling is batched** — pods in one context share a single `kubectl get pod a b c …` per tick. Status was ~90% of all kubectl traffic (862 of 957 calls in a measured 7-pod run), and at 19 concurrent recipes that traffic self-congested (mean `get pod` latency 0.2s → 1.3s). Terminal detection is at most ~1.5s staler. -- **The output collect is bounded and no longer intrudes into running pods.** It emits a heartbeat every 30s and, on timeout, names `collect_grace_seconds` and `collect_node_local_output` in the message. It no longer `kubectl exec`s into a live pod — that used to put 42 execs into a serving TRT-LLM pod. -- **`collect_node_local_output: false`** (backend, default `true`) turns the collect machinery off entirely: no in-pod `EXIT` trap, no driver-side copy. Task completion then depends only on pod status, probes, and the merge-pod marker, and outputs must reach you via a shared filesystem, `uploads:`, or a PVC. -- **The RBAC preflight no longer blocks on permissions sflow does not need.** `get nodes` and `get deviceclasses` are now *optional* — a denial warns and degrades node-level detection (set `gpus_per_node` explicitly) instead of failing the run. A namespace-scoped ServiceAccount on a shared multi-tenant cluster now works out of the box. -- **`--extra-kubectl-apply-args`** (repeatable) passes a flag to the `kubectl apply` **subcommand** (`--validate=false`, `--server-side`, `--force-conflicts`). kubectl takes global flags *before* the verb, so these cannot ride on `--extra-kubectl-args` — sflow now warns if it spots an apply-only flag there. - -### 5. Merge-pod accepts dependencies between its own members - -v0.3.0 refused to merge tasks that depended on each other: merged tasks run concurrently in one pod, so a benchmark depending on a co-located server could not use merged pods at all — exactly the case merge-pod exists for. - -A **direct** member→member edge is now honored by gating rather than rejected. The dependent member waits on an in-pod gate; the driver opens it when the dependency reaches READY or COMPLETED. If the dependency failed, the gated member never starts and propagates its exit code. A member reachable only *transitively through a non-member* is still rejected. - -Visible in `sflow_summary.log` as `gated_on=` on the SUBMITTED row and a new `UNGATED` timeline event. A gated member's duration is measured from gate-open, not submission — a 5-second client no longer reads as 40 seconds. - -### 6. Monitoring you can aim - -- **Reports are on by default.** Declaring `monitor:` now writes the report folders; `report: {enabled: false}` opts out. Worth doing on large fan-outs — a report folder is a per-view *copy* of the samples, so cost scales with `samples × views`. -- **`monitor.window`** clips a report to the phase you actually measured, using markers from the task's own log: - - ```yaml - monitor: - window: - start: "Benchmark starting" - end: "re:Total throughput: [0-9.]+" - ``` - - Plain strings are case-sensitive literal substrings; prefix `re:` or `regex:` for a regex. `start` resolves first and `end` only from matches strictly after it. Task monitors only. The collector still runs for the whole task — only the report is clipped, so `sflow_monitor/lifecycle/` and `sflow_monitor/windowed/` sit side by side. If a marker never matches, sflow **warns and skips that report** rather than silently falling back to lifecycle timing, and writes `window_not_found.json` so you can see which pattern missed. -- **GPU charts draw one line per device**, each labelled with its own avg/max, instead of one averaged line that hid an idle card. Node-level scopes (cpu/memory/disk/network) stay averaged. -- **Multi-node reports split per node** — `timeline..svg` per node instead of one combined chart. CSVs stay combined. -- **Task-event markers were redesigned** — labels drawn in place with no legend to decode, near-simultaneous events merged into one labelled rule (`3 tasks submit +2 more`). Dotted = started, solid = ended. -- **Clock skew is corrected and warned about** — samples are shifted onto the driver's clock for reporting only when the estimated node offset excludes zero. Raw logs keep node timestamps. -- **Coverage fixes:** the workflow-level monitor now spans every monitorable backend instead of only the default one; a task on a backend no monitor covers no longer gets a bogus empty report folder; and a task that reserved no GPUs no longer picks up its node's GPUs in its report. - -### 7. `sflow upgrade` - -Reinstall sflow in place without hand-writing the `uv pip install` incantation: - -```bash -sflow upgrade # latest main of the public repo -sflow upgrade --branch develop # a specific ref -sflow upgrade --sflow-index-url # a private PyPI index -sflow upgrade --sflow-source-path ~/src/sflow # editable, from a local checkout -sflow upgrade --dry-run # print the resolved command and stop -``` - -Prefers `uv`, falls back to `pip`. It refuses to upgrade over an editable/source-tree dev install unless you pass `--force`. `sflow update` is an alias. - -Note the deliberate asymmetry: bare `sflow upgrade` installs **`main` of the public OSS repo**, whereas `sflow batch` installs whatever ref the *running* environment came from. - -### 8. Run output that explains itself - -- **`GPU Assignment`** in `sflow_summary.log` — per task, the physical GPUs it was planned onto next to the devices it actually saw, with a hint when the backend re-indexes inside the container. GPU/node charts now plot physical devices; previously every Docker task was drawn on GPU 0. -- **`Node Topology`** — the CPU/NUMA/GPU probe each backend captured at reservation time. -- **`External Command Health`** — call counts, failures, timeouts, and mean/max latency for `kubectl`/`srun`/`docker`, with a `healthy`/`DEGRADED` verdict. Written even when the run is cancelled or fails, which is when it matters. Backed by `command_trace.jsonl`, written lazily and only for *notable* calls (non-zero exit, or slower than 5s), so a healthy run leaves no file. A live warning fires when a control-plane call takes over 5s. -- **`loop_stalls.txt`** — if the driver's event loop stops being scheduled for 30s, sflow logs a warning and dumps every thread's Python stack here. Previously such a freeze produced no diagnostic output at all, because sflow's own logging runs on the thread that was stuck. Created only when a stall actually happens. -- **Console output is length-capped at 2000 characters per line**, on *every* backend. A single unbounded line was measured at ~6.3µs and ~300 bytes of RSS per character — a 48MB line cost ~5 CPU-minutes and ~14GB and froze the driver's event loop. **`.log` is unaffected and still holds every byte**; probes, `result:` parsing, and `output:` all read the file, so only the terminal changes. -- **Progress bars that end on a carriage return keep their final frame.** A bar whose last redraw ended in `\r` used to vanish from both the console and `.log`, and two consecutive redraw-terminated reads spliced into `50%60%` — in the log file, not just the console. -- **DAG cycle errors name the loop edge by edge** in `depends_on` phrasing, with a separate "Waiting behind it:" list, instead of a bare `Graph contains a cycle`. -- **`--dry-run`** now lists every planned monitor report folder with its group and, for marker windows, the patterns — so a marker typo surfaces before the run rather than after, as an empty report. -- **New doc page:** [Understanding run output](https://nvidia.github.io/nv-sflow/docs/user/run-output) — the output tree, a "what do you want to know?" lookup table, and a five-step failure triage. `sflow run` now prints the run directory on failure too, as soon as the folder exists. - -### 9. Kubernetes MPI: per-rank CPU binding - -`mpi.cpu_bind` (`core` | `numa` | `none`, default `core`) injects per-rank CPU binding, but **only when several ranks share a pod**, and never over a binding your recipe already passes. `core` gives each rank an isolated core slice — the tightest cap on the LLVM/OpenMP thread pools that `OMP_NUM_THREADS` alone does not reach. - -`mpi.cpu_bind_cores_per_rank` (default `8`, `0` = uncapped) bounds that slice: the launch-time value is `min(cores-in-cpuset / ranks-per-pod, this)`, and if the cpuset has fewer cores than ranks the binding is skipped rather than failing the launch. - ---- - -## Improvements - -- **Docker:** CPU-only tasks no longer see the host's GPUs (many CUDA images bake `NVIDIA_VISIBLE_DEVICES=all`). Container names carry the driver PID (`sflow-p--`) so concurrent runs never collide, and orphaned containers from a dead driver are reaped once per run. `--gpus device=0,1` is now quoted — docker used to parse the trailing `1` as a *count* and die with *"cannot set both Count and DeviceIDs on device request"*. A raw `--gpus` grant in `extra_args` now warns, because docker *accumulates* device requests and would widen the container past its reservation. -- **Slurm:** `extra_args` no longer silently drops repeated values — a bare value in `["-G","1","-N","1"]` used to delete an earlier identical one and produce a wrong allocation. A new driver-side topology probe records the index→UUID map per node. -- **`sflow batch`:** compound expressions such as `${{ variables.NUM_NODES * 2 }}` now resolve, so the generated sbatch no longer diverges from the dry run; a `--set` node override reaches `#SBATCH --nodes`; config errors keep their full multi-line pydantic detail instead of being cut to `Configuration validation failed:`; and `.cache` is excluded from the source-tree copy, which used to make rsync exit 24 and kill the job seconds in. -- **`--skip-artifact-check`** (`sflow run` and `sflow batch`) — a missing `fs://` path warns instead of failing, and is left alone rather than created as an empty directory. For paths that exist only where the task runs. `sflow batch` forwards it into the job, which is where the check actually runs. -- **Container image preflight was loosened** — an unrecognized reference now warns instead of aborting, and the regex accepts pyxis/enroot forms such as `nvcr.io#nvidia/ai-dynamo/sglang-runtime:1.2.0`. **Recipes that failed preflight on v0.3.0 now run.** -- **A readiness probe written with `match_pattern` no longer fails its own dry run.** The validator normalizes it into `regex_pattern`, so any dump-and-reload round trip (`sflow compose`, or the temp config `sflow batch` writes) handed both back and tripped the "only one of" check. -- **TUI:** the header no longer clips the elapsed clock and output directory, and ticks once a second so the clock does not freeze on an idle run. -- **Packaging:** the `parse` pin was loosened from `==1.16.0` to `>=1.16,<2`, which had made sflow uninstallable alongside anything needing a newer `parse`. A bare `pytest` no longer pulls images and launches containers. - ---- - -## Breaking Changes - -Ordered by how likely they are to affect an existing v0.3.0 recipe. - -1. **`monitor.report.enabled` now defaults to `true`.** Any recipe with a bare `monitor:` block now writes report folders — disk and post-processing you did not previously pay for. *Migration:* `report: {enabled: false}` to opt out. -2. **Monitor report paths gained a group segment.** `sflow_monitor//` → `sflow_monitor/lifecycle//` (or `windowed//`). `sflow_monitor/raw/` and `sflow_monitor.log` are unchanged. *Migration:* **any scraper, CI glob, or notebook reading `sflow_monitor//summary.csv` breaks** — insert the group segment. -3. **`resources.gpus.count` must now divide the assigned node count.** Previously the planner rounded *up* and reserved that many on **every** node: `nodes: 2` + `gpus: 1` silently consumed 2 GPUs, and `nodes: 2` + `gpus: 3` consumed 4; a `count: 10` against `(8, 2)` caps silently under-allocated to 4. Both are now rejected at plan time with a message naming a count that works. *Migration:* multiply by the node count — `nodes: 2` + `gpus: 1` becomes `gpus: 2`. Backend-agnostic. -4. **`fail_fast` now defaults per backend — `true` on Kubernetes.** A Kubernetes shell task whose script had a failing command masked by a later successful one (a trailing `echo`) now fails the task. Local, Docker, and Slurm are unchanged at `false`, and an explicit `fail_fast:` in the task always wins. -5. **`gpus.release_after` and `nodes.release_after` no longer default to `workflow_completion`** — an omitted value is now inferred. A **probe-less** GPU task that used to hold its GPUs for the whole workflow now releases at task completion, so downstream tasks may be packed onto them. An omitted node policy now means *placement only* (may overlap with other planned tasks) rather than exclusive. *Migration:* set `release_after: workflow_completion` explicitly to restore v0.3.0 behavior. -6. **A readiness-probed service that exits before becoming READY is now FAILED, even on exit 0.** It was previously marked COMPLETED, wrongly unblocking dependents against a dead server. sflow forces one final readiness scan first to avoid a false failure. The reason string changes from `process exit` to `service exited before readiness`. All backends. Probe-less tasks, and services that reached READY and then exited cleanly, are unaffected. -7. **Kubernetes `.log` is the streamed log, not a rebuilt one.** A multi-pod task's log is now ordered **chronologically, interleaved across pods**, instead of grouped per pod. `kubectl logs --prefix` still tags every line with its pod. *Migration:* a `result:` pattern that takes the *last* match now takes the last one **in time** across all pods — match on the pod prefix if you need a specific pod's value. -8. **Docker container names changed** — `sflow--` → `sflow-p--`. *Migration:* update any script matching the old name. -9. **Docker GPU tasks now fail fast when the host has no free GPUs**, because reservation is on by default. Foreign workloads above 512 MiB count as busy, so **on a workstation with an attached display every GPU can read busy**. The error names the busy GPUs and the escape hatches. *Migration:* `--wait-for-gpus`, `SFLOW_GPU_IGNORE_FOREIGN=1`, `SFLOW_GPU_BUSY_MEM_MIB`, or `SFLOW_GPU_RESERVATION=0`. -10. **Docker CPU-only containers no longer see GPUs** (`NVIDIA_VISIBLE_DEVICES=void`). Skipped when `extra_args` already grant GPUs. -11. **Slurm no longer exports `NVIDIA_VISIBLE_DEVICES` to srun steps.** Containers see all of the node's GPUs, so NVML consumers (`nvidia-smi`, DCGM) lose device isolation — the trade that makes the planned host-numbered slice addressable at all. Docker keeps isolation via `--gpus device=`. -12. **New Slurm in-step abort `exit 97`** when the planned GPUs cannot be honored — see [Feature 2](#2-slurm-the-gpu-plan-is-re-applied-inside-the-step). -13. **Kubernetes rejects `resources.gpus.indices` at plan time**, with a message pointing at `count`. -14. **Relative inline-content `file://` artifact URIs that escape the run output dir now hard-error.** `output_dir / raw` did not previously collapse `..`, so `file://../../x` silently wrote outside the run directory. Narrowly scoped: relative `file://` **with inline content** only — `fs://` model mounts and absolute `file://` are untouched. -15. **`sflow batch --bulk-input` refuses a `--nodes` that disagrees with the CSV node column.** `--nodes` sizes the sbatch allocation while the config's number sizes the workflow, so two different numbers allocate one size and plan another. Config-driven paths warn instead of refusing. -16. **`sflow batch` rejects a CSV data row with a blank `sflow_config_file`** — previously a raw `AttributeError` traceback, so this is strictly better, but it is a new hard failure. -17. **`timeout:` now warns on every load.** It was never enforced — no code path reads it, and `TaskStatus.TIMEOUT` is never assigned — but v0.3.0 said nothing. It is still accepted so existing recipes keep loading. *Migration:* bound the run with the backend's own limit (Slurm `--time`). The README and user docs have been corrected in this release; they previously implied it worked. - ---- - -## Tested Environments & Current Support - -**Validated setups** - -- vanilla bare-metal Kubernetes -- Google Kubernetes Engine (GKE) -- Slurm (GRES and non-GRES partitions) - -**Current limitations / work in progress** - -- **Kubernetes hardware monitoring is still not supported.** The built-in bare-node `monitor:` feature covers Local, Docker, and Slurm. On Kubernetes, monitor blocks are skipped because there is no DCGM/DaemonSet collector; sampling the driver host would produce misleading data. Unchanged from v0.3.0. -- **`resources.gpus.indices` is not supported on Kubernetes** — the device plugin or DRA assigns physical devices. Use `count`. -- **The GPU reservation registry covers the Docker backend only.** The Local backend also runs on the host but does not participate, so a Local run and a Docker run on the same box can still overlap. -- **DRA GPU allocation is supported but still WIP** — implemented, not yet broadly validated across Kubernetes distributions, versions, and NVIDIA DRA deployments. The device-plugin path remains the default. -- **Kubernetes execution is driver-attached.** Use interactive `sflow run`; detached Kubernetes batch execution is not supported. -- **The GPU reservation registry requires POSIX `fcntl`** and is inert on Windows. - ---- - -## Documentation - -Updated for this release: [Backends](https://nvidia.github.io/nv-sflow/docs/user/backends) (Slurm GPU placement, Docker GPU reservation, `collect_node_local_output`, `mpi.cpu_bind`), [CLI](https://nvidia.github.io/nv-sflow/docs/user/cli) (`sflow upgrade`, `--wait-for-gpus`, `--extra-kubectl-apply-args`, `--skip-artifact-check`, `batch --nodes` conflicts), [Resources](https://nvidia.github.io/nv-sflow/docs/user/resources) (`gpus.indices`), [Monitor](https://nvidia.github.io/nv-sflow/docs/user/monitor) (`window:`, report defaults), [Outputs & logs](https://nvidia.github.io/nv-sflow/docs/user/outputs) (new summary sections, `command_trace.jsonl`, `loop_stalls.txt`), and the new [Understanding run output](https://nvidia.github.io/nv-sflow/docs/user/run-output) page. - -Full documentation: [nvidia.github.io/nv-sflow](https://nvidia.github.io/nv-sflow/) - ---- - -## Upgrade Guide - -```bash -# From an existing install -sflow upgrade - -# Or fresh -uv pip install "sflow @ git+https://github.com/NVIDIA/nv-sflow.git@main" - -# Re-plan every recipe without consuming resources -- this surfaces the two -# breaking changes most likely to bite (gpus.count divisibility, gpus.indices on k8s) -sflow run -f your_recipe.yaml --dry-run - -# New samples worth a look -sflow sample self_contained/slurm/gpu_indices -sflow sample self_contained/slurm/monitor_mixed -sflow sample self_contained/docker/gpu_monitor -``` - -**A five-minute upgrade checklist:** - -1. `--dry-run` every recipe — `gpus.count` divisibility and `gpus.indices`-on-Kubernetes both fail at plan time, before anything is allocated. -2. Grep your tooling for `sflow_monitor//` and insert the `lifecycle/` or `windowed/` segment. -3. If you have a bare `monitor:` on a large fan-out, add `report: {enabled: false}`. -4. If a Kubernetes shell task relies on a trailing command masking an earlier failure, set `fail_fast: false` on it. -5. If a probe-less GPU task needs to hold its GPUs for the whole run, set `release_after: workflow_completion` explicitly. -6. If you parse a multi-pod Kubernetes `.log` for a *last* match, match on the pod prefix. - -Repository: [https://github.com/NVIDIA/nv-sflow](https://github.com/NVIDIA/nv-sflow) diff --git a/docs/user/architecture.md b/docs/user/architecture.md index 6b0b9a6..28f6d35 100644 --- a/docs/user/architecture.md +++ b/docs/user/architecture.md @@ -216,13 +216,12 @@ stateDiagram-v2 | Command | Purpose | Key Options | |---------|---------|-------------| -| **`sflow run`** | Execute a workflow | `--dry-run`, `--tui`, `--bulk-input/--row`, `--set/-s`, `--artifact/-a`, `--missable-tasks/-M`, `--skip-artifact-check`, `--wait-for-gpus`, `--extra-args/-e`, `--output-dir`, `--log-level`, `--enable-workflow-monitor`, `--enable-task-monitor`, `--include-nodes/--exclude-nodes`, `--kubeconfig`, `--kube-context`, `--kube-namespace`, `--kube-node-selector`, `--kube-compute-domain-channel`, `--kube-compute-domain-create`, `--extra-kubectl-args`, `--extra-kubectl-apply-args` | +| **`sflow run`** | Execute a workflow | `--dry-run`, `--tui`, `--bulk-input/--row`, `--set/-s`, `--artifact/-a`, `--missable-tasks/-M`, `--skip-artifact-check`, `--extra-args/-e`, `--output-dir`, `--log-level`, `--enable-workflow-monitor`, `--enable-task-monitor`, `--include-nodes/--exclude-nodes`, `--kubeconfig`, `--kube-context`, `--kube-namespace`, `--kube-node-selector`, `--kube-compute-domain-channel`, `--kube-compute-domain-create`, `--extra-kubectl-args` | | **`sflow batch`** | Generate Slurm sbatch scripts | `--submit`, `--bulk-input` (CSV sweeps), `--bulk-submit` (YAML folders), `--row`, `--nodes`, `--partition`, `--account`, `--time`, `--resolve`, `--skip-artifact-check`, `--sflow-version`, `--enable-workflow-monitor`, `--enable-task-monitor`, `--include-nodes/--exclude-nodes` | | **`sflow compose`** | Merge multiple YAMLs into one | `--resolve`, `--validate`, `--bulk-input`, `--row`, `--missable-tasks/-M`, `-o/--output` | | **`sflow visualize`** | Render DAG as image/mermaid | `--format` (png/svg/pdf/mermaid/dot), `--show-variables`, `--set/-s`, `--artifact/-a`, `--missable-tasks/-M` | | **`sflow sample`** | List/copy example workflows | `--list`, `--force`, `-o/--output` | | **`sflow skill`** | Copy agent skills into project (merges into existing directory) | `--list`, `--force` (overwrite existing files), `-o/--output` | -| **`sflow upgrade`** (alias `sflow update`) | Reinstall sflow in place from git or an index | `--repo`, `--branch`, `--sflow-version`, `--sflow-index-url`, `--sflow-source-path`, `--force`, `--dry-run` | ### Multi-file Input diff --git a/docs/user/backends.md b/docs/user/backends.md index 8f8b365..730f5e0 100644 --- a/docs/user/backends.md +++ b/docs/user/backends.md @@ -174,40 +174,6 @@ sflow batch -f workflow.yaml -e "--gpus-per-node=8" -e "--segment=2" This is useful for quick adjustments or when testing different cluster configurations. ::: -### GPU placement inside the step - -On a GRES partition `slurmstepd` overwrites the `CUDA_VISIBLE_DEVICES` sflow exports, so -every step sees the whole allocation and every rank picks device 0 — several tasks that -were planned onto different GPUs all pile onto the same one. sflow therefore no longer -trusts the inherited value. A prelude sourced inside each Slurm step probes the devices -the step can actually see, looks up the physical UUIDs the driver resolved this task's -plan to (`SFLOW_PLANNED_GPU_UUIDS`), and re-exports the indices those same cards have -*here*. Matching by UUID rather than by index is also what fixes pyxis/enroot containers, -which renumber devices from 0. - -This is **on by default** for any Slurm task with a `resources.gpus` slice. The one thing -that opts out is srun `gpus_per_task`, because Slurm already carves GPUs per rank there. - -- **Audit record.** Each task writes `/sflow_gpus.log` (`sflow_gpus..log` per - node on multi-node tasks): planned indices and UUIDs, the inherited environment, the - visible index→UUID map, and the final selection. It is deliberately not dot-prefixed so - artifact browsers show it. The `GPU Assignment` section of `sflow_summary.log` is built - from it. -- **Hard failure `exit 97`.** If a planned card is not visible at all, the step holds fewer - GPUs than planned, or a planned slot is out of range, the step aborts with exit code 97 - rather than silently running on the wrong device. When Slurm — not sflow — chose the - devices, sflow degrades to index arithmetic instead of failing. -- **Graceful skips.** If `CUDA_VISIBLE_DEVICES` is not a plain comma-separated list of - non-negative integers (for example a workflow variable of that name shadowing it), or the - placement script cannot be staged, sflow warns and falls back to the previous behavior. - -:::note -`NVIDIA_VISIBLE_DEVICES` is no longer exported to srun steps. Containers therefore see all -of the node's GPUs — NVML consumers such as `nvidia-smi` and DCGM lose device isolation — -in exchange for the planned host-numbered slice being addressable at all. The Docker -backend is unaffected: it keeps isolation via `--gpus device=`. -::: - ## Selecting or excluding nodes (all backends) Restrict which cluster nodes a run may use with two backend-agnostic controls that @@ -366,6 +332,11 @@ Two things to know about `SFLOW_WAIT_FOR_GPUS`: Reservation requires POSIX file locking and is inert on Windows. +To see it work on a real GPU host, `examples/gpu_reservation/` ships a runnable demo +(`demo.yaml`) plus two harnesses: `prove.sh` asserts the four guarantees above +(exact pinning, disjoint concurrent tasks, fail-fast, `--wait-for-gpus`) and exits +non-zero if any fails, and `stress.sh` fires many concurrent runs at one GPU pool. + > **Changed behavior.** Container names gained a driver-PID segment > (`sflow--` → `sflow-p--`) so concurrent runs on one > host never collide; scripts matching the old name must be updated. A task that diff --git a/docs/user/cli.md b/docs/user/cli.md index 32a9eba..edb61b0 100644 --- a/docs/user/cli.md +++ b/docs/user/cli.md @@ -196,8 +196,6 @@ Common options: - `--account, -A `: Slurm account (auto-detected if not specified) - `--time `: time limit (e.g., `02:00:00`) - `--nodes, -N `: number of nodes. If omitted, single-job and bulk-submit modes derive it from the config's Slurm backend `nodes` field. Bulk-input mode requires either this flag or a CSV node-count column (`SLURM_NODES`, `NUM_SLURM_NODES`, or `NUM_NODES`). - - **When the two node counts disagree.** `--nodes` sizes the *sbatch allocation*; the config's own number sizes the *workflow inside it* — the backend's node count and the `match_count` of readiness probes. The same number in both is the normal way to run, but two different numbers means the job is allocated one size and the recipe plans for another: probes wait on a node that was never allocated, or the job holds nodes nothing will ever use. `--bulk-input` **rejects** the run outright (the CSV states the size per row, so a disagreement is unambiguously a mistake) and names up to five offending rows; the config-driven single-job and `--bulk-submit` paths **warn**. A `--set` of the node variable counts as the row's value, since it overrides the CSV cell. - `--gpus-per-node, -G `: number of GPUs per node for cluster topology. Config `gpus_per_node` wins when present. Applied to sflow validation and planning only, not as a Slurm directive. Use `-e '--gpus-per-node=N'` for `sflow batch`, or backend `extra_args` for `sflow run`, if your cluster requires the Slurm allocation flag. - `--job-name, -J `: Slurm job name (default: `sflow`) - `--set, -s KEY=VALUE`: override variables (repeatable) diff --git a/docs/user/configuration.md b/docs/user/configuration.md index 870f96f..2cb3816 100644 --- a/docs/user/configuration.md +++ b/docs/user/configuration.md @@ -336,7 +336,7 @@ Beyond `depends_on`, `resources`, `replicas`, and `probes`, a task supports: - `operator` / `backend`: name (or inline override object) of the operator or backend for this task. - `ports`: service ports the task exposes (each with `port` and an optional `name`). -- `timeout`: **accepted but not enforced.** Both the per-task and the workflow `timeout` are parsed and merged, but nothing reads them — a task that sets one runs unbounded, and `sflow run` logs a warning naming every task that does. Bound the run with the backend's own limit instead (Slurm `--time`, a Kubernetes `activeDeadlineSeconds`). Kept so existing recipes keep loading. +- `timeout`: per-task timeout (seconds or a string like `30m`); the workflow also has its own `timeout`. - `fail_fast`: bool, **default depends on the backend** — `true` on Kubernetes (a failed command in a pod should fail the task), `false` on Slurm/local/docker (shell default: only the last command's exit code counts). Leave unset to take the backend default, or set explicitly (`true`/`false`) to override per task. When effective-true, sflow prepends `set -e` to shell-operator scripts so any failed command fails the task. Applies to shell operators only (never `python`, whose script is Python source). - `variables`: task-scoped variables (same format as top-level `variables`). - `retries`: retry policy (`count`, `interval`, `backoff`) for a failed task. diff --git a/docs/user/outputs.md b/docs/user/outputs.md index 799f38f..ad12864 100644 --- a/docs/user/outputs.md +++ b/docs/user/outputs.md @@ -59,20 +59,10 @@ After a successful run, `sflow run` prints the output folder, summary path, and - task duration timeline and task event timeline - probe traces — the last attempt of every readiness/failure probe — when any task defines probes - GPU and node usage charts when resource placement data exists -- **`GPU Assignment`** — per task, the physical GPUs it was planned onto next to the devices it actually saw, read back from the placement record the step wrote. When the backend re-indexes devices inside the container (pyxis/enroot renumbering from 0), the section says so rather than leaving you to reconcile two different numbering schemes. -- **`Node Topology`** — the CPU / NUMA / GPU probe each backend captured at reservation time, printed as one `[backend ]` block per backend. -- **`External Command Health`** — call counts, failures, timeouts, and mean/max latency for the external commands sflow shells out to (`kubectl`, `srun`, `docker`), plus a list of the slow and failed calls sharing the Timeline's `+elapsed` column. Written even when the run is cancelled or fails, which is when it matters most. - command-log paths - workflow DAG and dependency list - failure hints with task name, attempts, reason, and task log path when a task fails or is cancelled -The task event Timeline also carries an `UNGATED` event for members of a merged Kubernetes pod, marking the moment their in-pod gate opened and the task was allowed to start. - -Two companion files are written next to the summary, but only when there is something to say: - -- **`command_trace.jsonl`** — one record per *notable* external command (non-zero exit, or slower than 5s). A healthy run, or a run with no notable external-command calls, leaves no file. This is the machine-readable form of the `External Command Health` section; the pre-existing `*_cmds.log` records *what* was issued, this records *how it went*. -- **`loop_stalls.txt`** — if sflow's own event loop stops being scheduled for 30s, sflow logs a warning and dumps every thread's Python stack here, then logs a recovery line with the worst observed lag. Previously such a freeze produced no diagnostic output at all, because sflow's logging runs on the thread that was stuck. Created only when a stall actually happens. - Example `sflow_summary.log`: ```text diff --git a/docs/user/quick-reference.md b/docs/user/quick-reference.md index fb6cc58..6201fcc 100644 --- a/docs/user/quick-reference.md +++ b/docs/user/quick-reference.md @@ -319,7 +319,7 @@ and `addressing_style` (`auto` / `virtual` / `path`). |-------|----------|------|---------|-------------| | `name` | Yes | string | — | Workflow name. | | `tasks` | Yes | list | — | List of task definitions (must be non-empty). | -| `timeout` | | string / int | `null` | **Not enforced** — accepted and merged, but nothing reads it; a workflow that sets it runs unbounded and `sflow run` warns. Use the backend's own limit (Slurm `--time`). | +| `timeout` | | string / int | `null` | Workflow-level timeout (e.g. `1h`, `115m`). | | `variables` | | dict / list | `null` | Workflow-scoped variables (same format as root `variables`). | | `upload_all` | | object | `null` | Zip the whole workflow output dir and upload it to a `storage` target (see [Workflow Upload-All](#workflow-upload-all)). | | `monitor` | | object | `null` | Workflow-level hardware monitor (see [Monitor](#monitor)). | @@ -336,7 +336,7 @@ and `addressing_style` (`auto` / `virtual` / `path`). | `backend` | | string / dict | `null` | Backend name, or inline backend override. | | `depends_on` | | list[string] | `null` | Names of tasks this task depends on. | | `required_by` | | list[string] | `null` | Reverse dependency: `A required_by: [B]` is folded into `B depends_on: [A]` at load (lets an optional fragment attach to a hub without editing it). | -| `timeout` | | int / string | `null` | **Not enforced** — see [Workflow](#workflow). Bound the task with the backend's own limit instead. | +| `timeout` | | int / string | `null` | Task-level timeout. | | `variables` | | dict / list | `null` | Task-scoped variables. | | `resources` | | object | `null` | Node / GPU resource requirements. | | `replicas` | | object | `null` | Replication configuration. | @@ -358,9 +358,8 @@ and `addressing_style` (`auto` / `virtual` / `path`). | `nodes.indices` | | list[int / expr] | `null` | Specific node indices (e.g. `[0]`). | | `nodes.count` | | int / expr | `null` | Number of nodes. | | `nodes.exclude` | | int / list[int] / expr | `null` | Node indices to remove from the placement pool before `indices`, `count`, or GPU packing. | -| `nodes.release_after` | | string | `null` | When node reservations can be reused: `workflow_completion`, `task_ready`, or `task_completion`. sflow reserves nodes **only when this is set explicitly**; omitted, `nodes.indices`/`count` are non-exclusive placement constraints (see note below). | -| `gpus.count` | One of `count` / `indices` | int / expr | `null` | Number of GPUs (sets `CUDA_VISIBLE_DEVICES`). Index-agnostic on its own: the planner packs the task into any contiguous idle GPU run on one node. | -| `gpus.indices` | One of `count` / `indices` | list[int / expr] | `null` | Pin the task to specific **0-based, non-negative, unique** device ids. Combined with `count`, `count` is the total across nodes and `indices` the per-node slice, so the task fans out over `count / len(indices)` nodes. | +| `nodes.release_after` | | string | `workflow_completion` | When node reservations can be reused: `workflow_completion`, `task_ready`, or `task_completion`. sflow reserves nodes **only when this is set explicitly**; omitted, `nodes.indices`/`count` are non-exclusive placement constraints (see note below). | +| `gpus.count` | If `gpus` is set | int / expr | — | Number of GPUs (sets `CUDA_VISIBLE_DEVICES`). | | `gpus.release_after` | | string | inferred | When GPU reservations can be reused: `workflow_completion`, `task_ready`, or `task_completion`. | For nodes, `release_after` only creates an exclusive node reservation when explicitly set; omitted `nodes.indices` and `nodes.count` are placement constraints and may overlap with other planned tasks. For GPUs, omitted `release_after` is inferred: tasks without readiness probes release GPUs after task completion for downstream dependents, while tasks with readiness probes keep GPUs until workflow completion unless explicitly set to `task_ready`. `task_ready` releases after readiness succeeds. `task_completion` releases after any terminal task status (`COMPLETED`, `FAILED`, `TIMEOUT`, or `CANCELLED`). Dry-run rehearses these resource lifetimes across the DAG. @@ -490,9 +489,7 @@ To publish a metric literally named `file`, use `patterns:`; a top-level `file:` | `interval` | | int | `5000` | Sampling interval (ms) for built-in scopes without their own. | | `scopes` | | object | `null` (all) | Which scopes to collect — `cpu`, `gpu`, `memory`, `disk`, `network`, `custom`; omit ⇒ all built-ins active. | | `resources` | | object | `null` | Which hardware to target — `nodes` / `gpus` (like task resources) or `used_by_tasks: [names]`. | -| `report` | | object | `{enabled: true}` | Post-run report. `format`: any of `csv`, `svg`, `png` (`png` needs the `sflow[monitor]` extra). | -| `report.enabled` | | bool | `true` | **On by default** whenever `monitor:` is set. Set `false` to opt out — worth doing on large fan-outs, since report cost scales with samples × views. Raw samples and the overview are still written. | -| `window` | | object | `null` | `{start, end}` task-log markers bounding the reported window, so the report covers the benchmark rather than the whole task lifecycle. Task monitors only (rejected on `workflow.monitor`) and requires `report.enabled: true`. | +| `report` | | object | `null` (csv + svg) | Opt-in post-run report `format`: any of `csv`, `svg`, `png` (`png` needs the `sflow[monitor]` extra). | ## Expression Syntax diff --git a/docs/user/resources.md b/docs/user/resources.md index 7be9332..95a4adb 100644 --- a/docs/user/resources.md +++ b/docs/user/resources.md @@ -185,15 +185,6 @@ split across two nodes. Write `gpus.count: 2` for one GPU on each of two nodes. This `CUDA_VISIBLE_DEVICES` packing applies to the **local**, **slurm**, and **docker** backends. Kubernetes assigns GPUs differently — see below. -:::note `gpus_per_task` hands device selection to Slurm -Setting `gpus_per_task` on an `srun` operator makes the job step request GRES, so Slurm -carves devices per **rank** and picks which ones. sflow does not re-apply its own slice -there — `resources.gpus` still sizes the request and drives packing and conflict -detection, but the physical devices are Slurm's choice, and run reporting falls back to -showing the planned slice. Leave `gpus_per_task` unset if you want sflow's exact device -pin (`resources.gpus.indices`) to hold. -::: - ### Pin specific GPUs with `indices` Use `resources.gpus.indices` when a task must land on particular device IDs — NUMA/NVLink affinity, reproducing a vendor benchmark topology, or steering around a known-bad device. diff --git a/docs/user/run-output.md b/docs/user/run-output.md index 78fc958..cd7e6ff 100644 --- a/docs/user/run-output.md +++ b/docs/user/run-output.md @@ -48,9 +48,6 @@ annotated example. | Why did readiness never fire? | probe traces in `sflow_summary.log` | [Probes](./probes.md) | | How busy were the GPUs / CPU / network? | `sflow_monitor.log`, `sflow_monitor/` | [Monitor](./monitor.md) | | Which GPUs and nodes did each task get? | usage charts in `sflow_summary.log` | [Resources](./resources.md) | -| Which *physical* GPUs did this task really end up on? | `/sflow_gpus.log`, plus the `GPU Assignment` section of `sflow_summary.log` | [Resources](./resources.md) | -| Was the cluster control plane slow or flaky, rather than my job? | `External Command Health` in `sflow_summary.log`, `command_trace.jsonl` | [Outputs & logs](./outputs.md#execution-summary) | -| sflow itself went unresponsive — what was it doing? | `loop_stalls.txt` | [Outputs & logs](./outputs.md#execution-summary) | | How do I get all this off the cluster? | storage targets + `uploads:` | [Uploads](./uploads.md) | ## The output tree at a glance @@ -61,14 +58,11 @@ annotated example. ├── sflow.log # orchestration + command/status lines (no task stdout) ├── *_cmds.log # launch commands, grouped by family (bash/slurm/docker/ssh/python) ├── results.json # workflow-level metric index (only with `result:`) -├── command_trace.jsonl # slow/failed external commands (only when some call was notable) -├── loop_stalls.txt # all-thread stacks on a driver stall (only when one happened) ├── sflow_monitor.log # resource overview (only with `monitor:`) -├── sflow_monitor/ # raw samples; per-task reports only when `report.enabled: true` (only with `monitor:`) +├── sflow_monitor/ # raw samples + per-task reports (only with `monitor:`) └── / ├── .log # full per-task stdout/stderr ├── result.json # canonical per-task metrics (only with `result:`) - ├── sflow_gpus.log # physical GPU placement record (Slurm steps that pick their own devices) └── ... # anything your scripts write ``` @@ -106,8 +100,8 @@ downstream tasks and external tooling — see [Results](./results.md). ## Beyond the local folder -- [Monitor](./monitor.md) — hardware sampling and reports. Setting `monitor:` writes - raw CSV samples *and* the per-task charts and summaries; reports are on by default - (`report: { enabled: false }` opts out). +- [Monitor](./monitor.md) — hardware sampling and reports. Raw CSV samples are + always written when `monitor:` is set; the per-task charts and summaries need + `report: { enabled: true }`. - [Uploads](./uploads.md) — declare storage targets and per-task `uploads:` to ship files to S3 as each task completes, so partial results survive a cancelled run. diff --git a/docs/user/samples.md b/docs/user/samples.md index c08cd9d..70f3e27 100644 --- a/docs/user/samples.md +++ b/docs/user/samples.md @@ -41,14 +41,10 @@ Two families ship with sflow: | docker | `self_contained/docker/hello_world` | Single container task via `docker_run` | | docker | `self_contained/docker/multi_node` | Multi-host Docker (`docker_host`/`context`) | | docker | `self_contained/docker/sglang_qwen3` | SGLang Qwen3 server + client in containers | -| docker | `self_contained/docker/gpu_monitor` | GPU monitor with a log-marker report `window:` — runs a stock CUDA `nbody` container as a deliberate idle/burst/idle square wave, so the lifecycle report and the windowed report visibly differ | | slurm | `self_contained/slurm/sglang_server_client` | Server + client on Slurm with readiness probes | | slurm | `self_contained/slurm/aiperf_template` | AIPerf benchmark template | | slurm | `self_contained/slurm/auto_replica` | Replica fan-out sized from a variable sweep | | slurm | `self_contained/slurm/resource_release_after` | `release_after` GPU/node lifetimes | -| slurm | `self_contained/slurm/gpu_indices` | `resources.gpus` — `count` vs `indices` vs both, with the 4-node × 4-GPU planner output annotated | -| slurm | `self_contained/slurm/gpu_placement_matrix` | GPU-placement regression matrix: container vs bare step, high slice offset, two tasks sharing a node, multi-node — proves placement by UUID rather than by GPU count | -| slurm | `self_contained/slurm/monitor_mixed` | The broadest single-job regression net: two Slurm pools/operators, replicas with cross-task refs, `release_after` GPU reuse, the placement matrix, and a monitor with a log-marker window. **If you only run one recipe on a new cluster, run this one.** | | slurm | `self_contained/slurm/multi_backend` | One workflow spanning multiple backends | | slurm | `self_contained/slurm/trtllm_serve_disagg` | TRT-LLM disaggregated serving | | slurm | `self_contained/slurm/infmax_v1_ds_r1` | InfMax DeepSeek-R1 benchmark | @@ -1408,7 +1404,7 @@ sflow run -f dynamo_trtllm_disagg.yaml \ Modular samples are folders containing multiple composable YAML files. Instead of one monolithic config, the workflow is split into reusable building blocks. -### Modular inference recipe (inference_x_v2) +### inference_x_v2 A modular inference benchmark setup supporting multiple frameworks (SGLang, vLLM, TensorRT-LLM) with disaggregated prefill/decode servers. diff --git a/examples/modular/inference_x_v2/benchmark_aiperf.yaml b/examples/modular/inference_x_v2/benchmark_aiperf.yaml index c2dd7f5..247ca73 100644 --- a/examples/modular/inference_x_v2/benchmark_aiperf.yaml +++ b/examples/modular/inference_x_v2/benchmark_aiperf.yaml @@ -59,6 +59,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/modular/inference_x_v2/bulk_input.csv b/examples/modular/inference_x_v2/bulk_input.csv index 90037c8..1f9962f 100644 --- a/examples/modular/inference_x_v2/bulk_input.csv +++ b/examples/modular/inference_x_v2/bulk_input.csv @@ -2,9 +2,9 @@ sflow_config_file,SLURM_NODES,GPUS_PER_NODE,NUM_CTX_SERVERS,CTX_TP_SIZE,NUM_GEN_ slurm_config.yaml common_workflow.yaml trtllm/prefill.yaml trtllm/decode.yaml benchmark_infmax.yaml,1,4,1,2,1,2,,,1,[16],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2,agg_server slurm_config.yaml common_workflow.yaml trtllm/prefill.yaml trtllm/decode.yaml benchmark_infmax.yaml,2,4,4,1,1,4,,,2,[32],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2,agg_server slurm_config.yaml common_workflow.yaml sglang/prefill.yaml sglang/decode.yaml benchmark_infmax.yaml,1,4,2,1,1,2,,,1,"[1,4]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,lmsysorg/sglang:v0.5.8.post1-cu130,agg_server -slurm_config.yaml common_workflow.yaml sglang/prefill.yaml sglang/decode.yaml benchmark_infmax.yaml,2,4,2,2,1,4,,,2,"[2,4,6]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0,agg_server +slurm_config.yaml common_workflow.yaml sglang/prefill.yaml sglang/decode.yaml benchmark_infmax.yaml,2,4,2,2,1,4,,,2,"[2,4,6]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0,agg_server slurm_config.yaml common_workflow.yaml vllm/prefill.yaml vllm/decode.yaml benchmark_infmax.yaml,2,4,2,1,3,2,,,1,[8],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0,agg_server slurm_config.yaml common_workflow.yaml vllm/prefill.yaml vllm/decode.yaml benchmark_infmax.yaml,3,4,4,1,1,8,,,3,"[64,32]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0,agg_server slurm_config.yaml common_workflow.yaml trtllm/agg.yaml benchmark_aiperf.yaml,1,4,,,,,1,4,1,"[1,3,5]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2,prefill_server decode_server -slurm_config.yaml common_workflow.yaml sglang/agg.yaml benchmark_aiperf.yaml,2,4,,,,,8,1,2,[10],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0,prefill_server decode_server +slurm_config.yaml common_workflow.yaml sglang/agg.yaml benchmark_aiperf.yaml,2,4,,,,,1,8,2,[10],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0,prefill_server decode_server slurm_config.yaml common_workflow.yaml vllm/agg.yaml benchmark_aiperf.yaml,1,4,,,,,2,2,1,[128],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0,prefill_server decode_server \ No newline at end of file diff --git a/examples/modular/inference_x_v2/common_workflow.yaml b/examples/modular/inference_x_v2/common_workflow.yaml index 6cf5b79..bb51329 100644 --- a/examples/modular/inference_x_v2/common_workflow.yaml +++ b/examples/modular/inference_x_v2/common_workflow.yaml @@ -54,7 +54,7 @@ variables: # Dependencies versions DYNAMO_VERSION: description: "Dynamo version to install when missing from the container image" - value: 1.3.0 + value: 0.8.0 NATS_VERSION: description: "NATS server version to install when missing" value: v2.10.28 diff --git a/examples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml b/examples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml index 5c770c1..2d18141 100644 --- a/examples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml +++ b/examples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml @@ -10,7 +10,7 @@ variables: - name: NUM_AGG_SERVERS description: Number of aggregated servers type: integer - value: 8 + value: 1 - name: CONCURRENCY description: Concurrency @@ -42,7 +42,7 @@ operators: container_writable: true mpi: pmix extra_args: - - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 + - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 - name: nginx type: srun @@ -101,8 +101,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 + echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 else echo "dynamo already installed." fi @@ -324,17 +324,17 @@ workflow: - export NODE_RANK=${SLURM_NODEID} - export FIRST_NODE_IP=$(echo ${SFLOW_TASK_ASSIGNED_NODE_IPS} | cut -d',' -f1) - | - if [ "1" -gt 1 ]; then - export MULTI_NODE_EXTRA_ARGS="--dist-init-addr ${FIRST_NODE_IP}:29500 --nnodes 1 --node-rank ${NODE_RANK}" + if [ "2" -gt 1 ]; then + export MULTI_NODE_EXTRA_ARGS="--dist-init-addr ${FIRST_NODE_IP}:29500 --nnodes 2 --node-rank ${NODE_RANK}" else export MULTI_NODE_EXTRA_ARGS="" fi - export VLLM_NIXL_SIDE_CHANNEL_PORT=$((5557 + ${FIRST_CUDA_DEVICE})) - export DYN_SYSTEM_PORT=$((8082 + ${FIRST_CUDA_DEVICE})) - - python3 -m dynamo.sglang --model-path ${{ artifacts.LOCAL_MODEL_PATH.path }} --served-model-name Qwen3-8B-FP8 --tensor-parallel-size 1 --pipeline-parallel-size 1 --data-parallel-size 1 --expert-parallel-size 1 --moe-dense-tp-size 1 --max-running-requests 128 --max-prefill-tokens 4096 --trust-remote-code --skip-tokenizer-init --cuda-graph-max-bs 128 ${MULTI_NODE_EXTRA_ARGS} --host 0.0.0.0 + - python3 -m dynamo.sglang --model-path ${{ artifacts.LOCAL_MODEL_PATH.path }} --served-model-name Qwen3-8B-FP8 --tensor-parallel-size 8 --pipeline-parallel-size 1 --data-parallel-size 1 --expert-parallel-size 1 --moe-dense-tp-size 1 --max-running-requests 128 --max-prefill-tokens 4096 --trust-remote-code --skip-tokenizer-init --cuda-graph-max-bs 128 ${MULTI_NODE_EXTRA_ARGS} --host 0.0.0.0 resources: gpus: - count: 1 + count: 8 depends_on: - frontend_server probes: @@ -355,7 +355,7 @@ workflow: - set -x - pip install aiperf==0.3.0 - export COLUMNS=200 - - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' + - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' - echo "Benchmarking finished" resources: nodes: diff --git a/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml b/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml index 61466d5..6d0b599 100644 --- a/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml +++ b/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml @@ -101,8 +101,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 + echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 else echo "dynamo already installed." fi diff --git a/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml b/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml index d5ef927..348a61b 100644 --- a/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml +++ b/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml @@ -49,7 +49,7 @@ operators: container_writable: true mpi: pmix extra_args: - - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 + - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 - name: nginx type: srun @@ -102,8 +102,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 + echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 else echo "dynamo already installed." fi diff --git a/examples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml b/examples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml index 5c4ff88..610c420 100644 --- a/examples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml +++ b/examples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml @@ -123,8 +123,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 + echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 else echo "dynamo already installed." fi @@ -378,7 +378,7 @@ workflow: - set -x - pip install aiperf==0.3.0 - export COLUMNS=200 - - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' + - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' - echo "Benchmarking finished" resources: nodes: diff --git a/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml b/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml index e8b8fea..bce8623 100644 --- a/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml +++ b/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml @@ -144,8 +144,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 + echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 else echo "dynamo already installed." fi @@ -390,6 +390,10 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 - name: decode_server operator: @@ -428,6 +432,10 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 - name: benchmark_infmax operator: diff --git a/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml b/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml index 6d054b7..4117bc9 100644 --- a/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml +++ b/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml @@ -144,8 +144,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 + echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 else echo "dynamo already installed." fi @@ -390,6 +390,10 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 - name: decode_server operator: @@ -428,6 +432,10 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 - name: benchmark_infmax operator: diff --git a/examples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml b/examples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml index 4319237..4be77a1 100644 --- a/examples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml +++ b/examples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml @@ -393,7 +393,7 @@ workflow: - set -x - pip install aiperf==0.3.0 - export COLUMNS=200 - - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' + - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' - echo "Benchmarking finished" resources: nodes: diff --git a/examples/modular/inference_x_v2/sglang/agg.yaml b/examples/modular/inference_x_v2/sglang/agg.yaml index 36efd83..06ddc8f 100644 --- a/examples/modular/inference_x_v2/sglang/agg.yaml +++ b/examples/modular/inference_x_v2/sglang/agg.yaml @@ -1,7 +1,7 @@ # SGLang aggregated (non-disaggregated) server task and its configuration. # Merge all files for a complete run: # sflow compose slurm_config.yaml common_workflow.yaml sglang/agg.yaml benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 \ +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 \ # --missable-tasks prefill_server --missable-tasks decode_server version: "0.1" diff --git a/examples/modular/inference_x_v2/sglang/decode.yaml b/examples/modular/inference_x_v2/sglang/decode.yaml index 4ee8ffe..3cc22e6 100644 --- a/examples/modular/inference_x_v2/sglang/decode.yaml +++ b/examples/modular/inference_x_v2/sglang/decode.yaml @@ -1,7 +1,7 @@ # SGLang decode server task and its configuration. # Merge all files for a complete run: # sflow run -f slurm_config.yaml -f common_workflow.yaml -f sglang_prefill.yaml -f sglang_decode.yaml -f benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 version: "0.1" diff --git a/examples/modular/inference_x_v2/sglang/prefill.yaml b/examples/modular/inference_x_v2/sglang/prefill.yaml index 828434d..62462e7 100644 --- a/examples/modular/inference_x_v2/sglang/prefill.yaml +++ b/examples/modular/inference_x_v2/sglang/prefill.yaml @@ -1,7 +1,7 @@ # SGLang prefill server task and its configuration. # Merge all files for a complete run: # sflow run -f slurm_config.yaml -f common_workflow.yaml -f sglang_prefill.yaml -f sglang_decode.yaml -f benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 version: "0.1" diff --git a/examples/modular/inference_x_v2/trtllm/agg.yaml b/examples/modular/inference_x_v2/trtllm/agg.yaml index 198f54e..b79ad63 100644 --- a/examples/modular/inference_x_v2/trtllm/agg.yaml +++ b/examples/modular/inference_x_v2/trtllm/agg.yaml @@ -1,7 +1,7 @@ # TensorRT-LLM aggregated (non-disaggregated) server task and its configuration. # Merge all files for a complete run: # sflow compose slurm_config.yaml common_workflow.yaml trtllm/agg.yaml benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 \ +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 \ # --missable-tasks prefill_server --missable-tasks decode_server version: "0.1" diff --git a/examples/modular/inference_x_v2/trtllm/decode.yaml b/examples/modular/inference_x_v2/trtllm/decode.yaml index 04e72dd..049e1bf 100644 --- a/examples/modular/inference_x_v2/trtllm/decode.yaml +++ b/examples/modular/inference_x_v2/trtllm/decode.yaml @@ -126,3 +126,7 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 diff --git a/examples/modular/inference_x_v2/trtllm/prefill.yaml b/examples/modular/inference_x_v2/trtllm/prefill.yaml index bc9bd33..74e62fe 100644 --- a/examples/modular/inference_x_v2/trtllm/prefill.yaml +++ b/examples/modular/inference_x_v2/trtllm/prefill.yaml @@ -124,3 +124,7 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 diff --git a/examples/self_contained/kubernetes/dynamo_sglang_agg.yaml b/examples/self_contained/kubernetes/dynamo_sglang_agg.yaml index 2c2044c..ef782fd 100644 --- a/examples/self_contained/kubernetes/dynamo_sglang_agg.yaml +++ b/examples/self_contained/kubernetes/dynamo_sglang_agg.yaml @@ -433,6 +433,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/kubernetes/dynamo_sglang_disagg.yaml b/examples/self_contained/kubernetes/dynamo_sglang_disagg.yaml index 40d6b4d..997852c 100644 --- a/examples/self_contained/kubernetes/dynamo_sglang_disagg.yaml +++ b/examples/self_contained/kubernetes/dynamo_sglang_disagg.yaml @@ -695,6 +695,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml b/examples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml index f925aea..2b3c995 100644 --- a/examples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml +++ b/examples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml @@ -719,6 +719,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/kubernetes/dynamo_trtllm_agg.yaml b/examples/self_contained/kubernetes/dynamo_trtllm_agg.yaml index 182117d..85c7416 100644 --- a/examples/self_contained/kubernetes/dynamo_trtllm_agg.yaml +++ b/examples/self_contained/kubernetes/dynamo_trtllm_agg.yaml @@ -383,6 +383,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml b/examples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml index 54cbdc7..4f5fccc 100644 --- a/examples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml +++ b/examples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml @@ -523,6 +523,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/kubernetes/dynamo_vllm_agg.yaml b/examples/self_contained/kubernetes/dynamo_vllm_agg.yaml index 520fd02..a4ed1a0 100644 --- a/examples/self_contained/kubernetes/dynamo_vllm_agg.yaml +++ b/examples/self_contained/kubernetes/dynamo_vllm_agg.yaml @@ -468,6 +468,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/kubernetes/dynamo_vllm_disagg.yaml b/examples/self_contained/kubernetes/dynamo_vllm_disagg.yaml index d363258..02d96d2 100644 --- a/examples/self_contained/kubernetes/dynamo_vllm_disagg.yaml +++ b/examples/self_contained/kubernetes/dynamo_vllm_disagg.yaml @@ -668,6 +668,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/aiperf_template.yaml b/examples/self_contained/slurm/aiperf_template.yaml index 13828ff..92087bc 100644 --- a/examples/self_contained/slurm/aiperf_template.yaml +++ b/examples/self_contained/slurm/aiperf_template.yaml @@ -109,6 +109,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/dynamo_sglang_agg.yaml b/examples/self_contained/slurm/dynamo_sglang_agg.yaml index fab64dd..5800360 100644 --- a/examples/self_contained/slurm/dynamo_sglang_agg.yaml +++ b/examples/self_contained/slurm/dynamo_sglang_agg.yaml @@ -115,7 +115,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 + value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -307,6 +307,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/dynamo_sglang_disagg.yaml b/examples/self_contained/slurm/dynamo_sglang_disagg.yaml index ca16e44..e551f75 100644 --- a/examples/self_contained/slurm/dynamo_sglang_disagg.yaml +++ b/examples/self_contained/slurm/dynamo_sglang_disagg.yaml @@ -170,7 +170,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 + value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -424,6 +424,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/dynamo_trtllm_agg.yaml b/examples/self_contained/slurm/dynamo_trtllm_agg.yaml index 000c9c6..771b71b 100644 --- a/examples/self_contained/slurm/dynamo_trtllm_agg.yaml +++ b/examples/self_contained/slurm/dynamo_trtllm_agg.yaml @@ -111,7 +111,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 + value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -321,6 +321,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/dynamo_trtllm_disagg.yaml b/examples/self_contained/slurm/dynamo_trtllm_disagg.yaml index 3e539a8..1db803e 100644 --- a/examples/self_contained/slurm/dynamo_trtllm_disagg.yaml +++ b/examples/self_contained/slurm/dynamo_trtllm_disagg.yaml @@ -154,7 +154,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 + value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -372,6 +372,10 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 - name: decode_server operator: @@ -415,6 +419,10 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 - name: benchmark operator: @@ -439,6 +447,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/dynamo_vllm_agg.yaml b/examples/self_contained/slurm/dynamo_vllm_agg.yaml index 56f337e..fafb786 100644 --- a/examples/self_contained/slurm/dynamo_vllm_agg.yaml +++ b/examples/self_contained/slurm/dynamo_vllm_agg.yaml @@ -337,6 +337,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/dynamo_vllm_disagg.yaml b/examples/self_contained/slurm/dynamo_vllm_disagg.yaml index 4eeadcf..3f6c8ce 100644 --- a/examples/self_contained/slurm/dynamo_vllm_disagg.yaml +++ b/examples/self_contained/slurm/dynamo_vllm_disagg.yaml @@ -304,16 +304,6 @@ workflow: - set -x - echo ${CUDA_VISIBLE_DEVICES} - export FIRST_CUDA_DEVICE=$(echo ${CUDA_VISIBLE_DEVICES} | cut -d',' -f1) - # Port derived from the first visible device, so two workers sharing a - # node never collide. This works because sflow does NOT hand - # NVIDIA_VISIBLE_DEVICES to an srun step: the container therefore sees the - # node's GPUs with HOST numbering intact, and the first entry of - # CUDA_VISIBLE_DEVICES is a real host index -- unique per worker. - # - # Keep it this way deliberately. When NVIDIA_VISIBLE_DEVICES was exported, - # the runtime carved and renumbered every worker's devices from 0, so both - # prefill and decode computed 5557 and the second to bind died with - # "Address already in use". This line is the canary that found it. - export VLLM_NIXL_SIDE_CHANNEL_PORT=$((5557 + ${FIRST_CUDA_DEVICE})) - export NODE_RANK=${SLURM_NODEID} - export FIRST_NODE_IP=$(echo ${SFLOW_TASK_ASSIGNED_NODE_IPS} | cut -d',' -f1) @@ -391,16 +381,6 @@ workflow: - set -x - echo ${CUDA_VISIBLE_DEVICES} - export FIRST_CUDA_DEVICE=$(echo ${CUDA_VISIBLE_DEVICES} | cut -d',' -f1) - # Port derived from the first visible device, so two workers sharing a - # node never collide. This works because sflow does NOT hand - # NVIDIA_VISIBLE_DEVICES to an srun step: the container therefore sees the - # node's GPUs with HOST numbering intact, and the first entry of - # CUDA_VISIBLE_DEVICES is a real host index -- unique per worker. - # - # Keep it this way deliberately. When NVIDIA_VISIBLE_DEVICES was exported, - # the runtime carved and renumbered every worker's devices from 0, so both - # prefill and decode computed 5557 and the second to bind died with - # "Address already in use". This line is the canary that found it. - export VLLM_NIXL_SIDE_CHANNEL_PORT=$((5557 + ${FIRST_CUDA_DEVICE})) - export NODE_RANK=${SLURM_NODEID} - export FIRST_NODE_IP=$(echo ${SFLOW_TASK_ASSIGNED_NODE_IPS} | cut -d',' -f1) @@ -489,6 +469,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/gpu_placement_matrix.yaml b/examples/self_contained/slurm/gpu_placement_matrix.yaml deleted file mode 100644 index 0ea0493..0000000 --- a/examples/self_contained/slurm/gpu_placement_matrix.yaml +++ /dev/null @@ -1,366 +0,0 @@ -# GPU placement regression matrix: every combination that can break the slice. -# -# sflow plans a per-task CUDA_VISIBLE_DEVICES and re-applies it inside the step. -# Four things independently decide what a task ends up holding: -# -# * container or not -- pyxis/enroot pass through only the task's devices and -# RENUMBER them from 0, so a task planned for 2,3 sees -# them as 0,1. A bare srun step sees the whole node. -# * slice offset -- a plan starting at 0 survives renumbering by accident, -# which is why only the high-offset task ever broke. -# * concurrency -- two tasks sharing a node must land on disjoint GPUs. -# * node count -- a multi-node plan is ONE node's slice, applied -# independently on every node. sflow guarantees the slice -# is uniform (gpus.count is divided by the node count and -# every node takes the same offsets), so the same flat -# plan must resolve correctly in each node's own step. -# -# Counting GPUs is not enough: a task can hold the right NUMBER of the WRONG -# devices and look perfectly healthy. So the first task records EACH node's -# index -> UUID map on bare metal, before anything is carved, and every later task -# proves by UUID that it holds the physical GPUs it was planned for -- per node, -# because a multi-node task's nodes each have their own map. -# -# Asserting rather than echoing is the point. The recipes that only printed their -# devices passed while holding nothing at all -- a decode server planned for 2,3 -# inside a 2-GPU container numbered 0,1 saw no GPU, and the first CUDA call died -# reporting an unsupported architecture. -version: "0.1" - -variables: - SLURM_ACCOUNT: - description: "SLURM account" - value: your_slurm_account - SLURM_PARTITION: - description: "SLURM partition" - value: your_slurm_partition - SLURM_TIMELIMIT: - description: "SLURM time limit" - value: "00:10:00" - SLURM_NODES: - description: >- - Number of Slurm nodes. Two, so the multi-node cases are real: a per-node - slice that is right on node 0 and wrong on node 1 is invisible at one node, - and that is the shape every multi-node server recipe uses. - # Typed, because boxed_multinode_high multiplies it. Untyped it is a STRING, - # and `${{ variables.SLURM_NODES * 2 }}` repeats it into "22" rather than - # doubling it -- valid YAML, valid expression, wrong number. - type: integer - value: 2 - GPUS_PER_NODE: - description: "GPUs per node" - value: 4 - PLACEMENT_IMAGE: - description: "Any GPU-enabled image; nvidia-smi is injected by the runtime." - value: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda12.5.0 - -artifacts: - # Declared so sample/bulk tests can override the standard sample artifact. - - name: LOCAL_MODEL_PATH - uri: fs:///path/to/your/model - - # One assertion, mounted at the same path on every node, instead of the same - # forty lines of shell pasted into each task. EXPECT_HOST is the only input. - - name: ASSERT_PLACEMENT - uri: file://assert_placement.sh - content: | - # Prove by UUID that THIS rank holds the physical GPUs it was planned for. - # Reads EXPECT_HOST (planned HOST indices, comma separated) from the env. - set -u - me=$(basename "$SFLOW_TASK_OUTPUT_DIR") - run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") - # Per NODE, not per task: every rank of a multi-node task runs this body, and - # each node has its own index -> UUID map. Reading node 0's map on node 1 - # would compare against the wrong hardware and "pass" for the wrong reason. - node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" - map=$(ls "$run_dir"/node_gpu_map*/map."$node".txt 2>/dev/null | head -1) - - echo "SFLOW_GPU_PROBE task=$me node=$node planned_host=$EXPECT_HOST CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" - nvidia-smi -L | sed "s/^/SFLOW_GPU_PROBE $node visible: /" - - if [ -z "$map" ]; then - echo "FAIL: no index -> UUID map recorded for node $node" >&2 - exit 1 - fi - # Are the planned ordinals even addressable in THIS namespace? A carved - # container renumbers from 0, so host ordinals can name nothing at all. - # An UNSET CUDA_VISIBLE_DEVICES is legal and means "every visible device", - # so `-i` must be omitted rather than passed empty -- and it must not be - # dereferenced bare under `set -u`. - if ! got=$(nvidia-smi ${CUDA_VISIBLE_DEVICES:+-i "$CUDA_VISIBLE_DEVICES"} \ - --query-gpu=uuid --format=csv,noheader); then - echo "FAIL: CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-} is not addressable on $node" >&2 - exit 1 - fi - # Per-node filenames: ranks of a multi-node task share one output dir, so a - # single gpus.txt would have them overwriting each other's evidence. - printf '%s\n' "$got" | sort > "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" - # And are they the RIGHT ones? Resolve the planned HOST indices through this - # node's bare-metal map; holding the correct count of the wrong devices is - # the failure mode a count-only check cannot see. - for i in $(echo "$EXPECT_HOST" | tr ',' ' '); do - grep "^$i," "$map" | cut -d, -f2 - done | sort > "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" - if ! diff -q "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" \ - "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" >/dev/null; then - echo "FAIL: $node planned host GPU(s) $EXPECT_HOST, but holds different devices" >&2 - echo " want: $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt")" >&2 - echo " got : $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt")" >&2 - exit 1 - fi - echo "OK: $node holds host GPU(s) $EXPECT_HOST, confirmed by UUID" - -backends: - - name: slurm_cluster - type: slurm - default: true - account: ${{ variables.SLURM_ACCOUNT }} - partition: ${{ variables.SLURM_PARTITION }} - time: ${{ variables.SLURM_TIMELIMIT }} - nodes: ${{ variables.SLURM_NODES }} - gpus_per_node: ${{ variables.GPUS_PER_NODE }} - -operators: - # One rank per node throughout: the assertion describes what a NODE holds, and - # extra ranks would each re-run it against the same devices for no new signal. - - name: bare - type: srun - ntasks_per_node: 1 - - name: boxed - type: srun - container_image: ${{ variables.PLACEMENT_IMAGE }} - ntasks_per_node: 1 - -workflow: - name: gpu_placement_matrix - tasks: - # Ground truth, taken bare metal with no `resources.gpus` so nothing is carved - # and no placement prelude runs. Runs on EVERY node (no `resources.nodes`), and - # each rank writes its own file, so the multi-node assertions below have a map - # for the node they actually landed on. Every assertion is relative to this. - - name: node_gpu_map - operator: bare - script: - - env | grep SLURM - - | - set -u - node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" - nvidia-smi --query-gpu=index,uuid --format=csv,noheader | tr -d ' ' \ - > "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" - sed "s/^/SFLOW_GPU_PROBE node_map $node: /" "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" - - # --- bare srun: the step sees the whole node, so the planned ordinals must - # --- survive untouched. Both run at once, on disjoint halves. - - name: bare_low - operator: bare - depends_on: [node_gpu_map] - resources: - gpus: - indices: [0, 1] - script: - - export EXPECT_HOST="0,1" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - - name: bare_high - operator: bare - depends_on: [node_gpu_map] - resources: - gpus: - indices: [2, 3] - script: - - export EXPECT_HOST="2,3" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # --- containerised: the runtime already carved and renumbered, so re-applying - # --- host ordinals here is what broke. boxed_high is the regression case. - - name: boxed_low - operator: boxed - depends_on: [bare_low, bare_high] - resources: - gpus: - indices: [0, 1] - script: - - export EXPECT_HOST="0,1" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - - name: boxed_high - operator: boxed - depends_on: [bare_low, bare_high] - resources: - gpus: - indices: [2, 3] - script: - - export EXPECT_HOST="2,3" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # A single GPU at a high offset: renumbering turns 3 into 0, so an off-by-one - # in the remap shows up here and nowhere else. - - name: boxed_one_high - operator: boxed - depends_on: [boxed_low, boxed_high] - resources: - gpus: - indices: [3] - script: - - export EXPECT_HOST="3" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # --- MULTI-NODE. `gpus.count` is the TOTAL over the task's nodes and every - # --- node takes the SAME offsets, so count/nodes must divide evenly. One flat - # --- plan therefore has to resolve correctly in each node's own step -- and - # --- each node proves it against its OWN bare-metal map, so a slice that is - # --- right on node 0 and wrong on node 1 fails here instead of shipping. - - name: bare_multinode - operator: bare - depends_on: [boxed_one_high] - resources: - # `nodes.count` is what forces the spread: `gpus.count` alone would fit on - # one node and the planner would place it there. count/nodes = 1 GPU per - # node, so the plan is "0" and both nodes must resolve it against their own - # hardware. This is the divide-the-total path every server recipe uses. - nodes: - count: ${{ variables.SLURM_NODES }} - gpus: - count: ${{ variables.SLURM_NODES }} - script: - - export EXPECT_HOST="0" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # The multi-node case that matters: containerised AND at a high offset, so - # every renumbering hazard is in play on more than one node at once. Pinned - # indices repeat on each node, which is the other multi-node planner path. - - name: boxed_multinode_high - operator: boxed - depends_on: [bare_multinode] - resources: - gpus: - # A TOTAL instead of a topology. With no `nodes:` here, `indices` is - # the per-node slice and the planner spreads the task over - # count / len(indices) nodes -- 4 / 2 = 2. This is the one route where - # the node count is an OUTPUT of the GPU ask rather than an input, which - # is how a recipe states things when it knows how many GPUs it needs but - # not how they are laid out. - count: ${{ variables.SLURM_NODES * 2 }} # 2 = len(indices) below - indices: [2, 3] - script: - - export EXPECT_HOST="2,3" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # --- AUTO-PLANNED (`gpus.count`, no `indices`). Everything above pins the - # --- devices by hand, so it proves the RE-APPLY path but never the PLANNER. - # --- These let sflow choose, which is what every real recipe does: the task - # --- cannot state an expected slice up front, so it asserts against the plan - # --- the prelude exported ($SFLOW_GPU_PLAN) -- still resolved to UUIDs - # --- through this node's bare-metal map, so "the planner picked something" - # --- and "the step actually holds it" remain separate claims. - # --- - # --- Chained rather than concurrent: 2x2 + 4x1 + 8 would need 16 GPUs and the - # --- allocation has 8. - - # Two replicas of two GPUs: the packing case. The planner must hand replica 0 - # and replica 1 DISJOINT pairs on the same node, which is the collision the - # whole slice mechanism exists to prevent. - - name: bare_count_2x2 - operator: bare - depends_on: [boxed_multinode_high] - replicas: - count: 2 - policy: parallel - resources: - gpus: - count: 2 - script: - - export EXPECT_HOST="$SFLOW_GPU_PLAN" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - - name: boxed_count_2x2 - operator: boxed - depends_on: [bare_count_2x2] - replicas: - count: 2 - policy: parallel - resources: - gpus: - count: 2 - script: - - export EXPECT_HOST="$SFLOW_GPU_PLAN" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # One GPU each, four ways: the finest packing, and the shape most likely to - # hand two replicas the same device if the planner ever miscounts. - - name: bare_count_1x4 - operator: bare - depends_on: [boxed_count_2x2] - replicas: - count: 4 - policy: parallel - resources: - gpus: - count: 1 - script: - - export EXPECT_HOST="$SFLOW_GPU_PLAN" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - - name: boxed_count_1x4 - operator: boxed - depends_on: [bare_count_1x4] - replicas: - count: 4 - policy: parallel - resources: - gpus: - count: 1 - script: - - export EXPECT_HOST="$SFLOW_GPU_PLAN" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # More GPUs than one node has, with no `nodes:` hint: the planner has to - # spread it itself and divide the total evenly, so every node runs the same - # flat plan against different physical cards. - - name: bare_count_8 - operator: bare - depends_on: [boxed_count_1x4] - resources: - gpus: - count: 8 - script: - - export EXPECT_HOST="$SFLOW_GPU_PLAN" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - - name: boxed_count_8 - operator: boxed - depends_on: [bare_count_8] - resources: - gpus: - count: 8 - script: - - export EXPECT_HOST="$SFLOW_GPU_PLAN" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # Concurrency: boxed_low and boxed_high held the node at the same time, so - # their devices must be disjoint. Sharing one is the collision this placement - # logic exists to prevent, and it is invisible from inside either task. - - name: verify_disjoint - operator: bare - # Terminal on purpose: the e2e checker treats this task's OK line as proof - # that the WHOLE matrix ran, so it has to sit downstream of the last case. - depends_on: [boxed_count_8] - resources: - nodes: - indices: [0] - script: - - | - set -u - run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") - low=$(ls "$run_dir"/boxed_low*/gpus.*.txt 2>/dev/null | head -1) - high=$(ls "$run_dir"/boxed_high*/gpus.*.txt 2>/dev/null | head -1) - if [ -z "$low" ] || [ -z "$high" ]; then - echo "FAIL: concurrent tasks left no gpus.*.txt to compare" >&2 - exit 1 - fi - shared=$(sort "$low" "$high" | uniq -d) - if [ -n "$shared" ]; then - echo "FAIL: concurrent tasks shared GPU(s): $shared" >&2 - exit 1 - fi - echo "OK: concurrent tasks held disjoint GPUs" diff --git a/examples/self_contained/slurm/infmax_v1_ds_r1.yaml b/examples/self_contained/slurm/infmax_v1_ds_r1.yaml index c5c8dd5..c235712 100644 --- a/examples/self_contained/slurm/infmax_v1_ds_r1.yaml +++ b/examples/self_contained/slurm/infmax_v1_ds_r1.yaml @@ -140,7 +140,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 + value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 artifacts: # file type artifacts will be generated by sflow with the content defined in this yaml file, accessible as `${{ artifacts.NAME.path }}` in task scripts @@ -420,6 +420,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/monitor_mixed.yaml b/examples/self_contained/slurm/monitor_mixed.yaml index d16bcf0..5e4b000 100644 --- a/examples/self_contained/slurm/monitor_mixed.yaml +++ b/examples/self_contained/slurm/monitor_mixed.yaml @@ -2,24 +2,13 @@ version: "0.1" # All-in-one Slurm sample: one `sflow run` exercises several features at once, so # a single job on a real cluster verifies them together instead of one at a time. -# This is the broadest regression net in examples/ -- if only one recipe gets run -# on a new cluster, run this one. # -# Mixes four existing samples and adds monitoring: +# Mixes three existing samples and adds monitoring: # multi_backend.yaml -> two Slurm pools, two operators/images # auto_replica.yaml -> replicas + cross-task `${{ task.x[i]... }}` access # resource_release_after.yaml-> readiness probes + GPU reuse (release_after) -# gpu_placement_matrix.yaml -> GPU placement PROVEN by UUID: container vs bare, -# slice offset, concurrency, and multi-node # NEW -> monitor with a log-marker report window # -# The placement stage runs first and releases its GPUs before the servers start, so -# it adds assertions without adding nodes. It is the only part here that can catch a -# task holding the right NUMBER of the WRONG GPUs -- everything below merely echoes -# CUDA_VISIBLE_DEVICES, which is how a decode server planned for 2,3 passed while -# sitting in a 2-GPU container numbered 0,1 with no usable device at all. -# gpu_placement_matrix.yaml keeps the fuller matrix as a standalone 2-node recipe. -# # What to check after a run (//sflow_monitor/): # # MARKER-CLIPPED -- sflow_monitor/windowed/ (each with window.json): @@ -107,54 +96,6 @@ artifacts: - name: LOCAL_MODEL_PATH uri: fs:///path/to/your/model - # Placement proof, mounted at the same path on every node. Kept in a file:// - # artifact rather than pasted into each task: EXPECT_HOST is the only input, so - # one copy serves every case below. - - name: ASSERT_PLACEMENT - uri: file://assert_placement.sh - content: | - # Prove by UUID that THIS rank holds the physical GPUs it was planned for. - # Reads EXPECT_HOST (planned HOST indices, comma separated) from the env. - set -u - me=$(basename "$SFLOW_TASK_OUTPUT_DIR") - run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") - # Per NODE, not per task: every rank of a multi-node task runs this body, and - # each node has its own index -> UUID map. Reading node 0's map on node 1 - # would compare against the wrong hardware and "pass" for the wrong reason. - node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" - map=$(ls "$run_dir"/node_gpu_map*/map."$node".txt 2>/dev/null | head -1) - - echo "SFLOW_GPU_PROBE task=$me node=$node planned_host=$EXPECT_HOST CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" - nvidia-smi -L | sed "s/^/SFLOW_GPU_PROBE $node visible: /" - - if [ -z "$map" ]; then - echo "FAIL: no index -> UUID map recorded for node $node" >&2 - exit 1 - fi - # Are the planned ordinals even addressable in THIS namespace? A carved - # container renumbers from 0, so host ordinals can name nothing at all. - if ! got=$(nvidia-smi -i "$CUDA_VISIBLE_DEVICES" --query-gpu=uuid --format=csv,noheader); then - echo "FAIL: CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES is not addressable on $node" >&2 - exit 1 - fi - # Per-node filenames: ranks of a multi-node task share one output dir, so a - # single gpus.txt would have them overwriting each other's evidence. - printf '%s\n' "$got" | sort > "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" - # And are they the RIGHT ones? Resolve the planned HOST indices through this - # node's bare-metal map; holding the correct count of the wrong devices is - # the failure mode a count-only check cannot see. - for i in $(echo "$EXPECT_HOST" | tr ',' ' '); do - grep "^$i," "$map" | cut -d, -f2 - done | sort > "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" - if ! diff -q "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" \ - "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" >/dev/null; then - echo "FAIL: $node planned host GPU(s) $EXPECT_HOST, but holds different devices" >&2 - echo " want: $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt")" >&2 - echo " got : $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt")" >&2 - exit 1 - fi - echo "OK: $node holds host GPU(s) $EXPECT_HOST, confirmed by UUID" - # Two independent pools: each allocates its own Slurm job and its own nodes. backends: - name: gpu_pool @@ -182,11 +123,6 @@ operators: type: srun container_image: ${{ variables.CPU_IMAGE }} ntasks_per_node: 1 - # No container, so the placement ground truth is taken on the raw node before any - # runtime has renumbered anything. Reuses the host driver's nvidia-smi. - - name: worker_bare - type: srun - ntasks_per_node: 1 workflow: name: monitor_mixed_slurm @@ -199,112 +135,11 @@ workflow: enabled: true tasks: - # --------------------------------------------------------------------------- - # -- gpu_placement_matrix: PROVE the planned slice, by UUID, before anything - # -- else runs. Counting GPUs is not enough -- a task can hold the right NUMBER - # -- of the WRONG devices and look perfectly healthy, which is how a decode - # -- server planned for 2,3 ran in a 2-GPU container numbered 0,1 and died on - # -- its first CUDA call. The servers below only ECHO their devices, so they - # -- cannot catch that; these tasks can. - # -- - # -- This whole stage completes (and releases its GPUs) before env_check, so it - # -- costs no extra nodes and cannot collide with the servers' reservations. - # --------------------------------------------------------------------------- - - # Ground truth, bare metal, nothing carved. Runs on EVERY node (no - # `resources.nodes`), one file each, so the multi-node case below can check the - # node it actually landed on. - - name: node_gpu_map - operator: worker_bare - script: - - | - set -u - node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" - nvidia-smi --query-gpu=index,uuid --format=csv,noheader | tr -d ' ' \ - > "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" - sed "s/^/SFLOW_GPU_PROBE node_map $node: /" "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" - - # Bare srun sees the whole node, so a non-zero offset must survive untouched. - - name: placement_bare_high - operator: worker_bare - depends_on: [node_gpu_map] - resources: - gpus: - indices: [2, 3] - script: - - export EXPECT_HOST="2,3" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # Containerised, low and high offset, CONCURRENTLY on one node. A plan starting - # at 0 survives container renumbering by accident, which is why only the - # high-offset task ever broke -- both sides are needed to see it. - - name: placement_boxed_low - operator: worker_gpu - depends_on: [placement_bare_high] - resources: - gpus: - indices: [0, 1] - script: - - export EXPECT_HOST="0,1" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - - name: placement_boxed_high - operator: worker_gpu - depends_on: [placement_bare_high] - resources: - gpus: - indices: [2, 3] - script: - - export EXPECT_HOST="2,3" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # MULTI-NODE, containerised, high offset -- every renumbering hazard in play on - # more than one node at once. A multi-node plan is ONE node's slice applied - # independently per node, so a slice that is right on node 0 and wrong on node 1 - # is invisible at a single node. Pinned indices repeat on each node. - - name: placement_multinode_high - operator: worker_gpu - depends_on: [placement_boxed_low, placement_boxed_high] - resources: - nodes: - count: 2 - gpus: - indices: [2, 3] - script: - - export EXPECT_HOST="2,3" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # Collisions are invisible from inside either task: boxed_low and boxed_high - # held one node at the same time, so their devices must be disjoint. - - name: placement_disjoint - operator: worker_bare - depends_on: [placement_multinode_high] - resources: - nodes: - indices: [0] - script: - - | - set -u - run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") - low=$(ls "$run_dir"/placement_boxed_low*/gpus.*.txt 2>/dev/null | head -1) - high=$(ls "$run_dir"/placement_boxed_high*/gpus.*.txt 2>/dev/null | head -1) - if [ -z "$low" ] || [ -z "$high" ]; then - echo "FAIL: concurrent tasks left no gpus.*.txt to compare" >&2 - exit 1 - fi - shared=$(sort "$low" "$high" | uniq -d) - if [ -n "$shared" ]; then - echo "FAIL: concurrent tasks shared GPU(s): $shared" >&2 - exit 1 - fi - echo "OK: concurrent tasks held disjoint GPUs" - # -- resource_release_after: grabs every GPU on a node, then frees them on # completion (no readiness probe -> sflow infers task_completion), so the # servers below can reuse them. - name: env_check operator: worker_gpu - depends_on: [placement_disjoint] resources: gpus: count: ${{ variables.GPUS_PER_NODE }} diff --git a/examples/self_contained/slurm/sglang_server_client.yaml b/examples/self_contained/slurm/sglang_server_client.yaml index 0594453..7409814 100644 --- a/examples/self_contained/slurm/sglang_server_client.yaml +++ b/examples/self_contained/slurm/sglang_server_client.yaml @@ -176,6 +176,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/trtllm_serve_disagg.yaml b/examples/self_contained/slurm/trtllm_serve_disagg.yaml index 478244c..37e870e 100644 --- a/examples/self_contained/slurm/trtllm_serve_disagg.yaml +++ b/examples/self_contained/slurm/trtllm_serve_disagg.yaml @@ -328,15 +328,6 @@ workflow: script: - set -x - echo ${CUDA_VISIBLE_DEVICES} - # GPU placement probe. sflow plans a per-task slice and re-applies it - # inside the step; this prints what the container ACTUALLY has next to - # what was planned, which is the only way to tell the two failure modes - # apart: if the planned ordinals are absent from `nvidia-smi -L`, the - # slice never reached the container (carved and renumbered by the - # runtime); if they are present, placement worked and the launcher chose - # the wrong device. Cheap, and it makes every future triage one grep. - - echo "SFLOW_GPU_PROBE task=${SLURM_JOB_NAME:-?} procid=${SLURM_PROCID:-?} localid=${SLURM_LOCALID:-?} planned_CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" - - nvidia-smi -L 2>&1 | sed "s/^/SFLOW_GPU_PROBE visible: /" || true - export TLLM_LOG_LEVEL=INFO - export TRTLLM_SERVER_DISABLE_GC=1 - export TRTLLM_WORKER_DISABLE_GC=1 @@ -383,15 +374,6 @@ workflow: script: - set -x - echo ${CUDA_VISIBLE_DEVICES} - # GPU placement probe. sflow plans a per-task slice and re-applies it - # inside the step; this prints what the container ACTUALLY has next to - # what was planned, which is the only way to tell the two failure modes - # apart: if the planned ordinals are absent from `nvidia-smi -L`, the - # slice never reached the container (carved and renumbered by the - # runtime); if they are present, placement worked and the launcher chose - # the wrong device. Cheap, and it makes every future triage one grep. - - echo "SFLOW_GPU_PROBE task=${SLURM_JOB_NAME:-?} procid=${SLURM_PROCID:-?} localid=${SLURM_LOCALID:-?} planned_CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" - - nvidia-smi -L 2>&1 | sed "s/^/SFLOW_GPU_PROBE visible: /" || true - export TLLM_LOG_LEVEL=INFO - export TRTLLM_SERVER_DISABLE_GC=1 - export TRTLLM_WORKER_DISABLE_GC=1 diff --git a/pyproject.toml b/pyproject.toml index ca0ba6f..59cb48a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,6 @@ dev = [ "pytest-asyncio>=0.23.0", "pytest-cov>=7.0.0", "pytest-subprocess>=1.5.3", - "pytest-xdist>=3.8.0", ] s3 = [ "boto3>=1.34.0", @@ -79,9 +78,6 @@ where = ["src"] [tool.setuptools.package-data] "sflow.samples" = ["**/*.yaml", "**/*.csv", "**/*.md"] -# gpu_placement.sh is sourced by every Slurm GPU step; without it the operator -# skips GPU placement entirely, so it must ship in the wheel. -"sflow.plugins.operators" = ["*.sh"] "sflow.skills" = ["*.md", "**/*.md", "**/*.py"] "sflow.monitoring" = ["*.py"] @@ -94,7 +90,6 @@ dev = [ "pytest-asyncio>=0.23.0", "pytest-cov>=7.0.0", "pytest-subprocess>=1.5.3", - "pytest-xdist>=3.8.0", ] diff --git a/scripts/full_sample_tests.sh b/scripts/full_sample_tests.sh index 900298d..cadb7af 100755 --- a/scripts/full_sample_tests.sh +++ b/scripts/full_sample_tests.sh @@ -10,15 +10,13 @@ CLI_MODEL_PATH="" CLI_PARTITION="" CLI_ACCOUNT="" usage() { - echo "Usage: $0 [-a|-s|-m|-inf|--smoke|--min|--one] [-S] [-P] [-j N] [-M model_path] [-p partition] [-A account]" + echo "Usage: $0 [-a|-s|-m|-inf|--smoke|--min] [-S] [-P] [-j N] [-M model_path] [-p partition] [-A account]" echo " -a all tests (default)" echo " -s self-contained examples only" echo " -m modular examples only" echo " -inf infmax batch suites only" echo " --smoke curated Slurm smoke subset with broad coverage" echo " --min minimal Slurm submit set (one representative per validation type)" - echo " --one submit EXACTLY ONE Slurm job and skip the preflight sweep." - echo " Plumbing smoke for a new CI path / cluster, not coverage." echo " -S submit jobs to Slurm" echo " -P preflight checks only (skip job submission even if -S is set)" echo " -j max parallel jobs (default: 16, 0 for unlimited)" @@ -35,7 +33,6 @@ while [ $# -gt 0 ]; do -inf) TEST_TYPE="inf" ;; --smoke) TEST_TYPE="smoke" ;; --min) TEST_TYPE="min" ;; - --one) TEST_TYPE="one" ;; -S) SUBMIT="--submit" ;; -P) PREFLIGHT_ONLY="1" ;; -j) [ $# -ge 2 ] || { usage; exit 1; }; shift; MAX_JOBS="$1" ;; @@ -278,85 +275,7 @@ run_check() { # ========================================================================= # Preflight: CLI smoke tests (no jobs submitted) # ========================================================================= -# The colon-in-task-script fixture. Written OUTSIDE the preflight gate on -# purpose: sample_test.sh submits a focused Slurm e2e for it whenever -# SFLOW_COLON_SCRIPT_FIXTURE points at a real file, so leaving it inside the -# sweep would silently drop that real job from --min/--smoke whenever the -# sweep is skipped. Writing a small YAML costs nothing; the run_check probes -# that USE it stay in the sweep below. -COLON_SCRIPT_DIR="$PREFLIGHT_DIR/colon_in_task_script" -COLON_SCRIPT_FIXTURE="$COLON_SCRIPT_DIR/colon_in_task_script.yaml" -COLON_SCRIPT_DRYRUN_LOG="$COLON_SCRIPT_DIR/dry_run.log" -COLON_SCRIPT_COMPOSED="$COLON_SCRIPT_DIR/colon_in_task_script_composed.yaml" -COLON_SCRIPT_BATCH="$COLON_SCRIPT_DIR/colon_in_task_script_batch.sh" -COLON_SCRIPT_BATCH_CONFIG="$COLON_SCRIPT_DIR/colon_in_task_script_batch.yaml" -mkdir -p "$COLON_SCRIPT_DIR" -cat > "$COLON_SCRIPT_FIXTURE" <<'EOF' -version: "0.1" - -variables: - SLURM_ACCOUNT: - value: dummy_acct - SLURM_PARTITION: - value: dummy_part - SLURM_TIMELIMIT: - value: "00:10:00" - SLURM_NODES: - value: 1 - GPUS_PER_NODE: - value: 4 - -backends: - - name: slurm_cluster - type: slurm - default: true - account: ${{ variables.SLURM_ACCOUNT }} - partition: ${{ variables.SLURM_PARTITION }} - time: ${{ variables.SLURM_TIMELIMIT }} - nodes: ${{ variables.SLURM_NODES }} - gpus_per_node: ${{ variables.GPUS_PER_NODE }} - -operators: - - name: srun_no_container - type: srun - ntasks_per_node: 1 - mpi: pmix - -workflow: - name: colon_in_task_script - tasks: - - name: worker - operator: srun_no_container - resources: - gpus: - count: 1 - script: - - echo "My GPUs: $CUDA_VISIBLE_DEVICES" - - echo "COLON_SCRIPT_E2E_PASS" -EOF - -# Whether to run the preflight sweep at all. -# -t one -> never: the single-job smoke exists to prove the -# Slurm path, and the sweep is the slowest part of -# a run while proving nothing about it. -# SFLOW_E2E_SKIP_PREFLIGHT=1 -> caller states the sweep already ran elsewhere. -# sflow's CI runs exactly this (`-P`) in its own -# container job every pipeline, so repeating all -# 182 checks on a SHARED login node costs ~3.5min -# of the e2e and someone else's CPU for no new -# signal. Default off, so a manual/local run still -# gets the safety net. -RUN_PREFLIGHT="1" -if [ "$TEST_TYPE" = "one" ] || [ "${SFLOW_E2E_SKIP_PREFLIGHT:-}" = "1" ]; then - RUN_PREFLIGHT="" -fi -# ...unless -P was asked for explicitly. -P IS the sweep, so a skip switch that -# silences it turns the only job that runs these checks into a vacuous exit 0. -if [ -n "$PREFLIGHT_ONLY" ]; then - RUN_PREFLIGHT="1" -fi - -if [ -n "$RUN_PREFLIGHT" ]; then +if true; then echo "" echo "===== Preflight: CLI smoke tests (no Slurm submission) =====" echo "===== Running tests in parallel (max_jobs=${MAX_JOBS:-unlimited}) =====" @@ -481,53 +400,44 @@ if [ -n "$RUN_PREFLIGHT" ]; then grep -F -- 'operator: docker_run' \"$DOCKER_MULTI_DRYRUN_LOG\" && \ grep -F -- 'CUDA_VISIBLE_DEVICES: 0' \"$DOCKER_MULTI_DRYRUN_LOG\" && \ grep -E -- 'gpus: device=0[^0-9,]' \"$DOCKER_MULTI_DRYRUN_LOG\"" - # The gpu_reservation samples are internal-only: they are excluded from the - # public sync, and sample_test.sh refuses to package them. Guard rather than - # assume, so a checkout without them SKIPs here instead of failing four checks - # on a missing file, which says nothing about sflow. - if [ -d "$EXAMPLES_DIR/gpu_reservation" ]; then - run_check "dry-run gpu_reservation demo pins the planned device slice" \ - bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/demo.yaml\" --dry-run --verbose > \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\" 2>&1 && \ - grep -F -- 'operator: docker_run' \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\" && \ - grep -E -- 'gpus: device=0,1[^0-9,]' \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\" && \ - grep -F -- 'Dry-run complete: gpu_reservation_demo' \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\"" - run_check "dry-run gpu_reservation hog workload plans its GPU claim" \ - bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/hog.yaml\" --dry-run --verbose > \"$DOCKER_GPU_HOG_DRYRUN_LOG\" 2>&1 && \ - grep -F -- 'operator: docker_run' \"$DOCKER_GPU_HOG_DRYRUN_LOG\" && \ - grep -F -- 'Dry-run complete: gpu_hog' \"$DOCKER_GPU_HOG_DRYRUN_LOG\"" - # The pipeline sample fills a 4-GPU board exactly: pinned_service holds device 0 - # for the whole run, server_a holds 1-2 and server_b holds 3 (both released at - # READY), and merged_consumer then takes server_a's PAIR back -- so device=1,2 - # appears twice while device=3 is left over for whoever wants it next. That exact - # layout is the assertion: it is what makes the leftover deterministic, which the - # e2e suite then relies on to prove a concurrent run picks up device 3 and not - # some other one. Drop either `release_after: task_ready` and this fails with - # "merged_consumer needs GPU N, but it is blocked by server_a/server_b". - run_check "dry-run gpu_reservation pipeline reuses one server's GPUs and frees the other's" \ - bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/pipeline.yaml\" --dry-run --verbose > \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\" 2>&1 && \ - grep -F -- 'server_a: releases GPUs after task readiness' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\" && \ - grep -F -- 'server_b: releases GPUs after task completion' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\" && \ - [ \"\$(grep -c -E -- 'gpus: device=0[^0-9,]' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\")\" -eq 1 ] && \ - [ \"\$(grep -c -E -- 'gpus: device=1,2[^0-9,]' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\")\" -eq 2 ] && \ - [ \"\$(grep -c -E -- 'gpus: device=3[^0-9,]' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\")\" -eq 1 ] && \ - grep -F -- 'Dry-run complete: gpu_reservation_pipeline' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\"" - # The scheduling smoke drains and refills a full 8-GPU board in four waves. Wave - # 1 splits it into four pairs and wave 2 reuses each pair, so every pair spec - # appears TWICE -- that count is the assertion, and a scheduler that double-books - # or scatters a task off its predecessor's devices breaks it. The later waves are - # forced by the DAG once the pairs hold, so they need no counts of their own. - run_check "dry-run gpu_reservation scheduling smoke refills a full board in waves" \ - bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/scheduling_smoke.yaml\" --dry-run --verbose > \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\" 2>&1 && \ - [ \"\$(grep -c -E -- 'gpus: device=0,1[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ - [ \"\$(grep -c -E -- 'gpus: device=2,3[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ - [ \"\$(grep -c -E -- 'gpus: device=4,5[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ - [ \"\$(grep -c -E -- 'gpus: device=6,7[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ - [ \"\$(grep -c -F -- 'releases GPUs after task completion' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 11 ] && \ - grep -F -- 'Dry-run complete: docker_gpu_scheduling_smoke' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\"" - else - record_preflight_skip "gpu_reservation dry-run checks skipped: examples/gpu_reservation is not present" - echo " SKIP: examples/gpu_reservation not present; skipping its dry-run checks." - fi + run_check "dry-run gpu_reservation demo pins the planned device slice" \ + bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/demo.yaml\" --dry-run --verbose > \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\" 2>&1 && \ + grep -F -- 'operator: docker_run' \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\" && \ + grep -E -- 'gpus: device=0,1[^0-9,]' \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\" && \ + grep -F -- 'Dry-run complete: gpu_reservation_demo' \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\"" + run_check "dry-run gpu_reservation hog workload plans its GPU claim" \ + bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/hog.yaml\" --dry-run --verbose > \"$DOCKER_GPU_HOG_DRYRUN_LOG\" 2>&1 && \ + grep -F -- 'operator: docker_run' \"$DOCKER_GPU_HOG_DRYRUN_LOG\" && \ + grep -F -- 'Dry-run complete: gpu_hog' \"$DOCKER_GPU_HOG_DRYRUN_LOG\"" + # The pipeline sample fills a 4-GPU board exactly: pinned_service holds device 0 + # for the whole run, server_a holds 1-2 and server_b holds 3 (both released at + # READY), and merged_consumer then takes server_a's PAIR back -- so device=1,2 + # appears twice while device=3 is left over for whoever wants it next. That exact + # layout is the assertion: it is what makes the leftover deterministic, which the + # e2e suite then relies on to prove a concurrent run picks up device 3 and not + # some other one. Drop either `release_after: task_ready` and this fails with + # "merged_consumer needs GPU N, but it is blocked by server_a/server_b". + run_check "dry-run gpu_reservation pipeline reuses one server's GPUs and frees the other's" \ + bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/pipeline.yaml\" --dry-run --verbose > \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\" 2>&1 && \ + grep -F -- 'server_a: releases GPUs after task readiness' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\" && \ + grep -F -- 'server_b: releases GPUs after task completion' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\" && \ + [ \"\$(grep -c -E -- 'gpus: device=0[^0-9,]' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\")\" -eq 1 ] && \ + [ \"\$(grep -c -E -- 'gpus: device=1,2[^0-9,]' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\")\" -eq 2 ] && \ + [ \"\$(grep -c -E -- 'gpus: device=3[^0-9,]' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\")\" -eq 1 ] && \ + grep -F -- 'Dry-run complete: gpu_reservation_pipeline' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\"" + # The scheduling smoke drains and refills a full 8-GPU board in four waves. Wave + # 1 splits it into four pairs and wave 2 reuses each pair, so every pair spec + # appears TWICE -- that count is the assertion, and a scheduler that double-books + # or scatters a task off its predecessor's devices breaks it. The later waves are + # forced by the DAG once the pairs hold, so they need no counts of their own. + run_check "dry-run gpu_reservation scheduling smoke refills a full board in waves" \ + bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/scheduling_smoke.yaml\" --dry-run --verbose > \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\" 2>&1 && \ + [ \"\$(grep -c -E -- 'gpus: device=0,1[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ + [ \"\$(grep -c -E -- 'gpus: device=2,3[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ + [ \"\$(grep -c -E -- 'gpus: device=4,5[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ + [ \"\$(grep -c -E -- 'gpus: device=6,7[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ + [ \"\$(grep -c -F -- 'releases GPUs after task completion' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 11 ] && \ + grep -F -- 'Dry-run complete: docker_gpu_scheduling_smoke' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\"" run_check "dry-run kubernetes_hello_world uses k8s operator" \ bash -c "sflow run \"$EXAMPLES_DIR/self_contained/kubernetes/hello_world.yaml\" --dry-run --verbose > \"$KUBERNETES_HELLO_DRYRUN_LOG\" 2>&1 && \ grep -F -- 'id=kubernetes' \"$KUBERNETES_HELLO_DRYRUN_LOG\" && \ @@ -1117,6 +1027,56 @@ EOF bash "$OFFLOAD_TUI_CHECK" "$OFFLOAD_TUI_FIXTURE" "$OFFLOAD_TUI_DIR" # -- sflow run/batch: plain script commands containing ':' must stay strings -- + COLON_SCRIPT_DIR="$PREFLIGHT_DIR/colon_in_task_script" + COLON_SCRIPT_FIXTURE="$COLON_SCRIPT_DIR/colon_in_task_script.yaml" + COLON_SCRIPT_DRYRUN_LOG="$COLON_SCRIPT_DIR/dry_run.log" + COLON_SCRIPT_COMPOSED="$COLON_SCRIPT_DIR/colon_in_task_script_composed.yaml" + COLON_SCRIPT_BATCH="$COLON_SCRIPT_DIR/colon_in_task_script_batch.sh" + COLON_SCRIPT_BATCH_CONFIG="$COLON_SCRIPT_DIR/colon_in_task_script_batch.yaml" + mkdir -p "$COLON_SCRIPT_DIR" + cat > "$COLON_SCRIPT_FIXTURE" <<'EOF' +version: "0.1" + +variables: + SLURM_ACCOUNT: + value: dummy_acct + SLURM_PARTITION: + value: dummy_part + SLURM_TIMELIMIT: + value: "00:10:00" + SLURM_NODES: + value: 1 + GPUS_PER_NODE: + value: 4 + +backends: + - name: slurm_cluster + type: slurm + default: true + account: ${{ variables.SLURM_ACCOUNT }} + partition: ${{ variables.SLURM_PARTITION }} + time: ${{ variables.SLURM_TIMELIMIT }} + nodes: ${{ variables.SLURM_NODES }} + gpus_per_node: ${{ variables.GPUS_PER_NODE }} + +operators: + - name: srun_no_container + type: srun + ntasks_per_node: 1 + mpi: pmix + +workflow: + name: colon_in_task_script + tasks: + - name: worker + operator: srun_no_container + resources: + gpus: + count: 1 + script: + - echo "My GPUs: $CUDA_VISIBLE_DEVICES" + - echo "COLON_SCRIPT_E2E_PASS" +EOF run_check "run colon in task script (dry-run)" \ bash -c "sflow run \"$COLON_SCRIPT_FIXTURE\" --dry-run > \"$COLON_SCRIPT_DRYRUN_LOG\" 2>&1" run_check "compose colon in task script" \ @@ -1852,7 +1812,7 @@ EOF COMMON="$EXAMPLES_DIR/modular/inference_x_v2/common_workflow.yaml" BENCH_INFMAX="$EXAMPLES_DIR/modular/inference_x_v2/benchmark_infmax.yaml" BENCH_AIPERF="$EXAMPLES_DIR/modular/inference_x_v2/benchmark_aiperf.yaml" - DYNAMO_IMAGE="${DYNAMO_IMAGE:-nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0}" + DYNAMO_IMAGE="${DYNAMO_IMAGE:-nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0}" MODULAR_MISSABLE=(-M agg_server -M prefill_server -M decode_server -M benchmark_infmax -M benchmark_aiperf) MODULAR_OVERRIDES=(-a "LOCAL_MODEL_PATH=fs://$MODEL_PATH" -s "DYNAMO_IMAGE=$DYNAMO_IMAGE") for framework in trtllm sglang vllm; do @@ -3033,11 +2993,7 @@ fi # ========================================================================= if [ -n "$SUBMIT" ] && [ -z "$PREFLIGHT_ONLY" ]; then echo "" - if [ -z "$RUN_PREFLIGHT" ]; then - echo "===== Preflight sweep skipped — proceeding straight to job submission =====" - else - echo "===== All preflight checks passed — proceeding to job submission =====" - fi + echo "===== All preflight checks passed — proceeding to job submission =====" echo "" set -x cd "$SCRIPT_DIR/../tests/e2e_tests" @@ -3045,12 +3001,12 @@ if [ -n "$SUBMIT" ] && [ -z "$PREFLIGHT_ONLY" ]; then E2E_ACCOUNT="${CLI_ACCOUNT:-user}" E2E_SBATCH_OUTPUT="$REPO_DIR/sflow_output/%j-sflow-submit.out" E2E_SBATCH_ERROR="$REPO_DIR/sflow_output/%j-sflow-submit.err" - # Nodes to drain from e2e, via $SLURM_E2E_EXCLUDE_NODES. Default EMPTY on - # purpose: node names are cluster-specific and sbatch rejects the WHOLE - # submission with "Invalid node name specified" if one is unknown, so a - # hardcoded list is a landmine the moment the e2e moves cluster (stale - # gb-nvl-137-* names failed every job on ptyche exactly this way). - E2E_EXCLUDE_NODES="${SLURM_E2E_EXCLUDE_NODES:-}" + # Known-broken/flaky Slurm nodes to keep every e2e submission off of. CI pins + # the live list via the $SLURM_E2E_EXCLUDE_NODES job variable (forwarded to the + # remote by run_slurm_e2e_over_ssh.py); the default here is the fallback for + # local/manual runs. Add a node here (or to the CI variable) to drain it from + # e2e without touching any recipe. + E2E_EXCLUDE_NODES="${SLURM_E2E_EXCLUDE_NODES:-gb-nvl-137-compute02,gb-nvl-137-compute14}" E2E_BATCH_EXTRA_ARGS=( "--sbatch-output" "$E2E_SBATCH_OUTPUT" "--sbatch-error" "$E2E_SBATCH_ERROR" @@ -3060,29 +3016,6 @@ if [ -n "$SUBMIT" ] && [ -z "$PREFLIGHT_ONLY" ]; then # regardless of the workflow's own pass/fail. "--enable-workflow-monitor" ) - # Some clusters (GB200: ptyche/poly/lyris) require --segment on every - # submission, sized to the job's OWN node count -- and this suite submits - # recipes spanning 1..N nodes in a single bulk submit, so a fixed number cannot - # be right for all of them. "auto" expands to sflow's expression, which sflow - # resolves per workflow (verified: 1-node recipe -> --segment=1, 3-node -> - # --segment=3, same submit). - # - # It is a KEYWORD rather than the expression itself in the CI config on - # purpose: a literal expression there would have to survive CI variable - # expansion upstream, expansion AGAIN when the trigger job forwards it to the - # downstream pipeline, then python and shell quoting -- four chances to be - # silently mangled into something sbatch rejects. "auto" has no "$" to mangle. - # Any other value is passed through verbatim (e.g. a fixed node count). - # Empty default, so clusters without segment support are unaffected. - E2E_SEGMENT="${SLURM_E2E_SEGMENT:-}" - if [ "$E2E_SEGMENT" = "auto" ]; then - # Single quotes: this must reach sflow as a literal expression, and bash - # would fail on "${{...}}" as a bad substitution if it tried to expand it. - E2E_SEGMENT='${{SLURM_NODES}}' - fi - if [ -n "$E2E_SEGMENT" ]; then - E2E_BATCH_EXTRA_ARGS+=("-e" "--segment=$E2E_SEGMENT") - fi # Flag and value MUST be separate argv tokens ("-e" "--exclude=..."); a glued # single token folds a leading space into the value and the secondary-backend # salloc drops the exclude silently. @@ -3096,13 +3029,8 @@ if [ -n "$SUBMIT" ] && [ -z "$PREFLIGHT_ONLY" ]; then else ./sample_test.sh -p "$E2E_PARTITION" -A "$E2E_ACCOUNT" -m "$MODEL_PATH" -t "$TEST_TYPE" --submit -- "${E2E_BATCH_EXTRA_ARGS[@]}" # 09 has some GPU issues fi - e2e_rc=$? set +x - # `set +x` is a command and it always succeeds, so without this the script's - # exit status is ITS status, not the suite's -- the "green despite exit 1" - # family. Nothing runs after this branch, so exiting here is safe. - exit "$e2e_rc" elif [ -z "$SUBMIT" ]; then echo "Preflight only (no -S flag). To submit jobs, re-run with -S." else diff --git a/scripts/use_under_dev_sflow.sh b/scripts/use_under_dev_sflow.sh index 7092435..81cabff 100644 --- a/scripts/use_under_dev_sflow.sh +++ b/scripts/use_under_dev_sflow.sh @@ -15,48 +15,55 @@ resolve_under_dev_sflow_ref() { fi } -assert_sflow_installed_from() { - # The suite exists to validate THIS checkout, so prove the sflow on PATH came - # from it. Not by grepping `sflow --version` for "source : local editable - # dev" -- that made the runtime-info format load-bearing and rejected install - # shapes that were the right code (a wheel built from the checkout, a - # direct-URL install). direct_url.json records the path pip installed FROM - # whatever the shape, so compare that and do not require "editable". - # - # Worth keeping now that the CI workspace and its venv are reused across - # runs: without it a stale install silently passes as this branch. +assert_under_dev_sflow_editable_install() { local repo_dir="$1" - # PYTHONPATH deliberately cleared: setup_under_dev_sflow puts $repo_dir/src on - # it, and importlib.metadata would then discover src/sflow.egg-info -- which - # has no direct_url.json -- ahead of the real dist-info in site-packages. The - # question here is what is INSTALLED, not what is importable, so the source - # tree must not be on the path. (Otherwise this rejects every checkout that - # ever ran a plain `pip install -e .` and left an egg-info behind.) - PYTHONPATH= "$SFLOW_TEST_PYTHON" - "$repo_dir" <<'SFLOW_INSTALL_CHECK' -import json, sys -from importlib import metadata + "$SFLOW_TEST_PYTHON" - "$repo_dir" <<'PY' +import json +import sys +from importlib import metadata as importlib_metadata from pathlib import Path from urllib.parse import unquote, urlparse repo_dir = Path(sys.argv[1]).resolve() + +try: + dist = importlib_metadata.distribution("sflow") +except importlib_metadata.PackageNotFoundError: + print( + "ERROR: sflow is not installed in the selected Python environment.", + file=sys.stderr, + ) + print( + f" Install this checkout editable first: {sys.executable} -m pip install -e {repo_dir}", + file=sys.stderr, + ) + sys.exit(1) + +direct_url_text = dist.read_text("direct_url.json") try: - dist = metadata.distribution("sflow") -except metadata.PackageNotFoundError: - sys.exit(f"ERROR: sflow is not installed in {sys.executable}.") - -url = json.loads(dist.read_text("direct_url.json") or "{}").get("url") or "" -parsed = urlparse(url) -src = Path(unquote(parsed.path)).resolve() if parsed.scheme == "file" else None -if src != repo_dir: - sys.exit( - "ERROR: the sflow on PATH was not installed from this checkout.\n" - f" Selected Python : {sys.executable}\n" - f" Expected repo : {repo_dir}\n" - f" Installed from : {src or url or 'a package index'}\n" - f" Fix with : {sys.executable} -m pip install -e {repo_dir}" + direct_url = json.loads(direct_url_text or "{}") +except json.JSONDecodeError: + direct_url = {} + +parsed = urlparse(direct_url.get("url") or "") +install_path = Path(unquote(parsed.path)).resolve() if parsed.scheme == "file" else None +is_editable = bool(direct_url.get("dir_info", {}).get("editable")) + +if not is_editable or install_path != repo_dir: + print( + "ERROR: full sample tests must run against this checkout installed editable.", + file=sys.stderr, ) -SFLOW_INSTALL_CHECK + print(f" Selected Python : {sys.executable}", file=sys.stderr) + print(f" Expected repo : {repo_dir}", file=sys.stderr) + print(f" Installed path : {install_path or 'not a local editable install'}", file=sys.stderr) + print( + f" Fix with : {sys.executable} -m pip install -e {repo_dir}", + file=sys.stderr, + ) + sys.exit(1) +PY } setup_under_dev_sflow() { @@ -80,6 +87,8 @@ setup_under_dev_sflow() { return 1 fi + assert_under_dev_sflow_editable_install "$repo_dir" + if [ -z "${SFLOW_UNDER_DEV_REF:-}" ]; then SFLOW_UNDER_DEV_REF="$(resolve_under_dev_sflow_ref "$repo_dir" || true)" fi @@ -126,8 +135,14 @@ EOF echo "Using under-dev sflow from $SFLOW_UNDER_DEV_REPO (ref: $SFLOW_UNDER_DEV_REF)" echo "Submitted Slurm jobs install this checkout editable (--sflow-source-path), so uncommitted working-tree changes are included." - sflow --version - assert_sflow_installed_from "$repo_dir" + local sflow_runtime_info + sflow_runtime_info="$(sflow --version)" + printf '%s\n' "$sflow_runtime_info" + if ! printf '%s\n' "$sflow_runtime_info" | grep -q "source : local editable dev"; then + echo "ERROR: sflow runtime source is not local editable dev." >&2 + echo " full_sample_tests.sh is intended to validate local editable sflow changes." >&2 + return 1 + fi } cleanup_under_dev_sflow() { diff --git a/src/sflow/app/assembly.py b/src/sflow/app/assembly.py index 09005f6..2c356df 100644 --- a/src/sflow/app/assembly.py +++ b/src/sflow/app/assembly.py @@ -1662,19 +1662,12 @@ async def build_state( """ Build runtime state from configuration (composition root). - Kept out of core because this is the PLAN-TIME side of the split: it turns a - validated config into runtime objects, while core owns what happens once - they exist. Not because core may not import plugins -- it already does, in - the four registry self-population calls (core/backend_registry.py:56 and - its siblings), so defending that invariant here would be defending one - that is already false. + This is intentionally kept out of core to avoid core importing plugins. """ from pathlib import Path - # Assigned unconditionally: `resolver` is module-global, so a conditional - # assignment leaves the PREVIOUS run's files in place and visualize() (which - # passes none) then points its error hints at the wrong YAML. - resolver.source_files = [Path(f) for f in source_files] if source_files else [] + if source_files: + resolver.source_files = [Path(f) for f in source_files] # Seed an empty workflow/state; we will populate task graph after resolution/allocation. wf = Workflow(name=config.workflow.name, task_graph=TaskGraph()) diff --git a/src/sflow/cli/batch.py b/src/sflow/cli/batch.py index 3b036de..41f2e93 100644 --- a/src/sflow/cli/batch.py +++ b/src/sflow/cli/batch.py @@ -748,13 +748,6 @@ def _generate_sbatch_script( "dist", "*.egg-info", "__pycache__", - # NOT just size: pip/uv write partial `*.tmp` files under .cache while - # other jobs are still bootstrapping, and rsync exits 24 ("some files - # vanished") when one disappears mid-transfer -- which the bootstrap - # treats as fatal, so the whole Slurm job dies seconds in with no output - # directory at all. A shared, concurrently-written cache must never be - # part of the copy source. - ".cache", ".pytest_cache", ".ruff_cache", ".mypy_cache", diff --git a/src/sflow/cli/sample.py b/src/sflow/cli/sample.py index ea10ce0..e7e58e2 100644 --- a/src/sflow/cli/sample.py +++ b/src/sflow/cli/sample.py @@ -378,6 +378,9 @@ def _get_sample_node_info(sample_path: Path) -> str | None: if line_stripped.startswith("SLURM_NODES:"): # Multi-line format, look for value in next lines continue + if "value:" in line_stripped and nodes is None: + # Check if this is under SLURM_NODES by looking at context + pass # Direct value extraction for SLURM_NODES if "SLURM_NODES:" in line and "value:" in line: # Inline format diff --git a/src/sflow/config/loader.py b/src/sflow/config/loader.py index 24ab25a..da158ac 100644 --- a/src/sflow/config/loader.py +++ b/src/sflow/config/loader.py @@ -514,26 +514,6 @@ def __init__(self): self.source_files: List[Path] = [] self.file_contributions: List[Dict[str, Any]] = [] - def _warn_unenforced_timeout(self, config: SflowConfig) -> None: - """Say out loud that `timeout:` does nothing. - - The field is accepted and deep-merged but never read, so a recipe that - declares one looks bounded and is not -- and a hung workflow then runs - until something external kills it. Warning is the honest middle: removing - the field would reject every config that sets it (the models forbid extra - keys), and enforcing it would start killing runs that have never been - held to it. - """ - where = [] - if config.workflow.timeout is not None: - where.append("workflow") - where += [t.name for t in config.workflow.tasks if t.timeout is not None] - if where: - _logger.warning( - f"`timeout:` is set on {', '.join(where)} but sflow does not enforce it; " - "these run unbounded. Use the backend's own limit (e.g. Slurm --time)." - ) - def load_config( self, path: Path, @@ -592,7 +572,6 @@ def load_config( except ValidationError as e: raise ValueError(f"Configuration validation failed:\n{e}") - self._warn_unenforced_timeout(config) self.config = config return config @@ -667,7 +646,6 @@ def load_configs( except ValidationError as e: raise ValueError(f"Merged configuration validation failed:\n{e}") - self._warn_unenforced_timeout(config) self.config = config return config diff --git a/src/sflow/config/schema.py b/src/sflow/config/schema.py index 4178b94..8320d79 100644 --- a/src/sflow/config/schema.py +++ b/src/sflow/config/schema.py @@ -1018,12 +1018,6 @@ def script_must_not_be_empty(cls, v: List[str]) -> List[str]: required_by: Optional[List[str]] = None replicas: Optional[ReplicaConfig] = None retries: Optional[RetryConfig] = None - # NOT ENFORCED. Accepted so existing recipes keep loading (this model forbids - # extra keys, so removing it would reject every config that sets it) and so - # multi-file merge can carry it, but nothing reads it: no code path passes it - # to Orchestrator._launch_task_with_timeout, and TaskStatus.TIMEOUT is never - # assigned. Bound a task with the backend's own limit (Slurm `--time`) until - # this is wired. load_config WARNs when it is set. timeout: Optional[Union[int, str]] = None variables: Optional[ Annotated[List[VariableConfig], BeforeValidator(_normalize_to_list)] @@ -1037,8 +1031,6 @@ class WorkflowConfig(StrictBaseModel): """Configuration for the workflow execution.""" name: str - # NOT ENFORCED -- see TaskConfig.timeout above. A workflow declaring - # `timeout: 115m` today runs unbounded. timeout: Optional[Union[str, int]] = None variables: Optional[ Annotated[List[VariableConfig], BeforeValidator(_normalize_to_list)] diff --git a/src/sflow/core/compute_node.py b/src/sflow/core/compute_node.py index eafc902..1848ae3 100644 --- a/src/sflow/core/compute_node.py +++ b/src/sflow/core/compute_node.py @@ -12,15 +12,6 @@ class ComputeNode: index: int # GPU count available on this node (if known). Used for CUDA_VISIBLE_DEVICES packing/validation. num_gpus: int | None = None - # Physical GPU UUIDs on this node, ordered by HOST device index, as read on - # bare metal before anything was carved. This is ground truth: it is the only - # way to tell "the right number of GPUs" from "the right GPUs", and it is what - # lets a task step name the indices its planned cards turned out to have, - # whatever layer renumbered them. None when the backend could not - # probe (no GPUs, no nvidia-smi, probe failed) -- callers must degrade, not - # assume. Per node, not per backend: nodes in one allocation can differ, and - # two Slurm backends can have different gpus_per_node entirely. - gpu_uuids: list[str] | None = None def to_dict(self) -> dict[str, Any]: return { @@ -28,5 +19,4 @@ def to_dict(self) -> dict[str, Any]: "ip_address": self.ip_address, "index": self.index, "num_gpus": self.num_gpus, - "gpu_uuids": self.gpu_uuids, } diff --git a/src/sflow/core/execution_summary.py b/src/sflow/core/execution_summary.py index 6ddf06c..aed025a 100644 --- a/src/sflow/core/execution_summary.py +++ b/src/sflow/core/execution_summary.py @@ -15,11 +15,7 @@ from typing import Any from sflow.core.loop_watchdog import EventLoopWatchdog -from sflow.utils.gpu import ( - parse_cuda_visible_devices, - task_gpu_indices, - task_gpu_record, -) +from sflow.utils.gpu import parse_cuda_visible_devices, task_gpu_indices from .command_trace import get_command_trace from .task import Task, TaskStatus @@ -674,19 +670,10 @@ def _gpu_assignment_lines(self, tasks: list[Task]) -> list[str]: physical = self._task_gpu_ids(task) if not physical: continue - # The step's OWN numbering, straight from its placement record. The - # planner's env is the HOST slice, so reading it here printed the - # physical ids in the in-container column and vice versa: a task - # planned for host 2,3 that a container renumbered to 0,1 was reported - # as physical 0,1 / in-container 2,3 -- backwards, and impossible. - record = task_gpu_record(task) - in_step = record.get("cuda_visible_devices", "") - if in_step.startswith("<"): # / - in_step = "" visible = [ str(i) for i in parse_cuda_visible_devices( - in_step or task.envs.get("CUDA_VISIBLE_DEVICES") + task.envs.get("CUDA_VISIBLE_DEVICES") ) ] rows.append((task.name, ",".join(physical), ",".join(visible) or "-")) diff --git a/src/sflow/core/launcher.py b/src/sflow/core/launcher.py index 480dde4..cad767f 100644 --- a/src/sflow/core/launcher.py +++ b/src/sflow/core/launcher.py @@ -106,6 +106,39 @@ def _console_prefix(self, task_name: str | None) -> str: """ return f"[{task_name}] " if task_name else "" + async def _terminate_process(self, process: asyncio.subprocess.Process) -> None: + """ + Best-effort terminate a running subprocess. + """ + try: + if process.returncode is not None: + return + process.terminate() + except ProcessLookupError: + return + except Exception: + # Fall back to kill below. + pass + + try: + await asyncio.wait_for(process.wait(), timeout=5) + return + except Exception: + pass + + try: + if process.returncode is None: + process.kill() + except ProcessLookupError: + return + except Exception: + return + + try: + await asyncio.wait_for(process.wait(), timeout=5) + except Exception: + return + async def run_async( self, command: Command | str | list[str], @@ -393,3 +426,161 @@ def _on_readable() -> None: os.close(master_fd) except OSError: pass + + # async def run_pipe_async( + # self, + # commands: list[Command | str | list[str]], + # shell: bool = False, + # output_logger: Optional[logging.Logger] = None, + # env: Mapping[str, str] | None = None, + # task_name: str | None = None, + # ) -> int: + # """Execute multiple commands connected by pipes asynchronously. + + # Args: + # commands: List of commands to pipe together + # shell: Whether to use shell for individual commands + # output_logger: Optional logger to use for output logging + + # Returns: + # int: Exit code of the final command + # """ + # pfx = self._console_prefix(task_name) + # _logger.info(f"{pfx}========== Commands ==========") + # _logger.info(f"{pfx}{' | '.join([format_command(cmd) for cmd in commands])}") + # _logger.info(f"{pfx}=============================") + + # if not commands: + # raise ValueError("At least one command required") + + # processes = [] + # pipes = [] + # proc_env = None + # if env is not None: + # proc_env = os.environ.copy() + # proc_env.update({str(k): str(v) for k, v in env.items()}) + + # try: + # for i, cmd in enumerate(commands): + # if isinstance(cmd, Command): + # cmd = cmd.as_list() + + # stdin = None + # if i > 0: + # stdin = pipes[-1][0] # Read end of previous pipe + + # if i < len(commands) - 1: + # r, w = os.pipe() + # pipes.append((r, w)) + # stdout = w + # else: + # stdout = asyncio.subprocess.PIPE + + # # Intermediate processes: inherit stderr (goes to console) + # # Last process: merge stderr to stdout (captured in output) + # stderr = asyncio.subprocess.STDOUT if i == len(commands) - 1 else None + + # create_subprocess = ( + # asyncio.create_subprocess_shell + # if shell + # else asyncio.create_subprocess_exec + # ) + + # args = [] + # if shell: + # if isinstance(cmd, list): + # args = [shlex.join(cmd)] + # else: + # args = [cmd] + # else: + # if isinstance(cmd, str): + # args = shlex.split(cmd) + # else: + # args = cmd + + # proc = await create_subprocess( + # *args, + # stdin=stdin, + # stdout=stdout, + # stderr=stderr, + # env=proc_env, + # ) + + # processes.append(proc) + + # # Close parent's copy of the write pipe end immediately + # if i < len(commands) - 1: + # os.close(stdout) + + # # Close parent's copy of the read pipe end used as stdin + # if i > 0: + # os.close(stdin) + + # # Read output from last process + # last_proc = processes[-1] + # try: + # # Read output in chunks to handle progress bars and special characters + # # that use \r without \n (which would cause readline() to hang) + # buffer = b"" + # while True: + # try: + # # Read available data in chunks (non-blocking when data is available) + # chunk = await last_proc.stdout.read(4096) + # if not chunk: + # # Process any remaining data in buffer + # if buffer: + # line_str = _strip_ansi(buffer.decode("utf-8", errors="replace")).rstrip() + # if line_str: + # _logger.info(f"{pfx}{line_str}") + # if output_logger: + # output_logger.info(line_str) + # break + + # buffer += chunk + + # # Split on both \n and \r to handle progress bars + # # Replace \r\n with \n first to avoid double processing + # text = buffer.decode("utf-8", errors="replace") + # text = text.replace("\r\n", "\n").replace("\r", "\n") + + # # Split into lines, keeping incomplete line in buffer + # lines = text.split("\n") + # buffer = lines[-1].encode("utf-8") # Keep incomplete line + + # for line_str in lines[:-1]: + # # Strip ANSI escape sequences for cleaner logs + # line_str = _strip_ansi(line_str).rstrip() + # if line_str: # Skip empty lines from progress bar overwrites + # _logger.info(f"{pfx}{line_str}") + # if output_logger: + # output_logger.info(line_str) + # except Exception as e: + # _logger.warning(f"{pfx}Error reading output: {e}") + # break + + # # Wait for all processes + # exit_codes = await asyncio.gather(*[p.wait() for p in processes]) + # last_exit_code = exit_codes[-1] + + # return last_exit_code + # except asyncio.CancelledError: + # # Terminate the whole pipeline on cancellation. + # for p in processes: + # try: + # await self._terminate_process(p) + # except Exception: + # pass + # raise + + # except Exception: + # # Cleanup pipes if error + # for r, w in pipes: + # try: + # os.close(r) + # except OSError: + # pass + # try: + # os.close(w) + # except OSError: + # pass + # raise diff --git a/src/sflow/core/task_graph.py b/src/sflow/core/task_graph.py index 3c78ae3..5cb4702 100644 --- a/src/sflow/core/task_graph.py +++ b/src/sflow/core/task_graph.py @@ -66,3 +66,34 @@ def get_submittable_tasks(self) -> List[Task]: return to_submit + def update_task_status( + self, + task_name: str, + status: TaskStatus | str, + ) -> None: + """Update the status of a task. + + Args: + task_name: Name of the task to update + status: New status (can be TaskStatus enum or string) + """ + task = self.dag.nodes[task_name] + + if isinstance(status, str): + status = TaskStatus(status) + + task.status = status + + _logger.debug(f"Task '{task_name}' status updated to {status}") + + def mark_all_cancelled(self) -> None: + """Mark all non-terminal tasks as cancelled.""" + for task in self.get_tasks(): + if task.status not in [ + TaskStatus.COMPLETED, + TaskStatus.FAILED, + TaskStatus.TIMEOUT, + TaskStatus.CANCELLED, + ]: + task.status = TaskStatus.CANCELLED + _logger.debug(f"Task '{task.name}' marked as CANCELLED") diff --git a/src/sflow/exceptions.py b/src/sflow/exceptions.py new file mode 100644 index 0000000..8cb3b90 --- /dev/null +++ b/src/sflow/exceptions.py @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + + +class AllocationRequiredError(Exception): + """ + Exception raised when an allocation is required but not available. + """ + + pass diff --git a/src/sflow/monitoring/postprocess_monitor_timeline.py b/src/sflow/monitoring/postprocess_monitor_timeline.py index 176d65e..3f01dd8 100644 --- a/src/sflow/monitoring/postprocess_monitor_timeline.py +++ b/src/sflow/monitoring/postprocess_monitor_timeline.py @@ -138,7 +138,7 @@ def _select_marker_events( _MARKER_LABEL_INK = "#5f6772" # `fail` -- and ONLY fail -- gets a reserved status colour: a cancel is usually # sflow tearing a service down on purpose, so flagging it red would cry wolf on -# every healthy run. Deliberately darker than the `#d55e00` in `_SERIES_COLORS`, +# every healthy run. Deliberately darker than the `#d62728` in `_SERIES_COLORS`, # so a rule can never be mistaken for a GPU line on the same canvas; it always # ships alongside its own text label, never as colour alone. _MARKER_FAIL_INK = "#a61b1b" @@ -231,8 +231,7 @@ def _build_device_legend_svg( colour = _SERIES_COLORS[idx % len(_SERIES_COLORS)] parts.append( f'' + f'y2="{y - 3:.1f}" stroke="{colour}" stroke-width="2"/>' ) parts.append( f'' @@ -662,96 +661,13 @@ def _series_for_metric( return [v for _ts, v in _metric_timeseries(rows, resource_type, metric_name)] -# Line colors for per-resource series (one per GPU): the Okabe-Ito "Color -# Universal Design" palette, verbatim and in its published order. Eight is -# exactly enough -- a node carries at most 8 GPUs, so the cycle never wraps in -# practice and no two devices on one panel can share a colour. -# -# Chosen over a wider hue spread (a true red plus a grey reads as more separated -# to normal vision) because this palette holds up under all three dichromacies as -# well, and these charts get pasted into bugs and shared with people we cannot -# survey. `_SERIES_DASHES` covers the residual: it separates any pair that is not -# exactly 4 slots apart, with no colour at all. -# Reordered from the published sequence -- same eight colours, different slots. -# The palette contains three near-pairs (blue/sky blue, and orange/vermilion/ -# yellow among the warms) and the published order puts BOTH members of two of -# them inside the first four slots. A 4-GPU task only ever draws slots 0-3, which -# made orange-vs-vermilion the common case rather than the rare one. -# -# Two rules pin this order, and both break if you sort it back: -# 1. Slots 0-3 take one colour from each family (blue / warm / green / purple), -# so the common 4-GPU chart is maximally separated by hue alone. -# 2. No near-pair sits exactly 4 apart, because `_SERIES_DASHES` cycles every 4 -# and slots i and i+4 therefore share a dash pattern -- which is precisely -# the redundancy those pairs need. Blue/sky blue are 0 and 5; the warms are -# 1, 6 and 7. +# Line colors for per-resource series (one per GPU). Cycled; chosen to stay +# distinguishable in both the SVG and a greyscale print. _SERIES_COLORS = ( - "#0072b2", # blue - "#d55e00", # vermilion - "#009e73", # bluish green - "#cc79a7", # reddish purple - "#000000", # black - "#56b4e9", # sky blue - "#e69f00", # orange - "#f0e442", # yellow + "#1f77b4", "#d62728", "#2ca02c", "#ff7f0e", "#9467bd", + "#8c564b", "#17becf", "#e377c2", "#7f7f7f", "#bcbd22", ) -# Per-slot stroke-width multiplier, keyed to the order above. Okabe-Ito's yellow -# (slot 7) is the one entry that is genuinely faint on white, and the fix is -# weight rather than a darker hue -- darkening it collapses it toward the -# `#e69f00` orange, which is the pairing the palette exists to avoid. Black -# (slot 4) is nudged the other way for the same reason in reverse: at equal width -# it reads heavier than the rest and draws the eye to whichever GPU lands there. -_SERIES_WIDTH_SCALE = (1.0, 1.0, 1.0, 1.0, 0.875, 1.0, 1.0, 1.25) - -# Dash pattern per device, cycled alongside _SERIES_COLORS and keyed on the same -# index. Colour alone cannot separate two lines that COINCIDE, and coincidence is -# the normal case for a tensor-parallel task: every rank allocates the same -# weights + KV-cache footprint, so `gpu_memory_used_mib` for its GPUs is often -# identical to the byte. The later line then hides exactly under the earlier one -# and the panel is indistinguishable from one where the second device was never -# drawn at all -- which reads as a collection bug rather than as real data. -_SERIES_DASHES = ("", "5,3", "1.5,2.5", "7,2,1.5,2") - - -def _series_dash(idx: int) -> str: - """SVG dash attribute for series *idx*, or "" for the solid first series.""" - pattern = _SERIES_DASHES[idx % len(_SERIES_DASHES)] - return f' stroke-dasharray="{pattern}"' if pattern else "" - - -def _series_width(idx: int, base: float) -> float: - """Stroke width for series *idx*, scaled per `_SERIES_WIDTH_SCALE`.""" - return round(base * _SERIES_WIDTH_SCALE[idx % len(_SERIES_WIDTH_SCALE)], 2) - - -def _mpl_dashes(idx: int) -> tuple[float, ...]: - """`_series_dash` as matplotlib's `dashes=`; empty tuple means solid.""" - pattern = _SERIES_DASHES[idx % len(_SERIES_DASHES)] - return tuple(float(x) for x in pattern.split(",")) if pattern else () - - -def _device_labels( - rows: list[dict[str, object]], families: "list[tuple[str, str, str]]" -) -> list[str]: - """Every GPU drawn anywhere in this image, in stable colour-assignment order. - - The union across ALL gpu families, not just the first: a device that reports - N/A for one field (MIG mode, a per-field collector error) drops out of that - family only. Both renderers key a line's colour on this list's index rather - than on the device's position within its own panel -- otherwise the survivor - of a drop-out inherits the colour of the device that vanished, and the shared - legend below the chart then names the wrong GPU. - """ - labels: list[str] = [] - for _label, resource_type, metric_name in families: - if resource_type != "gpu": - continue - for lab, _ser in _metric_series_by_resource(rows, resource_type, metric_name): - if lab not in labels: - labels.append(lab) - return labels - def gpu_label(gpus: object) -> str: """Render a report's GPU subset for humans. @@ -1133,7 +1049,16 @@ def _render_svg( caption_y = panels_bottom + 22 + marker_band_bottom # Device colours are shared by every panel, so their legend is built once. - device_labels = _device_labels(rows, families) + # Union across ALL gpu families, not just the first: a device that reports + # N/A for one field (MIG mode, a per-field collector error) drops out of that + # family only, and keying the colour on each panel's own index would then + # shift every line after it against this legend. + device_labels: list[str] = [] + for _lbl, _rtype, _mname in families: + if _rtype == "gpu": + for lab, _ser in _metric_series_by_resource(rows, _rtype, _mname): + if lab not in device_labels: + device_labels.append(lab) device_parts, device_h = ( _build_device_legend_svg(device_labels, x0=8, max_x=width - 8) if device_labels @@ -1213,19 +1138,16 @@ def _y(v: float, _lo: float = ymin, _hi: float = ymax) -> float: f'y2="{gy:.1f}" stroke="#eeeeee"/>' ) for series_label, series in multi: - # Look the style up BY LABEL so it always matches the shared legend. - # Index 0 for a panel with no per-device series (cpu/mem/disk/net): - # first colour, no dash, i.e. unchanged from a single solid line. - idx = ( - device_labels.index(series_label) + # Look the colour up BY LABEL so it always matches the shared legend. + colour = ( + _SERIES_COLORS[device_labels.index(series_label) % len(_SERIES_COLORS)] if series_label in device_labels - else 0 + else "#1f77b4" ) - colour = _SERIES_COLORS[idx % len(_SERIES_COLORS)] points = " ".join(f"{_x(t):.1f},{_y(v):.1f}" for t, v in series) parts.append( f'' + f'stroke-width="1.5"/>' ) parts.append( f' None: # Extra pad so the axis title clears the `ready`/`done` band drawn beneath it. axes[-1].set_xlabel("Elapsed time (s)", labelpad=28) figure.suptitle(title, fontsize=14) - # Device legend UNDER the panels, matching the SVG. In-panel it sat on top of - # the data it was explaining -- and the busiest corner at that, since GPU load - # ramps toward the end of a run. Once per figure, not once per panel: a - # device keeps its colour on every panel, so repeating it is pure noise. - if device_labels: - handles = [ - plt.Line2D( - [], - [], - color=_SERIES_COLORS[i % len(_SERIES_COLORS)], - dashes=_mpl_dashes(i), - linewidth=_series_width(i, 1.4), - ) - for i in range(len(device_labels)) - ] - figure.legend( - handles, - device_labels, - loc="upper center", - bbox_to_anchor=(0.5, 0.0), - ncol=min(len(device_labels), 8), - fontsize=8, - frameon=False, - ) png_path.parent.mkdir(parents=True, exist_ok=True) figure.savefig(png_path, dpi=140, bbox_inches="tight") plt.close(figure) diff --git a/src/sflow/plugins/backends/slurm.py b/src/sflow/plugins/backends/slurm.py index 5995c0a..cf0e633 100644 --- a/src/sflow/plugins/backends/slurm.py +++ b/src/sflow/plugins/backends/slurm.py @@ -60,71 +60,6 @@ def planning_node_count(self) -> Resolvable[int] | None: return self.nodes -def _planned_gpu_uuids( - cuda_visible_devices: str | None, allocation: "Allocation | None" -) -> str: - """Encode "which physical GPUs was this task planned for, on each node". - - Format: ``node=uuid,uuid;node=uuid,uuid`` -- one entry per node whose topology - is known. The task step looks up its OWN node and compares against what it can - actually see. - - The plan is a flat list of HOST indices applied identically on every node the - task spans, so the same slot resolves to a DIFFERENT physical card per node -- - which is exactly why this is emitted per node rather than as one list. - - Returns "" when there is nothing trustworthy to say (no plan, no allocation, - no probe, or a planned index beyond a node's device count). Silence means - "fall back", never "no GPUs". - """ - if not cuda_visible_devices or allocation is None: - return "" - slots: list[int] = [] - for token in cuda_visible_devices.split(","): - token = token.strip() - if not token.isdigit(): - # UUID-form or anything non-ordinal: nothing to resolve against. - return "" - slots.append(int(token)) - if not slots: - return "" - - entries: list[str] = [] - for node in allocation.nodes: - uuids = node.gpu_uuids - if not uuids: - continue - over = [slot for slot in slots if slot >= len(uuids)] - if over: - # A partial answer is worse than none: it would let a step "verify" - # against a map that cannot contain the card it was planned for. But - # say so -- this is the one case where the truth is in hand and the - # config is simply wrong (gpus_per_node larger than the node really - # has), and staying quiet just drops the task to weaker checking. - _logger.warning( - "Node %s reports %d GPU(s) but this task was planned for device %s; " - "check the backend's gpus_per_node. Placement for this task falls " - "back to device-index arithmetic instead of UUID verification.", - node.name, - len(uuids), - ",".join(str(slot) for slot in over), - ) - continue - entries.append(f"{node.name}=" + ",".join(uuids[slot] for slot in slots)) - return ";".join(entries) - - -class _OutputCapture(logging.Handler): - """Collect a subprocess's log lines into *sink*, for the srun/salloc probes.""" - - def __init__(self, sink: list[str]) -> None: - super().__init__() - self._sink = sink - - def emit(self, record: logging.LogRecord) -> None: - self._sink.append(record.getMessage()) - - @register_backend("slurm", SlurmBackendConfig) class SlurmBackend(Backend): """ @@ -208,52 +143,8 @@ def dry_run_details(self) -> list[tuple[str, str]]: details.append(("extra_args", str(list(self._extra_args)))) return details - @property - def node_topology_report(self) -> str | None: - """The allocation's GPU topology, for the summary's Node Topology section. - - This is the bare-metal index -> UUID map taken before any task carved - anything. Recorded so a run can be judged AFTER the fact: each GPU task - also writes what it actually saw inside its step, and comparing the two - is what distinguishes "sflow placed this wrong" from "the recipe used the - wrong device" -- neither of which a bare index list can settle. - """ - allocation = self.allocation - if allocation is None: - return None - blocks: list[str] = [] - for node in allocation.nodes: - if not node.gpu_uuids: - continue - blocks.append(f"{node.name}: {len(node.gpu_uuids)} GPU(s)") - blocks.extend( - f" [{index}] {uuid}" for index, uuid in enumerate(node.gpu_uuids) - ) - return "\n".join(blocks) if blocks else None - def resource_env(self, *, cuda_visible_devices: str | None = None) -> dict[str, str]: env = super().resource_env(cuda_visible_devices=cuda_visible_devices) - # Do NOT hand NVIDIA_VISIBLE_DEVICES to an srun step. It is read by the - # container runtime (pyxis/enroot) at container CREATION, and naming a - # subset there makes the runtime carve the container and RENUMBER those - # devices from 0 -- after which the CUDA_VISIBLE_DEVICES we exported - # alongside it, in HOST numbering, addresses nothing. That is the whole - # origin of the placement problem: a worker planned for host 2,3 landed in - # a 2-GPU container numbered 0,1 and died with "No CUDA GPUs are - # available", while a worker planned for 0,1 survived by coincidence. - # - # Without it the container sees every GPU on the node with host numbering - # intact, so the slice we planned is directly addressable and - # CUDA_VISIBLE_DEVICES alone -- the variable CUDA actually reads -- decides - # what the task uses. Where something still carves the step (a GRES - # partition with ConstrainDevices), the in-step placement script detects it - # by UUID and re-selects. - # - # The trade is deliberate: this drops device-level isolation (an NVML - # consumer such as nvidia-smi or DCGM can now SEE the node's other GPUs) - # in exchange for the slice being addressable at all. Docker keeps its own - # isolation via `--gpus device=` and overrides this method entirely. - env.pop("NVIDIA_VISIBLE_DEVICES", None) env.update( { key: value @@ -262,17 +153,6 @@ def resource_env(self, *, cuda_visible_devices: str | None = None) -> dict[str, } ) - # Resolve the planned HOST indices to physical UUIDs, per node, so the step - # can check whether it already holds them instead of rewriting - # CUDA_VISIBLE_DEVICES unconditionally. Built from THIS backend's own - # allocation: with several Slurm backends the node sets and gpus_per_node - # differ, and a UUID is the only identifier that survives a container - # renumbering devices from 0. Absent when the probe found nothing, which - # the step reads as "fall back to index arithmetic". - planned = _planned_gpu_uuids(cuda_visible_devices, self.allocation) - if planned: - env["SFLOW_PLANNED_GPU_UUIDS"] = planned - allocation = self.allocation job_id: str | None = env.get("SLURM_JOB_ID") or env.get("SLURM_JOBID") if not job_id and allocation is not None: @@ -345,90 +225,6 @@ async def _resolve_nodes_via_scontrol( ) return nodes - async def _discover_gpu_uuids( - self, *, nodes: list[ComputeNode], job_id: str | None = None - ) -> None: - """Record each node's HOST GPU index -> UUID map, in place. - - One bare srun across the allocation, before any task carves anything, so - what nvidia-smi reports here really is the host numbering. That map is the - only thing that can answer "does this step already hold the GPUs it was - planned for?" -- device indices cannot, because a container renumbers them - from 0 and an index says nothing about which physical card it names. - - Per node on purpose. Nodes in one allocation can differ, and two Slurm - backends can have different gpus_per_node, so a single per-backend count - is not a substitute. - - Best effort: on any failure the maps stay None and the step falls back to - the index arithmetic it used before. A placement probe must never be the - reason a workflow cannot start. - """ - if not nodes: - return - nodelist = ",".join(node.name for node in nodes) - cmd: list[str] = ["srun", "--nodelist", nodelist, "--ntasks-per-node=1"] - if job_id: - cmd.extend(["--jobid", job_id]) - # --overlap: this shares the allocation with the real job steps rather - # than waiting for one, and it must not hold resources of its own. - cmd.append("--overlap") - cmd.extend( - [ - "bash", - "-c", - "timeout 10 nvidia-smi --query-gpu=index,uuid --format=csv,noheader " - '| tr -d " " | sed "s|^|${SLURMD_NODENAME:-$(hostname -s)} |"', - ] - ) - _logger.debug(f"Discovering GPU topology via srun: {' '.join(cmd)}") - - output_lines: list[str] = [] - - capture_logger = isolated_logger("slurm.gpu_discovery") - try: - with temporary_handler(capture_logger, _OutputCapture(output_lines)): - exit_code = await self._subprocess_launcher.run_async( - cmd, output_logger=capture_logger - ) - except Exception as exc: # pragma: no cover - defensive - _logger.debug(f"GPU topology discovery failed ({exc}); placement will " - "fall back to device-index arithmetic.") - return - if exit_code != 0: - _logger.debug( - f"GPU topology discovery exited {exit_code}; placement will fall " - f"back to device-index arithmetic. Output: {output_lines}" - ) - return - - by_node: dict[str, dict[int, str]] = {} - for line in output_lines: - parts = line.split() - if len(parts) < 2 or "," not in parts[-1]: - continue - node_name = parts[-2] - index_text, _, uuid = parts[-1].partition(",") - if not index_text.isdigit() or not uuid.startswith("GPU-"): - continue - by_node.setdefault(node_name, {})[int(index_text)] = uuid - - for node in nodes: - found = by_node.get(node.name) - if not found: - continue - # Ordered by host index, and only a contiguous 0..N-1 run is usable: - # a gap means the reading is partial, and a partial map would resolve - # planned indices to the wrong cards. - if set(found) == set(range(len(found))): - node.gpu_uuids = [found[i] for i in range(len(found))] - _logger.debug( - "GPU topology: " - + ", ".join( - f"{n.name}={len(n.gpu_uuids or [])}" for n in nodes - ) - ) - async def _resolve_nodes_via_srun( self, *, nodelist: str, job_id: str | None = None ) -> list[ComputeNode]: @@ -455,7 +251,11 @@ async def _resolve_nodes_via_srun( # Capture output output_lines: list[str] = [] - capture_handler = _OutputCapture(output_lines) + class OutputCaptureHandler(logging.Handler): + def emit(self, record: logging.LogRecord): + output_lines.append(record.getMessage()) + + capture_handler = OutputCaptureHandler() # Use a per-call isolated logger so concurrent backend allocations don't # capture each other's output (see isolated_logger docstring). capture_logger = isolated_logger("slurm.srun_resolve") @@ -583,8 +383,6 @@ async def allocate(self) -> Allocation: ) # Important: we do NOT own this allocation; do not scancel on exit. - if self._gpu_per_node: - await self._discover_gpu_uuids(nodes=nodes, job_id=job_id) return Allocation(allocation_id=str(job_id), nodes=nodes, owned=False) command = ( @@ -604,12 +402,8 @@ async def allocate(self) -> Allocation: if self._exclude_nodes: command.add_opt("--exclude", ",".join(self._exclude_nodes)) - # add_arg, not add_opt: extra_args is a verbatim passthrough of already - # tokenized argv. add_opt() de-dups by option name, which treats a bare - # value ("1" in `-G 1`) as an option and lets a later identical value - # ("1" in `-N 1`) delete it. Same as srun/docker's extra_args handling. for arg in self._extra_args: - command.add_arg(arg) + command.add_opt(arg) parser = ParseLogHandler( patterns=[ @@ -620,7 +414,11 @@ async def allocate(self) -> Allocation: # Capture all output lines for error reporting output_lines: list[str] = [] - capture_handler = _OutputCapture(output_lines) + class OutputCaptureHandler(logging.Handler): + def emit(self, record: logging.LogRecord): + output_lines.append(record.getMessage()) + + capture_handler = OutputCaptureHandler() # Use a per-call isolated logger so concurrent backend allocations don't # capture each other's salloc output. Routing through the shared module # logger caused parsers from sibling allocations to see each other's @@ -681,8 +479,6 @@ async def allocate(self) -> Allocation: ) raise - if self._gpu_per_node: - await self._discover_gpu_uuids(nodes=nodes, job_id=allocation_id) return Allocation( allocation_id=allocation_id, nodes=nodes, diff --git a/src/sflow/plugins/operators/gpu_placement.sh b/src/sflow/plugins/operators/gpu_placement.sh deleted file mode 100644 index 54d4fc1..0000000 --- a/src/sflow/plugins/operators/gpu_placement.sh +++ /dev/null @@ -1,260 +0,0 @@ -# sflow GPU placement -- sourced (never executed) by each Slurm job step. -# -# MUST be sourced: it exports CUDA_VISIBLE_DEVICES into the task's own shell, and -# a subprocess could not. -# -# Inputs (environment): -# SFLOW_GPU_PLAN planned HOST device indices, e.g. "2,3" -# SFLOW_PLANNED_GPU_UUIDS "=,;..." from the driver's -# bare-metal probe; absent => fall back to the old -# index arithmetic -# SFLOW_GPU_MARKER record filename under SFLOW_TASK_OUTPUT_DIR -# -# It answers ONE question, the same way every time: which indices do the planned -# cards have HERE? It probes the devices this step can actually see, looks each -# planned UUID up among them, and names the indices it found. That single rule -# covers every shape -- more devices visible than planned (narrow to them), -# exactly as many (fix a stale value naming the wrong ordinals), a container that -# renumbered from 0 (just a different lookup result), nothing set at all (name -# them explicitly, because recipes deref this under `set -u`). There is no -# separate no-op / pin / narrow mode, because those were all this one question. -# -# UUID is the identity, never the index: an index stops meaning anything the -# moment a layer renumbers. A planned card that is not visible AT ALL is a hard -# error (exit 97) when sflow chose the devices, and a degrade to index arithmetic -# when Slurm did -- GRES may have granted cards the planner never saw, and there -# the plan can only mean a position into the grant. -# -# Where the UUIDs cannot be checked at all (no map for this node, or no nvidia-smi -# here) it degrades to that same index arithmetic rather than stopping. Narrowing -# a step handed the whole allocation is the collision this exists to prevent, and -# a KNOWN plan must never end up doing less than an unknown one. -# -# It leaves an audit record next to the task's logs: what was planned, what the -# step could see (index -> UUID), what CUDA ended up selecting, and what arrived -# beforehand -- inherited equal to final IS the "nothing needed changing" signal. -# Compared with the Node Topology section of the run summary, that is what tells -# a bad placement apart from a recipe using the wrong device. -[ -n "${SFLOW_GPU_PLAN:-}" ] || return 0 - -__sflow_plan="${SFLOW_GPU_PLAN:-}" -__sflow_node="${SLURMD_NODENAME:-$(hostname -s)}" -# Physical GPUs planned for THIS node; empty => fall back to index arithmetic. -__sflow_want="" -if [ -n "${SFLOW_PLANNED_GPU_UUIDS:-}" ]; then - __sflow_oifs="$IFS"; IFS=';' - for __sflow_e in ${SFLOW_PLANNED_GPU_UUIDS}; do - case "$__sflow_e" in - "$__sflow_node="*) __sflow_want="${__sflow_e#*=}" ;; - esac - done - IFS="$__sflow_oifs" - # A map that names other nodes but not this one means the driver's node names - # and $SLURMD_NODENAME disagree (short vs FQDN is the usual cause). Verification - # silently switches off there, so say it once. - [ -n "$__sflow_want" ] || echo "sflow: no planned-GPU entry for node '$__sflow_node' in SFLOW_PLANNED_GPU_UUIDS; falling back to device-index placement" >&2 -fi - -# What arrived in the step, before this script touches anything. Without it the -# record shows only the post-state and a reader cannot tell what changed or why. -# -# BOTH variables, because they answer different questions. sflow exports only -# CUDA_VISIBLE_DEVICES (Backend.resource_env pops NVIDIA_VISIBLE_DEVICES -- it is -# consumed at container CREATION and setting it carved the container down before -# the plan could be applied). So whatever NVIDIA_VISIBLE_DEVICES holds here came -# from the runtime, and recording it verbatim is how a reader attributes a -# surprising device set to that layer rather than to sflow. -__sflow_in="${CUDA_VISIBLE_DEVICES:-}" -# Describe a variable VERBATIM. UNSET and SET-BUT-EMPTY are different states and -# must not read alike, so they get and -- but no -# interpretation beyond that: what "unset" or "all" means to CUDA or to a -# container runtime is the reader's call, and a gloss here would be sflow -# asserting semantics it does not own (and would be wrong on any stack that -# differs). One function, so the two sentinels have one spelling. -__sflow_desc() { - [ -n "${!1+x}" ] || { printf ''; return; } - printf '%s' "${!1:-}" -} -__sflow_in_desc="$(__sflow_desc CUDA_VISIBLE_DEVICES)" -__sflow_nvd_desc="$(__sflow_desc NVIDIA_VISIBLE_DEVICES)" - -# THE RECORD FORMAT IS A CONTRACT -- three independent parsers read it: -# * this function (producer) -# * sflow.utils.gpu.task_gpu_record (Python; run reporting + the summary) -# * sample_test.sh::gpu_placement_verified (shell `sed`; the e2e verdict) -# Line 1 is the bare device list and must stay parseable on its own. Every other -# line is `key=value`; `visible=`/`selected=` repeat, the rest are scalars. -# Renaming a key breaks the shell reader SILENTLY, so the key set is pinned by -# test_the_marker_format_keys_are_the_contract -- change both, or neither. -__sflow_record() { - printf '%s\n' "${CUDA_VISIBLE_DEVICES:-}" - printf 'node=%s\n' "$__sflow_node" - printf 'action=%s\n' "$__sflow_act" - printf 'reason=%s\n' "${__sflow_reason:-(placement left to device-index arithmetic)}" - printf 'cuda_visible_devices_inherited=%s\n' "$__sflow_in_desc" - printf 'nvidia_visible_devices_inherited=%s\n' "$__sflow_nvd_desc" - # No nvidia_visible_devices= counterpart: sflow never writes that variable, so - # a post-state would equal the inherited line in every record ever produced. - printf 'cuda_visible_devices=%s\n' "$(__sflow_desc CUDA_VISIBLE_DEVICES)" - printf 'planned_host_indices=%s\n' "$__sflow_plan" - printf 'planned_uuids=%s\n' "${__sflow_want:-(not resolved)}" - printf 'visible_gpu_count=%s\n' "${#__sflow_uuid[@]}" - for __sflow_i in "${!__sflow_uuid[@]}"; do - printf 'visible=%s %s\n' "$__sflow_i" "${__sflow_uuid[$__sflow_i]}" - done - __sflow_now="${CUDA_VISIBLE_DEVICES:-}" - for __sflow_t in ${__sflow_now//,/ }; do - case "$__sflow_t" in - GPU-*) printf 'selected=%s %s\n' "?" "$__sflow_t" ;; - ''|*[!0-9]*) printf 'selected=%s (unresolvable)\n' "$__sflow_t" ;; - *) printf 'selected=%s %s\n' "$__sflow_t" "${__sflow_uuid[$__sflow_t]:-(not visible here)}" ;; - esac - done -} -# Called before every exit too, not just at the end: a step that aborts on a -# mis-placement is exactly when the record is worth having. -__sflow_save() { - [ -n "${SFLOW_TASK_OUTPUT_DIR:-}" ] && [ "${SLURM_LOCALID:-0}" = 0 ] || return 0 - __sflow_marker="${SFLOW_GPU_MARKER:-sflow_gpus.log}" - if [ "${SLURM_STEP_NUM_NODES:-1}" = 1 ] && [ "${SLURM_PROCID:-0}" = 0 ]; then - __sflow_record > "$SFLOW_TASK_OUTPUT_DIR/$__sflow_marker" 2>/dev/null || true - else - __sflow_record > "$SFLOW_TASK_OUTPUT_DIR/${__sflow_marker%.log}.$__sflow_node.log" 2>/dev/null || true - fi -} - -__sflow_done="" -__sflow_act="fallback" -__sflow_reason="" - -# What this namespace can really see, probed HERE (inside the container/cgroup, so -# it is the step's own view, not the driver's). Keyed by the index nvidia-smi -# REPORTS rather than by row position, so the record states a detected index and -# an ordinal lookup stays correct even if the numbering is ever not 0..N-1. -# Indexed (NOT associative): bash indexed arrays are sparse, and ${!arr[@]} -# yields subscripts in ascending numeric order -- an associative array would -# iterate in hash order and silently scramble the ordered comparison below. -__sflow_uuid=() -while IFS=, read -r __sflow_i __sflow_u; do - case "$__sflow_i" in ''|*[!0-9]*) continue ;; esac - [ -n "$__sflow_u" ] && __sflow_uuid[$__sflow_i]="$__sflow_u" -done < <(timeout 10 nvidia-smi --query-gpu=index,uuid --format=csv,noheader 2>/dev/null | tr -d ' \r') -if [ -n "$__sflow_want" ]; then - if [ "${#__sflow_uuid[@]}" -eq 0 ]; then - # Nothing to look the planned UUIDs up against. Record that the placement is - # unproven -- but do NOT stop here. The index arithmetic below still narrows a - # step that was handed the whole allocation, which is the collision this - # exists to prevent, and it is exactly what a task with NO planned UUIDs gets. - # Returning early would make a KNOWN plan do less than an unknown one. - __sflow_act="unverified" - __sflow_reason="no nvidia-smi in this step, so the planned UUIDs could not be checked" - else - # ONE rule, whatever set the devices: look up each planned card by UUID among - # the ones this step can actually see, and name their indices. That covers - # every shape without special cases -- more visible than planned narrows to - # the right subset; exactly as many still fixes a stale value that names the - # wrong ordinals; a container renumbering from 0 is just a different lookup - # result. UUID is the only identifier that survives every layer's renumbering. - __sflow_sel="" - __sflow_miss="" - for __sflow_w in ${__sflow_want//,/ }; do - __sflow_hit="" - for __sflow_i in "${!__sflow_uuid[@]}"; do - if [ "${__sflow_uuid[$__sflow_i]}" = "$__sflow_w" ]; then __sflow_hit="$__sflow_i"; break; fi - done - if [ -z "$__sflow_hit" ]; then __sflow_miss="$__sflow_w"; break; fi - __sflow_sel="${__sflow_sel:+$__sflow_sel,}$__sflow_hit" - done - if [ -n "$__sflow_miss" ]; then - # A planned card is not here. Who chose the devices decides what that means: - # * Slurm did (GRES granted them, or our CUDA_VISIBLE_DEVICES was replaced) - # -> it may have picked cards the planner never assumed, so the plan is a - # POSITION into what Slurm gave us. Fall through to index arithmetic - # rather than fail a healthy run. - # * sflow did -> the step holds something it was never given, which is the - # silent mis-placement this exists to catch. Fail loudly. - # - # Both signals below are STEP-scoped, deliberately. SLURM_JOB_GPUS is not: - # it says the JOB has GPUs, and Backend.resource_env copies every SLURM_* - # var out of the DRIVER's environment into every task -- so on the `batch - # --submit` path (driver inside the sbatch job) it was set for every step on - # any GRES cluster. Steps also run --overlap, where Slurm does not carve per - # step and never sets SLURM_STEP_GPUS, so that leaked job-level value was the - # only signal in play and the exit 97 below could not fire on the very - # clusters it was written for. Do not add it back. - if [ -n "${SLURM_STEP_GPUS:-}" ] || - { [ -n "$__sflow_in" ] && [ "$__sflow_in" != "$__sflow_plan" ]; }; then - # Keep $__sflow_want: those UUIDs WERE resolved, they just are not here, - # and that distinction is the whole diagnosis. Clearing it to signal - # "fall through" made the record claim planned_uuids=(not resolved), - # which points a reader at the driver probe instead of at the grant. - # Falling through is already what an empty $__sflow_done means. - echo "sflow: planned GPU $__sflow_miss is not among the devices Slurm granted this step; using the planned slice as a position into them instead" >&2 - __sflow_reason="planned GPU $__sflow_miss is not among the devices Slurm granted this step" - else - echo "sflow: planned GPU $__sflow_miss is not visible on $__sflow_node (visible: ${__sflow_uuid[*]})" >&2 - __sflow_act="missing"; __sflow_reason="planned GPU $__sflow_miss is not visible on this node" - __sflow_save - exit 97 - fi - else - # Export only when it actually differs. The record keeps both the inherited - # and the final value, so "did this change anything?" is answered there - # rather than by a separate action name. - # - # CUDA_VISIBLE_DEVICES only. NVIDIA_VISIBLE_DEVICES is consumed by the - # container runtime at CREATION time -- the container already exists, so - # writing it cannot change what is exposed, and the runtime has already - # rewritten it to describe this namespace (enroot sets "all"). Overwriting - # that with CUDA ordinals states something false about a different layer. - [ "$__sflow_sel" = "${CUDA_VISIBLE_DEVICES:-}" ] || export CUDA_VISIBLE_DEVICES="$__sflow_sel" - __sflow_done=1; __sflow_act="verified" - __sflow_reason="planned GPUs located by UUID among ${#__sflow_uuid[@]} visible device(s)" - fi - fi -fi -if [ -z "$__sflow_done" ]; then -__sflow_seen="${CUDA_VISIBLE_DEVICES:-}" -__sflow_real="" -if [ "${#__sflow_uuid[@]}" -gt 0 ]; then - for __sflow_i in "${!__sflow_uuid[@]}"; do - __sflow_real="${__sflow_real:+$__sflow_real,}$__sflow_i" - done - for __sflow_d in ${__sflow_seen//,/ }; do - case ",$__sflow_real," in - *",$__sflow_d,"*) ;; - *) __sflow_seen="" ;; - esac - done - [ -n "$__sflow_seen" ] || __sflow_seen="$__sflow_real" -else - echo "sflow: no nvidia-smi here, so CUDA_VISIBLE_DEVICES=${__sflow_seen:-} is taken on trust; a runtime that renumbered this task's devices from 0 cannot be detected, and placement may be wrong" >&2 -fi -if [ -z "$__sflow_seen" ]; then - __sflow_sel="$__sflow_plan" -else - IFS=, read -r -a __sflow_dev <<< "$__sflow_seen" - IFS=, read -r -a __sflow_slot <<< "$__sflow_plan" - if [ "${#__sflow_dev[@]}" -lt "${#__sflow_slot[@]}" ]; then - echo "sflow: step has ${#__sflow_dev[@]} GPU(s) but this task was planned for ${#__sflow_slot[@]} (CUDA_VISIBLE_DEVICES=$__sflow_seen)" >&2 - __sflow_act="too-few"; __sflow_reason="step has ${#__sflow_dev[@]} GPU(s), task was planned for ${#__sflow_slot[@]}" - __sflow_save - exit 97 - elif [ "${#__sflow_dev[@]}" -eq "${#__sflow_slot[@]}" ]; then - __sflow_sel="$__sflow_seen" - else - __sflow_sel="" - for __sflow_i in "${__sflow_slot[@]}"; do - if [ -z "${__sflow_dev[$__sflow_i]:-}" ]; then - echo "sflow: planned GPU slot $__sflow_i is outside CUDA_VISIBLE_DEVICES=$__sflow_seen" >&2 - __sflow_act="out-of-range"; __sflow_reason="planned GPU slot $__sflow_i is outside the visible devices" - __sflow_save - exit 97 - fi - __sflow_sel="${__sflow_sel:+$__sflow_sel,}${__sflow_dev[$__sflow_i]}" - done - fi -fi -export CUDA_VISIBLE_DEVICES="$__sflow_sel" -fi -__sflow_save diff --git a/src/sflow/plugins/operators/srun.py b/src/sflow/plugins/operators/srun.py index 4251956..1a3a3c4 100644 --- a/src/sflow/plugins/operators/srun.py +++ b/src/sflow/plugins/operators/srun.py @@ -24,7 +24,6 @@ from sflow.core.operator_registry import register_operator from sflow.utils.gpu import GPU_MARKER_FILE from sflow.logging import get_logger -from sflow.utils.extra_args import normalize_extra_args from sflow.utils.container import ( append_runtime_mounts as append_runtime_mount_specs, extract_container_images_from_extra_args, @@ -81,9 +80,10 @@ def _slurm_runtime_env_prelude() -> list[str]: # planned placement. _GPU_PLACEMENT_BANNER = """\ # --- sflow GPU placement (begin) ------------------------------------------- -# Names this task's planned GPUs by looking their UUIDs up among the devices the -# step can really see. Without it a step handed the whole allocation (GRES) or -# renumbered by a container runtime lands on the wrong cards. +# On a GRES partition slurmstepd rewrites CUDA_VISIBLE_DEVICES after --export, +# so sflow re-applies its planned slice here (last writer wins). Without this +# every concurrent step sees the whole allocation and collides on GPU 0. +# No-op when the step already sees exactly its own devices. """ # Closing half of the wrap, so it is obvious where sflow's block stops and the @@ -92,46 +92,53 @@ def _slurm_runtime_env_prelude() -> list[str]: # --- sflow GPU placement (end) --------------------------------------------- """ -# The placement logic lives in gpu_placement.sh next to this module, staged once -# per run and SOURCED by each step, rather than pasted into every srun command. -# Sourced, not executed: it exports CUDA_VISIBLE_DEVICES into the task's shell. -# Read that file for the reasoning; keeping it out of the command line means a -# failing srun line stays readable, and every task shares one copy. -_GPU_PLACEMENT_SCRIPT = Path(__file__).with_name("gpu_placement.sh") - - -def _stage_gpu_placement_script(workflow_out_dir: str | None) -> str | None: - """Write the placement script into the run's output dir; return its path. - - Lands under the workflow output dir, which is shared storage on Slurm by - construction -- that is what lets every node source the same file. - - Idempotent: many tasks launch at once and would otherwise fight over it. - Returns None when it cannot be written, and the caller then skips placement - entirely rather than running it from somewhere the nodes cannot read. - """ - if not workflow_out_dir: - return None - try: - target = Path(workflow_out_dir) / ".sflow" / "gpu_placement.sh" - target.parent.mkdir(parents=True, exist_ok=True) - body = _GPU_PLACEMENT_SCRIPT.read_text() - if not target.exists() or target.read_text() != body: - # Atomic: a step may be sourcing this path while another driver writes it. - tmp = target.with_name(f"{target.name}.{os.getpid()}.tmp") - tmp.write_text(body) - os.replace(tmp, target) - return str(target) - except OSError: - return None - - -def _gpu_placement_prelude( - cuda_visible_devices: str | None, - *, - gpus_per_task: str | None = None, - workflow_out_dir: str | None = None, -) -> list[str]: +_GPU_PLACEMENT_PRELUDE = """\ +__sflow_plan='{plan}' +__sflow_seen="${{CUDA_VISIBLE_DEVICES:-}}" +if [ -z "$__sflow_seen" ]; then + __sflow_sel="$__sflow_plan" +else + IFS=, read -r -a __sflow_dev <<< "$__sflow_seen" + IFS=, read -r -a __sflow_slot <<< "$__sflow_plan" + if [ "${{#__sflow_dev[@]}}" -lt "${{#__sflow_slot[@]}}" ]; then + echo "sflow: step has ${{#__sflow_dev[@]}} GPU(s) but this task was planned for ${{#__sflow_slot[@]}} (CUDA_VISIBLE_DEVICES=$__sflow_seen)" >&2 + exit 97 + elif [ "${{#__sflow_dev[@]}}" -eq "${{#__sflow_slot[@]}}" ]; then + __sflow_sel="$__sflow_seen" + else + __sflow_sel="" + for __sflow_i in "${{__sflow_slot[@]}}"; do + if [ -z "${{__sflow_dev[$__sflow_i]:-}}" ]; then + echo "sflow: planned GPU slot $__sflow_i is outside CUDA_VISIBLE_DEVICES=$__sflow_seen" >&2 + exit 97 + fi + __sflow_sel="${{__sflow_sel:+$__sflow_sel,}}${{__sflow_dev[$__sflow_i]}}" + done + fi +fi +export CUDA_VISIBLE_DEVICES="$__sflow_sel" NVIDIA_VISIBLE_DEVICES="$__sflow_sel" +[ -n "${{SFLOW_TASK_OUTPUT_DIR:-}}" ] && [ "${{SLURM_PROCID:-0}}" = 0 ] \ + && [ "${{SLURM_NNODES:-1}}" = 1 ] \ + && printf '%s\n' "$__sflow_sel" > "$SFLOW_TASK_OUTPUT_DIR/{marker}" 2>/dev/null || true +""" +# Why the marker write above is guarded the way it is. Kept OUT of the template: +# every line of that string is echoed into the generated srun command, where a +# ten-line rationale is noise for whoever is reading the failing command. +# +# Report the devices actually selected back to the driver: the plan is only the +# same thing when the step saw the whole allocation. +# rank 0 only -- every rank runs this body, and with --gpus-per-task they hold +# DIFFERENT devices, so letting them all truncate one path is a race whose +# winner is arbitrary. +# 1 node only -- mirrors the reader (utils/gpu.task_gpu_indices), which discounts +# the marker for a multi-node task because one flat list cannot speak for nodes +# holding different devices. Writing it anyway would leave a file that is right +# for node 0 and wrong for every other, reading as authoritative to anyone who +# opens it. +# `|| true` -- reporting must never fail a task. + + +def _gpu_placement_prelude(cuda_visible_devices: str | None) -> list[str]: """Re-apply sflow's planned GPU slice from *inside* the job step. sflow exports ``CUDA_VISIBLE_DEVICES`` from the driver and relies on srun @@ -152,49 +159,12 @@ def _gpu_placement_prelude( runtime -- last writer wins, and placement is restored without adding a single Slurm flag. - ``CUDA_VISIBLE_DEVICES`` cannot be trusted as an OBSERVATION of what the step - sees. sflow exports its plan and srun runs with ``--export=ALL``, so the value - arriving in the step is often just that plan echoed back -- using it to decide - "what do I have" is circular. A container runtime (pyxis/enroot) carves by - passing through only this task's devices and renumbers them from 0, so the - inherited value can name HOST ordinals that do not exist here at all: on - ptyche a decode server planned for ``2,3`` ran in a container holding exactly - two GPUs numbered ``0,1``, kept ``2,3`` because the counts matched, and died - with "No CUDA GPUs are available". The prefill server planned for ``0,1`` - survived only because its plan happened to match the renumbering. - - So the staged script does not reason about that value at all. It probes the - devices the step can really see and looks up the physical UUIDs the driver - resolved this task's plan to (``SFLOW_PLANNED_GPU_UUIDS``), then names the - indices they turned out to have here -- one rule for every shape, because a - UUID is the only identity that survives a layer renumbering from 0. Where - there is no map to check against (probe failed, node names disagree, no - nvidia-smi in the step) it degrades to the older index arithmetic: equal - counts keep the step's own numbering, a larger visible set narrows - positionally. - - Only ``--gpus-per-task`` skips this, not ``gres``/``gpus``. Those two make - Slurm carve per STEP, not per rank, so every rank still sees the same set and - the checks below stay meaningful -- a step granted fewer devices than the task - was planned for is then a real over-ask, and aborting is the right answer. - ``--gpus-per-task`` is the one that carves per RANK, which is what breaks the - premise: - that flag makes the step request GRES, so Slurm carves per RANK instead of - handing the step the whole allocation. Every rank then sees only its own - devices -- fewer than the task's slice -- and the count check below would abort - all of them with "step has 1 GPU(s) but this task was planned for 8". Slurm's - own GRES accounting already keeps those per-rank sets disjoint, which is the - collision this prelude exists to prevent, so there is nothing left to re-apply. + Inert where it should be: on a non-GRES partition the step observes exactly + the slice sflow exported, and on a step Slurm already carved it observes + exactly its own devices; both take the "keep what I see" branch. """ if not cuda_visible_devices: return [] - if gpus_per_task: - _logger.debug( - "srun --gpus-per-task=%s carves GPUs per rank, so Slurm already owns " - "this task's placement; skipping sflow's in-step GPU remap.", - gpus_per_task, - ) - return [] # The plan is interpolated into shell text, so it is validated first. Every value # sflow's planner produces is a comma-joined list of non-negative ints, but this # reads `envs`, and ANY workflow variable named CUDA_VISIBLE_DEVICES lands there @@ -212,29 +182,13 @@ def _gpu_placement_prelude( cuda_visible_devices, ) return [] - staged = _stage_gpu_placement_script(workflow_out_dir) - if not staged: - # Nowhere the compute nodes can read it from. Skip placement rather than - # paste a second copy of the logic into the command line: that would mean - # two delivery paths to keep honest, and the failure that gets us here -- - # an unwritable workflow output dir -- has already broken the run's logs. - # Skipping leaves CUDA_VISIBLE_DEVICES exactly as exported, which is the - # behaviour from before this prelude existed. - _logger.warning( - "Could not stage the GPU placement script under %r; skipping sflow's " - "in-step GPU placement for this task. On a GRES-configured partition " - "its placement is then Slurm's, not sflow's.", - workflow_out_dir, + return [ + _GPU_PLACEMENT_BANNER + + _GPU_PLACEMENT_PRELUDE.format( + plan=cuda_visible_devices, marker=GPU_MARKER_FILE ) - return [] - # One shared copy, sourced. `.` and not `bash`: the script exports into this - # shell, which a child process could not do. - body = ( - f"export SFLOW_GPU_PLAN='{cuda_visible_devices}'\n" - f"export SFLOW_GPU_MARKER='{GPU_MARKER_FILE}'\n" - f'. "{staged}"\n' - ) - return [_GPU_PLACEMENT_BANNER + body + _GPU_PLACEMENT_FOOTER] + + _GPU_PLACEMENT_FOOTER + ] def _is_valid_container_image(image: str) -> bool: @@ -581,8 +535,8 @@ def build_command( if _has_container and all_mounts: command.add_opt("--container-mounts", ",".join(all_mounts)) - for arg in normalize_extra_args(filtered_extra_args): - command.add_arg(arg) + for arg in filtered_extra_args: + command.add_opt(arg) command.add_arg("bash") command.add_arg("-c") @@ -592,11 +546,7 @@ def build_command( script_body = "\n".join( [ *_slurm_runtime_env_prelude(), - *_gpu_placement_prelude( - envs.get("CUDA_VISIBLE_DEVICES"), - gpus_per_task=c.gpus_per_task, - workflow_out_dir=envs.get("SFLOW_WORKFLOW_OUTPUT_DIR"), - ), + *_gpu_placement_prelude(envs.get("CUDA_VISIBLE_DEVICES")), *list(script), ] ) diff --git a/src/sflow/samples/modular/inference_x_v2/benchmark_aiperf.yaml b/src/sflow/samples/modular/inference_x_v2/benchmark_aiperf.yaml index c2dd7f5..247ca73 100644 --- a/src/sflow/samples/modular/inference_x_v2/benchmark_aiperf.yaml +++ b/src/sflow/samples/modular/inference_x_v2/benchmark_aiperf.yaml @@ -59,6 +59,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/modular/inference_x_v2/bulk_input.csv b/src/sflow/samples/modular/inference_x_v2/bulk_input.csv index 90037c8..1f9962f 100644 --- a/src/sflow/samples/modular/inference_x_v2/bulk_input.csv +++ b/src/sflow/samples/modular/inference_x_v2/bulk_input.csv @@ -2,9 +2,9 @@ sflow_config_file,SLURM_NODES,GPUS_PER_NODE,NUM_CTX_SERVERS,CTX_TP_SIZE,NUM_GEN_ slurm_config.yaml common_workflow.yaml trtllm/prefill.yaml trtllm/decode.yaml benchmark_infmax.yaml,1,4,1,2,1,2,,,1,[16],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2,agg_server slurm_config.yaml common_workflow.yaml trtllm/prefill.yaml trtllm/decode.yaml benchmark_infmax.yaml,2,4,4,1,1,4,,,2,[32],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2,agg_server slurm_config.yaml common_workflow.yaml sglang/prefill.yaml sglang/decode.yaml benchmark_infmax.yaml,1,4,2,1,1,2,,,1,"[1,4]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,lmsysorg/sglang:v0.5.8.post1-cu130,agg_server -slurm_config.yaml common_workflow.yaml sglang/prefill.yaml sglang/decode.yaml benchmark_infmax.yaml,2,4,2,2,1,4,,,2,"[2,4,6]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0,agg_server +slurm_config.yaml common_workflow.yaml sglang/prefill.yaml sglang/decode.yaml benchmark_infmax.yaml,2,4,2,2,1,4,,,2,"[2,4,6]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0,agg_server slurm_config.yaml common_workflow.yaml vllm/prefill.yaml vllm/decode.yaml benchmark_infmax.yaml,2,4,2,1,3,2,,,1,[8],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0,agg_server slurm_config.yaml common_workflow.yaml vllm/prefill.yaml vllm/decode.yaml benchmark_infmax.yaml,3,4,4,1,1,8,,,3,"[64,32]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0,agg_server slurm_config.yaml common_workflow.yaml trtllm/agg.yaml benchmark_aiperf.yaml,1,4,,,,,1,4,1,"[1,3,5]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2,prefill_server decode_server -slurm_config.yaml common_workflow.yaml sglang/agg.yaml benchmark_aiperf.yaml,2,4,,,,,8,1,2,[10],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0,prefill_server decode_server +slurm_config.yaml common_workflow.yaml sglang/agg.yaml benchmark_aiperf.yaml,2,4,,,,,1,8,2,[10],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0,prefill_server decode_server slurm_config.yaml common_workflow.yaml vllm/agg.yaml benchmark_aiperf.yaml,1,4,,,,,2,2,1,[128],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0,prefill_server decode_server \ No newline at end of file diff --git a/src/sflow/samples/modular/inference_x_v2/common_workflow.yaml b/src/sflow/samples/modular/inference_x_v2/common_workflow.yaml index 6cf5b79..bb51329 100644 --- a/src/sflow/samples/modular/inference_x_v2/common_workflow.yaml +++ b/src/sflow/samples/modular/inference_x_v2/common_workflow.yaml @@ -54,7 +54,7 @@ variables: # Dependencies versions DYNAMO_VERSION: description: "Dynamo version to install when missing from the container image" - value: 1.3.0 + value: 0.8.0 NATS_VERSION: description: "NATS server version to install when missing" value: v2.10.28 diff --git a/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml b/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml index 5c770c1..2d18141 100644 --- a/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml +++ b/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml @@ -10,7 +10,7 @@ variables: - name: NUM_AGG_SERVERS description: Number of aggregated servers type: integer - value: 8 + value: 1 - name: CONCURRENCY description: Concurrency @@ -42,7 +42,7 @@ operators: container_writable: true mpi: pmix extra_args: - - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 + - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 - name: nginx type: srun @@ -101,8 +101,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 + echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 else echo "dynamo already installed." fi @@ -324,17 +324,17 @@ workflow: - export NODE_RANK=${SLURM_NODEID} - export FIRST_NODE_IP=$(echo ${SFLOW_TASK_ASSIGNED_NODE_IPS} | cut -d',' -f1) - | - if [ "1" -gt 1 ]; then - export MULTI_NODE_EXTRA_ARGS="--dist-init-addr ${FIRST_NODE_IP}:29500 --nnodes 1 --node-rank ${NODE_RANK}" + if [ "2" -gt 1 ]; then + export MULTI_NODE_EXTRA_ARGS="--dist-init-addr ${FIRST_NODE_IP}:29500 --nnodes 2 --node-rank ${NODE_RANK}" else export MULTI_NODE_EXTRA_ARGS="" fi - export VLLM_NIXL_SIDE_CHANNEL_PORT=$((5557 + ${FIRST_CUDA_DEVICE})) - export DYN_SYSTEM_PORT=$((8082 + ${FIRST_CUDA_DEVICE})) - - python3 -m dynamo.sglang --model-path ${{ artifacts.LOCAL_MODEL_PATH.path }} --served-model-name Qwen3-8B-FP8 --tensor-parallel-size 1 --pipeline-parallel-size 1 --data-parallel-size 1 --expert-parallel-size 1 --moe-dense-tp-size 1 --max-running-requests 128 --max-prefill-tokens 4096 --trust-remote-code --skip-tokenizer-init --cuda-graph-max-bs 128 ${MULTI_NODE_EXTRA_ARGS} --host 0.0.0.0 + - python3 -m dynamo.sglang --model-path ${{ artifacts.LOCAL_MODEL_PATH.path }} --served-model-name Qwen3-8B-FP8 --tensor-parallel-size 8 --pipeline-parallel-size 1 --data-parallel-size 1 --expert-parallel-size 1 --moe-dense-tp-size 1 --max-running-requests 128 --max-prefill-tokens 4096 --trust-remote-code --skip-tokenizer-init --cuda-graph-max-bs 128 ${MULTI_NODE_EXTRA_ARGS} --host 0.0.0.0 resources: gpus: - count: 1 + count: 8 depends_on: - frontend_server probes: @@ -355,7 +355,7 @@ workflow: - set -x - pip install aiperf==0.3.0 - export COLUMNS=200 - - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' + - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' - echo "Benchmarking finished" resources: nodes: diff --git a/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml b/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml index 61466d5..6d0b599 100644 --- a/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml +++ b/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml @@ -101,8 +101,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 + echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 else echo "dynamo already installed." fi diff --git a/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml b/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml index d5ef927..348a61b 100644 --- a/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml +++ b/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml @@ -49,7 +49,7 @@ operators: container_writable: true mpi: pmix extra_args: - - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 + - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 - name: nginx type: srun @@ -102,8 +102,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 + echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 else echo "dynamo already installed." fi diff --git a/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml b/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml index 5c4ff88..610c420 100644 --- a/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml +++ b/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml @@ -123,8 +123,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 + echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 else echo "dynamo already installed." fi @@ -378,7 +378,7 @@ workflow: - set -x - pip install aiperf==0.3.0 - export COLUMNS=200 - - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' + - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' - echo "Benchmarking finished" resources: nodes: diff --git a/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml b/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml index e8b8fea..bce8623 100644 --- a/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml +++ b/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml @@ -144,8 +144,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 + echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 else echo "dynamo already installed." fi @@ -390,6 +390,10 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 - name: decode_server operator: @@ -428,6 +432,10 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 - name: benchmark_infmax operator: diff --git a/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml b/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml index 6d054b7..4117bc9 100644 --- a/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml +++ b/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml @@ -144,8 +144,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 + echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 else echo "dynamo already installed." fi @@ -390,6 +390,10 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 - name: decode_server operator: @@ -428,6 +432,10 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 - name: benchmark_infmax operator: diff --git a/src/sflow/samples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml b/src/sflow/samples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml index 4319237..4be77a1 100644 --- a/src/sflow/samples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml +++ b/src/sflow/samples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml @@ -393,7 +393,7 @@ workflow: - set -x - pip install aiperf==0.3.0 - export COLUMNS=200 - - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' + - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' - echo "Benchmarking finished" resources: nodes: diff --git a/src/sflow/samples/modular/inference_x_v2/sglang/agg.yaml b/src/sflow/samples/modular/inference_x_v2/sglang/agg.yaml index 36efd83..06ddc8f 100644 --- a/src/sflow/samples/modular/inference_x_v2/sglang/agg.yaml +++ b/src/sflow/samples/modular/inference_x_v2/sglang/agg.yaml @@ -1,7 +1,7 @@ # SGLang aggregated (non-disaggregated) server task and its configuration. # Merge all files for a complete run: # sflow compose slurm_config.yaml common_workflow.yaml sglang/agg.yaml benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 \ +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 \ # --missable-tasks prefill_server --missable-tasks decode_server version: "0.1" diff --git a/src/sflow/samples/modular/inference_x_v2/sglang/decode.yaml b/src/sflow/samples/modular/inference_x_v2/sglang/decode.yaml index 4ee8ffe..3cc22e6 100644 --- a/src/sflow/samples/modular/inference_x_v2/sglang/decode.yaml +++ b/src/sflow/samples/modular/inference_x_v2/sglang/decode.yaml @@ -1,7 +1,7 @@ # SGLang decode server task and its configuration. # Merge all files for a complete run: # sflow run -f slurm_config.yaml -f common_workflow.yaml -f sglang_prefill.yaml -f sglang_decode.yaml -f benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 version: "0.1" diff --git a/src/sflow/samples/modular/inference_x_v2/sglang/prefill.yaml b/src/sflow/samples/modular/inference_x_v2/sglang/prefill.yaml index 828434d..62462e7 100644 --- a/src/sflow/samples/modular/inference_x_v2/sglang/prefill.yaml +++ b/src/sflow/samples/modular/inference_x_v2/sglang/prefill.yaml @@ -1,7 +1,7 @@ # SGLang prefill server task and its configuration. # Merge all files for a complete run: # sflow run -f slurm_config.yaml -f common_workflow.yaml -f sglang_prefill.yaml -f sglang_decode.yaml -f benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 version: "0.1" diff --git a/src/sflow/samples/modular/inference_x_v2/trtllm/agg.yaml b/src/sflow/samples/modular/inference_x_v2/trtllm/agg.yaml index 198f54e..b79ad63 100644 --- a/src/sflow/samples/modular/inference_x_v2/trtllm/agg.yaml +++ b/src/sflow/samples/modular/inference_x_v2/trtllm/agg.yaml @@ -1,7 +1,7 @@ # TensorRT-LLM aggregated (non-disaggregated) server task and its configuration. # Merge all files for a complete run: # sflow compose slurm_config.yaml common_workflow.yaml trtllm/agg.yaml benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 \ +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 \ # --missable-tasks prefill_server --missable-tasks decode_server version: "0.1" diff --git a/src/sflow/samples/modular/inference_x_v2/trtllm/decode.yaml b/src/sflow/samples/modular/inference_x_v2/trtllm/decode.yaml index 04e72dd..049e1bf 100644 --- a/src/sflow/samples/modular/inference_x_v2/trtllm/decode.yaml +++ b/src/sflow/samples/modular/inference_x_v2/trtllm/decode.yaml @@ -126,3 +126,7 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 diff --git a/src/sflow/samples/modular/inference_x_v2/trtllm/prefill.yaml b/src/sflow/samples/modular/inference_x_v2/trtllm/prefill.yaml index bc9bd33..74e62fe 100644 --- a/src/sflow/samples/modular/inference_x_v2/trtllm/prefill.yaml +++ b/src/sflow/samples/modular/inference_x_v2/trtllm/prefill.yaml @@ -124,3 +124,7 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 diff --git a/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_agg.yaml b/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_agg.yaml index 2c2044c..ef782fd 100644 --- a/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_agg.yaml +++ b/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_agg.yaml @@ -433,6 +433,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg.yaml b/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg.yaml index 40d6b4d..997852c 100644 --- a/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg.yaml +++ b/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg.yaml @@ -695,6 +695,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml b/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml index f925aea..2b3c995 100644 --- a/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml +++ b/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml @@ -719,6 +719,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_agg.yaml b/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_agg.yaml index 182117d..85c7416 100644 --- a/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_agg.yaml +++ b/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_agg.yaml @@ -383,6 +383,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml b/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml index 54cbdc7..4f5fccc 100644 --- a/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml +++ b/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml @@ -523,6 +523,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_agg.yaml b/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_agg.yaml index 520fd02..a4ed1a0 100644 --- a/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_agg.yaml +++ b/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_agg.yaml @@ -468,6 +468,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_disagg.yaml b/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_disagg.yaml index d363258..02d96d2 100644 --- a/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_disagg.yaml +++ b/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_disagg.yaml @@ -668,6 +668,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/aiperf_template.yaml b/src/sflow/samples/self_contained/slurm/aiperf_template.yaml index 13828ff..92087bc 100644 --- a/src/sflow/samples/self_contained/slurm/aiperf_template.yaml +++ b/src/sflow/samples/self_contained/slurm/aiperf_template.yaml @@ -109,6 +109,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/dynamo_sglang_agg.yaml b/src/sflow/samples/self_contained/slurm/dynamo_sglang_agg.yaml index fab64dd..5800360 100644 --- a/src/sflow/samples/self_contained/slurm/dynamo_sglang_agg.yaml +++ b/src/sflow/samples/self_contained/slurm/dynamo_sglang_agg.yaml @@ -115,7 +115,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 + value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -307,6 +307,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/dynamo_sglang_disagg.yaml b/src/sflow/samples/self_contained/slurm/dynamo_sglang_disagg.yaml index ca16e44..e551f75 100644 --- a/src/sflow/samples/self_contained/slurm/dynamo_sglang_disagg.yaml +++ b/src/sflow/samples/self_contained/slurm/dynamo_sglang_disagg.yaml @@ -170,7 +170,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 + value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -424,6 +424,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/dynamo_trtllm_agg.yaml b/src/sflow/samples/self_contained/slurm/dynamo_trtllm_agg.yaml index 000c9c6..771b71b 100644 --- a/src/sflow/samples/self_contained/slurm/dynamo_trtllm_agg.yaml +++ b/src/sflow/samples/self_contained/slurm/dynamo_trtllm_agg.yaml @@ -111,7 +111,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 + value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -321,6 +321,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/dynamo_trtllm_disagg.yaml b/src/sflow/samples/self_contained/slurm/dynamo_trtllm_disagg.yaml index 3e539a8..1db803e 100644 --- a/src/sflow/samples/self_contained/slurm/dynamo_trtllm_disagg.yaml +++ b/src/sflow/samples/self_contained/slurm/dynamo_trtllm_disagg.yaml @@ -154,7 +154,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 + value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -372,6 +372,10 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 - name: decode_server operator: @@ -415,6 +419,10 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 + retries: + count: 3 + interval: 30 + backoff: 2 - name: benchmark operator: @@ -439,6 +447,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/dynamo_vllm_agg.yaml b/src/sflow/samples/self_contained/slurm/dynamo_vllm_agg.yaml index 56f337e..fafb786 100644 --- a/src/sflow/samples/self_contained/slurm/dynamo_vllm_agg.yaml +++ b/src/sflow/samples/self_contained/slurm/dynamo_vllm_agg.yaml @@ -337,6 +337,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/dynamo_vllm_disagg.yaml b/src/sflow/samples/self_contained/slurm/dynamo_vllm_disagg.yaml index 4eeadcf..3f6c8ce 100644 --- a/src/sflow/samples/self_contained/slurm/dynamo_vllm_disagg.yaml +++ b/src/sflow/samples/self_contained/slurm/dynamo_vllm_disagg.yaml @@ -304,16 +304,6 @@ workflow: - set -x - echo ${CUDA_VISIBLE_DEVICES} - export FIRST_CUDA_DEVICE=$(echo ${CUDA_VISIBLE_DEVICES} | cut -d',' -f1) - # Port derived from the first visible device, so two workers sharing a - # node never collide. This works because sflow does NOT hand - # NVIDIA_VISIBLE_DEVICES to an srun step: the container therefore sees the - # node's GPUs with HOST numbering intact, and the first entry of - # CUDA_VISIBLE_DEVICES is a real host index -- unique per worker. - # - # Keep it this way deliberately. When NVIDIA_VISIBLE_DEVICES was exported, - # the runtime carved and renumbered every worker's devices from 0, so both - # prefill and decode computed 5557 and the second to bind died with - # "Address already in use". This line is the canary that found it. - export VLLM_NIXL_SIDE_CHANNEL_PORT=$((5557 + ${FIRST_CUDA_DEVICE})) - export NODE_RANK=${SLURM_NODEID} - export FIRST_NODE_IP=$(echo ${SFLOW_TASK_ASSIGNED_NODE_IPS} | cut -d',' -f1) @@ -391,16 +381,6 @@ workflow: - set -x - echo ${CUDA_VISIBLE_DEVICES} - export FIRST_CUDA_DEVICE=$(echo ${CUDA_VISIBLE_DEVICES} | cut -d',' -f1) - # Port derived from the first visible device, so two workers sharing a - # node never collide. This works because sflow does NOT hand - # NVIDIA_VISIBLE_DEVICES to an srun step: the container therefore sees the - # node's GPUs with HOST numbering intact, and the first entry of - # CUDA_VISIBLE_DEVICES is a real host index -- unique per worker. - # - # Keep it this way deliberately. When NVIDIA_VISIBLE_DEVICES was exported, - # the runtime carved and renumbered every worker's devices from 0, so both - # prefill and decode computed 5557 and the second to bind died with - # "Address already in use". This line is the canary that found it. - export VLLM_NIXL_SIDE_CHANNEL_PORT=$((5557 + ${FIRST_CUDA_DEVICE})) - export NODE_RANK=${SLURM_NODEID} - export FIRST_NODE_IP=$(echo ${SFLOW_TASK_ASSIGNED_NODE_IPS} | cut -d',' -f1) @@ -489,6 +469,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/gpu_placement_matrix.yaml b/src/sflow/samples/self_contained/slurm/gpu_placement_matrix.yaml deleted file mode 100644 index 0ea0493..0000000 --- a/src/sflow/samples/self_contained/slurm/gpu_placement_matrix.yaml +++ /dev/null @@ -1,366 +0,0 @@ -# GPU placement regression matrix: every combination that can break the slice. -# -# sflow plans a per-task CUDA_VISIBLE_DEVICES and re-applies it inside the step. -# Four things independently decide what a task ends up holding: -# -# * container or not -- pyxis/enroot pass through only the task's devices and -# RENUMBER them from 0, so a task planned for 2,3 sees -# them as 0,1. A bare srun step sees the whole node. -# * slice offset -- a plan starting at 0 survives renumbering by accident, -# which is why only the high-offset task ever broke. -# * concurrency -- two tasks sharing a node must land on disjoint GPUs. -# * node count -- a multi-node plan is ONE node's slice, applied -# independently on every node. sflow guarantees the slice -# is uniform (gpus.count is divided by the node count and -# every node takes the same offsets), so the same flat -# plan must resolve correctly in each node's own step. -# -# Counting GPUs is not enough: a task can hold the right NUMBER of the WRONG -# devices and look perfectly healthy. So the first task records EACH node's -# index -> UUID map on bare metal, before anything is carved, and every later task -# proves by UUID that it holds the physical GPUs it was planned for -- per node, -# because a multi-node task's nodes each have their own map. -# -# Asserting rather than echoing is the point. The recipes that only printed their -# devices passed while holding nothing at all -- a decode server planned for 2,3 -# inside a 2-GPU container numbered 0,1 saw no GPU, and the first CUDA call died -# reporting an unsupported architecture. -version: "0.1" - -variables: - SLURM_ACCOUNT: - description: "SLURM account" - value: your_slurm_account - SLURM_PARTITION: - description: "SLURM partition" - value: your_slurm_partition - SLURM_TIMELIMIT: - description: "SLURM time limit" - value: "00:10:00" - SLURM_NODES: - description: >- - Number of Slurm nodes. Two, so the multi-node cases are real: a per-node - slice that is right on node 0 and wrong on node 1 is invisible at one node, - and that is the shape every multi-node server recipe uses. - # Typed, because boxed_multinode_high multiplies it. Untyped it is a STRING, - # and `${{ variables.SLURM_NODES * 2 }}` repeats it into "22" rather than - # doubling it -- valid YAML, valid expression, wrong number. - type: integer - value: 2 - GPUS_PER_NODE: - description: "GPUs per node" - value: 4 - PLACEMENT_IMAGE: - description: "Any GPU-enabled image; nvidia-smi is injected by the runtime." - value: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda12.5.0 - -artifacts: - # Declared so sample/bulk tests can override the standard sample artifact. - - name: LOCAL_MODEL_PATH - uri: fs:///path/to/your/model - - # One assertion, mounted at the same path on every node, instead of the same - # forty lines of shell pasted into each task. EXPECT_HOST is the only input. - - name: ASSERT_PLACEMENT - uri: file://assert_placement.sh - content: | - # Prove by UUID that THIS rank holds the physical GPUs it was planned for. - # Reads EXPECT_HOST (planned HOST indices, comma separated) from the env. - set -u - me=$(basename "$SFLOW_TASK_OUTPUT_DIR") - run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") - # Per NODE, not per task: every rank of a multi-node task runs this body, and - # each node has its own index -> UUID map. Reading node 0's map on node 1 - # would compare against the wrong hardware and "pass" for the wrong reason. - node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" - map=$(ls "$run_dir"/node_gpu_map*/map."$node".txt 2>/dev/null | head -1) - - echo "SFLOW_GPU_PROBE task=$me node=$node planned_host=$EXPECT_HOST CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" - nvidia-smi -L | sed "s/^/SFLOW_GPU_PROBE $node visible: /" - - if [ -z "$map" ]; then - echo "FAIL: no index -> UUID map recorded for node $node" >&2 - exit 1 - fi - # Are the planned ordinals even addressable in THIS namespace? A carved - # container renumbers from 0, so host ordinals can name nothing at all. - # An UNSET CUDA_VISIBLE_DEVICES is legal and means "every visible device", - # so `-i` must be omitted rather than passed empty -- and it must not be - # dereferenced bare under `set -u`. - if ! got=$(nvidia-smi ${CUDA_VISIBLE_DEVICES:+-i "$CUDA_VISIBLE_DEVICES"} \ - --query-gpu=uuid --format=csv,noheader); then - echo "FAIL: CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-} is not addressable on $node" >&2 - exit 1 - fi - # Per-node filenames: ranks of a multi-node task share one output dir, so a - # single gpus.txt would have them overwriting each other's evidence. - printf '%s\n' "$got" | sort > "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" - # And are they the RIGHT ones? Resolve the planned HOST indices through this - # node's bare-metal map; holding the correct count of the wrong devices is - # the failure mode a count-only check cannot see. - for i in $(echo "$EXPECT_HOST" | tr ',' ' '); do - grep "^$i," "$map" | cut -d, -f2 - done | sort > "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" - if ! diff -q "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" \ - "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" >/dev/null; then - echo "FAIL: $node planned host GPU(s) $EXPECT_HOST, but holds different devices" >&2 - echo " want: $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt")" >&2 - echo " got : $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt")" >&2 - exit 1 - fi - echo "OK: $node holds host GPU(s) $EXPECT_HOST, confirmed by UUID" - -backends: - - name: slurm_cluster - type: slurm - default: true - account: ${{ variables.SLURM_ACCOUNT }} - partition: ${{ variables.SLURM_PARTITION }} - time: ${{ variables.SLURM_TIMELIMIT }} - nodes: ${{ variables.SLURM_NODES }} - gpus_per_node: ${{ variables.GPUS_PER_NODE }} - -operators: - # One rank per node throughout: the assertion describes what a NODE holds, and - # extra ranks would each re-run it against the same devices for no new signal. - - name: bare - type: srun - ntasks_per_node: 1 - - name: boxed - type: srun - container_image: ${{ variables.PLACEMENT_IMAGE }} - ntasks_per_node: 1 - -workflow: - name: gpu_placement_matrix - tasks: - # Ground truth, taken bare metal with no `resources.gpus` so nothing is carved - # and no placement prelude runs. Runs on EVERY node (no `resources.nodes`), and - # each rank writes its own file, so the multi-node assertions below have a map - # for the node they actually landed on. Every assertion is relative to this. - - name: node_gpu_map - operator: bare - script: - - env | grep SLURM - - | - set -u - node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" - nvidia-smi --query-gpu=index,uuid --format=csv,noheader | tr -d ' ' \ - > "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" - sed "s/^/SFLOW_GPU_PROBE node_map $node: /" "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" - - # --- bare srun: the step sees the whole node, so the planned ordinals must - # --- survive untouched. Both run at once, on disjoint halves. - - name: bare_low - operator: bare - depends_on: [node_gpu_map] - resources: - gpus: - indices: [0, 1] - script: - - export EXPECT_HOST="0,1" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - - name: bare_high - operator: bare - depends_on: [node_gpu_map] - resources: - gpus: - indices: [2, 3] - script: - - export EXPECT_HOST="2,3" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # --- containerised: the runtime already carved and renumbered, so re-applying - # --- host ordinals here is what broke. boxed_high is the regression case. - - name: boxed_low - operator: boxed - depends_on: [bare_low, bare_high] - resources: - gpus: - indices: [0, 1] - script: - - export EXPECT_HOST="0,1" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - - name: boxed_high - operator: boxed - depends_on: [bare_low, bare_high] - resources: - gpus: - indices: [2, 3] - script: - - export EXPECT_HOST="2,3" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # A single GPU at a high offset: renumbering turns 3 into 0, so an off-by-one - # in the remap shows up here and nowhere else. - - name: boxed_one_high - operator: boxed - depends_on: [boxed_low, boxed_high] - resources: - gpus: - indices: [3] - script: - - export EXPECT_HOST="3" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # --- MULTI-NODE. `gpus.count` is the TOTAL over the task's nodes and every - # --- node takes the SAME offsets, so count/nodes must divide evenly. One flat - # --- plan therefore has to resolve correctly in each node's own step -- and - # --- each node proves it against its OWN bare-metal map, so a slice that is - # --- right on node 0 and wrong on node 1 fails here instead of shipping. - - name: bare_multinode - operator: bare - depends_on: [boxed_one_high] - resources: - # `nodes.count` is what forces the spread: `gpus.count` alone would fit on - # one node and the planner would place it there. count/nodes = 1 GPU per - # node, so the plan is "0" and both nodes must resolve it against their own - # hardware. This is the divide-the-total path every server recipe uses. - nodes: - count: ${{ variables.SLURM_NODES }} - gpus: - count: ${{ variables.SLURM_NODES }} - script: - - export EXPECT_HOST="0" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # The multi-node case that matters: containerised AND at a high offset, so - # every renumbering hazard is in play on more than one node at once. Pinned - # indices repeat on each node, which is the other multi-node planner path. - - name: boxed_multinode_high - operator: boxed - depends_on: [bare_multinode] - resources: - gpus: - # A TOTAL instead of a topology. With no `nodes:` here, `indices` is - # the per-node slice and the planner spreads the task over - # count / len(indices) nodes -- 4 / 2 = 2. This is the one route where - # the node count is an OUTPUT of the GPU ask rather than an input, which - # is how a recipe states things when it knows how many GPUs it needs but - # not how they are laid out. - count: ${{ variables.SLURM_NODES * 2 }} # 2 = len(indices) below - indices: [2, 3] - script: - - export EXPECT_HOST="2,3" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # --- AUTO-PLANNED (`gpus.count`, no `indices`). Everything above pins the - # --- devices by hand, so it proves the RE-APPLY path but never the PLANNER. - # --- These let sflow choose, which is what every real recipe does: the task - # --- cannot state an expected slice up front, so it asserts against the plan - # --- the prelude exported ($SFLOW_GPU_PLAN) -- still resolved to UUIDs - # --- through this node's bare-metal map, so "the planner picked something" - # --- and "the step actually holds it" remain separate claims. - # --- - # --- Chained rather than concurrent: 2x2 + 4x1 + 8 would need 16 GPUs and the - # --- allocation has 8. - - # Two replicas of two GPUs: the packing case. The planner must hand replica 0 - # and replica 1 DISJOINT pairs on the same node, which is the collision the - # whole slice mechanism exists to prevent. - - name: bare_count_2x2 - operator: bare - depends_on: [boxed_multinode_high] - replicas: - count: 2 - policy: parallel - resources: - gpus: - count: 2 - script: - - export EXPECT_HOST="$SFLOW_GPU_PLAN" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - - name: boxed_count_2x2 - operator: boxed - depends_on: [bare_count_2x2] - replicas: - count: 2 - policy: parallel - resources: - gpus: - count: 2 - script: - - export EXPECT_HOST="$SFLOW_GPU_PLAN" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # One GPU each, four ways: the finest packing, and the shape most likely to - # hand two replicas the same device if the planner ever miscounts. - - name: bare_count_1x4 - operator: bare - depends_on: [boxed_count_2x2] - replicas: - count: 4 - policy: parallel - resources: - gpus: - count: 1 - script: - - export EXPECT_HOST="$SFLOW_GPU_PLAN" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - - name: boxed_count_1x4 - operator: boxed - depends_on: [bare_count_1x4] - replicas: - count: 4 - policy: parallel - resources: - gpus: - count: 1 - script: - - export EXPECT_HOST="$SFLOW_GPU_PLAN" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # More GPUs than one node has, with no `nodes:` hint: the planner has to - # spread it itself and divide the total evenly, so every node runs the same - # flat plan against different physical cards. - - name: bare_count_8 - operator: bare - depends_on: [boxed_count_1x4] - resources: - gpus: - count: 8 - script: - - export EXPECT_HOST="$SFLOW_GPU_PLAN" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - - name: boxed_count_8 - operator: boxed - depends_on: [bare_count_8] - resources: - gpus: - count: 8 - script: - - export EXPECT_HOST="$SFLOW_GPU_PLAN" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # Concurrency: boxed_low and boxed_high held the node at the same time, so - # their devices must be disjoint. Sharing one is the collision this placement - # logic exists to prevent, and it is invisible from inside either task. - - name: verify_disjoint - operator: bare - # Terminal on purpose: the e2e checker treats this task's OK line as proof - # that the WHOLE matrix ran, so it has to sit downstream of the last case. - depends_on: [boxed_count_8] - resources: - nodes: - indices: [0] - script: - - | - set -u - run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") - low=$(ls "$run_dir"/boxed_low*/gpus.*.txt 2>/dev/null | head -1) - high=$(ls "$run_dir"/boxed_high*/gpus.*.txt 2>/dev/null | head -1) - if [ -z "$low" ] || [ -z "$high" ]; then - echo "FAIL: concurrent tasks left no gpus.*.txt to compare" >&2 - exit 1 - fi - shared=$(sort "$low" "$high" | uniq -d) - if [ -n "$shared" ]; then - echo "FAIL: concurrent tasks shared GPU(s): $shared" >&2 - exit 1 - fi - echo "OK: concurrent tasks held disjoint GPUs" diff --git a/src/sflow/samples/self_contained/slurm/infmax_v1_ds_r1.yaml b/src/sflow/samples/self_contained/slurm/infmax_v1_ds_r1.yaml index c5c8dd5..c235712 100644 --- a/src/sflow/samples/self_contained/slurm/infmax_v1_ds_r1.yaml +++ b/src/sflow/samples/self_contained/slurm/infmax_v1_ds_r1.yaml @@ -140,7 +140,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 + value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 artifacts: # file type artifacts will be generated by sflow with the content defined in this yaml file, accessible as `${{ artifacts.NAME.path }}` in task scripts @@ -420,6 +420,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/monitor_mixed.yaml b/src/sflow/samples/self_contained/slurm/monitor_mixed.yaml index d16bcf0..5e4b000 100644 --- a/src/sflow/samples/self_contained/slurm/monitor_mixed.yaml +++ b/src/sflow/samples/self_contained/slurm/monitor_mixed.yaml @@ -2,24 +2,13 @@ version: "0.1" # All-in-one Slurm sample: one `sflow run` exercises several features at once, so # a single job on a real cluster verifies them together instead of one at a time. -# This is the broadest regression net in examples/ -- if only one recipe gets run -# on a new cluster, run this one. # -# Mixes four existing samples and adds monitoring: +# Mixes three existing samples and adds monitoring: # multi_backend.yaml -> two Slurm pools, two operators/images # auto_replica.yaml -> replicas + cross-task `${{ task.x[i]... }}` access # resource_release_after.yaml-> readiness probes + GPU reuse (release_after) -# gpu_placement_matrix.yaml -> GPU placement PROVEN by UUID: container vs bare, -# slice offset, concurrency, and multi-node # NEW -> monitor with a log-marker report window # -# The placement stage runs first and releases its GPUs before the servers start, so -# it adds assertions without adding nodes. It is the only part here that can catch a -# task holding the right NUMBER of the WRONG GPUs -- everything below merely echoes -# CUDA_VISIBLE_DEVICES, which is how a decode server planned for 2,3 passed while -# sitting in a 2-GPU container numbered 0,1 with no usable device at all. -# gpu_placement_matrix.yaml keeps the fuller matrix as a standalone 2-node recipe. -# # What to check after a run (//sflow_monitor/): # # MARKER-CLIPPED -- sflow_monitor/windowed/ (each with window.json): @@ -107,54 +96,6 @@ artifacts: - name: LOCAL_MODEL_PATH uri: fs:///path/to/your/model - # Placement proof, mounted at the same path on every node. Kept in a file:// - # artifact rather than pasted into each task: EXPECT_HOST is the only input, so - # one copy serves every case below. - - name: ASSERT_PLACEMENT - uri: file://assert_placement.sh - content: | - # Prove by UUID that THIS rank holds the physical GPUs it was planned for. - # Reads EXPECT_HOST (planned HOST indices, comma separated) from the env. - set -u - me=$(basename "$SFLOW_TASK_OUTPUT_DIR") - run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") - # Per NODE, not per task: every rank of a multi-node task runs this body, and - # each node has its own index -> UUID map. Reading node 0's map on node 1 - # would compare against the wrong hardware and "pass" for the wrong reason. - node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" - map=$(ls "$run_dir"/node_gpu_map*/map."$node".txt 2>/dev/null | head -1) - - echo "SFLOW_GPU_PROBE task=$me node=$node planned_host=$EXPECT_HOST CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" - nvidia-smi -L | sed "s/^/SFLOW_GPU_PROBE $node visible: /" - - if [ -z "$map" ]; then - echo "FAIL: no index -> UUID map recorded for node $node" >&2 - exit 1 - fi - # Are the planned ordinals even addressable in THIS namespace? A carved - # container renumbers from 0, so host ordinals can name nothing at all. - if ! got=$(nvidia-smi -i "$CUDA_VISIBLE_DEVICES" --query-gpu=uuid --format=csv,noheader); then - echo "FAIL: CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES is not addressable on $node" >&2 - exit 1 - fi - # Per-node filenames: ranks of a multi-node task share one output dir, so a - # single gpus.txt would have them overwriting each other's evidence. - printf '%s\n' "$got" | sort > "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" - # And are they the RIGHT ones? Resolve the planned HOST indices through this - # node's bare-metal map; holding the correct count of the wrong devices is - # the failure mode a count-only check cannot see. - for i in $(echo "$EXPECT_HOST" | tr ',' ' '); do - grep "^$i," "$map" | cut -d, -f2 - done | sort > "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" - if ! diff -q "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" \ - "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" >/dev/null; then - echo "FAIL: $node planned host GPU(s) $EXPECT_HOST, but holds different devices" >&2 - echo " want: $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt")" >&2 - echo " got : $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt")" >&2 - exit 1 - fi - echo "OK: $node holds host GPU(s) $EXPECT_HOST, confirmed by UUID" - # Two independent pools: each allocates its own Slurm job and its own nodes. backends: - name: gpu_pool @@ -182,11 +123,6 @@ operators: type: srun container_image: ${{ variables.CPU_IMAGE }} ntasks_per_node: 1 - # No container, so the placement ground truth is taken on the raw node before any - # runtime has renumbered anything. Reuses the host driver's nvidia-smi. - - name: worker_bare - type: srun - ntasks_per_node: 1 workflow: name: monitor_mixed_slurm @@ -199,112 +135,11 @@ workflow: enabled: true tasks: - # --------------------------------------------------------------------------- - # -- gpu_placement_matrix: PROVE the planned slice, by UUID, before anything - # -- else runs. Counting GPUs is not enough -- a task can hold the right NUMBER - # -- of the WRONG devices and look perfectly healthy, which is how a decode - # -- server planned for 2,3 ran in a 2-GPU container numbered 0,1 and died on - # -- its first CUDA call. The servers below only ECHO their devices, so they - # -- cannot catch that; these tasks can. - # -- - # -- This whole stage completes (and releases its GPUs) before env_check, so it - # -- costs no extra nodes and cannot collide with the servers' reservations. - # --------------------------------------------------------------------------- - - # Ground truth, bare metal, nothing carved. Runs on EVERY node (no - # `resources.nodes`), one file each, so the multi-node case below can check the - # node it actually landed on. - - name: node_gpu_map - operator: worker_bare - script: - - | - set -u - node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" - nvidia-smi --query-gpu=index,uuid --format=csv,noheader | tr -d ' ' \ - > "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" - sed "s/^/SFLOW_GPU_PROBE node_map $node: /" "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" - - # Bare srun sees the whole node, so a non-zero offset must survive untouched. - - name: placement_bare_high - operator: worker_bare - depends_on: [node_gpu_map] - resources: - gpus: - indices: [2, 3] - script: - - export EXPECT_HOST="2,3" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # Containerised, low and high offset, CONCURRENTLY on one node. A plan starting - # at 0 survives container renumbering by accident, which is why only the - # high-offset task ever broke -- both sides are needed to see it. - - name: placement_boxed_low - operator: worker_gpu - depends_on: [placement_bare_high] - resources: - gpus: - indices: [0, 1] - script: - - export EXPECT_HOST="0,1" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - - name: placement_boxed_high - operator: worker_gpu - depends_on: [placement_bare_high] - resources: - gpus: - indices: [2, 3] - script: - - export EXPECT_HOST="2,3" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # MULTI-NODE, containerised, high offset -- every renumbering hazard in play on - # more than one node at once. A multi-node plan is ONE node's slice applied - # independently per node, so a slice that is right on node 0 and wrong on node 1 - # is invisible at a single node. Pinned indices repeat on each node. - - name: placement_multinode_high - operator: worker_gpu - depends_on: [placement_boxed_low, placement_boxed_high] - resources: - nodes: - count: 2 - gpus: - indices: [2, 3] - script: - - export EXPECT_HOST="2,3" - - bash ${{ artifacts.ASSERT_PLACEMENT.path }} - - # Collisions are invisible from inside either task: boxed_low and boxed_high - # held one node at the same time, so their devices must be disjoint. - - name: placement_disjoint - operator: worker_bare - depends_on: [placement_multinode_high] - resources: - nodes: - indices: [0] - script: - - | - set -u - run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") - low=$(ls "$run_dir"/placement_boxed_low*/gpus.*.txt 2>/dev/null | head -1) - high=$(ls "$run_dir"/placement_boxed_high*/gpus.*.txt 2>/dev/null | head -1) - if [ -z "$low" ] || [ -z "$high" ]; then - echo "FAIL: concurrent tasks left no gpus.*.txt to compare" >&2 - exit 1 - fi - shared=$(sort "$low" "$high" | uniq -d) - if [ -n "$shared" ]; then - echo "FAIL: concurrent tasks shared GPU(s): $shared" >&2 - exit 1 - fi - echo "OK: concurrent tasks held disjoint GPUs" - # -- resource_release_after: grabs every GPU on a node, then frees them on # completion (no readiness probe -> sflow infers task_completion), so the # servers below can reuse them. - name: env_check operator: worker_gpu - depends_on: [placement_disjoint] resources: gpus: count: ${{ variables.GPUS_PER_NODE }} diff --git a/src/sflow/samples/self_contained/slurm/sglang_server_client.yaml b/src/sflow/samples/self_contained/slurm/sglang_server_client.yaml index 0594453..7409814 100644 --- a/src/sflow/samples/self_contained/slurm/sglang_server_client.yaml +++ b/src/sflow/samples/self_contained/slurm/sglang_server_client.yaml @@ -176,6 +176,7 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/trtllm_serve_disagg.yaml b/src/sflow/samples/self_contained/slurm/trtllm_serve_disagg.yaml index 478244c..37e870e 100644 --- a/src/sflow/samples/self_contained/slurm/trtllm_serve_disagg.yaml +++ b/src/sflow/samples/self_contained/slurm/trtllm_serve_disagg.yaml @@ -328,15 +328,6 @@ workflow: script: - set -x - echo ${CUDA_VISIBLE_DEVICES} - # GPU placement probe. sflow plans a per-task slice and re-applies it - # inside the step; this prints what the container ACTUALLY has next to - # what was planned, which is the only way to tell the two failure modes - # apart: if the planned ordinals are absent from `nvidia-smi -L`, the - # slice never reached the container (carved and renumbered by the - # runtime); if they are present, placement worked and the launcher chose - # the wrong device. Cheap, and it makes every future triage one grep. - - echo "SFLOW_GPU_PROBE task=${SLURM_JOB_NAME:-?} procid=${SLURM_PROCID:-?} localid=${SLURM_LOCALID:-?} planned_CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" - - nvidia-smi -L 2>&1 | sed "s/^/SFLOW_GPU_PROBE visible: /" || true - export TLLM_LOG_LEVEL=INFO - export TRTLLM_SERVER_DISABLE_GC=1 - export TRTLLM_WORKER_DISABLE_GC=1 @@ -383,15 +374,6 @@ workflow: script: - set -x - echo ${CUDA_VISIBLE_DEVICES} - # GPU placement probe. sflow plans a per-task slice and re-applies it - # inside the step; this prints what the container ACTUALLY has next to - # what was planned, which is the only way to tell the two failure modes - # apart: if the planned ordinals are absent from `nvidia-smi -L`, the - # slice never reached the container (carved and renumbered by the - # runtime); if they are present, placement worked and the launcher chose - # the wrong device. Cheap, and it makes every future triage one grep. - - echo "SFLOW_GPU_PROBE task=${SLURM_JOB_NAME:-?} procid=${SLURM_PROCID:-?} localid=${SLURM_LOCALID:-?} planned_CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" - - nvidia-smi -L 2>&1 | sed "s/^/SFLOW_GPU_PROBE visible: /" || true - export TLLM_LOG_LEVEL=INFO - export TRTLLM_SERVER_DISABLE_GC=1 - export TRTLLM_WORKER_DISABLE_GC=1 diff --git a/src/sflow/skills/writing-sflow-yaml/schema-reference.md b/src/sflow/skills/writing-sflow-yaml/schema-reference.md index e696994..6b2bea1 100644 --- a/src/sflow/skills/writing-sflow-yaml/schema-reference.md +++ b/src/sflow/skills/writing-sflow-yaml/schema-reference.md @@ -331,7 +331,6 @@ backends: # dra: { gpu_device_class: gpu.nvidia.com, rdma_device_class: rdma.nvidia.com } # compute_domain: { channel: auto } # Multi-Node NVLink (IMEX); top-level, not under dra # probe_pod_image: curlimages/curl:latest - # collect_node_local_output: true # default true; false = NO sflow collect machinery in the pod at all # collect_max_file_size: 10Mi # cap for syncing pod output back to driver (0 disables) # collect_grace_seconds: 120 # grace window (s) for copying node-local outputs back # reservation: { placeholder_image: my-mirror/bash:5 } # air-gapped placeholder-pod image @@ -579,8 +578,6 @@ The task then carries only its launch line (no keypair/hostfile/sshd/wait/`-x`): | `omp_num_threads` | int\|null | `8` | Per-rank `OMP_NUM_THREADS` (pod env, forwarded to ranks). Caps OpenMP so co-located ranks don't exhaust pthreads at model load; a recipe `export OMP_NUM_THREADS=...` overrides it; `null`/`0` disables | | `worker_setup_timeout_seconds` | int/expr | `900` | Per-node setup budget (image apt-install + weight staging) before a worker's readiness probe reaps it (operator route). Rendered as a probe with a fixed 5s poll and `failureThreshold = ceil(timeout/5)`. Raise for a large first-time weight download over slow storage | | `launcher_discovery_timeout` | int/expr | `600` | Seconds to wait for the mpi-operator controller to create the launcher pod after the MPIJob is applied (operator route) | -| `cpu_bind` | enum | `core` | `core` / `numa` / `none`. Per-rank CPU binding, injected **only when several ranks share a pod**, and never over a binding the recipe already passes. `core` gives each rank an isolated core slice — the tightest cap on the LLVM/OpenMP thread pools `OMP_NUM_THREADS` alone doesn't reach; `numa` binds one rank per NUMA domain (only partitions when the cpuset spans >1 domain); `none` injects nothing | -| `cpu_bind_cores_per_rank` | int | `8` | Upper bound on the cores bound to each rank under `cpu_bind: core`. Launch-time value is `min(cores-in-cpuset / ranks-per-pod, this)`, so a small cpuset still gets a smaller slice; if the cpuset has fewer cores than ranks the binding is skipped rather than failing the launch. `0` = uncapped | **Env forwarding.** `mpirun` over SSH gives remote ranks a *bare* environment, so env is forwarded with `-x`. sflow forwards a built-in transport/system set (`NCCL_`, diff --git a/src/sflow/utils/gpu.py b/src/sflow/utils/gpu.py index 355b4a6..329ba42 100644 --- a/src/sflow/utils/gpu.py +++ b/src/sflow/utils/gpu.py @@ -8,10 +8,7 @@ # Written by a job step that picks its own devices (slurm), read back by run # reporting. Lives here, next to its only reader, so the producer imports it. -# Deliberately NOT dot-prefixed: this is the placement record a human reads when -# a run looks mis-placed, and GitLab's artifact browser hides dot-files entirely -# (they ship in the zip but cannot be clicked). -GPU_MARKER_FILE = "sflow_gpus.log" +GPU_MARKER_FILE = ".sflow_gpus" def count_device_tokens(cuda_visible: str | None) -> int: @@ -43,33 +40,6 @@ def count_visible_devices(cuda_visible: str | None) -> int: ) -def task_gpu_record(task: Any) -> dict[str, str]: - """Parse the placement record a Slurm step leaves in :data:`GPU_MARKER_FILE`. - - Line 1 is the device list the step ended up using; the rest is ``key=value`` - (node, the branch taken, planned host indices, planned UUIDs) plus repeated - ``visible=``/``selected=`` lines. Returns {} when there is no record. - """ - envs = getattr(task, "envs", None) or {} - out_dir = envs.get("SFLOW_TASK_OUTPUT_DIR") - if not out_dir: - return {} - try: - text = (Path(out_dir) / GPU_MARKER_FILE).read_text() - except OSError: - return {} - lines = text.splitlines() - if not lines: - return {} - record: dict[str, str] = {"devices": lines[0].strip()} - for line in lines[1:]: - key, sep, value = line.partition("=") - # visible=/selected= repeat; the scalars are what callers need. - if sep and key not in ("visible", "selected"): - record[key] = value.strip() - return record - - def task_gpu_indices(task: Any) -> list[int]: """The **physical** GPU indices a task ran on, for run reporting. @@ -111,25 +81,13 @@ def task_gpu_indices(task: Any) -> list[int]: # node 0's devices for every node would be a confident wrong answer -- the plan, # which is uniform across nodes by construction, is the honest one there. if out_dir and len(getattr(task, "assigned_nodes", None) or []) <= 1: - record = task_gpu_record(task) - if record.get("action") == "verified": - # The step PROVED, by UUID, that it holds exactly the planned cards -- - # it exits 97 otherwise -- so the planned HOST indices are the physical - # ones, whatever the step's own numbering happened to be. - # - # Line 1 must not be used here: inside a carved container it is the - # container's numbering (a task planned for host 2,3 records 0,1), and - # reporting that as physical put every containerised task on the wrong - # card in the summary and made the monitor sample the wrong GPUs. - indices = parse_cuda_visible_devices(record.get("planned_host_indices", "")) - if indices: - return indices - # No verified record (older marker, or the driver could not probe the - # topology): line 1 is the best available answer, and under that path the - # step selected host ordinals, so it means what it used to mean. + try: + reported = (Path(out_dir) / GPU_MARKER_FILE).read_text() + except OSError: + reported = "" # An unparseable marker (e.g. CUDA UUID form) must fall through, not drop the # task out of run reporting entirely. - indices = parse_cuda_visible_devices(record.get("devices", "")) + indices = parse_cuda_visible_devices(reported.strip()) if indices: return indices return planned_gpu_indices(task) diff --git a/tests/conftest.py b/tests/conftest.py index f1e01b4..ecedbfa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,11 @@ import pytest +@pytest.fixture +def mock_config(): + return {} + + @pytest.fixture def image_warnings(): """Collect the warnings emitted for unrecognised container image references. diff --git a/tests/e2e_tests/sample_test.sh b/tests/e2e_tests/sample_test.sh index 739c0d9..aa0beb6 100755 --- a/tests/e2e_tests/sample_test.sh +++ b/tests/e2e_tests/sample_test.sh @@ -3,7 +3,7 @@ # set -x usage() { - echo "Usage: $0 -p -A -m [-G ] [-t s|m|inf|a|smoke|min|one] [--submit] [--check JOB_IDS] [-- ]" + echo "Usage: $0 -p -A -m [-G ] [-t s|m|inf|a|smoke|min] [--submit] [--check JOB_IDS] [-- ]" echo "" echo " -t s Self-contained examples only (--bulk-submit examples/self_contained/slurm/)" echo " -t m Modular examples only (--bulk-input modular/inference_x_v2/bulk_input.csv)" @@ -11,15 +11,6 @@ usage() { echo " -t a Both single and multi (default)" echo " -t smoke Curated Slurm smoke subset with broad coverage" echo " -t min Minimal representative set (one job per validation type)" - echo " -t one EXACTLY ONE Slurm job -- plumbing smoke, not coverage." - echo " Use it to prove a CI/cluster path end to end (submit -> run ->" - echo " sflow_output -> summary) before spending nodes on min/smoke." - echo "" - echo " SFLOW_E2E_RECIPE_CLASS=workload|sanity|all (env, default all)" - echo " Which HALF of the suite to submit. 'workload' is the" - echo " dynamo/trtllm/vllm/sglang/aiperf/infmax recipes (real servers," - echo " model loads, big pulls); 'sanity' is everything else and needs" - echo " no model at all. The two halves run on different clusters." echo "" echo " --check JOB_IDS Skip submission, only check results" echo " Accepts: comma-separated IDs and/or [START:END] ranges" @@ -157,168 +148,6 @@ colon_task_log_has_markers() { ' sh {} + 2>/dev/null } -workflow_summary_ok() { - # Every indicator below comes from a benchmark or client log, so a recipe that - # ships neither (monitor_mixed, gpu_indices, ...) scored FAIL however green it - # was. sflow already writes the authoritative verdict: sflow_summary.log says - # `Status : COMPLETED` only when every task reached COMPLETED/READY - # (core/execution_summary.py::_infer_status), and FAILED/CANCELLED/TIMEOUT on - # any other outcome. Judge those runs by that instead of per-recipe allowlists. - local dir="$1" - [ -n "$dir" ] || return 1 - local summary - summary=$(find "$dir" -maxdepth 2 -type f -name 'sflow_summary.log' 2>/dev/null | head -1) - [ -n "$summary" ] || return 1 - grep -Eq '^Status[[:space:]]*:[[:space:]]*COMPLETED[[:space:]]*$' "$summary" -} - -aiperf_tally_ok() { # -> 0 every aiperf run benchmarked, 1 one did not, 2 no aiperf here - # A benchmark that measured NOTHING is the one failure this suite could not see. - # aiperf 0.3.0 exits 0 even when every single request failed, so the task rc is - # 0, sflow's own `Status : COMPLETED` is green, and the run is scored PASS while - # its CSV holds nothing but `Error Request Count`. - # - # Not hypothetical: dynamo >= 1.3.0 dropped `ignore_eos` from its (strictly - # deserialized) NvExt struct, so one stale `--extra-inputs` 400'd all 1024 - # requests in six workflows -- and this suite printed "11/11 jobs passed". - # - # aiperf states the outcome itself, and it is unambiguous: - # Processed 1024 valid requests and 0 errors (1024 total). <- benchmarked - # Processed 0 valid requests and 1024 errors (1024 total). <- measured nothing - # - # Both markers this replaces were wrong. "0 valid" was read as a SUCCESS marker - # when it is precisely what a total failure prints -- that alone scored the six - # dead runs green -- and unanchored "0 errors" also matches "10 errors" and - # "1000 errors", so a mostly-failed run passed too. - # - # Returns 2, not 0, when no aiperf ran: "no evidence" is a different answer from - # "good evidence", and only the caller knows whether this recipe owed any. Same - # lesson as workload_placement_ok() -- a checker that passes on an empty - # directory scores a workflow that died before it started as a success. - local dir="$1" valid errors seen=0 bad=0 - [ -n "$dir" ] && [ -d "$dir" ] || return 2 - # grep -o pins the field positions, so the split below cannot drift: - # Processed valid requests and errors - while read -r _ valid _ _ _ errors _; do - seen=$((seen + 1)) - [ "${valid:-0}" -gt 0 ] && [ "${errors:-1}" -eq 0 ] && continue - bad=$((bad + 1)) - echo " AIPERF MEASURED NOTHING: ${valid:-?} valid / ${errors:-?} errors under $dir" >&2 - done < <(find "$dir" -type f -name '*.log' \ - -exec grep -hoE 'Processed [0-9]+ valid requests and [0-9]+ errors' {} + 2>/dev/null) - [ "$seen" -gt 0 ] || return 2 - [ "$bad" -eq 0 ] -} - -serving_tally_ok() { # -> 0 every benchmark_serving run completed, 1 one did not, 2 none here - # The benchmark_serving.py (InferenceX) half of the same question aiperf_tally_ok - # asks. The modular dynamo_benchmark rows drive this instead of aiperf, so - # without it they keep the old, far weaker gate. - # - # What the old markers did: PASS on `grep -l "Successful requests:"` -- the mere - # PRESENCE of the string -- and FAIL only on `Successful requests:\s+0\s*$`. - # So a run that completed 3 of 512 requests scored a clean PASS, and any - # zero-count formatted differently (trailing text, padding) slipped the FAIL too. - # - # benchmark_serving.py prints its own summary, and the run's own command line is - # in the same log, so the two can be cross-checked: - # python3 ... benchmark_serving.py ... --num-prompts 128 ... - # ============ Serving Benchmark Result ============ - # Successful requests: 128 - # Output token throughput (tok/s): 4132.77 - # - # A real benchmark therefore owes three things: it succeeded at all (got > 0), it - # generated tokens (throughput > 0 -- a run can "succeed" 512 times with empty - # responses, which is the same measured-nothing shape ignore_eos produced), and - # it completed the work it was ASKED for (got == --num-prompts). Every healthy - # run in CI matches exactly: 16/16, 32/32, 48/48, 128/128, 256/256, 512/512. - # - # want == 0 means no command line was captured in this log; the cross-check is - # then skipped rather than guessed at. Returns 2 for "no benchmark_serving here" - # for the same reason aiperf_tally_ok does -- no evidence is not good evidence. - local dir="$1" f seen=0 bad=0 - [ -n "$dir" ] && [ -d "$dir" ] || return 2 - while IFS= read -r f; do - seen=$((seen + 1)) - awk -v src="$f" ' - match($0, /--num-prompts[= ]+[0-9]+/) { - s = substr($0, RSTART, RLENGTH); gsub(/[^0-9]/, "", s); want = s + 0 - } - /Successful requests:/ { got = $NF + 0 } - /Output token throughput \(tok\/s\):/ { thpt = $NF + 0 } - END { - if (got > 0 && thpt > 0 && (want == 0 || got == want)) exit 0 - printf " BENCHMARK INCOMPLETE: %d/%d requests succeeded, %g tok/s -- %s\n", \ - got, want, thpt, src > "/dev/stderr" - exit 1 - } - ' "$f" || bad=$((bad + 1)) - done < <(find "$dir" -type f -name '*.log' \ - -exec grep -l 'Serving Benchmark Result' {} + 2>/dev/null) - [ "$seen" -gt 0 ] || return 2 - [ "$bad" -eq 0 ] -} - -recipe_is_client_only() { # -> 0 when the recipe starts no server of its own - # aiperf_template is a TEMPLATE: ONE CPU-only client task aimed at - # ${HEAD_NODE_IP}:8000, an endpoint it deliberately does NOT start -- the reader - # is meant to point it at a server they already run. Played standalone in CI - # nothing is listening, every request is ConnectionRefused, and aiperf cannot - # benchmark. That is the recipe working as designed, not a regression. - # - # So the claim here is deliberately narrow. Such a run is still expected to - # COMPLETE -- sflow `Status : COMPLETED`, benchmark task exit=0 -- it simply - # owes no metrics. Every OTHER recipe in the workload half still owes a real - # benchmark, which is the whole point of aiperf_tally_ok(). - # - # Structural rather than a name allowlist: a workflow declaring a single task - # cannot have started the server it benchmarks. aiperf_template declares 1; - # every serving recipe here declares 6-7 (servers + frontend + benchmark), and - # the modular compositions more. The recipe is copied into the run directory, - # so this reads what actually ran rather than what is on disk now. - local dir="$1" total=0 n yml - [ -n "$dir" ] && [ -d "$dir" ] || return 1 - for yml in "$dir"/*.y*ml; do - [ -f "$yml" ] || continue - n=$(sed -n '/^ tasks:/,$p' "$yml" | grep -cE '^ - name:') - total=$((total + n)) - done - [ "$total" -eq 1 ] -} - -gpu_placement_run_ok() { - # The placement matrix ships no benchmark: its result IS its assertions. Every - # task resolves its planned HOST indices through a per-node bare-metal - # index -> UUID map and exits non-zero on a mismatch, and verify_disjoint sits - # downstream of all of them -- so its OK line can only appear when every - # assertion passed on every node. - # - # Only per-task logs are searched (mindepth 2): sflow.log echoes each task's - # script verbatim, FAIL: branches included, so a recursive grep would match - # text that never ran. - local dir="$1" - [ -n "$dir" ] || return 1 - find "$dir" -mindepth 2 -maxdepth 2 -type f -name '*.log' \ - -exec grep -lF "OK: concurrent tasks held disjoint GPUs" {} + 2>/dev/null | - grep -q . || return 1 - # And the multi-node cases must actually have spanned nodes. A one-node - # allocation degenerates them into duplicates of the single-node cases, which - # still pass -- silently dropping the coverage they exist for. Each assertion - # names the node it proved, so two distinct names is the proof. - local task nodes found=0 - for task in "$dir"/*multinode*/; do - [ -d "$task" ] || continue - found=1 - # Per multi-node TASK. A union over every task in the run is not the same - # claim: two single-node tasks landing on different nodes satisfy it while - # both multi-node cases sat on one. - nodes=$(find "$task" -maxdepth 1 -type f -name '*.log' \ - -exec grep -hoE 'OK: [^ ]+ holds host GPU' {} + 2>/dev/null | sort -u | wc -l) - [ "${nodes:-0}" -ge 2 ] || return 1 - done - [ "$found" = 1 ] -} - is_multi_backend_result() { local jid="$1" local mbid @@ -373,50 +202,11 @@ if [ -z "$PARTITION" ] || [ -z "$ACCOUNT" ] || [ -z "$MODEL_PATH" ]; then usage fi -if [ "$TEST_TYPE" != "s" ] && [ "$TEST_TYPE" != "m" ] && [ "$TEST_TYPE" != "inf" ] && [ "$TEST_TYPE" != "a" ] && [ "$TEST_TYPE" != "smoke" ] && [ "$TEST_TYPE" != "min" ] && [ "$TEST_TYPE" != "one" ]; then - echo "ERROR: -t must be 's', 'm', 'inf', 'a', 'smoke', 'min', or 'one', got '$TEST_TYPE'" +if [ "$TEST_TYPE" != "s" ] && [ "$TEST_TYPE" != "m" ] && [ "$TEST_TYPE" != "inf" ] && [ "$TEST_TYPE" != "a" ] && [ "$TEST_TYPE" != "smoke" ] && [ "$TEST_TYPE" != "min" ]; then + echo "ERROR: -t must be 's', 'm', 'inf', 'a', 'smoke', or 'min', got '$TEST_TYPE'" usage fi -# Which HALF of the suite this run submits. A class, not a new -t mode, so it -# composes with every existing mode instead of multiplying them. -# -# workload -- dynamo / trtllm / vllm / sglang / infmax / aiperf. Real servers, -# real model loads, multi-GB image pulls, many GPU-hours. Belongs on -# the cluster that has the quota for it. -# sanity -- everything else: GPU placement, replicas, resource release, the -# monitor and multi-backend recipes. No framework and no model -- -# they only DECLARE LocalModelPath so the harness's `-a` override is -# accepted, and never read it -- so this half runs on any cluster -# with GPUs and is cheap enough to play constantly. -# all -- both. The default, so a manual or local run is unchanged. -RECIPE_CLASS="${SFLOW_E2E_RECIPE_CLASS:-all}" -if [ "$RECIPE_CLASS" != "all" ] && [ "$RECIPE_CLASS" != "workload" ] && [ "$RECIPE_CLASS" != "sanity" ]; then - echo "ERROR: SFLOW_E2E_RECIPE_CLASS must be 'all', 'workload' or 'sanity', got '$RECIPE_CLASS'" - exit 1 -fi - -is_workload_recipe() { - # Named by the framework they drive, which is exactly the line the split is - # drawn on. Everything else is a functionality check. - case "$(basename "$1")" in - dynamo_*|trtllm_*|sglang_*|vllm_*|infmax_*|aiperf_*) return 0 ;; - esac - return 1 -} - -recipe_in_class() { - # -t one is a single named plumbing smoke (the GPU placement matrix), and its - # job is to prove THIS cluster's path end to end. Both halves want that, so it - # is never filtered out. - [ "$TEST_TYPE" = "one" ] && return 0 - case "$RECIPE_CLASS" in - workload) is_workload_recipe "$1" ;; - sanity) ! is_workload_recipe "$1" ;; - *) return 0 ;; - esac -} - if [ ${#EXTRA_BATCH_ARGS[@]} -gt 0 ]; then echo "Extra sflow batch args: ${EXTRA_BATCH_ARGS[*]}" fi @@ -695,10 +485,6 @@ submit_colon_task_script_e2e() { if [ "$TEST_TYPE" != "s" ] && [ "$TEST_TYPE" != "a" ] && [ "$TEST_TYPE" != "smoke" ]; then return fi - # Functionality check, no framework: it belongs to the sanity half -- which - # runs via `sflow run` (run_sanity_recipes_with_sflow_run), not batch. So this - # batch-based path is only for the combined default. - [ "$RECIPE_CLASS" != "all" ] && return local colon_dir="${SFLOW_COLON_SCRIPT_OUTPUT_DIR:-$E2E_OUTPUT_DIR/colon_in_task_script}" local colon_sbatch="$colon_dir/colon_in_task_script.sh" @@ -713,23 +499,6 @@ submit_colon_task_script_e2e() { # the shared E2E_BATCH_EXTRA_ARGS), so enable the workflow monitor explicitly -- # otherwise this job would be the one workflow without an sflow_monitor.log and # would trip the independent monitor-coverage gate below. - # Built as an array rather than a nested ${a:-${b:-...}} default: with no nodes - # to exclude that expression collapses to a bare `--exclude=`, which is not the - # same as passing nothing. - COLON_EXTRA_ARGS=() - if [ -n "${SFLOW_COLON_SCRIPT_EXTRA_ARGS:-}" ]; then - COLON_EXTRA_ARGS=(-e "$SFLOW_COLON_SCRIPT_EXTRA_ARGS") - elif [ -n "${SLURM_E2E_EXCLUDE_NODES:-}" ]; then - COLON_EXTRA_ARGS=(-e "--exclude=$SLURM_E2E_EXCLUDE_NODES") - fi - COLON_SEGMENT="${SLURM_E2E_SEGMENT:-}" - if [ "$COLON_SEGMENT" = "auto" ]; then - COLON_SEGMENT='${{SLURM_NODES}}' - fi - if [ -n "$COLON_SEGMENT" ]; then - COLON_EXTRA_ARGS+=(-e "--segment=$COLON_SEGMENT") - fi - colon_output=$(sflow batch -f "$SFLOW_COLON_SCRIPT_FIXTURE" \ -p "$PARTITION" -A "$ACCOUNT" --log-level warn \ "${BATCH_WORKSPACE_ARGS[@]}" \ @@ -737,7 +506,7 @@ submit_colon_task_script_e2e() { --output-dir "$colon_dir" \ --job-name "colon_in_task_script" \ --enable-workflow-monitor \ - "${COLON_EXTRA_ARGS[@]}" \ + -e "${SFLOW_COLON_SCRIPT_EXTRA_ARGS:---exclude=${SLURM_E2E_EXCLUDE_NODES:-gb-nvl-137-compute02,gb-nvl-137-compute14}}" \ -o "$colon_sbatch" \ $SUBMIT 2>&1) colon_status=$? @@ -761,463 +530,6 @@ submit_colon_task_script_e2e() { COLON_JOB_IDS+=("$colon_job_id") } -# Strip a suite-wide `-e --segment=...` for ONE `sflow batch` call, into -# STRIPPED_SEGMENT_ARGS. That value is an sflow expression sized to a recipe's own -# SLURM_NODES, so a config WITHOUT that variable passes it to sbatch VERBATIM as -# "#SBATCH --segment=${{SLURM_NODES}}". Both multi-backend recipes are in that -# position (they fix their backends' node counts instead), and a single salloc-wide -# segment could not be right for two differently sized backends anyway. -# -# One copy, two callers on purpose: a second hand-written twin of this predicate -# drifting is how the verbatim expression reached sbatch to begin with. -strip_suite_segment_args() { - STRIPPED_SEGMENT_ARGS=() - local i=0 - local n=${#EXTRA_BATCH_ARGS[@]} - while [ "$i" -lt "$n" ]; do - local arg="${EXTRA_BATCH_ARGS[$i]}" - local next="" - if [ $((i + 1)) -lt "$n" ]; then - next="${EXTRA_BATCH_ARGS[$((i + 1))]}" - fi - case "$arg:$next" in - "-e:--segment="*) - i=$((i + 2)) - continue - ;; - esac - STRIPPED_SEGMENT_ARGS+=("$arg") - i=$((i + 1)) - done -} - -# The second partition every two-backend recipe needs. Defaults to the partition -# this run was given: hardcoded names are cluster-specific and simply do not exist -# elsewhere -- the stale genesisq / gamoraq defaults meant every submission on a new -# cluster died with "no Slurm allocation granted". Two DIFFERENT partitions are -# better coverage, so CI sets SLURM_E2E_PARTITION_B (ptyche: backfill) and an -# operator can still override per recipe. -E2E_PARTITION_B="${SLURM_E2E_PARTITION_B:-$PARTITION}" - -# ============================================================================= -# The SANITY half: one `sflow run` per recipe, concurrently, no sbatch. -# ============================================================================= -# `sflow batch --submit` sbatches a DRIVER that runs ON a compute node, so the -# checkout, the venv and the output dir all have to be visible from there. On a -# cluster whose login-node $HOME is not exported to the compute nodes that is -# simply impossible. `sflow run` keeps the driver on the login node and only -# srun's the task steps out, which is why this half does not batch. -# -# It also makes the verdict honest. A batched job is scored by hunting for a -# "success indicator" in its logs; a subprocess just has an EXIT CODE, and sflow -# already exits non-zero when any task fails. So none of the indicator guessing -# applies here -- rc is the answer. -# -# NOTE: recipes whose tasks exchange files through $SFLOW_TASK_OUTPUT_DIR -# (gpu_placement_matrix reads the node -> UUID map written by another task, and -# verify_disjoint compares two tasks' device lists) still need the OUTPUT DIR on -# storage the compute nodes share. Point -o/E2E_OUTPUT_DIR at shared scratch; -# only the driver moved, the task steps still run out on the nodes. -sanity_recipe_set_args() { - # `--set` of a variable a config does not declare is a hard error, and so is - # `--artifact` of an artifact it does not declare -- which is why these are - # built per recipe rather than shared. The colon fixture needs no model and - # declares no artifacts at all, so handing it the standard - # `-a LOCAL_MODEL_PATH=` killed it before the workflow started: - # "Artifact 'LOCAL_MODEL_PATH' specified in overrides is not defined". - if grep -qE '^[[:space:]]*-?[[:space:]]*name:[[:space:]]*LOCAL_MODEL_PATH' "$1"; then - printf '%s\n' "--artifact" "LOCAL_MODEL_PATH=fs://$MODEL_PATH" - fi - # Cap the Slurm allocation. This is the ONLY cap that bites: a recipe's - # workflow-level `timeout:` is accepted by the schema and enforced by nothing - # (see TaskConfig.timeout), so without --time a server that never becomes - # ready holds its nodes for the recipe's own limit -- up to 120 minutes for - # the workload recipes, which on this cluster cannot serve at all and will - # always wait the full time. - # - # Grepped, not assumed, for the same reason as the model artifact above: - # `--set` of a variable a config does not declare is a hard error, and the - # two spellings are NOT interchangeable -- multi_backend.yaml calls it - # TIME_LIMIT, everything else SLURM_TIMELIMIT, and the colon fixture declares - # neither. Unset means "leave each recipe's own value alone". - if [ -n "${SFLOW_E2E_SLURM_TIMELIMIT:-}" ]; then - if grep -qE '^[[:space:]]*SLURM_TIMELIMIT:' "$1"; then - printf '%s\n' "--set" "SLURM_TIMELIMIT=$SFLOW_E2E_SLURM_TIMELIMIT" - elif grep -qE '^[[:space:]]*TIME_LIMIT:' "$1"; then - printf '%s\n' "--set" "TIME_LIMIT=$SFLOW_E2E_SLURM_TIMELIMIT" - fi - fi - case "$(basename "$1")" in - multi_backend.yaml) - printf '%s\n' "--set" "PARTITION_A=$PARTITION" \ - "--set" "PARTITION_B=${E2E_PARTITION_B:-$PARTITION}" \ - "--set" "SLURM_ACCOUNT=$ACCOUNT" - ;; - monitor_mixed.yaml) - printf '%s\n' "--set" "PARTITION_A=$PARTITION" \ - "--set" "PARTITION_B=${E2E_PARTITION_B:-$PARTITION}" \ - "--set" "SLURM_ACCOUNT=$ACCOUNT" \ - "--set" "GPUS_PER_NODE=$GPUS_PER_NODE" - ;; - *) - printf '%s\n' "--set" "SLURM_PARTITION=$PARTITION" \ - "--set" "SLURM_ACCOUNT=$ACCOUNT" \ - "--set" "GPUS_PER_NODE=$GPUS_PER_NODE" - ;; - esac -} - -gpu_placement_verified() { # -> 0 when every GPU task PROVED its placement - # "The workflow completed" says nothing about WHICH cards it used, and until - # now only gpu_placement_matrix checked that -- every other recipe (gpu_indices - # pins devices! auto_replica and resource_release_after pack and re-use them) - # was scored on completion alone, and the workload half checked no placement at - # all. - # - # Every GPU task now leaves sflow_gpus.log recording what it was PLANNED for - # and what it actually SELECTED, both as physical UUIDs, so this audits any - # recipe without the recipe having to assert anything. - # - # A task whose record says `fallback`/`unverified` did not prove its placement: - # the driver could not resolve the plan to UUIDs (probe failed, node names - # disagree, gpus_per_node larger than the node really has) or the step had no - # nvidia-smi. On these clusters that is a regression, not a normal mode -- and - # it is precisely the silent degradation this suite exists to catch. - local dir="$1" - [ -n "$dir" ] && [ -d "$dir" ] || return 0 - local rec action planned selected total=0 unproven=0 verified=0 - while IFS= read -r rec; do - [ -n "$rec" ] || continue - total=$((total + 1)) - action=$(sed -n 's/^action=//p' "$rec" | head -1) - planned=$(sed -n 's/^planned_uuids=//p' "$rec" | head -1) - # The UUID of each device CUDA will really use, in the order it will see them. - selected=$(sed -n 's/^selected=[0-9?]* //p' "$rec" | paste -sd, -) - case "$action" in - verified) verified=$((verified + 1)) ;; - *) - # `fallback` (no UUID map, or Slurm granted cards the plan never - # named) and `unverified` (no nvidia-smi) both mean the placement - # was not proven against physical devices. - unproven=$((unproven + 1)) - echo " GPU PLACEMENT UNPROVEN: $(dirname "$rec" | xargs basename) recorded action='${action:-none}' in $rec" >&2 - continue - ;; - esac - if [ "$selected" != "$planned" ]; then - unproven=$((unproven + 1)) - echo " GPU PLACEMENT MISMATCH: $(dirname "$rec" | xargs basename) planned '$planned' but holds '$selected' ($rec)" >&2 - fi - done < <(find "$dir" -type f -name 'sflow_gpus*.log' 2>/dev/null | sort) - - if [ "$total" -eq 0 ]; then - # No GPU task in this workflow (or none reached the prelude). Nothing to - # prove; the recipe's own verdict still applies. - return 0 - fi - echo " GPU placement: $verified/$total task record(s) proven by UUID" - [ "$unproven" -eq 0 ] -} - -recipe_requests_gpus() { # -> 0 when a TASK asks for GPUs - # A `gpus:` block inside a task's `resources:`. Deliberately NOT - # `gpus_per_node:`, which is the BACKEND's allocation shape -- a recipe can - # size an allocation and still run nothing on a GPU. aiperf_template is - # exactly that: one CPU-only benchmark-client task (`resources: nodes:`), - # gpus_per_node on the backend, and no GPU task anywhere. - grep -qE '^[[:space:]]+gpus:[[:space:]]*$' "$1" -} - -workload_placement_ok() { # -> 0 when placement is PROVEN - # The verdict for a recipe whose application is expected to fail. Placement is - # the only claim, so it must be a POSITIVE one: gpu_placement_verified() - # answers 0 when it finds no records at all -- correct for a CPU-only recipe, - # badly wrong here, because a workload that died before any task started would - # score PASS on zero evidence. Require records to exist, then require every - # one of them to be proven. - local recipe="$1" dir="$2" - [ -n "$dir" ] && [ -d "$dir" ] || return 1 - # No GPU task => no placement to prove, ever. Demanding a record here would - # fail such a recipe on every run forever, which is what happened to - # aiperf_template: it completed cleanly and was scored FAIL for producing - # evidence it structurally cannot produce. Fall back to its own verdict -- - # the only signal that means anything for a recipe holding no GPU. - if ! recipe_requests_gpus "$recipe"; then - workflow_summary_ok "$dir" - return - fi - # It DOES ask for GPUs, so a missing record means no GPU task ever reached the - # placement prelude -- the workflow died first. That is unproven, not passing. - find "$dir" -type f -name 'sflow_gpus*.log' -print -quit 2>/dev/null | grep -q . || return 1 - gpu_placement_verified "$dir" -} - -sanity_recipe_content_ok() { # -> 0 when the run PROVED itself - # rc == 0 only says every task exited 0. It does NOT say the workflow did the - # thing it exists to prove, and these recipes exist to prove something: - # * gpu_placement_matrix can exit 0 while its assertions never ran, or while - # a one-node allocation quietly collapsed the multi-node cases -- so read - # the by-UUID evidence and the 2-distinct-node proof. - # * multi_backend can exit 0 with both backends on the SAME node, which is - # precisely the binding it is meant to disprove. - # * the colon fixture can exit 0 without ever emitting its markers. - # * everything else: sflow's own Status must say COMPLETED. - local name="$1" dir="$2" - [ -n "$dir" ] && [ -d "$dir" ] || return 1 - case "$name" in - gpu_placement_matrix) gpu_placement_run_ok "$dir" ;; - multi_backend*) multi_backend_run_ok "$dir" ;; - colon_in_task_script) colon_task_log_has_markers "$dir" ;; - *) workflow_summary_ok "$dir" ;; - esac -} - -sflow_run_safe_args() { - # EXTRA_BATCH_ARGS is built for `sflow batch`; two kinds of it do not carry - # over to `sflow run`. - # - # --sbatch-output / --sbatch-error name the sbatch JOB's stdout/stderr - # files, and there is no sbatch job here: the driver runs on the login - # node and its output is already captured per recipe. `sflow run` rejects - # them outright ("No such option: --sbatch-output"), which killed every - # sanity recipe before it started. - # -e is this harness's channel for raw Slurm submission flags - # (--segment=, --exclude=). On `sflow run` those are properties of the - # ALLOCATION, so they belong to --extra-salloc-args rather than the - # generic --extra-args. In practice the list is usually empty here: - # --segment is a GB200 requirement and the sanity cluster does not set it. - local out=() skip=0 want_salloc=0 a - for a in "$@"; do - if [ "$skip" = 1 ]; then skip=0; continue; fi - if [ "$want_salloc" = 1 ]; then - want_salloc=0 - out+=("--extra-salloc-args" "$a") - continue - fi - case "$a" in - --sbatch-*=*) ;; # value is inline; drop this token only - --sbatch-*) skip=1 ;; # value is the NEXT token; drop both - -e|--extra-args) want_salloc=1 ;; - *) out+=("$a") ;; - esac - done - [ ${#out[@]} -eq 0 ] || printf '%s\n' "${out[@]}" -} - -run_sanity_recipes_with_sflow_run() { - local recipes=() - local f - for f in "$EXAMPLES_DIR"/self_contained/slurm/*.yaml; do - [ "$TEST_TYPE" != "one" ] || [ "${f##*/}" = "gpu_placement_matrix.yaml" ] || continue - if recipe_in_class "$f"; then - recipes+=("$f") - elif [ "${SFLOW_E2E_INCLUDE_WORKLOAD_PLACEMENT:-0}" = "1" ] && is_workload_recipe "$f"; then - # Workload recipes on a cluster that cannot actually serve. They ride - # `sflow run` (not batch --submit) because that is the only path this - # cluster supports, and they are judged on PLACEMENT ALONE -- see the - # verdict below. - recipes+=("$f") - fi - done - # The colon-in-task-script fixture is written by full_sample_tests.sh; it is a - # functionality check like the rest, so it rides this half when present. - if [ -n "${SFLOW_COLON_SCRIPT_FIXTURE:-}" ] && [ -f "$SFLOW_COLON_SCRIPT_FIXTURE" ]; then - recipes+=("$SFLOW_COLON_SCRIPT_FIXTURE") - fi - if [ ${#recipes[@]} -eq 0 ]; then - echo "ERROR: no sanity recipes selected" - exit 1 - fi - - echo "" - local par_note="all at once" - [ "${SFLOW_E2E_MAX_PARALLEL:-0}" -gt 0 ] && par_note="${SFLOW_E2E_MAX_PARALLEL} at a time" - echo "===== Sanity half: ${#recipes[@]} recipe(s) via \`sflow run\` (no sbatch), $par_note =====" - echo "" - - # All at once by default. Every recipe is its own allocation, so the cluster's - # own scheduler is what orders them -- a starved salloc queues rather than - # failing, and the Slurm --time cap (SFLOW_E2E_SLURM_TIMELIMIT) is what stops - # a hung one holding nodes. SFLOW_E2E_MAX_PARALLEL>0 throttles to waves for a - # cluster where that is not welcome; 0/unset means no limit. - local names=() logs=() roots=() kinds=() files=() - local max_par="${SFLOW_E2E_MAX_PARALLEL:-0}" - for f in "${recipes[@]}"; do - local name set_args=() - name=$(basename "$f" .yaml) - mapfile -t set_args < <(sanity_recipe_set_args "$f") - local run_args=() - mapfile -t run_args < <(sflow_run_safe_args "${EXTRA_BATCH_ARGS[@]+"${EXTRA_BATCH_ARGS[@]}"}") - local root="$E2E_OUTPUT_DIR/$name" - local log="$root/${name}.sflow_run.log" - rm -rf "$root" - mkdir -p "$root" - # Guarded on >0: `-ge 0` is always true, and `wait -n` with no children - # returns immediately, so an unguarded loop would spin instead of launch. - if [ "$max_par" -gt 0 ]; then - while [ "$(jobs -rp | wc -l)" -ge "$max_par" ]; do wait -n; done - fi - echo " launching $name" - # Each run records its OWN exit status. `wait -n` above reaps children as - # they finish, so a later `wait $pid` would hit "not a child of this - # shell" and report 127 for a run that actually passed. - ( - sflow run -f "$f" \ - "${set_args[@]}" \ - --output-dir "$root" \ - --enable-workflow-monitor \ - "${run_args[@]+"${run_args[@]}"}" \ - > "$log" 2>&1 - echo $? > "$root/.rc" - ) & - names+=("$name") - logs+=("$log") - roots+=("$root") - files+=("$f") - if is_workload_recipe "$f"; then kinds+=("workload"); else kinds+=("sanity"); fi - done - wait - - echo "" - echo "===== Scoring ${#names[@]} sflow run(s) =====" - local i rc - for i in "${!names[@]}"; do - # Missing .rc means the subshell never got to write one -- treat as failure. - rc=$(cat "${roots[$i]}/.rc" 2>/dev/null || echo 1) - # No job id to look a run up by later, so find where it landed; the - # content check and the monitor gate below both read it. - local run_dir - run_dir=$(ls -d "${roots[$i]}"/*/ 2>/dev/null | head -1) - SANITY_RUN_DIRS+=("${run_dir:-$E2E_OUTPUT_DIR/${names[$i]}-NOT-FOUND}") - SANITY_RUN_NAMES+=("${names[$i]}") - - TOTAL=$((TOTAL + 1)) - if [ "${kinds[$i]}" = "workload" ]; then - # This cluster's GPUs cannot run real LLM inference, so the framework - # WILL fail and its own verdict answers nothing. These recipes are here - # for one reason -- to prove GPU placement on a second cluster and a - # second GPU generation -- so that is the entire test. The app's exit - # status is deliberately ignored; a placement regression is not. - # Say which of the two things actually happened. workload_placement_ok - # passes for two different reasons and one message for both CLAIMED - # EVIDENCE THAT DOES NOT EXIST: aiperf_template holds no GPU, wrote no - # record, and still reported "placement proven by UUID" -- the exact - # kind of line that misleads whoever audits these artifacts later. - local proved="placement proven by UUID; app rc=$rc ignored on this cluster" - local unproved="GPU placement not proven" - if ! recipe_requests_gpus "${files[$i]}"; then - proved="no GPU task, so no placement to prove; sflow reports COMPLETED" - unproved="no GPU task to place, and sflow's own verdict is not COMPLETED" - fi - # Say what aiperf actually measured here, without gating on it. This - # cluster cannot serve a real model, so demanding a benchmark would fail - # these recipes forever -- but a bare PASS next to an aiperf that - # measured nothing is how the ptyche half stayed green for six workflows. - # Whoever audits these artifacts should not have to open the CSV to find - # that out. aiperf_tally_ok() already prints the counts to stderr. - aiperf_tally_ok "$run_dir" - case $? in - 0) proved="$proved; aiperf benchmarked" ;; - 1) proved="$proved; aiperf measured nothing (expected here, not gated)" ;; - esac - if workload_placement_ok "${files[$i]}" "$run_dir"; then - PASSED=$((PASSED + 1)) - echo " ${names[$i]}: PASS ($proved)" - else - echo " ${names[$i]}: FAIL ($unproved; see ${run_dir:-${logs[$i]}})" - fi - elif [ "$rc" -ne 0 ]; then - if cuda_infra_failure "${roots[$i]}"; then - mark_cuda_excused "${names[$i]}" "${logs[$i]}" "(sflow run rc=$rc with a CUDA init failure)" - else - echo " ${names[$i]}: FAIL (sflow run exited $rc; see ${logs[$i]})" - fi - elif sanity_recipe_content_ok "${names[$i]}" "$run_dir" \ - && gpu_placement_verified "$run_dir"; then - PASSED=$((PASSED + 1)) - echo " ${names[$i]}: PASS (rc=0 and its own output proves it, under $run_dir)" - elif cuda_infra_failure "${roots[$i]}"; then - mark_cuda_excused "${names[$i]}" "${run_dir:-${logs[$i]}}" "(exited 0 but proved nothing; CUDA init failure on node)" - else - # The nastiest shape: green process, unproven run. Exactly what a - # silently-degraded placement or a collapsed two-backend run looks like. - echo " ${names[$i]}: FAIL (sflow run exited 0 but its output does not prove the run: ${run_dir:-no run dir found})" - fi - done -} - -run_monitor_mixed_real() { - # monitor_mixed.yaml is the broadest single-run regression net in examples/ - # (placement proven by UUID, replicas, release_after GPU reuse, readiness - # ordering, marker-clipped monitor reports, two Slurm pools). It needs its OWN - # `sflow batch` call for one reason: `--set PARTITION_A=...` is REJECTED by any - # config that does not declare that variable ("Variable 'PARTITION_A' ... is not - # defined"), so it cannot ride the shared bulk-submit args. Without the --set it - # submitted with the sample's `your_partition_a` placeholder and was a - # guaranteed sbatch rejection. - if [ -z "$SUBMIT" ]; then - return - fi - case "$TEST_TYPE" in - s|a|smoke|min) ;; - *) return ;; - esac - # No framework and no model load: the sanity half owns it, and that half runs - # through `sflow run` now -- so batch it only in the combined default. - [ "$RECIPE_CLASS" != "all" ] && return - - local part_a="${MONITOR_MIXED_PARTITION_A:-$PARTITION}" - local part_b="${MONITOR_MIXED_PARTITION_B:-$E2E_PARTITION_B}" - local mm_dir="$E2E_OUTPUT_DIR/monitor_mixed_real" - local mm_script="$mm_dir/monitor_mixed.sh" - mkdir -p "$mm_dir" - - echo "" - echo "===== Real monitor_mixed run (all-in-one regression net) =====" - echo " gpu_pool partition=$part_a, cpu_pool partition=$part_b" - echo "" - - # -G is IGNORED for a multi-backend config (each backend uses its own config - # values), so the per-node GPU count has to go in as a --set or the recipe keeps - # its own default and mis-plans the 2-node decode server. - strip_suite_segment_args - local mm_output mm_status - mm_output=$(sflow batch "$EXAMPLES_DIR/self_contained/slurm/monitor_mixed.yaml" \ - --set "SLURM_ACCOUNT=$ACCOUNT" \ - --set "PARTITION_A=$part_a" \ - --set "PARTITION_B=$part_b" \ - --set "GPUS_PER_NODE=$GPUS_PER_NODE" \ - -a "LOCAL_MODEL_PATH=fs://$MODEL_PATH" \ - -p "$part_a" \ - -A "$ACCOUNT" \ - --job-name "monitor_mixed_slurm" \ - "${BATCH_WORKSPACE_ARGS[@]}" \ - "${BATCH_OUTPUT_ARGS[@]}" \ - "${BATCH_VENV_ARGS[@]}" \ - -o "$mm_script" \ - $SUBMIT \ - "${STRIPPED_SEGMENT_ARGS[@]}" 2>&1) - mm_status=$? - echo "$mm_output" - if [ "$mm_status" -ne 0 ]; then - echo " monitor_mixed run: FAIL (sflow batch failed, rc=$mm_status; see output above)" - MONITOR_MIXED_LAUNCH_FAILED=1 - return - fi - - # Join JOB_IDS so the driver job flows through the shared wait (sacct) + - # validate loop like every other batched job. - local mm_job_id - mm_job_id=$(echo "$mm_output" | sed -n 's/.*Submitted batch job \([0-9]\+\).*/\1/p' | tail -1) - if [ -n "$mm_job_id" ]; then - JOB_IDS+=("$mm_job_id") - echo " monitor_mixed driver job id: $mm_job_id (script: $mm_script)" - else - echo " monitor_mixed run: FAIL (no Slurm job id reported by sflow batch)" - MONITOR_MIXED_LAUNCH_FAILED=1 - fi -} - run_multi_backend_real() { # Real multi-backend coverage via `sflow batch`: a >=2-Slurm-backend config # makes `sflow batch` emit one driver sbatch sized to the leader backend; @@ -1235,20 +547,9 @@ run_multi_backend_real() { s|a|smoke|min) ;; *) return ;; esac - # No framework and no model load: the sanity half owns it, and that half runs - # through `sflow run` now -- so batch it only in the combined default. - [ "$RECIPE_CLASS" != "all" ] && return - - # Default BOTH to the partition this run was given. Hardcoded names are - # cluster-specific and simply do not exist elsewhere -- the stale genesisq / - # gamoraq defaults meant every submission on a new cluster died with "no Slurm - # allocation granted". Two DIFFERENT partitions are better coverage, so an - # operator can still opt in via MULTI_BACKEND_PARTITION_A/_B; with one - # partition the test still proves what it is for, because each backend gets - # its OWN allocation and the check is that task_a and task_b land on - # different NODES. - local part_a="${MULTI_BACKEND_PARTITION_A:-$PARTITION}" - local part_b="${MULTI_BACKEND_PARTITION_B:-$E2E_PARTITION_B}" + + local part_a="${MULTI_BACKEND_PARTITION_A:-genesisq}" + local part_b="${MULTI_BACKEND_PARTITION_B:-gamoraq}" MULTI_BACKEND_RUN_DIR="$E2E_OUTPUT_DIR/multi_backend_real" local mb_dir="$MULTI_BACKEND_RUN_DIR" local mb_script="$mb_dir/multi_backend_hetjob.sh" @@ -1265,9 +566,6 @@ run_multi_backend_real() { # (sacct) + validate loop below like every other batched job. The CLI -p/-A # are required by the command but the driver is sized to the leader backend # (each backend uses its own resolved partition/account). - # multi_backend.yaml declares no SLURM_NODES either -- see strip_suite_segment_args. - strip_suite_segment_args - local mb_output mb_status mb_output=$(sflow batch "$EXAMPLES_DIR/self_contained/slurm/multi_backend.yaml" \ --set "SLURM_ACCOUNT=$ACCOUNT" \ @@ -1283,7 +581,7 @@ run_multi_backend_real() { "${BATCH_VENV_ARGS[@]}" \ -o "$mb_script" \ $SUBMIT \ - "${STRIPPED_SEGMENT_ARGS[@]}" 2>&1) + "${EXTRA_BATCH_ARGS[@]}" 2>&1) mb_status=$? echo "$mb_output" if [ "$mb_status" -ne 0 ]; then @@ -1307,17 +605,10 @@ run_multi_backend_real() { fi } -# Sync examples/ to src/sflow/samples/ so packaged samples stay up to date. -# gpu_reservation/ and mlperf/ are deliberately NOT packaged: they are local -# scratch (examples/mlperf/ is even gitignored), so copying them here only -# produced untracked dirs under src/sflow/samples/ that would ship with the -# wheel if anyone committed them. --delete does not clean an --exclude'd path, -# so remove any copy an earlier run already made. +# Sync examples/ to src/sflow/samples/ so packaged samples stay up to date echo "Syncing examples/ -> src/sflow/samples/ ..." rsync -a --delete --exclude='__pycache__' --exclude='*.pyc' --exclude='__init__.py' --exclude='sflow_output' \ - --exclude='gpu_reservation' --exclude='mlperf' \ "$EXAMPLES_DIR/" "$SAMPLES_DIR/" -rm -rf "$SAMPLES_DIR/gpu_reservation" "$SAMPLES_DIR/mlperf" echo "Done." # No shared runtime venv to pre-build. Under the per-job venv flow, each @@ -1333,21 +624,16 @@ echo "Done." JOB_IDS=() COLON_JOB_IDS=() MULTI_BACKEND_JOB_IDS=() -SANITY_RUN_DIRS=() -SANITY_RUN_NAMES=() MULTI_BACKEND_RUN_DIR="" MULTI_BACKEND_LAUNCH_FAILED="" -MONITOR_MIXED_LAUNCH_FAILED="" CSV_FILE="$EXAMPLES_DIR/modular/inference_x_v2/bulk_input.csv" # ============================================================================= # Part 1: Self-contained examples (--bulk-submit) # ============================================================================= -if { [ "$TEST_TYPE" = "s" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" ] || [ "$TEST_TYPE" = "min" ] || [ "$TEST_TYPE" = "one" ]; } && [ "$RECIPE_CLASS" != "sanity" ]; then +if [ "$TEST_TYPE" = "s" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" ] || [ "$TEST_TYPE" = "min" ]; then echo "" - if [ "$TEST_TYPE" = "one" ]; then - echo "===== Part 1: Single-job plumbing smoke (--bulk-submit one file) =====" - elif [ "$TEST_TYPE" = "min" ]; then + if [ "$TEST_TYPE" = "min" ]; then echo "===== Part 1: Min self-contained examples (--bulk-submit selected files) =====" elif [ "$TEST_TYPE" = "smoke" ]; then echo "===== Part 1: Smoke self-contained examples (--bulk-submit selected files) =====" @@ -1356,50 +642,19 @@ if { [ "$TEST_TYPE" = "s" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke fi echo "" - if [ "$TEST_TYPE" = "min" ] || [ "$TEST_TYPE" = "one" ]; then + if [ "$TEST_TYPE" = "min" ]; then MIN_SELF_CONTAINED=( "$EXAMPLES_DIR/self_contained/slurm/auto_replica.yaml" "$EXAMPLES_DIR/self_contained/slurm/dynamo_trtllm_disagg.yaml" - # One node, seconds of runtime, and it ASSERTS by UUID that every - # container/bare x offset x concurrency combination held the physical - # GPUs it was planned for. The recipes above only echo their devices, - # so they passed while a server held none. - "$EXAMPLES_DIR/self_contained/slurm/gpu_placement_matrix.yaml" "$EXAMPLES_DIR/self_contained/slurm/resource_release_after.yaml" "$EXAMPLES_DIR/self_contained/slurm/trtllm_serve_disagg.yaml" ) - if [ "$TEST_TYPE" = "one" ]; then - # Narrow to ONE job: the GPU placement matrix. Two nodes, a small - # container, no model to load, and seconds of compute -- so it stays - # cheap enough to play on every CI or cluster change. - # - # It is the recipe that ASSERTS rather than echoes. Every combination - # that can break the slice is covered (bare vs container, zero vs - # high offset, two tasks concurrent on one node, and multi-node), and - # each one resolves its planned HOST indices through a bare-metal - # index -> UUID map taken per node before anything was carved. That is - # the only way to tell "the right number of GPUs" from "the right - # GPUs", and it is the failure that actually reaches clusters: a task - # planned for 2,3 inside a 2-GPU container renumbered to 0,1. - # - # It replaced the single-node DISAGG recipe, which probed the same - # failure but only PRINTED its devices -- so it passed while a server - # held none -- and cost a multi-gigabyte pull plus a model load, which - # made a red run ambiguous between a broken CI path and a real - # regression. Here a red run means placement is genuinely wrong; read - # SFLOW_GPU_PROBE in the task logs for the planned slice next to the - # devices the step actually held. - MIN_SELF_CONTAINED=( - "$EXAMPLES_DIR/self_contained/slurm/gpu_placement_matrix.yaml" - ) - fi MIN_BULK_ARGS=() for yaml_file in "${MIN_SELF_CONTAINED[@]}"; do if [ ! -f "$yaml_file" ]; then echo "ERROR: min self-contained Slurm YAML not found: $yaml_file" exit 1 fi - recipe_in_class "$yaml_file" || continue MIN_BULK_ARGS+=(--bulk-submit "$yaml_file") done output=$(sflow batch \ @@ -1421,17 +676,14 @@ if { [ "$TEST_TYPE" = "s" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke dynamo_sglang_agg.yaml|dynamo_vllm_agg.yaml|sglang_server_client.yaml) continue ;; - multi_backend.yaml|monitor_mixed.yaml) - # Both declare PARTITION_A/PARTITION_B and are covered - # separately (run_multi_backend_real / run_monitor_mixed_real). - # They cannot ride the shared bulk args: `--set PARTITION_A=...` - # is REJECTED by every config that does not declare it, and - # without the --set they submit with the sample's - # `your_partition_a` placeholder and sbatch rejects them. + multi_backend.yaml) + # Covered separately by run_multi_backend_real as a `sflow + # batch` heterogeneous job; it needs two partitions + # (PARTITION_A/PARTITION_B), so skip the single-partition + # bulk-submit copy here. continue ;; esac - recipe_in_class "$yaml_file" || continue SMOKE_SELF_CONTAINED+=("$yaml_file") SMOKE_BULK_ARGS+=(--bulk-submit "$yaml_file") done @@ -1452,18 +704,16 @@ if { [ "$TEST_TYPE" = "s" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke $SUBMIT \ "${EXTRA_BATCH_ARGS[@]}" 2>&1) else - # Bulk-submit every example EXCEPT the two-partition configs: they declare - # PARTITION_A/PARTITION_B, which the shared bulk args cannot set (a --set of - # a variable a config does not declare is a hard error), and are covered - # separately by run_multi_backend_real / run_monitor_mixed_real. + # Bulk-submit every example EXCEPT the multi-backend config: it needs two + # partitions (PARTITION_A/PARTITION_B) and is covered separately by + # run_multi_backend_real as a `sflow batch` heterogeneous job. ALL_BULK_ARGS=() for yaml_file in "$EXAMPLES_DIR"/self_contained/slurm/*.yaml; do case "$(basename "$yaml_file")" in - multi_backend.yaml|monitor_mixed.yaml) + multi_backend.yaml) continue ;; esac - recipe_in_class "$yaml_file" || continue ALL_BULK_ARGS+=(--bulk-submit "$yaml_file") done output=$(sflow batch \ @@ -1496,7 +746,7 @@ fi # ============================================================================= # Part 2: Modular examples (--bulk-input) # ============================================================================= -if { [ "$TEST_TYPE" = "m" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" ] || [ "$TEST_TYPE" = "min" ]; } && [ "$RECIPE_CLASS" != "sanity" ]; then +if [ "$TEST_TYPE" = "m" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" ] || [ "$TEST_TYPE" = "min" ]; then echo "" if [ "$TEST_TYPE" = "min" ]; then echo "===== Part 2: Min modular example (--bulk-input selected row) =====" @@ -1558,7 +808,7 @@ if [ "${SFLOW_E2E_SKIP_INFMAX:-}" = "1" ]; then echo "" echo "===== Part 3: infmax suites SKIPPED (SFLOW_E2E_SKIP_INFMAX=1; run by prenyx CI) =====" echo "" -elif { [ "$TEST_TYPE" = "inf" ] || [ "$TEST_TYPE" = "m" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" ] || [ "$TEST_TYPE" = "min" ]; } && [ "$RECIPE_CLASS" != "sanity" ]; then +elif [ "$TEST_TYPE" = "inf" ] || [ "$TEST_TYPE" = "m" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" ] || [ "$TEST_TYPE" = "min" ]; then echo "" if [ "$TEST_TYPE" = "min" ]; then echo "===== Part 3: Min infmax multi-node batch suite =====" @@ -1585,7 +835,6 @@ submit_colon_task_script_e2e # after the async submissions so its salloc job ids join JOB_IDS and flow through # the shared wait (sacct) + validate loop below. run_multi_backend_real -run_monitor_mixed_real set +x @@ -1595,7 +844,7 @@ echo "===== Submitted Jobs =====" fi # end of --check else block -if [ ${#JOB_IDS[@]} -eq 0 ] && [ "$RECIPE_CLASS" != "sanity" ]; then +if [ ${#JOB_IDS[@]} -eq 0 ]; then echo "No job IDs captured." exit 0 fi @@ -1641,23 +890,14 @@ done cuda_infra_failure() { # -> 0 (true) if the job failed due to CUDA infra local out_dir="$1" [ -n "$out_dir" ] && [ -d "$out_dir" ] || return 1 - # These are GB200/driver "system not ready" signatures -- an ERROR from the - # driver, on a node that needs draining. 'system not yet initialized' covers - # both torch ('Error 802: system not yet initialized') and cupy - # ('cudaErrorSystemNotReady: system not yet initialized'). - # - # Every pattern here must be something ONLY a broken node produces. Excusing is - # not a soft verdict -- it removes the job from the pass/fail threshold, so a - # pattern that also matches healthy output turns real regressions into a green - # pipeline. 'No CUDA runtime is found' used to be in this list and did exactly - # that: torch prints it as a routine WARNING when cpp_extension cannot find - # nvcc for JIT ("No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda'"), - # which healthy runs emit constantly. It excused two genuinely failed disagg - # jobs and reported "PASS - 0/6 failed". + # These are GB200/driver "system not ready" + missing-CUDA-runtime signatures. + # 'system not yet initialized' covers both torch ('Error 802: system not yet + # initialized') and cupy ('cudaErrorSystemNotReady: system not yet initialized'). grep -rIqs --include='*.log' --include='*.out' \ -e 'system not yet initialized' \ -e 'cudaErrorSystemNotReady' \ -e 'CUDA initialization: Unexpected error from cudaGetDeviceCount' \ + -e 'No CUDA runtime is found' \ -e 'Failed to get device capability: Unexpected error from cudaGetDeviceCount' \ "$out_dir" } @@ -1677,9 +917,6 @@ echo "===== Results =====" TOTAL=0 PASSED=0 CUDA_INFRA=0 -if [ "$RECIPE_CLASS" = "sanity" ]; then - run_sanity_recipes_with_sflow_run -fi for jid in "${JOB_IDS[@]}"; do TOTAL=$((TOTAL + 1)) if is_multi_backend_result "$jid"; then @@ -1708,62 +945,26 @@ for jid in "${JOB_IDS[@]}"; do fi continue fi - case "$out_dir" in - *-gpu_placement_matrix-*) - if gpu_placement_run_ok "$out_dir"; then - PASSED=$((PASSED + 1)) - echo " Job $jid: PASS (GPU placement proven by UUID on 2+ nodes under $out_dir)" - elif cuda_infra_failure "$out_dir"; then - mark_cuda_excused "$jid" "$out_dir" "(placement assertions unproven; CUDA init failure on node)" - else - echo " Job $jid: FAIL (GPU placement not proven under $out_dir; read SFLOW_GPU_PROBE / FAIL: in the task logs)" - fi - continue - ;; - esac # Check for various success indicators across different workflow types - # aiperf: its own "Processed N valid requests and M errors" tally - # benchmark_serving: its own "Successful requests:" vs the --num-prompts asked for + # aiperf benchmark: '0 errors' in benchmark log + # aiperf template: '0 valid' in benchmark log + # infmax benchmark: 'Successful requests:' with non-zero value # auto_replica: 'Client Task Nodes' in client task log + count_aiperf_errors=$(find "$out_dir" -type f -name 'benchmark*.log' -exec grep -l "0 errors" {} + 2>/dev/null | wc -l) + count_aiperf_valid=$(find "$out_dir" -type f -name 'benchmark*.log' -exec grep -l "0 valid" {} + 2>/dev/null | wc -l) + count_zero_success=$(find "$out_dir" -type f -name 'benchmark*.log' -exec grep -lP "Successful requests:\s+0\s*$" {} + 2>/dev/null | wc -l) + count_any_success=$(find "$out_dir" -type f -name 'benchmark*.log' -exec grep -l "Successful requests:" {} + 2>/dev/null | wc -l) count_replica=$(find "$out_dir" -type f -name 'client*.log' -exec grep -l "Client Task Nodes" {} + 2>/dev/null | wc -l) - # Both: 0 = benchmarked, 1 = ran and measured nothing, 2 = that tool not used here. - aiperf_tally_ok "$out_dir" - aiperf_state=$? - serving_tally_ok "$out_dir" - serving_state=$? - - if [ "$aiperf_state" -eq 1 ] && recipe_is_client_only "$out_dir"; then - # The ONE expected-not-to-benchmark shape, claimed explicitly so it reads as - # a decision rather than a hole: this recipe starts no server, so aiperf had - # nothing to talk to. It still owes a clean completion. - if workflow_summary_ok "$out_dir"; then - PASSED=$((PASSED + 1)) - echo " Job $jid: PASS (client-only recipe: no server to benchmark by design, and it completed; $out_dir)" - else - echo " Job $jid: FAIL (client-only recipe is still expected to COMPLETE, and did not; $out_dir)" - fi - elif [ "$aiperf_state" -eq 1 ]; then - # Checked BEFORE the success indicators: this is the shape where every other - # signal in the run says green. A dead aiperf is a failed benchmark even when - # a sibling task in the same workflow reported requests of its own. - if cuda_infra_failure "$out_dir"; then - mark_cuda_excused "$jid" "$out_dir" "(aiperf measured nothing; CUDA init failure on node)" - else - echo " Job $jid: FAIL (aiperf ran but measured nothing; see the tally above, $out_dir)" - fi - elif [ "$serving_state" -eq 1 ]; then + + if [ "$count_zero_success" -gt 0 ]; then if cuda_infra_failure "$out_dir"; then - mark_cuda_excused "$jid" "$out_dir" "(benchmark_serving did not complete; CUDA init failure on node)" + mark_cuda_excused "$jid" "$out_dir" "('Successful requests: 0' with a CUDA init failure)" else - echo " Job $jid: FAIL (benchmark_serving ran but did not complete its requests; see the counts above, $out_dir)" + echo " Job $jid: FAIL ('Successful requests: 0' found in $out_dir)" fi - elif [ "$aiperf_state" -eq 0 ] || [ "$serving_state" -eq 0 ] || [ "$count_replica" -gt 0 ]; then + elif [ "$count_aiperf_errors" -gt 0 ] || [ "$count_aiperf_valid" -gt 0 ] || [ "$count_any_success" -gt 0 ] || [ "$count_replica" -gt 0 ]; then PASSED=$((PASSED + 1)) echo " Job $jid: PASS (under $out_dir)" - elif [ -z "$(find "$out_dir" -type f -name 'benchmark*.log' -print -quit 2>/dev/null)" ] \ - && workflow_summary_ok "$out_dir"; then - PASSED=$((PASSED + 1)) - echo " Job $jid: PASS (no benchmark log; sflow reports Status: COMPLETED under $out_dir)" elif cuda_infra_failure "$out_dir"; then mark_cuda_excused "$jid" "$out_dir" "(no success indicator; CUDA init failure on node)" else @@ -1778,13 +979,6 @@ if [ -n "${MULTI_BACKEND_LAUNCH_FAILED:-}" ]; then echo " Multi-backend run: FAIL (no Slurm allocation granted)" fi -if [ -n "${MONITOR_MIXED_LAUNCH_FAILED:-}" ]; then - # Same as above: never submitted, so the loop never visited it. Silence here - # let the broadest recipe in the suite vanish without touching the verdict. - TOTAL=$((TOTAL + 1)) - echo " monitor_mixed run: FAIL (no Slurm job submitted)" -fi - echo "" echo "===== Summary =====" echo "$PASSED/$TOTAL jobs passed" @@ -1826,17 +1020,6 @@ echo "===== Monitor Coverage =====" MONITOR_TOTAL=0 MONITOR_PRESENT=0 MONITOR_MISSING_LABELS="" -for _i in "${!SANITY_RUN_DIRS[@]}"; do - MONITOR_TOTAL=$((MONITOR_TOTAL + 1)) - _mon=$(find "${SANITY_RUN_DIRS[$_i]}" -maxdepth 2 -type f -name 'sflow_monitor.log' 2>/dev/null | head -1) - if [ -n "$_mon" ] && monitor_log_has_content "$_mon"; then - MONITOR_PRESENT=$((MONITOR_PRESENT + 1)) - echo " ${SANITY_RUN_NAMES[$_i]}: monitor overview OK ($_mon)" - else - MONITOR_MISSING_LABELS="$MONITOR_MISSING_LABELS - ${SANITY_RUN_NAMES[$_i]} (no populated sflow_monitor.log under ${SANITY_RUN_DIRS[$_i]})\n" - echo " ${SANITY_RUN_NAMES[$_i]}: MONITOR MISSING/EMPTY (${SANITY_RUN_DIRS[$_i]})" - fi -done for jid in "${JOB_IDS[@]}"; do MONITOR_TOTAL=$((MONITOR_TOTAL + 1)) mon_out_dir=$(workflow_output_dir_for_job "$jid") @@ -1913,14 +1096,3 @@ if [ -n "$TARGETING_FAIL_LABELS" ]; then echo "Disagg workflows whose monitor sampled the wrong resources:" echo -e "$TARGETING_FAIL_LABELS" fi - -# The verdict, as an exit status. Without this the script ended on an `if` and -# returned 0 however red the run was, which is what made full_sample_tests.sh's -# `exit "$e2e_rc"` unable to fail. -# CUDA-infra excusals are excluded from the threshold, matching what -# summarize_validation() does with the same numbers -- but an ALL-excused run -# proved nothing, so it is not a pass either. -[ "$PASSED" -gt 0 ] \ - && [ $((PASSED + CUDA_INFRA)) -eq "$TOTAL" ] \ - && [ "$MONITOR_PRESENT" -eq "$MONITOR_TOTAL" ] \ - && [ "$TARGETING_OK" -eq "$TARGETING_TOTAL" ] diff --git a/tests/integration/guide/sflow_dynamo.yaml b/tests/integration/guide/sflow_dynamo.yaml index 1dad24d..7115ef8 100644 --- a/tests/integration/guide/sflow_dynamo.yaml +++ b/tests/integration/guide/sflow_dynamo.yaml @@ -353,6 +353,7 @@ workflow: --extra-inputs "max_tokens:${OSL}" --extra-inputs "min_tokens:${OSL}" --extra-inputs "ignore_eos:true" + --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/tests/unit/golden/k8s_recipes/dynamo_sglang_disagg.yaml b/tests/unit/golden/k8s_recipes/dynamo_sglang_disagg.yaml index f4b4607..44759f3 100644 --- a/tests/unit/golden/k8s_recipes/dynamo_sglang_disagg.yaml +++ b/tests/unit/golden/k8s_recipes/dynamo_sglang_disagg.yaml @@ -763,10 +763,11 @@ items: Qwen3-8B-FP8 --tokenizer /models/Qwen3-8B-FP8 --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://0.0.0.1:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs - "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs - "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count - ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: - Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple + "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" + --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} + --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries + $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' + --record-processors 8 --ui simple echo "Benchmarking finished"' kind: ConfigMap diff --git a/tests/unit/golden/k8s_recipes/dynamo_trtllm_disagg.yaml b/tests/unit/golden/k8s_recipes/dynamo_trtllm_disagg.yaml index 3075dcb..2a46e62 100644 --- a/tests/unit/golden/k8s_recipes/dynamo_trtllm_disagg.yaml +++ b/tests/unit/golden/k8s_recipes/dynamo_trtllm_disagg.yaml @@ -820,10 +820,11 @@ items: Qwen3-8B-FP8 --tokenizer /models/Qwen3-8B-FP8 --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://0.0.0.1:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs - "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs - "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count - ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: - Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple + "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" + --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} + --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries + $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' + --record-processors 8 --ui simple echo "Benchmarking finished"' kind: ConfigMap diff --git a/tests/unit/golden/k8s_recipes/dynamo_vllm_agg.yaml b/tests/unit/golden/k8s_recipes/dynamo_vllm_agg.yaml index d778432..7a36fcd 100644 --- a/tests/unit/golden/k8s_recipes/dynamo_vllm_agg.yaml +++ b/tests/unit/golden/k8s_recipes/dynamo_vllm_agg.yaml @@ -488,10 +488,11 @@ items: Qwen3-8B-FP8 --tokenizer /models/Qwen3-8B-FP8 --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://0.0.0.1:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs - "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs - "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count - ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: - Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple + "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" + --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} + --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries + $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' + --record-processors 8 --ui simple echo "Benchmarking finished"' kind: ConfigMap diff --git a/tests/unit/test_app_assembly_resolve_variables.py b/tests/unit/test_app_assembly_resolve_variables.py index 15a2ec7..f86f818 100644 --- a/tests/unit/test_app_assembly_resolve_variables.py +++ b/tests/unit/test_app_assembly_resolve_variables.py @@ -199,24 +199,3 @@ def test_resolve_variables_exposes_domain_metadata_in_expressions(): out = resolve_global_variables(config, state) assert out.variables["BATCH_SIZE"].value == 128 # max([128, 512]) // 4 - - -def test_build_state_clears_source_files_from_a_previous_call(tmp_path): - """`resolver` is a module GLOBAL, so state survives between build_state calls. - - source_files was assigned only when truthy and never cleared, so a call that - passes none (visualize() does) inherited the last run's files and pointed its - expression-error location hints at the wrong YAML. - """ - import asyncio - - from sflow.app import assembly - - assembly.resolver.source_files = [tmp_path / "stale.yaml"] - - config = _minimal_config(variables=[]) - asyncio.run( - assembly.build_state(config, allocate=False, workspace_dir=str(tmp_path)) - ) - - assert assembly.resolver.source_files == [] diff --git a/tests/unit/test_cli_batch.py b/tests/unit/test_cli_batch.py index 0a30364..ea2cdfd 100644 --- a/tests/unit/test_cli_batch.py +++ b/tests/unit/test_cli_batch.py @@ -1611,7 +1611,7 @@ def test_absolute_path_skipped(self): def test_container_image_skipped(self): assert ( - _normalize_col_value("nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0") is None + _normalize_col_value("nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0") is None ) def test_container_image_with_org_skipped(self): @@ -1626,7 +1626,7 @@ def test_derive_row_name_container_image_skipped(): rows = [ { "sflow_config_file": "wf.yaml", - "IMAGE": "nvcr.io/nvidia/vllm-runtime:1.3.0", + "IMAGE": "nvcr.io/nvidia/vllm-runtime:0.8.0", "SLURM_NODES": "2", }, { @@ -5146,11 +5146,6 @@ def test_batch_script_installs_editable_from_source_path( # does not recurse into its own destination / sibling jobs' growing copies. assert "--exclude='.sflow_venv*'" in script_content assert "--exclude='.sflow_src*'" in script_content - # Also correctness-critical, and not about size: pip/uv write partial *.tmp - # files under .cache while sibling jobs bootstrap, and rsync exits 24 when one - # vanishes mid-transfer -- fatal to the bootstrap, so the job dies with no - # output dir at all. - assert "--exclude=.cache" in script_content # The per-job source copy is cleaned up with the venv on exit/signal. assert '${SFLOW_SRC_DIR:+"$SFLOW_SRC_DIR"}' in script_content diff --git a/tests/unit/test_config_loader.py b/tests/unit/test_config_loader.py index 4df8be3..f21cfda 100644 --- a/tests/unit/test_config_loader.py +++ b/tests/unit/test_config_loader.py @@ -266,56 +266,3 @@ def test_strip_missable_noop_without_workflow(): stripped = strip_missable_tasks(config, ["anything"]) assert stripped == [] assert config == {"version": "0.1"} - - -def test_load_config_warns_that_timeout_is_not_enforced(tmp_path, caplog): - """`timeout:` is accepted but nothing reads it, so a recipe that sets one - LOOKS bounded and is not. The field cannot simply be removed (these models - forbid extra keys, so every config setting it would stop loading), so the - warning is what stops it lying.""" - p = tmp_path / "sflow.yaml" - p.write_text( - """ -version: "0.1" -workflow: - name: wf - timeout: 115m - tasks: - - name: t1 - timeout: 30m - script: - - echo hi - - name: t2 - script: - - echo hi -""".lstrip() - ) - - with caplog.at_level("WARNING"): - ConfigLoader().load_config(p) - - msg = "\n".join(r.message for r in caplog.records) - assert "does not enforce it" in msg - assert "workflow" in msg and "t1" in msg - # t2 sets no timeout, so it must not be named. - assert "t2" not in msg - - -def test_load_config_is_quiet_when_no_timeout_is_set(tmp_path, caplog): - p = tmp_path / "sflow.yaml" - p.write_text( - """ -version: "0.1" -workflow: - name: wf - tasks: - - name: t1 - script: - - echo hi -""".lstrip() - ) - - with caplog.at_level("WARNING"): - ConfigLoader().load_config(p) - - assert "does not enforce it" not in "\n".join(r.message for r in caplog.records) diff --git a/tests/unit/test_core_execution_summary.py b/tests/unit/test_core_execution_summary.py index eb3d299..d544e92 100644 --- a/tests/unit/test_core_execution_summary.py +++ b/tests/unit/test_core_execution_summary.py @@ -1121,60 +1121,3 @@ def test_gpu_usage_chart_rows_sorted_by_node_then_gpu_index(tmp_path): "node10 GPU 3", ], rows assert rows == sorted(rows, key=_resource_row_key) - - -def test_gpu_assignment_reads_the_in_container_view_from_the_placement_record(tmp_path): - """The planner's env is the HOST slice, so it cannot be the in-container column. - - A Slurm step planned for host 2,3 that a container renumbered to 0,1 was - reported as physical 0,1 / in-container 2,3 -- backwards, and impossible. The - step's own numbering only exists in the record it writes. - """ - from sflow.utils.gpu import GPU_MARKER_FILE - - task = _gpu_task("worker", tmp_path, visible="2,3") - out = Path(task.envs["SFLOW_TASK_OUTPUT_DIR"]) - out.mkdir(parents=True, exist_ok=True) - (out / GPU_MARKER_FILE).write_text( - "0,1\n" - "action=verified\n" - "cuda_visible_devices=0,1\n" - "planned_host_indices=2,3\n" - ) - - row = [ - ln - for ln in _render(tmp_path, [task]).split("GPU Assignment", 1)[1].splitlines() - if ln.startswith("worker") - ][0] - # Physical = the planned HOST indices the step PROVED it holds, by UUID. - # In-container = what the step itself was numbered. - assert row.index("2,3") < row.index("0,1"), row - - -def test_gpu_assignment_falls_back_to_the_env_when_the_record_says_unset(tmp_path): - """`` is a sentinel, not a device list. - - The guard used to test for a parenthesised spelling the script never writes, - so it never fired -- it only looked right because the sentinel happens to - parse to no indices. - """ - from sflow.utils.gpu import GPU_MARKER_FILE - - task = _gpu_task("worker", tmp_path, visible="4,5") - out = Path(task.envs["SFLOW_TASK_OUTPUT_DIR"]) - out.mkdir(parents=True, exist_ok=True) - (out / GPU_MARKER_FILE).write_text( - "4,5\naction=fallback\ncuda_visible_devices=\n" - ) - - row = [ - ln - for ln in _render(tmp_path, [task]).split("GPU Assignment", 1)[1].splitlines() - if ln.startswith("worker") - ][0] - # Both columns must read 4,5: the sentinel is discarded and the task's env - # supplies the in-container view. With the old guard the sentinel survived, - # parsed to no indices, and that column silently rendered as "-". - assert row.count("4,5") == 2, row - assert "" not in row, row diff --git a/tests/unit/test_monitoring_postprocess.py b/tests/unit/test_monitoring_postprocess.py index be34f91..91b092c 100644 --- a/tests/unit/test_monitoring_postprocess.py +++ b/tests/unit/test_monitoring_postprocess.py @@ -1005,54 +1005,6 @@ def test_gpu_panel_draws_one_line_per_device_with_a_legend(tmp_path): assert "max 90.00" in svg and "min 0.00" in svg -def test_coincident_device_lines_stay_distinguishable(tmp_path): - """Two GPUs with IDENTICAL values must not render as one line. - - A tensor-parallel task allocates the same footprint on every rank, so - `gpu_memory_used_mib` for its GPUs is often identical to the byte -- the - second polyline lands exactly on the first and the panel looks like it only - ever had one device, which reads as a collection bug. Colour cannot fix that - (nothing of the lower line is visible to be coloured); the dash can. - """ - rows = _gpu_rows([("n1", 0, 50.0), ("n1", 1, 50.0)]) - - svg_path = tmp_path / "timeline.svg" - assert pp._render_svg(rows, svg_path, title="t") - svg = svg_path.read_text() - - lines = re.findall(r"]*>", svg) - assert len(lines) == 2, lines - dashes = [ - re.search(r'stroke-dasharray="([^"]+)"', ln).group(1) - if "stroke-dasharray" in ln - else "" - for ln in lines - ] - assert dashes[0] != dashes[1], dashes - # The legend swatch must carry its line's dash, or the legend stops matching. - legend, _h = pp._build_device_legend_svg(["GPU 0", "GPU 1"], x0=8, max_x=800) - swatches = [frag for frag in legend if frag.startswith("GPU 0<") == 1 and svg.count(">GPU 1<") == 1 -def test_png_gives_all_eight_gpus_a_distinct_palette_colour(tmp_path, monkeypatch): - """The PNG must use `_SERIES_COLORS`, not matplotlib's default tab10 cycle. - - A node carries at most 8 GPUs, so a full node has to be readable without the - cycle wrapping or two devices landing on near-identical hues. - """ - plt = pytest.importorskip("matplotlib.pyplot") - rows = _gpu_rows([("n1", g, 10.0 * g) for g in range(8)]) - - drawn: list[tuple[str, tuple]] = [] - real_plot = plt.Axes.plot - - def spy(self, *a, **kw): - if "color" in kw: - drawn.append((kw["color"], kw.get("dashes", ()))) - return real_plot(self, *a, **kw) - - monkeypatch.setattr(plt.Axes, "plot", spy) - assert pp._render_png(rows, tmp_path / "timeline.png", title="t") - - colours = [c for c, _d in drawn] - assert colours == list(pp._SERIES_COLORS), colours - assert len(set(colours)) == 8, "two GPUs share a colour" - assert drawn[0][1] == () and drawn[1][1] == (5.0, 3.0), drawn[:2] - - -def test_series_palette_is_the_okabe_ito_set(): - """The colour SET is Okabe-Ito exactly -- order is our own (see below). - - This palette is picked for a property (it survives all three dichromacies) - that cannot be checked by looking at the chart on a normal display, so - "nudging" a colour silently forfeits the reason it was chosen. - """ - assert set(pp._SERIES_COLORS) == { - "#0072b2", "#e69f00", "#009e73", "#d55e00", - "#56b4e9", "#cc79a7", "#f0e442", "#000000", - } - assert len(pp._SERIES_COLORS) == 8, "8 GPUs per node; the cycle must not wrap" - # Yellow carries extra weight, black slightly less -- see _SERIES_WIDTH_SCALE. - assert pp._series_width(7, 1.4) > 1.4 > pp._series_width(4, 1.4) - assert pp._series_width(0, 1.4) == 1.4 - - -# The three near-pairs in Okabe-Ito: two blues, and the warms among themselves. -# Colour alone does not reliably separate these at 1.5px. -_NEAR_PAIRS = ( - ("#0072b2", "#56b4e9"), # blue / sky blue - ("#e69f00", "#d55e00"), # orange / vermilion - ("#e69f00", "#f0e442"), # orange / yellow - ("#d55e00", "#f0e442"), # vermilion / yellow -) - - -def test_first_four_slots_hold_no_near_pair(): - """A 4-GPU task only draws slots 0-3, so those must separate by hue alone. - - The published Okabe-Ito order fails this: it puts orange in slot 1 and - vermilion in slot 3, making the palette's worst pairing the common case. - """ - head = pp._SERIES_COLORS[:4] - for a, b in _NEAR_PAIRS: - assert not (a in head and b in head), f"{a}/{b} both in the first four" - - -def test_near_pairs_do_not_share_a_dash_pattern(): - """Dash is the fallback for the pairs hue cannot separate -- so it must differ. - - `_SERIES_DASHES` cycles every 4, so slots i and i+4 are identical in dash and - would leave such a pair with no redundant encoding at all. - """ - for a, b in _NEAR_PAIRS: - i, j = pp._SERIES_COLORS.index(a), pp._SERIES_COLORS.index(b) - assert pp._series_dash(i) != pp._series_dash(j), ( - f"{a} (slot {i}) and {b} (slot {j}) share a dash pattern" - ) - - -def test_png_greys_its_chrome_so_the_black_series_is_not_mistaken_for_the_frame( - tmp_path, monkeypatch -): - """Slot 8 is pure black; matplotlib's default spines/ticks are black too.""" - plt = pytest.importorskip("matplotlib.pyplot") - rows = _gpu_rows([("n1", g, 10.0 * g) for g in range(8)]) - figs: list = [] - real = plt.Figure.savefig - monkeypatch.setattr( - plt.Figure, - "savefig", - lambda self, *a, **k: (figs.append(self), real(self, *a, **k))[1], - ) - assert pp._render_png(rows, tmp_path / "t.png", title="t") - - (figure,) = figs - for ax in figure.axes: - for spine in ax.spines.values(): - assert spine.get_edgecolor()[:3] != (0.0, 0.0, 0.0), "frame still black" - - -def test_png_device_legend_sits_below_the_panels_not_on_the_data(tmp_path, monkeypatch): - """The legend belongs under the chart, as in the SVG -- not over the lines. - - In-panel (`loc="upper right"`) it covered the top-right of the first panel, - which is exactly where GPU load ramps at the end of a run. - """ - plt = pytest.importorskip("matplotlib.pyplot") - rows = _gpu_rows([("n1", g, 10.0 * g) for g in range(4)]) - - figs: list = [] - real_savefig = plt.Figure.savefig - - def spy(self, *a, **kw): - figs.append(self) - return real_savefig(self, *a, **kw) - - monkeypatch.setattr(plt.Figure, "savefig", spy) - assert pp._render_png(rows, tmp_path / "timeline.png", title="t") - - (figure,) = figs - assert not any(ax.get_legend() for ax in figure.axes), "legend baked into a panel" - (legend,) = figure.legends - assert [t.get_text() for t in legend.get_texts()] == [ - "GPU 0", "GPU 1", "GPU 2", "GPU 3", - ] - # Below the lowest panel, so it can never overlap plotted data. Both extents - # in display pixels (y grows upward): the legend's TOP must clear the bottom - # of the lowest axes. Agg canvas because only a real backend has a renderer, - # and the drawn box -- not the anchor -- is what can overlap the data. - from matplotlib.backends.backend_agg import FigureCanvasAgg - - renderer = FigureCanvasAgg(figure).get_renderer() - assert legend.get_window_extent(renderer).y1 <= min( - ax.get_window_extent(renderer).y0 for ax in figure.axes - ) - - def test_empty_gpu_list_reports_no_gpus_rather_than_all(tmp_path, capsys): """`[]` (task reserved no GPU) must not read as `None` (no subset -> all). diff --git a/tests/unit/test_plugin_backends_slurm_backend.py b/tests/unit/test_plugin_backends_slurm_backend.py index 9eb8688..d598b78 100644 --- a/tests/unit/test_plugin_backends_slurm_backend.py +++ b/tests/unit/test_plugin_backends_slurm_backend.py @@ -3,13 +3,11 @@ import asyncio import logging -import unittest.mock import pytest import sflow.plugins.backends.slurm as slurm_mod from sflow.core.backend import Allocation -from sflow.core.compute_node import ComputeNode from sflow.plugins.backends.slurm import SlurmBackend, SlurmBackendConfig @@ -170,51 +168,6 @@ def test_salloc_tokenizes_extra_args_with_bundled_whitespace( assert not any(arg != arg.strip() for arg in salloc_cmd) -def test_salloc_extra_args_keep_repeated_values(monkeypatch, slurm_test_logger): - # `-e '-G 1 -p polar4 -A acct -N 1'` tokenizes to space-separated flag/value - # pairs. Feeding each token through Command.add_opt() treated the bare values - # as option names and de-duped them, so the second "1" deleted the first and - # salloc received `-G -p polar4 -A acct -N 1` -- a silent, wrong allocation. - monkeypatch.delenv("SLURM_JOB_ID", raising=False) - monkeypatch.delenv("SLURM_JOBID", raising=False) - monkeypatch.delenv("SLURM_JOB_NODELIST", raising=False) - monkeypatch.delenv("SLURM_NODELIST", raising=False) - - backend = SlurmBackend( - SlurmBackendConfig( - name="b", - type="slurm", - account="acct", - partition="batch", - nodes=1, - time="00:10:00", - job_name="job", - extra_args=["-G 1 -p polar4 -A general_perflab -N 1"], - gpus_per_node=8, - ) - ) - fake_launcher = _FakeSubprocessLauncher( - script=[ - (0, ["salloc: Granted job allocation 1", "salloc: Nodes node001 are ready for job"]), - (0, ["node001: 10.0.0.1:123"]), - ] - ) - backend._subprocess_launcher = fake_launcher - asyncio.run(backend.allocate()) - - salloc_cmd = list(fake_launcher.calls[0]["command"]) - assert salloc_cmd[-8:] == [ - "-G", - "1", - "-p", - "polar4", - "-A", - "general_perflab", - "-N", - "1", - ] - - def test_env_reuse_applies_exclude_filter(monkeypatch, slurm_test_logger): # A reused Slurm allocation can't take salloc flags, so exclude filters the # resolved node pool instead. @@ -1453,12 +1406,8 @@ def test_slurm_backend_resource_env_preserves_controller_slurm_envs(monkeypatch) env = backend.resource_env(cuda_visible_devices="0,1") assert env["CUDA_VISIBLE_DEVICES"] == "0,1" - # INTENTIONAL CHANGE: NVIDIA_VISIBLE_DEVICES is no longer handed to an srun - # step. It used to mirror the slice, but the container runtime reads it at - # container CREATION and carves + RENUMBERS those devices from 0 -- leaving - # the CUDA_VISIBLE_DEVICES we exported beside it, in host numbering, - # addressing nothing. See SlurmBackend.resource_env. - assert "NVIDIA_VISIBLE_DEVICES" not in env + # NVIDIA_VISIBLE_DEVICES mirrors the same slice (some stacks honor only that one). + assert env["NVIDIA_VISIBLE_DEVICES"] == "0,1" assert env["SLURM_JOB_ID"] == "2222222" assert env["SLURM_JOB_NODELIST"] == "node[001-002]" assert env["SLURM_NNODES"] == "2" @@ -1467,174 +1416,3 @@ def test_slurm_backend_resource_env_preserves_controller_slurm_envs(monkeypatch) assert env["SFLOW_BACKEND_JOB_ID"] == "2222222" assert env["SFLOW_BACKEND_NODELIST"] == "node[001-002]" assert env["SFLOW_BACKEND_NUM_NODES"] == "2" - - -def test_srun_steps_are_not_handed_nvidia_visible_devices(): - """The variable that carves the container must not name a subset. - - pyxis/enroot reads NVIDIA_VISIBLE_DEVICES when it CREATES the container: a - subset there makes it expose only those devices and renumber them from 0, so - the CUDA_VISIBLE_DEVICES exported alongside -- in host numbering -- then - addresses nothing. A worker planned for host 2,3 landed in a 2-GPU container - numbered 0,1 and died with "No CUDA GPUs are available"; one planned for 0,1 - survived only by coincidence. - - Not exporting it lets the container see the node's GPUs with host numbering - intact, so the planned slice is directly addressable and CUDA_VISIBLE_DEVICES - -- the variable CUDA actually reads -- is the only thing sflow sets. - - Docker is unaffected: it overrides resource_env and isolates with - `--gpus device=`. - """ - backend = SlurmBackend( - SlurmBackendConfig( - name="s", - type="slurm", - account="acct", - partition="batch", - nodes=1, - time="00:10:00", - gpus_per_node=8, - ) - ) - env = backend.resource_env(cuda_visible_devices="2,3") - assert env["CUDA_VISIBLE_DEVICES"] == "2,3" - assert "NVIDIA_VISIBLE_DEVICES" not in env - - # A task with no GPU slice gets neither. - assert "NVIDIA_VISIBLE_DEVICES" not in backend.resource_env(cuda_visible_devices=None) - - -def _discovery_backend(script): - """A backend whose only subprocess call is the GPU topology probe.""" - backend = SlurmBackend( - SlurmBackendConfig( - name="b", type="slurm", account="acct", partition="batch", - nodes=1, time="00:10:00", job_name="job", gpus_per_node=4, - ) - ) - backend._subprocess_launcher = _FakeSubprocessLauncher(script=script) - return backend - - -def test_gpu_topology_probe_records_each_nodes_index_to_uuid_map(): - """The probe's output is the ground truth every later check rests on. - - A step can only ask "am I on the cards I was planned for?" by UUID, and this - bare srun is where those UUIDs come from. Parsing it wrong does not fail - loudly -- it leaves the maps empty and silently drops every task in the run to - index arithmetic, so the parse needs a test of its own. - """ - nodes = [ComputeNode(name="n0", ip_address="10.0.0.1", index=0), - ComputeNode(name="n1", ip_address="10.0.0.2", index=1)] - backend = _discovery_backend([(0, [ - "n0 0,GPU-aaa", "n0 1,GPU-bbb", - "n1 0,GPU-ccc", "n1 1,GPU-ddd", - ])]) - - asyncio.run(backend._discover_gpu_uuids(nodes=nodes)) - - # Ordered by HOST index, per node: the same planned slot is a different - # physical card on n1 than on n0, which is why this is not one flat list. - assert nodes[0].gpu_uuids == ["GPU-aaa", "GPU-bbb"] - assert nodes[1].gpu_uuids == ["GPU-ccc", "GPU-ddd"] - - cmd = backend._subprocess_launcher.calls[0]["command"] - # --overlap: shares the allocation instead of queueing behind a real step. - assert "--overlap" in cmd and "--nodelist" in cmd - assert cmd[cmd.index("--nodelist") + 1] == "n0,n1" - - -def test_gpu_topology_probe_rejects_a_partial_reading(): - """A gap means indices are missing, and a partial map resolves to the WRONG card. - - With 0 and 2 reported, index 1 is unaccounted for; treating the two as a - 0,1 run would map planned slot 1 onto the physical card at slot 2. Better to - say nothing and let the step fall back. - """ - nodes = [ComputeNode(name="n0", ip_address="10.0.0.1", index=0)] - backend = _discovery_backend([(0, ["n0 0,GPU-aaa", "n0 2,GPU-ccc"])]) - - asyncio.run(backend._discover_gpu_uuids(nodes=nodes)) - - assert nodes[0].gpu_uuids is None - - -def test_gpu_topology_probe_ignores_noise_and_survives_failure(): - """Never the reason a workflow cannot start.""" - nodes = [ComputeNode(name="n0", ip_address="10.0.0.1", index=0)] - # Slurm prologue chatter, a malformed pair, and a non-UUID value. - backend = _discovery_backend([(0, [ - "srun: job 42 queued and waiting for resources", - "n0 notanindex,GPU-zzz", - "n0 0,NOT-A-UUID", - "n0 0,GPU-aaa", - ])]) - asyncio.run(backend._discover_gpu_uuids(nodes=nodes)) - assert nodes[0].gpu_uuids == ["GPU-aaa"] - - # A failed probe leaves the map untouched rather than raising. - other = [ComputeNode(name="n0", ip_address="10.0.0.1", index=0)] - asyncio.run(_discovery_backend([(1, ["srun: error"])])._discover_gpu_uuids(nodes=other)) - assert other[0].gpu_uuids is None - - -def test_resource_env_hands_the_step_its_planned_uuids(): - """The seam the whole feature hangs on: driver map -> step env. - - _planned_gpu_uuids and gpu_placement.sh were each covered in isolation, but - nothing asserted resource_env actually JOINS them. Drop this line and every - other test still passes while every step silently falls back to index - arithmetic -- the failure mode is invisible. - """ - backend = SlurmBackend( - SlurmBackendConfig( - name="b", type="slurm", account="acct", partition="batch", - nodes=1, time="00:10:00", job_name="job", gpus_per_node=4, - ) - ) - backend.allocation = Allocation( - allocation_id="1", - nodes=[ComputeNode(name="n0", ip_address="10.0.0.1", index=0, - gpu_uuids=["GPU-a", "GPU-b", "GPU-c", "GPU-d"])], - owned=False, - ) - env = backend.resource_env(cuda_visible_devices="2,3") - assert env["SFLOW_PLANNED_GPU_UUIDS"] == "n0=GPU-c,GPU-d" - - # Nothing trustworthy to say -> say nothing. Absent means "fall back", and an - # empty string would read as "this task was planned for no GPUs". - backend.allocation.nodes[0].gpu_uuids = None - assert "SFLOW_PLANNED_GPU_UUIDS" not in backend.resource_env(cuda_visible_devices="2,3") - - -def test_planned_uuids_warns_when_a_slot_is_past_the_nodes_device_count(): - """The warning IS the deliverable here. - - Returning "" already drops the task to index arithmetic; the log line is the - only thing that tells the user their gpus_per_node is bigger than the node. - Silence would leave them with weaker checking and no idea why. - """ - alloc = Allocation( - allocation_id="1", - nodes=[ComputeNode(name="n0", ip_address="10.0.0.1", index=0, - gpu_uuids=["GPU-a", "GPU-b"])], - owned=False, - ) - with unittest.mock.patch.object(slurm_mod._logger, "warning") as warn: - assert slurm_mod._planned_gpu_uuids("0,9", alloc) == "" - assert warn.called - assert "gpus_per_node" in warn.call_args.args[0] - - -def test_allocation_probes_the_gpu_topology(): - """Both allocate() paths must actually call the probe. - - It is best-effort and never raises, so a missing call site is silent: the - maps stay None and every task drops to index arithmetic while the suite - stays green. - """ - import inspect - - # Once for the pre-existing (unowned) allocation, once for the salloc path. - assert inspect.getsource(SlurmBackend).count("await self._discover_gpu_uuids(") == 2 diff --git a/tests/unit/test_srun_gpu_placement.py b/tests/unit/test_srun_gpu_placement.py index 10e65b4..7e691f3 100644 --- a/tests/unit/test_srun_gpu_placement.py +++ b/tests/unit/test_srun_gpu_placement.py @@ -7,12 +7,7 @@ script sflow ships actually resolves to the right devices. """ -import atexit -import itertools -import shutil import subprocess -import tempfile -from pathlib import Path import pytest @@ -30,74 +25,9 @@ def allow_real_bash(fake_process): fake_process.allow_unregistered(True) -def _no_driver_dir() -> str: - """A PATH prefix whose `nvidia-smi` reports nothing. - - The prelude ALWAYS asks the driver now, so without this every test here would - measure whatever GPU the developer's machine happens to have -- this one has - an RTX 3090, which silently rewrote five expectations. Tests that want a - driver state it explicitly via `_run_in_container`. - """ - d = Path(tempfile.mkdtemp(prefix="sflow_no_driver_")) - smi = d / "nvidia-smi" - smi.write_text("#!/bin/bash\nexit 1\n") - smi.chmod(0o755) - return str(d) - - -_NO_DRIVER = _no_driver_dir() -# The placement script is staged into the workflow output dir and sourced from -# there -- that is the only delivery path -- so tests need a dir to stage into. -_STAGE = tempfile.mkdtemp(prefix="sflow_stage_") -# Module-level, so pytest's tmp_path machinery never sees them: clean up by hand -# or every run leaks two /tmp directories. -atexit.register(shutil.rmtree, _NO_DRIVER, ignore_errors=True) -atexit.register(shutil.rmtree, _STAGE, ignore_errors=True) - - -@pytest.fixture -def no_driver(): - return _NO_DRIVER - - -def _run(plan: str, observed: str | None, path_prefix=None) -> subprocess.CompletedProcess: - script = "\n".join(_gpu_placement_prelude(plan, workflow_out_dir=_STAGE) + ['echo "$CUDA_VISIBLE_DEVICES"']) - prefix = f"{path_prefix or _NO_DRIVER}:" - env = {"PATH": f"{prefix}/usr/bin:/bin"} - if observed is not None: - env["CUDA_VISIBLE_DEVICES"] = observed - return subprocess.run( - ["bash", "-c", script], capture_output=True, text=True, env=env - ) - - -def _run_in_container( - plan: str, visible_gpus: int, tmp_path, observed: str | None = None -) -> subprocess.CompletedProcess: - """Run the prelude against a driver reporting N GPUs numbered 0..N-1. - - This is the pyxis/enroot shape: the runtime passes through only this task's - devices and renumbers them from 0. ``observed`` is what --export=ALL carried - into the step, which is usually sflow's own plan in HOST ordinals. - """ - smi = tmp_path / "nvidia-smi" - lines = "\n".join( - f"GPU {i}: NVIDIA GB200 (UUID: GPU-{i:08x})" for i in range(visible_gpus) - ) - pairs = "\n".join(f"{i}, GPU-{i:08x}" for i in range(visible_gpus)) - # Answers both forms: `-L` for the legacy path, `--query-gpu=index,uuid` for - # the in-step probe that records the index the driver actually reports. - smi.write_text( - "#!/bin/bash\n" - 'if [ "$*" != "${*/index,uuid/}" ]; then\n' - f"cat <<'EOF'\n{pairs}\nEOF\n" - "else\n" - f"cat <<'EOF'\n{lines}\nEOF\n" - "fi\n" - ) - smi.chmod(0o755) - script = "\n".join(_gpu_placement_prelude(plan, workflow_out_dir=_STAGE) + ['echo "$CUDA_VISIBLE_DEVICES"']) - env = {"PATH": f"{tmp_path}:/usr/bin:/bin"} +def _run(plan: str, observed: str | None) -> subprocess.CompletedProcess: + script = "\n".join(_gpu_placement_prelude(plan) + ['echo "$CUDA_VISIBLE_DEVICES"']) + env = {"PATH": "/usr/bin:/bin"} if observed is not None: env["CUDA_VISIBLE_DEVICES"] = observed return subprocess.run( @@ -105,174 +35,6 @@ def _run_in_container( ) -@pytest.mark.parametrize( - "plan, visible_gpus, expected", - [ - # A container carved to this task's 2 GPUs numbers them 0,1, so host - # ordinals 2,3 name devices that do not exist here. - ("2,3", 2, "0,1"), - # The task that accidentally worked: its plan already matched the - # renumbering, which is why only the decode server ever failed. - ("0,1", 2, "0,1"), - # Single-GPU carve, high host ordinal. - ("3", 1, "0"), - # NOT carved: the whole node is visible with the variable unset, so the - # positional narrowing must still happen or concurrent tasks collide. - ("2,3", 4, "2,3"), - ("0,1", 4, "0,1"), - ], -) -def test_container_carve_keeps_the_containers_own_numbering( - plan, visible_gpus, expected, tmp_path -): - result = _run_in_container(plan, visible_gpus, tmp_path) - - assert result.returncode == 0, result.stderr - assert result.stdout.strip() == expected - - -@pytest.mark.parametrize( - "plan, observed, visible_gpus, expected", - [ - # THE BUG, exactly as it reached the cluster. srun runs --export=ALL, so - # sflow's own plan arrives in the step as CUDA_VISIBLE_DEVICES. Trusting it - # as an observation is circular: the counts match the plan by construction, - # so the remap concluded "already carved, keep what I see" and kept host - # ordinals 2,3 inside a container holding only 0,1 -> no GPU at all. - ("2,3", "2,3", 2, "0,1"), - # The task that accidentally worked: plan already matched the renumbering. - ("0,1", "0,1", 2, "0,1"), - # Single GPU carve, high host ordinal echoed back. - ("3", "3", 1, "0"), - # NOT a container: slurmstepd set a real in-range slice on a 4-GPU node. - # That IS an observation and must be honoured, or the GRES fix regresses. - ("0,1", "0,1", 4, "0,1"), - ("2,3", "2,3", 4, "2,3"), - # Whole allocation visible and handed over: still narrow positionally. - ("2,3", "0,1,2,3", 4, "2,3"), - ("0,1", "3,5,6,7", 8, "3,5"), - # UUID-form CUDA_VISIBLE_DEVICES carries no positions to remap against. - ("2,3", "GPU-abc,GPU-def", 2, "0,1"), - ], -) -def test_inherited_cuda_visible_devices_is_validated_against_the_driver( - plan, observed, visible_gpus, expected, tmp_path -): - result = _run_in_container(plan, visible_gpus, tmp_path, observed=observed) - - assert result.returncode == 0, result.stderr - assert result.stdout.strip() == expected - - -def test_no_driver_falls_back_to_the_plan(no_driver): - """With no driver to ask, behave exactly as before this probe existed.""" - result = _run("2,3", None, path_prefix=no_driver) - - assert result.returncode == 0, result.stderr - assert result.stdout.strip() == "2,3" - - -def test_no_driver_says_so_instead_of_degrading_silently(no_driver): - """A GPU task with no nvidia-smi loses the container-renumbering check. - - That is the ptyche failure restored: the inherited value is trusted, and a - container holding 0,1 keeps a plan of 2,3 and sees no GPU. It is the right - fallback -- guessing is worse -- but a slim image must not lose the protection - without saying so, or the only symptom is a task that dies inside CUDA. - """ - result = _run("2,3", "2,3", path_prefix=no_driver) - - assert result.returncode == 0 - assert "no nvidia-smi here" in result.stderr - assert "placement may be wrong" in result.stderr - # The warning must not pollute what the task reads. - assert result.stdout.strip() == "2,3" - - -def test_a_known_plan_never_narrows_less_than_an_unknown_one(no_driver, tmp_path): - """No nvidia-smi + planned UUIDs must still narrow, not bail. - - Knowing MORE about a step must never make sflow do LESS to it. Without a - probe the planned UUIDs cannot be checked, and it is right to record the - placement as unproven -- but returning early there left - CUDA_VISIBLE_DEVICES exactly as inherited, which on a GRES partition is the - whole allocation. Every concurrent worker then picks ordinal 0 and collides - on one physical GPU: precisely the OOM this prelude exists to prevent, and it - only happened when a UUID map was available. A task with NO map, in the same - container on the same partition, was narrowed correctly. - - So the no-probe case degrades to the same index arithmetic instead of - stopping. `action` still says `unverified`, so the e2e audit keeps counting - it as unproven. - """ - out = tmp_path / "out" - out.mkdir() - script = "\n".join( - _gpu_placement_prelude("2,3", workflow_out_dir=_STAGE) - + ['echo "CVD=$CUDA_VISIBLE_DEVICES"'] - ) - env = { - "PATH": f"{no_driver}:/usr/bin:/bin", - "SLURMD_NODENAME": "n0", - "SFLOW_TASK_OUTPUT_DIR": str(out), - # Slurm handed this step the whole 4-GPU allocation, as a GRES step with - # no --gres of its own always is. - "CUDA_VISIBLE_DEVICES": "0,1,2,3", - "SLURM_STEP_GPUS": "0,1,2,3", - "SFLOW_PLANNED_GPU_UUIDS": "n0=GPU-c,GPU-d", - } - r = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) - - assert r.returncode == 0, r.stderr - assert "CVD=2,3" in r.stdout, "must narrow to its own slice, not keep all four" - lines = (out / GPU_MARKER_FILE).read_text().splitlines() - assert "action=unverified" in lines, "narrowed, but not PROVEN -- both are true" - - -def test_a_fallback_record_keeps_the_uuids_it_could_not_find(tmp_path, fp): - """`fallback` must say what was wanted, not claim it never knew. - - When Slurm grants cards the planner never saw, the planned UUIDs WERE - resolved -- they just are not in the grant. A record saying - `planned_uuids=(not resolved)` sends the reader after the driver probe, which - worked fine, instead of at the grant, which is the actual finding. - """ - fp.allow_unregistered(True) - - out = tmp_path / "out" - out.mkdir() - script = "\n".join(_gpu_placement_prelude("2,3", workflow_out_dir=_STAGE)) - r = subprocess.run( - ["bash", "-c", script], - capture_output=True, - text=True, - env={ - "PATH": f"{_smi_stub(tmp_path, ['GPU-x', 'GPU-y'])}:/usr/bin:/bin", - "SLURMD_NODENAME": "n0", - "SFLOW_TASK_OUTPUT_DIR": str(out), - "CUDA_VISIBLE_DEVICES": "0,1", - "SLURM_STEP_GPUS": "6,7", - "SFLOW_PLANNED_GPU_UUIDS": "n0=GPU-c,GPU-d", - }, - ) - assert r.returncode == 0, r.stderr - - lines = (out / GPU_MARKER_FILE).read_text().splitlines() - assert "action=fallback" in lines - assert "planned_uuids=GPU-c,GPU-d" in lines, "resolved, just not present" - # The reason belongs in the record, not only in a step log nobody greps. - assert any("not among the devices Slurm granted" in ln for ln in lines) - # ...and what it landed on instead is right there to compare against. - assert "selected=0 GPU-x" in lines and "selected=1 GPU-y" in lines - - -def test_no_warning_when_the_driver_answers(tmp_path): - result = _run_in_container("2,3", 2, tmp_path, observed="2,3") - - assert result.returncode == 0 - assert "no nvidia-smi" not in result.stderr - - @pytest.mark.parametrize( "plan, observed, expected", [ @@ -293,8 +55,8 @@ def test_no_warning_when_the_driver_answers(tmp_path): ("2,3", "", "2,3"), ], ) -def test_remap_selects_planned_devices(plan, observed, expected, no_driver): - result = _run(plan, observed, path_prefix=no_driver) +def test_remap_selects_planned_devices(plan, observed, expected): + result = _run(plan, observed) assert result.returncode == 0, result.stderr assert result.stdout.strip() == expected @@ -322,68 +84,15 @@ def test_fewer_devices_than_planned_aborts(): A 4-GPU task handed 2 devices used to be accepted as "already carved" -- the same silent-wrong-placement failure this prelude exists to prevent. - - This is only a real fault while the step holds the task's WHOLE slice. Under - --gpus-per-task it is the normal shape, which is why no prelude is emitted - there (see test_gpus_per_task_leaves_placement_to_slurm). """ result = _run("0,1,2,3", "0,1") assert result.returncode == 97 assert "planned for 4" in result.stderr -@pytest.mark.parametrize("gpus_per_task", ["1", "2"]) -def test_gpus_per_task_leaves_placement_to_slurm(gpus_per_task): - """Per-rank carving breaks the prelude's premise, so it must step aside. - - --gpus-per-task makes the step REQUEST GRES, so Slurm hands each rank only its - own devices instead of handing the step the whole allocation. Counting the - task's full slice against one rank's view then aborts every rank of a perfectly - valid config: 8 ranks at 1 GPU each died with "step has 1 GPU(s) but this task - was planned for 8". Slurm keeps those per-rank sets disjoint itself, so there is - nothing to re-apply. - """ - assert _gpu_placement_prelude("0,1,2,3,4,5,6,7", gpus_per_task=gpus_per_task, workflow_out_dir=_STAGE) == [] - # Without the flag the same plan is still enforced. - assert _gpu_placement_prelude("0,1,2,3,4,5,6,7", workflow_out_dir=_STAGE) != [] - - -def test_build_command_omits_prelude_under_gpus_per_task(): - op = SrunOperator( - SrunOperatorConfig( - name="t", log_to_file=False, ntasks_per_node=8, gpus_per_task="1" - ) - ) - body = op.build_command( - task_name="ranks", - script=["torchrun train.py"], - envs={ - "CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7", - # Staging must be POSSIBLE here, or this would pass via the - # "nowhere to stage it" path instead of the gpus_per_task guard. - "SFLOW_WORKFLOW_OUTPUT_DIR": _STAGE, - }, - ).as_list()[-1] - - assert "SFLOW_GPU_PLAN" not in body - assert "torchrun train.py" in body, "the task itself must still run" - - -def test_many_ranks_sharing_the_task_slice_all_get_it(): - """The shape the samples actually use: ntasks_per_node with no per-rank carving. - - Every rank sees the whole step allocation and must narrow to the same planned - slice -- the app then picks its device by local rank. - """ - selected = { - _run("0,1,2,3", "0,1,2,3,4,5,6,7").stdout.strip() for _ in range(4) - } - assert selected == {"0,1,2,3"} - - def test_no_prelude_without_planned_gpus(): - assert _gpu_placement_prelude(None, workflow_out_dir=_STAGE) == [] - assert _gpu_placement_prelude("", workflow_out_dir=_STAGE) == [] + assert _gpu_placement_prelude(None) == [] + assert _gpu_placement_prelude("") == [] def test_build_command_splices_prelude_into_step_body(): @@ -391,22 +100,18 @@ def test_build_command_splices_prelude_into_step_body(): cmd = op.build_command( task_name="worker", script=["python -c 'import torch'"], - envs={ - "CUDA_VISIBLE_DEVICES": "2,3", - # The script is staged here and sourced from there. - "SFLOW_WORKFLOW_OUTPUT_DIR": _STAGE, - }, + envs={"CUDA_VISIBLE_DEVICES": "2,3"}, ) body = cmd.as_list()[-1] - assert "export SFLOW_GPU_PLAN='2,3'" in body - # Must precede the user script: the placement is an export the task inherits. - assert body.index("SFLOW_GPU_PLAN") < body.index("import torch") + assert "__sflow_plan='2,3'" in body + # Must precede the user script: the remap is an export the task then inherits. + assert body.index("__sflow_plan") < body.index("import torch") def test_build_command_omits_prelude_for_cpu_task(): op = SrunOperator(SrunOperatorConfig(name="t", log_to_file=False)) cmd = op.build_command(task_name="cpu", script=["echo hi"], envs={}) - assert "SFLOW_GPU_PLAN" not in cmd.as_list()[-1] + assert "__sflow_plan" not in cmd.as_list()[-1] # The plan is interpolated into shell text. Every value sflow's planner emits is @@ -428,12 +133,12 @@ def test_build_command_omits_prelude_for_cpu_task(): ], ) def test_non_numeric_plan_never_reaches_the_shell(hostile): - assert _gpu_placement_prelude(hostile, workflow_out_dir=_STAGE) == [] + assert _gpu_placement_prelude(hostile) == [] @pytest.mark.parametrize("plan", ["0", "2,3", "0,1,2,3", "10,11"]) def test_real_plans_are_still_emitted(plan): - assert _gpu_placement_prelude(plan, workflow_out_dir=_STAGE) != [] + assert _gpu_placement_prelude(plan) != [] def test_injected_value_cannot_execute_a_command(): @@ -448,7 +153,7 @@ def test_injected_value_cannot_execute_a_command(): ["bash", "-c", body], capture_output=True, text=True, - env={"PATH": f"{_NO_DRIVER}:/usr/bin:/bin", "CUDA_VISIBLE_DEVICES": "0,1,2,3"}, + env={"PATH": "/usr/bin:/bin", "CUDA_VISIBLE_DEVICES": "0,1,2,3"}, ) assert "INJECTED-COMMAND-RAN" not in result.stdout assert "task-body" in result.stdout, "the task itself must still run" @@ -466,19 +171,19 @@ def test_step_reports_the_devices_it_actually_selected(tmp_path): from sflow.utils.gpu import GPU_MARKER_FILE, planned_gpu_indices, task_gpu_indices plan = "0,1" - body = "\n".join(_gpu_placement_prelude(plan, workflow_out_dir=_STAGE)) + body = "\n".join(_gpu_placement_prelude(plan)) result = subprocess.run( ["bash", "-c", body], capture_output=True, text=True, env={ - "PATH": f"{_NO_DRIVER}:/usr/bin:/bin", + "PATH": "/usr/bin:/bin", "CUDA_VISIBLE_DEVICES": "3,5,6,7", # partial allocation "SFLOW_TASK_OUTPUT_DIR": str(tmp_path), }, ) assert result.returncode == 0, result.stderr - assert _marker_devices(tmp_path) == "3,5" + assert (tmp_path / GPU_MARKER_FILE).read_text().strip() == "3,5" task = SimpleNamespace( cuda_visible_devices=plan, @@ -530,13 +235,12 @@ def test_unparseable_marker_falls_back_to_the_plan(tmp_path): def test_only_rank_zero_writes_the_marker(tmp_path): - """Every rank runs this body, so letting them all truncate one path is a race. + """Every rank runs this body; with --gpus-per-task they hold different devices. - Ranks that were handed different partial allocations resolve to different - devices, which makes the reported value a coin flip. + Letting them all truncate one path makes the reported devices a coin flip. """ plan = "0,1" - body = "\n".join(_gpu_placement_prelude(plan, workflow_out_dir=_STAGE)) + body = "\n".join(_gpu_placement_prelude(plan)) def rank(procid, seen): return subprocess.run( @@ -544,7 +248,7 @@ def rank(procid, seen): capture_output=True, text=True, env={ - "PATH": f"{_NO_DRIVER}:/usr/bin:/bin", + "PATH": "/usr/bin:/bin", "CUDA_VISIBLE_DEVICES": seen, "SLURM_PROCID": procid, "SFLOW_TASK_OUTPUT_DIR": str(tmp_path), @@ -556,11 +260,11 @@ def rank(procid, seen): assert not marker.exists(), "a non-zero rank must not write the marker" assert rank("0", "0,1,2,3").returncode == 0 - assert marker.read_text().splitlines()[0].strip() == "0,1" + assert marker.read_text().strip() == "0,1" # A later non-zero rank must not clobber rank 0's value. assert rank("2", "4,5,6,7").returncode == 0 - assert marker.read_text().splitlines()[0].strip() == "0,1" + assert marker.read_text().strip() == "0,1" def test_multi_node_task_reports_the_plan_not_one_node(tmp_path): @@ -581,22 +285,6 @@ def test_multi_node_task_reports_the_plan_not_one_node(tmp_path): assert task_gpu_indices(multi) == [0, 1], "multi node: fall back to the plan" -def _marker_step(tmp_path, **slurm_env) -> subprocess.CompletedProcess: - body = "\n".join(_gpu_placement_prelude("0,1,2,3", workflow_out_dir=_STAGE)) - return subprocess.run( - ["bash", "-c", body], - capture_output=True, - text=True, - env={ - "PATH": f"{_NO_DRIVER}:/usr/bin:/bin", - "CUDA_VISIBLE_DEVICES": "0,1,2,3", - "SLURM_PROCID": "0", - "SFLOW_TASK_OUTPUT_DIR": str(tmp_path), - **slurm_env, - }, - ) - - def test_multi_node_step_writes_no_marker(tmp_path): """The writer's rule must match the reader's, or the file is a lie on disk. @@ -604,66 +292,28 @@ def test_multi_node_step_writes_no_marker(tmp_path): an artifact that is right for node 0 and wrong for every other node -- and looks authoritative to anyone who opens the task output dir. """ - assert ( - _marker_step(tmp_path, SLURM_STEP_NUM_NODES="2").returncode == 0 - ), "a multi-node step must still run normally" - assert not (tmp_path / GPU_MARKER_FILE).exists() - - assert _marker_step(tmp_path, SLURM_STEP_NUM_NODES="1").returncode == 0 - assert _marker_devices(tmp_path) == "0,1,2,3" - - -def test_allocation_wide_node_count_does_not_suppress_the_marker(tmp_path): - """The guard is about the STEP's nodes, not the allocation's. - - SLURM_NNODES is the backwards-compat alias of SLURM_JOB_NUM_NODES, and - SlurmBackend.resource_env copies the driver's SLURM_* into the env handed to - srun -- so a 4-node allocation puts SLURM_NNODES=4 in every step, including - single-node ones. Reading it here silently killed the marker for every task on - any multi-node allocation, which is exactly where concurrent single-node tasks - get devices the plan cannot predict. - """ - result = _marker_step(tmp_path, SLURM_NNODES="4", SLURM_STEP_NUM_NODES="1") - - assert result.returncode == 0, result.stderr - assert _marker_devices(tmp_path) == "0,1,2,3" - - -def test_multi_node_task_applies_its_per_node_slice_on_every_node(tmp_path): - """A multi-node plan is ONE node's slice, and each node resolves it alone. - - The planner guarantees this: pinned indices repeat per node, the multi-node - count path divides by the node count and refuses nodes with different - allocation cursors. So the same flat plan is evaluated independently in every - node's step, and each must land on its own node's devices -- including when the - nodes were handed different partial allocations. - """ - plan = "0,1" # gpus.count=4 over 2 nodes -> 2 per node - body = "\n".join(_gpu_placement_prelude(plan, workflow_out_dir=_STAGE) + ['echo "$CUDA_VISIBLE_DEVICES"']) + body = "\n".join(_gpu_placement_prelude("0,1,2,3")) - def node(seen, procid, nodeid): + def step(nnodes): return subprocess.run( ["bash", "-c", body], capture_output=True, text=True, env={ - "PATH": f"{_NO_DRIVER}:/usr/bin:/bin", - "CUDA_VISIBLE_DEVICES": seen, - "SLURM_PROCID": procid, - "SLURM_NODEID": nodeid, - "SLURM_STEP_NUM_NODES": "2", + "PATH": "/usr/bin:/bin", + "CUDA_VISIBLE_DEVICES": "0,1,2,3", + "SLURM_PROCID": "0", + "SLURM_NNODES": nnodes, "SFLOW_TASK_OUTPUT_DIR": str(tmp_path), }, ) - # Both nodes handed the whole 4-GPU node: narrow to the planned slots. - assert node("0,1,2,3", "0", "0").stdout.strip() == "0,1" - assert node("0,1,2,3", "1", "1").stdout.strip() == "0,1" - # Node 1 handed a partial allocation: slots are positions, so it follows. - assert node("4,5,6,7", "1", "1").stdout.strip() == "4,5" - # And no node leaves a marker that would speak for the others. + assert step("2").returncode == 0, "a multi-node step must still run normally" assert not (tmp_path / GPU_MARKER_FILE).exists() + assert step("1").returncode == 0 + assert (tmp_path / GPU_MARKER_FILE).read_text().strip() == "0,1,2,3" + def test_prelude_carries_a_short_banner_but_not_the_rationale(): """The generated command explains itself in a few lines, not ten. @@ -672,17 +322,13 @@ def test_prelude_carries_a_short_banner_but_not_the_rationale(): CUDA_VISIBLE_DEVICES; they do not need the marker-write design notes, which live in the module instead. """ - body = _gpu_placement_prelude("1,3", workflow_out_dir=_STAGE)[0] + body = _gpu_placement_prelude("1,3")[0] comments = [line for line in body.splitlines() if line.startswith("#")] # Wrapped, so it is obvious where sflow's block stops and the task's starts. assert body.startswith("# --- sflow GPU placement (begin)") assert body.rstrip("\n").endswith("# --- sflow GPU placement (end) ---------------------------------------------") - # Names what it does and the two shapes that need it -- a reader staring at a - # failing srun line must not conclude this block is what broke their GPUs. - assert any("planned GPUs" in c for c in comments) - assert any("GRES" in c for c in comments) - assert any("container" in c for c in comments) + assert any("slurmstepd rewrites CUDA_VISIBLE_DEVICES" in c for c in comments) # Thin: a banner, not an essay. assert len(comments) <= 8, comments # The long rationale stays out of the shipped shell text. @@ -692,889 +338,7 @@ def test_prelude_carries_a_short_banner_but_not_the_rationale(): def test_banner_does_not_break_the_emitted_shell(): """A comment block is inert, but it is spliced into a `bash -c` body.""" - body = _gpu_placement_prelude("2", workflow_out_dir=_STAGE)[0] + body = _gpu_placement_prelude("2")[0] result = subprocess.run(["bash", "-n"], input=body, text=True, capture_output=True) assert result.returncode == 0, result.stderr - - -# --------------------------------------------------------------------------- -# UUID-verified placement: compare against the plan, and act only if it differs. -# --------------------------------------------------------------------------- - - -def _marker_devices(out_dir) -> str: - """The device list from a marker file. - - Line 1 is the bare list; the lines after it are the audit record (node, the - action taken, every visible index -> UUID, and what CUDA ended up selecting). - Keeping the list first is the contract utils.gpu.task_gpu_indices relies on. - """ - return (out_dir / GPU_MARKER_FILE).read_text().splitlines()[0].strip() - - -def _smi_stub(tmp_path, uuids: list[str]): - """An nvidia-smi that answers BOTH forms the prelude uses.""" - smi = tmp_path / "nvidia-smi" - listing = "\n".join(f"GPU {i}: Fake (UUID: {u})" for i, u in enumerate(uuids)) - plain = "\n".join(uuids) - pairs = "\n".join(f"{i}, {u}" for i, u in enumerate(uuids)) - smi.write_text( - "#!/bin/bash\n" - 'if [ "$1" = "-L" ]; then\n' - f"cat <<'EOF'\n{listing}\nEOF\n" - 'elif [ "$*" != "${*/index,uuid/}" ]; then\n' - f"cat <<'EOF'\n{pairs}\nEOF\n" - "else\n" - f"cat <<'EOF'\n{plain}\nEOF\n" - "fi\n" - ) - smi.chmod(0o755) - return smi.parent - - -_RUN_SEQ = [0] - - -def _run_verified( - tmp_path, *, plan: str, visible: list[str], observed: str | None, planned_map: str, - node: str = "nodeA", -): - """Render the real prelude and report the resulting CVD and the branch taken. - - `action` from the placement record is the authoritative signal for which - branch ran -- verified / fallback / unverified. (It used to be - inferred from NVIDIA_VISIBLE_DEVICES being set, but the prelude no longer - writes that: the container runtime owns it, and it is consumed at container - creation, so writing it afterwards states something false about another layer.) - """ - _RUN_SEQ[0] += 1 - out_dir = tmp_path / f"run{_RUN_SEQ[0]}" - out_dir.mkdir() - script = "\n".join( - _gpu_placement_prelude(plan, workflow_out_dir=_STAGE) - + ['echo "CVD=${CUDA_VISIBLE_DEVICES-}"'] - ) - env = { - "PATH": f"{_smi_stub(tmp_path, visible)}:/usr/bin:/bin", - "SLURMD_NODENAME": node, - "SFLOW_PLANNED_GPU_UUIDS": planned_map, - "SFLOW_TASK_OUTPUT_DIR": str(out_dir), - } - if observed is not None: - env["CUDA_VISIBLE_DEVICES"] = observed - r = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) - cvd = next( - (ln.split("=", 1)[1] for ln in r.stdout.splitlines() if ln.startswith("CVD=")), None - ) - action = None - marker = out_dir / GPU_MARKER_FILE - if marker.exists(): - action = next( - (ln.split("=", 1)[1] for ln in marker.read_text().splitlines() - if ln.startswith("action=")), None - ) - return r, cvd, action - - -def test_placement_names_the_planned_cards_by_uuid_in_every_shape(tmp_path, fp): - """One rule covers every shape: look the planned UUIDs up, name their indices. - - There is deliberately no separate no-op / pin / narrow / re-select branch. - Each of those was the SAME question -- "which indices do the planned cards - have HERE?" -- answered against a different number of visible devices, so - they collapse into one lookup. Whether the answer differs from what was - inherited is a fact the record shows (inherited vs final), not a mode. - - Identity is the UUID, never the index: a device index stops being an identity - the moment a container renumbers from 0. - """ - fp.allow_unregistered(True) # drives a real `bash` subprocess - - want = "nodeA=GPU-c,GPU-d" - - # Whole node visible, inherited value already correct -> same answer back. - _, cvd, action = _run_verified( - tmp_path, plan="2,3", visible=["GPU-a", "GPU-b", "GPU-c", "GPU-d"], - observed="2,3", planned_map=want, - ) - assert (cvd, action) == ("2,3", "verified"), "an already-correct slice must survive" - - # A carved container whose plan happens to start at 0. (This is the shape - # that survived by coincidence before any of this existed.) - _, cvd, action = _run_verified( - tmp_path, plan="0,1", visible=["GPU-c", "GPU-d"], - observed="0,1", planned_map="nodeA=GPU-c,GPU-d", - ) - assert (cvd, action) == ("0,1", "verified") - - # Carved, and CUDA_VISIBLE_DEVICES not set at all. The cards are right -- - # unset means "every visible device" -- but leaving it unset is not safe: - # recipes read this variable to derive ranks, port offsets and device counts, - # and under `set -u` an unset one is a hard error (this killed bare_count_8 - # on perfwg). The same cards get named explicitly, which changes nothing - # about which are used. - _, cvd, action = _run_verified( - tmp_path, plan="2,3", visible=["GPU-c", "GPU-d"], observed=None, planned_map=want, - ) - assert (cvd, action) == ("0,1", "verified"), "an unset CVD must be named, not left unset" - - # The regression this exists for: host ordinals inherited into a 2-GPU - # container name nothing there, so they resolve to 0,1. - _, cvd, action = _run_verified( - tmp_path, plan="2,3", visible=["GPU-c", "GPU-d"], observed="2,3", planned_map=want, - ) - assert (cvd, action) == ("0,1", "verified") - - # More visible than planned, nothing selected: the lookup narrows. - _, cvd, action = _run_verified( - tmp_path, plan="2,3", visible=["GPU-a", "GPU-b", "GPU-c", "GPU-d"], - observed=None, planned_map=want, - ) - assert (cvd, action) == ("2,3", "verified") - - # Holding the right NUMBER of the WRONG cards is the failure a count-based - # check cannot see. It must be loud, not a silent pass. - r, _, _ = _run_verified( - tmp_path, plan="2,3", visible=["GPU-a", "GPU-b"], observed="2,3", planned_map=want, - ) - assert r.returncode == 97 - assert "is not visible" in r.stderr - - -def test_planned_uuids_are_resolved_per_node_not_per_task(tmp_path, fp): - """One flat plan, different physical cards on each node. - - The plan is a list of HOST indices applied identically on every node a task - spans, so slot 2 is a different card on node B than on node A. A step must - read its OWN node's entry; reading another node's would "verify" against - hardware it is not running on. - """ - fp.allow_unregistered(True) - - both = "nodeA=GPU-a2,GPU-a3;nodeB=GPU-b2,GPU-b3" - # On node B the same plan must resolve to B's cards -- and since they are - # already the visible ones, nothing is rewritten. - _, cvd, action = _run_verified( - tmp_path, plan="2,3", visible=["GPU-b2", "GPU-b3"], observed="2,3", - planned_map=both, node="nodeB", - ) - assert (cvd, action) == ("0,1", "verified") - - # Node B seeing node A's cards is a real placement error. - r, _, _ = _run_verified( - tmp_path, plan="2,3", visible=["GPU-a2", "GPU-a3"], observed="2,3", - planned_map=both, node="nodeB", - ) - assert r.returncode == 97 - - -def test_without_a_uuid_map_the_previous_behaviour_is_unchanged(tmp_path, fp): - """Clusters that cannot be probed must keep working exactly as before. - - No SFLOW_PLANNED_GPU_UUIDS -> the old index arithmetic runs, untouched. - """ - fp.allow_unregistered(True) - - _, cvd, action = _run_verified( - tmp_path, plan="2,3", visible=["GPU-a", "GPU-b", "GPU-c", "GPU-d"], - observed="2,3", planned_map="", - ) - assert (cvd, action) == ("2,3", "fallback"), "legacy path still runs" - - # And the legacy container remap still happens. - _, cvd, _ = _run_verified( - tmp_path, plan="2,3", visible=["GPU-c", "GPU-d"], observed="2,3", planned_map="", - ) - assert cvd == "0,1" - - -def test_planned_uuid_map_is_per_backend_and_refuses_partial_answers(): - """Encoding the plan as physical cards, per node. - - Two Slurm backends can have different gpus_per_node and different node sets, - so this is built from the calling backend's OWN allocation. A node whose - topology is unknown, or whose device count cannot contain a planned slot, is - omitted entirely: a partial map would let a step "verify" against a reading - that cannot hold the card it was planned for, which is worse than falling back. - """ - from sflow.core.backend import Allocation - from sflow.core.compute_node import ComputeNode - from sflow.plugins.backends.slurm import _planned_gpu_uuids - - def node(name, uuids): - return ComputeNode( - name=name, ip_address="1.2.3.4", index=0, num_gpus=len(uuids or []), - gpu_uuids=uuids, - ) - - alloc = Allocation( - allocation_id="1", - nodes=[node("n0", ["A0", "A1", "A2", "A3"]), node("n1", ["B0", "B1", "B2", "B3"])], - owned=False, - ) - # The SAME flat plan resolves to different physical cards per node. - assert _planned_gpu_uuids("2,3", alloc) == "n0=A2,A3;n1=B2,B3" - # Order is the plan's order, not the device order. - assert _planned_gpu_uuids("3,0", alloc) == "n0=A3,A0;n1=B3,B0" - - # Nothing trustworthy to say -> say nothing, and the step falls back. - assert _planned_gpu_uuids(None, alloc) == "" - assert _planned_gpu_uuids("2,3", None) == "" - assert _planned_gpu_uuids("9", alloc) == "", "a slot past the device count" - assert _planned_gpu_uuids("GPU-abc", alloc) == "", "UUID-form plan" - assert _planned_gpu_uuids("2,3", Allocation( - allocation_id="1", nodes=[node("n0", None)], owned=False - )) == "", "unprobed node" - - # A heterogeneous allocation contributes only the nodes it can vouch for. - mixed = Allocation( - allocation_id="1", - nodes=[node("small", ["S0", "S1"]), node("big", ["G0", "G1", "G2", "G3"])], - owned=False, - ) - assert _planned_gpu_uuids("2,3", mixed) == "big=G2,G3" - - -def test_marker_is_an_audit_record_whose_first_line_stays_the_device_list(tmp_path, fp): - """A bare index list cannot settle "was this placed right?". - - An index means nothing once a container renumbers from 0, so the marker also - records what the step actually SAW (index -> UUID), what CUDA ended up - selecting, and which branch ran. Compared against the allocation topology in - the summary, that distinguishes a bad placement from a recipe using the wrong - device -- after the run, without reproducing it. - - Line 1 stays the plain device list: utils.gpu.task_gpu_indices reads it, and - markers written by an older sflow must still parse. - """ - fp.allow_unregistered(True) # drives a real `bash` subprocess - - out = tmp_path / "task_out" - out.mkdir() - script = "\n".join(_gpu_placement_prelude("2,3", workflow_out_dir=_STAGE)) - env = { - "PATH": f"{_smi_stub(tmp_path, ['GPU-c', 'GPU-d'])}:/usr/bin:/bin", - "SLURMD_NODENAME": "nodeA", - "SFLOW_PLANNED_GPU_UUIDS": "nodeA=GPU-c,GPU-d", - "SFLOW_TASK_OUTPUT_DIR": str(out), - # Host ordinals inherited into a carved container -> a re-select. Both - # variables, as sflow's driver exports them (Backend.resource_env). - "CUDA_VISIBLE_DEVICES": "2,3", - "NVIDIA_VISIBLE_DEVICES": "2,3", - } - r = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) - assert r.returncode == 0, r.stderr - - text = (out / GPU_MARKER_FILE).read_text() - lines = text.splitlines() - # The contract the reader depends on. - assert lines[0] == "0,1" - from sflow.utils.gpu import parse_cuda_visible_devices - - assert parse_cuda_visible_devices(lines[0]) == [0, 1] - - record = dict( - line.split("=", 1) for line in lines[1:] if "=" in line and not line.startswith("visible=") - ) - assert record["node"] == "nodeA" - assert record["action"] == "verified" - # The record must show what ARRIVED, not just the post-state: inherited "2,3" - # next to a final "0,1" is the whole story of the container renumbering, and - # it is also how "did sflow change anything?" is answered now that there is - # no separate no-op action to read. - assert record["cuda_visible_devices_inherited"] == "2,3" - assert "located by UUID" in record["reason"] - # NVIDIA_VISIBLE_DEVICES as it ARRIVED, so a diff can be attributed. sflow - # never exports it (Backend.resource_env pops it), so whatever is here came - # from the container runtime -- which is exactly what makes it evidence. - assert record["nvidia_visible_devices_inherited"] == "2,3" - assert record["planned_host_indices"] == "2,3" - assert record["planned_uuids"] == "GPU-c,GPU-d" - assert record["cuda_visible_devices"] == "0,1" - # Everything the step could see, and what CUDA will really use. - assert "visible=0 GPU-c" in lines and "visible=1 GPU-d" in lines - assert "selected=0 GPU-c" in lines and "selected=1 GPU-d" in lines - - -def test_marker_shows_an_unchanged_slice_as_inherited_equals_final(tmp_path, fp): - """When the lookup confirms what arrived, the record has to say so. - - There is no "noop" action to read any more, so the record carries the fact - instead: an inherited value equal to the final one means the step was already - on the planned cards and nothing was rewritten. - """ - fp.allow_unregistered(True) - - out = tmp_path / "task_out" - out.mkdir() - script = "\n".join(_gpu_placement_prelude("2,3", workflow_out_dir=_STAGE)) - env = { - "PATH": f"{_smi_stub(tmp_path, ['GPU-a', 'GPU-b', 'GPU-c', 'GPU-d'])}:/usr/bin:/bin", - "SLURMD_NODENAME": "nodeA", - "SFLOW_PLANNED_GPU_UUIDS": "nodeA=GPU-c,GPU-d", - "SFLOW_TASK_OUTPUT_DIR": str(out), - "CUDA_VISIBLE_DEVICES": "2,3", - } - assert subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env).returncode == 0 - - lines = (out / GPU_MARKER_FILE).read_text().splitlines() - assert lines[0] == "2,3" - assert "action=verified" in lines - assert "cuda_visible_devices_inherited=2,3" in lines - assert "cuda_visible_devices=2,3" in lines, "unchanged: inherited == final" - assert "visible_gpu_count=4" in lines - # It selected the planned cards out of the whole node. - assert "selected=2 GPU-c" in lines and "selected=3 GPU-d" in lines - - -def test_multi_node_step_records_per_node_instead_of_racing_one_file(tmp_path, fp): - """Every node of a multi-node task has its own devices. - - One shared marker would be a race whose winner is arbitrary, which is why the - plain marker stays single-node-only. The per-node record is suffixed with the - node name so each node's evidence survives. - """ - fp.allow_unregistered(True) - - out = tmp_path / "task_out" - out.mkdir() - script = "\n".join(_gpu_placement_prelude("2,3", workflow_out_dir=_STAGE)) - env = { - "PATH": f"{_smi_stub(tmp_path, ['GPU-c', 'GPU-d'])}:/usr/bin:/bin", - "SLURMD_NODENAME": "nodeB", - "SFLOW_PLANNED_GPU_UUIDS": "nodeB=GPU-c,GPU-d", - "SFLOW_TASK_OUTPUT_DIR": str(out), - "SLURM_STEP_NUM_NODES": "2", - "SLURM_PROCID": "1", - "SLURM_LOCALID": "0", - "CUDA_VISIBLE_DEVICES": "0,1", - } - assert subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env).returncode == 0 - - assert not (out / GPU_MARKER_FILE).exists(), "the flat marker stays single-node" - # ..log, so .log stays the extension for viewers and globs. - per_node = out / GPU_MARKER_FILE.replace(".log", ".nodeB.log") - assert per_node.exists() - assert "node=nodeB" in per_node.read_text() - - -def test_slurm_reports_discovered_gpu_topology_to_the_summary(): - """The allocation's bare-metal topology belongs in the run record. - - Without it the per-task records have nothing to be compared against: knowing a - task held GPU-c only answers "was that the right card?" if the run also says - which cards the nodes had. - """ - from sflow.core.backend import Allocation - from sflow.core.compute_node import ComputeNode - from sflow.plugins.backends.slurm import SlurmBackend, SlurmBackendConfig - - backend = SlurmBackend( - SlurmBackendConfig( - name="s", - type="slurm", - account="acct", - partition="batch", - nodes=2, - time="00:10:00", - gpus_per_node=2, - ) - ) - assert backend.node_topology_report is None, "nothing to say before allocation" - - backend.allocation = Allocation( - allocation_id="1", - nodes=[ - ComputeNode(name="n0", ip_address="1", index=0, num_gpus=2, - gpu_uuids=["GPU-a", "GPU-b"]), - ComputeNode(name="n1", ip_address="2", index=1, num_gpus=0, gpu_uuids=None), - ], - owned=False, - ) - report = backend.node_topology_report - assert "n0: 2 GPU(s)" in report - assert "[0] GPU-a" in report and "[1] GPU-b" in report - # A node with no probe contributes nothing rather than a misleading empty entry. - assert "n1" not in report - - -def test_placement_logic_is_staged_once_and_sourced_not_pasted(tmp_path, fp): - """The step body should point at the logic, not carry it. - - ~150 lines of shell in every srun command made a failing command line - unreadable and duplicated the same text per task. It is staged once into the - run's output dir -- shared storage on Slurm, so every node can read it -- and - sourced. Sourced, not executed: it exports CUDA_VISIBLE_DEVICES into the - task's own shell, which a child process could not. - """ - fp.allow_unregistered(True) # drives a real `bash` subprocess - - out = tmp_path / "run" - out.mkdir() - body = "\n".join(_gpu_placement_prelude("2,3", workflow_out_dir=str(out))) - - staged = out / ".sflow" / "gpu_placement.sh" - assert staged.exists(), "the script must be staged where every node can read it" - assert body.count("\n") < 12, f"the step body should stay short:\n{body}" - assert f'. "{staged}"' in body, "sourced" - assert "bash " + str(staged) not in body, "must not be run as a child process" - # Inputs travel as environment, so nothing is interpolated into shell text. - assert "export SFLOW_GPU_PLAN='2,3'" in body - assert f"export SFLOW_GPU_MARKER='{GPU_MARKER_FILE}'" in body - - # Staging twice is fine (many tasks launch concurrently). - again = "\n".join(_gpu_placement_prelude("0,1", workflow_out_dir=str(out))) - assert staged.exists() and f'. "{staged}"' in again - - # And the staged script still does the job when sourced. - smi = _smi_stub(tmp_path, ["GPU-c", "GPU-d"]) - r = subprocess.run( - ["bash", "-c", f'{body}\necho "CVD=$CUDA_VISIBLE_DEVICES"'], - capture_output=True, - text=True, - env={ - "PATH": f"{smi}:/usr/bin:/bin", - "SLURMD_NODENAME": "nodeA", - "SFLOW_PLANNED_GPU_UUIDS": "nodeA=GPU-c,GPU-d", - "CUDA_VISIBLE_DEVICES": "2,3", - }, - ) - assert r.returncode == 0, r.stderr - assert "CVD=0,1" in r.stdout - - -def test_placement_is_skipped_when_the_script_cannot_be_staged(caplog): - """Nowhere the nodes can read it from -> do nothing, loudly. - - The alternative was pasting a second copy of the logic into the command line, - which meant two delivery paths to keep honest (and a source rewrite so - `return` stayed valid outside a sourced file). It was also unreachable in - practice: run_support always sets SFLOW_WORKFLOW_OUTPUT_DIR, so the only - trigger is an unwritable output dir -- by which point the run's own logs are - already broken. - - Skipping leaves CUDA_VISIBLE_DEVICES exactly as exported, which is the - behaviour from before this prelude existed. - """ - import logging - - with caplog.at_level(logging.WARNING): - assert _gpu_placement_prelude("2,3", workflow_out_dir=None) == [] - assert "Could not stage the GPU placement script" in caplog.text - - -def test_the_record_states_detected_values_not_echoed_ones(tmp_path, fp): - """Everything in the record that claims to be runtime must BE runtime. - - The point of the record is to settle "did this run on the cards we meant?" - after the fact, so a field that merely echoes what the driver passed in would - be worse than absent -- it would agree with the plan by construction. - - Probed inside the step: the visible UUIDs AND their indices (nvidia-smi - ignores CUDA_VISIBLE_DEVICES, so this is the namespace's real view), plus the - effective CUDA_VISIBLE_DEVICES / NVIDIA_VISIBLE_DEVICES after the decision. - Passed in by the driver: planned_host_indices and planned_uuids -- named as - "planned" precisely so they are not mistaken for observations. - """ - fp.allow_unregistered(True) # drives a real `bash` subprocess - - out = tmp_path / "task_out" - out.mkdir() - # The step sees cards the PLAN never mentions, so nothing here can be an echo - # of the plan: a real probe is the only way these names appear. - visible = ["GPU-zz0", "GPU-zz1", "GPU-c", "GPU-d"] - script = "\n".join(_gpu_placement_prelude("2,3", workflow_out_dir=_STAGE)) - r = subprocess.run( - ["bash", "-c", script], - capture_output=True, - text=True, - env={ - "PATH": f"{_smi_stub(tmp_path, visible)}:/usr/bin:/bin", - "SLURMD_NODENAME": "nodeA", - "SFLOW_PLANNED_GPU_UUIDS": "nodeA=GPU-c,GPU-d", - "SFLOW_TASK_OUTPUT_DIR": str(out), - "CUDA_VISIBLE_DEVICES": "2,3", - }, - ) - assert r.returncode == 0, r.stderr - lines = (out / GPU_MARKER_FILE).read_text().splitlines() - - # Detected: every visible card, with the index nvidia-smi reported. - assert "visible_gpu_count=4" in lines - for index, uuid in enumerate(visible): - assert f"visible={index} {uuid}" in lines - # Cards the plan never named still show up -> this is an observation. - assert "visible=0 GPU-zz0" in lines - - # Detected: the effective environment after the decision. Nothing changed - # here (the plan already selected the right cards). - assert "action=verified" in lines - assert "cuda_visible_devices=2,3" in lines - # There is no nvidia_visible_devices= post-state, and that is deliberate: - # sflow never writes that variable, so such a line could only ever repeat - # nvidia_visible_devices_inherited= and would read as a second, independent - # observation that it is not. - assert "nvidia_visible_devices_inherited=" in lines - assert not any(ln.startswith("nvidia_visible_devices=") for ln in lines) - - # Passed in, and labelled as such. - assert "planned_host_indices=2,3" in lines - assert "planned_uuids=GPU-c,GPU-d" in lines - - # selected= is the join of the two: real CVD resolved through the real probe. - assert "selected=2 GPU-c" in lines and "selected=3 GPU-d" in lines - - -def test_an_unset_cuda_visible_devices_is_named_not_left_unset(tmp_path, fp): - """The whole-node case: right cards, but nothing names them. - - `gpus.count` equal to a node's device count plans every GPU, so an - already-correct step can arrive with CUDA_VISIBLE_DEVICES unset -- "all of - them" is the right answer. Leaving it unset is what "nothing to do" would - look like to a naive check, and it broke a real run: assert_placement.sh dereferences the variable under - `set -u` and bare_count_8 died with "CUDA_VISIBLE_DEVICES: unbound variable" - on both nodes. Recipes also derive ranks and port offsets from it. - - So the cards get named explicitly. Same devices, contract intact. - """ - fp.allow_unregistered(True) # drives a real `bash` subprocess - - out = tmp_path / "task_out" - out.mkdir() - whole_node = ["GPU-w", "GPU-x", "GPU-y", "GPU-z"] - script = "\n".join(_gpu_placement_prelude("0,1,2,3", workflow_out_dir=_STAGE)) - r = subprocess.run( - # `set -u` on purpose: that is how the failure surfaced. - ["bash", "-c", f'set -u\n{script}\necho "CVD=${{CUDA_VISIBLE_DEVICES}}"'], - capture_output=True, - text=True, - env={ - "PATH": f"{_smi_stub(tmp_path, whole_node)}:/usr/bin:/bin", - "SLURMD_NODENAME": "c01", - "SFLOW_PLANNED_GPU_UUIDS": "c01=" + ",".join(whole_node), - "SFLOW_TASK_OUTPUT_DIR": str(out), - # no CUDA_VISIBLE_DEVICES in the environment at all - }, - ) - assert r.returncode == 0, r.stderr - assert "unbound variable" not in r.stderr - assert "CVD=0,1,2,3" in r.stdout - - lines = (out / GPU_MARKER_FILE).read_text().splitlines() - assert "action=verified" in lines - assert "cuda_visible_devices_inherited=" in lines, "named, not left unset" - assert "cuda_visible_devices=0,1,2,3" in lines - # Still the planned cards -- naming them changed nothing about which. - for index, uuid in enumerate(whole_node): - assert f"selected={index} {uuid}" in lines - - -def test_record_distinguishes_unset_from_set_but_empty(tmp_path, fp): - """Unset and empty are different states, and reporting them alike hides a bug. - - Both variables are recorded VERBATIM -- only and - are substituted, so the two can be told apart. sflow does not annotate what a - value means to CUDA or to a container runtime: it does not own those - semantics, and a gloss would be wrong on any stack that differs. A reader - seeing `all` can conclude the runtime carved nothing; that is their call to - make from the raw value. - """ - fp.allow_unregistered(True) # drives a real `bash` subprocess - - def record(env_extra): - out = tmp_path / f"t{abs(hash(tuple(sorted(env_extra.items()))))}" - out.mkdir() - script = "\n".join(_gpu_placement_prelude("0", workflow_out_dir=_STAGE)) - env = { - "PATH": f"{_smi_stub(tmp_path, ['GPU-a'])}:/usr/bin:/bin", - "SLURMD_NODENAME": "n0", - "SFLOW_PLANNED_GPU_UUIDS": "n0=GPU-a", - "SFLOW_TASK_OUTPUT_DIR": str(out), - } - env.update(env_extra) - r = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) - assert r.returncode == 0, r.stderr - return dict( - line.split("=", 1) - for line in (out / GPU_MARKER_FILE).read_text().splitlines() - if "=" in line and not line.startswith(("visible=", "selected=")) - ) - - # Neither variable present at all. - rec = record({}) - assert rec["cuda_visible_devices_inherited"] == "" - assert rec["nvidia_visible_devices_inherited"] == "" - - # Present but empty -- a different state, and it must not read as "unset". - rec = record({"CUDA_VISIBLE_DEVICES": "", "NVIDIA_VISIBLE_DEVICES": "all"}) - assert rec["cuda_visible_devices_inherited"] == "" - # Recorded verbatim: "all" is the runtime's own value, not sflow's gloss on it. - assert rec["nvidia_visible_devices_inherited"] == "all" - - -def test_gres_clusters_resolve_by_uuid_first_and_fall_back_only_on_a_miss(tmp_path, fp): - """On a GRES cluster the UUID lookup still runs first, and usually wins. - - Where GRES allocates the GPUs, slurmstepd sets CUDA_VISIBLE_DEVICES itself and - a `--overlap` step can be handed the whole node's grant. The lookup handles - that with no special case: the planned cards are in the grant, so their - indices are found and the step narrows to exactly them -- by identity, not by - counting positions. - - Only when a planned card is genuinely ABSENT does who-chose-the-devices - matter. Slurm may have granted cards the planner never saw, in which case the - plan can only mean a POSITION into the grant; resolving it as a host index - would abort a healthy run with exit 97. Two independent signals say Slurm - owns the devices, either sufficient: - * SLURM_STEP_GPUS -- set by Slurm only when THIS STEP took GRES - * the CUDA_VISIBLE_DEVICES we exported did not survive into the step - - Both are STEP-scoped. SLURM_JOB_GPUS deliberately is NOT one of them: it says - the JOB has GPUs, and Backend.resource_env copies every SLURM_* var from the - DRIVER's environment into every task, so on the `batch --submit` path it was - set for every step on any GRES cluster -- and because steps run --overlap - (Slurm does not carve per step, so SLURM_STEP_GPUS is unset there) it was the - ONLY signal in play. The hard fail below could not fire on the very clusters - it was written for. Asserted here so it is not quietly reinstated. - """ - fp.allow_unregistered(True) # drives a real `bash` subprocess - - seq = itertools.count() - - def run(env_extra, visible): - out = tmp_path / f"g{next(seq)}" - out.mkdir() - script = "\n".join( - _gpu_placement_prelude("2,3", workflow_out_dir=_STAGE) - + ['echo "CVD=$CUDA_VISIBLE_DEVICES"'] - ) - env = { - "PATH": f"{_smi_stub(tmp_path, visible)}:/usr/bin:/bin", - "SLURMD_NODENAME": "n0", - # The driver resolved the plan against a bare-metal probe of the WHOLE - # node; Slurm then granted something else entirely. - "SFLOW_PLANNED_GPU_UUIDS": "n0=GPU-c,GPU-d", - "SFLOW_TASK_OUTPUT_DIR": str(out), - } - env.update(env_extra) - r = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) - action = next( - (ln.split("=", 1)[1] for ln in (out / GPU_MARKER_FILE).read_text().splitlines() - if ln.startswith("action=")), None - ) if (out / GPU_MARKER_FILE).exists() else None - return r, action - - granted = ["GPU-x", "GPU-y"] # not the planned cards -- Slurm chose these - - # The `--overlap` shape from the field report: every concurrent worker is - # handed the WHOLE node and they all race on device 0 unless something - # narrows them. The planned cards are in the grant, so the lookup finds them - # and narrows by identity -- no fallback, no counting. - r, action = run( - {"CUDA_VISIBLE_DEVICES": "0,1,2,3", "SLURM_STEP_GPUS": "0,1,2,3"}, - ["GPU-a", "GPU-b", "GPU-c", "GPU-d"], - ) - assert (r.returncode, action) == (0, "verified"), r.stderr - assert "CVD=2,3" in r.stdout, "must land on the planned cards, not keep all four" - - # Now the miss: Slurm granted cards the planner never saw. Deferring keeps a - # healthy run alive instead of aborting on a plan that was never a host index. - r, action = run({"CUDA_VISIBLE_DEVICES": "0,1", "SLURM_STEP_GPUS": "3,5"}, granted) - assert r.returncode == 0, f"must not abort a GRES step\n{r.stderr}" - assert "CVD=0,1" in r.stdout, "both granted devices are the planned slice here" - assert action == "fallback" - assert "not among the devices Slurm granted this step" in r.stderr - - # ...but the JOB-level variable alone must NOT defer: it is what the driver - # leaks into every task, and treating it as "Slurm chose this step's devices" - # is what made the hard fail unreachable. CVD here still equals the plan, so - # nothing step-scoped says Slurm touched it -> this is a real mis-placement. - r, action = run({"CUDA_VISIBLE_DEVICES": "2,3", "SLURM_JOB_GPUS": "0,1,2,3"}, granted) - assert r.returncode == 97, f"a leaked job-level var must not excuse a miss\n{r.stderr}" - assert action == "missing" - assert "is not visible on" in r.stderr - - # No GRES variables, but our export was replaced anyway -> same deference. - r, action = run({"CUDA_VISIBLE_DEVICES": "0,1"}, granted) - assert (r.returncode, action) == (0, "fallback"), r.stderr - - # Deferring still narrows: 4 granted, none of them planned, a 2-slot plan - # -> positions 2,3 of the grant. - r, action = run( - {"CUDA_VISIBLE_DEVICES": "0,1,2,3", "SLURM_STEP_GPUS": "0,1,2,3"}, - ["GPU-p", "GPU-q", "GPU-r", "GPU-s"], - ) - assert (r.returncode, action) == (0, "fallback"), r.stderr - assert "CVD=2,3" in r.stdout - - # And the container case is untouched: our export DID survive, so the UUID - # lookup still corrects the renumbering. - r, action = run({"CUDA_VISIBLE_DEVICES": "2,3"}, ["GPU-c", "GPU-d"]) - assert (r.returncode, action) == (0, "verified"), r.stderr - assert "CVD=0,1" in r.stdout - - -def test_placement_is_skipped_when_staging_raises(tmp_path): - """Staging raises -> no placement, not a second inlined copy of the logic. - - The unstageable dir is a path UNDER A REGULAR FILE, so mkdir raises - NotADirectoryError (an OSError). Not `chmod(0o500)`: CI runs the suite as - root, and root ignores directory permission bits -- the write simply - succeeded there and the test failed on a premise that was never true, rather - than on the behaviour it meant to pin. - """ - blocker = tmp_path / "not-a-dir" - blocker.write_text("") - - assert _gpu_placement_prelude("2,3", workflow_out_dir=str(blocker / "out")) == [] - - -def test_a_hard_placement_failure_still_leaves_the_record(tmp_path): - """exit 97 is exactly when someone needs to see what the step held. - - The writer used to sit below every exit, so the one run worth diagnosing was - the one that produced no record at all. - """ - from types import SimpleNamespace - - from sflow.utils.gpu import task_gpu_record - - out_dir = tmp_path / "run" - out_dir.mkdir() - script = "\n".join(_gpu_placement_prelude("2,3", workflow_out_dir=_STAGE)) - r = subprocess.run( - ["bash", "-c", script], - capture_output=True, - text=True, - env={ - # Planned cards resolved, but neither is present here. - "PATH": f"{_smi_stub(tmp_path, ['GPU-x', 'GPU-y'])}:/usr/bin:/bin", - "SLURMD_NODENAME": "nodeA", - "SFLOW_PLANNED_GPU_UUIDS": "nodeA=GPU-a,GPU-b", - "SFLOW_TASK_OUTPUT_DIR": str(out_dir), - }, - ) - assert r.returncode == 97, r.stdout + r.stderr - record = task_gpu_record(SimpleNamespace(envs={"SFLOW_TASK_OUTPUT_DIR": str(out_dir)})) - assert record["action"] == "missing" - assert record["planned_uuids"] == "GPU-a,GPU-b" - assert "not visible" in record["reason"] - - -def _drive(tmp_path, name, *, plan, env_extra, visible=None): - """Run the real staged script and hand back (result, record dict).""" - out = tmp_path / name - out.mkdir() - script = "\n".join(_gpu_placement_prelude(plan, workflow_out_dir=_STAGE)) - env = { - "PATH": f"{_smi_stub(tmp_path, visible) if visible else _NO_DRIVER}:/usr/bin:/bin", - "SLURMD_NODENAME": "nodeA", - "SFLOW_TASK_OUTPUT_DIR": str(out), - } - env.update(env_extra) - r = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) - from types import SimpleNamespace - - from sflow.utils.gpu import task_gpu_record - - return r, task_gpu_record(SimpleNamespace(envs={"SFLOW_TASK_OUTPUT_DIR": str(out)})) - - -def test_a_node_missing_from_the_planned_map_says_so_once(tmp_path): - """The FQDN-vs-short-name mismatch silently switches verification off. - - The map is keyed by the name the DRIVER saw; the step keys by - $SLURMD_NODENAME. When they disagree the step finds no entry, falls back to - index arithmetic, and every task in the run quietly stops being verified -- - with nothing failing. The warning is the only signal, so it needs a test. - """ - r, record = _drive( - tmp_path, "mismatch", plan="0,1", - env_extra={ - # Map names the short form; the step reports the FQDN. - "SLURMD_NODENAME": "nodeA.cluster.example.com", - "SFLOW_PLANNED_GPU_UUIDS": "nodeA=GPU-a,GPU-b", - "CUDA_VISIBLE_DEVICES": "0,1", - }, - visible=["GPU-a", "GPU-b"], - ) - assert r.returncode == 0, r.stderr - assert "no planned-GPU entry for node 'nodeA.cluster.example.com'" in r.stderr - assert "falling back to device-index placement" in r.stderr - # Unverified, and the record must say why rather than claim a proven placement. - assert record["action"] == "fallback" - assert record["planned_uuids"] == "(not resolved)" - - -def test_every_abort_path_leaves_a_record(tmp_path): - """exit 97 without a record is the worst outcome: a run that failed on - placement and cannot be diagnosed. Proven for all three aborts, not just one. - """ - # too-few: 2 visible, 4 planned. - r, record = _drive( - tmp_path, "toofew", plan="0,1,2,3", - env_extra={"CUDA_VISIBLE_DEVICES": "0,1"}, visible=["GPU-x", "GPU-y"], - ) - assert r.returncode == 97, r.stderr - assert record["action"] == "too-few", record - assert "planned for 4" in record["reason"] - - # out-of-range: slot 9 is outside a 4-device visible set. - r, record = _drive( - tmp_path, "oor", plan="9", - env_extra={"CUDA_VISIBLE_DEVICES": "0,1,2,3"}, - visible=["GPU-a", "GPU-b", "GPU-c", "GPU-d"], - ) - assert r.returncode == 97, r.stderr - assert record["action"] == "out-of-range", record - assert "outside the visible devices" in record["reason"] - - -def test_the_record_names_every_selected_device_or_says_it_cannot(tmp_path): - """`selected=` must resolve each device CUDA will use back to a UUID. - - An index the step cannot see, and a device named as a UUID rather than an - ordinal, are both real shapes -- the record must state them rather than drop - the line, or a reader silently sees fewer devices than the task used. - """ - out = tmp_path / "sel" - out.mkdir() - # Force a post-state CUDA cannot resolve: no driver, so placement falls back - # to the plan verbatim, and the plan names devices nvidia-smi never reported. - script = "\n".join(_gpu_placement_prelude("0,1", workflow_out_dir=_STAGE)) - r = subprocess.run( - ["bash", "-c", script], capture_output=True, text=True, - env={"PATH": f"{_NO_DRIVER}:/usr/bin:/bin", "SLURMD_NODENAME": "nodeA", - "SFLOW_TASK_OUTPUT_DIR": str(out)}, - ) - assert r.returncode == 0, r.stderr - lines = (out / GPU_MARKER_FILE).read_text().splitlines() - selected = [ln for ln in lines if ln.startswith("selected=")] - assert len(selected) == 2, lines - # No driver -> nothing to resolve against, so each says so rather than lying. - assert all("(not visible here)" in ln for ln in selected), selected - - -def test_the_marker_format_keys_are_the_contract(tmp_path): - """Three independent parsers read this file: utils/gpu.py (Python), - sample_test.sh::gpu_placement_verified (shell sed), and a human. Renaming a - key breaks the shell reader silently, so the key set is pinned here. - """ - r, record = _drive( - tmp_path, "keys", plan="0,1", - env_extra={"SFLOW_PLANNED_GPU_UUIDS": "nodeA=GPU-a,GPU-b", - "CUDA_VISIBLE_DEVICES": "0,1"}, - visible=["GPU-a", "GPU-b"], - ) - assert r.returncode == 0, r.stderr - assert set(record) == { - "devices", # line 1, the bare device list - "node", "action", "reason", - "cuda_visible_devices_inherited", "nvidia_visible_devices_inherited", - "cuda_visible_devices", "planned_host_indices", "planned_uuids", - "visible_gpu_count", - }, sorted(record) - # The two keys the shell reader greps for must carry parseable values. - assert record["action"] == "verified" - assert record["planned_uuids"] == "GPU-a,GPU-b" diff --git a/tests/unit/test_srun_operator.py b/tests/unit/test_srun_operator.py index da5c2f8..d2f192c 100644 --- a/tests/unit/test_srun_operator.py +++ b/tests/unit/test_srun_operator.py @@ -445,40 +445,3 @@ def test_srun_step_exports_all_without_overrides(): assert "ENROOT_RUNTIME_PATH=" not in s assert "SLURM_JOB_ID=" not in s assert "--jobid 777" in s - - -def test_srun_operator_splits_bundled_extra_args(): - """srun was the one extra_args consumer that skipped normalize_extra_args. - - A bundled entry then reached srun as a SINGLE argv token it cannot parse, and - srun ignores what it cannot parse -- so the flags were silently dropped rather - than rejected. Every other consumer (slurm/docker/k8s backends, docker_run, - kubectl_config) normalizes; this one is now consistent with them. - """ - op = SrunOperator( - SrunOperatorConfig(name="op_srun", extra_args=["--gres=gpu:8 --exclusive"]) - ) - argv = op.build_command(task_name="t1", script=["echo hi"], envs={}).as_list() - - assert "--gres=gpu:8" in argv - assert "--exclusive" in argv - assert "--gres=gpu:8 --exclusive" not in argv, "must not stay one token" - # Still ahead of the payload, which is what srun requires. - assert argv.index("--exclusive") < argv.index("bash") - - -def test_srun_operator_extra_args_cannot_delete_an_sflow_managed_flag(): - """add_opt(name) defaults to append=False, which REMOVES every existing option - of that name (core/command.py). Routed through it, a user flag that collided - with an sflow-managed one silently deleted sflow's own value instead of - sitting alongside it. add_arg appends, so both survive and srun decides. - """ - op = SrunOperator( - SrunOperatorConfig( - name="op_srun", partition="batch", extra_args=["--partition", "other"] - ) - ) - argv = op.build_command(task_name="t1", script=["echo hi"], envs={}).as_list() - - assert "batch" in argv, "sflow's own --partition must survive" - assert "other" in argv diff --git a/tests/unit/test_utils_gpu.py b/tests/unit/test_utils_gpu.py index 05cd274..e929cfd 100644 --- a/tests/unit/test_utils_gpu.py +++ b/tests/unit/test_utils_gpu.py @@ -132,87 +132,3 @@ def test_two_docker_tasks_do_not_collide_on_gpu_zero(): a = _Task(cuda_visible_devices="0,1", envs={"CUDA_VISIBLE_DEVICES": "0,1"}) b = _Task(cuda_visible_devices="2,3", envs={"CUDA_VISIBLE_DEVICES": "0,1"}) assert set(task_gpu_indices(a)).isdisjoint(task_gpu_indices(b)) - - -def test_a_carved_container_reports_the_physical_card_not_its_own_numbering(tmp_path): - """The container's numbering is not a physical id. - - A task planned for host GPUs 2,3 that a runtime carved and renumbered from 0 - records "0,1" as the devices it used. Reporting that as PHYSICAL put every - containerised task on the wrong card in the summary's GPU Assignment table - (physical 0,1 / in-container 2,3 -- backwards, and impossible), and made the - hardware monitor sample the wrong GPUs. - - The step proves by UUID that it holds exactly the planned cards, and exits 97 - when it does not, so the planned HOST indices are the physical ones. - """ - from sflow.utils.gpu import GPU_MARKER_FILE, task_gpu_indices, task_gpu_record - - out = tmp_path / "boxed_high" - out.mkdir() - (out / GPU_MARKER_FILE).write_text( - "0,1\n" - "node=ptyche0074\n" - "action=verified\n" - "cuda_visible_devices=0,1\n" - "planned_host_indices=2,3\n" - "planned_uuids=GPU-74b4,GPU-0989\n" - "visible=0 GPU-74b4\n" - "visible=1 GPU-0989\n" - "selected=0 GPU-74b4\n" - "selected=1 GPU-0989\n" - ) - - class _Task: - envs = {"SFLOW_TASK_OUTPUT_DIR": str(out), "CUDA_VISIBLE_DEVICES": "2,3"} - assigned_nodes = ["ptyche0074"] - cuda_visible_devices = "2,3" - - task = _Task() - assert task_gpu_indices(task) == [2, 3], "physical is the planned host slice" - # ...and the step's own view is still recoverable for the other column. - assert task_gpu_record(task)["cuda_visible_devices"] == "0,1" - assert task_gpu_record(task)["action"] == "verified" - - -def test_an_unverified_record_still_reports_what_the_step_selected(tmp_path): - """Without UUID proof the record's line 1 is the best answer, as before. - - On that path the step selected host ordinals itself, so line 1 means what it - always meant -- notably the GRES case where slurmstepd hands over a partial - allocation and plan 0,1 really is physical 3,5. - """ - from sflow.utils.gpu import GPU_MARKER_FILE, task_gpu_indices - - out = tmp_path / "gres_task" - out.mkdir() - (out / GPU_MARKER_FILE).write_text( - "3,5\nnode=n0\naction=fallback\ncuda_visible_devices=3,5\n" - "planned_host_indices=0,1\n" - ) - - class _Task: - envs = {"SFLOW_TASK_OUTPUT_DIR": str(out), "CUDA_VISIBLE_DEVICES": "0,1"} - assigned_nodes = ["n0"] - cuda_visible_devices = "0,1" - - assert task_gpu_indices(_Task()) == [3, 5] - - -def test_task_gpu_record_returns_empty_when_there_is_nothing_to_read(tmp_path): - """Three ways to have no record, one answer: {}. - - Callers branch on empty-vs-parsed, so an exception or a half-filled dict here - would surface as a confidently wrong device list in the run summary. - """ - from types import SimpleNamespace - - from sflow.utils.gpu import GPU_MARKER_FILE, task_gpu_record - - # No task output dir at all. - assert task_gpu_record(SimpleNamespace(envs={})) == {} - # Dir known, marker absent (OSError). - assert task_gpu_record(SimpleNamespace(envs={"SFLOW_TASK_OUTPUT_DIR": str(tmp_path)})) == {} - # Marker present but empty -- there is not even a device line to trust. - (tmp_path / GPU_MARKER_FILE).write_text("") - assert task_gpu_record(SimpleNamespace(envs={"SFLOW_TASK_OUTPUT_DIR": str(tmp_path)})) == {} From 9618ee24edfb7d94d9f578a1b294d8fc007f106f Mon Sep 17 00:00:00 2001 From: Mira Jiang Date: Mon, 31 Aug 2026 16:35:16 +0800 Subject: [PATCH 2/2] Revert "sync develop from internal repo 14da8ab" --- docs-site/scripts/prepare-versioned-docs.js | 32 +- .../scripts/prepare-versioned-docs.test.js | 20 + docs-site/static/sflow_intro.html | 258 +++- docs/release_notes/RELEASE_NOTES_v0.3.1.md | 255 ++++ docs/user/architecture.md | 3 +- docs/user/backends.md | 39 +- docs/user/cli.md | 2 + docs/user/configuration.md | 2 +- docs/user/outputs.md | 10 + docs/user/quick-reference.md | 13 +- docs/user/resources.md | 9 + docs/user/run-output.md | 14 +- docs/user/samples.md | 6 +- .../inference_x_v2/benchmark_aiperf.yaml | 1 - .../modular/inference_x_v2/bulk_input.csv | 4 +- .../inference_x_v2/common_workflow.yaml | 2 +- .../sglang_agg_benchmark_aiperf_2n_008.yaml | 18 +- .../sglang_prefill_decode_benchmar_003.yaml | 4 +- .../sglang_prefill_decode_benchmar_004.yaml | 6 +- .../trtllm_agg_benchmark_aiperf_1n_007.yaml | 6 +- .../trtllm_prefill_decode_benchmar_001.yaml | 12 +- .../trtllm_prefill_decode_benchmar_002.yaml | 12 +- .../vllm_agg_benchmark_aiperf_1n_2_009.yaml | 2 +- .../modular/inference_x_v2/sglang/agg.yaml | 2 +- .../modular/inference_x_v2/sglang/decode.yaml | 2 +- .../inference_x_v2/sglang/prefill.yaml | 2 +- .../modular/inference_x_v2/trtllm/agg.yaml | 2 +- .../modular/inference_x_v2/trtllm/decode.yaml | 4 - .../inference_x_v2/trtllm/prefill.yaml | 4 - .../kubernetes/dynamo_sglang_agg.yaml | 1 - .../kubernetes/dynamo_sglang_disagg.yaml | 1 - .../kubernetes/dynamo_sglang_disagg_dsr1.yaml | 1 - .../kubernetes/dynamo_trtllm_agg.yaml | 1 - .../kubernetes/dynamo_trtllm_disagg.yaml | 1 - .../kubernetes/dynamo_vllm_agg.yaml | 1 - .../kubernetes/dynamo_vllm_disagg.yaml | 1 - .../self_contained/slurm/aiperf_template.yaml | 1 - .../slurm/dynamo_sglang_agg.yaml | 3 +- .../slurm/dynamo_sglang_disagg.yaml | 3 +- .../slurm/dynamo_trtllm_agg.yaml | 3 +- .../slurm/dynamo_trtllm_disagg.yaml | 11 +- .../self_contained/slurm/dynamo_vllm_agg.yaml | 1 - .../slurm/dynamo_vllm_disagg.yaml | 21 +- .../slurm/gpu_placement_matrix.yaml | 366 +++++ .../self_contained/slurm/infmax_v1_ds_r1.yaml | 3 +- .../self_contained/slurm/monitor_mixed.yaml | 167 ++- .../slurm/sglang_server_client.yaml | 1 - .../slurm/trtllm_serve_disagg.yaml | 18 + pyproject.toml | 5 + scripts/full_sample_tests.sh | 268 ++-- scripts/use_under_dev_sflow.sh | 87 +- src/sflow/app/assembly.py | 13 +- src/sflow/cli/batch.py | 7 + src/sflow/cli/sample.py | 3 - src/sflow/config/loader.py | 22 + src/sflow/config/schema.py | 8 + src/sflow/core/compute_node.py | 10 + src/sflow/core/execution_summary.py | 17 +- src/sflow/core/launcher.py | 191 --- src/sflow/core/task_graph.py | 31 - src/sflow/exceptions.py | 10 - .../postprocess_monitor_timeline.py | 175 ++- src/sflow/plugins/backends/slurm.py | 226 ++- src/sflow/plugins/operators/gpu_placement.sh | 260 ++++ src/sflow/plugins/operators/srun.py | 176 ++- .../inference_x_v2/benchmark_aiperf.yaml | 1 - .../modular/inference_x_v2/bulk_input.csv | 4 +- .../inference_x_v2/common_workflow.yaml | 2 +- .../sglang_agg_benchmark_aiperf_2n_008.yaml | 18 +- .../sglang_prefill_decode_benchmar_003.yaml | 4 +- .../sglang_prefill_decode_benchmar_004.yaml | 6 +- .../trtllm_agg_benchmark_aiperf_1n_007.yaml | 6 +- .../trtllm_prefill_decode_benchmar_001.yaml | 12 +- .../trtllm_prefill_decode_benchmar_002.yaml | 12 +- .../vllm_agg_benchmark_aiperf_1n_2_009.yaml | 2 +- .../modular/inference_x_v2/sglang/agg.yaml | 2 +- .../modular/inference_x_v2/sglang/decode.yaml | 2 +- .../inference_x_v2/sglang/prefill.yaml | 2 +- .../modular/inference_x_v2/trtllm/agg.yaml | 2 +- .../modular/inference_x_v2/trtllm/decode.yaml | 4 - .../inference_x_v2/trtllm/prefill.yaml | 4 - .../kubernetes/dynamo_sglang_agg.yaml | 1 - .../kubernetes/dynamo_sglang_disagg.yaml | 1 - .../kubernetes/dynamo_sglang_disagg_dsr1.yaml | 1 - .../kubernetes/dynamo_trtllm_agg.yaml | 1 - .../kubernetes/dynamo_trtllm_disagg.yaml | 1 - .../kubernetes/dynamo_vllm_agg.yaml | 1 - .../kubernetes/dynamo_vllm_disagg.yaml | 1 - .../self_contained/slurm/aiperf_template.yaml | 1 - .../slurm/dynamo_sglang_agg.yaml | 3 +- .../slurm/dynamo_sglang_disagg.yaml | 3 +- .../slurm/dynamo_trtllm_agg.yaml | 3 +- .../slurm/dynamo_trtllm_disagg.yaml | 11 +- .../self_contained/slurm/dynamo_vllm_agg.yaml | 1 - .../slurm/dynamo_vllm_disagg.yaml | 21 +- .../slurm/gpu_placement_matrix.yaml | 366 +++++ .../self_contained/slurm/infmax_v1_ds_r1.yaml | 3 +- .../self_contained/slurm/monitor_mixed.yaml | 167 ++- .../slurm/sglang_server_client.yaml | 1 - .../slurm/trtllm_serve_disagg.yaml | 18 + .../writing-sflow-yaml/schema-reference.md | 3 + src/sflow/utils/gpu.py | 54 +- tests/conftest.py | 5 - tests/e2e_tests/sample_test.sh | 908 +++++++++++- tests/integration/guide/sflow_dynamo.yaml | 1 - .../k8s_recipes/dynamo_sglang_disagg.yaml | 9 +- .../k8s_recipes/dynamo_trtllm_disagg.yaml | 9 +- .../golden/k8s_recipes/dynamo_vllm_agg.yaml | 9 +- .../test_app_assembly_resolve_variables.py | 21 + tests/unit/test_cli_batch.py | 9 +- tests/unit/test_config_loader.py | 53 + tests/unit/test_core_execution_summary.py | 57 + tests/unit/test_monitoring_postprocess.py | 183 +++ .../test_plugin_backends_slurm_backend.py | 226 ++- tests/unit/test_srun_gpu_placement.py | 1310 ++++++++++++++++- tests/unit/test_srun_operator.py | 37 + tests/unit/test_utils_gpu.py | 84 ++ 117 files changed, 5712 insertions(+), 814 deletions(-) create mode 100644 docs/release_notes/RELEASE_NOTES_v0.3.1.md create mode 100644 examples/self_contained/slurm/gpu_placement_matrix.yaml delete mode 100644 src/sflow/exceptions.py create mode 100644 src/sflow/plugins/operators/gpu_placement.sh create mode 100644 src/sflow/samples/self_contained/slurm/gpu_placement_matrix.yaml diff --git a/docs-site/scripts/prepare-versioned-docs.js b/docs-site/scripts/prepare-versioned-docs.js index 833b931..c66c1c2 100644 --- a/docs-site/scripts/prepare-versioned-docs.js +++ b/docs-site/scripts/prepare-versioned-docs.js @@ -205,6 +205,33 @@ function writeJson(filePath, value) { fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`); } +// In-page anchors that were already broken in a published snapshot. A released tag +// cannot be re-cut, so for frozen versions the only place left to repair them is +// here, at extraction time. Current docs are deliberately NOT covered: those are +// fixed at source in docs/. +// +// Each entry carries the heading it assumes, and the repair only fires when that +// heading is actually present. `main` is a branch, not a frozen tag -- once the +// source fix lands there its snapshot already has the corrected heading, and an +// unconditional rewrite would break the link a second time in the other direction. +const FROZEN_ANCHOR_REPAIRS = [ + { + from: "#modular-inference-recipe-inference_x_v2", + to: "#inference_x_v2", + onlyWhenHeading: /^### inference_x_v2\s*$/m, + }, +]; + +function repairFrozenAnchors(text) { + let out = text; + for (const repair of FROZEN_ANCHOR_REPAIRS) { + if (out.includes(repair.from) && repair.onlyWhenHeading.test(out)) { + out = out.replaceAll(repair.from, repair.to); + } + } + return out; +} + function rewriteVersionedDocsLinks(rootDir, versionLabel) { const entries = fs.readdirSync(rootDir, { withFileTypes: true }); for (const entry of entries) { @@ -213,7 +240,9 @@ function rewriteVersionedDocsLinks(rootDir, versionLabel) { rewriteVersionedDocsLinks(entryPath, versionLabel); } else if (entry.isFile() && entry.name.endsWith(".md")) { const text = fs.readFileSync(entryPath, "utf8"); - const rewritten = text.replaceAll("](/docs/", `](/docs/${versionLabel}/`); + const rewritten = repairFrozenAnchors( + text.replaceAll("](/docs/", `](/docs/${versionLabel}/`), + ); if (rewritten !== text) { fs.writeFileSync(entryPath, rewritten); } @@ -284,6 +313,7 @@ if (require.main === module) { module.exports = { buildDocVersionPlan, + repairFrozenAnchors, currentDocsSource, docsPaths, isReleaseTag, diff --git a/docs-site/scripts/prepare-versioned-docs.test.js b/docs-site/scripts/prepare-versioned-docs.test.js index f8d82bf..1d7e6fb 100644 --- a/docs-site/scripts/prepare-versioned-docs.test.js +++ b/docs-site/scripts/prepare-versioned-docs.test.js @@ -11,6 +11,7 @@ const { safeVersionDirName, currentDocsSource, prepareVersionedDocs, + repairFrozenAnchors, } = require("./prepare-versioned-docs"); function git(cwd, args) { @@ -164,3 +165,22 @@ test("prepareVersionedDocs materializes current docs, versioned docs, sidebars, "See [SPP](/docs/v1.2.3/sflow_spp).\n", ); }); + +test("repairFrozenAnchors fixes a stale anchor only while the old heading is present", () => { + const stale = [ + "See [Modular inference recipe](#modular-inference-recipe-inference_x_v2).", + "", + "### inference_x_v2", + ].join("\n"); + assert.match(repairFrozenAnchors(stale), /\(#inference_x_v2\)/); + + // The guard that matters: `main` is a branch, so once the source fix lands its + // snapshot already carries the long heading and the link is correct. Rewriting + // unconditionally would break it a second time, in the other direction. + const fixed = [ + "See [Modular inference recipe](#modular-inference-recipe-inference_x_v2).", + "", + "### Modular inference recipe (inference_x_v2)", + ].join("\n"); + assert.equal(repairFrozenAnchors(fixed), fixed); +}); diff --git a/docs-site/static/sflow_intro.html b/docs-site/static/sflow_intro.html index 54a0db7..b7fcdca 100644 --- a/docs-site/static/sflow_intro.html +++ b/docs-site/static/sflow_intro.html @@ -3,7 +3,7 @@ -sflow — Declarative Workflow Descriptor +sflow — Declarative Workflow Descriptor for Massive GPU Clusters @@ -352,17 +549,21 @@
01 / 13 -
- -

NV-sflow

-

Declarative Workflow Descriptor

-

Describe once. Run anywhere._

-
-View Documentation → -Explore Feature Map → -GitHub +
+
NV-SFLOW
+

Workflow orchestrator for large-scale GPU clusters of any kind

+

Describe once. Run anywhere._

+
+vLLMSGLangTensorRT-LLMDynamoPyTorch… +sflow +KubernetesSlurmDocker
+
+View Documentation → +Feature Map → +GitHub
+
Scroll @@ -396,12 +597,12 @@

The workload stays the same. The plumbing multiplies.

03 / 13
-Cross-Backend Transplanting -

Keep what. Swap where. Change only a few lines.

+Backend Agnostic by Design +

Keep what. Swap where. Rewrite nothing.

-
✎

Describe Once

Portable YAML — tasks, deps, resources, launch methods

-
⇄

Swap the Substrate

Change the backend/operator block; keep DAG, scripts, probes, and results.

-
⚙

Pluggable Plugins

Probes, artifacts, replicas — no platform coupling.

+
✎

One Semantic

A deployment's logic never changes — only the infrastructure glue does.

+
⇄

Swap the Backend Block

The same sflow.yaml runs on Docker, Slurm and Kubernetes.

+
⚙

Delegate, Don't Reimplement

Backends defer to each platform's native ecosystem instead of rebuilding it.

@@ -482,7 +683,7 @@

Dependencies activate the next stage

05 / 13
-Resource Planning +Cluster-Level Orchestration at Scale

Topology-aware GPU Allocation

sflow topology plannerGPU capacity · node affinity · task dependencies
▼ backend-native assignment
@@ -509,6 +710,7 @@

Topology-aware GPU Allocation

+
Node and GPU placement, multi-node replicas and sweeps, readiness probes, and batch submission — so one descriptor drives hundreds of GPUs instead of a pile of hand-written bash.
@@ -554,7 +756,7 @@

Understand

Backend Adapters

One intent becomes four native execution plans

Portable workflow contract   DAG · resources · probes · artifacts · results
-
▼ sflow adapts the execution substrate
+
▼ sflow delegates to each platform's native ecosystem
>_

Local

bash process
synthetic node/GPU map

▣

Docker

docker run
container GPU devices

@@ -578,9 +780,9 @@

Co-located GPU tasks, one fast intra-node fabric

Separate pods · isolated views

-
worker AGPU 0 visible
+
worker AGPU 0-1 visible
IB / TCP
↔
-
worker BGPU 1 visible
+
worker BGPU 2-3 visible
sflow merge
➔
@@ -736,7 +938,7 @@

Structured Error Analysis

identifiers that the TCP router can't parse. Fix: Verify DYN_REQUEST_PLANE and frontend -networking config match SGLang disagg routing.
+networking config match Dynamo disagg routing.
@@ -751,12 +953,13 @@

CLI at a Glance

- - - - + + + + - + +
CommandPurposeKey Flags
sflow runExecute a workflow--dry-run --tui --set
sflow batchGenerate sbatch scripts--submit --bulk-input
sflow composeMerge multiple YAMLs--resolve --validate
sflow visualizeRender DAG image--format png/svg/mermaid
sflow runExecute a workflow--dry-run --tui --set -f (multi-file)
sflow batchGenerate sbatch scripts--submit --bulk-input --row
sflow composeMerge multiple YAMLs--resolve --missable-tasks -o
sflow visualizeRender DAG graph--format png/svg/mermaid
sflow sampleList / copy examples--list -o
sflow skillExport AI agent skills--list -o
sflow skillInstall AI agent skills--list -o
sflow upgradeReinstall in place (alias: sflow update)--branch --dry-run --force
@@ -852,6 +1055,7 @@

Install. Explore. Run.

Start locally, validate before allocating hardware, then use the docs and repository as your source of truth.

# install from the repository +$ uv venv --python python3 && source .venv/bin/activate $ uv pip install "sflow @ git+https://github.com/NVIDIA/nv-sflow.git@main" # verify and try a local workflow diff --git a/docs/release_notes/RELEASE_NOTES_v0.3.1.md b/docs/release_notes/RELEASE_NOTES_v0.3.1.md new file mode 100644 index 0000000..761cdef --- /dev/null +++ b/docs/release_notes/RELEASE_NOTES_v0.3.1.md @@ -0,0 +1,255 @@ +# sflow v0.3.1 Release Notes + +**Release date:** August 2026 +**Previous release:** [v0.3.0](https://github.com/NVIDIA/nv-sflow/releases/tag/v0.3.0) (July 2026) + +--- + +## Highlights + +v0.3.0 made a recipe portable across Local, Docker, Slurm, and Kubernetes. v0.3.1 is about trusting the run it produces: **knowing which physical GPU each task actually landed on**, and **not losing the run to a silent hang or a truncated log**. + +Three things drive the release. **GPU placement** became explicit — you can pin device indices, sflow re-applies its plan *inside* the Slurm step where `slurmstepd` used to overwrite it, and concurrent runs on one box no longer collide on GPU 0. **The driver stopped hanging** — every `kubectl` call is bounded, console output is length-capped, an event-loop watchdog captures stalls that previously left no trace at all, and the post-run log re-fetch that silently truncated long-running pods is gone. **Monitoring became aimable** — reports are on by default, a `window:` clips them to the phase you actually measured, and GPU charts draw one line per device instead of an average. + +| Area | v0.3.0 | v0.3.1 | +| --- | --- | --- | +| **GPU selection** | `gpus.count` only, index-agnostic | **+ `gpus.indices`** to pin device ids; `count` must divide the node count | +| **Slurm GPU placement** | `slurmstepd` overwrote `CUDA_VISIBLE_DEVICES` → every rank on GPU 0 | in-step re-select **by UUID** + a per-task `sflow_gpus.log` audit record | +| **Concurrent runs on one host** | each run packed from device 0, unaware of the others | machine-local **UUID reservation registry** (Docker) | +| **Kubernetes task logs** | post-run re-fetch, truncated to the last rotation window | exactly what `kubectl logs -f` delivered | +| **Kubernetes control plane** | one `get pod` per pod per tick; no call bounded | batched per context; **every call timeout-bounded** | +| **Monitoring** | reports opt-in, one averaged GPU line | reports **on by default**, `window:` markers, one line per device | +| **Merge-pod + intra-group deps** | rejected outright | **gated in-pod**, the run proceeds | +| **Upgrading sflow** | hand-written `uv pip install` incantation | **`sflow upgrade`** | + +> **Read the [Breaking Changes](#breaking-changes) before upgrading.** Despite the patch version, this release changes several defaults that affect existing recipes — most notably monitor report output, GPU count validation, and resource release timing. The [Upgrade Guide](#upgrade-guide) has a five-minute checklist. + +--- + +## New Features + +### 1. Pin GPUs by device index — `resources.gpus.indices` + +`resources.gpus` used to take only a `count`, which is index-agnostic: the planner packs the task into any contiguous idle run on one node. That is the right default, but it cannot express NUMA/NVLink affinity, reproduce a vendor benchmark's exact topology, or steer around a known-bad device. + +```yaml +resources: + gpus: + indices: [2, 3] # pin these device ids +``` + +Three modes, all on Local, Docker, and Slurm: + +| You write | You get | +| --- | --- | +| `count: 4` | any contiguous idle run of 4 on a single node (never straddles a node boundary) | +| `indices: [2, 3]` | the first node where **both** are free; the node scan restarts at node 0, so later tasks backfill | +| `count: 8` + `indices: [0, 1]` | fan-out — `count` is the **total across nodes**, `indices` the per-node slice, so the task spans `8 / 2 = 4` nodes | + +`CUDA_VISIBLE_DEVICES` preserves the order you wrote. Indices must be non-negative, unique, and non-empty — deliberately unlike `nodes.indices`, which allows `-1`. + +**Kubernetes rejects `indices` at plan time.** The cluster's device plugin or DRA assigns the physical devices, so sflow will not pretend otherwise; use `count` to size the request. + +Samples: `sflow sample self_contained/slurm/gpu_indices` (all three modes on a 4×4 board, with the expected placement map annotated) and `self_contained/slurm/gpu_placement_matrix` (a regression matrix that asserts placement by UUID, not by GPU count). + +### 2. Slurm: the GPU plan is re-applied inside the step + +On a GRES partition, `slurmstepd` overwrites the `CUDA_VISIBLE_DEVICES` sflow exports. Every step then saw the whole allocation and every rank picked device 0 — tasks planned onto different GPUs all piled onto the same one, and the run's numbers were quietly wrong rather than loudly broken. + +sflow no longer trusts the inherited value. A prelude sourced inside each Slurm step probes the devices the step can really see, looks up the physical UUIDs the driver resolved this task's plan to, and re-exports the indices those same cards have *there*. Matching by UUID rather than by index is also what fixes pyxis/enroot containers, which renumber devices from 0. + +On by default for any Slurm task with a `resources.gpus` slice; srun `gpus_per_task` opts out, since Slurm already carves per rank. + +- **`/sflow_gpus.log`** — a per-task audit record: planned indices and UUIDs, the inherited environment, the visible index→UUID map, and the final selection. Multi-node tasks write one per node. +- **`exit 97`** — a planned card not visible at all, a step holding fewer GPUs than planned, or a planned slot out of range now aborts the step instead of silently running on the wrong device. When *Slurm* chose the devices rather than sflow, this degrades to index arithmetic instead of failing. + +See [Backends → GPU placement inside the step](https://nvidia.github.io/nv-sflow/docs/user/backends). + +### 3. Concurrent runs on one host stop colliding (Docker) + +Several `sflow run` processes on one machine each packed GPUs from device 0 independently — the in-process planner had no idea another run existed. Two runs on one workstation fought over the same cards. + +A file-locked registry in machine-local temp now makes concurrent runs, and concurrent tasks within a run, claim **disjoint physical GPUs by nvidia-smi UUID**. It refuses to take a GPU a foreign workload is already on. The container is launched with `--gpus "device="` naming those exact cards. + +**On by default for the Docker backend**, with no YAML opt-in. It is skipped — each case logging which one you hit — for remote `hosts:` pools, any `nodes > 1`, and hosts where `nvidia-smi` will not run. It requires POSIX `fcntl`, so it is inert on Windows. + +| Environment variable | Default | Effect | +| --- | --- | --- | +| `SFLOW_GPU_RESERVATION` | `1` | `0` disables the registry entirely | +| `SFLOW_WAIT_FOR_GPUS` | unset | any value turns waiting on; unset restores fail-fast | +| `SFLOW_GPU_RESERVATION_DIR` | `$TMPDIR/sflow-gpu-reservations` | must stay machine-local — on NFS another host's records read as local | +| `SFLOW_GPU_BUSY_MEM_MIB` | `512` | how much foreign memory counts as "busy" | +| `SFLOW_GPU_IGNORE_FOREIGN` | unset | `1` when sflow owns the box | + +Reservations are held per task, not per run, and released in a `finally` bounded at 10s so `Ctrl-C` can never hang on the lock. Records whose owning PID is gone are reaped on the next claim, guarded against PID recycling; another user's records are never reaped. + +**New: `--wait-for-gpus `** (Docker) — when too few GPUs are free at reserve time, wait instead of failing fast. `0` waits forever, `N` bounds the wait, omitting it fails fast. Also settable per-recipe as the backend field `wait_for_gpus`. + +### 4. Kubernetes: the driver stops hanging, and logs stop lying + +None of this needs a config change. + +- **Task logs are now the streamed log.** The post-run one-shot `kubectl logs` re-fetch that replaced the streamed file is gone. Because the kubelet **rotates container logs**, that re-fetch returned only the last window — one hour of server output was persisted as its final ~11 seconds. +- **Every `kubectl` call is bounded** (30s poll, 300s delete) and retried on the next tick. A silently-dead TCP connection to the API server used to wedge the driver for 15–20 minutes with nothing logged. +- **Pod-status polling is batched** — pods in one context share a single `kubectl get pod a b c …` per tick. Status was ~90% of all kubectl traffic (862 of 957 calls in a measured 7-pod run), and at 19 concurrent recipes that traffic self-congested (mean `get pod` latency 0.2s → 1.3s). Terminal detection is at most ~1.5s staler. +- **The output collect is bounded and no longer intrudes into running pods.** It emits a heartbeat every 30s and, on timeout, names `collect_grace_seconds` and `collect_node_local_output` in the message. It no longer `kubectl exec`s into a live pod — that used to put 42 execs into a serving TRT-LLM pod. +- **`collect_node_local_output: false`** (backend, default `true`) turns the collect machinery off entirely: no in-pod `EXIT` trap, no driver-side copy. Task completion then depends only on pod status, probes, and the merge-pod marker, and outputs must reach you via a shared filesystem, `uploads:`, or a PVC. +- **The RBAC preflight no longer blocks on permissions sflow does not need.** `get nodes` and `get deviceclasses` are now *optional* — a denial warns and degrades node-level detection (set `gpus_per_node` explicitly) instead of failing the run. A namespace-scoped ServiceAccount on a shared multi-tenant cluster now works out of the box. +- **`--extra-kubectl-apply-args`** (repeatable) passes a flag to the `kubectl apply` **subcommand** (`--validate=false`, `--server-side`, `--force-conflicts`). kubectl takes global flags *before* the verb, so these cannot ride on `--extra-kubectl-args` — sflow now warns if it spots an apply-only flag there. + +### 5. Merge-pod accepts dependencies between its own members + +v0.3.0 refused to merge tasks that depended on each other: merged tasks run concurrently in one pod, so a benchmark depending on a co-located server could not use merged pods at all — exactly the case merge-pod exists for. + +A **direct** member→member edge is now honored by gating rather than rejected. The dependent member waits on an in-pod gate; the driver opens it when the dependency reaches READY or COMPLETED. If the dependency failed, the gated member never starts and propagates its exit code. A member reachable only *transitively through a non-member* is still rejected. + +Visible in `sflow_summary.log` as `gated_on=` on the SUBMITTED row and a new `UNGATED` timeline event. A gated member's duration is measured from gate-open, not submission — a 5-second client no longer reads as 40 seconds. + +### 6. Monitoring you can aim + +- **Reports are on by default.** Declaring `monitor:` now writes the report folders; `report: {enabled: false}` opts out. Worth doing on large fan-outs — a report folder is a per-view *copy* of the samples, so cost scales with `samples × views`. +- **`monitor.window`** clips a report to the phase you actually measured, using markers from the task's own log: + + ```yaml + monitor: + window: + start: "Benchmark starting" + end: "re:Total throughput: [0-9.]+" + ``` + + Plain strings are case-sensitive literal substrings; prefix `re:` or `regex:` for a regex. `start` resolves first and `end` only from matches strictly after it. Task monitors only. The collector still runs for the whole task — only the report is clipped, so `sflow_monitor/lifecycle/` and `sflow_monitor/windowed/` sit side by side. If a marker never matches, sflow **warns and skips that report** rather than silently falling back to lifecycle timing, and writes `window_not_found.json` so you can see which pattern missed. +- **GPU charts draw one line per device**, each labelled with its own avg/max, instead of one averaged line that hid an idle card. Node-level scopes (cpu/memory/disk/network) stay averaged. +- **Multi-node reports split per node** — `timeline..svg` per node instead of one combined chart. CSVs stay combined. +- **Task-event markers were redesigned** — labels drawn in place with no legend to decode, near-simultaneous events merged into one labelled rule (`3 tasks submit +2 more`). Dotted = started, solid = ended. +- **Clock skew is corrected and warned about** — samples are shifted onto the driver's clock for reporting only when the estimated node offset excludes zero. Raw logs keep node timestamps. +- **Coverage fixes:** the workflow-level monitor now spans every monitorable backend instead of only the default one; a task on a backend no monitor covers no longer gets a bogus empty report folder; and a task that reserved no GPUs no longer picks up its node's GPUs in its report. + +### 7. `sflow upgrade` + +Reinstall sflow in place without hand-writing the `uv pip install` incantation: + +```bash +sflow upgrade # latest main of the public repo +sflow upgrade --branch develop # a specific ref +sflow upgrade --sflow-index-url # a private PyPI index +sflow upgrade --sflow-source-path ~/src/sflow # editable, from a local checkout +sflow upgrade --dry-run # print the resolved command and stop +``` + +Prefers `uv`, falls back to `pip`. It refuses to upgrade over an editable/source-tree dev install unless you pass `--force`. `sflow update` is an alias. + +Note the deliberate asymmetry: bare `sflow upgrade` installs **`main` of the public OSS repo**, whereas `sflow batch` installs whatever ref the *running* environment came from. + +### 8. Run output that explains itself + +- **`GPU Assignment`** in `sflow_summary.log` — per task, the physical GPUs it was planned onto next to the devices it actually saw, with a hint when the backend re-indexes inside the container. GPU/node charts now plot physical devices; previously every Docker task was drawn on GPU 0. +- **`Node Topology`** — the CPU/NUMA/GPU probe each backend captured at reservation time. +- **`External Command Health`** — call counts, failures, timeouts, and mean/max latency for `kubectl`/`srun`/`docker`, with a `healthy`/`DEGRADED` verdict. Written even when the run is cancelled or fails, which is when it matters. Backed by `command_trace.jsonl`, written lazily and only for *notable* calls (non-zero exit, or slower than 5s), so a healthy run leaves no file. A live warning fires when a control-plane call takes over 5s. +- **`loop_stalls.txt`** — if the driver's event loop stops being scheduled for 30s, sflow logs a warning and dumps every thread's Python stack here. Previously such a freeze produced no diagnostic output at all, because sflow's own logging runs on the thread that was stuck. Created only when a stall actually happens. +- **Console output is length-capped at 2000 characters per line**, on *every* backend. A single unbounded line was measured at ~6.3µs and ~300 bytes of RSS per character — a 48MB line cost ~5 CPU-minutes and ~14GB and froze the driver's event loop. **`.log` is unaffected and still holds every byte**; probes, `result:` parsing, and `output:` all read the file, so only the terminal changes. +- **Progress bars that end on a carriage return keep their final frame.** A bar whose last redraw ended in `\r` used to vanish from both the console and `.log`, and two consecutive redraw-terminated reads spliced into `50%60%` — in the log file, not just the console. +- **DAG cycle errors name the loop edge by edge** in `depends_on` phrasing, with a separate "Waiting behind it:" list, instead of a bare `Graph contains a cycle`. +- **`--dry-run`** now lists every planned monitor report folder with its group and, for marker windows, the patterns — so a marker typo surfaces before the run rather than after, as an empty report. +- **New doc page:** [Understanding run output](https://nvidia.github.io/nv-sflow/docs/user/run-output) — the output tree, a "what do you want to know?" lookup table, and a five-step failure triage. `sflow run` now prints the run directory on failure too, as soon as the folder exists. + +### 9. Kubernetes MPI: per-rank CPU binding + +`mpi.cpu_bind` (`core` | `numa` | `none`, default `core`) injects per-rank CPU binding, but **only when several ranks share a pod**, and never over a binding your recipe already passes. `core` gives each rank an isolated core slice — the tightest cap on the LLVM/OpenMP thread pools that `OMP_NUM_THREADS` alone does not reach. + +`mpi.cpu_bind_cores_per_rank` (default `8`, `0` = uncapped) bounds that slice: the launch-time value is `min(cores-in-cpuset / ranks-per-pod, this)`, and if the cpuset has fewer cores than ranks the binding is skipped rather than failing the launch. + +--- + +## Improvements + +- **Docker:** CPU-only tasks no longer see the host's GPUs (many CUDA images bake `NVIDIA_VISIBLE_DEVICES=all`). Container names carry the driver PID (`sflow-p--`) so concurrent runs never collide, and orphaned containers from a dead driver are reaped once per run. `--gpus device=0,1` is now quoted — docker used to parse the trailing `1` as a *count* and die with *"cannot set both Count and DeviceIDs on device request"*. A raw `--gpus` grant in `extra_args` now warns, because docker *accumulates* device requests and would widen the container past its reservation. +- **Slurm:** `extra_args` no longer silently drops repeated values — a bare value in `["-G","1","-N","1"]` used to delete an earlier identical one and produce a wrong allocation. A new driver-side topology probe records the index→UUID map per node. +- **`sflow batch`:** compound expressions such as `${{ variables.NUM_NODES * 2 }}` now resolve, so the generated sbatch no longer diverges from the dry run; a `--set` node override reaches `#SBATCH --nodes`; config errors keep their full multi-line pydantic detail instead of being cut to `Configuration validation failed:`; and `.cache` is excluded from the source-tree copy, which used to make rsync exit 24 and kill the job seconds in. +- **`--skip-artifact-check`** (`sflow run` and `sflow batch`) — a missing `fs://` path warns instead of failing, and is left alone rather than created as an empty directory. For paths that exist only where the task runs. `sflow batch` forwards it into the job, which is where the check actually runs. +- **Container image preflight was loosened** — an unrecognized reference now warns instead of aborting, and the regex accepts pyxis/enroot forms such as `nvcr.io#nvidia/ai-dynamo/sglang-runtime:1.2.0`. **Recipes that failed preflight on v0.3.0 now run.** +- **A readiness probe written with `match_pattern` no longer fails its own dry run.** The validator normalizes it into `regex_pattern`, so any dump-and-reload round trip (`sflow compose`, or the temp config `sflow batch` writes) handed both back and tripped the "only one of" check. +- **TUI:** the header no longer clips the elapsed clock and output directory, and ticks once a second so the clock does not freeze on an idle run. +- **Packaging:** the `parse` pin was loosened from `==1.16.0` to `>=1.16,<2`, which had made sflow uninstallable alongside anything needing a newer `parse`. A bare `pytest` no longer pulls images and launches containers. + +--- + +## Breaking Changes + +Ordered by how likely they are to affect an existing v0.3.0 recipe. + +1. **`monitor.report.enabled` now defaults to `true`.** Any recipe with a bare `monitor:` block now writes report folders — disk and post-processing you did not previously pay for. *Migration:* `report: {enabled: false}` to opt out. +2. **Monitor report paths gained a group segment.** `sflow_monitor//` → `sflow_monitor/lifecycle//` (or `windowed//`). `sflow_monitor/raw/` and `sflow_monitor.log` are unchanged. *Migration:* **any scraper, CI glob, or notebook reading `sflow_monitor//summary.csv` breaks** — insert the group segment. +3. **`resources.gpus.count` must now divide the assigned node count.** Previously the planner rounded *up* and reserved that many on **every** node: `nodes: 2` + `gpus: 1` silently consumed 2 GPUs, and `nodes: 2` + `gpus: 3` consumed 4; a `count: 10` against `(8, 2)` caps silently under-allocated to 4. Both are now rejected at plan time with a message naming a count that works. *Migration:* multiply by the node count — `nodes: 2` + `gpus: 1` becomes `gpus: 2`. Backend-agnostic. +4. **`fail_fast` now defaults per backend — `true` on Kubernetes.** A Kubernetes shell task whose script had a failing command masked by a later successful one (a trailing `echo`) now fails the task. Local, Docker, and Slurm are unchanged at `false`, and an explicit `fail_fast:` in the task always wins. +5. **`gpus.release_after` and `nodes.release_after` no longer default to `workflow_completion`** — an omitted value is now inferred. A **probe-less** GPU task that used to hold its GPUs for the whole workflow now releases at task completion, so downstream tasks may be packed onto them. An omitted node policy now means *placement only* (may overlap with other planned tasks) rather than exclusive. *Migration:* set `release_after: workflow_completion` explicitly to restore v0.3.0 behavior. +6. **A readiness-probed service that exits before becoming READY is now FAILED, even on exit 0.** It was previously marked COMPLETED, wrongly unblocking dependents against a dead server. sflow forces one final readiness scan first to avoid a false failure. The reason string changes from `process exit` to `service exited before readiness`. All backends. Probe-less tasks, and services that reached READY and then exited cleanly, are unaffected. +7. **Kubernetes `.log` is the streamed log, not a rebuilt one.** A multi-pod task's log is now ordered **chronologically, interleaved across pods**, instead of grouped per pod. `kubectl logs --prefix` still tags every line with its pod. *Migration:* a `result:` pattern that takes the *last* match now takes the last one **in time** across all pods — match on the pod prefix if you need a specific pod's value. +8. **Docker container names changed** — `sflow--` → `sflow-p--`. *Migration:* update any script matching the old name. +9. **Docker GPU tasks now fail fast when the host has no free GPUs**, because reservation is on by default. Foreign workloads above 512 MiB count as busy, so **on a workstation with an attached display every GPU can read busy**. The error names the busy GPUs and the escape hatches. *Migration:* `--wait-for-gpus`, `SFLOW_GPU_IGNORE_FOREIGN=1`, `SFLOW_GPU_BUSY_MEM_MIB`, or `SFLOW_GPU_RESERVATION=0`. +10. **Docker CPU-only containers no longer see GPUs** (`NVIDIA_VISIBLE_DEVICES=void`). Skipped when `extra_args` already grant GPUs. +11. **Slurm no longer exports `NVIDIA_VISIBLE_DEVICES` to srun steps.** Containers see all of the node's GPUs, so NVML consumers (`nvidia-smi`, DCGM) lose device isolation — the trade that makes the planned host-numbered slice addressable at all. Docker keeps isolation via `--gpus device=`. +12. **New Slurm in-step abort `exit 97`** when the planned GPUs cannot be honored — see [Feature 2](#2-slurm-the-gpu-plan-is-re-applied-inside-the-step). +13. **Kubernetes rejects `resources.gpus.indices` at plan time**, with a message pointing at `count`. +14. **Relative inline-content `file://` artifact URIs that escape the run output dir now hard-error.** `output_dir / raw` did not previously collapse `..`, so `file://../../x` silently wrote outside the run directory. Narrowly scoped: relative `file://` **with inline content** only — `fs://` model mounts and absolute `file://` are untouched. +15. **`sflow batch --bulk-input` refuses a `--nodes` that disagrees with the CSV node column.** `--nodes` sizes the sbatch allocation while the config's number sizes the workflow, so two different numbers allocate one size and plan another. Config-driven paths warn instead of refusing. +16. **`sflow batch` rejects a CSV data row with a blank `sflow_config_file`** — previously a raw `AttributeError` traceback, so this is strictly better, but it is a new hard failure. +17. **`timeout:` now warns on every load.** It was never enforced — no code path reads it, and `TaskStatus.TIMEOUT` is never assigned — but v0.3.0 said nothing. It is still accepted so existing recipes keep loading. *Migration:* bound the run with the backend's own limit (Slurm `--time`). The README and user docs have been corrected in this release; they previously implied it worked. + +--- + +## Tested Environments & Current Support + +**Validated setups** + +- vanilla bare-metal Kubernetes +- Google Kubernetes Engine (GKE) +- Slurm (GRES and non-GRES partitions) + +**Current limitations / work in progress** + +- **Kubernetes hardware monitoring is still not supported.** The built-in bare-node `monitor:` feature covers Local, Docker, and Slurm. On Kubernetes, monitor blocks are skipped because there is no DCGM/DaemonSet collector; sampling the driver host would produce misleading data. Unchanged from v0.3.0. +- **`resources.gpus.indices` is not supported on Kubernetes** — the device plugin or DRA assigns physical devices. Use `count`. +- **The GPU reservation registry covers the Docker backend only.** The Local backend also runs on the host but does not participate, so a Local run and a Docker run on the same box can still overlap. +- **DRA GPU allocation is supported but still WIP** — implemented, not yet broadly validated across Kubernetes distributions, versions, and NVIDIA DRA deployments. The device-plugin path remains the default. +- **Kubernetes execution is driver-attached.** Use interactive `sflow run`; detached Kubernetes batch execution is not supported. +- **The GPU reservation registry requires POSIX `fcntl`** and is inert on Windows. + +--- + +## Documentation + +Updated for this release: [Backends](https://nvidia.github.io/nv-sflow/docs/user/backends) (Slurm GPU placement, Docker GPU reservation, `collect_node_local_output`, `mpi.cpu_bind`), [CLI](https://nvidia.github.io/nv-sflow/docs/user/cli) (`sflow upgrade`, `--wait-for-gpus`, `--extra-kubectl-apply-args`, `--skip-artifact-check`, `batch --nodes` conflicts), [Resources](https://nvidia.github.io/nv-sflow/docs/user/resources) (`gpus.indices`), [Monitor](https://nvidia.github.io/nv-sflow/docs/user/monitor) (`window:`, report defaults), [Outputs & logs](https://nvidia.github.io/nv-sflow/docs/user/outputs) (new summary sections, `command_trace.jsonl`, `loop_stalls.txt`), and the new [Understanding run output](https://nvidia.github.io/nv-sflow/docs/user/run-output) page. + +Full documentation: [nvidia.github.io/nv-sflow](https://nvidia.github.io/nv-sflow/) + +--- + +## Upgrade Guide + +```bash +# From an existing install +sflow upgrade + +# Or fresh +uv pip install "sflow @ git+https://github.com/NVIDIA/nv-sflow.git@main" + +# Re-plan every recipe without consuming resources -- this surfaces the two +# breaking changes most likely to bite (gpus.count divisibility, gpus.indices on k8s) +sflow run -f your_recipe.yaml --dry-run + +# New samples worth a look +sflow sample self_contained/slurm/gpu_indices +sflow sample self_contained/slurm/monitor_mixed +sflow sample self_contained/docker/gpu_monitor +``` + +**A five-minute upgrade checklist:** + +1. `--dry-run` every recipe — `gpus.count` divisibility and `gpus.indices`-on-Kubernetes both fail at plan time, before anything is allocated. +2. Grep your tooling for `sflow_monitor//` and insert the `lifecycle/` or `windowed/` segment. +3. If you have a bare `monitor:` on a large fan-out, add `report: {enabled: false}`. +4. If a Kubernetes shell task relies on a trailing command masking an earlier failure, set `fail_fast: false` on it. +5. If a probe-less GPU task needs to hold its GPUs for the whole run, set `release_after: workflow_completion` explicitly. +6. If you parse a multi-pod Kubernetes `.log` for a *last* match, match on the pod prefix. + +Repository: [https://github.com/NVIDIA/nv-sflow](https://github.com/NVIDIA/nv-sflow) diff --git a/docs/user/architecture.md b/docs/user/architecture.md index 28f6d35..6b0b9a6 100644 --- a/docs/user/architecture.md +++ b/docs/user/architecture.md @@ -216,12 +216,13 @@ stateDiagram-v2 | Command | Purpose | Key Options | |---------|---------|-------------| -| **`sflow run`** | Execute a workflow | `--dry-run`, `--tui`, `--bulk-input/--row`, `--set/-s`, `--artifact/-a`, `--missable-tasks/-M`, `--skip-artifact-check`, `--extra-args/-e`, `--output-dir`, `--log-level`, `--enable-workflow-monitor`, `--enable-task-monitor`, `--include-nodes/--exclude-nodes`, `--kubeconfig`, `--kube-context`, `--kube-namespace`, `--kube-node-selector`, `--kube-compute-domain-channel`, `--kube-compute-domain-create`, `--extra-kubectl-args` | +| **`sflow run`** | Execute a workflow | `--dry-run`, `--tui`, `--bulk-input/--row`, `--set/-s`, `--artifact/-a`, `--missable-tasks/-M`, `--skip-artifact-check`, `--wait-for-gpus`, `--extra-args/-e`, `--output-dir`, `--log-level`, `--enable-workflow-monitor`, `--enable-task-monitor`, `--include-nodes/--exclude-nodes`, `--kubeconfig`, `--kube-context`, `--kube-namespace`, `--kube-node-selector`, `--kube-compute-domain-channel`, `--kube-compute-domain-create`, `--extra-kubectl-args`, `--extra-kubectl-apply-args` | | **`sflow batch`** | Generate Slurm sbatch scripts | `--submit`, `--bulk-input` (CSV sweeps), `--bulk-submit` (YAML folders), `--row`, `--nodes`, `--partition`, `--account`, `--time`, `--resolve`, `--skip-artifact-check`, `--sflow-version`, `--enable-workflow-monitor`, `--enable-task-monitor`, `--include-nodes/--exclude-nodes` | | **`sflow compose`** | Merge multiple YAMLs into one | `--resolve`, `--validate`, `--bulk-input`, `--row`, `--missable-tasks/-M`, `-o/--output` | | **`sflow visualize`** | Render DAG as image/mermaid | `--format` (png/svg/pdf/mermaid/dot), `--show-variables`, `--set/-s`, `--artifact/-a`, `--missable-tasks/-M` | | **`sflow sample`** | List/copy example workflows | `--list`, `--force`, `-o/--output` | | **`sflow skill`** | Copy agent skills into project (merges into existing directory) | `--list`, `--force` (overwrite existing files), `-o/--output` | +| **`sflow upgrade`** (alias `sflow update`) | Reinstall sflow in place from git or an index | `--repo`, `--branch`, `--sflow-version`, `--sflow-index-url`, `--sflow-source-path`, `--force`, `--dry-run` | ### Multi-file Input diff --git a/docs/user/backends.md b/docs/user/backends.md index 730f5e0..8f8b365 100644 --- a/docs/user/backends.md +++ b/docs/user/backends.md @@ -174,6 +174,40 @@ sflow batch -f workflow.yaml -e "--gpus-per-node=8" -e "--segment=2" This is useful for quick adjustments or when testing different cluster configurations. ::: +### GPU placement inside the step + +On a GRES partition `slurmstepd` overwrites the `CUDA_VISIBLE_DEVICES` sflow exports, so +every step sees the whole allocation and every rank picks device 0 — several tasks that +were planned onto different GPUs all pile onto the same one. sflow therefore no longer +trusts the inherited value. A prelude sourced inside each Slurm step probes the devices +the step can actually see, looks up the physical UUIDs the driver resolved this task's +plan to (`SFLOW_PLANNED_GPU_UUIDS`), and re-exports the indices those same cards have +*here*. Matching by UUID rather than by index is also what fixes pyxis/enroot containers, +which renumber devices from 0. + +This is **on by default** for any Slurm task with a `resources.gpus` slice. The one thing +that opts out is srun `gpus_per_task`, because Slurm already carves GPUs per rank there. + +- **Audit record.** Each task writes `/sflow_gpus.log` (`sflow_gpus..log` per + node on multi-node tasks): planned indices and UUIDs, the inherited environment, the + visible index→UUID map, and the final selection. It is deliberately not dot-prefixed so + artifact browsers show it. The `GPU Assignment` section of `sflow_summary.log` is built + from it. +- **Hard failure `exit 97`.** If a planned card is not visible at all, the step holds fewer + GPUs than planned, or a planned slot is out of range, the step aborts with exit code 97 + rather than silently running on the wrong device. When Slurm — not sflow — chose the + devices, sflow degrades to index arithmetic instead of failing. +- **Graceful skips.** If `CUDA_VISIBLE_DEVICES` is not a plain comma-separated list of + non-negative integers (for example a workflow variable of that name shadowing it), or the + placement script cannot be staged, sflow warns and falls back to the previous behavior. + +:::note +`NVIDIA_VISIBLE_DEVICES` is no longer exported to srun steps. Containers therefore see all +of the node's GPUs — NVML consumers such as `nvidia-smi` and DCGM lose device isolation — +in exchange for the planned host-numbered slice being addressable at all. The Docker +backend is unaffected: it keeps isolation via `--gpus device=`. +::: + ## Selecting or excluding nodes (all backends) Restrict which cluster nodes a run may use with two backend-agnostic controls that @@ -332,11 +366,6 @@ Two things to know about `SFLOW_WAIT_FOR_GPUS`: Reservation requires POSIX file locking and is inert on Windows. -To see it work on a real GPU host, `examples/gpu_reservation/` ships a runnable demo -(`demo.yaml`) plus two harnesses: `prove.sh` asserts the four guarantees above -(exact pinning, disjoint concurrent tasks, fail-fast, `--wait-for-gpus`) and exits -non-zero if any fails, and `stress.sh` fires many concurrent runs at one GPU pool. - > **Changed behavior.** Container names gained a driver-PID segment > (`sflow--` → `sflow-p--`) so concurrent runs on one > host never collide; scripts matching the old name must be updated. A task that diff --git a/docs/user/cli.md b/docs/user/cli.md index edb61b0..32a9eba 100644 --- a/docs/user/cli.md +++ b/docs/user/cli.md @@ -196,6 +196,8 @@ Common options: - `--account, -A `: Slurm account (auto-detected if not specified) - `--time `: time limit (e.g., `02:00:00`) - `--nodes, -N `: number of nodes. If omitted, single-job and bulk-submit modes derive it from the config's Slurm backend `nodes` field. Bulk-input mode requires either this flag or a CSV node-count column (`SLURM_NODES`, `NUM_SLURM_NODES`, or `NUM_NODES`). + + **When the two node counts disagree.** `--nodes` sizes the *sbatch allocation*; the config's own number sizes the *workflow inside it* — the backend's node count and the `match_count` of readiness probes. The same number in both is the normal way to run, but two different numbers means the job is allocated one size and the recipe plans for another: probes wait on a node that was never allocated, or the job holds nodes nothing will ever use. `--bulk-input` **rejects** the run outright (the CSV states the size per row, so a disagreement is unambiguously a mistake) and names up to five offending rows; the config-driven single-job and `--bulk-submit` paths **warn**. A `--set` of the node variable counts as the row's value, since it overrides the CSV cell. - `--gpus-per-node, -G `: number of GPUs per node for cluster topology. Config `gpus_per_node` wins when present. Applied to sflow validation and planning only, not as a Slurm directive. Use `-e '--gpus-per-node=N'` for `sflow batch`, or backend `extra_args` for `sflow run`, if your cluster requires the Slurm allocation flag. - `--job-name, -J `: Slurm job name (default: `sflow`) - `--set, -s KEY=VALUE`: override variables (repeatable) diff --git a/docs/user/configuration.md b/docs/user/configuration.md index 2cb3816..870f96f 100644 --- a/docs/user/configuration.md +++ b/docs/user/configuration.md @@ -336,7 +336,7 @@ Beyond `depends_on`, `resources`, `replicas`, and `probes`, a task supports: - `operator` / `backend`: name (or inline override object) of the operator or backend for this task. - `ports`: service ports the task exposes (each with `port` and an optional `name`). -- `timeout`: per-task timeout (seconds or a string like `30m`); the workflow also has its own `timeout`. +- `timeout`: **accepted but not enforced.** Both the per-task and the workflow `timeout` are parsed and merged, but nothing reads them — a task that sets one runs unbounded, and `sflow run` logs a warning naming every task that does. Bound the run with the backend's own limit instead (Slurm `--time`, a Kubernetes `activeDeadlineSeconds`). Kept so existing recipes keep loading. - `fail_fast`: bool, **default depends on the backend** — `true` on Kubernetes (a failed command in a pod should fail the task), `false` on Slurm/local/docker (shell default: only the last command's exit code counts). Leave unset to take the backend default, or set explicitly (`true`/`false`) to override per task. When effective-true, sflow prepends `set -e` to shell-operator scripts so any failed command fails the task. Applies to shell operators only (never `python`, whose script is Python source). - `variables`: task-scoped variables (same format as top-level `variables`). - `retries`: retry policy (`count`, `interval`, `backoff`) for a failed task. diff --git a/docs/user/outputs.md b/docs/user/outputs.md index ad12864..799f38f 100644 --- a/docs/user/outputs.md +++ b/docs/user/outputs.md @@ -59,10 +59,20 @@ After a successful run, `sflow run` prints the output folder, summary path, and - task duration timeline and task event timeline - probe traces — the last attempt of every readiness/failure probe — when any task defines probes - GPU and node usage charts when resource placement data exists +- **`GPU Assignment`** — per task, the physical GPUs it was planned onto next to the devices it actually saw, read back from the placement record the step wrote. When the backend re-indexes devices inside the container (pyxis/enroot renumbering from 0), the section says so rather than leaving you to reconcile two different numbering schemes. +- **`Node Topology`** — the CPU / NUMA / GPU probe each backend captured at reservation time, printed as one `[backend ]` block per backend. +- **`External Command Health`** — call counts, failures, timeouts, and mean/max latency for the external commands sflow shells out to (`kubectl`, `srun`, `docker`), plus a list of the slow and failed calls sharing the Timeline's `+elapsed` column. Written even when the run is cancelled or fails, which is when it matters most. - command-log paths - workflow DAG and dependency list - failure hints with task name, attempts, reason, and task log path when a task fails or is cancelled +The task event Timeline also carries an `UNGATED` event for members of a merged Kubernetes pod, marking the moment their in-pod gate opened and the task was allowed to start. + +Two companion files are written next to the summary, but only when there is something to say: + +- **`command_trace.jsonl`** — one record per *notable* external command (non-zero exit, or slower than 5s). A healthy run, or a run with no notable external-command calls, leaves no file. This is the machine-readable form of the `External Command Health` section; the pre-existing `*_cmds.log` records *what* was issued, this records *how it went*. +- **`loop_stalls.txt`** — if sflow's own event loop stops being scheduled for 30s, sflow logs a warning and dumps every thread's Python stack here, then logs a recovery line with the worst observed lag. Previously such a freeze produced no diagnostic output at all, because sflow's logging runs on the thread that was stuck. Created only when a stall actually happens. + Example `sflow_summary.log`: ```text diff --git a/docs/user/quick-reference.md b/docs/user/quick-reference.md index 6201fcc..fb6cc58 100644 --- a/docs/user/quick-reference.md +++ b/docs/user/quick-reference.md @@ -319,7 +319,7 @@ and `addressing_style` (`auto` / `virtual` / `path`). |-------|----------|------|---------|-------------| | `name` | Yes | string | — | Workflow name. | | `tasks` | Yes | list | — | List of task definitions (must be non-empty). | -| `timeout` | | string / int | `null` | Workflow-level timeout (e.g. `1h`, `115m`). | +| `timeout` | | string / int | `null` | **Not enforced** — accepted and merged, but nothing reads it; a workflow that sets it runs unbounded and `sflow run` warns. Use the backend's own limit (Slurm `--time`). | | `variables` | | dict / list | `null` | Workflow-scoped variables (same format as root `variables`). | | `upload_all` | | object | `null` | Zip the whole workflow output dir and upload it to a `storage` target (see [Workflow Upload-All](#workflow-upload-all)). | | `monitor` | | object | `null` | Workflow-level hardware monitor (see [Monitor](#monitor)). | @@ -336,7 +336,7 @@ and `addressing_style` (`auto` / `virtual` / `path`). | `backend` | | string / dict | `null` | Backend name, or inline backend override. | | `depends_on` | | list[string] | `null` | Names of tasks this task depends on. | | `required_by` | | list[string] | `null` | Reverse dependency: `A required_by: [B]` is folded into `B depends_on: [A]` at load (lets an optional fragment attach to a hub without editing it). | -| `timeout` | | int / string | `null` | Task-level timeout. | +| `timeout` | | int / string | `null` | **Not enforced** — see [Workflow](#workflow). Bound the task with the backend's own limit instead. | | `variables` | | dict / list | `null` | Task-scoped variables. | | `resources` | | object | `null` | Node / GPU resource requirements. | | `replicas` | | object | `null` | Replication configuration. | @@ -358,8 +358,9 @@ and `addressing_style` (`auto` / `virtual` / `path`). | `nodes.indices` | | list[int / expr] | `null` | Specific node indices (e.g. `[0]`). | | `nodes.count` | | int / expr | `null` | Number of nodes. | | `nodes.exclude` | | int / list[int] / expr | `null` | Node indices to remove from the placement pool before `indices`, `count`, or GPU packing. | -| `nodes.release_after` | | string | `workflow_completion` | When node reservations can be reused: `workflow_completion`, `task_ready`, or `task_completion`. sflow reserves nodes **only when this is set explicitly**; omitted, `nodes.indices`/`count` are non-exclusive placement constraints (see note below). | -| `gpus.count` | If `gpus` is set | int / expr | — | Number of GPUs (sets `CUDA_VISIBLE_DEVICES`). | +| `nodes.release_after` | | string | `null` | When node reservations can be reused: `workflow_completion`, `task_ready`, or `task_completion`. sflow reserves nodes **only when this is set explicitly**; omitted, `nodes.indices`/`count` are non-exclusive placement constraints (see note below). | +| `gpus.count` | One of `count` / `indices` | int / expr | `null` | Number of GPUs (sets `CUDA_VISIBLE_DEVICES`). Index-agnostic on its own: the planner packs the task into any contiguous idle GPU run on one node. | +| `gpus.indices` | One of `count` / `indices` | list[int / expr] | `null` | Pin the task to specific **0-based, non-negative, unique** device ids. Combined with `count`, `count` is the total across nodes and `indices` the per-node slice, so the task fans out over `count / len(indices)` nodes. | | `gpus.release_after` | | string | inferred | When GPU reservations can be reused: `workflow_completion`, `task_ready`, or `task_completion`. | For nodes, `release_after` only creates an exclusive node reservation when explicitly set; omitted `nodes.indices` and `nodes.count` are placement constraints and may overlap with other planned tasks. For GPUs, omitted `release_after` is inferred: tasks without readiness probes release GPUs after task completion for downstream dependents, while tasks with readiness probes keep GPUs until workflow completion unless explicitly set to `task_ready`. `task_ready` releases after readiness succeeds. `task_completion` releases after any terminal task status (`COMPLETED`, `FAILED`, `TIMEOUT`, or `CANCELLED`). Dry-run rehearses these resource lifetimes across the DAG. @@ -489,7 +490,9 @@ To publish a metric literally named `file`, use `patterns:`; a top-level `file:` | `interval` | | int | `5000` | Sampling interval (ms) for built-in scopes without their own. | | `scopes` | | object | `null` (all) | Which scopes to collect — `cpu`, `gpu`, `memory`, `disk`, `network`, `custom`; omit ⇒ all built-ins active. | | `resources` | | object | `null` | Which hardware to target — `nodes` / `gpus` (like task resources) or `used_by_tasks: [names]`. | -| `report` | | object | `null` (csv + svg) | Opt-in post-run report `format`: any of `csv`, `svg`, `png` (`png` needs the `sflow[monitor]` extra). | +| `report` | | object | `{enabled: true}` | Post-run report. `format`: any of `csv`, `svg`, `png` (`png` needs the `sflow[monitor]` extra). | +| `report.enabled` | | bool | `true` | **On by default** whenever `monitor:` is set. Set `false` to opt out — worth doing on large fan-outs, since report cost scales with samples × views. Raw samples and the overview are still written. | +| `window` | | object | `null` | `{start, end}` task-log markers bounding the reported window, so the report covers the benchmark rather than the whole task lifecycle. Task monitors only (rejected on `workflow.monitor`) and requires `report.enabled: true`. | ## Expression Syntax diff --git a/docs/user/resources.md b/docs/user/resources.md index 95a4adb..7be9332 100644 --- a/docs/user/resources.md +++ b/docs/user/resources.md @@ -185,6 +185,15 @@ split across two nodes. Write `gpus.count: 2` for one GPU on each of two nodes. This `CUDA_VISIBLE_DEVICES` packing applies to the **local**, **slurm**, and **docker** backends. Kubernetes assigns GPUs differently — see below. +:::note `gpus_per_task` hands device selection to Slurm +Setting `gpus_per_task` on an `srun` operator makes the job step request GRES, so Slurm +carves devices per **rank** and picks which ones. sflow does not re-apply its own slice +there — `resources.gpus` still sizes the request and drives packing and conflict +detection, but the physical devices are Slurm's choice, and run reporting falls back to +showing the planned slice. Leave `gpus_per_task` unset if you want sflow's exact device +pin (`resources.gpus.indices`) to hold. +::: + ### Pin specific GPUs with `indices` Use `resources.gpus.indices` when a task must land on particular device IDs — NUMA/NVLink affinity, reproducing a vendor benchmark topology, or steering around a known-bad device. diff --git a/docs/user/run-output.md b/docs/user/run-output.md index cd7e6ff..78fc958 100644 --- a/docs/user/run-output.md +++ b/docs/user/run-output.md @@ -48,6 +48,9 @@ annotated example. | Why did readiness never fire? | probe traces in `sflow_summary.log` | [Probes](./probes.md) | | How busy were the GPUs / CPU / network? | `sflow_monitor.log`, `sflow_monitor/` | [Monitor](./monitor.md) | | Which GPUs and nodes did each task get? | usage charts in `sflow_summary.log` | [Resources](./resources.md) | +| Which *physical* GPUs did this task really end up on? | `/sflow_gpus.log`, plus the `GPU Assignment` section of `sflow_summary.log` | [Resources](./resources.md) | +| Was the cluster control plane slow or flaky, rather than my job? | `External Command Health` in `sflow_summary.log`, `command_trace.jsonl` | [Outputs & logs](./outputs.md#execution-summary) | +| sflow itself went unresponsive — what was it doing? | `loop_stalls.txt` | [Outputs & logs](./outputs.md#execution-summary) | | How do I get all this off the cluster? | storage targets + `uploads:` | [Uploads](./uploads.md) | ## The output tree at a glance @@ -58,11 +61,14 @@ annotated example. ├── sflow.log # orchestration + command/status lines (no task stdout) ├── *_cmds.log # launch commands, grouped by family (bash/slurm/docker/ssh/python) ├── results.json # workflow-level metric index (only with `result:`) +├── command_trace.jsonl # slow/failed external commands (only when some call was notable) +├── loop_stalls.txt # all-thread stacks on a driver stall (only when one happened) ├── sflow_monitor.log # resource overview (only with `monitor:`) -├── sflow_monitor/ # raw samples + per-task reports (only with `monitor:`) +├── sflow_monitor/ # raw samples; per-task reports only when `report.enabled: true` (only with `monitor:`) └── / ├── .log # full per-task stdout/stderr ├── result.json # canonical per-task metrics (only with `result:`) + ├── sflow_gpus.log # physical GPU placement record (Slurm steps that pick their own devices) └── ... # anything your scripts write ``` @@ -100,8 +106,8 @@ downstream tasks and external tooling — see [Results](./results.md). ## Beyond the local folder -- [Monitor](./monitor.md) — hardware sampling and reports. Raw CSV samples are - always written when `monitor:` is set; the per-task charts and summaries need - `report: { enabled: true }`. +- [Monitor](./monitor.md) — hardware sampling and reports. Setting `monitor:` writes + raw CSV samples *and* the per-task charts and summaries; reports are on by default + (`report: { enabled: false }` opts out). - [Uploads](./uploads.md) — declare storage targets and per-task `uploads:` to ship files to S3 as each task completes, so partial results survive a cancelled run. diff --git a/docs/user/samples.md b/docs/user/samples.md index 70f3e27..c08cd9d 100644 --- a/docs/user/samples.md +++ b/docs/user/samples.md @@ -41,10 +41,14 @@ Two families ship with sflow: | docker | `self_contained/docker/hello_world` | Single container task via `docker_run` | | docker | `self_contained/docker/multi_node` | Multi-host Docker (`docker_host`/`context`) | | docker | `self_contained/docker/sglang_qwen3` | SGLang Qwen3 server + client in containers | +| docker | `self_contained/docker/gpu_monitor` | GPU monitor with a log-marker report `window:` — runs a stock CUDA `nbody` container as a deliberate idle/burst/idle square wave, so the lifecycle report and the windowed report visibly differ | | slurm | `self_contained/slurm/sglang_server_client` | Server + client on Slurm with readiness probes | | slurm | `self_contained/slurm/aiperf_template` | AIPerf benchmark template | | slurm | `self_contained/slurm/auto_replica` | Replica fan-out sized from a variable sweep | | slurm | `self_contained/slurm/resource_release_after` | `release_after` GPU/node lifetimes | +| slurm | `self_contained/slurm/gpu_indices` | `resources.gpus` — `count` vs `indices` vs both, with the 4-node × 4-GPU planner output annotated | +| slurm | `self_contained/slurm/gpu_placement_matrix` | GPU-placement regression matrix: container vs bare step, high slice offset, two tasks sharing a node, multi-node — proves placement by UUID rather than by GPU count | +| slurm | `self_contained/slurm/monitor_mixed` | The broadest single-job regression net: two Slurm pools/operators, replicas with cross-task refs, `release_after` GPU reuse, the placement matrix, and a monitor with a log-marker window. **If you only run one recipe on a new cluster, run this one.** | | slurm | `self_contained/slurm/multi_backend` | One workflow spanning multiple backends | | slurm | `self_contained/slurm/trtllm_serve_disagg` | TRT-LLM disaggregated serving | | slurm | `self_contained/slurm/infmax_v1_ds_r1` | InfMax DeepSeek-R1 benchmark | @@ -1404,7 +1408,7 @@ sflow run -f dynamo_trtllm_disagg.yaml \ Modular samples are folders containing multiple composable YAML files. Instead of one monolithic config, the workflow is split into reusable building blocks. -### inference_x_v2 +### Modular inference recipe (inference_x_v2) A modular inference benchmark setup supporting multiple frameworks (SGLang, vLLM, TensorRT-LLM) with disaggregated prefill/decode servers. diff --git a/examples/modular/inference_x_v2/benchmark_aiperf.yaml b/examples/modular/inference_x_v2/benchmark_aiperf.yaml index 247ca73..c2dd7f5 100644 --- a/examples/modular/inference_x_v2/benchmark_aiperf.yaml +++ b/examples/modular/inference_x_v2/benchmark_aiperf.yaml @@ -59,7 +59,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/modular/inference_x_v2/bulk_input.csv b/examples/modular/inference_x_v2/bulk_input.csv index 1f9962f..90037c8 100644 --- a/examples/modular/inference_x_v2/bulk_input.csv +++ b/examples/modular/inference_x_v2/bulk_input.csv @@ -2,9 +2,9 @@ sflow_config_file,SLURM_NODES,GPUS_PER_NODE,NUM_CTX_SERVERS,CTX_TP_SIZE,NUM_GEN_ slurm_config.yaml common_workflow.yaml trtllm/prefill.yaml trtllm/decode.yaml benchmark_infmax.yaml,1,4,1,2,1,2,,,1,[16],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2,agg_server slurm_config.yaml common_workflow.yaml trtllm/prefill.yaml trtllm/decode.yaml benchmark_infmax.yaml,2,4,4,1,1,4,,,2,[32],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2,agg_server slurm_config.yaml common_workflow.yaml sglang/prefill.yaml sglang/decode.yaml benchmark_infmax.yaml,1,4,2,1,1,2,,,1,"[1,4]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,lmsysorg/sglang:v0.5.8.post1-cu130,agg_server -slurm_config.yaml common_workflow.yaml sglang/prefill.yaml sglang/decode.yaml benchmark_infmax.yaml,2,4,2,2,1,4,,,2,"[2,4,6]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0,agg_server +slurm_config.yaml common_workflow.yaml sglang/prefill.yaml sglang/decode.yaml benchmark_infmax.yaml,2,4,2,2,1,4,,,2,"[2,4,6]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0,agg_server slurm_config.yaml common_workflow.yaml vllm/prefill.yaml vllm/decode.yaml benchmark_infmax.yaml,2,4,2,1,3,2,,,1,[8],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0,agg_server slurm_config.yaml common_workflow.yaml vllm/prefill.yaml vllm/decode.yaml benchmark_infmax.yaml,3,4,4,1,1,8,,,3,"[64,32]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0,agg_server slurm_config.yaml common_workflow.yaml trtllm/agg.yaml benchmark_aiperf.yaml,1,4,,,,,1,4,1,"[1,3,5]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2,prefill_server decode_server -slurm_config.yaml common_workflow.yaml sglang/agg.yaml benchmark_aiperf.yaml,2,4,,,,,1,8,2,[10],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0,prefill_server decode_server +slurm_config.yaml common_workflow.yaml sglang/agg.yaml benchmark_aiperf.yaml,2,4,,,,,8,1,2,[10],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0,prefill_server decode_server slurm_config.yaml common_workflow.yaml vllm/agg.yaml benchmark_aiperf.yaml,1,4,,,,,2,2,1,[128],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0,prefill_server decode_server \ No newline at end of file diff --git a/examples/modular/inference_x_v2/common_workflow.yaml b/examples/modular/inference_x_v2/common_workflow.yaml index bb51329..6cf5b79 100644 --- a/examples/modular/inference_x_v2/common_workflow.yaml +++ b/examples/modular/inference_x_v2/common_workflow.yaml @@ -54,7 +54,7 @@ variables: # Dependencies versions DYNAMO_VERSION: description: "Dynamo version to install when missing from the container image" - value: 0.8.0 + value: 1.3.0 NATS_VERSION: description: "NATS server version to install when missing" value: v2.10.28 diff --git a/examples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml b/examples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml index 2d18141..5c770c1 100644 --- a/examples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml +++ b/examples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml @@ -10,7 +10,7 @@ variables: - name: NUM_AGG_SERVERS description: Number of aggregated servers type: integer - value: 1 + value: 8 - name: CONCURRENCY description: Concurrency @@ -42,7 +42,7 @@ operators: container_writable: true mpi: pmix extra_args: - - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 + - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 - name: nginx type: srun @@ -101,8 +101,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 + echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 else echo "dynamo already installed." fi @@ -324,17 +324,17 @@ workflow: - export NODE_RANK=${SLURM_NODEID} - export FIRST_NODE_IP=$(echo ${SFLOW_TASK_ASSIGNED_NODE_IPS} | cut -d',' -f1) - | - if [ "2" -gt 1 ]; then - export MULTI_NODE_EXTRA_ARGS="--dist-init-addr ${FIRST_NODE_IP}:29500 --nnodes 2 --node-rank ${NODE_RANK}" + if [ "1" -gt 1 ]; then + export MULTI_NODE_EXTRA_ARGS="--dist-init-addr ${FIRST_NODE_IP}:29500 --nnodes 1 --node-rank ${NODE_RANK}" else export MULTI_NODE_EXTRA_ARGS="" fi - export VLLM_NIXL_SIDE_CHANNEL_PORT=$((5557 + ${FIRST_CUDA_DEVICE})) - export DYN_SYSTEM_PORT=$((8082 + ${FIRST_CUDA_DEVICE})) - - python3 -m dynamo.sglang --model-path ${{ artifacts.LOCAL_MODEL_PATH.path }} --served-model-name Qwen3-8B-FP8 --tensor-parallel-size 8 --pipeline-parallel-size 1 --data-parallel-size 1 --expert-parallel-size 1 --moe-dense-tp-size 1 --max-running-requests 128 --max-prefill-tokens 4096 --trust-remote-code --skip-tokenizer-init --cuda-graph-max-bs 128 ${MULTI_NODE_EXTRA_ARGS} --host 0.0.0.0 + - python3 -m dynamo.sglang --model-path ${{ artifacts.LOCAL_MODEL_PATH.path }} --served-model-name Qwen3-8B-FP8 --tensor-parallel-size 1 --pipeline-parallel-size 1 --data-parallel-size 1 --expert-parallel-size 1 --moe-dense-tp-size 1 --max-running-requests 128 --max-prefill-tokens 4096 --trust-remote-code --skip-tokenizer-init --cuda-graph-max-bs 128 ${MULTI_NODE_EXTRA_ARGS} --host 0.0.0.0 resources: gpus: - count: 8 + count: 1 depends_on: - frontend_server probes: @@ -355,7 +355,7 @@ workflow: - set -x - pip install aiperf==0.3.0 - export COLUMNS=200 - - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' + - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' - echo "Benchmarking finished" resources: nodes: diff --git a/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml b/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml index 6d0b599..61466d5 100644 --- a/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml +++ b/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml @@ -101,8 +101,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 + echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 else echo "dynamo already installed." fi diff --git a/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml b/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml index 348a61b..d5ef927 100644 --- a/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml +++ b/examples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml @@ -49,7 +49,7 @@ operators: container_writable: true mpi: pmix extra_args: - - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 + - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 - name: nginx type: srun @@ -102,8 +102,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 + echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 else echo "dynamo already installed." fi diff --git a/examples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml b/examples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml index 610c420..5c4ff88 100644 --- a/examples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml +++ b/examples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml @@ -123,8 +123,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 + echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 else echo "dynamo already installed." fi @@ -378,7 +378,7 @@ workflow: - set -x - pip install aiperf==0.3.0 - export COLUMNS=200 - - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' + - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' - echo "Benchmarking finished" resources: nodes: diff --git a/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml b/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml index bce8623..e8b8fea 100644 --- a/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml +++ b/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml @@ -144,8 +144,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 + echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 else echo "dynamo already installed." fi @@ -390,10 +390,6 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 - name: decode_server operator: @@ -432,10 +428,6 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 - name: benchmark_infmax operator: diff --git a/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml b/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml index 4117bc9..6d054b7 100644 --- a/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml +++ b/examples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml @@ -144,8 +144,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 + echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 else echo "dynamo already installed." fi @@ -390,10 +390,6 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 - name: decode_server operator: @@ -432,10 +428,6 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 - name: benchmark_infmax operator: diff --git a/examples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml b/examples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml index 4be77a1..4319237 100644 --- a/examples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml +++ b/examples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml @@ -393,7 +393,7 @@ workflow: - set -x - pip install aiperf==0.3.0 - export COLUMNS=200 - - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' + - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' - echo "Benchmarking finished" resources: nodes: diff --git a/examples/modular/inference_x_v2/sglang/agg.yaml b/examples/modular/inference_x_v2/sglang/agg.yaml index 06ddc8f..36efd83 100644 --- a/examples/modular/inference_x_v2/sglang/agg.yaml +++ b/examples/modular/inference_x_v2/sglang/agg.yaml @@ -1,7 +1,7 @@ # SGLang aggregated (non-disaggregated) server task and its configuration. # Merge all files for a complete run: # sflow compose slurm_config.yaml common_workflow.yaml sglang/agg.yaml benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 \ +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 \ # --missable-tasks prefill_server --missable-tasks decode_server version: "0.1" diff --git a/examples/modular/inference_x_v2/sglang/decode.yaml b/examples/modular/inference_x_v2/sglang/decode.yaml index 3cc22e6..4ee8ffe 100644 --- a/examples/modular/inference_x_v2/sglang/decode.yaml +++ b/examples/modular/inference_x_v2/sglang/decode.yaml @@ -1,7 +1,7 @@ # SGLang decode server task and its configuration. # Merge all files for a complete run: # sflow run -f slurm_config.yaml -f common_workflow.yaml -f sglang_prefill.yaml -f sglang_decode.yaml -f benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 version: "0.1" diff --git a/examples/modular/inference_x_v2/sglang/prefill.yaml b/examples/modular/inference_x_v2/sglang/prefill.yaml index 62462e7..828434d 100644 --- a/examples/modular/inference_x_v2/sglang/prefill.yaml +++ b/examples/modular/inference_x_v2/sglang/prefill.yaml @@ -1,7 +1,7 @@ # SGLang prefill server task and its configuration. # Merge all files for a complete run: # sflow run -f slurm_config.yaml -f common_workflow.yaml -f sglang_prefill.yaml -f sglang_decode.yaml -f benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 version: "0.1" diff --git a/examples/modular/inference_x_v2/trtllm/agg.yaml b/examples/modular/inference_x_v2/trtllm/agg.yaml index b79ad63..198f54e 100644 --- a/examples/modular/inference_x_v2/trtllm/agg.yaml +++ b/examples/modular/inference_x_v2/trtllm/agg.yaml @@ -1,7 +1,7 @@ # TensorRT-LLM aggregated (non-disaggregated) server task and its configuration. # Merge all files for a complete run: # sflow compose slurm_config.yaml common_workflow.yaml trtllm/agg.yaml benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 \ +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 \ # --missable-tasks prefill_server --missable-tasks decode_server version: "0.1" diff --git a/examples/modular/inference_x_v2/trtllm/decode.yaml b/examples/modular/inference_x_v2/trtllm/decode.yaml index 049e1bf..04e72dd 100644 --- a/examples/modular/inference_x_v2/trtllm/decode.yaml +++ b/examples/modular/inference_x_v2/trtllm/decode.yaml @@ -126,7 +126,3 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 diff --git a/examples/modular/inference_x_v2/trtllm/prefill.yaml b/examples/modular/inference_x_v2/trtllm/prefill.yaml index 74e62fe..bc9bd33 100644 --- a/examples/modular/inference_x_v2/trtllm/prefill.yaml +++ b/examples/modular/inference_x_v2/trtllm/prefill.yaml @@ -124,7 +124,3 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 diff --git a/examples/self_contained/kubernetes/dynamo_sglang_agg.yaml b/examples/self_contained/kubernetes/dynamo_sglang_agg.yaml index ef782fd..2c2044c 100644 --- a/examples/self_contained/kubernetes/dynamo_sglang_agg.yaml +++ b/examples/self_contained/kubernetes/dynamo_sglang_agg.yaml @@ -433,7 +433,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/kubernetes/dynamo_sglang_disagg.yaml b/examples/self_contained/kubernetes/dynamo_sglang_disagg.yaml index 997852c..40d6b4d 100644 --- a/examples/self_contained/kubernetes/dynamo_sglang_disagg.yaml +++ b/examples/self_contained/kubernetes/dynamo_sglang_disagg.yaml @@ -695,7 +695,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml b/examples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml index 2b3c995..f925aea 100644 --- a/examples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml +++ b/examples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml @@ -719,7 +719,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/kubernetes/dynamo_trtllm_agg.yaml b/examples/self_contained/kubernetes/dynamo_trtllm_agg.yaml index 85c7416..182117d 100644 --- a/examples/self_contained/kubernetes/dynamo_trtllm_agg.yaml +++ b/examples/self_contained/kubernetes/dynamo_trtllm_agg.yaml @@ -383,7 +383,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml b/examples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml index 4f5fccc..54cbdc7 100644 --- a/examples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml +++ b/examples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml @@ -523,7 +523,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/kubernetes/dynamo_vllm_agg.yaml b/examples/self_contained/kubernetes/dynamo_vllm_agg.yaml index a4ed1a0..520fd02 100644 --- a/examples/self_contained/kubernetes/dynamo_vllm_agg.yaml +++ b/examples/self_contained/kubernetes/dynamo_vllm_agg.yaml @@ -468,7 +468,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/kubernetes/dynamo_vllm_disagg.yaml b/examples/self_contained/kubernetes/dynamo_vllm_disagg.yaml index 02d96d2..d363258 100644 --- a/examples/self_contained/kubernetes/dynamo_vllm_disagg.yaml +++ b/examples/self_contained/kubernetes/dynamo_vllm_disagg.yaml @@ -668,7 +668,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/aiperf_template.yaml b/examples/self_contained/slurm/aiperf_template.yaml index 92087bc..13828ff 100644 --- a/examples/self_contained/slurm/aiperf_template.yaml +++ b/examples/self_contained/slurm/aiperf_template.yaml @@ -109,7 +109,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/dynamo_sglang_agg.yaml b/examples/self_contained/slurm/dynamo_sglang_agg.yaml index 5800360..fab64dd 100644 --- a/examples/self_contained/slurm/dynamo_sglang_agg.yaml +++ b/examples/self_contained/slurm/dynamo_sglang_agg.yaml @@ -115,7 +115,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 + value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -307,7 +307,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/dynamo_sglang_disagg.yaml b/examples/self_contained/slurm/dynamo_sglang_disagg.yaml index e551f75..ca16e44 100644 --- a/examples/self_contained/slurm/dynamo_sglang_disagg.yaml +++ b/examples/self_contained/slurm/dynamo_sglang_disagg.yaml @@ -170,7 +170,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 + value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -424,7 +424,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/dynamo_trtllm_agg.yaml b/examples/self_contained/slurm/dynamo_trtllm_agg.yaml index 771b71b..000c9c6 100644 --- a/examples/self_contained/slurm/dynamo_trtllm_agg.yaml +++ b/examples/self_contained/slurm/dynamo_trtllm_agg.yaml @@ -111,7 +111,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 + value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -321,7 +321,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/dynamo_trtllm_disagg.yaml b/examples/self_contained/slurm/dynamo_trtllm_disagg.yaml index 1db803e..3e539a8 100644 --- a/examples/self_contained/slurm/dynamo_trtllm_disagg.yaml +++ b/examples/self_contained/slurm/dynamo_trtllm_disagg.yaml @@ -154,7 +154,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 + value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -372,10 +372,6 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 - name: decode_server operator: @@ -419,10 +415,6 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 - name: benchmark operator: @@ -447,7 +439,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/dynamo_vllm_agg.yaml b/examples/self_contained/slurm/dynamo_vllm_agg.yaml index fafb786..56f337e 100644 --- a/examples/self_contained/slurm/dynamo_vllm_agg.yaml +++ b/examples/self_contained/slurm/dynamo_vllm_agg.yaml @@ -337,7 +337,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/dynamo_vllm_disagg.yaml b/examples/self_contained/slurm/dynamo_vllm_disagg.yaml index 3f6c8ce..4eeadcf 100644 --- a/examples/self_contained/slurm/dynamo_vllm_disagg.yaml +++ b/examples/self_contained/slurm/dynamo_vllm_disagg.yaml @@ -304,6 +304,16 @@ workflow: - set -x - echo ${CUDA_VISIBLE_DEVICES} - export FIRST_CUDA_DEVICE=$(echo ${CUDA_VISIBLE_DEVICES} | cut -d',' -f1) + # Port derived from the first visible device, so two workers sharing a + # node never collide. This works because sflow does NOT hand + # NVIDIA_VISIBLE_DEVICES to an srun step: the container therefore sees the + # node's GPUs with HOST numbering intact, and the first entry of + # CUDA_VISIBLE_DEVICES is a real host index -- unique per worker. + # + # Keep it this way deliberately. When NVIDIA_VISIBLE_DEVICES was exported, + # the runtime carved and renumbered every worker's devices from 0, so both + # prefill and decode computed 5557 and the second to bind died with + # "Address already in use". This line is the canary that found it. - export VLLM_NIXL_SIDE_CHANNEL_PORT=$((5557 + ${FIRST_CUDA_DEVICE})) - export NODE_RANK=${SLURM_NODEID} - export FIRST_NODE_IP=$(echo ${SFLOW_TASK_ASSIGNED_NODE_IPS} | cut -d',' -f1) @@ -381,6 +391,16 @@ workflow: - set -x - echo ${CUDA_VISIBLE_DEVICES} - export FIRST_CUDA_DEVICE=$(echo ${CUDA_VISIBLE_DEVICES} | cut -d',' -f1) + # Port derived from the first visible device, so two workers sharing a + # node never collide. This works because sflow does NOT hand + # NVIDIA_VISIBLE_DEVICES to an srun step: the container therefore sees the + # node's GPUs with HOST numbering intact, and the first entry of + # CUDA_VISIBLE_DEVICES is a real host index -- unique per worker. + # + # Keep it this way deliberately. When NVIDIA_VISIBLE_DEVICES was exported, + # the runtime carved and renumbered every worker's devices from 0, so both + # prefill and decode computed 5557 and the second to bind died with + # "Address already in use". This line is the canary that found it. - export VLLM_NIXL_SIDE_CHANNEL_PORT=$((5557 + ${FIRST_CUDA_DEVICE})) - export NODE_RANK=${SLURM_NODEID} - export FIRST_NODE_IP=$(echo ${SFLOW_TASK_ASSIGNED_NODE_IPS} | cut -d',' -f1) @@ -469,7 +489,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/gpu_placement_matrix.yaml b/examples/self_contained/slurm/gpu_placement_matrix.yaml new file mode 100644 index 0000000..0ea0493 --- /dev/null +++ b/examples/self_contained/slurm/gpu_placement_matrix.yaml @@ -0,0 +1,366 @@ +# GPU placement regression matrix: every combination that can break the slice. +# +# sflow plans a per-task CUDA_VISIBLE_DEVICES and re-applies it inside the step. +# Four things independently decide what a task ends up holding: +# +# * container or not -- pyxis/enroot pass through only the task's devices and +# RENUMBER them from 0, so a task planned for 2,3 sees +# them as 0,1. A bare srun step sees the whole node. +# * slice offset -- a plan starting at 0 survives renumbering by accident, +# which is why only the high-offset task ever broke. +# * concurrency -- two tasks sharing a node must land on disjoint GPUs. +# * node count -- a multi-node plan is ONE node's slice, applied +# independently on every node. sflow guarantees the slice +# is uniform (gpus.count is divided by the node count and +# every node takes the same offsets), so the same flat +# plan must resolve correctly in each node's own step. +# +# Counting GPUs is not enough: a task can hold the right NUMBER of the WRONG +# devices and look perfectly healthy. So the first task records EACH node's +# index -> UUID map on bare metal, before anything is carved, and every later task +# proves by UUID that it holds the physical GPUs it was planned for -- per node, +# because a multi-node task's nodes each have their own map. +# +# Asserting rather than echoing is the point. The recipes that only printed their +# devices passed while holding nothing at all -- a decode server planned for 2,3 +# inside a 2-GPU container numbered 0,1 saw no GPU, and the first CUDA call died +# reporting an unsupported architecture. +version: "0.1" + +variables: + SLURM_ACCOUNT: + description: "SLURM account" + value: your_slurm_account + SLURM_PARTITION: + description: "SLURM partition" + value: your_slurm_partition + SLURM_TIMELIMIT: + description: "SLURM time limit" + value: "00:10:00" + SLURM_NODES: + description: >- + Number of Slurm nodes. Two, so the multi-node cases are real: a per-node + slice that is right on node 0 and wrong on node 1 is invisible at one node, + and that is the shape every multi-node server recipe uses. + # Typed, because boxed_multinode_high multiplies it. Untyped it is a STRING, + # and `${{ variables.SLURM_NODES * 2 }}` repeats it into "22" rather than + # doubling it -- valid YAML, valid expression, wrong number. + type: integer + value: 2 + GPUS_PER_NODE: + description: "GPUs per node" + value: 4 + PLACEMENT_IMAGE: + description: "Any GPU-enabled image; nvidia-smi is injected by the runtime." + value: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda12.5.0 + +artifacts: + # Declared so sample/bulk tests can override the standard sample artifact. + - name: LOCAL_MODEL_PATH + uri: fs:///path/to/your/model + + # One assertion, mounted at the same path on every node, instead of the same + # forty lines of shell pasted into each task. EXPECT_HOST is the only input. + - name: ASSERT_PLACEMENT + uri: file://assert_placement.sh + content: | + # Prove by UUID that THIS rank holds the physical GPUs it was planned for. + # Reads EXPECT_HOST (planned HOST indices, comma separated) from the env. + set -u + me=$(basename "$SFLOW_TASK_OUTPUT_DIR") + run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") + # Per NODE, not per task: every rank of a multi-node task runs this body, and + # each node has its own index -> UUID map. Reading node 0's map on node 1 + # would compare against the wrong hardware and "pass" for the wrong reason. + node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" + map=$(ls "$run_dir"/node_gpu_map*/map."$node".txt 2>/dev/null | head -1) + + echo "SFLOW_GPU_PROBE task=$me node=$node planned_host=$EXPECT_HOST CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" + nvidia-smi -L | sed "s/^/SFLOW_GPU_PROBE $node visible: /" + + if [ -z "$map" ]; then + echo "FAIL: no index -> UUID map recorded for node $node" >&2 + exit 1 + fi + # Are the planned ordinals even addressable in THIS namespace? A carved + # container renumbers from 0, so host ordinals can name nothing at all. + # An UNSET CUDA_VISIBLE_DEVICES is legal and means "every visible device", + # so `-i` must be omitted rather than passed empty -- and it must not be + # dereferenced bare under `set -u`. + if ! got=$(nvidia-smi ${CUDA_VISIBLE_DEVICES:+-i "$CUDA_VISIBLE_DEVICES"} \ + --query-gpu=uuid --format=csv,noheader); then + echo "FAIL: CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-} is not addressable on $node" >&2 + exit 1 + fi + # Per-node filenames: ranks of a multi-node task share one output dir, so a + # single gpus.txt would have them overwriting each other's evidence. + printf '%s\n' "$got" | sort > "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" + # And are they the RIGHT ones? Resolve the planned HOST indices through this + # node's bare-metal map; holding the correct count of the wrong devices is + # the failure mode a count-only check cannot see. + for i in $(echo "$EXPECT_HOST" | tr ',' ' '); do + grep "^$i," "$map" | cut -d, -f2 + done | sort > "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" + if ! diff -q "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" \ + "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" >/dev/null; then + echo "FAIL: $node planned host GPU(s) $EXPECT_HOST, but holds different devices" >&2 + echo " want: $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt")" >&2 + echo " got : $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt")" >&2 + exit 1 + fi + echo "OK: $node holds host GPU(s) $EXPECT_HOST, confirmed by UUID" + +backends: + - name: slurm_cluster + type: slurm + default: true + account: ${{ variables.SLURM_ACCOUNT }} + partition: ${{ variables.SLURM_PARTITION }} + time: ${{ variables.SLURM_TIMELIMIT }} + nodes: ${{ variables.SLURM_NODES }} + gpus_per_node: ${{ variables.GPUS_PER_NODE }} + +operators: + # One rank per node throughout: the assertion describes what a NODE holds, and + # extra ranks would each re-run it against the same devices for no new signal. + - name: bare + type: srun + ntasks_per_node: 1 + - name: boxed + type: srun + container_image: ${{ variables.PLACEMENT_IMAGE }} + ntasks_per_node: 1 + +workflow: + name: gpu_placement_matrix + tasks: + # Ground truth, taken bare metal with no `resources.gpus` so nothing is carved + # and no placement prelude runs. Runs on EVERY node (no `resources.nodes`), and + # each rank writes its own file, so the multi-node assertions below have a map + # for the node they actually landed on. Every assertion is relative to this. + - name: node_gpu_map + operator: bare + script: + - env | grep SLURM + - | + set -u + node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" + nvidia-smi --query-gpu=index,uuid --format=csv,noheader | tr -d ' ' \ + > "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" + sed "s/^/SFLOW_GPU_PROBE node_map $node: /" "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" + + # --- bare srun: the step sees the whole node, so the planned ordinals must + # --- survive untouched. Both run at once, on disjoint halves. + - name: bare_low + operator: bare + depends_on: [node_gpu_map] + resources: + gpus: + indices: [0, 1] + script: + - export EXPECT_HOST="0,1" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + - name: bare_high + operator: bare + depends_on: [node_gpu_map] + resources: + gpus: + indices: [2, 3] + script: + - export EXPECT_HOST="2,3" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # --- containerised: the runtime already carved and renumbered, so re-applying + # --- host ordinals here is what broke. boxed_high is the regression case. + - name: boxed_low + operator: boxed + depends_on: [bare_low, bare_high] + resources: + gpus: + indices: [0, 1] + script: + - export EXPECT_HOST="0,1" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + - name: boxed_high + operator: boxed + depends_on: [bare_low, bare_high] + resources: + gpus: + indices: [2, 3] + script: + - export EXPECT_HOST="2,3" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # A single GPU at a high offset: renumbering turns 3 into 0, so an off-by-one + # in the remap shows up here and nowhere else. + - name: boxed_one_high + operator: boxed + depends_on: [boxed_low, boxed_high] + resources: + gpus: + indices: [3] + script: + - export EXPECT_HOST="3" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # --- MULTI-NODE. `gpus.count` is the TOTAL over the task's nodes and every + # --- node takes the SAME offsets, so count/nodes must divide evenly. One flat + # --- plan therefore has to resolve correctly in each node's own step -- and + # --- each node proves it against its OWN bare-metal map, so a slice that is + # --- right on node 0 and wrong on node 1 fails here instead of shipping. + - name: bare_multinode + operator: bare + depends_on: [boxed_one_high] + resources: + # `nodes.count` is what forces the spread: `gpus.count` alone would fit on + # one node and the planner would place it there. count/nodes = 1 GPU per + # node, so the plan is "0" and both nodes must resolve it against their own + # hardware. This is the divide-the-total path every server recipe uses. + nodes: + count: ${{ variables.SLURM_NODES }} + gpus: + count: ${{ variables.SLURM_NODES }} + script: + - export EXPECT_HOST="0" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # The multi-node case that matters: containerised AND at a high offset, so + # every renumbering hazard is in play on more than one node at once. Pinned + # indices repeat on each node, which is the other multi-node planner path. + - name: boxed_multinode_high + operator: boxed + depends_on: [bare_multinode] + resources: + gpus: + # A TOTAL instead of a topology. With no `nodes:` here, `indices` is + # the per-node slice and the planner spreads the task over + # count / len(indices) nodes -- 4 / 2 = 2. This is the one route where + # the node count is an OUTPUT of the GPU ask rather than an input, which + # is how a recipe states things when it knows how many GPUs it needs but + # not how they are laid out. + count: ${{ variables.SLURM_NODES * 2 }} # 2 = len(indices) below + indices: [2, 3] + script: + - export EXPECT_HOST="2,3" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # --- AUTO-PLANNED (`gpus.count`, no `indices`). Everything above pins the + # --- devices by hand, so it proves the RE-APPLY path but never the PLANNER. + # --- These let sflow choose, which is what every real recipe does: the task + # --- cannot state an expected slice up front, so it asserts against the plan + # --- the prelude exported ($SFLOW_GPU_PLAN) -- still resolved to UUIDs + # --- through this node's bare-metal map, so "the planner picked something" + # --- and "the step actually holds it" remain separate claims. + # --- + # --- Chained rather than concurrent: 2x2 + 4x1 + 8 would need 16 GPUs and the + # --- allocation has 8. + + # Two replicas of two GPUs: the packing case. The planner must hand replica 0 + # and replica 1 DISJOINT pairs on the same node, which is the collision the + # whole slice mechanism exists to prevent. + - name: bare_count_2x2 + operator: bare + depends_on: [boxed_multinode_high] + replicas: + count: 2 + policy: parallel + resources: + gpus: + count: 2 + script: + - export EXPECT_HOST="$SFLOW_GPU_PLAN" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + - name: boxed_count_2x2 + operator: boxed + depends_on: [bare_count_2x2] + replicas: + count: 2 + policy: parallel + resources: + gpus: + count: 2 + script: + - export EXPECT_HOST="$SFLOW_GPU_PLAN" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # One GPU each, four ways: the finest packing, and the shape most likely to + # hand two replicas the same device if the planner ever miscounts. + - name: bare_count_1x4 + operator: bare + depends_on: [boxed_count_2x2] + replicas: + count: 4 + policy: parallel + resources: + gpus: + count: 1 + script: + - export EXPECT_HOST="$SFLOW_GPU_PLAN" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + - name: boxed_count_1x4 + operator: boxed + depends_on: [bare_count_1x4] + replicas: + count: 4 + policy: parallel + resources: + gpus: + count: 1 + script: + - export EXPECT_HOST="$SFLOW_GPU_PLAN" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # More GPUs than one node has, with no `nodes:` hint: the planner has to + # spread it itself and divide the total evenly, so every node runs the same + # flat plan against different physical cards. + - name: bare_count_8 + operator: bare + depends_on: [boxed_count_1x4] + resources: + gpus: + count: 8 + script: + - export EXPECT_HOST="$SFLOW_GPU_PLAN" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + - name: boxed_count_8 + operator: boxed + depends_on: [bare_count_8] + resources: + gpus: + count: 8 + script: + - export EXPECT_HOST="$SFLOW_GPU_PLAN" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # Concurrency: boxed_low and boxed_high held the node at the same time, so + # their devices must be disjoint. Sharing one is the collision this placement + # logic exists to prevent, and it is invisible from inside either task. + - name: verify_disjoint + operator: bare + # Terminal on purpose: the e2e checker treats this task's OK line as proof + # that the WHOLE matrix ran, so it has to sit downstream of the last case. + depends_on: [boxed_count_8] + resources: + nodes: + indices: [0] + script: + - | + set -u + run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") + low=$(ls "$run_dir"/boxed_low*/gpus.*.txt 2>/dev/null | head -1) + high=$(ls "$run_dir"/boxed_high*/gpus.*.txt 2>/dev/null | head -1) + if [ -z "$low" ] || [ -z "$high" ]; then + echo "FAIL: concurrent tasks left no gpus.*.txt to compare" >&2 + exit 1 + fi + shared=$(sort "$low" "$high" | uniq -d) + if [ -n "$shared" ]; then + echo "FAIL: concurrent tasks shared GPU(s): $shared" >&2 + exit 1 + fi + echo "OK: concurrent tasks held disjoint GPUs" diff --git a/examples/self_contained/slurm/infmax_v1_ds_r1.yaml b/examples/self_contained/slurm/infmax_v1_ds_r1.yaml index c235712..c5c8dd5 100644 --- a/examples/self_contained/slurm/infmax_v1_ds_r1.yaml +++ b/examples/self_contained/slurm/infmax_v1_ds_r1.yaml @@ -140,7 +140,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 + value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 artifacts: # file type artifacts will be generated by sflow with the content defined in this yaml file, accessible as `${{ artifacts.NAME.path }}` in task scripts @@ -420,7 +420,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/monitor_mixed.yaml b/examples/self_contained/slurm/monitor_mixed.yaml index 5e4b000..d16bcf0 100644 --- a/examples/self_contained/slurm/monitor_mixed.yaml +++ b/examples/self_contained/slurm/monitor_mixed.yaml @@ -2,13 +2,24 @@ version: "0.1" # All-in-one Slurm sample: one `sflow run` exercises several features at once, so # a single job on a real cluster verifies them together instead of one at a time. +# This is the broadest regression net in examples/ -- if only one recipe gets run +# on a new cluster, run this one. # -# Mixes three existing samples and adds monitoring: +# Mixes four existing samples and adds monitoring: # multi_backend.yaml -> two Slurm pools, two operators/images # auto_replica.yaml -> replicas + cross-task `${{ task.x[i]... }}` access # resource_release_after.yaml-> readiness probes + GPU reuse (release_after) +# gpu_placement_matrix.yaml -> GPU placement PROVEN by UUID: container vs bare, +# slice offset, concurrency, and multi-node # NEW -> monitor with a log-marker report window # +# The placement stage runs first and releases its GPUs before the servers start, so +# it adds assertions without adding nodes. It is the only part here that can catch a +# task holding the right NUMBER of the WRONG GPUs -- everything below merely echoes +# CUDA_VISIBLE_DEVICES, which is how a decode server planned for 2,3 passed while +# sitting in a 2-GPU container numbered 0,1 with no usable device at all. +# gpu_placement_matrix.yaml keeps the fuller matrix as a standalone 2-node recipe. +# # What to check after a run (//sflow_monitor/): # # MARKER-CLIPPED -- sflow_monitor/windowed/ (each with window.json): @@ -96,6 +107,54 @@ artifacts: - name: LOCAL_MODEL_PATH uri: fs:///path/to/your/model + # Placement proof, mounted at the same path on every node. Kept in a file:// + # artifact rather than pasted into each task: EXPECT_HOST is the only input, so + # one copy serves every case below. + - name: ASSERT_PLACEMENT + uri: file://assert_placement.sh + content: | + # Prove by UUID that THIS rank holds the physical GPUs it was planned for. + # Reads EXPECT_HOST (planned HOST indices, comma separated) from the env. + set -u + me=$(basename "$SFLOW_TASK_OUTPUT_DIR") + run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") + # Per NODE, not per task: every rank of a multi-node task runs this body, and + # each node has its own index -> UUID map. Reading node 0's map on node 1 + # would compare against the wrong hardware and "pass" for the wrong reason. + node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" + map=$(ls "$run_dir"/node_gpu_map*/map."$node".txt 2>/dev/null | head -1) + + echo "SFLOW_GPU_PROBE task=$me node=$node planned_host=$EXPECT_HOST CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" + nvidia-smi -L | sed "s/^/SFLOW_GPU_PROBE $node visible: /" + + if [ -z "$map" ]; then + echo "FAIL: no index -> UUID map recorded for node $node" >&2 + exit 1 + fi + # Are the planned ordinals even addressable in THIS namespace? A carved + # container renumbers from 0, so host ordinals can name nothing at all. + if ! got=$(nvidia-smi -i "$CUDA_VISIBLE_DEVICES" --query-gpu=uuid --format=csv,noheader); then + echo "FAIL: CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES is not addressable on $node" >&2 + exit 1 + fi + # Per-node filenames: ranks of a multi-node task share one output dir, so a + # single gpus.txt would have them overwriting each other's evidence. + printf '%s\n' "$got" | sort > "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" + # And are they the RIGHT ones? Resolve the planned HOST indices through this + # node's bare-metal map; holding the correct count of the wrong devices is + # the failure mode a count-only check cannot see. + for i in $(echo "$EXPECT_HOST" | tr ',' ' '); do + grep "^$i," "$map" | cut -d, -f2 + done | sort > "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" + if ! diff -q "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" \ + "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" >/dev/null; then + echo "FAIL: $node planned host GPU(s) $EXPECT_HOST, but holds different devices" >&2 + echo " want: $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt")" >&2 + echo " got : $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt")" >&2 + exit 1 + fi + echo "OK: $node holds host GPU(s) $EXPECT_HOST, confirmed by UUID" + # Two independent pools: each allocates its own Slurm job and its own nodes. backends: - name: gpu_pool @@ -123,6 +182,11 @@ operators: type: srun container_image: ${{ variables.CPU_IMAGE }} ntasks_per_node: 1 + # No container, so the placement ground truth is taken on the raw node before any + # runtime has renumbered anything. Reuses the host driver's nvidia-smi. + - name: worker_bare + type: srun + ntasks_per_node: 1 workflow: name: monitor_mixed_slurm @@ -135,11 +199,112 @@ workflow: enabled: true tasks: + # --------------------------------------------------------------------------- + # -- gpu_placement_matrix: PROVE the planned slice, by UUID, before anything + # -- else runs. Counting GPUs is not enough -- a task can hold the right NUMBER + # -- of the WRONG devices and look perfectly healthy, which is how a decode + # -- server planned for 2,3 ran in a 2-GPU container numbered 0,1 and died on + # -- its first CUDA call. The servers below only ECHO their devices, so they + # -- cannot catch that; these tasks can. + # -- + # -- This whole stage completes (and releases its GPUs) before env_check, so it + # -- costs no extra nodes and cannot collide with the servers' reservations. + # --------------------------------------------------------------------------- + + # Ground truth, bare metal, nothing carved. Runs on EVERY node (no + # `resources.nodes`), one file each, so the multi-node case below can check the + # node it actually landed on. + - name: node_gpu_map + operator: worker_bare + script: + - | + set -u + node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" + nvidia-smi --query-gpu=index,uuid --format=csv,noheader | tr -d ' ' \ + > "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" + sed "s/^/SFLOW_GPU_PROBE node_map $node: /" "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" + + # Bare srun sees the whole node, so a non-zero offset must survive untouched. + - name: placement_bare_high + operator: worker_bare + depends_on: [node_gpu_map] + resources: + gpus: + indices: [2, 3] + script: + - export EXPECT_HOST="2,3" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # Containerised, low and high offset, CONCURRENTLY on one node. A plan starting + # at 0 survives container renumbering by accident, which is why only the + # high-offset task ever broke -- both sides are needed to see it. + - name: placement_boxed_low + operator: worker_gpu + depends_on: [placement_bare_high] + resources: + gpus: + indices: [0, 1] + script: + - export EXPECT_HOST="0,1" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + - name: placement_boxed_high + operator: worker_gpu + depends_on: [placement_bare_high] + resources: + gpus: + indices: [2, 3] + script: + - export EXPECT_HOST="2,3" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # MULTI-NODE, containerised, high offset -- every renumbering hazard in play on + # more than one node at once. A multi-node plan is ONE node's slice applied + # independently per node, so a slice that is right on node 0 and wrong on node 1 + # is invisible at a single node. Pinned indices repeat on each node. + - name: placement_multinode_high + operator: worker_gpu + depends_on: [placement_boxed_low, placement_boxed_high] + resources: + nodes: + count: 2 + gpus: + indices: [2, 3] + script: + - export EXPECT_HOST="2,3" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # Collisions are invisible from inside either task: boxed_low and boxed_high + # held one node at the same time, so their devices must be disjoint. + - name: placement_disjoint + operator: worker_bare + depends_on: [placement_multinode_high] + resources: + nodes: + indices: [0] + script: + - | + set -u + run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") + low=$(ls "$run_dir"/placement_boxed_low*/gpus.*.txt 2>/dev/null | head -1) + high=$(ls "$run_dir"/placement_boxed_high*/gpus.*.txt 2>/dev/null | head -1) + if [ -z "$low" ] || [ -z "$high" ]; then + echo "FAIL: concurrent tasks left no gpus.*.txt to compare" >&2 + exit 1 + fi + shared=$(sort "$low" "$high" | uniq -d) + if [ -n "$shared" ]; then + echo "FAIL: concurrent tasks shared GPU(s): $shared" >&2 + exit 1 + fi + echo "OK: concurrent tasks held disjoint GPUs" + # -- resource_release_after: grabs every GPU on a node, then frees them on # completion (no readiness probe -> sflow infers task_completion), so the # servers below can reuse them. - name: env_check operator: worker_gpu + depends_on: [placement_disjoint] resources: gpus: count: ${{ variables.GPUS_PER_NODE }} diff --git a/examples/self_contained/slurm/sglang_server_client.yaml b/examples/self_contained/slurm/sglang_server_client.yaml index 7409814..0594453 100644 --- a/examples/self_contained/slurm/sglang_server_client.yaml +++ b/examples/self_contained/slurm/sglang_server_client.yaml @@ -176,7 +176,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/examples/self_contained/slurm/trtllm_serve_disagg.yaml b/examples/self_contained/slurm/trtllm_serve_disagg.yaml index 37e870e..478244c 100644 --- a/examples/self_contained/slurm/trtllm_serve_disagg.yaml +++ b/examples/self_contained/slurm/trtllm_serve_disagg.yaml @@ -328,6 +328,15 @@ workflow: script: - set -x - echo ${CUDA_VISIBLE_DEVICES} + # GPU placement probe. sflow plans a per-task slice and re-applies it + # inside the step; this prints what the container ACTUALLY has next to + # what was planned, which is the only way to tell the two failure modes + # apart: if the planned ordinals are absent from `nvidia-smi -L`, the + # slice never reached the container (carved and renumbered by the + # runtime); if they are present, placement worked and the launcher chose + # the wrong device. Cheap, and it makes every future triage one grep. + - echo "SFLOW_GPU_PROBE task=${SLURM_JOB_NAME:-?} procid=${SLURM_PROCID:-?} localid=${SLURM_LOCALID:-?} planned_CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" + - nvidia-smi -L 2>&1 | sed "s/^/SFLOW_GPU_PROBE visible: /" || true - export TLLM_LOG_LEVEL=INFO - export TRTLLM_SERVER_DISABLE_GC=1 - export TRTLLM_WORKER_DISABLE_GC=1 @@ -374,6 +383,15 @@ workflow: script: - set -x - echo ${CUDA_VISIBLE_DEVICES} + # GPU placement probe. sflow plans a per-task slice and re-applies it + # inside the step; this prints what the container ACTUALLY has next to + # what was planned, which is the only way to tell the two failure modes + # apart: if the planned ordinals are absent from `nvidia-smi -L`, the + # slice never reached the container (carved and renumbered by the + # runtime); if they are present, placement worked and the launcher chose + # the wrong device. Cheap, and it makes every future triage one grep. + - echo "SFLOW_GPU_PROBE task=${SLURM_JOB_NAME:-?} procid=${SLURM_PROCID:-?} localid=${SLURM_LOCALID:-?} planned_CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" + - nvidia-smi -L 2>&1 | sed "s/^/SFLOW_GPU_PROBE visible: /" || true - export TLLM_LOG_LEVEL=INFO - export TRTLLM_SERVER_DISABLE_GC=1 - export TRTLLM_WORKER_DISABLE_GC=1 diff --git a/pyproject.toml b/pyproject.toml index 59cb48a..ca0ba6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ dev = [ "pytest-asyncio>=0.23.0", "pytest-cov>=7.0.0", "pytest-subprocess>=1.5.3", + "pytest-xdist>=3.8.0", ] s3 = [ "boto3>=1.34.0", @@ -78,6 +79,9 @@ where = ["src"] [tool.setuptools.package-data] "sflow.samples" = ["**/*.yaml", "**/*.csv", "**/*.md"] +# gpu_placement.sh is sourced by every Slurm GPU step; without it the operator +# skips GPU placement entirely, so it must ship in the wheel. +"sflow.plugins.operators" = ["*.sh"] "sflow.skills" = ["*.md", "**/*.md", "**/*.py"] "sflow.monitoring" = ["*.py"] @@ -90,6 +94,7 @@ dev = [ "pytest-asyncio>=0.23.0", "pytest-cov>=7.0.0", "pytest-subprocess>=1.5.3", + "pytest-xdist>=3.8.0", ] diff --git a/scripts/full_sample_tests.sh b/scripts/full_sample_tests.sh index cadb7af..900298d 100755 --- a/scripts/full_sample_tests.sh +++ b/scripts/full_sample_tests.sh @@ -10,13 +10,15 @@ CLI_MODEL_PATH="" CLI_PARTITION="" CLI_ACCOUNT="" usage() { - echo "Usage: $0 [-a|-s|-m|-inf|--smoke|--min] [-S] [-P] [-j N] [-M model_path] [-p partition] [-A account]" + echo "Usage: $0 [-a|-s|-m|-inf|--smoke|--min|--one] [-S] [-P] [-j N] [-M model_path] [-p partition] [-A account]" echo " -a all tests (default)" echo " -s self-contained examples only" echo " -m modular examples only" echo " -inf infmax batch suites only" echo " --smoke curated Slurm smoke subset with broad coverage" echo " --min minimal Slurm submit set (one representative per validation type)" + echo " --one submit EXACTLY ONE Slurm job and skip the preflight sweep." + echo " Plumbing smoke for a new CI path / cluster, not coverage." echo " -S submit jobs to Slurm" echo " -P preflight checks only (skip job submission even if -S is set)" echo " -j max parallel jobs (default: 16, 0 for unlimited)" @@ -33,6 +35,7 @@ while [ $# -gt 0 ]; do -inf) TEST_TYPE="inf" ;; --smoke) TEST_TYPE="smoke" ;; --min) TEST_TYPE="min" ;; + --one) TEST_TYPE="one" ;; -S) SUBMIT="--submit" ;; -P) PREFLIGHT_ONLY="1" ;; -j) [ $# -ge 2 ] || { usage; exit 1; }; shift; MAX_JOBS="$1" ;; @@ -275,7 +278,85 @@ run_check() { # ========================================================================= # Preflight: CLI smoke tests (no jobs submitted) # ========================================================================= -if true; then +# The colon-in-task-script fixture. Written OUTSIDE the preflight gate on +# purpose: sample_test.sh submits a focused Slurm e2e for it whenever +# SFLOW_COLON_SCRIPT_FIXTURE points at a real file, so leaving it inside the +# sweep would silently drop that real job from --min/--smoke whenever the +# sweep is skipped. Writing a small YAML costs nothing; the run_check probes +# that USE it stay in the sweep below. +COLON_SCRIPT_DIR="$PREFLIGHT_DIR/colon_in_task_script" +COLON_SCRIPT_FIXTURE="$COLON_SCRIPT_DIR/colon_in_task_script.yaml" +COLON_SCRIPT_DRYRUN_LOG="$COLON_SCRIPT_DIR/dry_run.log" +COLON_SCRIPT_COMPOSED="$COLON_SCRIPT_DIR/colon_in_task_script_composed.yaml" +COLON_SCRIPT_BATCH="$COLON_SCRIPT_DIR/colon_in_task_script_batch.sh" +COLON_SCRIPT_BATCH_CONFIG="$COLON_SCRIPT_DIR/colon_in_task_script_batch.yaml" +mkdir -p "$COLON_SCRIPT_DIR" +cat > "$COLON_SCRIPT_FIXTURE" <<'EOF' +version: "0.1" + +variables: + SLURM_ACCOUNT: + value: dummy_acct + SLURM_PARTITION: + value: dummy_part + SLURM_TIMELIMIT: + value: "00:10:00" + SLURM_NODES: + value: 1 + GPUS_PER_NODE: + value: 4 + +backends: + - name: slurm_cluster + type: slurm + default: true + account: ${{ variables.SLURM_ACCOUNT }} + partition: ${{ variables.SLURM_PARTITION }} + time: ${{ variables.SLURM_TIMELIMIT }} + nodes: ${{ variables.SLURM_NODES }} + gpus_per_node: ${{ variables.GPUS_PER_NODE }} + +operators: + - name: srun_no_container + type: srun + ntasks_per_node: 1 + mpi: pmix + +workflow: + name: colon_in_task_script + tasks: + - name: worker + operator: srun_no_container + resources: + gpus: + count: 1 + script: + - echo "My GPUs: $CUDA_VISIBLE_DEVICES" + - echo "COLON_SCRIPT_E2E_PASS" +EOF + +# Whether to run the preflight sweep at all. +# -t one -> never: the single-job smoke exists to prove the +# Slurm path, and the sweep is the slowest part of +# a run while proving nothing about it. +# SFLOW_E2E_SKIP_PREFLIGHT=1 -> caller states the sweep already ran elsewhere. +# sflow's CI runs exactly this (`-P`) in its own +# container job every pipeline, so repeating all +# 182 checks on a SHARED login node costs ~3.5min +# of the e2e and someone else's CPU for no new +# signal. Default off, so a manual/local run still +# gets the safety net. +RUN_PREFLIGHT="1" +if [ "$TEST_TYPE" = "one" ] || [ "${SFLOW_E2E_SKIP_PREFLIGHT:-}" = "1" ]; then + RUN_PREFLIGHT="" +fi +# ...unless -P was asked for explicitly. -P IS the sweep, so a skip switch that +# silences it turns the only job that runs these checks into a vacuous exit 0. +if [ -n "$PREFLIGHT_ONLY" ]; then + RUN_PREFLIGHT="1" +fi + +if [ -n "$RUN_PREFLIGHT" ]; then echo "" echo "===== Preflight: CLI smoke tests (no Slurm submission) =====" echo "===== Running tests in parallel (max_jobs=${MAX_JOBS:-unlimited}) =====" @@ -400,44 +481,53 @@ if true; then grep -F -- 'operator: docker_run' \"$DOCKER_MULTI_DRYRUN_LOG\" && \ grep -F -- 'CUDA_VISIBLE_DEVICES: 0' \"$DOCKER_MULTI_DRYRUN_LOG\" && \ grep -E -- 'gpus: device=0[^0-9,]' \"$DOCKER_MULTI_DRYRUN_LOG\"" - run_check "dry-run gpu_reservation demo pins the planned device slice" \ - bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/demo.yaml\" --dry-run --verbose > \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\" 2>&1 && \ - grep -F -- 'operator: docker_run' \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\" && \ - grep -E -- 'gpus: device=0,1[^0-9,]' \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\" && \ - grep -F -- 'Dry-run complete: gpu_reservation_demo' \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\"" - run_check "dry-run gpu_reservation hog workload plans its GPU claim" \ - bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/hog.yaml\" --dry-run --verbose > \"$DOCKER_GPU_HOG_DRYRUN_LOG\" 2>&1 && \ - grep -F -- 'operator: docker_run' \"$DOCKER_GPU_HOG_DRYRUN_LOG\" && \ - grep -F -- 'Dry-run complete: gpu_hog' \"$DOCKER_GPU_HOG_DRYRUN_LOG\"" - # The pipeline sample fills a 4-GPU board exactly: pinned_service holds device 0 - # for the whole run, server_a holds 1-2 and server_b holds 3 (both released at - # READY), and merged_consumer then takes server_a's PAIR back -- so device=1,2 - # appears twice while device=3 is left over for whoever wants it next. That exact - # layout is the assertion: it is what makes the leftover deterministic, which the - # e2e suite then relies on to prove a concurrent run picks up device 3 and not - # some other one. Drop either `release_after: task_ready` and this fails with - # "merged_consumer needs GPU N, but it is blocked by server_a/server_b". - run_check "dry-run gpu_reservation pipeline reuses one server's GPUs and frees the other's" \ - bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/pipeline.yaml\" --dry-run --verbose > \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\" 2>&1 && \ - grep -F -- 'server_a: releases GPUs after task readiness' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\" && \ - grep -F -- 'server_b: releases GPUs after task completion' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\" && \ - [ \"\$(grep -c -E -- 'gpus: device=0[^0-9,]' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\")\" -eq 1 ] && \ - [ \"\$(grep -c -E -- 'gpus: device=1,2[^0-9,]' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\")\" -eq 2 ] && \ - [ \"\$(grep -c -E -- 'gpus: device=3[^0-9,]' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\")\" -eq 1 ] && \ - grep -F -- 'Dry-run complete: gpu_reservation_pipeline' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\"" - # The scheduling smoke drains and refills a full 8-GPU board in four waves. Wave - # 1 splits it into four pairs and wave 2 reuses each pair, so every pair spec - # appears TWICE -- that count is the assertion, and a scheduler that double-books - # or scatters a task off its predecessor's devices breaks it. The later waves are - # forced by the DAG once the pairs hold, so they need no counts of their own. - run_check "dry-run gpu_reservation scheduling smoke refills a full board in waves" \ - bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/scheduling_smoke.yaml\" --dry-run --verbose > \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\" 2>&1 && \ - [ \"\$(grep -c -E -- 'gpus: device=0,1[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ - [ \"\$(grep -c -E -- 'gpus: device=2,3[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ - [ \"\$(grep -c -E -- 'gpus: device=4,5[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ - [ \"\$(grep -c -E -- 'gpus: device=6,7[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ - [ \"\$(grep -c -F -- 'releases GPUs after task completion' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 11 ] && \ - grep -F -- 'Dry-run complete: docker_gpu_scheduling_smoke' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\"" + # The gpu_reservation samples are internal-only: they are excluded from the + # public sync, and sample_test.sh refuses to package them. Guard rather than + # assume, so a checkout without them SKIPs here instead of failing four checks + # on a missing file, which says nothing about sflow. + if [ -d "$EXAMPLES_DIR/gpu_reservation" ]; then + run_check "dry-run gpu_reservation demo pins the planned device slice" \ + bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/demo.yaml\" --dry-run --verbose > \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\" 2>&1 && \ + grep -F -- 'operator: docker_run' \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\" && \ + grep -E -- 'gpus: device=0,1[^0-9,]' \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\" && \ + grep -F -- 'Dry-run complete: gpu_reservation_demo' \"$DOCKER_GPU_RESERVATION_DRYRUN_LOG\"" + run_check "dry-run gpu_reservation hog workload plans its GPU claim" \ + bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/hog.yaml\" --dry-run --verbose > \"$DOCKER_GPU_HOG_DRYRUN_LOG\" 2>&1 && \ + grep -F -- 'operator: docker_run' \"$DOCKER_GPU_HOG_DRYRUN_LOG\" && \ + grep -F -- 'Dry-run complete: gpu_hog' \"$DOCKER_GPU_HOG_DRYRUN_LOG\"" + # The pipeline sample fills a 4-GPU board exactly: pinned_service holds device 0 + # for the whole run, server_a holds 1-2 and server_b holds 3 (both released at + # READY), and merged_consumer then takes server_a's PAIR back -- so device=1,2 + # appears twice while device=3 is left over for whoever wants it next. That exact + # layout is the assertion: it is what makes the leftover deterministic, which the + # e2e suite then relies on to prove a concurrent run picks up device 3 and not + # some other one. Drop either `release_after: task_ready` and this fails with + # "merged_consumer needs GPU N, but it is blocked by server_a/server_b". + run_check "dry-run gpu_reservation pipeline reuses one server's GPUs and frees the other's" \ + bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/pipeline.yaml\" --dry-run --verbose > \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\" 2>&1 && \ + grep -F -- 'server_a: releases GPUs after task readiness' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\" && \ + grep -F -- 'server_b: releases GPUs after task completion' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\" && \ + [ \"\$(grep -c -E -- 'gpus: device=0[^0-9,]' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\")\" -eq 1 ] && \ + [ \"\$(grep -c -E -- 'gpus: device=1,2[^0-9,]' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\")\" -eq 2 ] && \ + [ \"\$(grep -c -E -- 'gpus: device=3[^0-9,]' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\")\" -eq 1 ] && \ + grep -F -- 'Dry-run complete: gpu_reservation_pipeline' \"$DOCKER_GPU_PIPELINE_DRYRUN_LOG\"" + # The scheduling smoke drains and refills a full 8-GPU board in four waves. Wave + # 1 splits it into four pairs and wave 2 reuses each pair, so every pair spec + # appears TWICE -- that count is the assertion, and a scheduler that double-books + # or scatters a task off its predecessor's devices breaks it. The later waves are + # forced by the DAG once the pairs hold, so they need no counts of their own. + run_check "dry-run gpu_reservation scheduling smoke refills a full board in waves" \ + bash -c "sflow run \"$EXAMPLES_DIR/gpu_reservation/scheduling_smoke.yaml\" --dry-run --verbose > \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\" 2>&1 && \ + [ \"\$(grep -c -E -- 'gpus: device=0,1[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ + [ \"\$(grep -c -E -- 'gpus: device=2,3[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ + [ \"\$(grep -c -E -- 'gpus: device=4,5[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ + [ \"\$(grep -c -E -- 'gpus: device=6,7[^0-9,]' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 2 ] && \ + [ \"\$(grep -c -F -- 'releases GPUs after task completion' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\")\" -eq 11 ] && \ + grep -F -- 'Dry-run complete: docker_gpu_scheduling_smoke' \"$DOCKER_GPU_SCHEDULING_DRYRUN_LOG\"" + else + record_preflight_skip "gpu_reservation dry-run checks skipped: examples/gpu_reservation is not present" + echo " SKIP: examples/gpu_reservation not present; skipping its dry-run checks." + fi run_check "dry-run kubernetes_hello_world uses k8s operator" \ bash -c "sflow run \"$EXAMPLES_DIR/self_contained/kubernetes/hello_world.yaml\" --dry-run --verbose > \"$KUBERNETES_HELLO_DRYRUN_LOG\" 2>&1 && \ grep -F -- 'id=kubernetes' \"$KUBERNETES_HELLO_DRYRUN_LOG\" && \ @@ -1027,56 +1117,6 @@ EOF bash "$OFFLOAD_TUI_CHECK" "$OFFLOAD_TUI_FIXTURE" "$OFFLOAD_TUI_DIR" # -- sflow run/batch: plain script commands containing ':' must stay strings -- - COLON_SCRIPT_DIR="$PREFLIGHT_DIR/colon_in_task_script" - COLON_SCRIPT_FIXTURE="$COLON_SCRIPT_DIR/colon_in_task_script.yaml" - COLON_SCRIPT_DRYRUN_LOG="$COLON_SCRIPT_DIR/dry_run.log" - COLON_SCRIPT_COMPOSED="$COLON_SCRIPT_DIR/colon_in_task_script_composed.yaml" - COLON_SCRIPT_BATCH="$COLON_SCRIPT_DIR/colon_in_task_script_batch.sh" - COLON_SCRIPT_BATCH_CONFIG="$COLON_SCRIPT_DIR/colon_in_task_script_batch.yaml" - mkdir -p "$COLON_SCRIPT_DIR" - cat > "$COLON_SCRIPT_FIXTURE" <<'EOF' -version: "0.1" - -variables: - SLURM_ACCOUNT: - value: dummy_acct - SLURM_PARTITION: - value: dummy_part - SLURM_TIMELIMIT: - value: "00:10:00" - SLURM_NODES: - value: 1 - GPUS_PER_NODE: - value: 4 - -backends: - - name: slurm_cluster - type: slurm - default: true - account: ${{ variables.SLURM_ACCOUNT }} - partition: ${{ variables.SLURM_PARTITION }} - time: ${{ variables.SLURM_TIMELIMIT }} - nodes: ${{ variables.SLURM_NODES }} - gpus_per_node: ${{ variables.GPUS_PER_NODE }} - -operators: - - name: srun_no_container - type: srun - ntasks_per_node: 1 - mpi: pmix - -workflow: - name: colon_in_task_script - tasks: - - name: worker - operator: srun_no_container - resources: - gpus: - count: 1 - script: - - echo "My GPUs: $CUDA_VISIBLE_DEVICES" - - echo "COLON_SCRIPT_E2E_PASS" -EOF run_check "run colon in task script (dry-run)" \ bash -c "sflow run \"$COLON_SCRIPT_FIXTURE\" --dry-run > \"$COLON_SCRIPT_DRYRUN_LOG\" 2>&1" run_check "compose colon in task script" \ @@ -1812,7 +1852,7 @@ EOF COMMON="$EXAMPLES_DIR/modular/inference_x_v2/common_workflow.yaml" BENCH_INFMAX="$EXAMPLES_DIR/modular/inference_x_v2/benchmark_infmax.yaml" BENCH_AIPERF="$EXAMPLES_DIR/modular/inference_x_v2/benchmark_aiperf.yaml" - DYNAMO_IMAGE="${DYNAMO_IMAGE:-nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0}" + DYNAMO_IMAGE="${DYNAMO_IMAGE:-nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0}" MODULAR_MISSABLE=(-M agg_server -M prefill_server -M decode_server -M benchmark_infmax -M benchmark_aiperf) MODULAR_OVERRIDES=(-a "LOCAL_MODEL_PATH=fs://$MODEL_PATH" -s "DYNAMO_IMAGE=$DYNAMO_IMAGE") for framework in trtllm sglang vllm; do @@ -2993,7 +3033,11 @@ fi # ========================================================================= if [ -n "$SUBMIT" ] && [ -z "$PREFLIGHT_ONLY" ]; then echo "" - echo "===== All preflight checks passed — proceeding to job submission =====" + if [ -z "$RUN_PREFLIGHT" ]; then + echo "===== Preflight sweep skipped — proceeding straight to job submission =====" + else + echo "===== All preflight checks passed — proceeding to job submission =====" + fi echo "" set -x cd "$SCRIPT_DIR/../tests/e2e_tests" @@ -3001,12 +3045,12 @@ if [ -n "$SUBMIT" ] && [ -z "$PREFLIGHT_ONLY" ]; then E2E_ACCOUNT="${CLI_ACCOUNT:-user}" E2E_SBATCH_OUTPUT="$REPO_DIR/sflow_output/%j-sflow-submit.out" E2E_SBATCH_ERROR="$REPO_DIR/sflow_output/%j-sflow-submit.err" - # Known-broken/flaky Slurm nodes to keep every e2e submission off of. CI pins - # the live list via the $SLURM_E2E_EXCLUDE_NODES job variable (forwarded to the - # remote by run_slurm_e2e_over_ssh.py); the default here is the fallback for - # local/manual runs. Add a node here (or to the CI variable) to drain it from - # e2e without touching any recipe. - E2E_EXCLUDE_NODES="${SLURM_E2E_EXCLUDE_NODES:-gb-nvl-137-compute02,gb-nvl-137-compute14}" + # Nodes to drain from e2e, via $SLURM_E2E_EXCLUDE_NODES. Default EMPTY on + # purpose: node names are cluster-specific and sbatch rejects the WHOLE + # submission with "Invalid node name specified" if one is unknown, so a + # hardcoded list is a landmine the moment the e2e moves cluster (stale + # gb-nvl-137-* names failed every job on ptyche exactly this way). + E2E_EXCLUDE_NODES="${SLURM_E2E_EXCLUDE_NODES:-}" E2E_BATCH_EXTRA_ARGS=( "--sbatch-output" "$E2E_SBATCH_OUTPUT" "--sbatch-error" "$E2E_SBATCH_ERROR" @@ -3016,6 +3060,29 @@ if [ -n "$SUBMIT" ] && [ -z "$PREFLIGHT_ONLY" ]; then # regardless of the workflow's own pass/fail. "--enable-workflow-monitor" ) + # Some clusters (GB200: ptyche/poly/lyris) require --segment on every + # submission, sized to the job's OWN node count -- and this suite submits + # recipes spanning 1..N nodes in a single bulk submit, so a fixed number cannot + # be right for all of them. "auto" expands to sflow's expression, which sflow + # resolves per workflow (verified: 1-node recipe -> --segment=1, 3-node -> + # --segment=3, same submit). + # + # It is a KEYWORD rather than the expression itself in the CI config on + # purpose: a literal expression there would have to survive CI variable + # expansion upstream, expansion AGAIN when the trigger job forwards it to the + # downstream pipeline, then python and shell quoting -- four chances to be + # silently mangled into something sbatch rejects. "auto" has no "$" to mangle. + # Any other value is passed through verbatim (e.g. a fixed node count). + # Empty default, so clusters without segment support are unaffected. + E2E_SEGMENT="${SLURM_E2E_SEGMENT:-}" + if [ "$E2E_SEGMENT" = "auto" ]; then + # Single quotes: this must reach sflow as a literal expression, and bash + # would fail on "${{...}}" as a bad substitution if it tried to expand it. + E2E_SEGMENT='${{SLURM_NODES}}' + fi + if [ -n "$E2E_SEGMENT" ]; then + E2E_BATCH_EXTRA_ARGS+=("-e" "--segment=$E2E_SEGMENT") + fi # Flag and value MUST be separate argv tokens ("-e" "--exclude=..."); a glued # single token folds a leading space into the value and the secondary-backend # salloc drops the exclude silently. @@ -3029,8 +3096,13 @@ if [ -n "$SUBMIT" ] && [ -z "$PREFLIGHT_ONLY" ]; then else ./sample_test.sh -p "$E2E_PARTITION" -A "$E2E_ACCOUNT" -m "$MODEL_PATH" -t "$TEST_TYPE" --submit -- "${E2E_BATCH_EXTRA_ARGS[@]}" # 09 has some GPU issues fi + e2e_rc=$? set +x + # `set +x` is a command and it always succeeds, so without this the script's + # exit status is ITS status, not the suite's -- the "green despite exit 1" + # family. Nothing runs after this branch, so exiting here is safe. + exit "$e2e_rc" elif [ -z "$SUBMIT" ]; then echo "Preflight only (no -S flag). To submit jobs, re-run with -S." else diff --git a/scripts/use_under_dev_sflow.sh b/scripts/use_under_dev_sflow.sh index 81cabff..7092435 100644 --- a/scripts/use_under_dev_sflow.sh +++ b/scripts/use_under_dev_sflow.sh @@ -15,55 +15,48 @@ resolve_under_dev_sflow_ref() { fi } -assert_under_dev_sflow_editable_install() { +assert_sflow_installed_from() { + # The suite exists to validate THIS checkout, so prove the sflow on PATH came + # from it. Not by grepping `sflow --version` for "source : local editable + # dev" -- that made the runtime-info format load-bearing and rejected install + # shapes that were the right code (a wheel built from the checkout, a + # direct-URL install). direct_url.json records the path pip installed FROM + # whatever the shape, so compare that and do not require "editable". + # + # Worth keeping now that the CI workspace and its venv are reused across + # runs: without it a stale install silently passes as this branch. local repo_dir="$1" - "$SFLOW_TEST_PYTHON" - "$repo_dir" <<'PY' -import json -import sys -from importlib import metadata as importlib_metadata + # PYTHONPATH deliberately cleared: setup_under_dev_sflow puts $repo_dir/src on + # it, and importlib.metadata would then discover src/sflow.egg-info -- which + # has no direct_url.json -- ahead of the real dist-info in site-packages. The + # question here is what is INSTALLED, not what is importable, so the source + # tree must not be on the path. (Otherwise this rejects every checkout that + # ever ran a plain `pip install -e .` and left an egg-info behind.) + PYTHONPATH= "$SFLOW_TEST_PYTHON" - "$repo_dir" <<'SFLOW_INSTALL_CHECK' +import json, sys +from importlib import metadata from pathlib import Path from urllib.parse import unquote, urlparse repo_dir = Path(sys.argv[1]).resolve() - -try: - dist = importlib_metadata.distribution("sflow") -except importlib_metadata.PackageNotFoundError: - print( - "ERROR: sflow is not installed in the selected Python environment.", - file=sys.stderr, - ) - print( - f" Install this checkout editable first: {sys.executable} -m pip install -e {repo_dir}", - file=sys.stderr, - ) - sys.exit(1) - -direct_url_text = dist.read_text("direct_url.json") try: - direct_url = json.loads(direct_url_text or "{}") -except json.JSONDecodeError: - direct_url = {} - -parsed = urlparse(direct_url.get("url") or "") -install_path = Path(unquote(parsed.path)).resolve() if parsed.scheme == "file" else None -is_editable = bool(direct_url.get("dir_info", {}).get("editable")) - -if not is_editable or install_path != repo_dir: - print( - "ERROR: full sample tests must run against this checkout installed editable.", - file=sys.stderr, + dist = metadata.distribution("sflow") +except metadata.PackageNotFoundError: + sys.exit(f"ERROR: sflow is not installed in {sys.executable}.") + +url = json.loads(dist.read_text("direct_url.json") or "{}").get("url") or "" +parsed = urlparse(url) +src = Path(unquote(parsed.path)).resolve() if parsed.scheme == "file" else None +if src != repo_dir: + sys.exit( + "ERROR: the sflow on PATH was not installed from this checkout.\n" + f" Selected Python : {sys.executable}\n" + f" Expected repo : {repo_dir}\n" + f" Installed from : {src or url or 'a package index'}\n" + f" Fix with : {sys.executable} -m pip install -e {repo_dir}" ) - print(f" Selected Python : {sys.executable}", file=sys.stderr) - print(f" Expected repo : {repo_dir}", file=sys.stderr) - print(f" Installed path : {install_path or 'not a local editable install'}", file=sys.stderr) - print( - f" Fix with : {sys.executable} -m pip install -e {repo_dir}", - file=sys.stderr, - ) - sys.exit(1) -PY +SFLOW_INSTALL_CHECK } setup_under_dev_sflow() { @@ -87,8 +80,6 @@ setup_under_dev_sflow() { return 1 fi - assert_under_dev_sflow_editable_install "$repo_dir" - if [ -z "${SFLOW_UNDER_DEV_REF:-}" ]; then SFLOW_UNDER_DEV_REF="$(resolve_under_dev_sflow_ref "$repo_dir" || true)" fi @@ -135,14 +126,8 @@ EOF echo "Using under-dev sflow from $SFLOW_UNDER_DEV_REPO (ref: $SFLOW_UNDER_DEV_REF)" echo "Submitted Slurm jobs install this checkout editable (--sflow-source-path), so uncommitted working-tree changes are included." - local sflow_runtime_info - sflow_runtime_info="$(sflow --version)" - printf '%s\n' "$sflow_runtime_info" - if ! printf '%s\n' "$sflow_runtime_info" | grep -q "source : local editable dev"; then - echo "ERROR: sflow runtime source is not local editable dev." >&2 - echo " full_sample_tests.sh is intended to validate local editable sflow changes." >&2 - return 1 - fi + sflow --version + assert_sflow_installed_from "$repo_dir" } cleanup_under_dev_sflow() { diff --git a/src/sflow/app/assembly.py b/src/sflow/app/assembly.py index 2c356df..09005f6 100644 --- a/src/sflow/app/assembly.py +++ b/src/sflow/app/assembly.py @@ -1662,12 +1662,19 @@ async def build_state( """ Build runtime state from configuration (composition root). - This is intentionally kept out of core to avoid core importing plugins. + Kept out of core because this is the PLAN-TIME side of the split: it turns a + validated config into runtime objects, while core owns what happens once + they exist. Not because core may not import plugins -- it already does, in + the four registry self-population calls (core/backend_registry.py:56 and + its siblings), so defending that invariant here would be defending one + that is already false. """ from pathlib import Path - if source_files: - resolver.source_files = [Path(f) for f in source_files] + # Assigned unconditionally: `resolver` is module-global, so a conditional + # assignment leaves the PREVIOUS run's files in place and visualize() (which + # passes none) then points its error hints at the wrong YAML. + resolver.source_files = [Path(f) for f in source_files] if source_files else [] # Seed an empty workflow/state; we will populate task graph after resolution/allocation. wf = Workflow(name=config.workflow.name, task_graph=TaskGraph()) diff --git a/src/sflow/cli/batch.py b/src/sflow/cli/batch.py index 41f2e93..3b036de 100644 --- a/src/sflow/cli/batch.py +++ b/src/sflow/cli/batch.py @@ -748,6 +748,13 @@ def _generate_sbatch_script( "dist", "*.egg-info", "__pycache__", + # NOT just size: pip/uv write partial `*.tmp` files under .cache while + # other jobs are still bootstrapping, and rsync exits 24 ("some files + # vanished") when one disappears mid-transfer -- which the bootstrap + # treats as fatal, so the whole Slurm job dies seconds in with no output + # directory at all. A shared, concurrently-written cache must never be + # part of the copy source. + ".cache", ".pytest_cache", ".ruff_cache", ".mypy_cache", diff --git a/src/sflow/cli/sample.py b/src/sflow/cli/sample.py index e7e58e2..ea10ce0 100644 --- a/src/sflow/cli/sample.py +++ b/src/sflow/cli/sample.py @@ -378,9 +378,6 @@ def _get_sample_node_info(sample_path: Path) -> str | None: if line_stripped.startswith("SLURM_NODES:"): # Multi-line format, look for value in next lines continue - if "value:" in line_stripped and nodes is None: - # Check if this is under SLURM_NODES by looking at context - pass # Direct value extraction for SLURM_NODES if "SLURM_NODES:" in line and "value:" in line: # Inline format diff --git a/src/sflow/config/loader.py b/src/sflow/config/loader.py index da158ac..24ab25a 100644 --- a/src/sflow/config/loader.py +++ b/src/sflow/config/loader.py @@ -514,6 +514,26 @@ def __init__(self): self.source_files: List[Path] = [] self.file_contributions: List[Dict[str, Any]] = [] + def _warn_unenforced_timeout(self, config: SflowConfig) -> None: + """Say out loud that `timeout:` does nothing. + + The field is accepted and deep-merged but never read, so a recipe that + declares one looks bounded and is not -- and a hung workflow then runs + until something external kills it. Warning is the honest middle: removing + the field would reject every config that sets it (the models forbid extra + keys), and enforcing it would start killing runs that have never been + held to it. + """ + where = [] + if config.workflow.timeout is not None: + where.append("workflow") + where += [t.name for t in config.workflow.tasks if t.timeout is not None] + if where: + _logger.warning( + f"`timeout:` is set on {', '.join(where)} but sflow does not enforce it; " + "these run unbounded. Use the backend's own limit (e.g. Slurm --time)." + ) + def load_config( self, path: Path, @@ -572,6 +592,7 @@ def load_config( except ValidationError as e: raise ValueError(f"Configuration validation failed:\n{e}") + self._warn_unenforced_timeout(config) self.config = config return config @@ -646,6 +667,7 @@ def load_configs( except ValidationError as e: raise ValueError(f"Merged configuration validation failed:\n{e}") + self._warn_unenforced_timeout(config) self.config = config return config diff --git a/src/sflow/config/schema.py b/src/sflow/config/schema.py index 8320d79..4178b94 100644 --- a/src/sflow/config/schema.py +++ b/src/sflow/config/schema.py @@ -1018,6 +1018,12 @@ def script_must_not_be_empty(cls, v: List[str]) -> List[str]: required_by: Optional[List[str]] = None replicas: Optional[ReplicaConfig] = None retries: Optional[RetryConfig] = None + # NOT ENFORCED. Accepted so existing recipes keep loading (this model forbids + # extra keys, so removing it would reject every config that sets it) and so + # multi-file merge can carry it, but nothing reads it: no code path passes it + # to Orchestrator._launch_task_with_timeout, and TaskStatus.TIMEOUT is never + # assigned. Bound a task with the backend's own limit (Slurm `--time`) until + # this is wired. load_config WARNs when it is set. timeout: Optional[Union[int, str]] = None variables: Optional[ Annotated[List[VariableConfig], BeforeValidator(_normalize_to_list)] @@ -1031,6 +1037,8 @@ class WorkflowConfig(StrictBaseModel): """Configuration for the workflow execution.""" name: str + # NOT ENFORCED -- see TaskConfig.timeout above. A workflow declaring + # `timeout: 115m` today runs unbounded. timeout: Optional[Union[str, int]] = None variables: Optional[ Annotated[List[VariableConfig], BeforeValidator(_normalize_to_list)] diff --git a/src/sflow/core/compute_node.py b/src/sflow/core/compute_node.py index 1848ae3..eafc902 100644 --- a/src/sflow/core/compute_node.py +++ b/src/sflow/core/compute_node.py @@ -12,6 +12,15 @@ class ComputeNode: index: int # GPU count available on this node (if known). Used for CUDA_VISIBLE_DEVICES packing/validation. num_gpus: int | None = None + # Physical GPU UUIDs on this node, ordered by HOST device index, as read on + # bare metal before anything was carved. This is ground truth: it is the only + # way to tell "the right number of GPUs" from "the right GPUs", and it is what + # lets a task step name the indices its planned cards turned out to have, + # whatever layer renumbered them. None when the backend could not + # probe (no GPUs, no nvidia-smi, probe failed) -- callers must degrade, not + # assume. Per node, not per backend: nodes in one allocation can differ, and + # two Slurm backends can have different gpus_per_node entirely. + gpu_uuids: list[str] | None = None def to_dict(self) -> dict[str, Any]: return { @@ -19,4 +28,5 @@ def to_dict(self) -> dict[str, Any]: "ip_address": self.ip_address, "index": self.index, "num_gpus": self.num_gpus, + "gpu_uuids": self.gpu_uuids, } diff --git a/src/sflow/core/execution_summary.py b/src/sflow/core/execution_summary.py index aed025a..6ddf06c 100644 --- a/src/sflow/core/execution_summary.py +++ b/src/sflow/core/execution_summary.py @@ -15,7 +15,11 @@ from typing import Any from sflow.core.loop_watchdog import EventLoopWatchdog -from sflow.utils.gpu import parse_cuda_visible_devices, task_gpu_indices +from sflow.utils.gpu import ( + parse_cuda_visible_devices, + task_gpu_indices, + task_gpu_record, +) from .command_trace import get_command_trace from .task import Task, TaskStatus @@ -670,10 +674,19 @@ def _gpu_assignment_lines(self, tasks: list[Task]) -> list[str]: physical = self._task_gpu_ids(task) if not physical: continue + # The step's OWN numbering, straight from its placement record. The + # planner's env is the HOST slice, so reading it here printed the + # physical ids in the in-container column and vice versa: a task + # planned for host 2,3 that a container renumbered to 0,1 was reported + # as physical 0,1 / in-container 2,3 -- backwards, and impossible. + record = task_gpu_record(task) + in_step = record.get("cuda_visible_devices", "") + if in_step.startswith("<"): # / + in_step = "" visible = [ str(i) for i in parse_cuda_visible_devices( - task.envs.get("CUDA_VISIBLE_DEVICES") + in_step or task.envs.get("CUDA_VISIBLE_DEVICES") ) ] rows.append((task.name, ",".join(physical), ",".join(visible) or "-")) diff --git a/src/sflow/core/launcher.py b/src/sflow/core/launcher.py index cad767f..480dde4 100644 --- a/src/sflow/core/launcher.py +++ b/src/sflow/core/launcher.py @@ -106,39 +106,6 @@ def _console_prefix(self, task_name: str | None) -> str: """ return f"[{task_name}] " if task_name else "" - async def _terminate_process(self, process: asyncio.subprocess.Process) -> None: - """ - Best-effort terminate a running subprocess. - """ - try: - if process.returncode is not None: - return - process.terminate() - except ProcessLookupError: - return - except Exception: - # Fall back to kill below. - pass - - try: - await asyncio.wait_for(process.wait(), timeout=5) - return - except Exception: - pass - - try: - if process.returncode is None: - process.kill() - except ProcessLookupError: - return - except Exception: - return - - try: - await asyncio.wait_for(process.wait(), timeout=5) - except Exception: - return - async def run_async( self, command: Command | str | list[str], @@ -426,161 +393,3 @@ def _on_readable() -> None: os.close(master_fd) except OSError: pass - - # async def run_pipe_async( - # self, - # commands: list[Command | str | list[str]], - # shell: bool = False, - # output_logger: Optional[logging.Logger] = None, - # env: Mapping[str, str] | None = None, - # task_name: str | None = None, - # ) -> int: - # """Execute multiple commands connected by pipes asynchronously. - - # Args: - # commands: List of commands to pipe together - # shell: Whether to use shell for individual commands - # output_logger: Optional logger to use for output logging - - # Returns: - # int: Exit code of the final command - # """ - # pfx = self._console_prefix(task_name) - # _logger.info(f"{pfx}========== Commands ==========") - # _logger.info(f"{pfx}{' | '.join([format_command(cmd) for cmd in commands])}") - # _logger.info(f"{pfx}=============================") - - # if not commands: - # raise ValueError("At least one command required") - - # processes = [] - # pipes = [] - # proc_env = None - # if env is not None: - # proc_env = os.environ.copy() - # proc_env.update({str(k): str(v) for k, v in env.items()}) - - # try: - # for i, cmd in enumerate(commands): - # if isinstance(cmd, Command): - # cmd = cmd.as_list() - - # stdin = None - # if i > 0: - # stdin = pipes[-1][0] # Read end of previous pipe - - # if i < len(commands) - 1: - # r, w = os.pipe() - # pipes.append((r, w)) - # stdout = w - # else: - # stdout = asyncio.subprocess.PIPE - - # # Intermediate processes: inherit stderr (goes to console) - # # Last process: merge stderr to stdout (captured in output) - # stderr = asyncio.subprocess.STDOUT if i == len(commands) - 1 else None - - # create_subprocess = ( - # asyncio.create_subprocess_shell - # if shell - # else asyncio.create_subprocess_exec - # ) - - # args = [] - # if shell: - # if isinstance(cmd, list): - # args = [shlex.join(cmd)] - # else: - # args = [cmd] - # else: - # if isinstance(cmd, str): - # args = shlex.split(cmd) - # else: - # args = cmd - - # proc = await create_subprocess( - # *args, - # stdin=stdin, - # stdout=stdout, - # stderr=stderr, - # env=proc_env, - # ) - - # processes.append(proc) - - # # Close parent's copy of the write pipe end immediately - # if i < len(commands) - 1: - # os.close(stdout) - - # # Close parent's copy of the read pipe end used as stdin - # if i > 0: - # os.close(stdin) - - # # Read output from last process - # last_proc = processes[-1] - # try: - # # Read output in chunks to handle progress bars and special characters - # # that use \r without \n (which would cause readline() to hang) - # buffer = b"" - # while True: - # try: - # # Read available data in chunks (non-blocking when data is available) - # chunk = await last_proc.stdout.read(4096) - # if not chunk: - # # Process any remaining data in buffer - # if buffer: - # line_str = _strip_ansi(buffer.decode("utf-8", errors="replace")).rstrip() - # if line_str: - # _logger.info(f"{pfx}{line_str}") - # if output_logger: - # output_logger.info(line_str) - # break - - # buffer += chunk - - # # Split on both \n and \r to handle progress bars - # # Replace \r\n with \n first to avoid double processing - # text = buffer.decode("utf-8", errors="replace") - # text = text.replace("\r\n", "\n").replace("\r", "\n") - - # # Split into lines, keeping incomplete line in buffer - # lines = text.split("\n") - # buffer = lines[-1].encode("utf-8") # Keep incomplete line - - # for line_str in lines[:-1]: - # # Strip ANSI escape sequences for cleaner logs - # line_str = _strip_ansi(line_str).rstrip() - # if line_str: # Skip empty lines from progress bar overwrites - # _logger.info(f"{pfx}{line_str}") - # if output_logger: - # output_logger.info(line_str) - # except Exception as e: - # _logger.warning(f"{pfx}Error reading output: {e}") - # break - - # # Wait for all processes - # exit_codes = await asyncio.gather(*[p.wait() for p in processes]) - # last_exit_code = exit_codes[-1] - - # return last_exit_code - # except asyncio.CancelledError: - # # Terminate the whole pipeline on cancellation. - # for p in processes: - # try: - # await self._terminate_process(p) - # except Exception: - # pass - # raise - - # except Exception: - # # Cleanup pipes if error - # for r, w in pipes: - # try: - # os.close(r) - # except OSError: - # pass - # try: - # os.close(w) - # except OSError: - # pass - # raise diff --git a/src/sflow/core/task_graph.py b/src/sflow/core/task_graph.py index 5cb4702..3c78ae3 100644 --- a/src/sflow/core/task_graph.py +++ b/src/sflow/core/task_graph.py @@ -66,34 +66,3 @@ def get_submittable_tasks(self) -> List[Task]: return to_submit - def update_task_status( - self, - task_name: str, - status: TaskStatus | str, - ) -> None: - """Update the status of a task. - - Args: - task_name: Name of the task to update - status: New status (can be TaskStatus enum or string) - """ - task = self.dag.nodes[task_name] - - if isinstance(status, str): - status = TaskStatus(status) - - task.status = status - - _logger.debug(f"Task '{task_name}' status updated to {status}") - - def mark_all_cancelled(self) -> None: - """Mark all non-terminal tasks as cancelled.""" - for task in self.get_tasks(): - if task.status not in [ - TaskStatus.COMPLETED, - TaskStatus.FAILED, - TaskStatus.TIMEOUT, - TaskStatus.CANCELLED, - ]: - task.status = TaskStatus.CANCELLED - _logger.debug(f"Task '{task.name}' marked as CANCELLED") diff --git a/src/sflow/exceptions.py b/src/sflow/exceptions.py deleted file mode 100644 index 8cb3b90..0000000 --- a/src/sflow/exceptions.py +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - - -class AllocationRequiredError(Exception): - """ - Exception raised when an allocation is required but not available. - """ - - pass diff --git a/src/sflow/monitoring/postprocess_monitor_timeline.py b/src/sflow/monitoring/postprocess_monitor_timeline.py index 3f01dd8..176d65e 100644 --- a/src/sflow/monitoring/postprocess_monitor_timeline.py +++ b/src/sflow/monitoring/postprocess_monitor_timeline.py @@ -138,7 +138,7 @@ def _select_marker_events( _MARKER_LABEL_INK = "#5f6772" # `fail` -- and ONLY fail -- gets a reserved status colour: a cancel is usually # sflow tearing a service down on purpose, so flagging it red would cry wolf on -# every healthy run. Deliberately darker than the `#d62728` in `_SERIES_COLORS`, +# every healthy run. Deliberately darker than the `#d55e00` in `_SERIES_COLORS`, # so a rule can never be mistaken for a GPU line on the same canvas; it always # ships alongside its own text label, never as colour alone. _MARKER_FAIL_INK = "#a61b1b" @@ -231,7 +231,8 @@ def _build_device_legend_svg( colour = _SERIES_COLORS[idx % len(_SERIES_COLORS)] parts.append( f'' + f'y2="{y - 3:.1f}" stroke="{colour}" ' + f'stroke-width="{_series_width(idx, 2.0)}"{_series_dash(idx)}/>' ) parts.append( f'' @@ -661,13 +662,96 @@ def _series_for_metric( return [v for _ts, v in _metric_timeseries(rows, resource_type, metric_name)] -# Line colors for per-resource series (one per GPU). Cycled; chosen to stay -# distinguishable in both the SVG and a greyscale print. +# Line colors for per-resource series (one per GPU): the Okabe-Ito "Color +# Universal Design" palette, verbatim and in its published order. Eight is +# exactly enough -- a node carries at most 8 GPUs, so the cycle never wraps in +# practice and no two devices on one panel can share a colour. +# +# Chosen over a wider hue spread (a true red plus a grey reads as more separated +# to normal vision) because this palette holds up under all three dichromacies as +# well, and these charts get pasted into bugs and shared with people we cannot +# survey. `_SERIES_DASHES` covers the residual: it separates any pair that is not +# exactly 4 slots apart, with no colour at all. +# Reordered from the published sequence -- same eight colours, different slots. +# The palette contains three near-pairs (blue/sky blue, and orange/vermilion/ +# yellow among the warms) and the published order puts BOTH members of two of +# them inside the first four slots. A 4-GPU task only ever draws slots 0-3, which +# made orange-vs-vermilion the common case rather than the rare one. +# +# Two rules pin this order, and both break if you sort it back: +# 1. Slots 0-3 take one colour from each family (blue / warm / green / purple), +# so the common 4-GPU chart is maximally separated by hue alone. +# 2. No near-pair sits exactly 4 apart, because `_SERIES_DASHES` cycles every 4 +# and slots i and i+4 therefore share a dash pattern -- which is precisely +# the redundancy those pairs need. Blue/sky blue are 0 and 5; the warms are +# 1, 6 and 7. _SERIES_COLORS = ( - "#1f77b4", "#d62728", "#2ca02c", "#ff7f0e", "#9467bd", - "#8c564b", "#17becf", "#e377c2", "#7f7f7f", "#bcbd22", + "#0072b2", # blue + "#d55e00", # vermilion + "#009e73", # bluish green + "#cc79a7", # reddish purple + "#000000", # black + "#56b4e9", # sky blue + "#e69f00", # orange + "#f0e442", # yellow ) +# Per-slot stroke-width multiplier, keyed to the order above. Okabe-Ito's yellow +# (slot 7) is the one entry that is genuinely faint on white, and the fix is +# weight rather than a darker hue -- darkening it collapses it toward the +# `#e69f00` orange, which is the pairing the palette exists to avoid. Black +# (slot 4) is nudged the other way for the same reason in reverse: at equal width +# it reads heavier than the rest and draws the eye to whichever GPU lands there. +_SERIES_WIDTH_SCALE = (1.0, 1.0, 1.0, 1.0, 0.875, 1.0, 1.0, 1.25) + +# Dash pattern per device, cycled alongside _SERIES_COLORS and keyed on the same +# index. Colour alone cannot separate two lines that COINCIDE, and coincidence is +# the normal case for a tensor-parallel task: every rank allocates the same +# weights + KV-cache footprint, so `gpu_memory_used_mib` for its GPUs is often +# identical to the byte. The later line then hides exactly under the earlier one +# and the panel is indistinguishable from one where the second device was never +# drawn at all -- which reads as a collection bug rather than as real data. +_SERIES_DASHES = ("", "5,3", "1.5,2.5", "7,2,1.5,2") + + +def _series_dash(idx: int) -> str: + """SVG dash attribute for series *idx*, or "" for the solid first series.""" + pattern = _SERIES_DASHES[idx % len(_SERIES_DASHES)] + return f' stroke-dasharray="{pattern}"' if pattern else "" + + +def _series_width(idx: int, base: float) -> float: + """Stroke width for series *idx*, scaled per `_SERIES_WIDTH_SCALE`.""" + return round(base * _SERIES_WIDTH_SCALE[idx % len(_SERIES_WIDTH_SCALE)], 2) + + +def _mpl_dashes(idx: int) -> tuple[float, ...]: + """`_series_dash` as matplotlib's `dashes=`; empty tuple means solid.""" + pattern = _SERIES_DASHES[idx % len(_SERIES_DASHES)] + return tuple(float(x) for x in pattern.split(",")) if pattern else () + + +def _device_labels( + rows: list[dict[str, object]], families: "list[tuple[str, str, str]]" +) -> list[str]: + """Every GPU drawn anywhere in this image, in stable colour-assignment order. + + The union across ALL gpu families, not just the first: a device that reports + N/A for one field (MIG mode, a per-field collector error) drops out of that + family only. Both renderers key a line's colour on this list's index rather + than on the device's position within its own panel -- otherwise the survivor + of a drop-out inherits the colour of the device that vanished, and the shared + legend below the chart then names the wrong GPU. + """ + labels: list[str] = [] + for _label, resource_type, metric_name in families: + if resource_type != "gpu": + continue + for lab, _ser in _metric_series_by_resource(rows, resource_type, metric_name): + if lab not in labels: + labels.append(lab) + return labels + def gpu_label(gpus: object) -> str: """Render a report's GPU subset for humans. @@ -1049,16 +1133,7 @@ def _render_svg( caption_y = panels_bottom + 22 + marker_band_bottom # Device colours are shared by every panel, so their legend is built once. - # Union across ALL gpu families, not just the first: a device that reports - # N/A for one field (MIG mode, a per-field collector error) drops out of that - # family only, and keying the colour on each panel's own index would then - # shift every line after it against this legend. - device_labels: list[str] = [] - for _lbl, _rtype, _mname in families: - if _rtype == "gpu": - for lab, _ser in _metric_series_by_resource(rows, _rtype, _mname): - if lab not in device_labels: - device_labels.append(lab) + device_labels = _device_labels(rows, families) device_parts, device_h = ( _build_device_legend_svg(device_labels, x0=8, max_x=width - 8) if device_labels @@ -1138,16 +1213,19 @@ def _y(v: float, _lo: float = ymin, _hi: float = ymax) -> float: f'y2="{gy:.1f}" stroke="#eeeeee"/>' ) for series_label, series in multi: - # Look the colour up BY LABEL so it always matches the shared legend. - colour = ( - _SERIES_COLORS[device_labels.index(series_label) % len(_SERIES_COLORS)] + # Look the style up BY LABEL so it always matches the shared legend. + # Index 0 for a panel with no per-device series (cpu/mem/disk/net): + # first colour, no dash, i.e. unchanged from a single solid line. + idx = ( + device_labels.index(series_label) if series_label in device_labels - else "#1f77b4" + else 0 ) + colour = _SERIES_COLORS[idx % len(_SERIES_COLORS)] points = " ".join(f"{_x(t):.1f},{_y(v):.1f}" for t, v in series) parts.append( f'' + f'stroke-width="{_series_width(idx, 1.5)}"{_series_dash(idx)}/>' ) parts.append( f' None: # Extra pad so the axis title clears the `ready`/`done` band drawn beneath it. axes[-1].set_xlabel("Elapsed time (s)", labelpad=28) figure.suptitle(title, fontsize=14) + # Device legend UNDER the panels, matching the SVG. In-panel it sat on top of + # the data it was explaining -- and the busiest corner at that, since GPU load + # ramps toward the end of a run. Once per figure, not once per panel: a + # device keeps its colour on every panel, so repeating it is pure noise. + if device_labels: + handles = [ + plt.Line2D( + [], + [], + color=_SERIES_COLORS[i % len(_SERIES_COLORS)], + dashes=_mpl_dashes(i), + linewidth=_series_width(i, 1.4), + ) + for i in range(len(device_labels)) + ] + figure.legend( + handles, + device_labels, + loc="upper center", + bbox_to_anchor=(0.5, 0.0), + ncol=min(len(device_labels), 8), + fontsize=8, + frameon=False, + ) png_path.parent.mkdir(parents=True, exist_ok=True) figure.savefig(png_path, dpi=140, bbox_inches="tight") plt.close(figure) diff --git a/src/sflow/plugins/backends/slurm.py b/src/sflow/plugins/backends/slurm.py index cf0e633..5995c0a 100644 --- a/src/sflow/plugins/backends/slurm.py +++ b/src/sflow/plugins/backends/slurm.py @@ -60,6 +60,71 @@ def planning_node_count(self) -> Resolvable[int] | None: return self.nodes +def _planned_gpu_uuids( + cuda_visible_devices: str | None, allocation: "Allocation | None" +) -> str: + """Encode "which physical GPUs was this task planned for, on each node". + + Format: ``node=uuid,uuid;node=uuid,uuid`` -- one entry per node whose topology + is known. The task step looks up its OWN node and compares against what it can + actually see. + + The plan is a flat list of HOST indices applied identically on every node the + task spans, so the same slot resolves to a DIFFERENT physical card per node -- + which is exactly why this is emitted per node rather than as one list. + + Returns "" when there is nothing trustworthy to say (no plan, no allocation, + no probe, or a planned index beyond a node's device count). Silence means + "fall back", never "no GPUs". + """ + if not cuda_visible_devices or allocation is None: + return "" + slots: list[int] = [] + for token in cuda_visible_devices.split(","): + token = token.strip() + if not token.isdigit(): + # UUID-form or anything non-ordinal: nothing to resolve against. + return "" + slots.append(int(token)) + if not slots: + return "" + + entries: list[str] = [] + for node in allocation.nodes: + uuids = node.gpu_uuids + if not uuids: + continue + over = [slot for slot in slots if slot >= len(uuids)] + if over: + # A partial answer is worse than none: it would let a step "verify" + # against a map that cannot contain the card it was planned for. But + # say so -- this is the one case where the truth is in hand and the + # config is simply wrong (gpus_per_node larger than the node really + # has), and staying quiet just drops the task to weaker checking. + _logger.warning( + "Node %s reports %d GPU(s) but this task was planned for device %s; " + "check the backend's gpus_per_node. Placement for this task falls " + "back to device-index arithmetic instead of UUID verification.", + node.name, + len(uuids), + ",".join(str(slot) for slot in over), + ) + continue + entries.append(f"{node.name}=" + ",".join(uuids[slot] for slot in slots)) + return ";".join(entries) + + +class _OutputCapture(logging.Handler): + """Collect a subprocess's log lines into *sink*, for the srun/salloc probes.""" + + def __init__(self, sink: list[str]) -> None: + super().__init__() + self._sink = sink + + def emit(self, record: logging.LogRecord) -> None: + self._sink.append(record.getMessage()) + + @register_backend("slurm", SlurmBackendConfig) class SlurmBackend(Backend): """ @@ -143,8 +208,52 @@ def dry_run_details(self) -> list[tuple[str, str]]: details.append(("extra_args", str(list(self._extra_args)))) return details + @property + def node_topology_report(self) -> str | None: + """The allocation's GPU topology, for the summary's Node Topology section. + + This is the bare-metal index -> UUID map taken before any task carved + anything. Recorded so a run can be judged AFTER the fact: each GPU task + also writes what it actually saw inside its step, and comparing the two + is what distinguishes "sflow placed this wrong" from "the recipe used the + wrong device" -- neither of which a bare index list can settle. + """ + allocation = self.allocation + if allocation is None: + return None + blocks: list[str] = [] + for node in allocation.nodes: + if not node.gpu_uuids: + continue + blocks.append(f"{node.name}: {len(node.gpu_uuids)} GPU(s)") + blocks.extend( + f" [{index}] {uuid}" for index, uuid in enumerate(node.gpu_uuids) + ) + return "\n".join(blocks) if blocks else None + def resource_env(self, *, cuda_visible_devices: str | None = None) -> dict[str, str]: env = super().resource_env(cuda_visible_devices=cuda_visible_devices) + # Do NOT hand NVIDIA_VISIBLE_DEVICES to an srun step. It is read by the + # container runtime (pyxis/enroot) at container CREATION, and naming a + # subset there makes the runtime carve the container and RENUMBER those + # devices from 0 -- after which the CUDA_VISIBLE_DEVICES we exported + # alongside it, in HOST numbering, addresses nothing. That is the whole + # origin of the placement problem: a worker planned for host 2,3 landed in + # a 2-GPU container numbered 0,1 and died with "No CUDA GPUs are + # available", while a worker planned for 0,1 survived by coincidence. + # + # Without it the container sees every GPU on the node with host numbering + # intact, so the slice we planned is directly addressable and + # CUDA_VISIBLE_DEVICES alone -- the variable CUDA actually reads -- decides + # what the task uses. Where something still carves the step (a GRES + # partition with ConstrainDevices), the in-step placement script detects it + # by UUID and re-selects. + # + # The trade is deliberate: this drops device-level isolation (an NVML + # consumer such as nvidia-smi or DCGM can now SEE the node's other GPUs) + # in exchange for the slice being addressable at all. Docker keeps its own + # isolation via `--gpus device=` and overrides this method entirely. + env.pop("NVIDIA_VISIBLE_DEVICES", None) env.update( { key: value @@ -153,6 +262,17 @@ def resource_env(self, *, cuda_visible_devices: str | None = None) -> dict[str, } ) + # Resolve the planned HOST indices to physical UUIDs, per node, so the step + # can check whether it already holds them instead of rewriting + # CUDA_VISIBLE_DEVICES unconditionally. Built from THIS backend's own + # allocation: with several Slurm backends the node sets and gpus_per_node + # differ, and a UUID is the only identifier that survives a container + # renumbering devices from 0. Absent when the probe found nothing, which + # the step reads as "fall back to index arithmetic". + planned = _planned_gpu_uuids(cuda_visible_devices, self.allocation) + if planned: + env["SFLOW_PLANNED_GPU_UUIDS"] = planned + allocation = self.allocation job_id: str | None = env.get("SLURM_JOB_ID") or env.get("SLURM_JOBID") if not job_id and allocation is not None: @@ -225,6 +345,90 @@ async def _resolve_nodes_via_scontrol( ) return nodes + async def _discover_gpu_uuids( + self, *, nodes: list[ComputeNode], job_id: str | None = None + ) -> None: + """Record each node's HOST GPU index -> UUID map, in place. + + One bare srun across the allocation, before any task carves anything, so + what nvidia-smi reports here really is the host numbering. That map is the + only thing that can answer "does this step already hold the GPUs it was + planned for?" -- device indices cannot, because a container renumbers them + from 0 and an index says nothing about which physical card it names. + + Per node on purpose. Nodes in one allocation can differ, and two Slurm + backends can have different gpus_per_node, so a single per-backend count + is not a substitute. + + Best effort: on any failure the maps stay None and the step falls back to + the index arithmetic it used before. A placement probe must never be the + reason a workflow cannot start. + """ + if not nodes: + return + nodelist = ",".join(node.name for node in nodes) + cmd: list[str] = ["srun", "--nodelist", nodelist, "--ntasks-per-node=1"] + if job_id: + cmd.extend(["--jobid", job_id]) + # --overlap: this shares the allocation with the real job steps rather + # than waiting for one, and it must not hold resources of its own. + cmd.append("--overlap") + cmd.extend( + [ + "bash", + "-c", + "timeout 10 nvidia-smi --query-gpu=index,uuid --format=csv,noheader " + '| tr -d " " | sed "s|^|${SLURMD_NODENAME:-$(hostname -s)} |"', + ] + ) + _logger.debug(f"Discovering GPU topology via srun: {' '.join(cmd)}") + + output_lines: list[str] = [] + + capture_logger = isolated_logger("slurm.gpu_discovery") + try: + with temporary_handler(capture_logger, _OutputCapture(output_lines)): + exit_code = await self._subprocess_launcher.run_async( + cmd, output_logger=capture_logger + ) + except Exception as exc: # pragma: no cover - defensive + _logger.debug(f"GPU topology discovery failed ({exc}); placement will " + "fall back to device-index arithmetic.") + return + if exit_code != 0: + _logger.debug( + f"GPU topology discovery exited {exit_code}; placement will fall " + f"back to device-index arithmetic. Output: {output_lines}" + ) + return + + by_node: dict[str, dict[int, str]] = {} + for line in output_lines: + parts = line.split() + if len(parts) < 2 or "," not in parts[-1]: + continue + node_name = parts[-2] + index_text, _, uuid = parts[-1].partition(",") + if not index_text.isdigit() or not uuid.startswith("GPU-"): + continue + by_node.setdefault(node_name, {})[int(index_text)] = uuid + + for node in nodes: + found = by_node.get(node.name) + if not found: + continue + # Ordered by host index, and only a contiguous 0..N-1 run is usable: + # a gap means the reading is partial, and a partial map would resolve + # planned indices to the wrong cards. + if set(found) == set(range(len(found))): + node.gpu_uuids = [found[i] for i in range(len(found))] + _logger.debug( + "GPU topology: " + + ", ".join( + f"{n.name}={len(n.gpu_uuids or [])}" for n in nodes + ) + ) + async def _resolve_nodes_via_srun( self, *, nodelist: str, job_id: str | None = None ) -> list[ComputeNode]: @@ -251,11 +455,7 @@ async def _resolve_nodes_via_srun( # Capture output output_lines: list[str] = [] - class OutputCaptureHandler(logging.Handler): - def emit(self, record: logging.LogRecord): - output_lines.append(record.getMessage()) - - capture_handler = OutputCaptureHandler() + capture_handler = _OutputCapture(output_lines) # Use a per-call isolated logger so concurrent backend allocations don't # capture each other's output (see isolated_logger docstring). capture_logger = isolated_logger("slurm.srun_resolve") @@ -383,6 +583,8 @@ async def allocate(self) -> Allocation: ) # Important: we do NOT own this allocation; do not scancel on exit. + if self._gpu_per_node: + await self._discover_gpu_uuids(nodes=nodes, job_id=job_id) return Allocation(allocation_id=str(job_id), nodes=nodes, owned=False) command = ( @@ -402,8 +604,12 @@ async def allocate(self) -> Allocation: if self._exclude_nodes: command.add_opt("--exclude", ",".join(self._exclude_nodes)) + # add_arg, not add_opt: extra_args is a verbatim passthrough of already + # tokenized argv. add_opt() de-dups by option name, which treats a bare + # value ("1" in `-G 1`) as an option and lets a later identical value + # ("1" in `-N 1`) delete it. Same as srun/docker's extra_args handling. for arg in self._extra_args: - command.add_opt(arg) + command.add_arg(arg) parser = ParseLogHandler( patterns=[ @@ -414,11 +620,7 @@ async def allocate(self) -> Allocation: # Capture all output lines for error reporting output_lines: list[str] = [] - class OutputCaptureHandler(logging.Handler): - def emit(self, record: logging.LogRecord): - output_lines.append(record.getMessage()) - - capture_handler = OutputCaptureHandler() + capture_handler = _OutputCapture(output_lines) # Use a per-call isolated logger so concurrent backend allocations don't # capture each other's salloc output. Routing through the shared module # logger caused parsers from sibling allocations to see each other's @@ -479,6 +681,8 @@ def emit(self, record: logging.LogRecord): ) raise + if self._gpu_per_node: + await self._discover_gpu_uuids(nodes=nodes, job_id=allocation_id) return Allocation( allocation_id=allocation_id, nodes=nodes, diff --git a/src/sflow/plugins/operators/gpu_placement.sh b/src/sflow/plugins/operators/gpu_placement.sh new file mode 100644 index 0000000..54d4fc1 --- /dev/null +++ b/src/sflow/plugins/operators/gpu_placement.sh @@ -0,0 +1,260 @@ +# sflow GPU placement -- sourced (never executed) by each Slurm job step. +# +# MUST be sourced: it exports CUDA_VISIBLE_DEVICES into the task's own shell, and +# a subprocess could not. +# +# Inputs (environment): +# SFLOW_GPU_PLAN planned HOST device indices, e.g. "2,3" +# SFLOW_PLANNED_GPU_UUIDS "=,;..." from the driver's +# bare-metal probe; absent => fall back to the old +# index arithmetic +# SFLOW_GPU_MARKER record filename under SFLOW_TASK_OUTPUT_DIR +# +# It answers ONE question, the same way every time: which indices do the planned +# cards have HERE? It probes the devices this step can actually see, looks each +# planned UUID up among them, and names the indices it found. That single rule +# covers every shape -- more devices visible than planned (narrow to them), +# exactly as many (fix a stale value naming the wrong ordinals), a container that +# renumbered from 0 (just a different lookup result), nothing set at all (name +# them explicitly, because recipes deref this under `set -u`). There is no +# separate no-op / pin / narrow mode, because those were all this one question. +# +# UUID is the identity, never the index: an index stops meaning anything the +# moment a layer renumbers. A planned card that is not visible AT ALL is a hard +# error (exit 97) when sflow chose the devices, and a degrade to index arithmetic +# when Slurm did -- GRES may have granted cards the planner never saw, and there +# the plan can only mean a position into the grant. +# +# Where the UUIDs cannot be checked at all (no map for this node, or no nvidia-smi +# here) it degrades to that same index arithmetic rather than stopping. Narrowing +# a step handed the whole allocation is the collision this exists to prevent, and +# a KNOWN plan must never end up doing less than an unknown one. +# +# It leaves an audit record next to the task's logs: what was planned, what the +# step could see (index -> UUID), what CUDA ended up selecting, and what arrived +# beforehand -- inherited equal to final IS the "nothing needed changing" signal. +# Compared with the Node Topology section of the run summary, that is what tells +# a bad placement apart from a recipe using the wrong device. +[ -n "${SFLOW_GPU_PLAN:-}" ] || return 0 + +__sflow_plan="${SFLOW_GPU_PLAN:-}" +__sflow_node="${SLURMD_NODENAME:-$(hostname -s)}" +# Physical GPUs planned for THIS node; empty => fall back to index arithmetic. +__sflow_want="" +if [ -n "${SFLOW_PLANNED_GPU_UUIDS:-}" ]; then + __sflow_oifs="$IFS"; IFS=';' + for __sflow_e in ${SFLOW_PLANNED_GPU_UUIDS}; do + case "$__sflow_e" in + "$__sflow_node="*) __sflow_want="${__sflow_e#*=}" ;; + esac + done + IFS="$__sflow_oifs" + # A map that names other nodes but not this one means the driver's node names + # and $SLURMD_NODENAME disagree (short vs FQDN is the usual cause). Verification + # silently switches off there, so say it once. + [ -n "$__sflow_want" ] || echo "sflow: no planned-GPU entry for node '$__sflow_node' in SFLOW_PLANNED_GPU_UUIDS; falling back to device-index placement" >&2 +fi + +# What arrived in the step, before this script touches anything. Without it the +# record shows only the post-state and a reader cannot tell what changed or why. +# +# BOTH variables, because they answer different questions. sflow exports only +# CUDA_VISIBLE_DEVICES (Backend.resource_env pops NVIDIA_VISIBLE_DEVICES -- it is +# consumed at container CREATION and setting it carved the container down before +# the plan could be applied). So whatever NVIDIA_VISIBLE_DEVICES holds here came +# from the runtime, and recording it verbatim is how a reader attributes a +# surprising device set to that layer rather than to sflow. +__sflow_in="${CUDA_VISIBLE_DEVICES:-}" +# Describe a variable VERBATIM. UNSET and SET-BUT-EMPTY are different states and +# must not read alike, so they get and -- but no +# interpretation beyond that: what "unset" or "all" means to CUDA or to a +# container runtime is the reader's call, and a gloss here would be sflow +# asserting semantics it does not own (and would be wrong on any stack that +# differs). One function, so the two sentinels have one spelling. +__sflow_desc() { + [ -n "${!1+x}" ] || { printf ''; return; } + printf '%s' "${!1:-}" +} +__sflow_in_desc="$(__sflow_desc CUDA_VISIBLE_DEVICES)" +__sflow_nvd_desc="$(__sflow_desc NVIDIA_VISIBLE_DEVICES)" + +# THE RECORD FORMAT IS A CONTRACT -- three independent parsers read it: +# * this function (producer) +# * sflow.utils.gpu.task_gpu_record (Python; run reporting + the summary) +# * sample_test.sh::gpu_placement_verified (shell `sed`; the e2e verdict) +# Line 1 is the bare device list and must stay parseable on its own. Every other +# line is `key=value`; `visible=`/`selected=` repeat, the rest are scalars. +# Renaming a key breaks the shell reader SILENTLY, so the key set is pinned by +# test_the_marker_format_keys_are_the_contract -- change both, or neither. +__sflow_record() { + printf '%s\n' "${CUDA_VISIBLE_DEVICES:-}" + printf 'node=%s\n' "$__sflow_node" + printf 'action=%s\n' "$__sflow_act" + printf 'reason=%s\n' "${__sflow_reason:-(placement left to device-index arithmetic)}" + printf 'cuda_visible_devices_inherited=%s\n' "$__sflow_in_desc" + printf 'nvidia_visible_devices_inherited=%s\n' "$__sflow_nvd_desc" + # No nvidia_visible_devices= counterpart: sflow never writes that variable, so + # a post-state would equal the inherited line in every record ever produced. + printf 'cuda_visible_devices=%s\n' "$(__sflow_desc CUDA_VISIBLE_DEVICES)" + printf 'planned_host_indices=%s\n' "$__sflow_plan" + printf 'planned_uuids=%s\n' "${__sflow_want:-(not resolved)}" + printf 'visible_gpu_count=%s\n' "${#__sflow_uuid[@]}" + for __sflow_i in "${!__sflow_uuid[@]}"; do + printf 'visible=%s %s\n' "$__sflow_i" "${__sflow_uuid[$__sflow_i]}" + done + __sflow_now="${CUDA_VISIBLE_DEVICES:-}" + for __sflow_t in ${__sflow_now//,/ }; do + case "$__sflow_t" in + GPU-*) printf 'selected=%s %s\n' "?" "$__sflow_t" ;; + ''|*[!0-9]*) printf 'selected=%s (unresolvable)\n' "$__sflow_t" ;; + *) printf 'selected=%s %s\n' "$__sflow_t" "${__sflow_uuid[$__sflow_t]:-(not visible here)}" ;; + esac + done +} +# Called before every exit too, not just at the end: a step that aborts on a +# mis-placement is exactly when the record is worth having. +__sflow_save() { + [ -n "${SFLOW_TASK_OUTPUT_DIR:-}" ] && [ "${SLURM_LOCALID:-0}" = 0 ] || return 0 + __sflow_marker="${SFLOW_GPU_MARKER:-sflow_gpus.log}" + if [ "${SLURM_STEP_NUM_NODES:-1}" = 1 ] && [ "${SLURM_PROCID:-0}" = 0 ]; then + __sflow_record > "$SFLOW_TASK_OUTPUT_DIR/$__sflow_marker" 2>/dev/null || true + else + __sflow_record > "$SFLOW_TASK_OUTPUT_DIR/${__sflow_marker%.log}.$__sflow_node.log" 2>/dev/null || true + fi +} + +__sflow_done="" +__sflow_act="fallback" +__sflow_reason="" + +# What this namespace can really see, probed HERE (inside the container/cgroup, so +# it is the step's own view, not the driver's). Keyed by the index nvidia-smi +# REPORTS rather than by row position, so the record states a detected index and +# an ordinal lookup stays correct even if the numbering is ever not 0..N-1. +# Indexed (NOT associative): bash indexed arrays are sparse, and ${!arr[@]} +# yields subscripts in ascending numeric order -- an associative array would +# iterate in hash order and silently scramble the ordered comparison below. +__sflow_uuid=() +while IFS=, read -r __sflow_i __sflow_u; do + case "$__sflow_i" in ''|*[!0-9]*) continue ;; esac + [ -n "$__sflow_u" ] && __sflow_uuid[$__sflow_i]="$__sflow_u" +done < <(timeout 10 nvidia-smi --query-gpu=index,uuid --format=csv,noheader 2>/dev/null | tr -d ' \r') +if [ -n "$__sflow_want" ]; then + if [ "${#__sflow_uuid[@]}" -eq 0 ]; then + # Nothing to look the planned UUIDs up against. Record that the placement is + # unproven -- but do NOT stop here. The index arithmetic below still narrows a + # step that was handed the whole allocation, which is the collision this + # exists to prevent, and it is exactly what a task with NO planned UUIDs gets. + # Returning early would make a KNOWN plan do less than an unknown one. + __sflow_act="unverified" + __sflow_reason="no nvidia-smi in this step, so the planned UUIDs could not be checked" + else + # ONE rule, whatever set the devices: look up each planned card by UUID among + # the ones this step can actually see, and name their indices. That covers + # every shape without special cases -- more visible than planned narrows to + # the right subset; exactly as many still fixes a stale value that names the + # wrong ordinals; a container renumbering from 0 is just a different lookup + # result. UUID is the only identifier that survives every layer's renumbering. + __sflow_sel="" + __sflow_miss="" + for __sflow_w in ${__sflow_want//,/ }; do + __sflow_hit="" + for __sflow_i in "${!__sflow_uuid[@]}"; do + if [ "${__sflow_uuid[$__sflow_i]}" = "$__sflow_w" ]; then __sflow_hit="$__sflow_i"; break; fi + done + if [ -z "$__sflow_hit" ]; then __sflow_miss="$__sflow_w"; break; fi + __sflow_sel="${__sflow_sel:+$__sflow_sel,}$__sflow_hit" + done + if [ -n "$__sflow_miss" ]; then + # A planned card is not here. Who chose the devices decides what that means: + # * Slurm did (GRES granted them, or our CUDA_VISIBLE_DEVICES was replaced) + # -> it may have picked cards the planner never assumed, so the plan is a + # POSITION into what Slurm gave us. Fall through to index arithmetic + # rather than fail a healthy run. + # * sflow did -> the step holds something it was never given, which is the + # silent mis-placement this exists to catch. Fail loudly. + # + # Both signals below are STEP-scoped, deliberately. SLURM_JOB_GPUS is not: + # it says the JOB has GPUs, and Backend.resource_env copies every SLURM_* + # var out of the DRIVER's environment into every task -- so on the `batch + # --submit` path (driver inside the sbatch job) it was set for every step on + # any GRES cluster. Steps also run --overlap, where Slurm does not carve per + # step and never sets SLURM_STEP_GPUS, so that leaked job-level value was the + # only signal in play and the exit 97 below could not fire on the very + # clusters it was written for. Do not add it back. + if [ -n "${SLURM_STEP_GPUS:-}" ] || + { [ -n "$__sflow_in" ] && [ "$__sflow_in" != "$__sflow_plan" ]; }; then + # Keep $__sflow_want: those UUIDs WERE resolved, they just are not here, + # and that distinction is the whole diagnosis. Clearing it to signal + # "fall through" made the record claim planned_uuids=(not resolved), + # which points a reader at the driver probe instead of at the grant. + # Falling through is already what an empty $__sflow_done means. + echo "sflow: planned GPU $__sflow_miss is not among the devices Slurm granted this step; using the planned slice as a position into them instead" >&2 + __sflow_reason="planned GPU $__sflow_miss is not among the devices Slurm granted this step" + else + echo "sflow: planned GPU $__sflow_miss is not visible on $__sflow_node (visible: ${__sflow_uuid[*]})" >&2 + __sflow_act="missing"; __sflow_reason="planned GPU $__sflow_miss is not visible on this node" + __sflow_save + exit 97 + fi + else + # Export only when it actually differs. The record keeps both the inherited + # and the final value, so "did this change anything?" is answered there + # rather than by a separate action name. + # + # CUDA_VISIBLE_DEVICES only. NVIDIA_VISIBLE_DEVICES is consumed by the + # container runtime at CREATION time -- the container already exists, so + # writing it cannot change what is exposed, and the runtime has already + # rewritten it to describe this namespace (enroot sets "all"). Overwriting + # that with CUDA ordinals states something false about a different layer. + [ "$__sflow_sel" = "${CUDA_VISIBLE_DEVICES:-}" ] || export CUDA_VISIBLE_DEVICES="$__sflow_sel" + __sflow_done=1; __sflow_act="verified" + __sflow_reason="planned GPUs located by UUID among ${#__sflow_uuid[@]} visible device(s)" + fi + fi +fi +if [ -z "$__sflow_done" ]; then +__sflow_seen="${CUDA_VISIBLE_DEVICES:-}" +__sflow_real="" +if [ "${#__sflow_uuid[@]}" -gt 0 ]; then + for __sflow_i in "${!__sflow_uuid[@]}"; do + __sflow_real="${__sflow_real:+$__sflow_real,}$__sflow_i" + done + for __sflow_d in ${__sflow_seen//,/ }; do + case ",$__sflow_real," in + *",$__sflow_d,"*) ;; + *) __sflow_seen="" ;; + esac + done + [ -n "$__sflow_seen" ] || __sflow_seen="$__sflow_real" +else + echo "sflow: no nvidia-smi here, so CUDA_VISIBLE_DEVICES=${__sflow_seen:-} is taken on trust; a runtime that renumbered this task's devices from 0 cannot be detected, and placement may be wrong" >&2 +fi +if [ -z "$__sflow_seen" ]; then + __sflow_sel="$__sflow_plan" +else + IFS=, read -r -a __sflow_dev <<< "$__sflow_seen" + IFS=, read -r -a __sflow_slot <<< "$__sflow_plan" + if [ "${#__sflow_dev[@]}" -lt "${#__sflow_slot[@]}" ]; then + echo "sflow: step has ${#__sflow_dev[@]} GPU(s) but this task was planned for ${#__sflow_slot[@]} (CUDA_VISIBLE_DEVICES=$__sflow_seen)" >&2 + __sflow_act="too-few"; __sflow_reason="step has ${#__sflow_dev[@]} GPU(s), task was planned for ${#__sflow_slot[@]}" + __sflow_save + exit 97 + elif [ "${#__sflow_dev[@]}" -eq "${#__sflow_slot[@]}" ]; then + __sflow_sel="$__sflow_seen" + else + __sflow_sel="" + for __sflow_i in "${__sflow_slot[@]}"; do + if [ -z "${__sflow_dev[$__sflow_i]:-}" ]; then + echo "sflow: planned GPU slot $__sflow_i is outside CUDA_VISIBLE_DEVICES=$__sflow_seen" >&2 + __sflow_act="out-of-range"; __sflow_reason="planned GPU slot $__sflow_i is outside the visible devices" + __sflow_save + exit 97 + fi + __sflow_sel="${__sflow_sel:+$__sflow_sel,}${__sflow_dev[$__sflow_i]}" + done + fi +fi +export CUDA_VISIBLE_DEVICES="$__sflow_sel" +fi +__sflow_save diff --git a/src/sflow/plugins/operators/srun.py b/src/sflow/plugins/operators/srun.py index 1a3a3c4..4251956 100644 --- a/src/sflow/plugins/operators/srun.py +++ b/src/sflow/plugins/operators/srun.py @@ -24,6 +24,7 @@ from sflow.core.operator_registry import register_operator from sflow.utils.gpu import GPU_MARKER_FILE from sflow.logging import get_logger +from sflow.utils.extra_args import normalize_extra_args from sflow.utils.container import ( append_runtime_mounts as append_runtime_mount_specs, extract_container_images_from_extra_args, @@ -80,10 +81,9 @@ def _slurm_runtime_env_prelude() -> list[str]: # planned placement. _GPU_PLACEMENT_BANNER = """\ # --- sflow GPU placement (begin) ------------------------------------------- -# On a GRES partition slurmstepd rewrites CUDA_VISIBLE_DEVICES after --export, -# so sflow re-applies its planned slice here (last writer wins). Without this -# every concurrent step sees the whole allocation and collides on GPU 0. -# No-op when the step already sees exactly its own devices. +# Names this task's planned GPUs by looking their UUIDs up among the devices the +# step can really see. Without it a step handed the whole allocation (GRES) or +# renumbered by a container runtime lands on the wrong cards. """ # Closing half of the wrap, so it is obvious where sflow's block stops and the @@ -92,53 +92,46 @@ def _slurm_runtime_env_prelude() -> list[str]: # --- sflow GPU placement (end) --------------------------------------------- """ -_GPU_PLACEMENT_PRELUDE = """\ -__sflow_plan='{plan}' -__sflow_seen="${{CUDA_VISIBLE_DEVICES:-}}" -if [ -z "$__sflow_seen" ]; then - __sflow_sel="$__sflow_plan" -else - IFS=, read -r -a __sflow_dev <<< "$__sflow_seen" - IFS=, read -r -a __sflow_slot <<< "$__sflow_plan" - if [ "${{#__sflow_dev[@]}}" -lt "${{#__sflow_slot[@]}}" ]; then - echo "sflow: step has ${{#__sflow_dev[@]}} GPU(s) but this task was planned for ${{#__sflow_slot[@]}} (CUDA_VISIBLE_DEVICES=$__sflow_seen)" >&2 - exit 97 - elif [ "${{#__sflow_dev[@]}}" -eq "${{#__sflow_slot[@]}}" ]; then - __sflow_sel="$__sflow_seen" - else - __sflow_sel="" - for __sflow_i in "${{__sflow_slot[@]}}"; do - if [ -z "${{__sflow_dev[$__sflow_i]:-}}" ]; then - echo "sflow: planned GPU slot $__sflow_i is outside CUDA_VISIBLE_DEVICES=$__sflow_seen" >&2 - exit 97 - fi - __sflow_sel="${{__sflow_sel:+$__sflow_sel,}}${{__sflow_dev[$__sflow_i]}}" - done - fi -fi -export CUDA_VISIBLE_DEVICES="$__sflow_sel" NVIDIA_VISIBLE_DEVICES="$__sflow_sel" -[ -n "${{SFLOW_TASK_OUTPUT_DIR:-}}" ] && [ "${{SLURM_PROCID:-0}}" = 0 ] \ - && [ "${{SLURM_NNODES:-1}}" = 1 ] \ - && printf '%s\n' "$__sflow_sel" > "$SFLOW_TASK_OUTPUT_DIR/{marker}" 2>/dev/null || true -""" -# Why the marker write above is guarded the way it is. Kept OUT of the template: -# every line of that string is echoed into the generated srun command, where a -# ten-line rationale is noise for whoever is reading the failing command. -# -# Report the devices actually selected back to the driver: the plan is only the -# same thing when the step saw the whole allocation. -# rank 0 only -- every rank runs this body, and with --gpus-per-task they hold -# DIFFERENT devices, so letting them all truncate one path is a race whose -# winner is arbitrary. -# 1 node only -- mirrors the reader (utils/gpu.task_gpu_indices), which discounts -# the marker for a multi-node task because one flat list cannot speak for nodes -# holding different devices. Writing it anyway would leave a file that is right -# for node 0 and wrong for every other, reading as authoritative to anyone who -# opens it. -# `|| true` -- reporting must never fail a task. - - -def _gpu_placement_prelude(cuda_visible_devices: str | None) -> list[str]: +# The placement logic lives in gpu_placement.sh next to this module, staged once +# per run and SOURCED by each step, rather than pasted into every srun command. +# Sourced, not executed: it exports CUDA_VISIBLE_DEVICES into the task's shell. +# Read that file for the reasoning; keeping it out of the command line means a +# failing srun line stays readable, and every task shares one copy. +_GPU_PLACEMENT_SCRIPT = Path(__file__).with_name("gpu_placement.sh") + + +def _stage_gpu_placement_script(workflow_out_dir: str | None) -> str | None: + """Write the placement script into the run's output dir; return its path. + + Lands under the workflow output dir, which is shared storage on Slurm by + construction -- that is what lets every node source the same file. + + Idempotent: many tasks launch at once and would otherwise fight over it. + Returns None when it cannot be written, and the caller then skips placement + entirely rather than running it from somewhere the nodes cannot read. + """ + if not workflow_out_dir: + return None + try: + target = Path(workflow_out_dir) / ".sflow" / "gpu_placement.sh" + target.parent.mkdir(parents=True, exist_ok=True) + body = _GPU_PLACEMENT_SCRIPT.read_text() + if not target.exists() or target.read_text() != body: + # Atomic: a step may be sourcing this path while another driver writes it. + tmp = target.with_name(f"{target.name}.{os.getpid()}.tmp") + tmp.write_text(body) + os.replace(tmp, target) + return str(target) + except OSError: + return None + + +def _gpu_placement_prelude( + cuda_visible_devices: str | None, + *, + gpus_per_task: str | None = None, + workflow_out_dir: str | None = None, +) -> list[str]: """Re-apply sflow's planned GPU slice from *inside* the job step. sflow exports ``CUDA_VISIBLE_DEVICES`` from the driver and relies on srun @@ -159,12 +152,49 @@ def _gpu_placement_prelude(cuda_visible_devices: str | None) -> list[str]: runtime -- last writer wins, and placement is restored without adding a single Slurm flag. - Inert where it should be: on a non-GRES partition the step observes exactly - the slice sflow exported, and on a step Slurm already carved it observes - exactly its own devices; both take the "keep what I see" branch. + ``CUDA_VISIBLE_DEVICES`` cannot be trusted as an OBSERVATION of what the step + sees. sflow exports its plan and srun runs with ``--export=ALL``, so the value + arriving in the step is often just that plan echoed back -- using it to decide + "what do I have" is circular. A container runtime (pyxis/enroot) carves by + passing through only this task's devices and renumbers them from 0, so the + inherited value can name HOST ordinals that do not exist here at all: on + ptyche a decode server planned for ``2,3`` ran in a container holding exactly + two GPUs numbered ``0,1``, kept ``2,3`` because the counts matched, and died + with "No CUDA GPUs are available". The prefill server planned for ``0,1`` + survived only because its plan happened to match the renumbering. + + So the staged script does not reason about that value at all. It probes the + devices the step can really see and looks up the physical UUIDs the driver + resolved this task's plan to (``SFLOW_PLANNED_GPU_UUIDS``), then names the + indices they turned out to have here -- one rule for every shape, because a + UUID is the only identity that survives a layer renumbering from 0. Where + there is no map to check against (probe failed, node names disagree, no + nvidia-smi in the step) it degrades to the older index arithmetic: equal + counts keep the step's own numbering, a larger visible set narrows + positionally. + + Only ``--gpus-per-task`` skips this, not ``gres``/``gpus``. Those two make + Slurm carve per STEP, not per rank, so every rank still sees the same set and + the checks below stay meaningful -- a step granted fewer devices than the task + was planned for is then a real over-ask, and aborting is the right answer. + ``--gpus-per-task`` is the one that carves per RANK, which is what breaks the + premise: + that flag makes the step request GRES, so Slurm carves per RANK instead of + handing the step the whole allocation. Every rank then sees only its own + devices -- fewer than the task's slice -- and the count check below would abort + all of them with "step has 1 GPU(s) but this task was planned for 8". Slurm's + own GRES accounting already keeps those per-rank sets disjoint, which is the + collision this prelude exists to prevent, so there is nothing left to re-apply. """ if not cuda_visible_devices: return [] + if gpus_per_task: + _logger.debug( + "srun --gpus-per-task=%s carves GPUs per rank, so Slurm already owns " + "this task's placement; skipping sflow's in-step GPU remap.", + gpus_per_task, + ) + return [] # The plan is interpolated into shell text, so it is validated first. Every value # sflow's planner produces is a comma-joined list of non-negative ints, but this # reads `envs`, and ANY workflow variable named CUDA_VISIBLE_DEVICES lands there @@ -182,13 +212,29 @@ def _gpu_placement_prelude(cuda_visible_devices: str | None) -> list[str]: cuda_visible_devices, ) return [] - return [ - _GPU_PLACEMENT_BANNER - + _GPU_PLACEMENT_PRELUDE.format( - plan=cuda_visible_devices, marker=GPU_MARKER_FILE + staged = _stage_gpu_placement_script(workflow_out_dir) + if not staged: + # Nowhere the compute nodes can read it from. Skip placement rather than + # paste a second copy of the logic into the command line: that would mean + # two delivery paths to keep honest, and the failure that gets us here -- + # an unwritable workflow output dir -- has already broken the run's logs. + # Skipping leaves CUDA_VISIBLE_DEVICES exactly as exported, which is the + # behaviour from before this prelude existed. + _logger.warning( + "Could not stage the GPU placement script under %r; skipping sflow's " + "in-step GPU placement for this task. On a GRES-configured partition " + "its placement is then Slurm's, not sflow's.", + workflow_out_dir, ) - + _GPU_PLACEMENT_FOOTER - ] + return [] + # One shared copy, sourced. `.` and not `bash`: the script exports into this + # shell, which a child process could not do. + body = ( + f"export SFLOW_GPU_PLAN='{cuda_visible_devices}'\n" + f"export SFLOW_GPU_MARKER='{GPU_MARKER_FILE}'\n" + f'. "{staged}"\n' + ) + return [_GPU_PLACEMENT_BANNER + body + _GPU_PLACEMENT_FOOTER] def _is_valid_container_image(image: str) -> bool: @@ -535,8 +581,8 @@ def build_command( if _has_container and all_mounts: command.add_opt("--container-mounts", ",".join(all_mounts)) - for arg in filtered_extra_args: - command.add_opt(arg) + for arg in normalize_extra_args(filtered_extra_args): + command.add_arg(arg) command.add_arg("bash") command.add_arg("-c") @@ -546,7 +592,11 @@ def build_command( script_body = "\n".join( [ *_slurm_runtime_env_prelude(), - *_gpu_placement_prelude(envs.get("CUDA_VISIBLE_DEVICES")), + *_gpu_placement_prelude( + envs.get("CUDA_VISIBLE_DEVICES"), + gpus_per_task=c.gpus_per_task, + workflow_out_dir=envs.get("SFLOW_WORKFLOW_OUTPUT_DIR"), + ), *list(script), ] ) diff --git a/src/sflow/samples/modular/inference_x_v2/benchmark_aiperf.yaml b/src/sflow/samples/modular/inference_x_v2/benchmark_aiperf.yaml index 247ca73..c2dd7f5 100644 --- a/src/sflow/samples/modular/inference_x_v2/benchmark_aiperf.yaml +++ b/src/sflow/samples/modular/inference_x_v2/benchmark_aiperf.yaml @@ -59,7 +59,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/modular/inference_x_v2/bulk_input.csv b/src/sflow/samples/modular/inference_x_v2/bulk_input.csv index 1f9962f..90037c8 100644 --- a/src/sflow/samples/modular/inference_x_v2/bulk_input.csv +++ b/src/sflow/samples/modular/inference_x_v2/bulk_input.csv @@ -2,9 +2,9 @@ sflow_config_file,SLURM_NODES,GPUS_PER_NODE,NUM_CTX_SERVERS,CTX_TP_SIZE,NUM_GEN_ slurm_config.yaml common_workflow.yaml trtllm/prefill.yaml trtllm/decode.yaml benchmark_infmax.yaml,1,4,1,2,1,2,,,1,[16],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2,agg_server slurm_config.yaml common_workflow.yaml trtllm/prefill.yaml trtllm/decode.yaml benchmark_infmax.yaml,2,4,4,1,1,4,,,2,[32],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2,agg_server slurm_config.yaml common_workflow.yaml sglang/prefill.yaml sglang/decode.yaml benchmark_infmax.yaml,1,4,2,1,1,2,,,1,"[1,4]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,lmsysorg/sglang:v0.5.8.post1-cu130,agg_server -slurm_config.yaml common_workflow.yaml sglang/prefill.yaml sglang/decode.yaml benchmark_infmax.yaml,2,4,2,2,1,4,,,2,"[2,4,6]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0,agg_server +slurm_config.yaml common_workflow.yaml sglang/prefill.yaml sglang/decode.yaml benchmark_infmax.yaml,2,4,2,2,1,4,,,2,"[2,4,6]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0,agg_server slurm_config.yaml common_workflow.yaml vllm/prefill.yaml vllm/decode.yaml benchmark_infmax.yaml,2,4,2,1,3,2,,,1,[8],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0,agg_server slurm_config.yaml common_workflow.yaml vllm/prefill.yaml vllm/decode.yaml benchmark_infmax.yaml,3,4,4,1,1,8,,,3,"[64,32]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0,agg_server slurm_config.yaml common_workflow.yaml trtllm/agg.yaml benchmark_aiperf.yaml,1,4,,,,,1,4,1,"[1,3,5]",Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2,prefill_server decode_server -slurm_config.yaml common_workflow.yaml sglang/agg.yaml benchmark_aiperf.yaml,2,4,,,,,1,8,2,[10],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0,prefill_server decode_server +slurm_config.yaml common_workflow.yaml sglang/agg.yaml benchmark_aiperf.yaml,2,4,,,,,8,1,2,[10],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0,prefill_server decode_server slurm_config.yaml common_workflow.yaml vllm/agg.yaml benchmark_aiperf.yaml,1,4,,,,,2,2,1,[128],Qwen3-8B-FP8,fs:///path/to/Qwen3-8B-FP8,nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0,prefill_server decode_server \ No newline at end of file diff --git a/src/sflow/samples/modular/inference_x_v2/common_workflow.yaml b/src/sflow/samples/modular/inference_x_v2/common_workflow.yaml index bb51329..6cf5b79 100644 --- a/src/sflow/samples/modular/inference_x_v2/common_workflow.yaml +++ b/src/sflow/samples/modular/inference_x_v2/common_workflow.yaml @@ -54,7 +54,7 @@ variables: # Dependencies versions DYNAMO_VERSION: description: "Dynamo version to install when missing from the container image" - value: 0.8.0 + value: 1.3.0 NATS_VERSION: description: "NATS server version to install when missing" value: v2.10.28 diff --git a/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml b/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml index 2d18141..5c770c1 100644 --- a/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml +++ b/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_agg_benchmark_aiperf_2n_008.yaml @@ -10,7 +10,7 @@ variables: - name: NUM_AGG_SERVERS description: Number of aggregated servers type: integer - value: 1 + value: 8 - name: CONCURRENCY description: Concurrency @@ -42,7 +42,7 @@ operators: container_writable: true mpi: pmix extra_args: - - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 + - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 - name: nginx type: srun @@ -101,8 +101,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 + echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 else echo "dynamo already installed." fi @@ -324,17 +324,17 @@ workflow: - export NODE_RANK=${SLURM_NODEID} - export FIRST_NODE_IP=$(echo ${SFLOW_TASK_ASSIGNED_NODE_IPS} | cut -d',' -f1) - | - if [ "2" -gt 1 ]; then - export MULTI_NODE_EXTRA_ARGS="--dist-init-addr ${FIRST_NODE_IP}:29500 --nnodes 2 --node-rank ${NODE_RANK}" + if [ "1" -gt 1 ]; then + export MULTI_NODE_EXTRA_ARGS="--dist-init-addr ${FIRST_NODE_IP}:29500 --nnodes 1 --node-rank ${NODE_RANK}" else export MULTI_NODE_EXTRA_ARGS="" fi - export VLLM_NIXL_SIDE_CHANNEL_PORT=$((5557 + ${FIRST_CUDA_DEVICE})) - export DYN_SYSTEM_PORT=$((8082 + ${FIRST_CUDA_DEVICE})) - - python3 -m dynamo.sglang --model-path ${{ artifacts.LOCAL_MODEL_PATH.path }} --served-model-name Qwen3-8B-FP8 --tensor-parallel-size 8 --pipeline-parallel-size 1 --data-parallel-size 1 --expert-parallel-size 1 --moe-dense-tp-size 1 --max-running-requests 128 --max-prefill-tokens 4096 --trust-remote-code --skip-tokenizer-init --cuda-graph-max-bs 128 ${MULTI_NODE_EXTRA_ARGS} --host 0.0.0.0 + - python3 -m dynamo.sglang --model-path ${{ artifacts.LOCAL_MODEL_PATH.path }} --served-model-name Qwen3-8B-FP8 --tensor-parallel-size 1 --pipeline-parallel-size 1 --data-parallel-size 1 --expert-parallel-size 1 --moe-dense-tp-size 1 --max-running-requests 128 --max-prefill-tokens 4096 --trust-remote-code --skip-tokenizer-init --cuda-graph-max-bs 128 ${MULTI_NODE_EXTRA_ARGS} --host 0.0.0.0 resources: gpus: - count: 8 + count: 1 depends_on: - frontend_server probes: @@ -355,7 +355,7 @@ workflow: - set -x - pip install aiperf==0.3.0 - export COLUMNS=200 - - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' + - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' - echo "Benchmarking finished" resources: nodes: diff --git a/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml b/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml index 6d0b599..61466d5 100644 --- a/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml +++ b/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_003.yaml @@ -101,8 +101,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 + echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 else echo "dynamo already installed." fi diff --git a/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml b/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml index 348a61b..d5ef927 100644 --- a/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml +++ b/src/sflow/samples/modular/inference_x_v2/composed_recipes/sglang_prefill_decode_benchmar_004.yaml @@ -49,7 +49,7 @@ operators: container_writable: true mpi: pmix extra_args: - - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 + - --container-image=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 - name: nginx type: srun @@ -102,8 +102,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 + echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 else echo "dynamo already installed." fi diff --git a/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml b/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml index 610c420..5c4ff88 100644 --- a/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml +++ b/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_agg_benchmark_aiperf_1n_007.yaml @@ -123,8 +123,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 + echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 else echo "dynamo already installed." fi @@ -378,7 +378,7 @@ workflow: - set -x - pip install aiperf==0.3.0 - export COLUMNS=200 - - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' + - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' - echo "Benchmarking finished" resources: nodes: diff --git a/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml b/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml index bce8623..e8b8fea 100644 --- a/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml +++ b/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_001.yaml @@ -144,8 +144,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 + echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 else echo "dynamo already installed." fi @@ -390,10 +390,6 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 - name: decode_server operator: @@ -432,10 +428,6 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 - name: benchmark_infmax operator: diff --git a/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml b/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml index 4117bc9..6d054b7 100644 --- a/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml +++ b/src/sflow/samples/modular/inference_x_v2/composed_recipes/trtllm_prefill_decode_benchmar_002.yaml @@ -144,8 +144,8 @@ workflow: # Install dynamo if missing (check for the base dynamo.frontend module which all backends use) if ! python3 -c "import dynamo" >/dev/null 2>&1; then - echo "dynamo not found, installing ai-dynamo packages (0.8.0)..." - pip install --break-system-packages --quiet ai-dynamo-runtime==0.8.0 ai-dynamo==0.8.0 + echo "dynamo not found, installing ai-dynamo packages (1.3.0)..." + pip install --break-system-packages --quiet ai-dynamo-runtime==1.3.0 ai-dynamo==1.3.0 else echo "dynamo already installed." fi @@ -390,10 +390,6 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 - name: decode_server operator: @@ -432,10 +428,6 @@ workflow: match_pattern: Traceback (most recent call last) match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 - name: benchmark_infmax operator: diff --git a/src/sflow/samples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml b/src/sflow/samples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml index 4be77a1..4319237 100644 --- a/src/sflow/samples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml +++ b/src/sflow/samples/modular/inference_x_v2/composed_recipes/vllm_agg_benchmark_aiperf_1n_2_009.yaml @@ -393,7 +393,7 @@ workflow: - set -x - pip install aiperf==0.3.0 - export COLUMNS=200 - - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' + - 'aiperf profile --artifact-dir ${SFLOW_WORKFLOW_OUTPUT_DIR}/aiperf_concurrency_${CONCURRENCY} --model Qwen3-8B-FP8 --tokenizer ${{ artifacts.LOCAL_MODEL_PATH.path }} --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://${{ variables.HEAD_NODE_IP }}:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple' - echo "Benchmarking finished" resources: nodes: diff --git a/src/sflow/samples/modular/inference_x_v2/sglang/agg.yaml b/src/sflow/samples/modular/inference_x_v2/sglang/agg.yaml index 06ddc8f..36efd83 100644 --- a/src/sflow/samples/modular/inference_x_v2/sglang/agg.yaml +++ b/src/sflow/samples/modular/inference_x_v2/sglang/agg.yaml @@ -1,7 +1,7 @@ # SGLang aggregated (non-disaggregated) server task and its configuration. # Merge all files for a complete run: # sflow compose slurm_config.yaml common_workflow.yaml sglang/agg.yaml benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 \ +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 \ # --missable-tasks prefill_server --missable-tasks decode_server version: "0.1" diff --git a/src/sflow/samples/modular/inference_x_v2/sglang/decode.yaml b/src/sflow/samples/modular/inference_x_v2/sglang/decode.yaml index 3cc22e6..4ee8ffe 100644 --- a/src/sflow/samples/modular/inference_x_v2/sglang/decode.yaml +++ b/src/sflow/samples/modular/inference_x_v2/sglang/decode.yaml @@ -1,7 +1,7 @@ # SGLang decode server task and its configuration. # Merge all files for a complete run: # sflow run -f slurm_config.yaml -f common_workflow.yaml -f sglang_prefill.yaml -f sglang_decode.yaml -f benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 version: "0.1" diff --git a/src/sflow/samples/modular/inference_x_v2/sglang/prefill.yaml b/src/sflow/samples/modular/inference_x_v2/sglang/prefill.yaml index 62462e7..828434d 100644 --- a/src/sflow/samples/modular/inference_x_v2/sglang/prefill.yaml +++ b/src/sflow/samples/modular/inference_x_v2/sglang/prefill.yaml @@ -1,7 +1,7 @@ # SGLang prefill server task and its configuration. # Merge all files for a complete run: # sflow run -f slurm_config.yaml -f common_workflow.yaml -f sglang_prefill.yaml -f sglang_decode.yaml -f benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 version: "0.1" diff --git a/src/sflow/samples/modular/inference_x_v2/trtllm/agg.yaml b/src/sflow/samples/modular/inference_x_v2/trtllm/agg.yaml index b79ad63..198f54e 100644 --- a/src/sflow/samples/modular/inference_x_v2/trtllm/agg.yaml +++ b/src/sflow/samples/modular/inference_x_v2/trtllm/agg.yaml @@ -1,7 +1,7 @@ # TensorRT-LLM aggregated (non-disaggregated) server task and its configuration. # Merge all files for a complete run: # sflow compose slurm_config.yaml common_workflow.yaml trtllm/agg.yaml benchmark_aiperf.yaml \ -# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 \ +# --set DYNAMO_IMAGE=nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 \ # --missable-tasks prefill_server --missable-tasks decode_server version: "0.1" diff --git a/src/sflow/samples/modular/inference_x_v2/trtllm/decode.yaml b/src/sflow/samples/modular/inference_x_v2/trtllm/decode.yaml index 049e1bf..04e72dd 100644 --- a/src/sflow/samples/modular/inference_x_v2/trtllm/decode.yaml +++ b/src/sflow/samples/modular/inference_x_v2/trtllm/decode.yaml @@ -126,7 +126,3 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 diff --git a/src/sflow/samples/modular/inference_x_v2/trtllm/prefill.yaml b/src/sflow/samples/modular/inference_x_v2/trtllm/prefill.yaml index 74e62fe..bc9bd33 100644 --- a/src/sflow/samples/modular/inference_x_v2/trtllm/prefill.yaml +++ b/src/sflow/samples/modular/inference_x_v2/trtllm/prefill.yaml @@ -124,7 +124,3 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 diff --git a/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_agg.yaml b/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_agg.yaml index ef782fd..2c2044c 100644 --- a/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_agg.yaml +++ b/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_agg.yaml @@ -433,7 +433,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg.yaml b/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg.yaml index 997852c..40d6b4d 100644 --- a/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg.yaml +++ b/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg.yaml @@ -695,7 +695,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml b/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml index 2b3c995..f925aea 100644 --- a/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml +++ b/src/sflow/samples/self_contained/kubernetes/dynamo_sglang_disagg_dsr1.yaml @@ -719,7 +719,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_agg.yaml b/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_agg.yaml index 85c7416..182117d 100644 --- a/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_agg.yaml +++ b/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_agg.yaml @@ -383,7 +383,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml b/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml index 4f5fccc..54cbdc7 100644 --- a/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml +++ b/src/sflow/samples/self_contained/kubernetes/dynamo_trtllm_disagg.yaml @@ -523,7 +523,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_agg.yaml b/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_agg.yaml index a4ed1a0..520fd02 100644 --- a/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_agg.yaml +++ b/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_agg.yaml @@ -468,7 +468,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_disagg.yaml b/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_disagg.yaml index 02d96d2..d363258 100644 --- a/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_disagg.yaml +++ b/src/sflow/samples/self_contained/kubernetes/dynamo_vllm_disagg.yaml @@ -668,7 +668,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/aiperf_template.yaml b/src/sflow/samples/self_contained/slurm/aiperf_template.yaml index 92087bc..13828ff 100644 --- a/src/sflow/samples/self_contained/slurm/aiperf_template.yaml +++ b/src/sflow/samples/self_contained/slurm/aiperf_template.yaml @@ -109,7 +109,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/dynamo_sglang_agg.yaml b/src/sflow/samples/self_contained/slurm/dynamo_sglang_agg.yaml index 5800360..fab64dd 100644 --- a/src/sflow/samples/self_contained/slurm/dynamo_sglang_agg.yaml +++ b/src/sflow/samples/self_contained/slurm/dynamo_sglang_agg.yaml @@ -115,7 +115,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 + value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -307,7 +307,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/dynamo_sglang_disagg.yaml b/src/sflow/samples/self_contained/slurm/dynamo_sglang_disagg.yaml index e551f75..ca16e44 100644 --- a/src/sflow/samples/self_contained/slurm/dynamo_sglang_disagg.yaml +++ b/src/sflow/samples/self_contained/slurm/dynamo_sglang_disagg.yaml @@ -170,7 +170,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.8.0 + value: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -424,7 +424,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/dynamo_trtllm_agg.yaml b/src/sflow/samples/self_contained/slurm/dynamo_trtllm_agg.yaml index 771b71b..000c9c6 100644 --- a/src/sflow/samples/self_contained/slurm/dynamo_trtllm_agg.yaml +++ b/src/sflow/samples/self_contained/slurm/dynamo_trtllm_agg.yaml @@ -111,7 +111,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 + value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -321,7 +321,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/dynamo_trtllm_disagg.yaml b/src/sflow/samples/self_contained/slurm/dynamo_trtllm_disagg.yaml index 1db803e..3e539a8 100644 --- a/src/sflow/samples/self_contained/slurm/dynamo_trtllm_disagg.yaml +++ b/src/sflow/samples/self_contained/slurm/dynamo_trtllm_disagg.yaml @@ -154,7 +154,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 + value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 # Network device information, change per your cluster or comment out if not needed # NCCL_SOCKET_IFNAME: @@ -372,10 +372,6 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 - name: decode_server operator: @@ -419,10 +415,6 @@ workflow: match_pattern: "Traceback (most recent call last)" match_count: 5 interval: 30 - retries: - count: 3 - interval: 30 - backoff: 2 - name: benchmark operator: @@ -447,7 +439,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/dynamo_vllm_agg.yaml b/src/sflow/samples/self_contained/slurm/dynamo_vllm_agg.yaml index fafb786..56f337e 100644 --- a/src/sflow/samples/self_contained/slurm/dynamo_vllm_agg.yaml +++ b/src/sflow/samples/self_contained/slurm/dynamo_vllm_agg.yaml @@ -337,7 +337,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/dynamo_vllm_disagg.yaml b/src/sflow/samples/self_contained/slurm/dynamo_vllm_disagg.yaml index 3f6c8ce..4eeadcf 100644 --- a/src/sflow/samples/self_contained/slurm/dynamo_vllm_disagg.yaml +++ b/src/sflow/samples/self_contained/slurm/dynamo_vllm_disagg.yaml @@ -304,6 +304,16 @@ workflow: - set -x - echo ${CUDA_VISIBLE_DEVICES} - export FIRST_CUDA_DEVICE=$(echo ${CUDA_VISIBLE_DEVICES} | cut -d',' -f1) + # Port derived from the first visible device, so two workers sharing a + # node never collide. This works because sflow does NOT hand + # NVIDIA_VISIBLE_DEVICES to an srun step: the container therefore sees the + # node's GPUs with HOST numbering intact, and the first entry of + # CUDA_VISIBLE_DEVICES is a real host index -- unique per worker. + # + # Keep it this way deliberately. When NVIDIA_VISIBLE_DEVICES was exported, + # the runtime carved and renumbered every worker's devices from 0, so both + # prefill and decode computed 5557 and the second to bind died with + # "Address already in use". This line is the canary that found it. - export VLLM_NIXL_SIDE_CHANNEL_PORT=$((5557 + ${FIRST_CUDA_DEVICE})) - export NODE_RANK=${SLURM_NODEID} - export FIRST_NODE_IP=$(echo ${SFLOW_TASK_ASSIGNED_NODE_IPS} | cut -d',' -f1) @@ -381,6 +391,16 @@ workflow: - set -x - echo ${CUDA_VISIBLE_DEVICES} - export FIRST_CUDA_DEVICE=$(echo ${CUDA_VISIBLE_DEVICES} | cut -d',' -f1) + # Port derived from the first visible device, so two workers sharing a + # node never collide. This works because sflow does NOT hand + # NVIDIA_VISIBLE_DEVICES to an srun step: the container therefore sees the + # node's GPUs with HOST numbering intact, and the first entry of + # CUDA_VISIBLE_DEVICES is a real host index -- unique per worker. + # + # Keep it this way deliberately. When NVIDIA_VISIBLE_DEVICES was exported, + # the runtime carved and renumbered every worker's devices from 0, so both + # prefill and decode computed 5557 and the second to bind died with + # "Address already in use". This line is the canary that found it. - export VLLM_NIXL_SIDE_CHANNEL_PORT=$((5557 + ${FIRST_CUDA_DEVICE})) - export NODE_RANK=${SLURM_NODEID} - export FIRST_NODE_IP=$(echo ${SFLOW_TASK_ASSIGNED_NODE_IPS} | cut -d',' -f1) @@ -469,7 +489,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/gpu_placement_matrix.yaml b/src/sflow/samples/self_contained/slurm/gpu_placement_matrix.yaml new file mode 100644 index 0000000..0ea0493 --- /dev/null +++ b/src/sflow/samples/self_contained/slurm/gpu_placement_matrix.yaml @@ -0,0 +1,366 @@ +# GPU placement regression matrix: every combination that can break the slice. +# +# sflow plans a per-task CUDA_VISIBLE_DEVICES and re-applies it inside the step. +# Four things independently decide what a task ends up holding: +# +# * container or not -- pyxis/enroot pass through only the task's devices and +# RENUMBER them from 0, so a task planned for 2,3 sees +# them as 0,1. A bare srun step sees the whole node. +# * slice offset -- a plan starting at 0 survives renumbering by accident, +# which is why only the high-offset task ever broke. +# * concurrency -- two tasks sharing a node must land on disjoint GPUs. +# * node count -- a multi-node plan is ONE node's slice, applied +# independently on every node. sflow guarantees the slice +# is uniform (gpus.count is divided by the node count and +# every node takes the same offsets), so the same flat +# plan must resolve correctly in each node's own step. +# +# Counting GPUs is not enough: a task can hold the right NUMBER of the WRONG +# devices and look perfectly healthy. So the first task records EACH node's +# index -> UUID map on bare metal, before anything is carved, and every later task +# proves by UUID that it holds the physical GPUs it was planned for -- per node, +# because a multi-node task's nodes each have their own map. +# +# Asserting rather than echoing is the point. The recipes that only printed their +# devices passed while holding nothing at all -- a decode server planned for 2,3 +# inside a 2-GPU container numbered 0,1 saw no GPU, and the first CUDA call died +# reporting an unsupported architecture. +version: "0.1" + +variables: + SLURM_ACCOUNT: + description: "SLURM account" + value: your_slurm_account + SLURM_PARTITION: + description: "SLURM partition" + value: your_slurm_partition + SLURM_TIMELIMIT: + description: "SLURM time limit" + value: "00:10:00" + SLURM_NODES: + description: >- + Number of Slurm nodes. Two, so the multi-node cases are real: a per-node + slice that is right on node 0 and wrong on node 1 is invisible at one node, + and that is the shape every multi-node server recipe uses. + # Typed, because boxed_multinode_high multiplies it. Untyped it is a STRING, + # and `${{ variables.SLURM_NODES * 2 }}` repeats it into "22" rather than + # doubling it -- valid YAML, valid expression, wrong number. + type: integer + value: 2 + GPUS_PER_NODE: + description: "GPUs per node" + value: 4 + PLACEMENT_IMAGE: + description: "Any GPU-enabled image; nvidia-smi is injected by the runtime." + value: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda12.5.0 + +artifacts: + # Declared so sample/bulk tests can override the standard sample artifact. + - name: LOCAL_MODEL_PATH + uri: fs:///path/to/your/model + + # One assertion, mounted at the same path on every node, instead of the same + # forty lines of shell pasted into each task. EXPECT_HOST is the only input. + - name: ASSERT_PLACEMENT + uri: file://assert_placement.sh + content: | + # Prove by UUID that THIS rank holds the physical GPUs it was planned for. + # Reads EXPECT_HOST (planned HOST indices, comma separated) from the env. + set -u + me=$(basename "$SFLOW_TASK_OUTPUT_DIR") + run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") + # Per NODE, not per task: every rank of a multi-node task runs this body, and + # each node has its own index -> UUID map. Reading node 0's map on node 1 + # would compare against the wrong hardware and "pass" for the wrong reason. + node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" + map=$(ls "$run_dir"/node_gpu_map*/map."$node".txt 2>/dev/null | head -1) + + echo "SFLOW_GPU_PROBE task=$me node=$node planned_host=$EXPECT_HOST CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" + nvidia-smi -L | sed "s/^/SFLOW_GPU_PROBE $node visible: /" + + if [ -z "$map" ]; then + echo "FAIL: no index -> UUID map recorded for node $node" >&2 + exit 1 + fi + # Are the planned ordinals even addressable in THIS namespace? A carved + # container renumbers from 0, so host ordinals can name nothing at all. + # An UNSET CUDA_VISIBLE_DEVICES is legal and means "every visible device", + # so `-i` must be omitted rather than passed empty -- and it must not be + # dereferenced bare under `set -u`. + if ! got=$(nvidia-smi ${CUDA_VISIBLE_DEVICES:+-i "$CUDA_VISIBLE_DEVICES"} \ + --query-gpu=uuid --format=csv,noheader); then + echo "FAIL: CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-} is not addressable on $node" >&2 + exit 1 + fi + # Per-node filenames: ranks of a multi-node task share one output dir, so a + # single gpus.txt would have them overwriting each other's evidence. + printf '%s\n' "$got" | sort > "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" + # And are they the RIGHT ones? Resolve the planned HOST indices through this + # node's bare-metal map; holding the correct count of the wrong devices is + # the failure mode a count-only check cannot see. + for i in $(echo "$EXPECT_HOST" | tr ',' ' '); do + grep "^$i," "$map" | cut -d, -f2 + done | sort > "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" + if ! diff -q "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" \ + "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" >/dev/null; then + echo "FAIL: $node planned host GPU(s) $EXPECT_HOST, but holds different devices" >&2 + echo " want: $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt")" >&2 + echo " got : $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt")" >&2 + exit 1 + fi + echo "OK: $node holds host GPU(s) $EXPECT_HOST, confirmed by UUID" + +backends: + - name: slurm_cluster + type: slurm + default: true + account: ${{ variables.SLURM_ACCOUNT }} + partition: ${{ variables.SLURM_PARTITION }} + time: ${{ variables.SLURM_TIMELIMIT }} + nodes: ${{ variables.SLURM_NODES }} + gpus_per_node: ${{ variables.GPUS_PER_NODE }} + +operators: + # One rank per node throughout: the assertion describes what a NODE holds, and + # extra ranks would each re-run it against the same devices for no new signal. + - name: bare + type: srun + ntasks_per_node: 1 + - name: boxed + type: srun + container_image: ${{ variables.PLACEMENT_IMAGE }} + ntasks_per_node: 1 + +workflow: + name: gpu_placement_matrix + tasks: + # Ground truth, taken bare metal with no `resources.gpus` so nothing is carved + # and no placement prelude runs. Runs on EVERY node (no `resources.nodes`), and + # each rank writes its own file, so the multi-node assertions below have a map + # for the node they actually landed on. Every assertion is relative to this. + - name: node_gpu_map + operator: bare + script: + - env | grep SLURM + - | + set -u + node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" + nvidia-smi --query-gpu=index,uuid --format=csv,noheader | tr -d ' ' \ + > "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" + sed "s/^/SFLOW_GPU_PROBE node_map $node: /" "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" + + # --- bare srun: the step sees the whole node, so the planned ordinals must + # --- survive untouched. Both run at once, on disjoint halves. + - name: bare_low + operator: bare + depends_on: [node_gpu_map] + resources: + gpus: + indices: [0, 1] + script: + - export EXPECT_HOST="0,1" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + - name: bare_high + operator: bare + depends_on: [node_gpu_map] + resources: + gpus: + indices: [2, 3] + script: + - export EXPECT_HOST="2,3" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # --- containerised: the runtime already carved and renumbered, so re-applying + # --- host ordinals here is what broke. boxed_high is the regression case. + - name: boxed_low + operator: boxed + depends_on: [bare_low, bare_high] + resources: + gpus: + indices: [0, 1] + script: + - export EXPECT_HOST="0,1" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + - name: boxed_high + operator: boxed + depends_on: [bare_low, bare_high] + resources: + gpus: + indices: [2, 3] + script: + - export EXPECT_HOST="2,3" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # A single GPU at a high offset: renumbering turns 3 into 0, so an off-by-one + # in the remap shows up here and nowhere else. + - name: boxed_one_high + operator: boxed + depends_on: [boxed_low, boxed_high] + resources: + gpus: + indices: [3] + script: + - export EXPECT_HOST="3" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # --- MULTI-NODE. `gpus.count` is the TOTAL over the task's nodes and every + # --- node takes the SAME offsets, so count/nodes must divide evenly. One flat + # --- plan therefore has to resolve correctly in each node's own step -- and + # --- each node proves it against its OWN bare-metal map, so a slice that is + # --- right on node 0 and wrong on node 1 fails here instead of shipping. + - name: bare_multinode + operator: bare + depends_on: [boxed_one_high] + resources: + # `nodes.count` is what forces the spread: `gpus.count` alone would fit on + # one node and the planner would place it there. count/nodes = 1 GPU per + # node, so the plan is "0" and both nodes must resolve it against their own + # hardware. This is the divide-the-total path every server recipe uses. + nodes: + count: ${{ variables.SLURM_NODES }} + gpus: + count: ${{ variables.SLURM_NODES }} + script: + - export EXPECT_HOST="0" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # The multi-node case that matters: containerised AND at a high offset, so + # every renumbering hazard is in play on more than one node at once. Pinned + # indices repeat on each node, which is the other multi-node planner path. + - name: boxed_multinode_high + operator: boxed + depends_on: [bare_multinode] + resources: + gpus: + # A TOTAL instead of a topology. With no `nodes:` here, `indices` is + # the per-node slice and the planner spreads the task over + # count / len(indices) nodes -- 4 / 2 = 2. This is the one route where + # the node count is an OUTPUT of the GPU ask rather than an input, which + # is how a recipe states things when it knows how many GPUs it needs but + # not how they are laid out. + count: ${{ variables.SLURM_NODES * 2 }} # 2 = len(indices) below + indices: [2, 3] + script: + - export EXPECT_HOST="2,3" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # --- AUTO-PLANNED (`gpus.count`, no `indices`). Everything above pins the + # --- devices by hand, so it proves the RE-APPLY path but never the PLANNER. + # --- These let sflow choose, which is what every real recipe does: the task + # --- cannot state an expected slice up front, so it asserts against the plan + # --- the prelude exported ($SFLOW_GPU_PLAN) -- still resolved to UUIDs + # --- through this node's bare-metal map, so "the planner picked something" + # --- and "the step actually holds it" remain separate claims. + # --- + # --- Chained rather than concurrent: 2x2 + 4x1 + 8 would need 16 GPUs and the + # --- allocation has 8. + + # Two replicas of two GPUs: the packing case. The planner must hand replica 0 + # and replica 1 DISJOINT pairs on the same node, which is the collision the + # whole slice mechanism exists to prevent. + - name: bare_count_2x2 + operator: bare + depends_on: [boxed_multinode_high] + replicas: + count: 2 + policy: parallel + resources: + gpus: + count: 2 + script: + - export EXPECT_HOST="$SFLOW_GPU_PLAN" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + - name: boxed_count_2x2 + operator: boxed + depends_on: [bare_count_2x2] + replicas: + count: 2 + policy: parallel + resources: + gpus: + count: 2 + script: + - export EXPECT_HOST="$SFLOW_GPU_PLAN" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # One GPU each, four ways: the finest packing, and the shape most likely to + # hand two replicas the same device if the planner ever miscounts. + - name: bare_count_1x4 + operator: bare + depends_on: [boxed_count_2x2] + replicas: + count: 4 + policy: parallel + resources: + gpus: + count: 1 + script: + - export EXPECT_HOST="$SFLOW_GPU_PLAN" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + - name: boxed_count_1x4 + operator: boxed + depends_on: [bare_count_1x4] + replicas: + count: 4 + policy: parallel + resources: + gpus: + count: 1 + script: + - export EXPECT_HOST="$SFLOW_GPU_PLAN" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # More GPUs than one node has, with no `nodes:` hint: the planner has to + # spread it itself and divide the total evenly, so every node runs the same + # flat plan against different physical cards. + - name: bare_count_8 + operator: bare + depends_on: [boxed_count_1x4] + resources: + gpus: + count: 8 + script: + - export EXPECT_HOST="$SFLOW_GPU_PLAN" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + - name: boxed_count_8 + operator: boxed + depends_on: [bare_count_8] + resources: + gpus: + count: 8 + script: + - export EXPECT_HOST="$SFLOW_GPU_PLAN" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # Concurrency: boxed_low and boxed_high held the node at the same time, so + # their devices must be disjoint. Sharing one is the collision this placement + # logic exists to prevent, and it is invisible from inside either task. + - name: verify_disjoint + operator: bare + # Terminal on purpose: the e2e checker treats this task's OK line as proof + # that the WHOLE matrix ran, so it has to sit downstream of the last case. + depends_on: [boxed_count_8] + resources: + nodes: + indices: [0] + script: + - | + set -u + run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") + low=$(ls "$run_dir"/boxed_low*/gpus.*.txt 2>/dev/null | head -1) + high=$(ls "$run_dir"/boxed_high*/gpus.*.txt 2>/dev/null | head -1) + if [ -z "$low" ] || [ -z "$high" ]; then + echo "FAIL: concurrent tasks left no gpus.*.txt to compare" >&2 + exit 1 + fi + shared=$(sort "$low" "$high" | uniq -d) + if [ -n "$shared" ]; then + echo "FAIL: concurrent tasks shared GPU(s): $shared" >&2 + exit 1 + fi + echo "OK: concurrent tasks held disjoint GPUs" diff --git a/src/sflow/samples/self_contained/slurm/infmax_v1_ds_r1.yaml b/src/sflow/samples/self_contained/slurm/infmax_v1_ds_r1.yaml index c235712..c5c8dd5 100644 --- a/src/sflow/samples/self_contained/slurm/infmax_v1_ds_r1.yaml +++ b/src/sflow/samples/self_contained/slurm/infmax_v1_ds_r1.yaml @@ -140,7 +140,7 @@ variables: value: python:3.12-slim DYNAMO_IMAGE: description: "Dynamo TRTLLM container image" - value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.0 + value: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0 artifacts: # file type artifacts will be generated by sflow with the content defined in this yaml file, accessible as `${{ artifacts.NAME.path }}` in task scripts @@ -420,7 +420,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/monitor_mixed.yaml b/src/sflow/samples/self_contained/slurm/monitor_mixed.yaml index 5e4b000..d16bcf0 100644 --- a/src/sflow/samples/self_contained/slurm/monitor_mixed.yaml +++ b/src/sflow/samples/self_contained/slurm/monitor_mixed.yaml @@ -2,13 +2,24 @@ version: "0.1" # All-in-one Slurm sample: one `sflow run` exercises several features at once, so # a single job on a real cluster verifies them together instead of one at a time. +# This is the broadest regression net in examples/ -- if only one recipe gets run +# on a new cluster, run this one. # -# Mixes three existing samples and adds monitoring: +# Mixes four existing samples and adds monitoring: # multi_backend.yaml -> two Slurm pools, two operators/images # auto_replica.yaml -> replicas + cross-task `${{ task.x[i]... }}` access # resource_release_after.yaml-> readiness probes + GPU reuse (release_after) +# gpu_placement_matrix.yaml -> GPU placement PROVEN by UUID: container vs bare, +# slice offset, concurrency, and multi-node # NEW -> monitor with a log-marker report window # +# The placement stage runs first and releases its GPUs before the servers start, so +# it adds assertions without adding nodes. It is the only part here that can catch a +# task holding the right NUMBER of the WRONG GPUs -- everything below merely echoes +# CUDA_VISIBLE_DEVICES, which is how a decode server planned for 2,3 passed while +# sitting in a 2-GPU container numbered 0,1 with no usable device at all. +# gpu_placement_matrix.yaml keeps the fuller matrix as a standalone 2-node recipe. +# # What to check after a run (//sflow_monitor/): # # MARKER-CLIPPED -- sflow_monitor/windowed/ (each with window.json): @@ -96,6 +107,54 @@ artifacts: - name: LOCAL_MODEL_PATH uri: fs:///path/to/your/model + # Placement proof, mounted at the same path on every node. Kept in a file:// + # artifact rather than pasted into each task: EXPECT_HOST is the only input, so + # one copy serves every case below. + - name: ASSERT_PLACEMENT + uri: file://assert_placement.sh + content: | + # Prove by UUID that THIS rank holds the physical GPUs it was planned for. + # Reads EXPECT_HOST (planned HOST indices, comma separated) from the env. + set -u + me=$(basename "$SFLOW_TASK_OUTPUT_DIR") + run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") + # Per NODE, not per task: every rank of a multi-node task runs this body, and + # each node has its own index -> UUID map. Reading node 0's map on node 1 + # would compare against the wrong hardware and "pass" for the wrong reason. + node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" + map=$(ls "$run_dir"/node_gpu_map*/map."$node".txt 2>/dev/null | head -1) + + echo "SFLOW_GPU_PROBE task=$me node=$node planned_host=$EXPECT_HOST CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" + nvidia-smi -L | sed "s/^/SFLOW_GPU_PROBE $node visible: /" + + if [ -z "$map" ]; then + echo "FAIL: no index -> UUID map recorded for node $node" >&2 + exit 1 + fi + # Are the planned ordinals even addressable in THIS namespace? A carved + # container renumbers from 0, so host ordinals can name nothing at all. + if ! got=$(nvidia-smi -i "$CUDA_VISIBLE_DEVICES" --query-gpu=uuid --format=csv,noheader); then + echo "FAIL: CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES is not addressable on $node" >&2 + exit 1 + fi + # Per-node filenames: ranks of a multi-node task share one output dir, so a + # single gpus.txt would have them overwriting each other's evidence. + printf '%s\n' "$got" | sort > "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" + # And are they the RIGHT ones? Resolve the planned HOST indices through this + # node's bare-metal map; holding the correct count of the wrong devices is + # the failure mode a count-only check cannot see. + for i in $(echo "$EXPECT_HOST" | tr ',' ' '); do + grep "^$i," "$map" | cut -d, -f2 + done | sort > "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" + if ! diff -q "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt" \ + "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt" >/dev/null; then + echo "FAIL: $node planned host GPU(s) $EXPECT_HOST, but holds different devices" >&2 + echo " want: $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/want.$node.txt")" >&2 + echo " got : $(tr '\n' ' ' < "$SFLOW_TASK_OUTPUT_DIR/gpus.$node.txt")" >&2 + exit 1 + fi + echo "OK: $node holds host GPU(s) $EXPECT_HOST, confirmed by UUID" + # Two independent pools: each allocates its own Slurm job and its own nodes. backends: - name: gpu_pool @@ -123,6 +182,11 @@ operators: type: srun container_image: ${{ variables.CPU_IMAGE }} ntasks_per_node: 1 + # No container, so the placement ground truth is taken on the raw node before any + # runtime has renumbered anything. Reuses the host driver's nvidia-smi. + - name: worker_bare + type: srun + ntasks_per_node: 1 workflow: name: monitor_mixed_slurm @@ -135,11 +199,112 @@ workflow: enabled: true tasks: + # --------------------------------------------------------------------------- + # -- gpu_placement_matrix: PROVE the planned slice, by UUID, before anything + # -- else runs. Counting GPUs is not enough -- a task can hold the right NUMBER + # -- of the WRONG devices and look perfectly healthy, which is how a decode + # -- server planned for 2,3 ran in a 2-GPU container numbered 0,1 and died on + # -- its first CUDA call. The servers below only ECHO their devices, so they + # -- cannot catch that; these tasks can. + # -- + # -- This whole stage completes (and releases its GPUs) before env_check, so it + # -- costs no extra nodes and cannot collide with the servers' reservations. + # --------------------------------------------------------------------------- + + # Ground truth, bare metal, nothing carved. Runs on EVERY node (no + # `resources.nodes`), one file each, so the multi-node case below can check the + # node it actually landed on. + - name: node_gpu_map + operator: worker_bare + script: + - | + set -u + node="${SFLOW_TASK_NODE_NAME:-$(hostname -s)}" + nvidia-smi --query-gpu=index,uuid --format=csv,noheader | tr -d ' ' \ + > "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" + sed "s/^/SFLOW_GPU_PROBE node_map $node: /" "$SFLOW_TASK_OUTPUT_DIR/map.$node.txt" + + # Bare srun sees the whole node, so a non-zero offset must survive untouched. + - name: placement_bare_high + operator: worker_bare + depends_on: [node_gpu_map] + resources: + gpus: + indices: [2, 3] + script: + - export EXPECT_HOST="2,3" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # Containerised, low and high offset, CONCURRENTLY on one node. A plan starting + # at 0 survives container renumbering by accident, which is why only the + # high-offset task ever broke -- both sides are needed to see it. + - name: placement_boxed_low + operator: worker_gpu + depends_on: [placement_bare_high] + resources: + gpus: + indices: [0, 1] + script: + - export EXPECT_HOST="0,1" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + - name: placement_boxed_high + operator: worker_gpu + depends_on: [placement_bare_high] + resources: + gpus: + indices: [2, 3] + script: + - export EXPECT_HOST="2,3" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # MULTI-NODE, containerised, high offset -- every renumbering hazard in play on + # more than one node at once. A multi-node plan is ONE node's slice applied + # independently per node, so a slice that is right on node 0 and wrong on node 1 + # is invisible at a single node. Pinned indices repeat on each node. + - name: placement_multinode_high + operator: worker_gpu + depends_on: [placement_boxed_low, placement_boxed_high] + resources: + nodes: + count: 2 + gpus: + indices: [2, 3] + script: + - export EXPECT_HOST="2,3" + - bash ${{ artifacts.ASSERT_PLACEMENT.path }} + + # Collisions are invisible from inside either task: boxed_low and boxed_high + # held one node at the same time, so their devices must be disjoint. + - name: placement_disjoint + operator: worker_bare + depends_on: [placement_multinode_high] + resources: + nodes: + indices: [0] + script: + - | + set -u + run_dir=$(dirname "$SFLOW_TASK_OUTPUT_DIR") + low=$(ls "$run_dir"/placement_boxed_low*/gpus.*.txt 2>/dev/null | head -1) + high=$(ls "$run_dir"/placement_boxed_high*/gpus.*.txt 2>/dev/null | head -1) + if [ -z "$low" ] || [ -z "$high" ]; then + echo "FAIL: concurrent tasks left no gpus.*.txt to compare" >&2 + exit 1 + fi + shared=$(sort "$low" "$high" | uniq -d) + if [ -n "$shared" ]; then + echo "FAIL: concurrent tasks shared GPU(s): $shared" >&2 + exit 1 + fi + echo "OK: concurrent tasks held disjoint GPUs" + # -- resource_release_after: grabs every GPU on a node, then frees them on # completion (no readiness probe -> sflow infers task_completion), so the # servers below can reuse them. - name: env_check operator: worker_gpu + depends_on: [placement_disjoint] resources: gpus: count: ${{ variables.GPUS_PER_NODE }} diff --git a/src/sflow/samples/self_contained/slurm/sglang_server_client.yaml b/src/sflow/samples/self_contained/slurm/sglang_server_client.yaml index 7409814..0594453 100644 --- a/src/sflow/samples/self_contained/slurm/sglang_server_client.yaml +++ b/src/sflow/samples/self_contained/slurm/sglang_server_client.yaml @@ -176,7 +176,6 @@ workflow: --extra-inputs "max_tokens:${{ variables.OSL }}" --extra-inputs "min_tokens:${{ variables.OSL }}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/src/sflow/samples/self_contained/slurm/trtllm_serve_disagg.yaml b/src/sflow/samples/self_contained/slurm/trtllm_serve_disagg.yaml index 37e870e..478244c 100644 --- a/src/sflow/samples/self_contained/slurm/trtllm_serve_disagg.yaml +++ b/src/sflow/samples/self_contained/slurm/trtllm_serve_disagg.yaml @@ -328,6 +328,15 @@ workflow: script: - set -x - echo ${CUDA_VISIBLE_DEVICES} + # GPU placement probe. sflow plans a per-task slice and re-applies it + # inside the step; this prints what the container ACTUALLY has next to + # what was planned, which is the only way to tell the two failure modes + # apart: if the planned ordinals are absent from `nvidia-smi -L`, the + # slice never reached the container (carved and renumbered by the + # runtime); if they are present, placement worked and the launcher chose + # the wrong device. Cheap, and it makes every future triage one grep. + - echo "SFLOW_GPU_PROBE task=${SLURM_JOB_NAME:-?} procid=${SLURM_PROCID:-?} localid=${SLURM_LOCALID:-?} planned_CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" + - nvidia-smi -L 2>&1 | sed "s/^/SFLOW_GPU_PROBE visible: /" || true - export TLLM_LOG_LEVEL=INFO - export TRTLLM_SERVER_DISABLE_GC=1 - export TRTLLM_WORKER_DISABLE_GC=1 @@ -374,6 +383,15 @@ workflow: script: - set -x - echo ${CUDA_VISIBLE_DEVICES} + # GPU placement probe. sflow plans a per-task slice and re-applies it + # inside the step; this prints what the container ACTUALLY has next to + # what was planned, which is the only way to tell the two failure modes + # apart: if the planned ordinals are absent from `nvidia-smi -L`, the + # slice never reached the container (carved and renumbered by the + # runtime); if they are present, placement worked and the launcher chose + # the wrong device. Cheap, and it makes every future triage one grep. + - echo "SFLOW_GPU_PROBE task=${SLURM_JOB_NAME:-?} procid=${SLURM_PROCID:-?} localid=${SLURM_LOCALID:-?} planned_CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" + - nvidia-smi -L 2>&1 | sed "s/^/SFLOW_GPU_PROBE visible: /" || true - export TLLM_LOG_LEVEL=INFO - export TRTLLM_SERVER_DISABLE_GC=1 - export TRTLLM_WORKER_DISABLE_GC=1 diff --git a/src/sflow/skills/writing-sflow-yaml/schema-reference.md b/src/sflow/skills/writing-sflow-yaml/schema-reference.md index 6b2bea1..e696994 100644 --- a/src/sflow/skills/writing-sflow-yaml/schema-reference.md +++ b/src/sflow/skills/writing-sflow-yaml/schema-reference.md @@ -331,6 +331,7 @@ backends: # dra: { gpu_device_class: gpu.nvidia.com, rdma_device_class: rdma.nvidia.com } # compute_domain: { channel: auto } # Multi-Node NVLink (IMEX); top-level, not under dra # probe_pod_image: curlimages/curl:latest + # collect_node_local_output: true # default true; false = NO sflow collect machinery in the pod at all # collect_max_file_size: 10Mi # cap for syncing pod output back to driver (0 disables) # collect_grace_seconds: 120 # grace window (s) for copying node-local outputs back # reservation: { placeholder_image: my-mirror/bash:5 } # air-gapped placeholder-pod image @@ -578,6 +579,8 @@ The task then carries only its launch line (no keypair/hostfile/sshd/wait/`-x`): | `omp_num_threads` | int\|null | `8` | Per-rank `OMP_NUM_THREADS` (pod env, forwarded to ranks). Caps OpenMP so co-located ranks don't exhaust pthreads at model load; a recipe `export OMP_NUM_THREADS=...` overrides it; `null`/`0` disables | | `worker_setup_timeout_seconds` | int/expr | `900` | Per-node setup budget (image apt-install + weight staging) before a worker's readiness probe reaps it (operator route). Rendered as a probe with a fixed 5s poll and `failureThreshold = ceil(timeout/5)`. Raise for a large first-time weight download over slow storage | | `launcher_discovery_timeout` | int/expr | `600` | Seconds to wait for the mpi-operator controller to create the launcher pod after the MPIJob is applied (operator route) | +| `cpu_bind` | enum | `core` | `core` / `numa` / `none`. Per-rank CPU binding, injected **only when several ranks share a pod**, and never over a binding the recipe already passes. `core` gives each rank an isolated core slice — the tightest cap on the LLVM/OpenMP thread pools `OMP_NUM_THREADS` alone doesn't reach; `numa` binds one rank per NUMA domain (only partitions when the cpuset spans >1 domain); `none` injects nothing | +| `cpu_bind_cores_per_rank` | int | `8` | Upper bound on the cores bound to each rank under `cpu_bind: core`. Launch-time value is `min(cores-in-cpuset / ranks-per-pod, this)`, so a small cpuset still gets a smaller slice; if the cpuset has fewer cores than ranks the binding is skipped rather than failing the launch. `0` = uncapped | **Env forwarding.** `mpirun` over SSH gives remote ranks a *bare* environment, so env is forwarded with `-x`. sflow forwards a built-in transport/system set (`NCCL_`, diff --git a/src/sflow/utils/gpu.py b/src/sflow/utils/gpu.py index 329ba42..355b4a6 100644 --- a/src/sflow/utils/gpu.py +++ b/src/sflow/utils/gpu.py @@ -8,7 +8,10 @@ # Written by a job step that picks its own devices (slurm), read back by run # reporting. Lives here, next to its only reader, so the producer imports it. -GPU_MARKER_FILE = ".sflow_gpus" +# Deliberately NOT dot-prefixed: this is the placement record a human reads when +# a run looks mis-placed, and GitLab's artifact browser hides dot-files entirely +# (they ship in the zip but cannot be clicked). +GPU_MARKER_FILE = "sflow_gpus.log" def count_device_tokens(cuda_visible: str | None) -> int: @@ -40,6 +43,33 @@ def count_visible_devices(cuda_visible: str | None) -> int: ) +def task_gpu_record(task: Any) -> dict[str, str]: + """Parse the placement record a Slurm step leaves in :data:`GPU_MARKER_FILE`. + + Line 1 is the device list the step ended up using; the rest is ``key=value`` + (node, the branch taken, planned host indices, planned UUIDs) plus repeated + ``visible=``/``selected=`` lines. Returns {} when there is no record. + """ + envs = getattr(task, "envs", None) or {} + out_dir = envs.get("SFLOW_TASK_OUTPUT_DIR") + if not out_dir: + return {} + try: + text = (Path(out_dir) / GPU_MARKER_FILE).read_text() + except OSError: + return {} + lines = text.splitlines() + if not lines: + return {} + record: dict[str, str] = {"devices": lines[0].strip()} + for line in lines[1:]: + key, sep, value = line.partition("=") + # visible=/selected= repeat; the scalars are what callers need. + if sep and key not in ("visible", "selected"): + record[key] = value.strip() + return record + + def task_gpu_indices(task: Any) -> list[int]: """The **physical** GPU indices a task ran on, for run reporting. @@ -81,13 +111,25 @@ def task_gpu_indices(task: Any) -> list[int]: # node 0's devices for every node would be a confident wrong answer -- the plan, # which is uniform across nodes by construction, is the honest one there. if out_dir and len(getattr(task, "assigned_nodes", None) or []) <= 1: - try: - reported = (Path(out_dir) / GPU_MARKER_FILE).read_text() - except OSError: - reported = "" + record = task_gpu_record(task) + if record.get("action") == "verified": + # The step PROVED, by UUID, that it holds exactly the planned cards -- + # it exits 97 otherwise -- so the planned HOST indices are the physical + # ones, whatever the step's own numbering happened to be. + # + # Line 1 must not be used here: inside a carved container it is the + # container's numbering (a task planned for host 2,3 records 0,1), and + # reporting that as physical put every containerised task on the wrong + # card in the summary and made the monitor sample the wrong GPUs. + indices = parse_cuda_visible_devices(record.get("planned_host_indices", "")) + if indices: + return indices + # No verified record (older marker, or the driver could not probe the + # topology): line 1 is the best available answer, and under that path the + # step selected host ordinals, so it means what it used to mean. # An unparseable marker (e.g. CUDA UUID form) must fall through, not drop the # task out of run reporting entirely. - indices = parse_cuda_visible_devices(reported.strip()) + indices = parse_cuda_visible_devices(record.get("devices", "")) if indices: return indices return planned_gpu_indices(task) diff --git a/tests/conftest.py b/tests/conftest.py index ecedbfa..f1e01b4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,11 +6,6 @@ import pytest -@pytest.fixture -def mock_config(): - return {} - - @pytest.fixture def image_warnings(): """Collect the warnings emitted for unrecognised container image references. diff --git a/tests/e2e_tests/sample_test.sh b/tests/e2e_tests/sample_test.sh index aa0beb6..739c0d9 100755 --- a/tests/e2e_tests/sample_test.sh +++ b/tests/e2e_tests/sample_test.sh @@ -3,7 +3,7 @@ # set -x usage() { - echo "Usage: $0 -p -A -m [-G ] [-t s|m|inf|a|smoke|min] [--submit] [--check JOB_IDS] [-- ]" + echo "Usage: $0 -p -A -m [-G ] [-t s|m|inf|a|smoke|min|one] [--submit] [--check JOB_IDS] [-- ]" echo "" echo " -t s Self-contained examples only (--bulk-submit examples/self_contained/slurm/)" echo " -t m Modular examples only (--bulk-input modular/inference_x_v2/bulk_input.csv)" @@ -11,6 +11,15 @@ usage() { echo " -t a Both single and multi (default)" echo " -t smoke Curated Slurm smoke subset with broad coverage" echo " -t min Minimal representative set (one job per validation type)" + echo " -t one EXACTLY ONE Slurm job -- plumbing smoke, not coverage." + echo " Use it to prove a CI/cluster path end to end (submit -> run ->" + echo " sflow_output -> summary) before spending nodes on min/smoke." + echo "" + echo " SFLOW_E2E_RECIPE_CLASS=workload|sanity|all (env, default all)" + echo " Which HALF of the suite to submit. 'workload' is the" + echo " dynamo/trtllm/vllm/sglang/aiperf/infmax recipes (real servers," + echo " model loads, big pulls); 'sanity' is everything else and needs" + echo " no model at all. The two halves run on different clusters." echo "" echo " --check JOB_IDS Skip submission, only check results" echo " Accepts: comma-separated IDs and/or [START:END] ranges" @@ -148,6 +157,168 @@ colon_task_log_has_markers() { ' sh {} + 2>/dev/null } +workflow_summary_ok() { + # Every indicator below comes from a benchmark or client log, so a recipe that + # ships neither (monitor_mixed, gpu_indices, ...) scored FAIL however green it + # was. sflow already writes the authoritative verdict: sflow_summary.log says + # `Status : COMPLETED` only when every task reached COMPLETED/READY + # (core/execution_summary.py::_infer_status), and FAILED/CANCELLED/TIMEOUT on + # any other outcome. Judge those runs by that instead of per-recipe allowlists. + local dir="$1" + [ -n "$dir" ] || return 1 + local summary + summary=$(find "$dir" -maxdepth 2 -type f -name 'sflow_summary.log' 2>/dev/null | head -1) + [ -n "$summary" ] || return 1 + grep -Eq '^Status[[:space:]]*:[[:space:]]*COMPLETED[[:space:]]*$' "$summary" +} + +aiperf_tally_ok() { # -> 0 every aiperf run benchmarked, 1 one did not, 2 no aiperf here + # A benchmark that measured NOTHING is the one failure this suite could not see. + # aiperf 0.3.0 exits 0 even when every single request failed, so the task rc is + # 0, sflow's own `Status : COMPLETED` is green, and the run is scored PASS while + # its CSV holds nothing but `Error Request Count`. + # + # Not hypothetical: dynamo >= 1.3.0 dropped `ignore_eos` from its (strictly + # deserialized) NvExt struct, so one stale `--extra-inputs` 400'd all 1024 + # requests in six workflows -- and this suite printed "11/11 jobs passed". + # + # aiperf states the outcome itself, and it is unambiguous: + # Processed 1024 valid requests and 0 errors (1024 total). <- benchmarked + # Processed 0 valid requests and 1024 errors (1024 total). <- measured nothing + # + # Both markers this replaces were wrong. "0 valid" was read as a SUCCESS marker + # when it is precisely what a total failure prints -- that alone scored the six + # dead runs green -- and unanchored "0 errors" also matches "10 errors" and + # "1000 errors", so a mostly-failed run passed too. + # + # Returns 2, not 0, when no aiperf ran: "no evidence" is a different answer from + # "good evidence", and only the caller knows whether this recipe owed any. Same + # lesson as workload_placement_ok() -- a checker that passes on an empty + # directory scores a workflow that died before it started as a success. + local dir="$1" valid errors seen=0 bad=0 + [ -n "$dir" ] && [ -d "$dir" ] || return 2 + # grep -o pins the field positions, so the split below cannot drift: + # Processed valid requests and errors + while read -r _ valid _ _ _ errors _; do + seen=$((seen + 1)) + [ "${valid:-0}" -gt 0 ] && [ "${errors:-1}" -eq 0 ] && continue + bad=$((bad + 1)) + echo " AIPERF MEASURED NOTHING: ${valid:-?} valid / ${errors:-?} errors under $dir" >&2 + done < <(find "$dir" -type f -name '*.log' \ + -exec grep -hoE 'Processed [0-9]+ valid requests and [0-9]+ errors' {} + 2>/dev/null) + [ "$seen" -gt 0 ] || return 2 + [ "$bad" -eq 0 ] +} + +serving_tally_ok() { # -> 0 every benchmark_serving run completed, 1 one did not, 2 none here + # The benchmark_serving.py (InferenceX) half of the same question aiperf_tally_ok + # asks. The modular dynamo_benchmark rows drive this instead of aiperf, so + # without it they keep the old, far weaker gate. + # + # What the old markers did: PASS on `grep -l "Successful requests:"` -- the mere + # PRESENCE of the string -- and FAIL only on `Successful requests:\s+0\s*$`. + # So a run that completed 3 of 512 requests scored a clean PASS, and any + # zero-count formatted differently (trailing text, padding) slipped the FAIL too. + # + # benchmark_serving.py prints its own summary, and the run's own command line is + # in the same log, so the two can be cross-checked: + # python3 ... benchmark_serving.py ... --num-prompts 128 ... + # ============ Serving Benchmark Result ============ + # Successful requests: 128 + # Output token throughput (tok/s): 4132.77 + # + # A real benchmark therefore owes three things: it succeeded at all (got > 0), it + # generated tokens (throughput > 0 -- a run can "succeed" 512 times with empty + # responses, which is the same measured-nothing shape ignore_eos produced), and + # it completed the work it was ASKED for (got == --num-prompts). Every healthy + # run in CI matches exactly: 16/16, 32/32, 48/48, 128/128, 256/256, 512/512. + # + # want == 0 means no command line was captured in this log; the cross-check is + # then skipped rather than guessed at. Returns 2 for "no benchmark_serving here" + # for the same reason aiperf_tally_ok does -- no evidence is not good evidence. + local dir="$1" f seen=0 bad=0 + [ -n "$dir" ] && [ -d "$dir" ] || return 2 + while IFS= read -r f; do + seen=$((seen + 1)) + awk -v src="$f" ' + match($0, /--num-prompts[= ]+[0-9]+/) { + s = substr($0, RSTART, RLENGTH); gsub(/[^0-9]/, "", s); want = s + 0 + } + /Successful requests:/ { got = $NF + 0 } + /Output token throughput \(tok\/s\):/ { thpt = $NF + 0 } + END { + if (got > 0 && thpt > 0 && (want == 0 || got == want)) exit 0 + printf " BENCHMARK INCOMPLETE: %d/%d requests succeeded, %g tok/s -- %s\n", \ + got, want, thpt, src > "/dev/stderr" + exit 1 + } + ' "$f" || bad=$((bad + 1)) + done < <(find "$dir" -type f -name '*.log' \ + -exec grep -l 'Serving Benchmark Result' {} + 2>/dev/null) + [ "$seen" -gt 0 ] || return 2 + [ "$bad" -eq 0 ] +} + +recipe_is_client_only() { # -> 0 when the recipe starts no server of its own + # aiperf_template is a TEMPLATE: ONE CPU-only client task aimed at + # ${HEAD_NODE_IP}:8000, an endpoint it deliberately does NOT start -- the reader + # is meant to point it at a server they already run. Played standalone in CI + # nothing is listening, every request is ConnectionRefused, and aiperf cannot + # benchmark. That is the recipe working as designed, not a regression. + # + # So the claim here is deliberately narrow. Such a run is still expected to + # COMPLETE -- sflow `Status : COMPLETED`, benchmark task exit=0 -- it simply + # owes no metrics. Every OTHER recipe in the workload half still owes a real + # benchmark, which is the whole point of aiperf_tally_ok(). + # + # Structural rather than a name allowlist: a workflow declaring a single task + # cannot have started the server it benchmarks. aiperf_template declares 1; + # every serving recipe here declares 6-7 (servers + frontend + benchmark), and + # the modular compositions more. The recipe is copied into the run directory, + # so this reads what actually ran rather than what is on disk now. + local dir="$1" total=0 n yml + [ -n "$dir" ] && [ -d "$dir" ] || return 1 + for yml in "$dir"/*.y*ml; do + [ -f "$yml" ] || continue + n=$(sed -n '/^ tasks:/,$p' "$yml" | grep -cE '^ - name:') + total=$((total + n)) + done + [ "$total" -eq 1 ] +} + +gpu_placement_run_ok() { + # The placement matrix ships no benchmark: its result IS its assertions. Every + # task resolves its planned HOST indices through a per-node bare-metal + # index -> UUID map and exits non-zero on a mismatch, and verify_disjoint sits + # downstream of all of them -- so its OK line can only appear when every + # assertion passed on every node. + # + # Only per-task logs are searched (mindepth 2): sflow.log echoes each task's + # script verbatim, FAIL: branches included, so a recursive grep would match + # text that never ran. + local dir="$1" + [ -n "$dir" ] || return 1 + find "$dir" -mindepth 2 -maxdepth 2 -type f -name '*.log' \ + -exec grep -lF "OK: concurrent tasks held disjoint GPUs" {} + 2>/dev/null | + grep -q . || return 1 + # And the multi-node cases must actually have spanned nodes. A one-node + # allocation degenerates them into duplicates of the single-node cases, which + # still pass -- silently dropping the coverage they exist for. Each assertion + # names the node it proved, so two distinct names is the proof. + local task nodes found=0 + for task in "$dir"/*multinode*/; do + [ -d "$task" ] || continue + found=1 + # Per multi-node TASK. A union over every task in the run is not the same + # claim: two single-node tasks landing on different nodes satisfy it while + # both multi-node cases sat on one. + nodes=$(find "$task" -maxdepth 1 -type f -name '*.log' \ + -exec grep -hoE 'OK: [^ ]+ holds host GPU' {} + 2>/dev/null | sort -u | wc -l) + [ "${nodes:-0}" -ge 2 ] || return 1 + done + [ "$found" = 1 ] +} + is_multi_backend_result() { local jid="$1" local mbid @@ -202,11 +373,50 @@ if [ -z "$PARTITION" ] || [ -z "$ACCOUNT" ] || [ -z "$MODEL_PATH" ]; then usage fi -if [ "$TEST_TYPE" != "s" ] && [ "$TEST_TYPE" != "m" ] && [ "$TEST_TYPE" != "inf" ] && [ "$TEST_TYPE" != "a" ] && [ "$TEST_TYPE" != "smoke" ] && [ "$TEST_TYPE" != "min" ]; then - echo "ERROR: -t must be 's', 'm', 'inf', 'a', 'smoke', or 'min', got '$TEST_TYPE'" +if [ "$TEST_TYPE" != "s" ] && [ "$TEST_TYPE" != "m" ] && [ "$TEST_TYPE" != "inf" ] && [ "$TEST_TYPE" != "a" ] && [ "$TEST_TYPE" != "smoke" ] && [ "$TEST_TYPE" != "min" ] && [ "$TEST_TYPE" != "one" ]; then + echo "ERROR: -t must be 's', 'm', 'inf', 'a', 'smoke', 'min', or 'one', got '$TEST_TYPE'" usage fi +# Which HALF of the suite this run submits. A class, not a new -t mode, so it +# composes with every existing mode instead of multiplying them. +# +# workload -- dynamo / trtllm / vllm / sglang / infmax / aiperf. Real servers, +# real model loads, multi-GB image pulls, many GPU-hours. Belongs on +# the cluster that has the quota for it. +# sanity -- everything else: GPU placement, replicas, resource release, the +# monitor and multi-backend recipes. No framework and no model -- +# they only DECLARE LocalModelPath so the harness's `-a` override is +# accepted, and never read it -- so this half runs on any cluster +# with GPUs and is cheap enough to play constantly. +# all -- both. The default, so a manual or local run is unchanged. +RECIPE_CLASS="${SFLOW_E2E_RECIPE_CLASS:-all}" +if [ "$RECIPE_CLASS" != "all" ] && [ "$RECIPE_CLASS" != "workload" ] && [ "$RECIPE_CLASS" != "sanity" ]; then + echo "ERROR: SFLOW_E2E_RECIPE_CLASS must be 'all', 'workload' or 'sanity', got '$RECIPE_CLASS'" + exit 1 +fi + +is_workload_recipe() { + # Named by the framework they drive, which is exactly the line the split is + # drawn on. Everything else is a functionality check. + case "$(basename "$1")" in + dynamo_*|trtllm_*|sglang_*|vllm_*|infmax_*|aiperf_*) return 0 ;; + esac + return 1 +} + +recipe_in_class() { + # -t one is a single named plumbing smoke (the GPU placement matrix), and its + # job is to prove THIS cluster's path end to end. Both halves want that, so it + # is never filtered out. + [ "$TEST_TYPE" = "one" ] && return 0 + case "$RECIPE_CLASS" in + workload) is_workload_recipe "$1" ;; + sanity) ! is_workload_recipe "$1" ;; + *) return 0 ;; + esac +} + if [ ${#EXTRA_BATCH_ARGS[@]} -gt 0 ]; then echo "Extra sflow batch args: ${EXTRA_BATCH_ARGS[*]}" fi @@ -485,6 +695,10 @@ submit_colon_task_script_e2e() { if [ "$TEST_TYPE" != "s" ] && [ "$TEST_TYPE" != "a" ] && [ "$TEST_TYPE" != "smoke" ]; then return fi + # Functionality check, no framework: it belongs to the sanity half -- which + # runs via `sflow run` (run_sanity_recipes_with_sflow_run), not batch. So this + # batch-based path is only for the combined default. + [ "$RECIPE_CLASS" != "all" ] && return local colon_dir="${SFLOW_COLON_SCRIPT_OUTPUT_DIR:-$E2E_OUTPUT_DIR/colon_in_task_script}" local colon_sbatch="$colon_dir/colon_in_task_script.sh" @@ -499,6 +713,23 @@ submit_colon_task_script_e2e() { # the shared E2E_BATCH_EXTRA_ARGS), so enable the workflow monitor explicitly -- # otherwise this job would be the one workflow without an sflow_monitor.log and # would trip the independent monitor-coverage gate below. + # Built as an array rather than a nested ${a:-${b:-...}} default: with no nodes + # to exclude that expression collapses to a bare `--exclude=`, which is not the + # same as passing nothing. + COLON_EXTRA_ARGS=() + if [ -n "${SFLOW_COLON_SCRIPT_EXTRA_ARGS:-}" ]; then + COLON_EXTRA_ARGS=(-e "$SFLOW_COLON_SCRIPT_EXTRA_ARGS") + elif [ -n "${SLURM_E2E_EXCLUDE_NODES:-}" ]; then + COLON_EXTRA_ARGS=(-e "--exclude=$SLURM_E2E_EXCLUDE_NODES") + fi + COLON_SEGMENT="${SLURM_E2E_SEGMENT:-}" + if [ "$COLON_SEGMENT" = "auto" ]; then + COLON_SEGMENT='${{SLURM_NODES}}' + fi + if [ -n "$COLON_SEGMENT" ]; then + COLON_EXTRA_ARGS+=(-e "--segment=$COLON_SEGMENT") + fi + colon_output=$(sflow batch -f "$SFLOW_COLON_SCRIPT_FIXTURE" \ -p "$PARTITION" -A "$ACCOUNT" --log-level warn \ "${BATCH_WORKSPACE_ARGS[@]}" \ @@ -506,7 +737,7 @@ submit_colon_task_script_e2e() { --output-dir "$colon_dir" \ --job-name "colon_in_task_script" \ --enable-workflow-monitor \ - -e "${SFLOW_COLON_SCRIPT_EXTRA_ARGS:---exclude=${SLURM_E2E_EXCLUDE_NODES:-gb-nvl-137-compute02,gb-nvl-137-compute14}}" \ + "${COLON_EXTRA_ARGS[@]}" \ -o "$colon_sbatch" \ $SUBMIT 2>&1) colon_status=$? @@ -530,6 +761,463 @@ submit_colon_task_script_e2e() { COLON_JOB_IDS+=("$colon_job_id") } +# Strip a suite-wide `-e --segment=...` for ONE `sflow batch` call, into +# STRIPPED_SEGMENT_ARGS. That value is an sflow expression sized to a recipe's own +# SLURM_NODES, so a config WITHOUT that variable passes it to sbatch VERBATIM as +# "#SBATCH --segment=${{SLURM_NODES}}". Both multi-backend recipes are in that +# position (they fix their backends' node counts instead), and a single salloc-wide +# segment could not be right for two differently sized backends anyway. +# +# One copy, two callers on purpose: a second hand-written twin of this predicate +# drifting is how the verbatim expression reached sbatch to begin with. +strip_suite_segment_args() { + STRIPPED_SEGMENT_ARGS=() + local i=0 + local n=${#EXTRA_BATCH_ARGS[@]} + while [ "$i" -lt "$n" ]; do + local arg="${EXTRA_BATCH_ARGS[$i]}" + local next="" + if [ $((i + 1)) -lt "$n" ]; then + next="${EXTRA_BATCH_ARGS[$((i + 1))]}" + fi + case "$arg:$next" in + "-e:--segment="*) + i=$((i + 2)) + continue + ;; + esac + STRIPPED_SEGMENT_ARGS+=("$arg") + i=$((i + 1)) + done +} + +# The second partition every two-backend recipe needs. Defaults to the partition +# this run was given: hardcoded names are cluster-specific and simply do not exist +# elsewhere -- the stale genesisq / gamoraq defaults meant every submission on a new +# cluster died with "no Slurm allocation granted". Two DIFFERENT partitions are +# better coverage, so CI sets SLURM_E2E_PARTITION_B (ptyche: backfill) and an +# operator can still override per recipe. +E2E_PARTITION_B="${SLURM_E2E_PARTITION_B:-$PARTITION}" + +# ============================================================================= +# The SANITY half: one `sflow run` per recipe, concurrently, no sbatch. +# ============================================================================= +# `sflow batch --submit` sbatches a DRIVER that runs ON a compute node, so the +# checkout, the venv and the output dir all have to be visible from there. On a +# cluster whose login-node $HOME is not exported to the compute nodes that is +# simply impossible. `sflow run` keeps the driver on the login node and only +# srun's the task steps out, which is why this half does not batch. +# +# It also makes the verdict honest. A batched job is scored by hunting for a +# "success indicator" in its logs; a subprocess just has an EXIT CODE, and sflow +# already exits non-zero when any task fails. So none of the indicator guessing +# applies here -- rc is the answer. +# +# NOTE: recipes whose tasks exchange files through $SFLOW_TASK_OUTPUT_DIR +# (gpu_placement_matrix reads the node -> UUID map written by another task, and +# verify_disjoint compares two tasks' device lists) still need the OUTPUT DIR on +# storage the compute nodes share. Point -o/E2E_OUTPUT_DIR at shared scratch; +# only the driver moved, the task steps still run out on the nodes. +sanity_recipe_set_args() { + # `--set` of a variable a config does not declare is a hard error, and so is + # `--artifact` of an artifact it does not declare -- which is why these are + # built per recipe rather than shared. The colon fixture needs no model and + # declares no artifacts at all, so handing it the standard + # `-a LOCAL_MODEL_PATH=` killed it before the workflow started: + # "Artifact 'LOCAL_MODEL_PATH' specified in overrides is not defined". + if grep -qE '^[[:space:]]*-?[[:space:]]*name:[[:space:]]*LOCAL_MODEL_PATH' "$1"; then + printf '%s\n' "--artifact" "LOCAL_MODEL_PATH=fs://$MODEL_PATH" + fi + # Cap the Slurm allocation. This is the ONLY cap that bites: a recipe's + # workflow-level `timeout:` is accepted by the schema and enforced by nothing + # (see TaskConfig.timeout), so without --time a server that never becomes + # ready holds its nodes for the recipe's own limit -- up to 120 minutes for + # the workload recipes, which on this cluster cannot serve at all and will + # always wait the full time. + # + # Grepped, not assumed, for the same reason as the model artifact above: + # `--set` of a variable a config does not declare is a hard error, and the + # two spellings are NOT interchangeable -- multi_backend.yaml calls it + # TIME_LIMIT, everything else SLURM_TIMELIMIT, and the colon fixture declares + # neither. Unset means "leave each recipe's own value alone". + if [ -n "${SFLOW_E2E_SLURM_TIMELIMIT:-}" ]; then + if grep -qE '^[[:space:]]*SLURM_TIMELIMIT:' "$1"; then + printf '%s\n' "--set" "SLURM_TIMELIMIT=$SFLOW_E2E_SLURM_TIMELIMIT" + elif grep -qE '^[[:space:]]*TIME_LIMIT:' "$1"; then + printf '%s\n' "--set" "TIME_LIMIT=$SFLOW_E2E_SLURM_TIMELIMIT" + fi + fi + case "$(basename "$1")" in + multi_backend.yaml) + printf '%s\n' "--set" "PARTITION_A=$PARTITION" \ + "--set" "PARTITION_B=${E2E_PARTITION_B:-$PARTITION}" \ + "--set" "SLURM_ACCOUNT=$ACCOUNT" + ;; + monitor_mixed.yaml) + printf '%s\n' "--set" "PARTITION_A=$PARTITION" \ + "--set" "PARTITION_B=${E2E_PARTITION_B:-$PARTITION}" \ + "--set" "SLURM_ACCOUNT=$ACCOUNT" \ + "--set" "GPUS_PER_NODE=$GPUS_PER_NODE" + ;; + *) + printf '%s\n' "--set" "SLURM_PARTITION=$PARTITION" \ + "--set" "SLURM_ACCOUNT=$ACCOUNT" \ + "--set" "GPUS_PER_NODE=$GPUS_PER_NODE" + ;; + esac +} + +gpu_placement_verified() { # -> 0 when every GPU task PROVED its placement + # "The workflow completed" says nothing about WHICH cards it used, and until + # now only gpu_placement_matrix checked that -- every other recipe (gpu_indices + # pins devices! auto_replica and resource_release_after pack and re-use them) + # was scored on completion alone, and the workload half checked no placement at + # all. + # + # Every GPU task now leaves sflow_gpus.log recording what it was PLANNED for + # and what it actually SELECTED, both as physical UUIDs, so this audits any + # recipe without the recipe having to assert anything. + # + # A task whose record says `fallback`/`unverified` did not prove its placement: + # the driver could not resolve the plan to UUIDs (probe failed, node names + # disagree, gpus_per_node larger than the node really has) or the step had no + # nvidia-smi. On these clusters that is a regression, not a normal mode -- and + # it is precisely the silent degradation this suite exists to catch. + local dir="$1" + [ -n "$dir" ] && [ -d "$dir" ] || return 0 + local rec action planned selected total=0 unproven=0 verified=0 + while IFS= read -r rec; do + [ -n "$rec" ] || continue + total=$((total + 1)) + action=$(sed -n 's/^action=//p' "$rec" | head -1) + planned=$(sed -n 's/^planned_uuids=//p' "$rec" | head -1) + # The UUID of each device CUDA will really use, in the order it will see them. + selected=$(sed -n 's/^selected=[0-9?]* //p' "$rec" | paste -sd, -) + case "$action" in + verified) verified=$((verified + 1)) ;; + *) + # `fallback` (no UUID map, or Slurm granted cards the plan never + # named) and `unverified` (no nvidia-smi) both mean the placement + # was not proven against physical devices. + unproven=$((unproven + 1)) + echo " GPU PLACEMENT UNPROVEN: $(dirname "$rec" | xargs basename) recorded action='${action:-none}' in $rec" >&2 + continue + ;; + esac + if [ "$selected" != "$planned" ]; then + unproven=$((unproven + 1)) + echo " GPU PLACEMENT MISMATCH: $(dirname "$rec" | xargs basename) planned '$planned' but holds '$selected' ($rec)" >&2 + fi + done < <(find "$dir" -type f -name 'sflow_gpus*.log' 2>/dev/null | sort) + + if [ "$total" -eq 0 ]; then + # No GPU task in this workflow (or none reached the prelude). Nothing to + # prove; the recipe's own verdict still applies. + return 0 + fi + echo " GPU placement: $verified/$total task record(s) proven by UUID" + [ "$unproven" -eq 0 ] +} + +recipe_requests_gpus() { # -> 0 when a TASK asks for GPUs + # A `gpus:` block inside a task's `resources:`. Deliberately NOT + # `gpus_per_node:`, which is the BACKEND's allocation shape -- a recipe can + # size an allocation and still run nothing on a GPU. aiperf_template is + # exactly that: one CPU-only benchmark-client task (`resources: nodes:`), + # gpus_per_node on the backend, and no GPU task anywhere. + grep -qE '^[[:space:]]+gpus:[[:space:]]*$' "$1" +} + +workload_placement_ok() { # -> 0 when placement is PROVEN + # The verdict for a recipe whose application is expected to fail. Placement is + # the only claim, so it must be a POSITIVE one: gpu_placement_verified() + # answers 0 when it finds no records at all -- correct for a CPU-only recipe, + # badly wrong here, because a workload that died before any task started would + # score PASS on zero evidence. Require records to exist, then require every + # one of them to be proven. + local recipe="$1" dir="$2" + [ -n "$dir" ] && [ -d "$dir" ] || return 1 + # No GPU task => no placement to prove, ever. Demanding a record here would + # fail such a recipe on every run forever, which is what happened to + # aiperf_template: it completed cleanly and was scored FAIL for producing + # evidence it structurally cannot produce. Fall back to its own verdict -- + # the only signal that means anything for a recipe holding no GPU. + if ! recipe_requests_gpus "$recipe"; then + workflow_summary_ok "$dir" + return + fi + # It DOES ask for GPUs, so a missing record means no GPU task ever reached the + # placement prelude -- the workflow died first. That is unproven, not passing. + find "$dir" -type f -name 'sflow_gpus*.log' -print -quit 2>/dev/null | grep -q . || return 1 + gpu_placement_verified "$dir" +} + +sanity_recipe_content_ok() { # -> 0 when the run PROVED itself + # rc == 0 only says every task exited 0. It does NOT say the workflow did the + # thing it exists to prove, and these recipes exist to prove something: + # * gpu_placement_matrix can exit 0 while its assertions never ran, or while + # a one-node allocation quietly collapsed the multi-node cases -- so read + # the by-UUID evidence and the 2-distinct-node proof. + # * multi_backend can exit 0 with both backends on the SAME node, which is + # precisely the binding it is meant to disprove. + # * the colon fixture can exit 0 without ever emitting its markers. + # * everything else: sflow's own Status must say COMPLETED. + local name="$1" dir="$2" + [ -n "$dir" ] && [ -d "$dir" ] || return 1 + case "$name" in + gpu_placement_matrix) gpu_placement_run_ok "$dir" ;; + multi_backend*) multi_backend_run_ok "$dir" ;; + colon_in_task_script) colon_task_log_has_markers "$dir" ;; + *) workflow_summary_ok "$dir" ;; + esac +} + +sflow_run_safe_args() { + # EXTRA_BATCH_ARGS is built for `sflow batch`; two kinds of it do not carry + # over to `sflow run`. + # + # --sbatch-output / --sbatch-error name the sbatch JOB's stdout/stderr + # files, and there is no sbatch job here: the driver runs on the login + # node and its output is already captured per recipe. `sflow run` rejects + # them outright ("No such option: --sbatch-output"), which killed every + # sanity recipe before it started. + # -e is this harness's channel for raw Slurm submission flags + # (--segment=, --exclude=). On `sflow run` those are properties of the + # ALLOCATION, so they belong to --extra-salloc-args rather than the + # generic --extra-args. In practice the list is usually empty here: + # --segment is a GB200 requirement and the sanity cluster does not set it. + local out=() skip=0 want_salloc=0 a + for a in "$@"; do + if [ "$skip" = 1 ]; then skip=0; continue; fi + if [ "$want_salloc" = 1 ]; then + want_salloc=0 + out+=("--extra-salloc-args" "$a") + continue + fi + case "$a" in + --sbatch-*=*) ;; # value is inline; drop this token only + --sbatch-*) skip=1 ;; # value is the NEXT token; drop both + -e|--extra-args) want_salloc=1 ;; + *) out+=("$a") ;; + esac + done + [ ${#out[@]} -eq 0 ] || printf '%s\n' "${out[@]}" +} + +run_sanity_recipes_with_sflow_run() { + local recipes=() + local f + for f in "$EXAMPLES_DIR"/self_contained/slurm/*.yaml; do + [ "$TEST_TYPE" != "one" ] || [ "${f##*/}" = "gpu_placement_matrix.yaml" ] || continue + if recipe_in_class "$f"; then + recipes+=("$f") + elif [ "${SFLOW_E2E_INCLUDE_WORKLOAD_PLACEMENT:-0}" = "1" ] && is_workload_recipe "$f"; then + # Workload recipes on a cluster that cannot actually serve. They ride + # `sflow run` (not batch --submit) because that is the only path this + # cluster supports, and they are judged on PLACEMENT ALONE -- see the + # verdict below. + recipes+=("$f") + fi + done + # The colon-in-task-script fixture is written by full_sample_tests.sh; it is a + # functionality check like the rest, so it rides this half when present. + if [ -n "${SFLOW_COLON_SCRIPT_FIXTURE:-}" ] && [ -f "$SFLOW_COLON_SCRIPT_FIXTURE" ]; then + recipes+=("$SFLOW_COLON_SCRIPT_FIXTURE") + fi + if [ ${#recipes[@]} -eq 0 ]; then + echo "ERROR: no sanity recipes selected" + exit 1 + fi + + echo "" + local par_note="all at once" + [ "${SFLOW_E2E_MAX_PARALLEL:-0}" -gt 0 ] && par_note="${SFLOW_E2E_MAX_PARALLEL} at a time" + echo "===== Sanity half: ${#recipes[@]} recipe(s) via \`sflow run\` (no sbatch), $par_note =====" + echo "" + + # All at once by default. Every recipe is its own allocation, so the cluster's + # own scheduler is what orders them -- a starved salloc queues rather than + # failing, and the Slurm --time cap (SFLOW_E2E_SLURM_TIMELIMIT) is what stops + # a hung one holding nodes. SFLOW_E2E_MAX_PARALLEL>0 throttles to waves for a + # cluster where that is not welcome; 0/unset means no limit. + local names=() logs=() roots=() kinds=() files=() + local max_par="${SFLOW_E2E_MAX_PARALLEL:-0}" + for f in "${recipes[@]}"; do + local name set_args=() + name=$(basename "$f" .yaml) + mapfile -t set_args < <(sanity_recipe_set_args "$f") + local run_args=() + mapfile -t run_args < <(sflow_run_safe_args "${EXTRA_BATCH_ARGS[@]+"${EXTRA_BATCH_ARGS[@]}"}") + local root="$E2E_OUTPUT_DIR/$name" + local log="$root/${name}.sflow_run.log" + rm -rf "$root" + mkdir -p "$root" + # Guarded on >0: `-ge 0` is always true, and `wait -n` with no children + # returns immediately, so an unguarded loop would spin instead of launch. + if [ "$max_par" -gt 0 ]; then + while [ "$(jobs -rp | wc -l)" -ge "$max_par" ]; do wait -n; done + fi + echo " launching $name" + # Each run records its OWN exit status. `wait -n` above reaps children as + # they finish, so a later `wait $pid` would hit "not a child of this + # shell" and report 127 for a run that actually passed. + ( + sflow run -f "$f" \ + "${set_args[@]}" \ + --output-dir "$root" \ + --enable-workflow-monitor \ + "${run_args[@]+"${run_args[@]}"}" \ + > "$log" 2>&1 + echo $? > "$root/.rc" + ) & + names+=("$name") + logs+=("$log") + roots+=("$root") + files+=("$f") + if is_workload_recipe "$f"; then kinds+=("workload"); else kinds+=("sanity"); fi + done + wait + + echo "" + echo "===== Scoring ${#names[@]} sflow run(s) =====" + local i rc + for i in "${!names[@]}"; do + # Missing .rc means the subshell never got to write one -- treat as failure. + rc=$(cat "${roots[$i]}/.rc" 2>/dev/null || echo 1) + # No job id to look a run up by later, so find where it landed; the + # content check and the monitor gate below both read it. + local run_dir + run_dir=$(ls -d "${roots[$i]}"/*/ 2>/dev/null | head -1) + SANITY_RUN_DIRS+=("${run_dir:-$E2E_OUTPUT_DIR/${names[$i]}-NOT-FOUND}") + SANITY_RUN_NAMES+=("${names[$i]}") + + TOTAL=$((TOTAL + 1)) + if [ "${kinds[$i]}" = "workload" ]; then + # This cluster's GPUs cannot run real LLM inference, so the framework + # WILL fail and its own verdict answers nothing. These recipes are here + # for one reason -- to prove GPU placement on a second cluster and a + # second GPU generation -- so that is the entire test. The app's exit + # status is deliberately ignored; a placement regression is not. + # Say which of the two things actually happened. workload_placement_ok + # passes for two different reasons and one message for both CLAIMED + # EVIDENCE THAT DOES NOT EXIST: aiperf_template holds no GPU, wrote no + # record, and still reported "placement proven by UUID" -- the exact + # kind of line that misleads whoever audits these artifacts later. + local proved="placement proven by UUID; app rc=$rc ignored on this cluster" + local unproved="GPU placement not proven" + if ! recipe_requests_gpus "${files[$i]}"; then + proved="no GPU task, so no placement to prove; sflow reports COMPLETED" + unproved="no GPU task to place, and sflow's own verdict is not COMPLETED" + fi + # Say what aiperf actually measured here, without gating on it. This + # cluster cannot serve a real model, so demanding a benchmark would fail + # these recipes forever -- but a bare PASS next to an aiperf that + # measured nothing is how the ptyche half stayed green for six workflows. + # Whoever audits these artifacts should not have to open the CSV to find + # that out. aiperf_tally_ok() already prints the counts to stderr. + aiperf_tally_ok "$run_dir" + case $? in + 0) proved="$proved; aiperf benchmarked" ;; + 1) proved="$proved; aiperf measured nothing (expected here, not gated)" ;; + esac + if workload_placement_ok "${files[$i]}" "$run_dir"; then + PASSED=$((PASSED + 1)) + echo " ${names[$i]}: PASS ($proved)" + else + echo " ${names[$i]}: FAIL ($unproved; see ${run_dir:-${logs[$i]}})" + fi + elif [ "$rc" -ne 0 ]; then + if cuda_infra_failure "${roots[$i]}"; then + mark_cuda_excused "${names[$i]}" "${logs[$i]}" "(sflow run rc=$rc with a CUDA init failure)" + else + echo " ${names[$i]}: FAIL (sflow run exited $rc; see ${logs[$i]})" + fi + elif sanity_recipe_content_ok "${names[$i]}" "$run_dir" \ + && gpu_placement_verified "$run_dir"; then + PASSED=$((PASSED + 1)) + echo " ${names[$i]}: PASS (rc=0 and its own output proves it, under $run_dir)" + elif cuda_infra_failure "${roots[$i]}"; then + mark_cuda_excused "${names[$i]}" "${run_dir:-${logs[$i]}}" "(exited 0 but proved nothing; CUDA init failure on node)" + else + # The nastiest shape: green process, unproven run. Exactly what a + # silently-degraded placement or a collapsed two-backend run looks like. + echo " ${names[$i]}: FAIL (sflow run exited 0 but its output does not prove the run: ${run_dir:-no run dir found})" + fi + done +} + +run_monitor_mixed_real() { + # monitor_mixed.yaml is the broadest single-run regression net in examples/ + # (placement proven by UUID, replicas, release_after GPU reuse, readiness + # ordering, marker-clipped monitor reports, two Slurm pools). It needs its OWN + # `sflow batch` call for one reason: `--set PARTITION_A=...` is REJECTED by any + # config that does not declare that variable ("Variable 'PARTITION_A' ... is not + # defined"), so it cannot ride the shared bulk-submit args. Without the --set it + # submitted with the sample's `your_partition_a` placeholder and was a + # guaranteed sbatch rejection. + if [ -z "$SUBMIT" ]; then + return + fi + case "$TEST_TYPE" in + s|a|smoke|min) ;; + *) return ;; + esac + # No framework and no model load: the sanity half owns it, and that half runs + # through `sflow run` now -- so batch it only in the combined default. + [ "$RECIPE_CLASS" != "all" ] && return + + local part_a="${MONITOR_MIXED_PARTITION_A:-$PARTITION}" + local part_b="${MONITOR_MIXED_PARTITION_B:-$E2E_PARTITION_B}" + local mm_dir="$E2E_OUTPUT_DIR/monitor_mixed_real" + local mm_script="$mm_dir/monitor_mixed.sh" + mkdir -p "$mm_dir" + + echo "" + echo "===== Real monitor_mixed run (all-in-one regression net) =====" + echo " gpu_pool partition=$part_a, cpu_pool partition=$part_b" + echo "" + + # -G is IGNORED for a multi-backend config (each backend uses its own config + # values), so the per-node GPU count has to go in as a --set or the recipe keeps + # its own default and mis-plans the 2-node decode server. + strip_suite_segment_args + local mm_output mm_status + mm_output=$(sflow batch "$EXAMPLES_DIR/self_contained/slurm/monitor_mixed.yaml" \ + --set "SLURM_ACCOUNT=$ACCOUNT" \ + --set "PARTITION_A=$part_a" \ + --set "PARTITION_B=$part_b" \ + --set "GPUS_PER_NODE=$GPUS_PER_NODE" \ + -a "LOCAL_MODEL_PATH=fs://$MODEL_PATH" \ + -p "$part_a" \ + -A "$ACCOUNT" \ + --job-name "monitor_mixed_slurm" \ + "${BATCH_WORKSPACE_ARGS[@]}" \ + "${BATCH_OUTPUT_ARGS[@]}" \ + "${BATCH_VENV_ARGS[@]}" \ + -o "$mm_script" \ + $SUBMIT \ + "${STRIPPED_SEGMENT_ARGS[@]}" 2>&1) + mm_status=$? + echo "$mm_output" + if [ "$mm_status" -ne 0 ]; then + echo " monitor_mixed run: FAIL (sflow batch failed, rc=$mm_status; see output above)" + MONITOR_MIXED_LAUNCH_FAILED=1 + return + fi + + # Join JOB_IDS so the driver job flows through the shared wait (sacct) + + # validate loop like every other batched job. + local mm_job_id + mm_job_id=$(echo "$mm_output" | sed -n 's/.*Submitted batch job \([0-9]\+\).*/\1/p' | tail -1) + if [ -n "$mm_job_id" ]; then + JOB_IDS+=("$mm_job_id") + echo " monitor_mixed driver job id: $mm_job_id (script: $mm_script)" + else + echo " monitor_mixed run: FAIL (no Slurm job id reported by sflow batch)" + MONITOR_MIXED_LAUNCH_FAILED=1 + fi +} + run_multi_backend_real() { # Real multi-backend coverage via `sflow batch`: a >=2-Slurm-backend config # makes `sflow batch` emit one driver sbatch sized to the leader backend; @@ -547,9 +1235,20 @@ run_multi_backend_real() { s|a|smoke|min) ;; *) return ;; esac - - local part_a="${MULTI_BACKEND_PARTITION_A:-genesisq}" - local part_b="${MULTI_BACKEND_PARTITION_B:-gamoraq}" + # No framework and no model load: the sanity half owns it, and that half runs + # through `sflow run` now -- so batch it only in the combined default. + [ "$RECIPE_CLASS" != "all" ] && return + + # Default BOTH to the partition this run was given. Hardcoded names are + # cluster-specific and simply do not exist elsewhere -- the stale genesisq / + # gamoraq defaults meant every submission on a new cluster died with "no Slurm + # allocation granted". Two DIFFERENT partitions are better coverage, so an + # operator can still opt in via MULTI_BACKEND_PARTITION_A/_B; with one + # partition the test still proves what it is for, because each backend gets + # its OWN allocation and the check is that task_a and task_b land on + # different NODES. + local part_a="${MULTI_BACKEND_PARTITION_A:-$PARTITION}" + local part_b="${MULTI_BACKEND_PARTITION_B:-$E2E_PARTITION_B}" MULTI_BACKEND_RUN_DIR="$E2E_OUTPUT_DIR/multi_backend_real" local mb_dir="$MULTI_BACKEND_RUN_DIR" local mb_script="$mb_dir/multi_backend_hetjob.sh" @@ -566,6 +1265,9 @@ run_multi_backend_real() { # (sacct) + validate loop below like every other batched job. The CLI -p/-A # are required by the command but the driver is sized to the leader backend # (each backend uses its own resolved partition/account). + # multi_backend.yaml declares no SLURM_NODES either -- see strip_suite_segment_args. + strip_suite_segment_args + local mb_output mb_status mb_output=$(sflow batch "$EXAMPLES_DIR/self_contained/slurm/multi_backend.yaml" \ --set "SLURM_ACCOUNT=$ACCOUNT" \ @@ -581,7 +1283,7 @@ run_multi_backend_real() { "${BATCH_VENV_ARGS[@]}" \ -o "$mb_script" \ $SUBMIT \ - "${EXTRA_BATCH_ARGS[@]}" 2>&1) + "${STRIPPED_SEGMENT_ARGS[@]}" 2>&1) mb_status=$? echo "$mb_output" if [ "$mb_status" -ne 0 ]; then @@ -605,10 +1307,17 @@ run_multi_backend_real() { fi } -# Sync examples/ to src/sflow/samples/ so packaged samples stay up to date +# Sync examples/ to src/sflow/samples/ so packaged samples stay up to date. +# gpu_reservation/ and mlperf/ are deliberately NOT packaged: they are local +# scratch (examples/mlperf/ is even gitignored), so copying them here only +# produced untracked dirs under src/sflow/samples/ that would ship with the +# wheel if anyone committed them. --delete does not clean an --exclude'd path, +# so remove any copy an earlier run already made. echo "Syncing examples/ -> src/sflow/samples/ ..." rsync -a --delete --exclude='__pycache__' --exclude='*.pyc' --exclude='__init__.py' --exclude='sflow_output' \ + --exclude='gpu_reservation' --exclude='mlperf' \ "$EXAMPLES_DIR/" "$SAMPLES_DIR/" +rm -rf "$SAMPLES_DIR/gpu_reservation" "$SAMPLES_DIR/mlperf" echo "Done." # No shared runtime venv to pre-build. Under the per-job venv flow, each @@ -624,16 +1333,21 @@ echo "Done." JOB_IDS=() COLON_JOB_IDS=() MULTI_BACKEND_JOB_IDS=() +SANITY_RUN_DIRS=() +SANITY_RUN_NAMES=() MULTI_BACKEND_RUN_DIR="" MULTI_BACKEND_LAUNCH_FAILED="" +MONITOR_MIXED_LAUNCH_FAILED="" CSV_FILE="$EXAMPLES_DIR/modular/inference_x_v2/bulk_input.csv" # ============================================================================= # Part 1: Self-contained examples (--bulk-submit) # ============================================================================= -if [ "$TEST_TYPE" = "s" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" ] || [ "$TEST_TYPE" = "min" ]; then +if { [ "$TEST_TYPE" = "s" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" ] || [ "$TEST_TYPE" = "min" ] || [ "$TEST_TYPE" = "one" ]; } && [ "$RECIPE_CLASS" != "sanity" ]; then echo "" - if [ "$TEST_TYPE" = "min" ]; then + if [ "$TEST_TYPE" = "one" ]; then + echo "===== Part 1: Single-job plumbing smoke (--bulk-submit one file) =====" + elif [ "$TEST_TYPE" = "min" ]; then echo "===== Part 1: Min self-contained examples (--bulk-submit selected files) =====" elif [ "$TEST_TYPE" = "smoke" ]; then echo "===== Part 1: Smoke self-contained examples (--bulk-submit selected files) =====" @@ -642,19 +1356,50 @@ if [ "$TEST_TYPE" = "s" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" fi echo "" - if [ "$TEST_TYPE" = "min" ]; then + if [ "$TEST_TYPE" = "min" ] || [ "$TEST_TYPE" = "one" ]; then MIN_SELF_CONTAINED=( "$EXAMPLES_DIR/self_contained/slurm/auto_replica.yaml" "$EXAMPLES_DIR/self_contained/slurm/dynamo_trtllm_disagg.yaml" + # One node, seconds of runtime, and it ASSERTS by UUID that every + # container/bare x offset x concurrency combination held the physical + # GPUs it was planned for. The recipes above only echo their devices, + # so they passed while a server held none. + "$EXAMPLES_DIR/self_contained/slurm/gpu_placement_matrix.yaml" "$EXAMPLES_DIR/self_contained/slurm/resource_release_after.yaml" "$EXAMPLES_DIR/self_contained/slurm/trtllm_serve_disagg.yaml" ) + if [ "$TEST_TYPE" = "one" ]; then + # Narrow to ONE job: the GPU placement matrix. Two nodes, a small + # container, no model to load, and seconds of compute -- so it stays + # cheap enough to play on every CI or cluster change. + # + # It is the recipe that ASSERTS rather than echoes. Every combination + # that can break the slice is covered (bare vs container, zero vs + # high offset, two tasks concurrent on one node, and multi-node), and + # each one resolves its planned HOST indices through a bare-metal + # index -> UUID map taken per node before anything was carved. That is + # the only way to tell "the right number of GPUs" from "the right + # GPUs", and it is the failure that actually reaches clusters: a task + # planned for 2,3 inside a 2-GPU container renumbered to 0,1. + # + # It replaced the single-node DISAGG recipe, which probed the same + # failure but only PRINTED its devices -- so it passed while a server + # held none -- and cost a multi-gigabyte pull plus a model load, which + # made a red run ambiguous between a broken CI path and a real + # regression. Here a red run means placement is genuinely wrong; read + # SFLOW_GPU_PROBE in the task logs for the planned slice next to the + # devices the step actually held. + MIN_SELF_CONTAINED=( + "$EXAMPLES_DIR/self_contained/slurm/gpu_placement_matrix.yaml" + ) + fi MIN_BULK_ARGS=() for yaml_file in "${MIN_SELF_CONTAINED[@]}"; do if [ ! -f "$yaml_file" ]; then echo "ERROR: min self-contained Slurm YAML not found: $yaml_file" exit 1 fi + recipe_in_class "$yaml_file" || continue MIN_BULK_ARGS+=(--bulk-submit "$yaml_file") done output=$(sflow batch \ @@ -676,14 +1421,17 @@ if [ "$TEST_TYPE" = "s" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" dynamo_sglang_agg.yaml|dynamo_vllm_agg.yaml|sglang_server_client.yaml) continue ;; - multi_backend.yaml) - # Covered separately by run_multi_backend_real as a `sflow - # batch` heterogeneous job; it needs two partitions - # (PARTITION_A/PARTITION_B), so skip the single-partition - # bulk-submit copy here. + multi_backend.yaml|monitor_mixed.yaml) + # Both declare PARTITION_A/PARTITION_B and are covered + # separately (run_multi_backend_real / run_monitor_mixed_real). + # They cannot ride the shared bulk args: `--set PARTITION_A=...` + # is REJECTED by every config that does not declare it, and + # without the --set they submit with the sample's + # `your_partition_a` placeholder and sbatch rejects them. continue ;; esac + recipe_in_class "$yaml_file" || continue SMOKE_SELF_CONTAINED+=("$yaml_file") SMOKE_BULK_ARGS+=(--bulk-submit "$yaml_file") done @@ -704,16 +1452,18 @@ if [ "$TEST_TYPE" = "s" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" $SUBMIT \ "${EXTRA_BATCH_ARGS[@]}" 2>&1) else - # Bulk-submit every example EXCEPT the multi-backend config: it needs two - # partitions (PARTITION_A/PARTITION_B) and is covered separately by - # run_multi_backend_real as a `sflow batch` heterogeneous job. + # Bulk-submit every example EXCEPT the two-partition configs: they declare + # PARTITION_A/PARTITION_B, which the shared bulk args cannot set (a --set of + # a variable a config does not declare is a hard error), and are covered + # separately by run_multi_backend_real / run_monitor_mixed_real. ALL_BULK_ARGS=() for yaml_file in "$EXAMPLES_DIR"/self_contained/slurm/*.yaml; do case "$(basename "$yaml_file")" in - multi_backend.yaml) + multi_backend.yaml|monitor_mixed.yaml) continue ;; esac + recipe_in_class "$yaml_file" || continue ALL_BULK_ARGS+=(--bulk-submit "$yaml_file") done output=$(sflow batch \ @@ -746,7 +1496,7 @@ fi # ============================================================================= # Part 2: Modular examples (--bulk-input) # ============================================================================= -if [ "$TEST_TYPE" = "m" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" ] || [ "$TEST_TYPE" = "min" ]; then +if { [ "$TEST_TYPE" = "m" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" ] || [ "$TEST_TYPE" = "min" ]; } && [ "$RECIPE_CLASS" != "sanity" ]; then echo "" if [ "$TEST_TYPE" = "min" ]; then echo "===== Part 2: Min modular example (--bulk-input selected row) =====" @@ -808,7 +1558,7 @@ if [ "${SFLOW_E2E_SKIP_INFMAX:-}" = "1" ]; then echo "" echo "===== Part 3: infmax suites SKIPPED (SFLOW_E2E_SKIP_INFMAX=1; run by prenyx CI) =====" echo "" -elif [ "$TEST_TYPE" = "inf" ] || [ "$TEST_TYPE" = "m" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" ] || [ "$TEST_TYPE" = "min" ]; then +elif { [ "$TEST_TYPE" = "inf" ] || [ "$TEST_TYPE" = "m" ] || [ "$TEST_TYPE" = "a" ] || [ "$TEST_TYPE" = "smoke" ] || [ "$TEST_TYPE" = "min" ]; } && [ "$RECIPE_CLASS" != "sanity" ]; then echo "" if [ "$TEST_TYPE" = "min" ]; then echo "===== Part 3: Min infmax multi-node batch suite =====" @@ -835,6 +1585,7 @@ submit_colon_task_script_e2e # after the async submissions so its salloc job ids join JOB_IDS and flow through # the shared wait (sacct) + validate loop below. run_multi_backend_real +run_monitor_mixed_real set +x @@ -844,7 +1595,7 @@ echo "===== Submitted Jobs =====" fi # end of --check else block -if [ ${#JOB_IDS[@]} -eq 0 ]; then +if [ ${#JOB_IDS[@]} -eq 0 ] && [ "$RECIPE_CLASS" != "sanity" ]; then echo "No job IDs captured." exit 0 fi @@ -890,14 +1641,23 @@ done cuda_infra_failure() { # -> 0 (true) if the job failed due to CUDA infra local out_dir="$1" [ -n "$out_dir" ] && [ -d "$out_dir" ] || return 1 - # These are GB200/driver "system not ready" + missing-CUDA-runtime signatures. - # 'system not yet initialized' covers both torch ('Error 802: system not yet - # initialized') and cupy ('cudaErrorSystemNotReady: system not yet initialized'). + # These are GB200/driver "system not ready" signatures -- an ERROR from the + # driver, on a node that needs draining. 'system not yet initialized' covers + # both torch ('Error 802: system not yet initialized') and cupy + # ('cudaErrorSystemNotReady: system not yet initialized'). + # + # Every pattern here must be something ONLY a broken node produces. Excusing is + # not a soft verdict -- it removes the job from the pass/fail threshold, so a + # pattern that also matches healthy output turns real regressions into a green + # pipeline. 'No CUDA runtime is found' used to be in this list and did exactly + # that: torch prints it as a routine WARNING when cpp_extension cannot find + # nvcc for JIT ("No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda'"), + # which healthy runs emit constantly. It excused two genuinely failed disagg + # jobs and reported "PASS - 0/6 failed". grep -rIqs --include='*.log' --include='*.out' \ -e 'system not yet initialized' \ -e 'cudaErrorSystemNotReady' \ -e 'CUDA initialization: Unexpected error from cudaGetDeviceCount' \ - -e 'No CUDA runtime is found' \ -e 'Failed to get device capability: Unexpected error from cudaGetDeviceCount' \ "$out_dir" } @@ -917,6 +1677,9 @@ echo "===== Results =====" TOTAL=0 PASSED=0 CUDA_INFRA=0 +if [ "$RECIPE_CLASS" = "sanity" ]; then + run_sanity_recipes_with_sflow_run +fi for jid in "${JOB_IDS[@]}"; do TOTAL=$((TOTAL + 1)) if is_multi_backend_result "$jid"; then @@ -945,26 +1708,62 @@ for jid in "${JOB_IDS[@]}"; do fi continue fi + case "$out_dir" in + *-gpu_placement_matrix-*) + if gpu_placement_run_ok "$out_dir"; then + PASSED=$((PASSED + 1)) + echo " Job $jid: PASS (GPU placement proven by UUID on 2+ nodes under $out_dir)" + elif cuda_infra_failure "$out_dir"; then + mark_cuda_excused "$jid" "$out_dir" "(placement assertions unproven; CUDA init failure on node)" + else + echo " Job $jid: FAIL (GPU placement not proven under $out_dir; read SFLOW_GPU_PROBE / FAIL: in the task logs)" + fi + continue + ;; + esac # Check for various success indicators across different workflow types - # aiperf benchmark: '0 errors' in benchmark log - # aiperf template: '0 valid' in benchmark log - # infmax benchmark: 'Successful requests:' with non-zero value + # aiperf: its own "Processed N valid requests and M errors" tally + # benchmark_serving: its own "Successful requests:" vs the --num-prompts asked for # auto_replica: 'Client Task Nodes' in client task log - count_aiperf_errors=$(find "$out_dir" -type f -name 'benchmark*.log' -exec grep -l "0 errors" {} + 2>/dev/null | wc -l) - count_aiperf_valid=$(find "$out_dir" -type f -name 'benchmark*.log' -exec grep -l "0 valid" {} + 2>/dev/null | wc -l) - count_zero_success=$(find "$out_dir" -type f -name 'benchmark*.log' -exec grep -lP "Successful requests:\s+0\s*$" {} + 2>/dev/null | wc -l) - count_any_success=$(find "$out_dir" -type f -name 'benchmark*.log' -exec grep -l "Successful requests:" {} + 2>/dev/null | wc -l) count_replica=$(find "$out_dir" -type f -name 'client*.log' -exec grep -l "Client Task Nodes" {} + 2>/dev/null | wc -l) - - if [ "$count_zero_success" -gt 0 ]; then + # Both: 0 = benchmarked, 1 = ran and measured nothing, 2 = that tool not used here. + aiperf_tally_ok "$out_dir" + aiperf_state=$? + serving_tally_ok "$out_dir" + serving_state=$? + + if [ "$aiperf_state" -eq 1 ] && recipe_is_client_only "$out_dir"; then + # The ONE expected-not-to-benchmark shape, claimed explicitly so it reads as + # a decision rather than a hole: this recipe starts no server, so aiperf had + # nothing to talk to. It still owes a clean completion. + if workflow_summary_ok "$out_dir"; then + PASSED=$((PASSED + 1)) + echo " Job $jid: PASS (client-only recipe: no server to benchmark by design, and it completed; $out_dir)" + else + echo " Job $jid: FAIL (client-only recipe is still expected to COMPLETE, and did not; $out_dir)" + fi + elif [ "$aiperf_state" -eq 1 ]; then + # Checked BEFORE the success indicators: this is the shape where every other + # signal in the run says green. A dead aiperf is a failed benchmark even when + # a sibling task in the same workflow reported requests of its own. if cuda_infra_failure "$out_dir"; then - mark_cuda_excused "$jid" "$out_dir" "('Successful requests: 0' with a CUDA init failure)" + mark_cuda_excused "$jid" "$out_dir" "(aiperf measured nothing; CUDA init failure on node)" else - echo " Job $jid: FAIL ('Successful requests: 0' found in $out_dir)" + echo " Job $jid: FAIL (aiperf ran but measured nothing; see the tally above, $out_dir)" fi - elif [ "$count_aiperf_errors" -gt 0 ] || [ "$count_aiperf_valid" -gt 0 ] || [ "$count_any_success" -gt 0 ] || [ "$count_replica" -gt 0 ]; then + elif [ "$serving_state" -eq 1 ]; then + if cuda_infra_failure "$out_dir"; then + mark_cuda_excused "$jid" "$out_dir" "(benchmark_serving did not complete; CUDA init failure on node)" + else + echo " Job $jid: FAIL (benchmark_serving ran but did not complete its requests; see the counts above, $out_dir)" + fi + elif [ "$aiperf_state" -eq 0 ] || [ "$serving_state" -eq 0 ] || [ "$count_replica" -gt 0 ]; then PASSED=$((PASSED + 1)) echo " Job $jid: PASS (under $out_dir)" + elif [ -z "$(find "$out_dir" -type f -name 'benchmark*.log' -print -quit 2>/dev/null)" ] \ + && workflow_summary_ok "$out_dir"; then + PASSED=$((PASSED + 1)) + echo " Job $jid: PASS (no benchmark log; sflow reports Status: COMPLETED under $out_dir)" elif cuda_infra_failure "$out_dir"; then mark_cuda_excused "$jid" "$out_dir" "(no success indicator; CUDA init failure on node)" else @@ -979,6 +1778,13 @@ if [ -n "${MULTI_BACKEND_LAUNCH_FAILED:-}" ]; then echo " Multi-backend run: FAIL (no Slurm allocation granted)" fi +if [ -n "${MONITOR_MIXED_LAUNCH_FAILED:-}" ]; then + # Same as above: never submitted, so the loop never visited it. Silence here + # let the broadest recipe in the suite vanish without touching the verdict. + TOTAL=$((TOTAL + 1)) + echo " monitor_mixed run: FAIL (no Slurm job submitted)" +fi + echo "" echo "===== Summary =====" echo "$PASSED/$TOTAL jobs passed" @@ -1020,6 +1826,17 @@ echo "===== Monitor Coverage =====" MONITOR_TOTAL=0 MONITOR_PRESENT=0 MONITOR_MISSING_LABELS="" +for _i in "${!SANITY_RUN_DIRS[@]}"; do + MONITOR_TOTAL=$((MONITOR_TOTAL + 1)) + _mon=$(find "${SANITY_RUN_DIRS[$_i]}" -maxdepth 2 -type f -name 'sflow_monitor.log' 2>/dev/null | head -1) + if [ -n "$_mon" ] && monitor_log_has_content "$_mon"; then + MONITOR_PRESENT=$((MONITOR_PRESENT + 1)) + echo " ${SANITY_RUN_NAMES[$_i]}: monitor overview OK ($_mon)" + else + MONITOR_MISSING_LABELS="$MONITOR_MISSING_LABELS - ${SANITY_RUN_NAMES[$_i]} (no populated sflow_monitor.log under ${SANITY_RUN_DIRS[$_i]})\n" + echo " ${SANITY_RUN_NAMES[$_i]}: MONITOR MISSING/EMPTY (${SANITY_RUN_DIRS[$_i]})" + fi +done for jid in "${JOB_IDS[@]}"; do MONITOR_TOTAL=$((MONITOR_TOTAL + 1)) mon_out_dir=$(workflow_output_dir_for_job "$jid") @@ -1096,3 +1913,14 @@ if [ -n "$TARGETING_FAIL_LABELS" ]; then echo "Disagg workflows whose monitor sampled the wrong resources:" echo -e "$TARGETING_FAIL_LABELS" fi + +# The verdict, as an exit status. Without this the script ended on an `if` and +# returned 0 however red the run was, which is what made full_sample_tests.sh's +# `exit "$e2e_rc"` unable to fail. +# CUDA-infra excusals are excluded from the threshold, matching what +# summarize_validation() does with the same numbers -- but an ALL-excused run +# proved nothing, so it is not a pass either. +[ "$PASSED" -gt 0 ] \ + && [ $((PASSED + CUDA_INFRA)) -eq "$TOTAL" ] \ + && [ "$MONITOR_PRESENT" -eq "$MONITOR_TOTAL" ] \ + && [ "$TARGETING_OK" -eq "$TARGETING_TOTAL" ] diff --git a/tests/integration/guide/sflow_dynamo.yaml b/tests/integration/guide/sflow_dynamo.yaml index 7115ef8..1dad24d 100644 --- a/tests/integration/guide/sflow_dynamo.yaml +++ b/tests/integration/guide/sflow_dynamo.yaml @@ -353,7 +353,6 @@ workflow: --extra-inputs "max_tokens:${OSL}" --extra-inputs "min_tokens:${OSL}" --extra-inputs "ignore_eos:true" - --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} diff --git a/tests/unit/golden/k8s_recipes/dynamo_sglang_disagg.yaml b/tests/unit/golden/k8s_recipes/dynamo_sglang_disagg.yaml index 44759f3..f4b4607 100644 --- a/tests/unit/golden/k8s_recipes/dynamo_sglang_disagg.yaml +++ b/tests/unit/golden/k8s_recipes/dynamo_sglang_disagg.yaml @@ -763,11 +763,10 @@ items: Qwen3-8B-FP8 --tokenizer /models/Qwen3-8B-FP8 --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://0.0.0.1:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs - "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" - --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} - --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries - $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' - --record-processors 8 --ui simple + "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs + "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count + ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: + Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple echo "Benchmarking finished"' kind: ConfigMap diff --git a/tests/unit/golden/k8s_recipes/dynamo_trtllm_disagg.yaml b/tests/unit/golden/k8s_recipes/dynamo_trtllm_disagg.yaml index 2a46e62..3075dcb 100644 --- a/tests/unit/golden/k8s_recipes/dynamo_trtllm_disagg.yaml +++ b/tests/unit/golden/k8s_recipes/dynamo_trtllm_disagg.yaml @@ -820,11 +820,10 @@ items: Qwen3-8B-FP8 --tokenizer /models/Qwen3-8B-FP8 --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://0.0.0.1:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs - "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" - --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} - --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries - $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' - --record-processors 8 --ui simple + "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs + "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count + ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: + Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple echo "Benchmarking finished"' kind: ConfigMap diff --git a/tests/unit/golden/k8s_recipes/dynamo_vllm_agg.yaml b/tests/unit/golden/k8s_recipes/dynamo_vllm_agg.yaml index 7a36fcd..d778432 100644 --- a/tests/unit/golden/k8s_recipes/dynamo_vllm_agg.yaml +++ b/tests/unit/golden/k8s_recipes/dynamo_vllm_agg.yaml @@ -488,11 +488,10 @@ items: Qwen3-8B-FP8 --tokenizer /models/Qwen3-8B-FP8 --endpoint-type chat --endpoint /v1/chat/completions --streaming --url http://0.0.0.1:8000 --synthetic-input-tokens-mean 1024 --synthetic-input-tokens-stddev 0 --output-tokens-mean 1024 --output-tokens-stddev 0 --extra-inputs "max_tokens:1024" --extra-inputs - "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "{\"nvext\":{\"ignore_eos\":true}}" - --extra-inputs "repetition_penalty:1.0" --extra-inputs "temperature: 0.0" --concurrency ${CONCURRENCY} - --request-count $((8*${CONCURRENCY})) --warmup-request-count ${CONCURRENCY} --num-dataset-entries - $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: Bearer NOT USED'' -H ''Accept: text/event-stream'' - --record-processors 8 --ui simple + "min_tokens:1024" --extra-inputs "ignore_eos:true" --extra-inputs "repetition_penalty:1.0" --extra-inputs + "temperature: 0.0" --concurrency ${CONCURRENCY} --request-count $((8*${CONCURRENCY})) --warmup-request-count + ${CONCURRENCY} --num-dataset-entries $((8*${CONCURRENCY})) --random-seed 100 -H ''Authorization: + Bearer NOT USED'' -H ''Accept: text/event-stream'' --record-processors 8 --ui simple echo "Benchmarking finished"' kind: ConfigMap diff --git a/tests/unit/test_app_assembly_resolve_variables.py b/tests/unit/test_app_assembly_resolve_variables.py index f86f818..15a2ec7 100644 --- a/tests/unit/test_app_assembly_resolve_variables.py +++ b/tests/unit/test_app_assembly_resolve_variables.py @@ -199,3 +199,24 @@ def test_resolve_variables_exposes_domain_metadata_in_expressions(): out = resolve_global_variables(config, state) assert out.variables["BATCH_SIZE"].value == 128 # max([128, 512]) // 4 + + +def test_build_state_clears_source_files_from_a_previous_call(tmp_path): + """`resolver` is a module GLOBAL, so state survives between build_state calls. + + source_files was assigned only when truthy and never cleared, so a call that + passes none (visualize() does) inherited the last run's files and pointed its + expression-error location hints at the wrong YAML. + """ + import asyncio + + from sflow.app import assembly + + assembly.resolver.source_files = [tmp_path / "stale.yaml"] + + config = _minimal_config(variables=[]) + asyncio.run( + assembly.build_state(config, allocate=False, workspace_dir=str(tmp_path)) + ) + + assert assembly.resolver.source_files == [] diff --git a/tests/unit/test_cli_batch.py b/tests/unit/test_cli_batch.py index ea2cdfd..0a30364 100644 --- a/tests/unit/test_cli_batch.py +++ b/tests/unit/test_cli_batch.py @@ -1611,7 +1611,7 @@ def test_absolute_path_skipped(self): def test_container_image_skipped(self): assert ( - _normalize_col_value("nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.8.0") is None + _normalize_col_value("nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0") is None ) def test_container_image_with_org_skipped(self): @@ -1626,7 +1626,7 @@ def test_derive_row_name_container_image_skipped(): rows = [ { "sflow_config_file": "wf.yaml", - "IMAGE": "nvcr.io/nvidia/vllm-runtime:0.8.0", + "IMAGE": "nvcr.io/nvidia/vllm-runtime:1.3.0", "SLURM_NODES": "2", }, { @@ -5146,6 +5146,11 @@ def test_batch_script_installs_editable_from_source_path( # does not recurse into its own destination / sibling jobs' growing copies. assert "--exclude='.sflow_venv*'" in script_content assert "--exclude='.sflow_src*'" in script_content + # Also correctness-critical, and not about size: pip/uv write partial *.tmp + # files under .cache while sibling jobs bootstrap, and rsync exits 24 when one + # vanishes mid-transfer -- fatal to the bootstrap, so the job dies with no + # output dir at all. + assert "--exclude=.cache" in script_content # The per-job source copy is cleaned up with the venv on exit/signal. assert '${SFLOW_SRC_DIR:+"$SFLOW_SRC_DIR"}' in script_content diff --git a/tests/unit/test_config_loader.py b/tests/unit/test_config_loader.py index f21cfda..4df8be3 100644 --- a/tests/unit/test_config_loader.py +++ b/tests/unit/test_config_loader.py @@ -266,3 +266,56 @@ def test_strip_missable_noop_without_workflow(): stripped = strip_missable_tasks(config, ["anything"]) assert stripped == [] assert config == {"version": "0.1"} + + +def test_load_config_warns_that_timeout_is_not_enforced(tmp_path, caplog): + """`timeout:` is accepted but nothing reads it, so a recipe that sets one + LOOKS bounded and is not. The field cannot simply be removed (these models + forbid extra keys, so every config setting it would stop loading), so the + warning is what stops it lying.""" + p = tmp_path / "sflow.yaml" + p.write_text( + """ +version: "0.1" +workflow: + name: wf + timeout: 115m + tasks: + - name: t1 + timeout: 30m + script: + - echo hi + - name: t2 + script: + - echo hi +""".lstrip() + ) + + with caplog.at_level("WARNING"): + ConfigLoader().load_config(p) + + msg = "\n".join(r.message for r in caplog.records) + assert "does not enforce it" in msg + assert "workflow" in msg and "t1" in msg + # t2 sets no timeout, so it must not be named. + assert "t2" not in msg + + +def test_load_config_is_quiet_when_no_timeout_is_set(tmp_path, caplog): + p = tmp_path / "sflow.yaml" + p.write_text( + """ +version: "0.1" +workflow: + name: wf + tasks: + - name: t1 + script: + - echo hi +""".lstrip() + ) + + with caplog.at_level("WARNING"): + ConfigLoader().load_config(p) + + assert "does not enforce it" not in "\n".join(r.message for r in caplog.records) diff --git a/tests/unit/test_core_execution_summary.py b/tests/unit/test_core_execution_summary.py index d544e92..eb3d299 100644 --- a/tests/unit/test_core_execution_summary.py +++ b/tests/unit/test_core_execution_summary.py @@ -1121,3 +1121,60 @@ def test_gpu_usage_chart_rows_sorted_by_node_then_gpu_index(tmp_path): "node10 GPU 3", ], rows assert rows == sorted(rows, key=_resource_row_key) + + +def test_gpu_assignment_reads_the_in_container_view_from_the_placement_record(tmp_path): + """The planner's env is the HOST slice, so it cannot be the in-container column. + + A Slurm step planned for host 2,3 that a container renumbered to 0,1 was + reported as physical 0,1 / in-container 2,3 -- backwards, and impossible. The + step's own numbering only exists in the record it writes. + """ + from sflow.utils.gpu import GPU_MARKER_FILE + + task = _gpu_task("worker", tmp_path, visible="2,3") + out = Path(task.envs["SFLOW_TASK_OUTPUT_DIR"]) + out.mkdir(parents=True, exist_ok=True) + (out / GPU_MARKER_FILE).write_text( + "0,1\n" + "action=verified\n" + "cuda_visible_devices=0,1\n" + "planned_host_indices=2,3\n" + ) + + row = [ + ln + for ln in _render(tmp_path, [task]).split("GPU Assignment", 1)[1].splitlines() + if ln.startswith("worker") + ][0] + # Physical = the planned HOST indices the step PROVED it holds, by UUID. + # In-container = what the step itself was numbered. + assert row.index("2,3") < row.index("0,1"), row + + +def test_gpu_assignment_falls_back_to_the_env_when_the_record_says_unset(tmp_path): + """`` is a sentinel, not a device list. + + The guard used to test for a parenthesised spelling the script never writes, + so it never fired -- it only looked right because the sentinel happens to + parse to no indices. + """ + from sflow.utils.gpu import GPU_MARKER_FILE + + task = _gpu_task("worker", tmp_path, visible="4,5") + out = Path(task.envs["SFLOW_TASK_OUTPUT_DIR"]) + out.mkdir(parents=True, exist_ok=True) + (out / GPU_MARKER_FILE).write_text( + "4,5\naction=fallback\ncuda_visible_devices=\n" + ) + + row = [ + ln + for ln in _render(tmp_path, [task]).split("GPU Assignment", 1)[1].splitlines() + if ln.startswith("worker") + ][0] + # Both columns must read 4,5: the sentinel is discarded and the task's env + # supplies the in-container view. With the old guard the sentinel survived, + # parsed to no indices, and that column silently rendered as "-". + assert row.count("4,5") == 2, row + assert "" not in row, row diff --git a/tests/unit/test_monitoring_postprocess.py b/tests/unit/test_monitoring_postprocess.py index 91b092c..be34f91 100644 --- a/tests/unit/test_monitoring_postprocess.py +++ b/tests/unit/test_monitoring_postprocess.py @@ -1005,6 +1005,54 @@ def test_gpu_panel_draws_one_line_per_device_with_a_legend(tmp_path): assert "max 90.00" in svg and "min 0.00" in svg +def test_coincident_device_lines_stay_distinguishable(tmp_path): + """Two GPUs with IDENTICAL values must not render as one line. + + A tensor-parallel task allocates the same footprint on every rank, so + `gpu_memory_used_mib` for its GPUs is often identical to the byte -- the + second polyline lands exactly on the first and the panel looks like it only + ever had one device, which reads as a collection bug. Colour cannot fix that + (nothing of the lower line is visible to be coloured); the dash can. + """ + rows = _gpu_rows([("n1", 0, 50.0), ("n1", 1, 50.0)]) + + svg_path = tmp_path / "timeline.svg" + assert pp._render_svg(rows, svg_path, title="t") + svg = svg_path.read_text() + + lines = re.findall(r"]*>", svg) + assert len(lines) == 2, lines + dashes = [ + re.search(r'stroke-dasharray="([^"]+)"', ln).group(1) + if "stroke-dasharray" in ln + else "" + for ln in lines + ] + assert dashes[0] != dashes[1], dashes + # The legend swatch must carry its line's dash, or the legend stops matching. + legend, _h = pp._build_device_legend_svg(["GPU 0", "GPU 1"], x0=8, max_x=800) + swatches = [frag for frag in legend if frag.startswith("GPU 0<") == 1 and svg.count(">GPU 1<") == 1 +def test_png_gives_all_eight_gpus_a_distinct_palette_colour(tmp_path, monkeypatch): + """The PNG must use `_SERIES_COLORS`, not matplotlib's default tab10 cycle. + + A node carries at most 8 GPUs, so a full node has to be readable without the + cycle wrapping or two devices landing on near-identical hues. + """ + plt = pytest.importorskip("matplotlib.pyplot") + rows = _gpu_rows([("n1", g, 10.0 * g) for g in range(8)]) + + drawn: list[tuple[str, tuple]] = [] + real_plot = plt.Axes.plot + + def spy(self, *a, **kw): + if "color" in kw: + drawn.append((kw["color"], kw.get("dashes", ()))) + return real_plot(self, *a, **kw) + + monkeypatch.setattr(plt.Axes, "plot", spy) + assert pp._render_png(rows, tmp_path / "timeline.png", title="t") + + colours = [c for c, _d in drawn] + assert colours == list(pp._SERIES_COLORS), colours + assert len(set(colours)) == 8, "two GPUs share a colour" + assert drawn[0][1] == () and drawn[1][1] == (5.0, 3.0), drawn[:2] + + +def test_series_palette_is_the_okabe_ito_set(): + """The colour SET is Okabe-Ito exactly -- order is our own (see below). + + This palette is picked for a property (it survives all three dichromacies) + that cannot be checked by looking at the chart on a normal display, so + "nudging" a colour silently forfeits the reason it was chosen. + """ + assert set(pp._SERIES_COLORS) == { + "#0072b2", "#e69f00", "#009e73", "#d55e00", + "#56b4e9", "#cc79a7", "#f0e442", "#000000", + } + assert len(pp._SERIES_COLORS) == 8, "8 GPUs per node; the cycle must not wrap" + # Yellow carries extra weight, black slightly less -- see _SERIES_WIDTH_SCALE. + assert pp._series_width(7, 1.4) > 1.4 > pp._series_width(4, 1.4) + assert pp._series_width(0, 1.4) == 1.4 + + +# The three near-pairs in Okabe-Ito: two blues, and the warms among themselves. +# Colour alone does not reliably separate these at 1.5px. +_NEAR_PAIRS = ( + ("#0072b2", "#56b4e9"), # blue / sky blue + ("#e69f00", "#d55e00"), # orange / vermilion + ("#e69f00", "#f0e442"), # orange / yellow + ("#d55e00", "#f0e442"), # vermilion / yellow +) + + +def test_first_four_slots_hold_no_near_pair(): + """A 4-GPU task only draws slots 0-3, so those must separate by hue alone. + + The published Okabe-Ito order fails this: it puts orange in slot 1 and + vermilion in slot 3, making the palette's worst pairing the common case. + """ + head = pp._SERIES_COLORS[:4] + for a, b in _NEAR_PAIRS: + assert not (a in head and b in head), f"{a}/{b} both in the first four" + + +def test_near_pairs_do_not_share_a_dash_pattern(): + """Dash is the fallback for the pairs hue cannot separate -- so it must differ. + + `_SERIES_DASHES` cycles every 4, so slots i and i+4 are identical in dash and + would leave such a pair with no redundant encoding at all. + """ + for a, b in _NEAR_PAIRS: + i, j = pp._SERIES_COLORS.index(a), pp._SERIES_COLORS.index(b) + assert pp._series_dash(i) != pp._series_dash(j), ( + f"{a} (slot {i}) and {b} (slot {j}) share a dash pattern" + ) + + +def test_png_greys_its_chrome_so_the_black_series_is_not_mistaken_for_the_frame( + tmp_path, monkeypatch +): + """Slot 8 is pure black; matplotlib's default spines/ticks are black too.""" + plt = pytest.importorskip("matplotlib.pyplot") + rows = _gpu_rows([("n1", g, 10.0 * g) for g in range(8)]) + figs: list = [] + real = plt.Figure.savefig + monkeypatch.setattr( + plt.Figure, + "savefig", + lambda self, *a, **k: (figs.append(self), real(self, *a, **k))[1], + ) + assert pp._render_png(rows, tmp_path / "t.png", title="t") + + (figure,) = figs + for ax in figure.axes: + for spine in ax.spines.values(): + assert spine.get_edgecolor()[:3] != (0.0, 0.0, 0.0), "frame still black" + + +def test_png_device_legend_sits_below_the_panels_not_on_the_data(tmp_path, monkeypatch): + """The legend belongs under the chart, as in the SVG -- not over the lines. + + In-panel (`loc="upper right"`) it covered the top-right of the first panel, + which is exactly where GPU load ramps at the end of a run. + """ + plt = pytest.importorskip("matplotlib.pyplot") + rows = _gpu_rows([("n1", g, 10.0 * g) for g in range(4)]) + + figs: list = [] + real_savefig = plt.Figure.savefig + + def spy(self, *a, **kw): + figs.append(self) + return real_savefig(self, *a, **kw) + + monkeypatch.setattr(plt.Figure, "savefig", spy) + assert pp._render_png(rows, tmp_path / "timeline.png", title="t") + + (figure,) = figs + assert not any(ax.get_legend() for ax in figure.axes), "legend baked into a panel" + (legend,) = figure.legends + assert [t.get_text() for t in legend.get_texts()] == [ + "GPU 0", "GPU 1", "GPU 2", "GPU 3", + ] + # Below the lowest panel, so it can never overlap plotted data. Both extents + # in display pixels (y grows upward): the legend's TOP must clear the bottom + # of the lowest axes. Agg canvas because only a real backend has a renderer, + # and the drawn box -- not the anchor -- is what can overlap the data. + from matplotlib.backends.backend_agg import FigureCanvasAgg + + renderer = FigureCanvasAgg(figure).get_renderer() + assert legend.get_window_extent(renderer).y1 <= min( + ax.get_window_extent(renderer).y0 for ax in figure.axes + ) + + def test_empty_gpu_list_reports_no_gpus_rather_than_all(tmp_path, capsys): """`[]` (task reserved no GPU) must not read as `None` (no subset -> all). diff --git a/tests/unit/test_plugin_backends_slurm_backend.py b/tests/unit/test_plugin_backends_slurm_backend.py index d598b78..9eb8688 100644 --- a/tests/unit/test_plugin_backends_slurm_backend.py +++ b/tests/unit/test_plugin_backends_slurm_backend.py @@ -3,11 +3,13 @@ import asyncio import logging +import unittest.mock import pytest import sflow.plugins.backends.slurm as slurm_mod from sflow.core.backend import Allocation +from sflow.core.compute_node import ComputeNode from sflow.plugins.backends.slurm import SlurmBackend, SlurmBackendConfig @@ -168,6 +170,51 @@ def test_salloc_tokenizes_extra_args_with_bundled_whitespace( assert not any(arg != arg.strip() for arg in salloc_cmd) +def test_salloc_extra_args_keep_repeated_values(monkeypatch, slurm_test_logger): + # `-e '-G 1 -p polar4 -A acct -N 1'` tokenizes to space-separated flag/value + # pairs. Feeding each token through Command.add_opt() treated the bare values + # as option names and de-duped them, so the second "1" deleted the first and + # salloc received `-G -p polar4 -A acct -N 1` -- a silent, wrong allocation. + monkeypatch.delenv("SLURM_JOB_ID", raising=False) + monkeypatch.delenv("SLURM_JOBID", raising=False) + monkeypatch.delenv("SLURM_JOB_NODELIST", raising=False) + monkeypatch.delenv("SLURM_NODELIST", raising=False) + + backend = SlurmBackend( + SlurmBackendConfig( + name="b", + type="slurm", + account="acct", + partition="batch", + nodes=1, + time="00:10:00", + job_name="job", + extra_args=["-G 1 -p polar4 -A general_perflab -N 1"], + gpus_per_node=8, + ) + ) + fake_launcher = _FakeSubprocessLauncher( + script=[ + (0, ["salloc: Granted job allocation 1", "salloc: Nodes node001 are ready for job"]), + (0, ["node001: 10.0.0.1:123"]), + ] + ) + backend._subprocess_launcher = fake_launcher + asyncio.run(backend.allocate()) + + salloc_cmd = list(fake_launcher.calls[0]["command"]) + assert salloc_cmd[-8:] == [ + "-G", + "1", + "-p", + "polar4", + "-A", + "general_perflab", + "-N", + "1", + ] + + def test_env_reuse_applies_exclude_filter(monkeypatch, slurm_test_logger): # A reused Slurm allocation can't take salloc flags, so exclude filters the # resolved node pool instead. @@ -1406,8 +1453,12 @@ def test_slurm_backend_resource_env_preserves_controller_slurm_envs(monkeypatch) env = backend.resource_env(cuda_visible_devices="0,1") assert env["CUDA_VISIBLE_DEVICES"] == "0,1" - # NVIDIA_VISIBLE_DEVICES mirrors the same slice (some stacks honor only that one). - assert env["NVIDIA_VISIBLE_DEVICES"] == "0,1" + # INTENTIONAL CHANGE: NVIDIA_VISIBLE_DEVICES is no longer handed to an srun + # step. It used to mirror the slice, but the container runtime reads it at + # container CREATION and carves + RENUMBERS those devices from 0 -- leaving + # the CUDA_VISIBLE_DEVICES we exported beside it, in host numbering, + # addressing nothing. See SlurmBackend.resource_env. + assert "NVIDIA_VISIBLE_DEVICES" not in env assert env["SLURM_JOB_ID"] == "2222222" assert env["SLURM_JOB_NODELIST"] == "node[001-002]" assert env["SLURM_NNODES"] == "2" @@ -1416,3 +1467,174 @@ def test_slurm_backend_resource_env_preserves_controller_slurm_envs(monkeypatch) assert env["SFLOW_BACKEND_JOB_ID"] == "2222222" assert env["SFLOW_BACKEND_NODELIST"] == "node[001-002]" assert env["SFLOW_BACKEND_NUM_NODES"] == "2" + + +def test_srun_steps_are_not_handed_nvidia_visible_devices(): + """The variable that carves the container must not name a subset. + + pyxis/enroot reads NVIDIA_VISIBLE_DEVICES when it CREATES the container: a + subset there makes it expose only those devices and renumber them from 0, so + the CUDA_VISIBLE_DEVICES exported alongside -- in host numbering -- then + addresses nothing. A worker planned for host 2,3 landed in a 2-GPU container + numbered 0,1 and died with "No CUDA GPUs are available"; one planned for 0,1 + survived only by coincidence. + + Not exporting it lets the container see the node's GPUs with host numbering + intact, so the planned slice is directly addressable and CUDA_VISIBLE_DEVICES + -- the variable CUDA actually reads -- is the only thing sflow sets. + + Docker is unaffected: it overrides resource_env and isolates with + `--gpus device=`. + """ + backend = SlurmBackend( + SlurmBackendConfig( + name="s", + type="slurm", + account="acct", + partition="batch", + nodes=1, + time="00:10:00", + gpus_per_node=8, + ) + ) + env = backend.resource_env(cuda_visible_devices="2,3") + assert env["CUDA_VISIBLE_DEVICES"] == "2,3" + assert "NVIDIA_VISIBLE_DEVICES" not in env + + # A task with no GPU slice gets neither. + assert "NVIDIA_VISIBLE_DEVICES" not in backend.resource_env(cuda_visible_devices=None) + + +def _discovery_backend(script): + """A backend whose only subprocess call is the GPU topology probe.""" + backend = SlurmBackend( + SlurmBackendConfig( + name="b", type="slurm", account="acct", partition="batch", + nodes=1, time="00:10:00", job_name="job", gpus_per_node=4, + ) + ) + backend._subprocess_launcher = _FakeSubprocessLauncher(script=script) + return backend + + +def test_gpu_topology_probe_records_each_nodes_index_to_uuid_map(): + """The probe's output is the ground truth every later check rests on. + + A step can only ask "am I on the cards I was planned for?" by UUID, and this + bare srun is where those UUIDs come from. Parsing it wrong does not fail + loudly -- it leaves the maps empty and silently drops every task in the run to + index arithmetic, so the parse needs a test of its own. + """ + nodes = [ComputeNode(name="n0", ip_address="10.0.0.1", index=0), + ComputeNode(name="n1", ip_address="10.0.0.2", index=1)] + backend = _discovery_backend([(0, [ + "n0 0,GPU-aaa", "n0 1,GPU-bbb", + "n1 0,GPU-ccc", "n1 1,GPU-ddd", + ])]) + + asyncio.run(backend._discover_gpu_uuids(nodes=nodes)) + + # Ordered by HOST index, per node: the same planned slot is a different + # physical card on n1 than on n0, which is why this is not one flat list. + assert nodes[0].gpu_uuids == ["GPU-aaa", "GPU-bbb"] + assert nodes[1].gpu_uuids == ["GPU-ccc", "GPU-ddd"] + + cmd = backend._subprocess_launcher.calls[0]["command"] + # --overlap: shares the allocation instead of queueing behind a real step. + assert "--overlap" in cmd and "--nodelist" in cmd + assert cmd[cmd.index("--nodelist") + 1] == "n0,n1" + + +def test_gpu_topology_probe_rejects_a_partial_reading(): + """A gap means indices are missing, and a partial map resolves to the WRONG card. + + With 0 and 2 reported, index 1 is unaccounted for; treating the two as a + 0,1 run would map planned slot 1 onto the physical card at slot 2. Better to + say nothing and let the step fall back. + """ + nodes = [ComputeNode(name="n0", ip_address="10.0.0.1", index=0)] + backend = _discovery_backend([(0, ["n0 0,GPU-aaa", "n0 2,GPU-ccc"])]) + + asyncio.run(backend._discover_gpu_uuids(nodes=nodes)) + + assert nodes[0].gpu_uuids is None + + +def test_gpu_topology_probe_ignores_noise_and_survives_failure(): + """Never the reason a workflow cannot start.""" + nodes = [ComputeNode(name="n0", ip_address="10.0.0.1", index=0)] + # Slurm prologue chatter, a malformed pair, and a non-UUID value. + backend = _discovery_backend([(0, [ + "srun: job 42 queued and waiting for resources", + "n0 notanindex,GPU-zzz", + "n0 0,NOT-A-UUID", + "n0 0,GPU-aaa", + ])]) + asyncio.run(backend._discover_gpu_uuids(nodes=nodes)) + assert nodes[0].gpu_uuids == ["GPU-aaa"] + + # A failed probe leaves the map untouched rather than raising. + other = [ComputeNode(name="n0", ip_address="10.0.0.1", index=0)] + asyncio.run(_discovery_backend([(1, ["srun: error"])])._discover_gpu_uuids(nodes=other)) + assert other[0].gpu_uuids is None + + +def test_resource_env_hands_the_step_its_planned_uuids(): + """The seam the whole feature hangs on: driver map -> step env. + + _planned_gpu_uuids and gpu_placement.sh were each covered in isolation, but + nothing asserted resource_env actually JOINS them. Drop this line and every + other test still passes while every step silently falls back to index + arithmetic -- the failure mode is invisible. + """ + backend = SlurmBackend( + SlurmBackendConfig( + name="b", type="slurm", account="acct", partition="batch", + nodes=1, time="00:10:00", job_name="job", gpus_per_node=4, + ) + ) + backend.allocation = Allocation( + allocation_id="1", + nodes=[ComputeNode(name="n0", ip_address="10.0.0.1", index=0, + gpu_uuids=["GPU-a", "GPU-b", "GPU-c", "GPU-d"])], + owned=False, + ) + env = backend.resource_env(cuda_visible_devices="2,3") + assert env["SFLOW_PLANNED_GPU_UUIDS"] == "n0=GPU-c,GPU-d" + + # Nothing trustworthy to say -> say nothing. Absent means "fall back", and an + # empty string would read as "this task was planned for no GPUs". + backend.allocation.nodes[0].gpu_uuids = None + assert "SFLOW_PLANNED_GPU_UUIDS" not in backend.resource_env(cuda_visible_devices="2,3") + + +def test_planned_uuids_warns_when_a_slot_is_past_the_nodes_device_count(): + """The warning IS the deliverable here. + + Returning "" already drops the task to index arithmetic; the log line is the + only thing that tells the user their gpus_per_node is bigger than the node. + Silence would leave them with weaker checking and no idea why. + """ + alloc = Allocation( + allocation_id="1", + nodes=[ComputeNode(name="n0", ip_address="10.0.0.1", index=0, + gpu_uuids=["GPU-a", "GPU-b"])], + owned=False, + ) + with unittest.mock.patch.object(slurm_mod._logger, "warning") as warn: + assert slurm_mod._planned_gpu_uuids("0,9", alloc) == "" + assert warn.called + assert "gpus_per_node" in warn.call_args.args[0] + + +def test_allocation_probes_the_gpu_topology(): + """Both allocate() paths must actually call the probe. + + It is best-effort and never raises, so a missing call site is silent: the + maps stay None and every task drops to index arithmetic while the suite + stays green. + """ + import inspect + + # Once for the pre-existing (unowned) allocation, once for the salloc path. + assert inspect.getsource(SlurmBackend).count("await self._discover_gpu_uuids(") == 2 diff --git a/tests/unit/test_srun_gpu_placement.py b/tests/unit/test_srun_gpu_placement.py index 7e691f3..10e65b4 100644 --- a/tests/unit/test_srun_gpu_placement.py +++ b/tests/unit/test_srun_gpu_placement.py @@ -7,7 +7,12 @@ script sflow ships actually resolves to the right devices. """ +import atexit +import itertools +import shutil import subprocess +import tempfile +from pathlib import Path import pytest @@ -25,9 +30,74 @@ def allow_real_bash(fake_process): fake_process.allow_unregistered(True) -def _run(plan: str, observed: str | None) -> subprocess.CompletedProcess: - script = "\n".join(_gpu_placement_prelude(plan) + ['echo "$CUDA_VISIBLE_DEVICES"']) - env = {"PATH": "/usr/bin:/bin"} +def _no_driver_dir() -> str: + """A PATH prefix whose `nvidia-smi` reports nothing. + + The prelude ALWAYS asks the driver now, so without this every test here would + measure whatever GPU the developer's machine happens to have -- this one has + an RTX 3090, which silently rewrote five expectations. Tests that want a + driver state it explicitly via `_run_in_container`. + """ + d = Path(tempfile.mkdtemp(prefix="sflow_no_driver_")) + smi = d / "nvidia-smi" + smi.write_text("#!/bin/bash\nexit 1\n") + smi.chmod(0o755) + return str(d) + + +_NO_DRIVER = _no_driver_dir() +# The placement script is staged into the workflow output dir and sourced from +# there -- that is the only delivery path -- so tests need a dir to stage into. +_STAGE = tempfile.mkdtemp(prefix="sflow_stage_") +# Module-level, so pytest's tmp_path machinery never sees them: clean up by hand +# or every run leaks two /tmp directories. +atexit.register(shutil.rmtree, _NO_DRIVER, ignore_errors=True) +atexit.register(shutil.rmtree, _STAGE, ignore_errors=True) + + +@pytest.fixture +def no_driver(): + return _NO_DRIVER + + +def _run(plan: str, observed: str | None, path_prefix=None) -> subprocess.CompletedProcess: + script = "\n".join(_gpu_placement_prelude(plan, workflow_out_dir=_STAGE) + ['echo "$CUDA_VISIBLE_DEVICES"']) + prefix = f"{path_prefix or _NO_DRIVER}:" + env = {"PATH": f"{prefix}/usr/bin:/bin"} + if observed is not None: + env["CUDA_VISIBLE_DEVICES"] = observed + return subprocess.run( + ["bash", "-c", script], capture_output=True, text=True, env=env + ) + + +def _run_in_container( + plan: str, visible_gpus: int, tmp_path, observed: str | None = None +) -> subprocess.CompletedProcess: + """Run the prelude against a driver reporting N GPUs numbered 0..N-1. + + This is the pyxis/enroot shape: the runtime passes through only this task's + devices and renumbers them from 0. ``observed`` is what --export=ALL carried + into the step, which is usually sflow's own plan in HOST ordinals. + """ + smi = tmp_path / "nvidia-smi" + lines = "\n".join( + f"GPU {i}: NVIDIA GB200 (UUID: GPU-{i:08x})" for i in range(visible_gpus) + ) + pairs = "\n".join(f"{i}, GPU-{i:08x}" for i in range(visible_gpus)) + # Answers both forms: `-L` for the legacy path, `--query-gpu=index,uuid` for + # the in-step probe that records the index the driver actually reports. + smi.write_text( + "#!/bin/bash\n" + 'if [ "$*" != "${*/index,uuid/}" ]; then\n' + f"cat <<'EOF'\n{pairs}\nEOF\n" + "else\n" + f"cat <<'EOF'\n{lines}\nEOF\n" + "fi\n" + ) + smi.chmod(0o755) + script = "\n".join(_gpu_placement_prelude(plan, workflow_out_dir=_STAGE) + ['echo "$CUDA_VISIBLE_DEVICES"']) + env = {"PATH": f"{tmp_path}:/usr/bin:/bin"} if observed is not None: env["CUDA_VISIBLE_DEVICES"] = observed return subprocess.run( @@ -35,6 +105,174 @@ def _run(plan: str, observed: str | None) -> subprocess.CompletedProcess: ) +@pytest.mark.parametrize( + "plan, visible_gpus, expected", + [ + # A container carved to this task's 2 GPUs numbers them 0,1, so host + # ordinals 2,3 name devices that do not exist here. + ("2,3", 2, "0,1"), + # The task that accidentally worked: its plan already matched the + # renumbering, which is why only the decode server ever failed. + ("0,1", 2, "0,1"), + # Single-GPU carve, high host ordinal. + ("3", 1, "0"), + # NOT carved: the whole node is visible with the variable unset, so the + # positional narrowing must still happen or concurrent tasks collide. + ("2,3", 4, "2,3"), + ("0,1", 4, "0,1"), + ], +) +def test_container_carve_keeps_the_containers_own_numbering( + plan, visible_gpus, expected, tmp_path +): + result = _run_in_container(plan, visible_gpus, tmp_path) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == expected + + +@pytest.mark.parametrize( + "plan, observed, visible_gpus, expected", + [ + # THE BUG, exactly as it reached the cluster. srun runs --export=ALL, so + # sflow's own plan arrives in the step as CUDA_VISIBLE_DEVICES. Trusting it + # as an observation is circular: the counts match the plan by construction, + # so the remap concluded "already carved, keep what I see" and kept host + # ordinals 2,3 inside a container holding only 0,1 -> no GPU at all. + ("2,3", "2,3", 2, "0,1"), + # The task that accidentally worked: plan already matched the renumbering. + ("0,1", "0,1", 2, "0,1"), + # Single GPU carve, high host ordinal echoed back. + ("3", "3", 1, "0"), + # NOT a container: slurmstepd set a real in-range slice on a 4-GPU node. + # That IS an observation and must be honoured, or the GRES fix regresses. + ("0,1", "0,1", 4, "0,1"), + ("2,3", "2,3", 4, "2,3"), + # Whole allocation visible and handed over: still narrow positionally. + ("2,3", "0,1,2,3", 4, "2,3"), + ("0,1", "3,5,6,7", 8, "3,5"), + # UUID-form CUDA_VISIBLE_DEVICES carries no positions to remap against. + ("2,3", "GPU-abc,GPU-def", 2, "0,1"), + ], +) +def test_inherited_cuda_visible_devices_is_validated_against_the_driver( + plan, observed, visible_gpus, expected, tmp_path +): + result = _run_in_container(plan, visible_gpus, tmp_path, observed=observed) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == expected + + +def test_no_driver_falls_back_to_the_plan(no_driver): + """With no driver to ask, behave exactly as before this probe existed.""" + result = _run("2,3", None, path_prefix=no_driver) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "2,3" + + +def test_no_driver_says_so_instead_of_degrading_silently(no_driver): + """A GPU task with no nvidia-smi loses the container-renumbering check. + + That is the ptyche failure restored: the inherited value is trusted, and a + container holding 0,1 keeps a plan of 2,3 and sees no GPU. It is the right + fallback -- guessing is worse -- but a slim image must not lose the protection + without saying so, or the only symptom is a task that dies inside CUDA. + """ + result = _run("2,3", "2,3", path_prefix=no_driver) + + assert result.returncode == 0 + assert "no nvidia-smi here" in result.stderr + assert "placement may be wrong" in result.stderr + # The warning must not pollute what the task reads. + assert result.stdout.strip() == "2,3" + + +def test_a_known_plan_never_narrows_less_than_an_unknown_one(no_driver, tmp_path): + """No nvidia-smi + planned UUIDs must still narrow, not bail. + + Knowing MORE about a step must never make sflow do LESS to it. Without a + probe the planned UUIDs cannot be checked, and it is right to record the + placement as unproven -- but returning early there left + CUDA_VISIBLE_DEVICES exactly as inherited, which on a GRES partition is the + whole allocation. Every concurrent worker then picks ordinal 0 and collides + on one physical GPU: precisely the OOM this prelude exists to prevent, and it + only happened when a UUID map was available. A task with NO map, in the same + container on the same partition, was narrowed correctly. + + So the no-probe case degrades to the same index arithmetic instead of + stopping. `action` still says `unverified`, so the e2e audit keeps counting + it as unproven. + """ + out = tmp_path / "out" + out.mkdir() + script = "\n".join( + _gpu_placement_prelude("2,3", workflow_out_dir=_STAGE) + + ['echo "CVD=$CUDA_VISIBLE_DEVICES"'] + ) + env = { + "PATH": f"{no_driver}:/usr/bin:/bin", + "SLURMD_NODENAME": "n0", + "SFLOW_TASK_OUTPUT_DIR": str(out), + # Slurm handed this step the whole 4-GPU allocation, as a GRES step with + # no --gres of its own always is. + "CUDA_VISIBLE_DEVICES": "0,1,2,3", + "SLURM_STEP_GPUS": "0,1,2,3", + "SFLOW_PLANNED_GPU_UUIDS": "n0=GPU-c,GPU-d", + } + r = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) + + assert r.returncode == 0, r.stderr + assert "CVD=2,3" in r.stdout, "must narrow to its own slice, not keep all four" + lines = (out / GPU_MARKER_FILE).read_text().splitlines() + assert "action=unverified" in lines, "narrowed, but not PROVEN -- both are true" + + +def test_a_fallback_record_keeps_the_uuids_it_could_not_find(tmp_path, fp): + """`fallback` must say what was wanted, not claim it never knew. + + When Slurm grants cards the planner never saw, the planned UUIDs WERE + resolved -- they just are not in the grant. A record saying + `planned_uuids=(not resolved)` sends the reader after the driver probe, which + worked fine, instead of at the grant, which is the actual finding. + """ + fp.allow_unregistered(True) + + out = tmp_path / "out" + out.mkdir() + script = "\n".join(_gpu_placement_prelude("2,3", workflow_out_dir=_STAGE)) + r = subprocess.run( + ["bash", "-c", script], + capture_output=True, + text=True, + env={ + "PATH": f"{_smi_stub(tmp_path, ['GPU-x', 'GPU-y'])}:/usr/bin:/bin", + "SLURMD_NODENAME": "n0", + "SFLOW_TASK_OUTPUT_DIR": str(out), + "CUDA_VISIBLE_DEVICES": "0,1", + "SLURM_STEP_GPUS": "6,7", + "SFLOW_PLANNED_GPU_UUIDS": "n0=GPU-c,GPU-d", + }, + ) + assert r.returncode == 0, r.stderr + + lines = (out / GPU_MARKER_FILE).read_text().splitlines() + assert "action=fallback" in lines + assert "planned_uuids=GPU-c,GPU-d" in lines, "resolved, just not present" + # The reason belongs in the record, not only in a step log nobody greps. + assert any("not among the devices Slurm granted" in ln for ln in lines) + # ...and what it landed on instead is right there to compare against. + assert "selected=0 GPU-x" in lines and "selected=1 GPU-y" in lines + + +def test_no_warning_when_the_driver_answers(tmp_path): + result = _run_in_container("2,3", 2, tmp_path, observed="2,3") + + assert result.returncode == 0 + assert "no nvidia-smi" not in result.stderr + + @pytest.mark.parametrize( "plan, observed, expected", [ @@ -55,8 +293,8 @@ def _run(plan: str, observed: str | None) -> subprocess.CompletedProcess: ("2,3", "", "2,3"), ], ) -def test_remap_selects_planned_devices(plan, observed, expected): - result = _run(plan, observed) +def test_remap_selects_planned_devices(plan, observed, expected, no_driver): + result = _run(plan, observed, path_prefix=no_driver) assert result.returncode == 0, result.stderr assert result.stdout.strip() == expected @@ -84,15 +322,68 @@ def test_fewer_devices_than_planned_aborts(): A 4-GPU task handed 2 devices used to be accepted as "already carved" -- the same silent-wrong-placement failure this prelude exists to prevent. + + This is only a real fault while the step holds the task's WHOLE slice. Under + --gpus-per-task it is the normal shape, which is why no prelude is emitted + there (see test_gpus_per_task_leaves_placement_to_slurm). """ result = _run("0,1,2,3", "0,1") assert result.returncode == 97 assert "planned for 4" in result.stderr +@pytest.mark.parametrize("gpus_per_task", ["1", "2"]) +def test_gpus_per_task_leaves_placement_to_slurm(gpus_per_task): + """Per-rank carving breaks the prelude's premise, so it must step aside. + + --gpus-per-task makes the step REQUEST GRES, so Slurm hands each rank only its + own devices instead of handing the step the whole allocation. Counting the + task's full slice against one rank's view then aborts every rank of a perfectly + valid config: 8 ranks at 1 GPU each died with "step has 1 GPU(s) but this task + was planned for 8". Slurm keeps those per-rank sets disjoint itself, so there is + nothing to re-apply. + """ + assert _gpu_placement_prelude("0,1,2,3,4,5,6,7", gpus_per_task=gpus_per_task, workflow_out_dir=_STAGE) == [] + # Without the flag the same plan is still enforced. + assert _gpu_placement_prelude("0,1,2,3,4,5,6,7", workflow_out_dir=_STAGE) != [] + + +def test_build_command_omits_prelude_under_gpus_per_task(): + op = SrunOperator( + SrunOperatorConfig( + name="t", log_to_file=False, ntasks_per_node=8, gpus_per_task="1" + ) + ) + body = op.build_command( + task_name="ranks", + script=["torchrun train.py"], + envs={ + "CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7", + # Staging must be POSSIBLE here, or this would pass via the + # "nowhere to stage it" path instead of the gpus_per_task guard. + "SFLOW_WORKFLOW_OUTPUT_DIR": _STAGE, + }, + ).as_list()[-1] + + assert "SFLOW_GPU_PLAN" not in body + assert "torchrun train.py" in body, "the task itself must still run" + + +def test_many_ranks_sharing_the_task_slice_all_get_it(): + """The shape the samples actually use: ntasks_per_node with no per-rank carving. + + Every rank sees the whole step allocation and must narrow to the same planned + slice -- the app then picks its device by local rank. + """ + selected = { + _run("0,1,2,3", "0,1,2,3,4,5,6,7").stdout.strip() for _ in range(4) + } + assert selected == {"0,1,2,3"} + + def test_no_prelude_without_planned_gpus(): - assert _gpu_placement_prelude(None) == [] - assert _gpu_placement_prelude("") == [] + assert _gpu_placement_prelude(None, workflow_out_dir=_STAGE) == [] + assert _gpu_placement_prelude("", workflow_out_dir=_STAGE) == [] def test_build_command_splices_prelude_into_step_body(): @@ -100,18 +391,22 @@ def test_build_command_splices_prelude_into_step_body(): cmd = op.build_command( task_name="worker", script=["python -c 'import torch'"], - envs={"CUDA_VISIBLE_DEVICES": "2,3"}, + envs={ + "CUDA_VISIBLE_DEVICES": "2,3", + # The script is staged here and sourced from there. + "SFLOW_WORKFLOW_OUTPUT_DIR": _STAGE, + }, ) body = cmd.as_list()[-1] - assert "__sflow_plan='2,3'" in body - # Must precede the user script: the remap is an export the task then inherits. - assert body.index("__sflow_plan") < body.index("import torch") + assert "export SFLOW_GPU_PLAN='2,3'" in body + # Must precede the user script: the placement is an export the task inherits. + assert body.index("SFLOW_GPU_PLAN") < body.index("import torch") def test_build_command_omits_prelude_for_cpu_task(): op = SrunOperator(SrunOperatorConfig(name="t", log_to_file=False)) cmd = op.build_command(task_name="cpu", script=["echo hi"], envs={}) - assert "__sflow_plan" not in cmd.as_list()[-1] + assert "SFLOW_GPU_PLAN" not in cmd.as_list()[-1] # The plan is interpolated into shell text. Every value sflow's planner emits is @@ -133,12 +428,12 @@ def test_build_command_omits_prelude_for_cpu_task(): ], ) def test_non_numeric_plan_never_reaches_the_shell(hostile): - assert _gpu_placement_prelude(hostile) == [] + assert _gpu_placement_prelude(hostile, workflow_out_dir=_STAGE) == [] @pytest.mark.parametrize("plan", ["0", "2,3", "0,1,2,3", "10,11"]) def test_real_plans_are_still_emitted(plan): - assert _gpu_placement_prelude(plan) != [] + assert _gpu_placement_prelude(plan, workflow_out_dir=_STAGE) != [] def test_injected_value_cannot_execute_a_command(): @@ -153,7 +448,7 @@ def test_injected_value_cannot_execute_a_command(): ["bash", "-c", body], capture_output=True, text=True, - env={"PATH": "/usr/bin:/bin", "CUDA_VISIBLE_DEVICES": "0,1,2,3"}, + env={"PATH": f"{_NO_DRIVER}:/usr/bin:/bin", "CUDA_VISIBLE_DEVICES": "0,1,2,3"}, ) assert "INJECTED-COMMAND-RAN" not in result.stdout assert "task-body" in result.stdout, "the task itself must still run" @@ -171,19 +466,19 @@ def test_step_reports_the_devices_it_actually_selected(tmp_path): from sflow.utils.gpu import GPU_MARKER_FILE, planned_gpu_indices, task_gpu_indices plan = "0,1" - body = "\n".join(_gpu_placement_prelude(plan)) + body = "\n".join(_gpu_placement_prelude(plan, workflow_out_dir=_STAGE)) result = subprocess.run( ["bash", "-c", body], capture_output=True, text=True, env={ - "PATH": "/usr/bin:/bin", + "PATH": f"{_NO_DRIVER}:/usr/bin:/bin", "CUDA_VISIBLE_DEVICES": "3,5,6,7", # partial allocation "SFLOW_TASK_OUTPUT_DIR": str(tmp_path), }, ) assert result.returncode == 0, result.stderr - assert (tmp_path / GPU_MARKER_FILE).read_text().strip() == "3,5" + assert _marker_devices(tmp_path) == "3,5" task = SimpleNamespace( cuda_visible_devices=plan, @@ -235,12 +530,13 @@ def test_unparseable_marker_falls_back_to_the_plan(tmp_path): def test_only_rank_zero_writes_the_marker(tmp_path): - """Every rank runs this body; with --gpus-per-task they hold different devices. + """Every rank runs this body, so letting them all truncate one path is a race. - Letting them all truncate one path makes the reported devices a coin flip. + Ranks that were handed different partial allocations resolve to different + devices, which makes the reported value a coin flip. """ plan = "0,1" - body = "\n".join(_gpu_placement_prelude(plan)) + body = "\n".join(_gpu_placement_prelude(plan, workflow_out_dir=_STAGE)) def rank(procid, seen): return subprocess.run( @@ -248,7 +544,7 @@ def rank(procid, seen): capture_output=True, text=True, env={ - "PATH": "/usr/bin:/bin", + "PATH": f"{_NO_DRIVER}:/usr/bin:/bin", "CUDA_VISIBLE_DEVICES": seen, "SLURM_PROCID": procid, "SFLOW_TASK_OUTPUT_DIR": str(tmp_path), @@ -260,11 +556,11 @@ def rank(procid, seen): assert not marker.exists(), "a non-zero rank must not write the marker" assert rank("0", "0,1,2,3").returncode == 0 - assert marker.read_text().strip() == "0,1" + assert marker.read_text().splitlines()[0].strip() == "0,1" # A later non-zero rank must not clobber rank 0's value. assert rank("2", "4,5,6,7").returncode == 0 - assert marker.read_text().strip() == "0,1" + assert marker.read_text().splitlines()[0].strip() == "0,1" def test_multi_node_task_reports_the_plan_not_one_node(tmp_path): @@ -285,6 +581,22 @@ def test_multi_node_task_reports_the_plan_not_one_node(tmp_path): assert task_gpu_indices(multi) == [0, 1], "multi node: fall back to the plan" +def _marker_step(tmp_path, **slurm_env) -> subprocess.CompletedProcess: + body = "\n".join(_gpu_placement_prelude("0,1,2,3", workflow_out_dir=_STAGE)) + return subprocess.run( + ["bash", "-c", body], + capture_output=True, + text=True, + env={ + "PATH": f"{_NO_DRIVER}:/usr/bin:/bin", + "CUDA_VISIBLE_DEVICES": "0,1,2,3", + "SLURM_PROCID": "0", + "SFLOW_TASK_OUTPUT_DIR": str(tmp_path), + **slurm_env, + }, + ) + + def test_multi_node_step_writes_no_marker(tmp_path): """The writer's rule must match the reader's, or the file is a lie on disk. @@ -292,28 +604,66 @@ def test_multi_node_step_writes_no_marker(tmp_path): an artifact that is right for node 0 and wrong for every other node -- and looks authoritative to anyone who opens the task output dir. """ - body = "\n".join(_gpu_placement_prelude("0,1,2,3")) + assert ( + _marker_step(tmp_path, SLURM_STEP_NUM_NODES="2").returncode == 0 + ), "a multi-node step must still run normally" + assert not (tmp_path / GPU_MARKER_FILE).exists() + + assert _marker_step(tmp_path, SLURM_STEP_NUM_NODES="1").returncode == 0 + assert _marker_devices(tmp_path) == "0,1,2,3" + + +def test_allocation_wide_node_count_does_not_suppress_the_marker(tmp_path): + """The guard is about the STEP's nodes, not the allocation's. + + SLURM_NNODES is the backwards-compat alias of SLURM_JOB_NUM_NODES, and + SlurmBackend.resource_env copies the driver's SLURM_* into the env handed to + srun -- so a 4-node allocation puts SLURM_NNODES=4 in every step, including + single-node ones. Reading it here silently killed the marker for every task on + any multi-node allocation, which is exactly where concurrent single-node tasks + get devices the plan cannot predict. + """ + result = _marker_step(tmp_path, SLURM_NNODES="4", SLURM_STEP_NUM_NODES="1") + + assert result.returncode == 0, result.stderr + assert _marker_devices(tmp_path) == "0,1,2,3" + + +def test_multi_node_task_applies_its_per_node_slice_on_every_node(tmp_path): + """A multi-node plan is ONE node's slice, and each node resolves it alone. + + The planner guarantees this: pinned indices repeat per node, the multi-node + count path divides by the node count and refuses nodes with different + allocation cursors. So the same flat plan is evaluated independently in every + node's step, and each must land on its own node's devices -- including when the + nodes were handed different partial allocations. + """ + plan = "0,1" # gpus.count=4 over 2 nodes -> 2 per node + body = "\n".join(_gpu_placement_prelude(plan, workflow_out_dir=_STAGE) + ['echo "$CUDA_VISIBLE_DEVICES"']) - def step(nnodes): + def node(seen, procid, nodeid): return subprocess.run( ["bash", "-c", body], capture_output=True, text=True, env={ - "PATH": "/usr/bin:/bin", - "CUDA_VISIBLE_DEVICES": "0,1,2,3", - "SLURM_PROCID": "0", - "SLURM_NNODES": nnodes, + "PATH": f"{_NO_DRIVER}:/usr/bin:/bin", + "CUDA_VISIBLE_DEVICES": seen, + "SLURM_PROCID": procid, + "SLURM_NODEID": nodeid, + "SLURM_STEP_NUM_NODES": "2", "SFLOW_TASK_OUTPUT_DIR": str(tmp_path), }, ) - assert step("2").returncode == 0, "a multi-node step must still run normally" + # Both nodes handed the whole 4-GPU node: narrow to the planned slots. + assert node("0,1,2,3", "0", "0").stdout.strip() == "0,1" + assert node("0,1,2,3", "1", "1").stdout.strip() == "0,1" + # Node 1 handed a partial allocation: slots are positions, so it follows. + assert node("4,5,6,7", "1", "1").stdout.strip() == "4,5" + # And no node leaves a marker that would speak for the others. assert not (tmp_path / GPU_MARKER_FILE).exists() - assert step("1").returncode == 0 - assert (tmp_path / GPU_MARKER_FILE).read_text().strip() == "0,1,2,3" - def test_prelude_carries_a_short_banner_but_not_the_rationale(): """The generated command explains itself in a few lines, not ten. @@ -322,13 +672,17 @@ def test_prelude_carries_a_short_banner_but_not_the_rationale(): CUDA_VISIBLE_DEVICES; they do not need the marker-write design notes, which live in the module instead. """ - body = _gpu_placement_prelude("1,3")[0] + body = _gpu_placement_prelude("1,3", workflow_out_dir=_STAGE)[0] comments = [line for line in body.splitlines() if line.startswith("#")] # Wrapped, so it is obvious where sflow's block stops and the task's starts. assert body.startswith("# --- sflow GPU placement (begin)") assert body.rstrip("\n").endswith("# --- sflow GPU placement (end) ---------------------------------------------") - assert any("slurmstepd rewrites CUDA_VISIBLE_DEVICES" in c for c in comments) + # Names what it does and the two shapes that need it -- a reader staring at a + # failing srun line must not conclude this block is what broke their GPUs. + assert any("planned GPUs" in c for c in comments) + assert any("GRES" in c for c in comments) + assert any("container" in c for c in comments) # Thin: a banner, not an essay. assert len(comments) <= 8, comments # The long rationale stays out of the shipped shell text. @@ -338,7 +692,889 @@ def test_prelude_carries_a_short_banner_but_not_the_rationale(): def test_banner_does_not_break_the_emitted_shell(): """A comment block is inert, but it is spliced into a `bash -c` body.""" - body = _gpu_placement_prelude("2")[0] + body = _gpu_placement_prelude("2", workflow_out_dir=_STAGE)[0] result = subprocess.run(["bash", "-n"], input=body, text=True, capture_output=True) assert result.returncode == 0, result.stderr + + +# --------------------------------------------------------------------------- +# UUID-verified placement: compare against the plan, and act only if it differs. +# --------------------------------------------------------------------------- + + +def _marker_devices(out_dir) -> str: + """The device list from a marker file. + + Line 1 is the bare list; the lines after it are the audit record (node, the + action taken, every visible index -> UUID, and what CUDA ended up selecting). + Keeping the list first is the contract utils.gpu.task_gpu_indices relies on. + """ + return (out_dir / GPU_MARKER_FILE).read_text().splitlines()[0].strip() + + +def _smi_stub(tmp_path, uuids: list[str]): + """An nvidia-smi that answers BOTH forms the prelude uses.""" + smi = tmp_path / "nvidia-smi" + listing = "\n".join(f"GPU {i}: Fake (UUID: {u})" for i, u in enumerate(uuids)) + plain = "\n".join(uuids) + pairs = "\n".join(f"{i}, {u}" for i, u in enumerate(uuids)) + smi.write_text( + "#!/bin/bash\n" + 'if [ "$1" = "-L" ]; then\n' + f"cat <<'EOF'\n{listing}\nEOF\n" + 'elif [ "$*" != "${*/index,uuid/}" ]; then\n' + f"cat <<'EOF'\n{pairs}\nEOF\n" + "else\n" + f"cat <<'EOF'\n{plain}\nEOF\n" + "fi\n" + ) + smi.chmod(0o755) + return smi.parent + + +_RUN_SEQ = [0] + + +def _run_verified( + tmp_path, *, plan: str, visible: list[str], observed: str | None, planned_map: str, + node: str = "nodeA", +): + """Render the real prelude and report the resulting CVD and the branch taken. + + `action` from the placement record is the authoritative signal for which + branch ran -- verified / fallback / unverified. (It used to be + inferred from NVIDIA_VISIBLE_DEVICES being set, but the prelude no longer + writes that: the container runtime owns it, and it is consumed at container + creation, so writing it afterwards states something false about another layer.) + """ + _RUN_SEQ[0] += 1 + out_dir = tmp_path / f"run{_RUN_SEQ[0]}" + out_dir.mkdir() + script = "\n".join( + _gpu_placement_prelude(plan, workflow_out_dir=_STAGE) + + ['echo "CVD=${CUDA_VISIBLE_DEVICES-}"'] + ) + env = { + "PATH": f"{_smi_stub(tmp_path, visible)}:/usr/bin:/bin", + "SLURMD_NODENAME": node, + "SFLOW_PLANNED_GPU_UUIDS": planned_map, + "SFLOW_TASK_OUTPUT_DIR": str(out_dir), + } + if observed is not None: + env["CUDA_VISIBLE_DEVICES"] = observed + r = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) + cvd = next( + (ln.split("=", 1)[1] for ln in r.stdout.splitlines() if ln.startswith("CVD=")), None + ) + action = None + marker = out_dir / GPU_MARKER_FILE + if marker.exists(): + action = next( + (ln.split("=", 1)[1] for ln in marker.read_text().splitlines() + if ln.startswith("action=")), None + ) + return r, cvd, action + + +def test_placement_names_the_planned_cards_by_uuid_in_every_shape(tmp_path, fp): + """One rule covers every shape: look the planned UUIDs up, name their indices. + + There is deliberately no separate no-op / pin / narrow / re-select branch. + Each of those was the SAME question -- "which indices do the planned cards + have HERE?" -- answered against a different number of visible devices, so + they collapse into one lookup. Whether the answer differs from what was + inherited is a fact the record shows (inherited vs final), not a mode. + + Identity is the UUID, never the index: a device index stops being an identity + the moment a container renumbers from 0. + """ + fp.allow_unregistered(True) # drives a real `bash` subprocess + + want = "nodeA=GPU-c,GPU-d" + + # Whole node visible, inherited value already correct -> same answer back. + _, cvd, action = _run_verified( + tmp_path, plan="2,3", visible=["GPU-a", "GPU-b", "GPU-c", "GPU-d"], + observed="2,3", planned_map=want, + ) + assert (cvd, action) == ("2,3", "verified"), "an already-correct slice must survive" + + # A carved container whose plan happens to start at 0. (This is the shape + # that survived by coincidence before any of this existed.) + _, cvd, action = _run_verified( + tmp_path, plan="0,1", visible=["GPU-c", "GPU-d"], + observed="0,1", planned_map="nodeA=GPU-c,GPU-d", + ) + assert (cvd, action) == ("0,1", "verified") + + # Carved, and CUDA_VISIBLE_DEVICES not set at all. The cards are right -- + # unset means "every visible device" -- but leaving it unset is not safe: + # recipes read this variable to derive ranks, port offsets and device counts, + # and under `set -u` an unset one is a hard error (this killed bare_count_8 + # on perfwg). The same cards get named explicitly, which changes nothing + # about which are used. + _, cvd, action = _run_verified( + tmp_path, plan="2,3", visible=["GPU-c", "GPU-d"], observed=None, planned_map=want, + ) + assert (cvd, action) == ("0,1", "verified"), "an unset CVD must be named, not left unset" + + # The regression this exists for: host ordinals inherited into a 2-GPU + # container name nothing there, so they resolve to 0,1. + _, cvd, action = _run_verified( + tmp_path, plan="2,3", visible=["GPU-c", "GPU-d"], observed="2,3", planned_map=want, + ) + assert (cvd, action) == ("0,1", "verified") + + # More visible than planned, nothing selected: the lookup narrows. + _, cvd, action = _run_verified( + tmp_path, plan="2,3", visible=["GPU-a", "GPU-b", "GPU-c", "GPU-d"], + observed=None, planned_map=want, + ) + assert (cvd, action) == ("2,3", "verified") + + # Holding the right NUMBER of the WRONG cards is the failure a count-based + # check cannot see. It must be loud, not a silent pass. + r, _, _ = _run_verified( + tmp_path, plan="2,3", visible=["GPU-a", "GPU-b"], observed="2,3", planned_map=want, + ) + assert r.returncode == 97 + assert "is not visible" in r.stderr + + +def test_planned_uuids_are_resolved_per_node_not_per_task(tmp_path, fp): + """One flat plan, different physical cards on each node. + + The plan is a list of HOST indices applied identically on every node a task + spans, so slot 2 is a different card on node B than on node A. A step must + read its OWN node's entry; reading another node's would "verify" against + hardware it is not running on. + """ + fp.allow_unregistered(True) + + both = "nodeA=GPU-a2,GPU-a3;nodeB=GPU-b2,GPU-b3" + # On node B the same plan must resolve to B's cards -- and since they are + # already the visible ones, nothing is rewritten. + _, cvd, action = _run_verified( + tmp_path, plan="2,3", visible=["GPU-b2", "GPU-b3"], observed="2,3", + planned_map=both, node="nodeB", + ) + assert (cvd, action) == ("0,1", "verified") + + # Node B seeing node A's cards is a real placement error. + r, _, _ = _run_verified( + tmp_path, plan="2,3", visible=["GPU-a2", "GPU-a3"], observed="2,3", + planned_map=both, node="nodeB", + ) + assert r.returncode == 97 + + +def test_without_a_uuid_map_the_previous_behaviour_is_unchanged(tmp_path, fp): + """Clusters that cannot be probed must keep working exactly as before. + + No SFLOW_PLANNED_GPU_UUIDS -> the old index arithmetic runs, untouched. + """ + fp.allow_unregistered(True) + + _, cvd, action = _run_verified( + tmp_path, plan="2,3", visible=["GPU-a", "GPU-b", "GPU-c", "GPU-d"], + observed="2,3", planned_map="", + ) + assert (cvd, action) == ("2,3", "fallback"), "legacy path still runs" + + # And the legacy container remap still happens. + _, cvd, _ = _run_verified( + tmp_path, plan="2,3", visible=["GPU-c", "GPU-d"], observed="2,3", planned_map="", + ) + assert cvd == "0,1" + + +def test_planned_uuid_map_is_per_backend_and_refuses_partial_answers(): + """Encoding the plan as physical cards, per node. + + Two Slurm backends can have different gpus_per_node and different node sets, + so this is built from the calling backend's OWN allocation. A node whose + topology is unknown, or whose device count cannot contain a planned slot, is + omitted entirely: a partial map would let a step "verify" against a reading + that cannot hold the card it was planned for, which is worse than falling back. + """ + from sflow.core.backend import Allocation + from sflow.core.compute_node import ComputeNode + from sflow.plugins.backends.slurm import _planned_gpu_uuids + + def node(name, uuids): + return ComputeNode( + name=name, ip_address="1.2.3.4", index=0, num_gpus=len(uuids or []), + gpu_uuids=uuids, + ) + + alloc = Allocation( + allocation_id="1", + nodes=[node("n0", ["A0", "A1", "A2", "A3"]), node("n1", ["B0", "B1", "B2", "B3"])], + owned=False, + ) + # The SAME flat plan resolves to different physical cards per node. + assert _planned_gpu_uuids("2,3", alloc) == "n0=A2,A3;n1=B2,B3" + # Order is the plan's order, not the device order. + assert _planned_gpu_uuids("3,0", alloc) == "n0=A3,A0;n1=B3,B0" + + # Nothing trustworthy to say -> say nothing, and the step falls back. + assert _planned_gpu_uuids(None, alloc) == "" + assert _planned_gpu_uuids("2,3", None) == "" + assert _planned_gpu_uuids("9", alloc) == "", "a slot past the device count" + assert _planned_gpu_uuids("GPU-abc", alloc) == "", "UUID-form plan" + assert _planned_gpu_uuids("2,3", Allocation( + allocation_id="1", nodes=[node("n0", None)], owned=False + )) == "", "unprobed node" + + # A heterogeneous allocation contributes only the nodes it can vouch for. + mixed = Allocation( + allocation_id="1", + nodes=[node("small", ["S0", "S1"]), node("big", ["G0", "G1", "G2", "G3"])], + owned=False, + ) + assert _planned_gpu_uuids("2,3", mixed) == "big=G2,G3" + + +def test_marker_is_an_audit_record_whose_first_line_stays_the_device_list(tmp_path, fp): + """A bare index list cannot settle "was this placed right?". + + An index means nothing once a container renumbers from 0, so the marker also + records what the step actually SAW (index -> UUID), what CUDA ended up + selecting, and which branch ran. Compared against the allocation topology in + the summary, that distinguishes a bad placement from a recipe using the wrong + device -- after the run, without reproducing it. + + Line 1 stays the plain device list: utils.gpu.task_gpu_indices reads it, and + markers written by an older sflow must still parse. + """ + fp.allow_unregistered(True) # drives a real `bash` subprocess + + out = tmp_path / "task_out" + out.mkdir() + script = "\n".join(_gpu_placement_prelude("2,3", workflow_out_dir=_STAGE)) + env = { + "PATH": f"{_smi_stub(tmp_path, ['GPU-c', 'GPU-d'])}:/usr/bin:/bin", + "SLURMD_NODENAME": "nodeA", + "SFLOW_PLANNED_GPU_UUIDS": "nodeA=GPU-c,GPU-d", + "SFLOW_TASK_OUTPUT_DIR": str(out), + # Host ordinals inherited into a carved container -> a re-select. Both + # variables, as sflow's driver exports them (Backend.resource_env). + "CUDA_VISIBLE_DEVICES": "2,3", + "NVIDIA_VISIBLE_DEVICES": "2,3", + } + r = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) + assert r.returncode == 0, r.stderr + + text = (out / GPU_MARKER_FILE).read_text() + lines = text.splitlines() + # The contract the reader depends on. + assert lines[0] == "0,1" + from sflow.utils.gpu import parse_cuda_visible_devices + + assert parse_cuda_visible_devices(lines[0]) == [0, 1] + + record = dict( + line.split("=", 1) for line in lines[1:] if "=" in line and not line.startswith("visible=") + ) + assert record["node"] == "nodeA" + assert record["action"] == "verified" + # The record must show what ARRIVED, not just the post-state: inherited "2,3" + # next to a final "0,1" is the whole story of the container renumbering, and + # it is also how "did sflow change anything?" is answered now that there is + # no separate no-op action to read. + assert record["cuda_visible_devices_inherited"] == "2,3" + assert "located by UUID" in record["reason"] + # NVIDIA_VISIBLE_DEVICES as it ARRIVED, so a diff can be attributed. sflow + # never exports it (Backend.resource_env pops it), so whatever is here came + # from the container runtime -- which is exactly what makes it evidence. + assert record["nvidia_visible_devices_inherited"] == "2,3" + assert record["planned_host_indices"] == "2,3" + assert record["planned_uuids"] == "GPU-c,GPU-d" + assert record["cuda_visible_devices"] == "0,1" + # Everything the step could see, and what CUDA will really use. + assert "visible=0 GPU-c" in lines and "visible=1 GPU-d" in lines + assert "selected=0 GPU-c" in lines and "selected=1 GPU-d" in lines + + +def test_marker_shows_an_unchanged_slice_as_inherited_equals_final(tmp_path, fp): + """When the lookup confirms what arrived, the record has to say so. + + There is no "noop" action to read any more, so the record carries the fact + instead: an inherited value equal to the final one means the step was already + on the planned cards and nothing was rewritten. + """ + fp.allow_unregistered(True) + + out = tmp_path / "task_out" + out.mkdir() + script = "\n".join(_gpu_placement_prelude("2,3", workflow_out_dir=_STAGE)) + env = { + "PATH": f"{_smi_stub(tmp_path, ['GPU-a', 'GPU-b', 'GPU-c', 'GPU-d'])}:/usr/bin:/bin", + "SLURMD_NODENAME": "nodeA", + "SFLOW_PLANNED_GPU_UUIDS": "nodeA=GPU-c,GPU-d", + "SFLOW_TASK_OUTPUT_DIR": str(out), + "CUDA_VISIBLE_DEVICES": "2,3", + } + assert subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env).returncode == 0 + + lines = (out / GPU_MARKER_FILE).read_text().splitlines() + assert lines[0] == "2,3" + assert "action=verified" in lines + assert "cuda_visible_devices_inherited=2,3" in lines + assert "cuda_visible_devices=2,3" in lines, "unchanged: inherited == final" + assert "visible_gpu_count=4" in lines + # It selected the planned cards out of the whole node. + assert "selected=2 GPU-c" in lines and "selected=3 GPU-d" in lines + + +def test_multi_node_step_records_per_node_instead_of_racing_one_file(tmp_path, fp): + """Every node of a multi-node task has its own devices. + + One shared marker would be a race whose winner is arbitrary, which is why the + plain marker stays single-node-only. The per-node record is suffixed with the + node name so each node's evidence survives. + """ + fp.allow_unregistered(True) + + out = tmp_path / "task_out" + out.mkdir() + script = "\n".join(_gpu_placement_prelude("2,3", workflow_out_dir=_STAGE)) + env = { + "PATH": f"{_smi_stub(tmp_path, ['GPU-c', 'GPU-d'])}:/usr/bin:/bin", + "SLURMD_NODENAME": "nodeB", + "SFLOW_PLANNED_GPU_UUIDS": "nodeB=GPU-c,GPU-d", + "SFLOW_TASK_OUTPUT_DIR": str(out), + "SLURM_STEP_NUM_NODES": "2", + "SLURM_PROCID": "1", + "SLURM_LOCALID": "0", + "CUDA_VISIBLE_DEVICES": "0,1", + } + assert subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env).returncode == 0 + + assert not (out / GPU_MARKER_FILE).exists(), "the flat marker stays single-node" + # ..log, so .log stays the extension for viewers and globs. + per_node = out / GPU_MARKER_FILE.replace(".log", ".nodeB.log") + assert per_node.exists() + assert "node=nodeB" in per_node.read_text() + + +def test_slurm_reports_discovered_gpu_topology_to_the_summary(): + """The allocation's bare-metal topology belongs in the run record. + + Without it the per-task records have nothing to be compared against: knowing a + task held GPU-c only answers "was that the right card?" if the run also says + which cards the nodes had. + """ + from sflow.core.backend import Allocation + from sflow.core.compute_node import ComputeNode + from sflow.plugins.backends.slurm import SlurmBackend, SlurmBackendConfig + + backend = SlurmBackend( + SlurmBackendConfig( + name="s", + type="slurm", + account="acct", + partition="batch", + nodes=2, + time="00:10:00", + gpus_per_node=2, + ) + ) + assert backend.node_topology_report is None, "nothing to say before allocation" + + backend.allocation = Allocation( + allocation_id="1", + nodes=[ + ComputeNode(name="n0", ip_address="1", index=0, num_gpus=2, + gpu_uuids=["GPU-a", "GPU-b"]), + ComputeNode(name="n1", ip_address="2", index=1, num_gpus=0, gpu_uuids=None), + ], + owned=False, + ) + report = backend.node_topology_report + assert "n0: 2 GPU(s)" in report + assert "[0] GPU-a" in report and "[1] GPU-b" in report + # A node with no probe contributes nothing rather than a misleading empty entry. + assert "n1" not in report + + +def test_placement_logic_is_staged_once_and_sourced_not_pasted(tmp_path, fp): + """The step body should point at the logic, not carry it. + + ~150 lines of shell in every srun command made a failing command line + unreadable and duplicated the same text per task. It is staged once into the + run's output dir -- shared storage on Slurm, so every node can read it -- and + sourced. Sourced, not executed: it exports CUDA_VISIBLE_DEVICES into the + task's own shell, which a child process could not. + """ + fp.allow_unregistered(True) # drives a real `bash` subprocess + + out = tmp_path / "run" + out.mkdir() + body = "\n".join(_gpu_placement_prelude("2,3", workflow_out_dir=str(out))) + + staged = out / ".sflow" / "gpu_placement.sh" + assert staged.exists(), "the script must be staged where every node can read it" + assert body.count("\n") < 12, f"the step body should stay short:\n{body}" + assert f'. "{staged}"' in body, "sourced" + assert "bash " + str(staged) not in body, "must not be run as a child process" + # Inputs travel as environment, so nothing is interpolated into shell text. + assert "export SFLOW_GPU_PLAN='2,3'" in body + assert f"export SFLOW_GPU_MARKER='{GPU_MARKER_FILE}'" in body + + # Staging twice is fine (many tasks launch concurrently). + again = "\n".join(_gpu_placement_prelude("0,1", workflow_out_dir=str(out))) + assert staged.exists() and f'. "{staged}"' in again + + # And the staged script still does the job when sourced. + smi = _smi_stub(tmp_path, ["GPU-c", "GPU-d"]) + r = subprocess.run( + ["bash", "-c", f'{body}\necho "CVD=$CUDA_VISIBLE_DEVICES"'], + capture_output=True, + text=True, + env={ + "PATH": f"{smi}:/usr/bin:/bin", + "SLURMD_NODENAME": "nodeA", + "SFLOW_PLANNED_GPU_UUIDS": "nodeA=GPU-c,GPU-d", + "CUDA_VISIBLE_DEVICES": "2,3", + }, + ) + assert r.returncode == 0, r.stderr + assert "CVD=0,1" in r.stdout + + +def test_placement_is_skipped_when_the_script_cannot_be_staged(caplog): + """Nowhere the nodes can read it from -> do nothing, loudly. + + The alternative was pasting a second copy of the logic into the command line, + which meant two delivery paths to keep honest (and a source rewrite so + `return` stayed valid outside a sourced file). It was also unreachable in + practice: run_support always sets SFLOW_WORKFLOW_OUTPUT_DIR, so the only + trigger is an unwritable output dir -- by which point the run's own logs are + already broken. + + Skipping leaves CUDA_VISIBLE_DEVICES exactly as exported, which is the + behaviour from before this prelude existed. + """ + import logging + + with caplog.at_level(logging.WARNING): + assert _gpu_placement_prelude("2,3", workflow_out_dir=None) == [] + assert "Could not stage the GPU placement script" in caplog.text + + +def test_the_record_states_detected_values_not_echoed_ones(tmp_path, fp): + """Everything in the record that claims to be runtime must BE runtime. + + The point of the record is to settle "did this run on the cards we meant?" + after the fact, so a field that merely echoes what the driver passed in would + be worse than absent -- it would agree with the plan by construction. + + Probed inside the step: the visible UUIDs AND their indices (nvidia-smi + ignores CUDA_VISIBLE_DEVICES, so this is the namespace's real view), plus the + effective CUDA_VISIBLE_DEVICES / NVIDIA_VISIBLE_DEVICES after the decision. + Passed in by the driver: planned_host_indices and planned_uuids -- named as + "planned" precisely so they are not mistaken for observations. + """ + fp.allow_unregistered(True) # drives a real `bash` subprocess + + out = tmp_path / "task_out" + out.mkdir() + # The step sees cards the PLAN never mentions, so nothing here can be an echo + # of the plan: a real probe is the only way these names appear. + visible = ["GPU-zz0", "GPU-zz1", "GPU-c", "GPU-d"] + script = "\n".join(_gpu_placement_prelude("2,3", workflow_out_dir=_STAGE)) + r = subprocess.run( + ["bash", "-c", script], + capture_output=True, + text=True, + env={ + "PATH": f"{_smi_stub(tmp_path, visible)}:/usr/bin:/bin", + "SLURMD_NODENAME": "nodeA", + "SFLOW_PLANNED_GPU_UUIDS": "nodeA=GPU-c,GPU-d", + "SFLOW_TASK_OUTPUT_DIR": str(out), + "CUDA_VISIBLE_DEVICES": "2,3", + }, + ) + assert r.returncode == 0, r.stderr + lines = (out / GPU_MARKER_FILE).read_text().splitlines() + + # Detected: every visible card, with the index nvidia-smi reported. + assert "visible_gpu_count=4" in lines + for index, uuid in enumerate(visible): + assert f"visible={index} {uuid}" in lines + # Cards the plan never named still show up -> this is an observation. + assert "visible=0 GPU-zz0" in lines + + # Detected: the effective environment after the decision. Nothing changed + # here (the plan already selected the right cards). + assert "action=verified" in lines + assert "cuda_visible_devices=2,3" in lines + # There is no nvidia_visible_devices= post-state, and that is deliberate: + # sflow never writes that variable, so such a line could only ever repeat + # nvidia_visible_devices_inherited= and would read as a second, independent + # observation that it is not. + assert "nvidia_visible_devices_inherited=" in lines + assert not any(ln.startswith("nvidia_visible_devices=") for ln in lines) + + # Passed in, and labelled as such. + assert "planned_host_indices=2,3" in lines + assert "planned_uuids=GPU-c,GPU-d" in lines + + # selected= is the join of the two: real CVD resolved through the real probe. + assert "selected=2 GPU-c" in lines and "selected=3 GPU-d" in lines + + +def test_an_unset_cuda_visible_devices_is_named_not_left_unset(tmp_path, fp): + """The whole-node case: right cards, but nothing names them. + + `gpus.count` equal to a node's device count plans every GPU, so an + already-correct step can arrive with CUDA_VISIBLE_DEVICES unset -- "all of + them" is the right answer. Leaving it unset is what "nothing to do" would + look like to a naive check, and it broke a real run: assert_placement.sh dereferences the variable under + `set -u` and bare_count_8 died with "CUDA_VISIBLE_DEVICES: unbound variable" + on both nodes. Recipes also derive ranks and port offsets from it. + + So the cards get named explicitly. Same devices, contract intact. + """ + fp.allow_unregistered(True) # drives a real `bash` subprocess + + out = tmp_path / "task_out" + out.mkdir() + whole_node = ["GPU-w", "GPU-x", "GPU-y", "GPU-z"] + script = "\n".join(_gpu_placement_prelude("0,1,2,3", workflow_out_dir=_STAGE)) + r = subprocess.run( + # `set -u` on purpose: that is how the failure surfaced. + ["bash", "-c", f'set -u\n{script}\necho "CVD=${{CUDA_VISIBLE_DEVICES}}"'], + capture_output=True, + text=True, + env={ + "PATH": f"{_smi_stub(tmp_path, whole_node)}:/usr/bin:/bin", + "SLURMD_NODENAME": "c01", + "SFLOW_PLANNED_GPU_UUIDS": "c01=" + ",".join(whole_node), + "SFLOW_TASK_OUTPUT_DIR": str(out), + # no CUDA_VISIBLE_DEVICES in the environment at all + }, + ) + assert r.returncode == 0, r.stderr + assert "unbound variable" not in r.stderr + assert "CVD=0,1,2,3" in r.stdout + + lines = (out / GPU_MARKER_FILE).read_text().splitlines() + assert "action=verified" in lines + assert "cuda_visible_devices_inherited=" in lines, "named, not left unset" + assert "cuda_visible_devices=0,1,2,3" in lines + # Still the planned cards -- naming them changed nothing about which. + for index, uuid in enumerate(whole_node): + assert f"selected={index} {uuid}" in lines + + +def test_record_distinguishes_unset_from_set_but_empty(tmp_path, fp): + """Unset and empty are different states, and reporting them alike hides a bug. + + Both variables are recorded VERBATIM -- only and + are substituted, so the two can be told apart. sflow does not annotate what a + value means to CUDA or to a container runtime: it does not own those + semantics, and a gloss would be wrong on any stack that differs. A reader + seeing `all` can conclude the runtime carved nothing; that is their call to + make from the raw value. + """ + fp.allow_unregistered(True) # drives a real `bash` subprocess + + def record(env_extra): + out = tmp_path / f"t{abs(hash(tuple(sorted(env_extra.items()))))}" + out.mkdir() + script = "\n".join(_gpu_placement_prelude("0", workflow_out_dir=_STAGE)) + env = { + "PATH": f"{_smi_stub(tmp_path, ['GPU-a'])}:/usr/bin:/bin", + "SLURMD_NODENAME": "n0", + "SFLOW_PLANNED_GPU_UUIDS": "n0=GPU-a", + "SFLOW_TASK_OUTPUT_DIR": str(out), + } + env.update(env_extra) + r = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) + assert r.returncode == 0, r.stderr + return dict( + line.split("=", 1) + for line in (out / GPU_MARKER_FILE).read_text().splitlines() + if "=" in line and not line.startswith(("visible=", "selected=")) + ) + + # Neither variable present at all. + rec = record({}) + assert rec["cuda_visible_devices_inherited"] == "" + assert rec["nvidia_visible_devices_inherited"] == "" + + # Present but empty -- a different state, and it must not read as "unset". + rec = record({"CUDA_VISIBLE_DEVICES": "", "NVIDIA_VISIBLE_DEVICES": "all"}) + assert rec["cuda_visible_devices_inherited"] == "" + # Recorded verbatim: "all" is the runtime's own value, not sflow's gloss on it. + assert rec["nvidia_visible_devices_inherited"] == "all" + + +def test_gres_clusters_resolve_by_uuid_first_and_fall_back_only_on_a_miss(tmp_path, fp): + """On a GRES cluster the UUID lookup still runs first, and usually wins. + + Where GRES allocates the GPUs, slurmstepd sets CUDA_VISIBLE_DEVICES itself and + a `--overlap` step can be handed the whole node's grant. The lookup handles + that with no special case: the planned cards are in the grant, so their + indices are found and the step narrows to exactly them -- by identity, not by + counting positions. + + Only when a planned card is genuinely ABSENT does who-chose-the-devices + matter. Slurm may have granted cards the planner never saw, in which case the + plan can only mean a POSITION into the grant; resolving it as a host index + would abort a healthy run with exit 97. Two independent signals say Slurm + owns the devices, either sufficient: + * SLURM_STEP_GPUS -- set by Slurm only when THIS STEP took GRES + * the CUDA_VISIBLE_DEVICES we exported did not survive into the step + + Both are STEP-scoped. SLURM_JOB_GPUS deliberately is NOT one of them: it says + the JOB has GPUs, and Backend.resource_env copies every SLURM_* var from the + DRIVER's environment into every task, so on the `batch --submit` path it was + set for every step on any GRES cluster -- and because steps run --overlap + (Slurm does not carve per step, so SLURM_STEP_GPUS is unset there) it was the + ONLY signal in play. The hard fail below could not fire on the very clusters + it was written for. Asserted here so it is not quietly reinstated. + """ + fp.allow_unregistered(True) # drives a real `bash` subprocess + + seq = itertools.count() + + def run(env_extra, visible): + out = tmp_path / f"g{next(seq)}" + out.mkdir() + script = "\n".join( + _gpu_placement_prelude("2,3", workflow_out_dir=_STAGE) + + ['echo "CVD=$CUDA_VISIBLE_DEVICES"'] + ) + env = { + "PATH": f"{_smi_stub(tmp_path, visible)}:/usr/bin:/bin", + "SLURMD_NODENAME": "n0", + # The driver resolved the plan against a bare-metal probe of the WHOLE + # node; Slurm then granted something else entirely. + "SFLOW_PLANNED_GPU_UUIDS": "n0=GPU-c,GPU-d", + "SFLOW_TASK_OUTPUT_DIR": str(out), + } + env.update(env_extra) + r = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) + action = next( + (ln.split("=", 1)[1] for ln in (out / GPU_MARKER_FILE).read_text().splitlines() + if ln.startswith("action=")), None + ) if (out / GPU_MARKER_FILE).exists() else None + return r, action + + granted = ["GPU-x", "GPU-y"] # not the planned cards -- Slurm chose these + + # The `--overlap` shape from the field report: every concurrent worker is + # handed the WHOLE node and they all race on device 0 unless something + # narrows them. The planned cards are in the grant, so the lookup finds them + # and narrows by identity -- no fallback, no counting. + r, action = run( + {"CUDA_VISIBLE_DEVICES": "0,1,2,3", "SLURM_STEP_GPUS": "0,1,2,3"}, + ["GPU-a", "GPU-b", "GPU-c", "GPU-d"], + ) + assert (r.returncode, action) == (0, "verified"), r.stderr + assert "CVD=2,3" in r.stdout, "must land on the planned cards, not keep all four" + + # Now the miss: Slurm granted cards the planner never saw. Deferring keeps a + # healthy run alive instead of aborting on a plan that was never a host index. + r, action = run({"CUDA_VISIBLE_DEVICES": "0,1", "SLURM_STEP_GPUS": "3,5"}, granted) + assert r.returncode == 0, f"must not abort a GRES step\n{r.stderr}" + assert "CVD=0,1" in r.stdout, "both granted devices are the planned slice here" + assert action == "fallback" + assert "not among the devices Slurm granted this step" in r.stderr + + # ...but the JOB-level variable alone must NOT defer: it is what the driver + # leaks into every task, and treating it as "Slurm chose this step's devices" + # is what made the hard fail unreachable. CVD here still equals the plan, so + # nothing step-scoped says Slurm touched it -> this is a real mis-placement. + r, action = run({"CUDA_VISIBLE_DEVICES": "2,3", "SLURM_JOB_GPUS": "0,1,2,3"}, granted) + assert r.returncode == 97, f"a leaked job-level var must not excuse a miss\n{r.stderr}" + assert action == "missing" + assert "is not visible on" in r.stderr + + # No GRES variables, but our export was replaced anyway -> same deference. + r, action = run({"CUDA_VISIBLE_DEVICES": "0,1"}, granted) + assert (r.returncode, action) == (0, "fallback"), r.stderr + + # Deferring still narrows: 4 granted, none of them planned, a 2-slot plan + # -> positions 2,3 of the grant. + r, action = run( + {"CUDA_VISIBLE_DEVICES": "0,1,2,3", "SLURM_STEP_GPUS": "0,1,2,3"}, + ["GPU-p", "GPU-q", "GPU-r", "GPU-s"], + ) + assert (r.returncode, action) == (0, "fallback"), r.stderr + assert "CVD=2,3" in r.stdout + + # And the container case is untouched: our export DID survive, so the UUID + # lookup still corrects the renumbering. + r, action = run({"CUDA_VISIBLE_DEVICES": "2,3"}, ["GPU-c", "GPU-d"]) + assert (r.returncode, action) == (0, "verified"), r.stderr + assert "CVD=0,1" in r.stdout + + +def test_placement_is_skipped_when_staging_raises(tmp_path): + """Staging raises -> no placement, not a second inlined copy of the logic. + + The unstageable dir is a path UNDER A REGULAR FILE, so mkdir raises + NotADirectoryError (an OSError). Not `chmod(0o500)`: CI runs the suite as + root, and root ignores directory permission bits -- the write simply + succeeded there and the test failed on a premise that was never true, rather + than on the behaviour it meant to pin. + """ + blocker = tmp_path / "not-a-dir" + blocker.write_text("") + + assert _gpu_placement_prelude("2,3", workflow_out_dir=str(blocker / "out")) == [] + + +def test_a_hard_placement_failure_still_leaves_the_record(tmp_path): + """exit 97 is exactly when someone needs to see what the step held. + + The writer used to sit below every exit, so the one run worth diagnosing was + the one that produced no record at all. + """ + from types import SimpleNamespace + + from sflow.utils.gpu import task_gpu_record + + out_dir = tmp_path / "run" + out_dir.mkdir() + script = "\n".join(_gpu_placement_prelude("2,3", workflow_out_dir=_STAGE)) + r = subprocess.run( + ["bash", "-c", script], + capture_output=True, + text=True, + env={ + # Planned cards resolved, but neither is present here. + "PATH": f"{_smi_stub(tmp_path, ['GPU-x', 'GPU-y'])}:/usr/bin:/bin", + "SLURMD_NODENAME": "nodeA", + "SFLOW_PLANNED_GPU_UUIDS": "nodeA=GPU-a,GPU-b", + "SFLOW_TASK_OUTPUT_DIR": str(out_dir), + }, + ) + assert r.returncode == 97, r.stdout + r.stderr + record = task_gpu_record(SimpleNamespace(envs={"SFLOW_TASK_OUTPUT_DIR": str(out_dir)})) + assert record["action"] == "missing" + assert record["planned_uuids"] == "GPU-a,GPU-b" + assert "not visible" in record["reason"] + + +def _drive(tmp_path, name, *, plan, env_extra, visible=None): + """Run the real staged script and hand back (result, record dict).""" + out = tmp_path / name + out.mkdir() + script = "\n".join(_gpu_placement_prelude(plan, workflow_out_dir=_STAGE)) + env = { + "PATH": f"{_smi_stub(tmp_path, visible) if visible else _NO_DRIVER}:/usr/bin:/bin", + "SLURMD_NODENAME": "nodeA", + "SFLOW_TASK_OUTPUT_DIR": str(out), + } + env.update(env_extra) + r = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) + from types import SimpleNamespace + + from sflow.utils.gpu import task_gpu_record + + return r, task_gpu_record(SimpleNamespace(envs={"SFLOW_TASK_OUTPUT_DIR": str(out)})) + + +def test_a_node_missing_from_the_planned_map_says_so_once(tmp_path): + """The FQDN-vs-short-name mismatch silently switches verification off. + + The map is keyed by the name the DRIVER saw; the step keys by + $SLURMD_NODENAME. When they disagree the step finds no entry, falls back to + index arithmetic, and every task in the run quietly stops being verified -- + with nothing failing. The warning is the only signal, so it needs a test. + """ + r, record = _drive( + tmp_path, "mismatch", plan="0,1", + env_extra={ + # Map names the short form; the step reports the FQDN. + "SLURMD_NODENAME": "nodeA.cluster.example.com", + "SFLOW_PLANNED_GPU_UUIDS": "nodeA=GPU-a,GPU-b", + "CUDA_VISIBLE_DEVICES": "0,1", + }, + visible=["GPU-a", "GPU-b"], + ) + assert r.returncode == 0, r.stderr + assert "no planned-GPU entry for node 'nodeA.cluster.example.com'" in r.stderr + assert "falling back to device-index placement" in r.stderr + # Unverified, and the record must say why rather than claim a proven placement. + assert record["action"] == "fallback" + assert record["planned_uuids"] == "(not resolved)" + + +def test_every_abort_path_leaves_a_record(tmp_path): + """exit 97 without a record is the worst outcome: a run that failed on + placement and cannot be diagnosed. Proven for all three aborts, not just one. + """ + # too-few: 2 visible, 4 planned. + r, record = _drive( + tmp_path, "toofew", plan="0,1,2,3", + env_extra={"CUDA_VISIBLE_DEVICES": "0,1"}, visible=["GPU-x", "GPU-y"], + ) + assert r.returncode == 97, r.stderr + assert record["action"] == "too-few", record + assert "planned for 4" in record["reason"] + + # out-of-range: slot 9 is outside a 4-device visible set. + r, record = _drive( + tmp_path, "oor", plan="9", + env_extra={"CUDA_VISIBLE_DEVICES": "0,1,2,3"}, + visible=["GPU-a", "GPU-b", "GPU-c", "GPU-d"], + ) + assert r.returncode == 97, r.stderr + assert record["action"] == "out-of-range", record + assert "outside the visible devices" in record["reason"] + + +def test_the_record_names_every_selected_device_or_says_it_cannot(tmp_path): + """`selected=` must resolve each device CUDA will use back to a UUID. + + An index the step cannot see, and a device named as a UUID rather than an + ordinal, are both real shapes -- the record must state them rather than drop + the line, or a reader silently sees fewer devices than the task used. + """ + out = tmp_path / "sel" + out.mkdir() + # Force a post-state CUDA cannot resolve: no driver, so placement falls back + # to the plan verbatim, and the plan names devices nvidia-smi never reported. + script = "\n".join(_gpu_placement_prelude("0,1", workflow_out_dir=_STAGE)) + r = subprocess.run( + ["bash", "-c", script], capture_output=True, text=True, + env={"PATH": f"{_NO_DRIVER}:/usr/bin:/bin", "SLURMD_NODENAME": "nodeA", + "SFLOW_TASK_OUTPUT_DIR": str(out)}, + ) + assert r.returncode == 0, r.stderr + lines = (out / GPU_MARKER_FILE).read_text().splitlines() + selected = [ln for ln in lines if ln.startswith("selected=")] + assert len(selected) == 2, lines + # No driver -> nothing to resolve against, so each says so rather than lying. + assert all("(not visible here)" in ln for ln in selected), selected + + +def test_the_marker_format_keys_are_the_contract(tmp_path): + """Three independent parsers read this file: utils/gpu.py (Python), + sample_test.sh::gpu_placement_verified (shell sed), and a human. Renaming a + key breaks the shell reader silently, so the key set is pinned here. + """ + r, record = _drive( + tmp_path, "keys", plan="0,1", + env_extra={"SFLOW_PLANNED_GPU_UUIDS": "nodeA=GPU-a,GPU-b", + "CUDA_VISIBLE_DEVICES": "0,1"}, + visible=["GPU-a", "GPU-b"], + ) + assert r.returncode == 0, r.stderr + assert set(record) == { + "devices", # line 1, the bare device list + "node", "action", "reason", + "cuda_visible_devices_inherited", "nvidia_visible_devices_inherited", + "cuda_visible_devices", "planned_host_indices", "planned_uuids", + "visible_gpu_count", + }, sorted(record) + # The two keys the shell reader greps for must carry parseable values. + assert record["action"] == "verified" + assert record["planned_uuids"] == "GPU-a,GPU-b" diff --git a/tests/unit/test_srun_operator.py b/tests/unit/test_srun_operator.py index d2f192c..da5c2f8 100644 --- a/tests/unit/test_srun_operator.py +++ b/tests/unit/test_srun_operator.py @@ -445,3 +445,40 @@ def test_srun_step_exports_all_without_overrides(): assert "ENROOT_RUNTIME_PATH=" not in s assert "SLURM_JOB_ID=" not in s assert "--jobid 777" in s + + +def test_srun_operator_splits_bundled_extra_args(): + """srun was the one extra_args consumer that skipped normalize_extra_args. + + A bundled entry then reached srun as a SINGLE argv token it cannot parse, and + srun ignores what it cannot parse -- so the flags were silently dropped rather + than rejected. Every other consumer (slurm/docker/k8s backends, docker_run, + kubectl_config) normalizes; this one is now consistent with them. + """ + op = SrunOperator( + SrunOperatorConfig(name="op_srun", extra_args=["--gres=gpu:8 --exclusive"]) + ) + argv = op.build_command(task_name="t1", script=["echo hi"], envs={}).as_list() + + assert "--gres=gpu:8" in argv + assert "--exclusive" in argv + assert "--gres=gpu:8 --exclusive" not in argv, "must not stay one token" + # Still ahead of the payload, which is what srun requires. + assert argv.index("--exclusive") < argv.index("bash") + + +def test_srun_operator_extra_args_cannot_delete_an_sflow_managed_flag(): + """add_opt(name) defaults to append=False, which REMOVES every existing option + of that name (core/command.py). Routed through it, a user flag that collided + with an sflow-managed one silently deleted sflow's own value instead of + sitting alongside it. add_arg appends, so both survive and srun decides. + """ + op = SrunOperator( + SrunOperatorConfig( + name="op_srun", partition="batch", extra_args=["--partition", "other"] + ) + ) + argv = op.build_command(task_name="t1", script=["echo hi"], envs={}).as_list() + + assert "batch" in argv, "sflow's own --partition must survive" + assert "other" in argv diff --git a/tests/unit/test_utils_gpu.py b/tests/unit/test_utils_gpu.py index e929cfd..05cd274 100644 --- a/tests/unit/test_utils_gpu.py +++ b/tests/unit/test_utils_gpu.py @@ -132,3 +132,87 @@ def test_two_docker_tasks_do_not_collide_on_gpu_zero(): a = _Task(cuda_visible_devices="0,1", envs={"CUDA_VISIBLE_DEVICES": "0,1"}) b = _Task(cuda_visible_devices="2,3", envs={"CUDA_VISIBLE_DEVICES": "0,1"}) assert set(task_gpu_indices(a)).isdisjoint(task_gpu_indices(b)) + + +def test_a_carved_container_reports_the_physical_card_not_its_own_numbering(tmp_path): + """The container's numbering is not a physical id. + + A task planned for host GPUs 2,3 that a runtime carved and renumbered from 0 + records "0,1" as the devices it used. Reporting that as PHYSICAL put every + containerised task on the wrong card in the summary's GPU Assignment table + (physical 0,1 / in-container 2,3 -- backwards, and impossible), and made the + hardware monitor sample the wrong GPUs. + + The step proves by UUID that it holds exactly the planned cards, and exits 97 + when it does not, so the planned HOST indices are the physical ones. + """ + from sflow.utils.gpu import GPU_MARKER_FILE, task_gpu_indices, task_gpu_record + + out = tmp_path / "boxed_high" + out.mkdir() + (out / GPU_MARKER_FILE).write_text( + "0,1\n" + "node=ptyche0074\n" + "action=verified\n" + "cuda_visible_devices=0,1\n" + "planned_host_indices=2,3\n" + "planned_uuids=GPU-74b4,GPU-0989\n" + "visible=0 GPU-74b4\n" + "visible=1 GPU-0989\n" + "selected=0 GPU-74b4\n" + "selected=1 GPU-0989\n" + ) + + class _Task: + envs = {"SFLOW_TASK_OUTPUT_DIR": str(out), "CUDA_VISIBLE_DEVICES": "2,3"} + assigned_nodes = ["ptyche0074"] + cuda_visible_devices = "2,3" + + task = _Task() + assert task_gpu_indices(task) == [2, 3], "physical is the planned host slice" + # ...and the step's own view is still recoverable for the other column. + assert task_gpu_record(task)["cuda_visible_devices"] == "0,1" + assert task_gpu_record(task)["action"] == "verified" + + +def test_an_unverified_record_still_reports_what_the_step_selected(tmp_path): + """Without UUID proof the record's line 1 is the best answer, as before. + + On that path the step selected host ordinals itself, so line 1 means what it + always meant -- notably the GRES case where slurmstepd hands over a partial + allocation and plan 0,1 really is physical 3,5. + """ + from sflow.utils.gpu import GPU_MARKER_FILE, task_gpu_indices + + out = tmp_path / "gres_task" + out.mkdir() + (out / GPU_MARKER_FILE).write_text( + "3,5\nnode=n0\naction=fallback\ncuda_visible_devices=3,5\n" + "planned_host_indices=0,1\n" + ) + + class _Task: + envs = {"SFLOW_TASK_OUTPUT_DIR": str(out), "CUDA_VISIBLE_DEVICES": "0,1"} + assigned_nodes = ["n0"] + cuda_visible_devices = "0,1" + + assert task_gpu_indices(_Task()) == [3, 5] + + +def test_task_gpu_record_returns_empty_when_there_is_nothing_to_read(tmp_path): + """Three ways to have no record, one answer: {}. + + Callers branch on empty-vs-parsed, so an exception or a half-filled dict here + would surface as a confidently wrong device list in the run summary. + """ + from types import SimpleNamespace + + from sflow.utils.gpu import GPU_MARKER_FILE, task_gpu_record + + # No task output dir at all. + assert task_gpu_record(SimpleNamespace(envs={})) == {} + # Dir known, marker absent (OSError). + assert task_gpu_record(SimpleNamespace(envs={"SFLOW_TASK_OUTPUT_DIR": str(tmp_path)})) == {} + # Marker present but empty -- there is not even a device line to trust. + (tmp_path / GPU_MARKER_FILE).write_text("") + assert task_gpu_record(SimpleNamespace(envs={"SFLOW_TASK_OUTPUT_DIR": str(tmp_path)})) == {}