From 13cbf2c04fe3fff1a4f231654c20a5a99237a6aa Mon Sep 17 00:00:00 2001 From: xinyuej Date: Tue, 1 Sep 2026 15:55:59 +0800 Subject: [PATCH 1/3] docs: align user guide intro with the README description --- docs/user/backends.md | 9 +- docs/user/configuration.md | 7 +- docs/user/intro.md | 16 +- docs/user/quick-reference.md | 2 +- .../self_contained/kubernetes/multinode.yaml | 94 ++++++- src/sflow/cli/batch.py | 18 +- src/sflow/cli/compose.py | 3 +- src/sflow/config/loader.py | 68 ++++- src/sflow/config/schema.py | 4 +- src/sflow/skills/writing-sflow-yaml/SKILL.md | 2 +- .../scripts/check_gpu_plan.py | 8 +- .../scripts/validate_sflow_yaml.py | 12 +- tests/e2e_tests/sample_test.sh | 264 +++++++++++++++++- tests/unit/test_config_loader.py | 177 +++++++++++- tests/unit/test_config_loader_merge.py | 16 +- .../test_plugin_backends_slurm_backend.py | 57 ++++ 16 files changed, 694 insertions(+), 63 deletions(-) diff --git a/docs/user/backends.md b/docs/user/backends.md index 8f8b365..345678c 100644 --- a/docs/user/backends.md +++ b/docs/user/backends.md @@ -119,7 +119,14 @@ also accepts: | `job_name` | workflow name | `salloc --job-name`. Falls back to the backend name, then is set to the workflow name at resolve time. | | `offload_task_logs` | `true` | Have `srun` write each task's `.log` on the compute side (via `--output`) instead of streaming every line through the driver. Auto-falls back to streaming on an interactive TTY / `--tui`. Also toggled by `--offload-task-logs` / `--no-offload-task-logs` or `SFLOW_OFFLOAD_TASK_LOGS`. | -> `time` accepts either an `"HH:MM:SS"` string or an integer number of minutes. +> `time` accepts either an `HH:MM:SS` walltime or an integer number of minutes. +> Quoting is optional: `time: 10:00:00` and `time: "10:00:00"` are equivalent. +> A bare integer (`time: 5400`) means **minutes**, matching `sbatch --time`. +> +> Earlier sflow versions required the quotes. Unquoted, YAML 1.1 read `10:00:00` +> as the base-60 integer `36000`, which Slurm then interpreted as 36000 *minutes* +> — a 10-hour request silently became 25 days. sflow now parses config scalars +> with YAML 1.2 rules, so the unquoted form means what it looks like. ### Cluster-specific flags (`extra_args`) diff --git a/docs/user/configuration.md b/docs/user/configuration.md index 870f96f..d7b110c 100644 --- a/docs/user/configuration.md +++ b/docs/user/configuration.md @@ -23,12 +23,15 @@ Looking for a quick lookup of all config fields? See the [Quick Reference](./qui ## version -Currently supported: +Optional. Omit it and sflow uses `"0.1"`, the only value that has ever existed: ```yaml -version: "0.1" +version: "0.1" # optional ``` +If you do declare it, it must be `"0.1"` — any other value is rejected. When +several files are merged, they must not declare conflicting versions. + ## variables Variables can be written as a **dict** or a **list** (they are normalized internally). diff --git a/docs/user/intro.md b/docs/user/intro.md index 77259f5..3d75ec1 100644 --- a/docs/user/intro.md +++ b/docs/user/intro.md @@ -3,22 +3,26 @@ title: Introduction sidebar_position: 1 --- -`sflow` is a **declarative workflow descriptor** that separates _what to deploy_ from _where to deploy it_. +## What is sflow + +A **declarative workflow descriptor for massive GPU clusters** that separates _what to deploy_ from _where to deploy it_. :::tip Find the right feature Not sure where to start? Open the [Feature Map](/feature-map) to choose a goal, see which sflow features apply, and jump to the relevant docs. Building with an AI coding agent? See [Agent Skills](/docs/agents/intro). ::: -An application's deployment steps are usually logically the same regardless of the underlying infrastructure. Take [NVIDIA Dynamo](https://github.com/ai-dynamo/dynamo) as an example: you start etcd and NATS, launch a frontend server, spin up workers that register to the frontend, and the service is up. That logical flow never changes — but making it actually run on Slurm, Docker Compose, or Kubernetes requires a different set of infrastructure-specific scripts, resource management, and networking tweaks each time, and the effort must be repeated for every new platform. - -`sflow` is trying to eliminate this duplication. You describe the workflow once in a portable YAML format — tasks, dependencies, resources, and launch methods — and `sflow` delegates execution to the target infrastructure through swappable backends, leveraging each platform's native ecosystem rather than reimplementing it (e.g. Kubernetes, Helm charts, Argo Workflows). +**One semantic across every platform — backend agnostic by design.** A deployment's logic never changes: start etcd and NATS, launch a frontend, spin up workers, run the benchmark. Only the infrastructure glue does — and today that glue is rewritten from scratch for every platform. `sflow` consolidates it into a single portable YAML: tasks, dependencies, resources, and launch methods. The **same `sflow.yaml` runs on Docker, Slurm, and Kubernetes** — swap the backend fragment, rewrite nothing. Backends delegate to each platform's native ecosystem (`srun`/MPI, `docker run`, pods and MPI jobs) rather than reimplementing it. -Pluggable extensions such as probes and artifacts integrate naturally without coupling your workflow to any specific platform. Write one `sflow.yaml` and run it across environments with minimal changes. +**Cluster-level orchestration at scale.** Topology-aware node and GPU placement, multi-node replicas and sweeps, readiness/failure probes, and batch submission — so one descriptor drives hundreds of GPUs instead of a pile of hand-written bash. -The current focus is **Slurm**, which — unlike Kubernetes or Docker — lacks a built-in workflow orchestration layer, making multi-step deployments especially cumbersome. As of the v0.3.0 release, the Docker and Kubernetes backends ship as well, alongside `local` and `slurm`. Kubernetes support is new in v0.3.0 and has known limitations (interactive `sflow run` only, `monitor:` not yet supported, tested on bare-metal Kubernetes and GKE) — see [Backends](./backends.md#kubernetes-backend). +All four backends ship today: `local`, `docker`, `slurm`, and `kubernetes` (`k8s` / `k8s_mpi`). It is light enough to write and debug a recipe on your laptop and submit that same file to the cluster. ![sflow TUI](/img/sflow_tui.gif) +Define _what to run_ in a `sflow.yaml` — tasks, dependencies, how to launch each task, and required resources. `sflow` executes the DAG in order, collects logs, and organizes outputs into a consistent directory structure. Example of a dynamo PD disaggregation LLM inference service workflow: + +![Workflow DAG Example](/img/workflow-dag.png) + ## Docs versions The docs site version selector intentionally shows only maintained documentation streams: diff --git a/docs/user/quick-reference.md b/docs/user/quick-reference.md index fb6cc58..203ce84 100644 --- a/docs/user/quick-reference.md +++ b/docs/user/quick-reference.md @@ -11,7 +11,7 @@ For detailed explanations and examples, see [Configuration](./configuration.md). | Field | Required | Type | Default | Description | |-------|----------|------|---------|-------------| -| `version` | Yes | string | — | Schema version. Must be `"0.1"`. | +| `version` | No | string | `"0.1"` | Schema version. If declared, must be `"0.1"`. | | `variables` | | dict / list | — | Global variables available to expressions and task env. | | `artifacts` | | dict / list | — | Named resources referenced by URI. | | `backends` | | dict / list | — | Compute backends (`local`, `slurm`, `docker`, `kubernetes`). | diff --git a/examples/self_contained/kubernetes/multinode.yaml b/examples/self_contained/kubernetes/multinode.yaml index 3c3a269..b4dff85 100644 --- a/examples/self_contained/kubernetes/multinode.yaml +++ b/examples/self_contained/kubernetes/multinode.yaml @@ -3,21 +3,90 @@ version: "0.1" # Multi-node distributed training on Kubernetes via sflow's default device-plugin # GPU scheduling (scheduling: device_plugin -> nvidia.com/gpu limits). # -# The reservation reserves 4 GPU nodes; the single task is split into 4 pods (one -# per reserved node, leader = index 0) each requesting 8 GPUs via the device-plugin -# limit. sflow injects SFLOW_TASK_NODE_INDEX and SFLOW_LEADER_ADDRESS -# per pod (plus the shared SFLOW_TASK_ASSIGNED_NODE_IPS) so torchrun can rendezvous. +# The reservation reserves NUM_NODES GPU nodes; the single task is split into one +# pod per reserved node (leader = index 0), each requesting GPUS_PER_NODE GPUs via +# the device-plugin limit. sflow injects SFLOW_TASK_NODE_INDEX and +# SFLOW_LEADER_ADDRESS per pod so torchrun can rendezvous. +# +# The topology lives in variables and NOTHING restates it. torchrun's --nnodes must +# equal the pod count or the c10d rendezvous blocks forever waiting for ranks that +# will never join -- a silent hang, not an error. Deriving both from one variable is +# why that cannot drift. # # `scheduling: device_plugin` is sflow's default; swap to `scheduling: dra` (a work in # progress) on a cluster with nvidia-dra-driver-gpu to request GPUs via DRA ResourceClaims. +variables: + NUM_NODES: + description: "Nodes to reserve == the number of pods == torchrun --nnodes." + type: integer + value: 2 + GPUS_PER_NODE: + description: "GPUs per node == torchrun --nproc_per_node." + type: integer + value: 2 + TOTAL_GPUS: + description: "GPUs across the whole task; sflow divides it per pod." + type: integer + value: ${{ variables.NUM_NODES * variables.GPUS_PER_NODE }} + +artifacts: + # The recipe used to invoke a bare `train.py` that existed nowhere -- the run only + # ever got that far after the rendezvous, so the missing file was masked by the + # hang. Ship the smallest thing that PROVES the point of the example: every rank + # joined and NCCL actually carried a collective between the nodes. + - name: TRAIN_SCRIPT + uri: file://train.py + content: | + import os + import socket + import torch + import torch.distributed as dist + + host = socket.gethostname() + node = os.environ.get("SFLOW_TASK_NODE_INDEX", "?") + local_rank = int(os.environ["LOCAL_RANK"]) + + def log(msg): + # Rank-tagged and flushed: torchrun interleaves every rank onto one stream, + # and an unflushed print is lost entirely if a peer dies mid-collective. + print(f"[node {node} | rank {os.environ.get('RANK', '?')} | {host}] {msg}", flush=True) + + log(f"starting: local_rank={local_rank} master={os.environ.get('MASTER_ADDR')}:" + f"{os.environ.get('MASTER_PORT')} world_size={os.environ.get('WORLD_SIZE')}") + + # Everything before this line is local. If the run hangs HERE, the rendezvous + # never completed -- which is exactly the --nnodes mismatch this recipe hit. + log("entering rendezvous (init_process_group)...") + dist.init_process_group("nccl") + rank, world = dist.get_rank(), dist.get_world_size() + torch.cuda.set_device(local_rank) + log(f"rendezvous OK: rank {rank}/{world} on {torch.cuda.get_device_name(local_rank)}") + + # Sum of all ranks -- wrong if any rank is missing or NCCL silently no-ops. + t = torch.full((1,), float(rank), device="cuda") + dist.all_reduce(t) + expected = world * (world - 1) / 2 + log(f"all_reduce -> {t.item():.0f} (expected {expected:.0f})") + assert t.item() == expected, f"all_reduce={t.item()} expected={expected}" + + dist.barrier() + if rank == 0: + hosts = [None] * world + dist.all_gather_object(hosts, host) + log(f"OK: {world} ranks across {len(set(hosts))} host(s) agreed: {sorted(set(hosts))}") + else: + dist.all_gather_object([None] * world, host) + dist.destroy_process_group() + log("done") + backends: - name: k8s type: kubernetes default: true namespace: default - nodes: 4 - gpus_per_node: 8 + nodes: ${{ variables.NUM_NODES }} + gpus_per_node: ${{ variables.GPUS_PER_NODE }} host_network: true scheduling: device_plugin @@ -33,17 +102,18 @@ workflow: operator: trainer resources: nodes: - count: 4 + count: ${{ variables.NUM_NODES }} gpus: - count: 32 # 8 GPUs per pod across 4 nodes + count: ${{ variables.TOTAL_GPUS }} script: # SFLOW_LEADER_ADDRESS is the leader (node 0) IP; SFLOW_TASK_NODE_INDEX is - # 0 on the leader and 1..N-1 on workers. Keep NNODES in sync with nodes.count. + # 0 on the leader and 1..N-1 on workers. - export MASTER_ADDR="$SFLOW_LEADER_ADDRESS" - export MASTER_PORT=12345 - - export NNODES=4 - - export NPROC_PER_NODE=8 + - export NNODES=${{ variables.NUM_NODES }} + - export NPROC_PER_NODE=${{ variables.GPUS_PER_NODE }} - export NODE_RANK="$SFLOW_TASK_NODE_INDEX" + - nvidia-smi - > torchrun --nnodes=$NNODES @@ -51,4 +121,4 @@ workflow: --node_rank=$NODE_RANK --master_addr=$MASTER_ADDR --master_port=$MASTER_PORT - train.py + ${{ artifacts.TRAIN_SCRIPT.path }} diff --git a/src/sflow/cli/batch.py b/src/sflow/cli/batch.py index 3b036de..7ae5a33 100644 --- a/src/sflow/cli/batch.py +++ b/src/sflow/cli/batch.py @@ -22,6 +22,7 @@ import yaml as _yaml from sflow.app.sflow import SflowApp +from sflow.config.loader import safe_load from sflow.cli import DOCS_URL, app from sflow.cli._args import ( # split_list_arg re-exported for back-compat EnableTaskMonitorOption, @@ -279,10 +280,9 @@ def _resolve_sbatch_extra_args( domain_map: dict[str, list[Any]] = {} for cfg_path in config_files: try: - import yaml as _yaml with open(cfg_path) as fh: - data = _yaml.safe_load(fh) + data = safe_load(fh) if data: var_map.update(_build_var_map(data)) domain_map.update(extract_domains_from_raw_config(data)) @@ -1803,7 +1803,6 @@ def _derive_backend_int( runs when the regex returns None, so currently-resolving configs are unchanged and partial fragments (which the pipeline can't validate) keep their regex result. """ - import yaml as _yaml merged_var_map: dict[str, Any] = {} all_data: list[dict] = [] @@ -1811,7 +1810,7 @@ def _derive_backend_int( for f in config_files: try: with open(f) as fh: - raw = _yaml.safe_load(fh) + raw = safe_load(fh) if isinstance(raw, dict): all_data.append(raw) merged_var_map.update(_build_var_map(raw)) @@ -2108,7 +2107,6 @@ def _scan_sflow_yamls(paths: list[Path]) -> list[Path]: """ import glob as _glob - import yaml as _yaml candidates: list[Path] = [] for p in paths: @@ -2132,8 +2130,8 @@ def _scan_sflow_yamls(paths: list[Path]) -> list[Path]: for f in candidates: try: with open(f) as fh: - data = _yaml.safe_load(fh) - if isinstance(data, dict) and "version" in data: + data = safe_load(fh) + if isinstance(data, dict) and "workflow" in data: valid.append(f.resolve()) except Exception: continue @@ -2216,10 +2214,9 @@ def _run_bulk_submit( # Warn about CLI variable overrides if cli_var_keys: try: - import yaml as _yaml with open(yaml_file) as fh: - data = _yaml.safe_load(fh) + data = safe_load(fh) config_var_names: set[str] = set() raw_vars = data.get("variables") or [] if isinstance(raw_vars, dict): @@ -2326,10 +2323,9 @@ def _run_bulk_submit( # (loading the YAML) when the backend has no resolvable ``nodes`` field. row_nodes = _derive_nodes([yaml_file], cli_overrides=cli_set_var) if row_nodes is None: - import yaml as _yaml with open(yaml_file) as fh: - data = _yaml.safe_load(fh) + data = safe_load(fh) row_nodes = _first_node_column_int( _build_var_map(data, cli_overrides=cli_set_var) ) diff --git a/src/sflow/cli/compose.py b/src/sflow/cli/compose.py index d813088..d368899 100644 --- a/src/sflow/cli/compose.py +++ b/src/sflow/cli/compose.py @@ -18,6 +18,7 @@ ConfigLoader, _normalize_script_plain_mappings, merge_config_dicts, + safe_load, ) from sflow.logging import configure_logging, get_logger from sflow.resolution import ExpressionResolver, resolve_variables_inline @@ -85,7 +86,7 @@ def _compose_files( config_dicts: List[Dict[str, Any]] = [] for path in files: with open(path, "r") as f: - data = yaml.safe_load(f) + data = safe_load(f) if data is None: raise ValueError(f"Configuration file is empty: {path}") _normalize_script_plain_mappings(data) diff --git a/src/sflow/config/loader.py b/src/sflow/config/loader.py index 24ab25a..068563d 100644 --- a/src/sflow/config/loader.py +++ b/src/sflow/config/loader.py @@ -1,8 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import re from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import IO, Any, Dict, List, Optional import yaml from pydantic import ValidationError @@ -17,6 +18,63 @@ _logger = get_logger(__name__) +# --- YAML 1.2 scalar rules --------------------------------------------------- +# PyYAML implements YAML 1.1, which resolves `10:00:00` to the base-60 integer +# 36000. Slurm walltimes are written in exactly that shape, so an unquoted +# `time: 10:00:00` became `--time=36000` -- and Slurm reads a bare integer as +# *minutes*, turning a 10-hour request into 25 days. The sexagesimal float form +# (`10:00:00.5`) failed schema validation outright. +# +# Why this is fixed here and not with a pydantic validator on the field: the +# loss happens strictly upstream of pydantic. By the time a validator runs it +# has received the integer 36000, which is indistinguishable from a legitimate +# `time: 5400` (Slurm reads a bare int as minutes). A validator could reject, +# never repair, and would only cover `slurm.time` rather than every field. +# +# YAML 1.2 removed sexagesimal scalars. Below are PyYAML's own int/float +# patterns with only the `(:[0-5]?[0-9])+` branch dropped; every other rule is +# copied verbatim, including the quirk that an exponent requires a sign (so +# `1e10` stays a string exactly as before). Only `H:MM:SS`-shaped values change. +# `test_patterns_stay_in_sync_with_pyyaml` fails loudly if PyYAML edits these. +_INT_WITHOUT_SEXAGESIMAL = re.compile( + r"""^(?:[-+]?0b[0-1_]+ + |[-+]?0[0-7_]+ + |[-+]?(?:0|[1-9][0-9_]*) + |[-+]?0x[0-9a-fA-F_]+)$""", + re.X, +) +_FLOAT_WITHOUT_SEXAGESIMAL = re.compile( + r"""^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)? + |\.[0-9][0-9_]*(?:[eE][-+][0-9]+)? + |[-+]?\.(?:inf|Inf|INF) + |\.(?:nan|NaN|NAN))$""", + re.X, +) +_PATCHED_PATTERNS = { + "tag:yaml.org,2002:int": _INT_WITHOUT_SEXAGESIMAL, + "tag:yaml.org,2002:float": _FLOAT_WITHOUT_SEXAGESIMAL, +} + + +class SflowSafeLoader(yaml.SafeLoader): + """``yaml.SafeLoader`` without YAML 1.1 base-60 int/float scalars.""" + + +# Copying the table first is required: `yaml_implicit_resolvers` is inherited by +# reference, so mutating it in place would reconfigure `yaml.safe_load` process-wide. +SflowSafeLoader.yaml_implicit_resolvers = { + first_char: [ + (tag, _PATCHED_PATTERNS.get(tag, pattern)) for tag, pattern in resolvers + ] + for first_char, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items() +} + + +def safe_load(stream: str | bytes | IO[str] | IO[bytes]) -> Any: + """Drop-in ``yaml.safe_load`` that keeps ``H:MM:SS`` values as strings.""" + return yaml.load(stream, Loader=SflowSafeLoader) + + def strip_missable_tasks( config_data: Dict[str, Any], missable_patterns: List[str], @@ -443,8 +501,8 @@ def merge_config_dicts( ) errors: list[str] = [] - if "version" not in merged: - errors.append("No 'version' field found in any input file") + # `version` is optional and defaults to "0.1" in the schema. A conflict + # between files is still a hard error (handled above), but absence is fine. wf = merged.get("workflow") if not wf: errors.append("No 'workflow' section found in any input file") @@ -561,7 +619,7 @@ def load_config( try: with open(path, "r") as f: - config_data = yaml.safe_load(f) + config_data = safe_load(f) except yaml.YAMLError as e: raise ValueError(f"Error parsing YAML configuration: {e}") @@ -625,7 +683,7 @@ def load_configs( raise FileNotFoundError(f"Configuration file not found: {path}") try: with open(path, "r") as f: - data = yaml.safe_load(f) + data = safe_load(f) except yaml.YAMLError as e: raise ValueError(f"Error parsing YAML configuration ({path}): {e}") if data is None: diff --git a/src/sflow/config/schema.py b/src/sflow/config/schema.py index 4178b94..9f434fc 100644 --- a/src/sflow/config/schema.py +++ b/src/sflow/config/schema.py @@ -1194,7 +1194,9 @@ class SflowConfig(StrictBaseModel): Main configuration model for Sflow. """ - version: str + # Optional. Only "0.1" has ever existed, so requiring it bought nothing but a + # line of boilerplate at the top of every recipe. Omitting it means "0.1". + version: str = "0.1" variables: Optional[ Annotated[List[VariableConfig], BeforeValidator(_normalize_to_list)] ] = None diff --git a/src/sflow/skills/writing-sflow-yaml/SKILL.md b/src/sflow/skills/writing-sflow-yaml/SKILL.md index c5abd2b..69d054b 100644 --- a/src/sflow/skills/writing-sflow-yaml/SKILL.md +++ b/src/sflow/skills/writing-sflow-yaml/SKILL.md @@ -29,7 +29,7 @@ Every recipe has the same top-level shape. You design `workflow.tasks` first; th optional scaffolding you add only when a step calls for it. ```yaml -version: "0.1" # schema version (NOT the sflow release) — required in EVERY file +version: "0.1" # optional — schema version (NOT the sflow release); omit ⇒ "0.1" variables: { ... } # optional — ${{ variables.X }} in YAML, ${X} in scripts artifacts: [ ... ] # optional — named paths/URIs, auto-mounted at the same path backends: [ ... ] # optional — omit ⇒ a default `local` backend diff --git a/src/sflow/skills/writing-sflow-yaml/scripts/check_gpu_plan.py b/src/sflow/skills/writing-sflow-yaml/scripts/check_gpu_plan.py index d40c8b2..b2bc366 100644 --- a/src/sflow/skills/writing-sflow-yaml/scripts/check_gpu_plan.py +++ b/src/sflow/skills/writing-sflow-yaml/scripts/check_gpu_plan.py @@ -17,7 +17,11 @@ import sys from pathlib import Path -import yaml + +# Parse exactly as the runtime does: sflow's loader keeps `10:00:00` a string +# instead of letting YAML 1.1 read it as the base-60 integer 36000. A validator +# that disagrees with the runtime is worse than one that requires sflow. +from sflow.config.loader import safe_load EXPRESSION_PATTERN = re.compile(r"\$\{\{(.+?)\}\}", re.DOTALL) @@ -31,7 +35,7 @@ def _load_and_merge(filepaths: list[str]) -> dict: print(f"Warning: file not found: {fp}", file=sys.stderr) continue with open(fp) as f: - config = yaml.safe_load(f.read()) + config = safe_load(f.read()) if not isinstance(config, dict): continue if not merged: diff --git a/src/sflow/skills/writing-sflow-yaml/scripts/validate_sflow_yaml.py b/src/sflow/skills/writing-sflow-yaml/scripts/validate_sflow_yaml.py index ca771f1..3463a6c 100755 --- a/src/sflow/skills/writing-sflow-yaml/scripts/validate_sflow_yaml.py +++ b/src/sflow/skills/writing-sflow-yaml/scripts/validate_sflow_yaml.py @@ -5,7 +5,7 @@ python validate_sflow_yaml.py [ ...] Checks performed: - - version field is present and set to "0.1" + - version field, if declared, is "0.1" (the field is optional) - Top-level keys are from the allowed set - Variable references (${{ }}) have valid syntax - depends_on references exist as task names @@ -25,6 +25,11 @@ import yaml +# Parse exactly as the runtime does: sflow's loader keeps `10:00:00` a string +# instead of letting YAML 1.1 read it as the base-60 integer 36000. A validator +# that disagrees with the runtime is worse than one that requires sflow. +from sflow.config.loader import safe_load + ALLOWED_TOP_LEVEL_KEYS = { "version", "variables", @@ -158,8 +163,9 @@ def _resolve_variable_value(variables: dict, name: str) -> int | float | str | N def check_version(config: dict, result: ValidationResult) -> None: + # `version` is optional and defaults to "0.1". Only that value has ever + # existed, so omitting it is fine; declaring anything else is not. if "version" not in config: - result.error("Missing required field: 'version'") return if str(config["version"]) != "0.1": result.error(f"Invalid version: '{config['version']}' (must be '0.1')") @@ -647,7 +653,7 @@ def validate_file(filepath: str) -> ValidationResult: return result try: - config = yaml.safe_load(content) + config = safe_load(content) except yaml.YAMLError as e: result.error(f"YAML syntax error: {e}") return result diff --git a/tests/e2e_tests/sample_test.sh b/tests/e2e_tests/sample_test.sh index 739c0d9..f3bec22 100755 --- a/tests/e2e_tests/sample_test.sh +++ b/tests/e2e_tests/sample_test.sh @@ -210,6 +210,22 @@ aiperf_tally_ok() { # -> 0 every aiperf run benchmarked, 1 one did no [ "$bad" -eq 0 ] } +# The one question both scoring paths ask: did a benchmark client actually +# measure something? Wraps the two clients so callers never have to remember that +# there are two -- the submitted-job loop checks both, and the sanity/`sflow run` +# path used to check only aiperf, silently passing every benchmark_serving recipe. +# 0 = a client benchmarked +# 1 = a client ran and measured nothing +# 2 = no benchmark client in this run +benchmark_tally_state() { # + local dir="$1" a s + aiperf_tally_ok "$dir"; a=$? + serving_tally_ok "$dir"; s=$? + [ "$a" -eq 0 ] || [ "$s" -eq 0 ] && return 0 + [ "$a" -eq 1 ] || [ "$s" -eq 1 ] && return 1 + return 2 +} + serving_tally_ok() { # -> 0 every benchmark_serving run completed, 1 one did not, 2 none here # The benchmark_serving.py (InferenceX) half of the same question aiperf_tally_ok # asks. The modular dynamo_benchmark rows drive this instead of aiperf, so @@ -818,6 +834,24 @@ E2E_PARTITION_B="${SLURM_E2E_PARTITION_B:-$PARTITION}" # verify_disjoint compares two tasks' device lists) still need the OUTPUT DIR on # storage the compute nodes share. Point -o/E2E_OUTPUT_DIR at shared scratch; # only the driver moved, the task steps still run out on the nodes. +# Emit `--set =` for a PRE-IMPORTED squashfs image, when it is there. +# pyxis takes a .sqsh straight from disk, so there is no registry import and no +# enroot cache write -- it cannot hit the shared-cache race that aborts two +# concurrent imports of the same image (see container_infra_failure). +# +# A missing file falls back to the recipe's own registry image: that keeps the run +# working, where a stale path would fail the recipe outright -- strictly worse than +# the race it avoids. The warning goes to STDERR because callers capture this +# function's stdout AS the argument list; an echo there becomes a bogus --set. +emit_sqsh_override() { # + [ -n "$2" ] || return 0 + if [ -f "$2" ]; then + printf '%s\n' "--set" "$1=$2" + else + echo "⚠ WARNING: $2 does not exist on this host; keeping the recipe's registry image for $1. The enroot image-cache race can recur -- run 'enroot import -o $2 docker://' to create it." >&2 + fi +} + sanity_recipe_set_args() { # `--set` of a variable a config does not declare is a hard error, and so is # `--artifact` of an artifact it does not declare -- which is why these are @@ -858,6 +892,18 @@ sanity_recipe_set_args() { "--set" "PARTITION_B=${E2E_PARTITION_B:-$PARTITION}" \ "--set" "SLURM_ACCOUNT=$ACCOUNT" \ "--set" "GPUS_PER_NODE=$GPUS_PER_NODE" + # Its gpu_pool tasks pull the same pytorch image concurrently, which is + # the other half of the enroot cache race. Same contract as above. + emit_sqsh_override GPU_IMAGE "${SFLOW_E2E_MONITOR_GPU_IMAGE:-}" + ;; + gpu_placement_matrix.yaml) + printf '%s\n' "--set" "SLURM_PARTITION=$PARTITION" \ + "--set" "SLURM_ACCOUNT=$ACCOUNT" \ + "--set" "GPUS_PER_NODE=$GPUS_PER_NODE" + # Overridden here rather than in the recipe: the path is cluster- + # specific, and the recipe's docker:// default is what every other user + # and cluster needs. Unset = leave it alone. + emit_sqsh_override PLACEMENT_IMAGE "${SFLOW_E2E_PLACEMENT_IMAGE:-}" ;; *) printf '%s\n' "--set" "SLURM_PARTITION=$PARTITION" \ @@ -865,6 +911,32 @@ sanity_recipe_set_args() { "--set" "GPUS_PER_NODE=$GPUS_PER_NODE" ;; esac + + # Right-size parallelism for the small stand-in model. The workload recipes + # carry LARGE-model sizing -- infmax_v1_ds_r1 is a DeepSeek-R1 recipe (GEN + # TP=8 + attention DP), dynamo_sglang_agg runs TP=4 -- which a 0.6B dense + # stand-in cannot satisfy: + # * TRT-LLM asserts "lm_head and vocab embedding should use the same TP + # size" when a tiny vocab is sharded 8 ways with attention DP on. + # * sglang's torch.compile dies with FailOnRecompileLimitHit and SIGKILLs + # its own process tree (exit 137). + # Both are model-vs-parallelism mismatches, not GPU or sflow problems -- the + # other 7 workload recipes serve real traffic on this cluster. + # + # Opt-in: unset leaves every recipe's own sizing alone, so a hand-run against + # a real checkpoint is never silently down-scaled. + if [ "${SFLOW_E2E_SMALL_MODEL_PARALLELISM:-0}" = "1" ]; then + case "$(basename "$1")" in + dynamo_sglang_agg.yaml) + printf '%s\n' "--set" "AGG_TP_SIZE=1" + ;; + infmax_v1_ds_r1.yaml) + printf '%s\n' "--set" "CTX_TP_SIZE=1" \ + "--set" "GEN_TP_SIZE=1" \ + "--set" "GEN_ENABLE_ATTENTION_DP=false" + ;; + esac + fi } gpu_placement_verified() { # -> 0 when every GPU task PROVED its placement @@ -1076,12 +1148,26 @@ run_sanity_recipes_with_sflow_run() { files+=("$f") if is_workload_recipe "$f"; then kinds+=("workload"); else kinds+=("sanity"); fi done - wait - echo "" - echo "===== Scoring ${#names[@]} sflow run(s) =====" - local i rc - for i in "${!names[@]}"; do + echo "===== Scoring ${#names[@]} sflow run(s) -- each is scored the moment it finishes =====" + # Score in COMPLETION order, not launch order: with recipes running in + # parallel and wildly different runtimes, a single `wait` for all of them + # meant a whole suite's worth of verdicts appeared only at the very end, and + # a hung recipe hid the results of every one that had already passed. + # + # `.rc` is written by each subshell after its `sflow run` returns, so its + # presence is the completion signal. -s, not -e: the file is created and + # written in two steps, and an empty read would score a finished run as rc="". + local i rc pending=("${!names[@]}") still=() progressed + while [ ${#pending[@]} -gt 0 ]; do + progressed=0 + still=() + for i in "${pending[@]}"; do + if [ ! -s "${roots[$i]}/.rc" ]; then + still+=("$i") + continue + fi + progressed=1 # Missing .rc means the subshell never got to write one -- treat as failure. rc=$(cat "${roots[$i]}/.rc" 2>/dev/null || echo 1) # No job id to look a run up by later, so find where it landed; the @@ -1103,7 +1189,12 @@ run_sanity_recipes_with_sflow_run() { # EVIDENCE THAT DOES NOT EXIST: aiperf_template holds no GPU, wrote no # record, and still reported "placement proven by UUID" -- the exact # kind of line that misleads whoever audits these artifacts later. - local proved="placement proven by UUID; app rc=$rc ignored on this cluster" + local proved="placement proven by UUID" + # Only the lenient mode disclaims the app's exit status; under strict + # scoring the rc is part of the verdict, so saying it was ignored would + # be a lie in the CI log. + [ "${SFLOW_E2E_STRICT_WORKLOAD:-0}" = "1" ] \ + || proved="$proved; app rc=$rc ignored on this cluster" local unproved="GPU placement not proven" if ! recipe_requests_gpus "${files[$i]}"; then proved="no GPU task, so no placement to prove; sflow reports COMPLETED" @@ -1114,13 +1205,53 @@ run_sanity_recipes_with_sflow_run() { # these recipes forever -- but a bare PASS next to an aiperf that # measured nothing is how the ptyche half stayed green for six workflows. # Whoever audits these artifacts should not have to open the CSV to find - # that out. aiperf_tally_ok() already prints the counts to stderr. - aiperf_tally_ok "$run_dir" - case $? in - 0) proved="$proved; aiperf benchmarked" ;; - 1) proved="$proved; aiperf measured nothing (expected here, not gated)" ;; + # that out. The tally helpers already print their counts to stderr. + benchmark_tally_state "$run_dir" + local tally=$? + # The ONE expected-not-to-benchmark shape, excused exactly as the ptyche + # half does it (see recipe_is_client_only): aiperf_template starts no + # server, so its client has nothing to talk to and can never produce a + # metric. Every OTHER workload recipe still owes a real benchmark. + # It does still owe a clean COMPLETE -- the workload_placement_ok elif + # below already demands exactly that, because a recipe with no GPU task + # falls back to workflow_summary_ok. + local client_only=1 + [ "$tally" -eq 1 ] && recipe_is_client_only "$run_dir" && client_only=0 + case "$tally" in + 0) proved="$proved; benchmarked" ;; + 1) if [ "$client_only" -eq 0 ]; then + proved="$proved; client-only recipe: no server to benchmark by design" + else + proved="$proved; benchmark client measured nothing" + fi ;; esac - if workload_placement_ok "${files[$i]}" "$run_dir"; then + if [ "${SFLOW_E2E_STRICT_WORKLOAD:-0}" = "1" ]; then + # STRICT: the cluster can serve, so the app's own verdict counts. + # A workload recipe must exit 0, prove its placement, AND have + # actually benchmarked -- placement alone is not a working stack. + # Infra failures are still excused (a broken node or an enroot + # cache race is not a regression), exactly as the sanity half does. + if [ "$rc" -ne 0 ]; then + if cuda_infra_failure "${roots[$i]}"; then + mark_cuda_excused "${names[$i]}" "${logs[$i]}" "(sflow run rc=$rc with a CUDA init failure)" + elif container_infra_failure "${roots[$i]}"; then + mark_container_excused "${names[$i]}" "${logs[$i]}" "(enroot image-cache race on its node)" + else + echo " ${names[$i]}: FAIL (sflow run exited $rc; see ${logs[$i]})" + fi + elif ! workload_placement_ok "${files[$i]}" "$run_dir"; then + echo " ${names[$i]}: FAIL ($unproved; see ${run_dir:-${logs[$i]}})" + elif [ "$tally" -eq 1 ] && [ "$client_only" -ne 0 ]; then + # tally 2 = this recipe runs no benchmark client at all, which + # is not a failure; 1 = one ran and measured nothing, which is + # -- unless it is the client-only recipe, which by design has + # no server to measure. + echo " ${names[$i]}: FAIL (benchmark client ran but measured nothing; see $run_dir)" + else + PASSED=$((PASSED + 1)) + echo " ${names[$i]}: PASS ($proved)" + fi + elif workload_placement_ok "${files[$i]}" "$run_dir"; then PASSED=$((PASSED + 1)) echo " ${names[$i]}: PASS ($proved)" else @@ -1129,6 +1260,8 @@ run_sanity_recipes_with_sflow_run() { elif [ "$rc" -ne 0 ]; then if cuda_infra_failure "${roots[$i]}"; then mark_cuda_excused "${names[$i]}" "${logs[$i]}" "(sflow run rc=$rc with a CUDA init failure)" + elif container_infra_failure "${roots[$i]}"; then + mark_container_excused "${names[$i]}" "${logs[$i]}" "(enroot image-cache race on its node)" else echo " ${names[$i]}: FAIL (sflow run exited $rc; see ${logs[$i]})" fi @@ -1138,12 +1271,21 @@ run_sanity_recipes_with_sflow_run() { echo " ${names[$i]}: PASS (rc=0 and its own output proves it, under $run_dir)" elif cuda_infra_failure "${roots[$i]}"; then mark_cuda_excused "${names[$i]}" "${run_dir:-${logs[$i]}}" "(exited 0 but proved nothing; CUDA init failure on node)" + elif container_infra_failure "${roots[$i]}"; then + mark_container_excused "${names[$i]}" "${run_dir:-${logs[$i]}}" "(enroot image-cache race on its node)" else # The nastiest shape: green process, unproven run. Exactly what a # silently-degraded placement or a collapsed two-backend run looks like. echo " ${names[$i]}: FAIL (sflow run exited 0 but its output does not prove the run: ${run_dir:-no run dir found})" fi + done + pending=("${still[@]+"${still[@]}"}") + # Only sleep when nothing finished this pass, so a burst of completions + # prints back-to-back instead of one every poll interval. + [ ${#pending[@]} -eq 0 ] || [ "$progressed" -eq 1 ] || sleep 5 done + # Reap the subshells; every one has already written its .rc by now. + wait } run_monitor_mixed_real() { @@ -1662,6 +1804,25 @@ cuda_infra_failure() { # -> 0 (true) if the job failed due to CUDA in "$out_dir" } +container_infra_failure() { # -> 0 (true) if the job failed due to container-runtime infra + local out_dir="$1" + [ -n "$out_dir" ] && [ -d "$out_dir" ] || return 1 + # Enroot populates its shared image cache with `mv --no-clobber`. When two + # concurrent srun steps import the same image (a cold cache -- e.g. the first + # run on a new cluster), the loser's mv refuses to replace the winner's file + # and pyxis aborts the step. Only that race produces this line. + # + # Deliberately NOT matched here, per the same rule as cuda_infra_failure above + # (a pattern that also fits healthy//buggy runs turns real regressions green): + # 'failed to import docker image' and 'couldn't start container' are also what + # a recipe naming a nonexistent image, or a broken registry credential, emits. + # Those are recipe/config bugs and must stay real failures. The mv collision + # cannot be produced by a bad recipe. + grep -rIqs --include='*.log' --include='*.out' \ + -e 'mv: not replacing .*enroot' \ + "$out_dir" +} + # Reclassify a would-be FAIL as a CUDA-infra excuse: bump the excused counter and # print a prominent warning. Excused jobs are NOT counted as failures by the CI # threshold, but they did NOT succeed -- the node should be investigated/drained. @@ -1671,12 +1832,21 @@ mark_cuda_excused() { # echo " ⚠ WARNING: Job $1 failed due to a CUDA/GPU infrastructure error on its node (e.g. 'CUDA initialization: Unexpected error from cudaGetDeviceCount()' / 'Error 802: system not yet initialized'). Excused from the pass/fail threshold, but this is NOT a successful run -- investigate/drain the node. See $2" } +# Same contract as mark_cuda_excused, for a container-runtime (enroot/pyxis) infra +# failure: excused from the threshold, but loudly reported -- it did NOT succeed. +mark_container_excused() { # + CONTAINER_INFRA=$((CONTAINER_INFRA + 1)) + echo " Job $1: CONTAINER-INFRA EXCUSED (enroot image-cache race; NOT a real pass) $3" + echo " ⚠ WARNING: Job $1 failed because two concurrent steps imported the same image into the shared enroot cache ('mv: not replacing ...enroot.../cache/...'), so pyxis could not start the container. Excused from the pass/fail threshold, but this is NOT a successful run -- re-run once the cache is warm, or pre-warm it on this cluster. See $2" +} + # Check results in output folders echo "" echo "===== Results =====" TOTAL=0 PASSED=0 CUDA_INFRA=0 +CONTAINER_INFRA=0 if [ "$RECIPE_CLASS" = "sanity" ]; then run_sanity_recipes_with_sflow_run fi @@ -1715,6 +1885,8 @@ for jid in "${JOB_IDS[@]}"; do echo " Job $jid: PASS (GPU placement proven by UUID on 2+ nodes under $out_dir)" elif cuda_infra_failure "$out_dir"; then mark_cuda_excused "$jid" "$out_dir" "(placement assertions unproven; CUDA init failure on node)" + elif container_infra_failure "$out_dir"; then + mark_container_excused "$jid" "$out_dir" "(enroot image-cache race on its node)" else echo " Job $jid: FAIL (GPU placement not proven under $out_dir; read SFLOW_GPU_PROBE / FAIL: in the task logs)" fi @@ -1748,12 +1920,16 @@ for jid in "${JOB_IDS[@]}"; do # a sibling task in the same workflow reported requests of its own. if cuda_infra_failure "$out_dir"; then mark_cuda_excused "$jid" "$out_dir" "(aiperf measured nothing; CUDA init failure on node)" + elif container_infra_failure "$out_dir"; then + mark_container_excused "$jid" "$out_dir" "(enroot image-cache race on its node)" else echo " Job $jid: FAIL (aiperf ran but measured nothing; see the tally above, $out_dir)" fi elif [ "$serving_state" -eq 1 ]; then if cuda_infra_failure "$out_dir"; then mark_cuda_excused "$jid" "$out_dir" "(benchmark_serving did not complete; CUDA init failure on node)" + elif container_infra_failure "$out_dir"; then + mark_container_excused "$jid" "$out_dir" "(enroot image-cache race on its node)" else echo " Job $jid: FAIL (benchmark_serving ran but did not complete its requests; see the counts above, $out_dir)" fi @@ -1766,6 +1942,8 @@ for jid in "${JOB_IDS[@]}"; do echo " Job $jid: PASS (no benchmark log; sflow reports Status: COMPLETED under $out_dir)" elif cuda_infra_failure "$out_dir"; then mark_cuda_excused "$jid" "$out_dir" "(no success indicator; CUDA init failure on node)" + elif container_infra_failure "$out_dir"; then + mark_container_excused "$jid" "$out_dir" "(enroot image-cache race on its node)" else echo " Job $jid: FAIL (no success indicator found in $out_dir)" fi @@ -1785,6 +1963,62 @@ if [ -n "${MONITOR_MIXED_LAUNCH_FAILED:-}" ]; then echo " monitor_mixed run: FAIL (no Slurm job submitted)" fi +# Print the benchmark client's own summary numbers from every benchmark task, so +# the CI log carries REAL perf. A green verdict only says the workflow exited 0 +# and placement was proven; these lines are what show the server actually served +# -- throughput and latency a human can sanity-check, and a non-zero request count. +# +# Both clients in this suite are covered, because the recipes are split between +# them (see aiperf_tally_ok / serving_tally_ok): +# * aiperf -- a boxed "NVIDIA AIPerf | LLM Metrics" table. +# * benchmark_serving -- sa-bench / InferenceX "Serving Benchmark Result". +# +# Matched on metric names, never on the bare words "Successful"/"Request": the +# benchmark task pip-installs its client first, and "Successfully installed ..." +# is in the same log. +print_benchmark_metrics() { + local root="${E2E_OUTPUT_DIR:-sflow_output}" found=0 log rows label client + + while IFS= read -r log; do + # Strip sflow's per-task log prefix ("... - INFO - 0: ") so both summaries + # line up in the CI log. + local body + body=$(sed 's/.*INFO - [0-9]*: //' "$log" 2>/dev/null) || continue + + # The ┃ line is aiperf's column header (avg/min/max/p99/p90/p50/std) -- + # without it the numbers below are unlabelled. + rows=$(printf '%s\n' "$body" | grep -E \ + '^┃ *Metric|^│ *(Time to First Token|Inter Token Latency|Output Token Throughput|Request Throughput|Request Count)' || true) + client="aiperf" + if [ -z "$rows" ]; then + # sa-bench / benchmark_serving.py prints a flat "label: value" block. + rows=$(printf '%s\n' "$body" | grep -E \ + '^(Successful requests|Benchmark duration|Total input tokens|Total generated tokens|Request throughput|Output token throughput|Total token throughput|Mean TTFT|Median TTFT|P99 TTFT|Mean TPOT|Median TPOT|Mean ITL|Median ITL) *\(?[^)]*\)? *:' || true) + client="benchmark_serving" + fi + [ -n "$rows" ] || continue + + if [ "$found" -eq 0 ]; then + echo "" + echo "===== Benchmark metrics =====" + found=1 + fi + # /, e.g. dynamo_trtllm_agg/benchmark_128 + label="$(basename "$(dirname "$(dirname "$(dirname "$log")")")")/$(basename "$log" .log)" + echo "" + echo " $label [$client]" + printf '%s\n' "$rows" | sed 's/^/ /' + done < <(find "$root" -type f -name 'benchmark*.log' 2>/dev/null | sort) + + if [ "$found" -eq 0 ]; then + echo "" + echo "===== Benchmark metrics =====" + echo " (none found -- no benchmark task produced an aiperf or benchmark_serving summary)" + fi +} + +print_benchmark_metrics + echo "" echo "===== Summary =====" echo "$PASSED/$TOTAL jobs passed" @@ -1792,6 +2026,10 @@ if [ "${CUDA_INFRA:-0}" -gt 0 ]; then echo "$CUDA_INFRA/$TOTAL jobs excused due to CUDA/GPU infrastructure failures (not counted as failures)" echo "⚠ WARNING: $CUDA_INFRA job(s) failed because of CUDA/GPU infrastructure errors on their nodes (driver/fabric not ready). They are EXCUSED from the pass/fail threshold but did NOT succeed -- investigate/drain the affected nodes." fi +if [ "${CONTAINER_INFRA:-0}" -gt 0 ]; then + echo "$CONTAINER_INFRA/$TOTAL jobs excused due to container-runtime infrastructure failures (not counted as failures)" + echo "⚠ WARNING: $CONTAINER_INFRA job(s) failed because of an enroot image-cache race on their nodes (concurrent imports of the same image). They are EXCUSED from the pass/fail threshold but did NOT succeed -- re-run with a warm cache, or pre-warm the images on this cluster." +fi # ============================================================================= # Independent monitor coverage check @@ -1921,6 +2159,6 @@ fi # summarize_validation() does with the same numbers -- but an ALL-excused run # proved nothing, so it is not a pass either. [ "$PASSED" -gt 0 ] \ - && [ $((PASSED + CUDA_INFRA)) -eq "$TOTAL" ] \ + && [ $((PASSED + CUDA_INFRA + CONTAINER_INFRA)) -eq "$TOTAL" ] \ && [ "$MONITOR_PRESENT" -eq "$MONITOR_TOTAL" ] \ && [ "$TARGETING_OK" -eq "$TARGETING_TOTAL" ] diff --git a/tests/unit/test_config_loader.py b/tests/unit/test_config_loader.py index 4df8be3..a9a16d9 100644 --- a/tests/unit/test_config_loader.py +++ b/tests/unit/test_config_loader.py @@ -1,9 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import re + import pytest +import yaml -from sflow.config.loader import ConfigLoader +from sflow.config.loader import ( + _FLOAT_WITHOUT_SEXAGESIMAL, + _INT_WITHOUT_SEXAGESIMAL, + ConfigLoader, + safe_load, +) def _vars_to_map(config) -> dict[str, object]: @@ -319,3 +327,170 @@ def test_load_config_is_quiet_when_no_timeout_is_set(tmp_path, caplog): ConfigLoader().load_config(p) assert "does not enforce it" not in "\n".join(r.message for r in caplog.records) + + +def test_unquoted_slurm_walltime_is_not_read_as_base_60(tmp_path): + """`time: 10:00:00` must reach the backend as a string, not YAML 1.1 base-60. + + PyYAML resolves `10:00:00` to the integer 36000, which Slurm reads as 36000 + *minutes*. Guards the ConfigLoader wiring, not just the loader helper. + """ + p = tmp_path / "sflow.yaml" + p.write_text( + """ +version: "0.1" +backends: + - name: slurm + type: slurm + default: true + partition: debug + account: test + nodes: 1 + gpus_per_node: 1 + time: 10:00:00 +workflow: + name: wf + tasks: + - name: t1 + script: + - echo hi +""".lstrip() + ) + + config = ConfigLoader().load_config(p) + + assert config.backends[0].time == "10:00:00" + + +def test_integer_walltime_still_means_minutes(tmp_path): + """A genuine integer must keep working — Slurm reads it as minutes.""" + p = tmp_path / "sflow.yaml" + p.write_text( + """ +version: "0.1" +backends: + - name: slurm + type: slurm + default: true + partition: debug + account: test + nodes: 1 + gpus_per_node: 1 + time: 5400 +workflow: + name: wf + tasks: + - name: t1 + script: + - echo hi +""".lstrip() + ) + + config = ConfigLoader().load_config(p) + + assert config.backends[0].time == 5400 + +@pytest.mark.parametrize( + "text", + [ + "10:00:00", # the reported case: 10 hours, not 36000 + "9:00:00", + "1:30:00", + "10:00", + "01:00:00", # leading zero already survived under YAML 1.1 + "00:10:00", + "1-00:00:00", # Slurm day-hour form + "10:00:00.5", # sexagesimal *float* — used to fail validation outright + ], +) +def test_walltime_stays_a_string(text: str) -> None: + assert safe_load(f"time: {text}") == {"time": text} + + +def test_only_sexagesimal_scalars_differ_from_pyyaml() -> None: + """Dropping base-60 must not disturb any other scalar type. + + `017` is YAML 1.1 octal (15); `1e10` stays a string because PyYAML requires a + signed exponent. Both quirks are deliberately preserved. + """ + document = ( + "a: 7\nb: -7\nc: 0\nd: 0x1f\ne: 017\nf: 1.5\ng: -0.25\n" + "h: true\ni: null\nj: slurm\nk: [1, 2]\nl: {x: 1}\nm: 1e10\n" + ) + assert safe_load(document) == yaml.safe_load(document) + + +def test_loader_does_not_mutate_pyyaml_global_state() -> None: + """Also pins the upstream behavior this loader exists to neutralize.""" + safe_load("time: 10:00:00") + assert yaml.safe_load("time: 10:00:00") == {"time": 36000} + + +def _without_sexagesimal(pattern: str) -> str: + """Delete the base-60 alternative from a PyYAML resolver pattern.""" + collapsed = re.sub(r"\s+", "", pattern) + return re.sub(r"\|[^|]*?\(\?::\[0-5\]\?\[0-9\]\)\+[^|)]*", "", collapsed) + + +@pytest.mark.parametrize( + ("tag", "ours"), + [ + ("tag:yaml.org,2002:int", _INT_WITHOUT_SEXAGESIMAL), + ("tag:yaml.org,2002:float", _FLOAT_WITHOUT_SEXAGESIMAL), + ], +) +def test_patterns_stay_in_sync_with_pyyaml(tag: str, ours) -> None: + """Fail loudly if PyYAML edits the patterns we copied. + + Our resolvers are PyYAML's own, minus the base-60 branch. That copy would + otherwise rot silently on a PyYAML upgrade, so pin the relationship rather + than the literal text: re-derive ours from upstream and compare. + """ + upstream = next( + pattern + for resolvers in yaml.SafeLoader.yaml_implicit_resolvers.values() + for resolver_tag, pattern in resolvers + if resolver_tag == tag + ) + assert "(?::[0-5]?[0-9])+" in upstream.pattern, ( + f"PyYAML no longer resolves {tag} as base-60; this workaround may be obsolete" + ) + assert _without_sexagesimal(upstream.pattern) == re.sub(r"\s+", "", ours.pattern) + + +def test_config_without_version_defaults_to_0_1(tmp_path): + """`version` is optional; omitting it means "0.1".""" + p = tmp_path / "sflow.yaml" + p.write_text( + """ +workflow: + name: wf + tasks: + - name: t1 + script: + - echo hi +""".lstrip() + ) + + config = ConfigLoader().load_config(p) + + assert config.version == "0.1" + + +def test_explicit_unsupported_version_still_rejected(tmp_path): + """Optional does not mean unvalidated: a declared bad version still fails.""" + p = tmp_path / "sflow.yaml" + p.write_text( + """ +version: "9.9" +workflow: + name: wf + tasks: + - name: t1 + script: + - echo hi +""".lstrip() + ) + + with pytest.raises(Exception, match="9.9"): + ConfigLoader().load_config(p) diff --git a/tests/unit/test_config_loader_merge.py b/tests/unit/test_config_loader_merge.py index 00b9317..92d5cb5 100644 --- a/tests/unit/test_config_loader_merge.py +++ b/tests/unit/test_config_loader_merge.py @@ -125,11 +125,21 @@ def test_missing_workflow_raises(self): with pytest.raises(ValueError, match="No 'workflow' section"): merge_config_dicts([a, b]) - def test_missing_version_raises(self): + def test_missing_version_is_allowed(self): + """`version` is optional; the schema defaults it to "0.1". + + Behavior change: this previously raised "No 'version' field found in any + input file". Only "0.1" has ever existed, so requiring it bought nothing. + A conflict between files is still an error -- see + ``test_version_must_be_consistent``. + """ a = {"workflow": {"name": "wf", "tasks": [{"name": "t", "script": ["echo"]}]}} b = {"variables": {"X": {"value": 1}}} - with pytest.raises(ValueError, match="No 'version' field"): - merge_config_dicts([a, b]) + + merged = merge_config_dicts([a, b]) + + assert "version" not in merged + assert merged["workflow"]["name"] == "wf" def test_missing_tasks_raises(self): a = {"version": "0.1", "workflow": {"name": "wf"}} diff --git a/tests/unit/test_plugin_backends_slurm_backend.py b/tests/unit/test_plugin_backends_slurm_backend.py index 9eb8688..e7de7ff 100644 --- a/tests/unit/test_plugin_backends_slurm_backend.py +++ b/tests/unit/test_plugin_backends_slurm_backend.py @@ -1638,3 +1638,60 @@ def test_allocation_probes_the_gpu_topology(): # Once for the pre-existing (unowned) allocation, once for the salloc path. assert inspect.getsource(SlurmBackend).count("await self._discover_gpu_uuids(") == 2 + + +def test_unquoted_walltime_from_yaml_reaches_the_slurm_time_flag(tmp_path, monkeypatch): + """`time: 10:00:00` must render as `--time 10:00:00`, not `--time 36000`. + + PyYAML resolves an unquoted `10:00:00` to the base-60 integer 36000, which + Slurm reads as 36000 *minutes*. Covers the whole path: YAML -> ConfigLoader + -> SlurmBackendConfig -> the rendered flag value. + """ + from sflow.config.loader import ConfigLoader + + path = tmp_path / "sflow.yaml" + path.write_text( + """ +version: "0.1" +backends: + - name: slurm + type: slurm + default: true + partition: debug + account: test + nodes: 1 + gpus_per_node: 1 + time: 10:00:00 +workflow: + name: wf + tasks: + - name: t1 + script: + - echo hi +""".lstrip() + ) + + monkeypatch.delenv("SLURM_JOB_ID", raising=False) + monkeypatch.delenv("SLURM_JOBID", raising=False) + monkeypatch.delenv("SLURM_JOB_NODELIST", raising=False) + monkeypatch.delenv("SLURM_NODELIST", raising=False) + + backend = SlurmBackend(ConfigLoader().load_config(path).backends[0]) + backend._subprocess_launcher = _FakeSubprocessLauncher( + script=[ + ( + 0, + [ + "salloc: Granted job allocation 2222222", + "salloc: Nodes node001 are ready for job", + ], + ), + (0, ["node001: 10.0.0.1:123"]), + ] + ) + + asyncio.run(backend.allocate()) + + salloc_cmd = list(backend._subprocess_launcher.calls[0]["command"]) + assert "--time" in salloc_cmd + assert salloc_cmd[salloc_cmd.index("--time") + 1] == "10:00:00" From b15051c856a69efc6c01c39c70c93094c15b8d05 Mon Sep 17 00:00:00 2001 From: rogliu Date: Thu, 10 Sep 2026 15:27:17 +0800 Subject: [PATCH 2/3] Add project page in ZH --- docs/developer/dev-notes/sflow_intro_zh.html | 1373 ++++++++++++++++++ 1 file changed, 1373 insertions(+) create mode 100644 docs/developer/dev-notes/sflow_intro_zh.html diff --git a/docs/developer/dev-notes/sflow_intro_zh.html b/docs/developer/dev-notes/sflow_intro_zh.html new file mode 100644 index 0000000..17ca36b --- /dev/null +++ b/docs/developer/dev-notes/sflow_intro_zh.html @@ -0,0 +1,1373 @@ + + + + + +sflow — 面向大规模 GPU 集群的声明式工作流描述器 + + + + + + +
+文档 → +功能地图 → +
+
+
+ +
按 P 开启自动播放
+ + +
+
+
+
+01 / 16 +
+
NV-SFLOW
+

面向大规模 GPU 集群的通用工作流编排引擎

+

一次描述，随处运行。_

+
+vLLMSGLangTensorRT-LLMDynamoPyTorch… +sflow +KubernetesSlurmDocker +
+
+查看文档 → +功能地图 → +GitHub +
+
+ +滚动 + + +
+ + +
+
+
+02 / 16 +
+工作负载演进 +

从本地单机容器，到集群级多阶段服务

+
+
+
部署规模跑在哪里
+
+ 过去 +
本地单机
+
1 个节点 · 1-8 张 GPU · 无跨节点依赖
+
+
➔
+
+ 现在 +
多节点集群级服务
+
数十至数百个节点 · 跨节点通信与拓扑 · 调度器、配额与生命周期
+
+
+
+
服务架构怎么被服务
+
+ 过去 +
Agg Server→Client
+
单阶段 · 一条命令拉起服务
+
+
➔
+
+ 现在 +
Router→PD 分离→KV Transfer→异构计算
+
多阶段 · 多组件协同 · 每个阶段各自的资源与就绪条件
+
+
+
+
复杂度就此转移：模型还是那个模型，但把它跑起来，已经从「起一个进程」变成「编排一整套分布式系统」。
+
+
+ + +
+
+
+03 / 16 +
+视角转变 +

更宏观的编排视角：从一台机器，到一个集群

+
+
单机视角五层都挤在同一台机器里，天然就是一致的 — 装好、跑起来，就结束了。
+
➔
+
集群视角同样五层要在每个节点上各来一份，还必须彼此对齐 — 任何一层错位，整个工作流都跑不起来。
+
+
+
↑自下而上
+
+
node-0node-1node-2node-3
+
+
应用工作流DAG · 任务依赖 · 探针
+
+
跨节点依赖与就绪对齐
+
+
+
基础设施平台Slurm · Kubernetes · Docker
+
+
跨节点调度、配额与网络
+
+
+
软件vLLM · SGLang · TensorRT-LLM · 镜像
+
+
跨节点版本与镜像一致
+
+
+
驱动CUDA · NCCL · 网卡驱动
+
+
跨节点驱动与固件对齐
+
+
+
硬件GPU · NVLink · IB · 拓扑
+
+
跨节点拓扑与可见性一致
+
+
+
+
于是编排的对象变了：不再是「把一个进程跑起来」，而是把五层当成一个整体来组合，并让节点之间共享同一份信息、保持同一套约定。
+
+
+ + +
+
+
+04 / 16 +
+基础设施差异 +

每个平台一套语义，迁移与适配成本倍增

+
+
+

Slurm

+
#SBATCH --gres=gpu:8 +srun --ntasks=16 ...
+
分区、账户与 QoS · sbatch 语法 · hostlist 展开规则
+
学习曲线
+
+
+

Kubernetes

+
resources.limits: + nvidia.com/gpu: 8
+
manifest 与 CRD · label 与 affinity · RBAC 与 namespace
+
学习曲线
+
+
+

Docker

+
docker run --gpus all + --network host ...
+
镜像与挂载 · 设备映射 · compose 语法
+
学习曲线
+
+
+
+
+ 应用迭代 +
+ 改一版 + 跑一次 + 卡住 — 等平台适配做完 + 改一版 +
+
+
+ 平台适配 +
+ + 学语法 · 对 schema · 调权限 · 补兼容 + +
+
+
+
真正被拖慢的是应用：模型和参数只改了一行，时间却全花在「让它能在这个平台上跑起来」— 换一个基础设施平台，就要重写整个工作流编排。
+
+
+ + +
+
+
+05 / 16 +
+问题 +

工作负载始终不变，平台适配成本却成倍增长。

+
稳定的工作流： etcd → NATS → 前端 → GPU 工作节点 → 基准测试
+
+
重复的关注点
Slurm
Docker
Kubernetes
+
启动
sbatch + srun
docker run
Pod / Job
+
GPU 放置
GPU 参数
设备映射
claims + Affinity
+
网络
hostlist + 端口
host 网络
Service + DNS
+
就绪检测
shell 轮询
healthcheck
probes
+
日志与清理
sacct + 文件
docker logs
日志转发 + 生命周期
+
+
5 类关注点 × 3 个平台围绕同一个稳定的工作流，重复着同样的编排工作
+
+
+ + +
+
+
+06 / 16 +
+后端无关的设计 +

保留做什么，替换在哪跑，无需重写。

+
+
✎

统一语义

部署逻辑始终不变 — 变的只是基础设施适配。

+
⇄

只换后端配置块

同一份 sflow.yaml 可运行在 Docker、Slurm 与 Kubernetes 上。

+
⚙

委托，而非重造

后端交由各平台的原生生态处理，而不是重新实现一遍。

+
+
+ + + + + + + +sflow.yaml +DAG · 脚本 · 探针 +资源 · 结果 · 产物 + + + + + + + + +sflow + + + + + + + + + + + + + +Local · bash + +Docker · docker_run + +Slurm · srun + +Kubernetes · k8s + + + + + +
+
+
+ + +
+
+
+07 / 16 +
+DAG 编排 +

任务与依赖，全部由你定义

+
+
+
有向无环图(DAG)   每个阶段在依赖就绪时自动启动
+
+
load_image
+
▼
+
install_dependency
gpu_monitor
+
▼
+
nats_server
etcd_server
+
▼
+
frontend_server_0
frontend_server_1
frontend_server_2
+
▼
+
nginx_server
prefill_server_0
prefill_server_1
prefill_server_2
prefill_server_3
decode_server_0
+
▼
+
benchmark_client
+
+
+
+
✎

任务 = 任意命令

bash、python、容器入口都可以。

+
⇄

依赖 = 任意 DAG

串行、并行、分支与汇聚，阶段数不设上限。

+
∞

不限定领域

推理、训练、评测、数据处理均可自由编排。

+
+
+
+
+ + +
+
+
+08 / 16 +
+大规模集群级编排 +

拓扑感知的 GPU 分配

+
sflow 拓扑规划器GPU 容量 · 节点亲和性 · 任务依赖
+
▼ 以后端原生方式下发
+
+
+
node-0NVLink 域 A · 4 张 GPU
+
+
GPUGPU 0prefill_0
+
GPUGPU 1prefill_1
+
GPUGPU 2prefill_2
+
GPUGPU 3prefill_3
+
+
+
+
node-1NVLink 域 B · 4 张 GPU
+
+ decode_0 · 4 GPU 任务 +
+
GPUGPU 0decode_0
+
GPUGPU 1decode_0
+
GPUGPU 2decode_0
+
GPUGPU 3decode_0
+
+
+
+
+
节点与 GPU 放置、多节点副本与参数扫描、就绪探针、批量提交 — 于是一份描述文件即可驱动数百张 GPU，而不是一堆手写的 bash 脚本。
+
+
+ + +
+
+
+09 / 16 +
+工作流生命周期 +

定义。执行。洞察。

+
+
+

定义

+
DAG任务与依赖
一份声明式工作流
+
{ }变量与产物
可移植的输入与表达式
+
✓探针
就绪与失败判定
+
+
➔
+
+

执行

+
GPU资源规划
节点、GPU、拓扑
+
N×副本与参数扫描
并行或串行
+
→后端适配器
Local、Docker、Slurm、K8S
+
+
➔
+
+

洞察

+
TUI实时状态
任务、日志、就绪
+
∿汇总与监控
时间线与硬件图表
+
⇧结果与上传
JSON 指标与 S3
+
+
+
+
+ + +
+
+
+10 / 16 +
+后端适配器 +

同一份意图，落地为四套原生执行方案

+
可移植的工作流契约   DAG · 资源 · 探针 · 产物 · 结果
+
▼ sflow 委托给各平台的原生生态
+
+
>_

Local

bash 进程
模拟的节点 / GPU 映射

+
▣

Docker

docker run
容器 GPU 设备

+
#

Slurm

salloc + srun
调度器分配

+
⎈

Kubernetes

pods + claims
绑定与生命周期

+
+
▼
+
一致的输出   task.log · result.json · summary · 产物
+
+
+ + +
+
+
+11 / 16 +
+Merge Pod +

GPU 任务同机部署，共享高速的节点内互联

+
+
+

独立 Pod · 视图相互隔离

+
+
worker A仅可见 GPU 0-1
+
IB / TCP
↔
+
worker B仅可见 GPU 2-3
+
+
+
sflow merge
➔
+
+

单个 Pod · 合并的 GPU 视图

+
worker Aworker B
+
GPU 0GPU 1GPU 2GPU 3
+
CUDA IPC · NVLink · NVSwitch
+
+
+
安全边界：同一工作流 + 同一后端 + 同一节点 + 同一镜像。典型场景：PD 分离的 KV 传输。同时契合当前每节点 IMEX 的约束。
+
+
+ + +
+
+
+12 / 16 +
+可观测性 +

每一次运行都会自我解释

+
+
sflow_summary.log +Task Duration Chart +prefill |####..........................| 58.412s READY +decode |##########################....| 6m14.882s READY +benchmark |..........................####| 47.906s COMPLETED + +Timeline +12:31:33 +00.000s prefill SUBMITTED +12:31:33 +00.000s decode SUBMITTED +12:32:31 +58.412s prefill READY +12:37:48 +06m15s decode READY +12:37:49 +06m16s benchmark SUBMITTED +12:38:36 +07m03s benchmark COMPLETED
+
sflow_monitor.log +Metric Summary +GPU util % min=42 avg=87 max=99 +GPU mem used GiB min=18 avg=61 max=76 +GPU power W min=310 avg=642 max=718 + +Timelines (cluster avg) +GPU util % ▁▂▅▇████▆▅ +GPU mem used GiB ▁▃▄▆▇█████ +Net RX MiB/s ▁▁▂▅▇▆▃▂▁
+
+
+
+
GPU 利用率87%
+ +
+
+
GPU 显存61 GiB
+ +
+
+
GPU 功耗642 W
+ +
+
+
+
+ + +
+
+
+13 / 16 +
+模块化组合 +

换掉一个叶子，复用整棵树。

+
+
recipe/组合根目录
+
+
+
backend/
+
local.yamldocker.yamlslurm.yamlk8s.yaml ✓
+
+
+
common/
+
common_workflow.yaml · 共享 DAG
+
+
+
workload/
+
vllm/ ✓sglang/trtllm/
+
+
+
benchmark/
+
aiperf.yaml ✓infmax.yaml
+
+
+
+
▼ 选中的叶子完成组合
+
composed.yaml   k8s + 共享工作流 + vLLM + AIPerf
+
只替换高亮的那个叶子，共享分支保持不变。
+
+
+ + +
+
+
+11 / 15 +
+真实场景调试 +

结构化的错误分析

+
+
+
Workflow: b200-fp8-low-latency-tep8-1p-1d +Model: DeepSeek R1 FP8 | 2 nodes × 8 GPUs | ISL=8192, OSL=1024 + +Allocation Map +├─ slurm-node-01 (node 0) +│ GPU 0-7: prefill_server_0 (TP=8) +│ Also: load_image, nats, etcd, frontend, benchmark_* +└─ slurm-node-02 (node 1) + GPU 0-7: decode_server_0 (TP=8) + Also: load_image, gpu_monitor + +Timeline +01:57:08 — load_image + install_aiperf submitted +01:59:10 — load_image COMPLETED on both nodes +01:59:43 — nats_server READY +01:59:45 — etcd_server READY +02:00:31 — frontend_server_0 READY (10.52.32.8) +02:05:14 — prefill + decode READY → benchmark_4 starts +02:05:20 — HTTP 500 — all benchmark requests fail +02:05:41 — Workflow finished (8m 33s) + +Error from frontend logs: +Invalid TCP address 'dynamo_prefill.generate-58b49ce145f56609' +Invalid TCP address 'dynamo_backend.generate-58b49ce145f5660b'
+
+
+

诊断 — sflow 编排层 vs 应用层错误

+ + + + + + + + + +
层级状态细节
sflow✓ OKDAG 已执行，全部任务已拉起，探针通过
GPU 分配✓ OK每节点 8 张 GPU，无重叠，每个 server TP=8
基础设施✓ OKetcd、NATS、frontend 均已 READY
路由✗ FAILfrontend 拿到的是服务名，而不是 host:port
基准测试✗ FAIL0/800 请求成功（全部 HTTP 500）
+
根因 +frontend 收到的是服务发现的服务名 +（例如 dynamo_prefill.generate-58b49...） +而不是 host:port 地址。 + +NATS/etcd 服务注册表返回的内部标识符 +TCP 路由无法解析。 + +修复：检查 DYN_REQUEST_PLANE 与 frontend +网络配置是否匹配 Dynamo 分离式路由。
+
+
+
+
+ + +
+
+12 / 15 +
+

CLI 速览

+ + + + + + + + + + + +
命令用途关键参数
sflow run执行工作流--dry-run --tui --set -f（多文件）
sflow batch生成 sbatch 脚本--submit --bulk-input --row
sflow compose合并多个 YAML--resolve --missable-tasks -o
sflow visualize渲染 DAG 图--format png/svg/mermaid
sflow sample列出 / 复制示例--list -o
sflow skill安装 AI Agent 技能--list -o
sflow upgrade原地重装（别名：sflow update）--branch --dry-run --force
+
+
+ + +
+
+
+
+14 / 16 +
+开发者体验 +

Agent 原生的工作流，每跑一次就更好一点

+
+
+
01

编写 · Agent 原生

AGENTS.md 工作流规则writing-sflow-yaml 技能schema + 示例
+
➔
+
02

校验

sflow run --dry-run解析后的变量资源 / GPU 方案
+
➔
+
03

运行与观测

带时间戳的任务事件TUI + 就绪探针sflow_monitor.log
+
➔
+
04

诊断与演进

sflow_summary.logtask.log + 失败提示result.json 指标
+
+
+ + AGENT 闭环 + + +
+
+
+
+ + +
+
+
+
+15 / 16 +
+愿景 +

异构计算

+

一份 recipe 就能把分离式流水线的每个阶段路由到最合适的资源池，同时保持同一份工作流契约。

+
+ + + + +Vera Rubin +GPU 资源池 • Prefill + +LPU Cluster +加速器 • Decode + + +sflow + + +prefill_server +Vera Rubin GPUs + +decode_server +LPU 加速器 + + + + + + + + + + + + +
+
一份 DAG · 专用加速器 · 同一份工作流契约
+
+
+ + +
+
+
+
+16 / 16 +
+立即上手 +

安装。探索。运行。

+

先在本地起步，在申请硬件之前完成校验，再以文档和代码仓库作为唯一事实来源。

+
+
# 从代码仓库安装 +$ uv venv --python python3 && source .venv/bin/activate +$ uv pip install "sflow @ git+https://github.com/NVIDIA/nv-sflow.git@main" + +# 验证并试跑一个本地工作流 +$ sflow --version +$ sflow sample self_contained/local/hello_world +$ sflow run -f hello_world.yaml --dry-run +✓ 先校验；确认方案无误后再运行
+
+ GITGitHub 仓库github.com/NVIDIA/nv-sflow→ + DOC文档nvidia.github.io/nv-sflow/docs→ + 01快速上手docs/user/quickstart→ + MAP功能地图按意图找到对应的指南→ +
+
+
本地DockerSlurmKubernetes同一份工作流契约
+
+
+ + + + From 6db2b64fccb64b3796fcfe7d1bc0b172f7727163 Mon Sep 17 00:00:00 2001 From: rogliu Date: Thu, 10 Sep 2026 17:56:08 +0800 Subject: [PATCH 3/3] Fix pages and add Zh-cn --- docs-site/docusaurus.config.js | 3 + docs-site/scripts/prepare-versioned-docs.js | 10 +- docs-site/sidebars.js | 10 +- docs-site/sidebarsConfig.js | 28 +- docs-site/sidebarsConfig.test.js | 24 +- docs-site/src/pages/index.js | 114 +++++- docs-site/static/sflow_intro.html | 356 ++++++++++++++++-- .../static}/sflow_intro_zh.html | 108 ++++-- 8 files changed, 559 insertions(+), 94 deletions(-) rename {docs/developer/dev-notes => docs-site/static}/sflow_intro_zh.html (96%) diff --git a/docs-site/docusaurus.config.js b/docs-site/docusaurus.config.js index db1708b..4dc2508 100644 --- a/docs-site/docusaurus.config.js +++ b/docs-site/docusaurus.config.js @@ -121,6 +121,9 @@ const config = { }, items: [ { type: "doc", docId: "user/intro", label: "Docs", position: "left" }, + // Agent Skills is a peer of Docs, not a subsection of it. Targets the + // standalone `agents` sidebar so the entry follows the version dropdown. + { type: "docSidebar", sidebarId: "agents", label: "Agent Skills", position: "left" }, { type: "search", position: "left" }, { type: "docsVersionDropdown", position: "right" }, { diff --git a/docs-site/scripts/prepare-versioned-docs.js b/docs-site/scripts/prepare-versioned-docs.js index c66c1c2..d9a47f8 100644 --- a/docs-site/scripts/prepare-versioned-docs.js +++ b/docs-site/scripts/prepare-versioned-docs.js @@ -5,7 +5,7 @@ const fs = require("fs"); const os = require("os"); const path = require("path"); -const { buildDocsSidebar } = require("../sidebarsConfig"); +const { buildDocsSidebar, buildAgentsSidebar } = require("../sidebarsConfig"); const { mirrorSkillsToAgents } = require("./mirror-skills"); const DOCS_SITE_DIR = path.resolve(__dirname, ".."); @@ -281,9 +281,15 @@ function prepareVersionedDocs(plan, options = {}) { handwrittenDir: paths.agentsSrcDir, destDir: path.join(versionDir, "agents"), }); + // Versions that predate the skills feature get no `agents` sidebar at all -- + // see buildAgentsSidebar. The navbar entry is guarded to match. + const agentsSidebar = buildAgentsSidebar(versionDir); writeJson( path.join(paths.versionedSidebarsDir, `${safeVersionDirName(version.label)}-sidebars.json`), - { docs: buildDocsSidebar(versionDir) }, + { + docs: buildDocsSidebar(versionDir), + ...(agentsSidebar ? { agents: agentsSidebar } : {}), + }, ); } diff --git a/docs-site/sidebars.js b/docs-site/sidebars.js index dbc187e..a02c00f 100644 --- a/docs-site/sidebars.js +++ b/docs-site/sidebars.js @@ -1,6 +1,6 @@ const path = require("path"); const fs = require("fs"); -const { buildDocsSidebar } = require("./sidebarsConfig"); +const { buildDocsSidebar, buildAgentsSidebar } = require("./sidebarsConfig"); // Mirror docusaurus.config.js: prefer the generated develop snapshot, otherwise // fall back to the repo-level docs/ directory for local dev. @@ -10,8 +10,14 @@ function currentDocsPath() { } /** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ +const currentDocs = currentDocsPath(); +const agents = buildAgentsSidebar(currentDocs); + const sidebars = { - docs: buildDocsSidebar(currentDocsPath()), + docs: buildDocsSidebar(currentDocs), + // Spread rather than assign: Docusaurus rejects an empty/undefined sidebar, so a + // snapshot without skills must omit the key entirely. + ...(agents ? { agents } : {}), }; module.exports = sidebars; diff --git a/docs-site/sidebarsConfig.js b/docs-site/sidebarsConfig.js index 9befa05..83f3663 100644 --- a/docs-site/sidebarsConfig.js +++ b/docs-site/sidebarsConfig.js @@ -96,16 +96,9 @@ function buildDocsSidebar(docsDir) { if (categories.length) { sidebar.push({ type: "category", label: "Sflow User Guide", collapsed: false, items: categories }); } - // Agent Skills: a labeled category wrapping the autogenerated agents/ tree, - // present only when this snapshot ships the skills mirror. - if (dirHasDocs(docsDir, "agents")) { - sidebar.push({ - type: "category", - label: "Agent Skills", - collapsed: false, - items: [{ type: "autogenerated", dirName: "agents" }], - }); - } + // Agent Skills deliberately does NOT appear here -- it is its own top-level + // sidebar (see buildAgentsSidebar) reached from its own navbar entry, so it is + // a peer of the user guide rather than a subsection buried inside it. for (const dir of AUTOGEN_SECTIONS) { if (!EXCLUDED_DIRS.has(dir) && dirHasDocs(docsDir, dir)) { sidebar.push({ type: "autogenerated", dirName: dir }); @@ -120,6 +113,20 @@ function buildDocsSidebar(docsDir) { return sidebar; } +// Build the standalone `agents` sidebar for the Agent Skills navbar entry. +// The mirrored agents/ tree already carries sidebar_position / _category_.json +// ordering, so plain autogeneration is enough -- no wrapper category, which would +// only repeat the navbar label back at the reader. +// +// Returns null when this snapshot ships no skills (e.g. v0.1.0, which predates the +// feature). Callers must omit the sidebar entirely in that case: Docusaurus rejects +// an empty sidebar array, and a version that never had agent skills should not +// advertise them. +function buildAgentsSidebar(docsDir) { + if (!dirHasDocs(docsDir, "agents")) return null; + return [{ type: "autogenerated", dirName: "agents" }]; +} + module.exports = { USER_GUIDE_CATEGORIES, AUTOGEN_SECTIONS, @@ -127,4 +134,5 @@ module.exports = { listUserDocIds, dirHasDocs, buildDocsSidebar, + buildAgentsSidebar, }; diff --git a/docs-site/sidebarsConfig.test.js b/docs-site/sidebarsConfig.test.js index f828a91..b078eb5 100644 --- a/docs-site/sidebarsConfig.test.js +++ b/docs-site/sidebarsConfig.test.js @@ -4,7 +4,7 @@ const fs = require("fs"); const os = require("os"); const path = require("path"); -const { buildDocsSidebar } = require("./sidebarsConfig"); +const { buildDocsSidebar, buildAgentsSidebar } = require("./sidebarsConfig"); function makeDocs(files) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "sflow-sidebar-")); @@ -43,16 +43,26 @@ test("groups user docs, keeps only existing ones, and buckets the rest under Mor { type: "category", label: "More", collapsed: false, items: ["user/zzz-extra"] }, ], }, - { - type: "category", - label: "Agent Skills", - collapsed: false, - items: [{ type: "autogenerated", dirName: "agents" }], - }, { type: "autogenerated", dirName: "release_notes" }, ]); }); +test("agent skills live in their own sidebar, not inside the docs sidebar", () => { + const dir = makeDocs(["user/intro.md", "agents/intro.md", "agents/writing-sflow-yaml/index.md"]); + + // The navbar entry owns the "Agent Skills" label, so the sidebar itself is just + // the autogenerated tree -- and it must not also appear under the docs sidebar. + assert.deepEqual(buildAgentsSidebar(dir), [{ type: "autogenerated", dirName: "agents" }]); + assert.equal(JSON.stringify(buildDocsSidebar(dir)).includes("agents"), false); +}); + +test("omits the agents sidebar entirely for snapshots that predate agent skills", () => { + // Docusaurus rejects an empty sidebar, and a version that never shipped skills + // should not advertise them -- so the key has to be absent, not empty. + assert.equal(buildAgentsSidebar(makeDocs(["user/intro.md"])), null); + assert.equal(buildAgentsSidebar(makeDocs(["agents/_category_.json"])), null); +}); + test("never includes the excluded plc or developer directories and falls back when nothing matches", () => { const dir = makeDocs(["plc/sflow_srd.md", "developer/note.md"]); assert.deepEqual(buildDocsSidebar(dir), [{ type: "autogenerated", dirName: "." }]); diff --git a/docs-site/src/pages/index.js b/docs-site/src/pages/index.js index 58e5f78..c27a600 100644 --- a/docs-site/src/pages/index.js +++ b/docs-site/src/pages/index.js @@ -1,36 +1,114 @@ import React from "react"; import Head from "@docusaurus/Head"; import useBaseUrl from "@docusaurus/useBaseUrl"; +import useIsBrowser from "@docusaurus/useIsBrowser"; +import { useLocation } from "@docusaurus/router"; + +// The homepage is a full-viewport frame around the static intro deck. Language is +// carried in the query string (`/?lang=zh`) rather than by swapping the iframe in +// place, so the Chinese deck is shareable, bookmarkable and survives a reload -- +// an in-place swap leaves the address bar on "/" and silently reverts on refresh. +// +// The deck itself drives this: when it detects it is framed, its language toggle +// rewrites the TOP url instead of navigating its own document. See the +// `langToggle` handler in static/sflow_intro*.html. +const DECKS = { + en: { + file: "/sflow_intro.html", + title: "NV-sflow — Declarative Workflow Descriptor", + description: + "Declarative workflow descriptor with swappable backends. Describe once, run anywhere.", + frameTitle: "NV-sflow Introduction", + htmlLang: "en", + }, + zh: { + file: "/sflow_intro_zh.html", + title: "NV-sflow — 面向大规模 GPU 集群的声明式工作流描述器", + description: "声明式工作流描述器，后端可自由切换。一次描述，随处运行。", + frameTitle: "NV-sflow 介绍", + htmlLang: "zh-CN", + }, +}; + +const DECK_BG = "#060a10"; // matches the deck's own background, so the pre-load frame is invisible export default function Home() { - const introUrl = useBaseUrl("/sflow_intro.html"); + const { search, hash } = useLocation(); + const isBrowser = useIsBrowser(); + + const lang = new URLSearchParams(search).get("lang") === "zh" ? "zh" : "en"; + const deck = DECKS[lang]; + + // Both are resolved unconditionally: useBaseUrl is a hook and cannot be called + // behind a branch without breaking the rules of hooks. + const enUrl = useBaseUrl(DECKS.en.file); + const zhUrl = useBaseUrl(DECKS.zh.file); + + // Forward the slide anchor (#s7) so switching language deep in the deck lands + // on the same slide instead of resetting to the title. + const slide = /^#s[0-9a-z]+$/i.test(hash) ? hash : ""; + const src = (lang === "zh" ? zhUrl : enUrl) + slide; return ( <> - NV-sflow — Declarative Workflow Descriptor - + + {deck.title} + + + -
-