Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions programs/sbd/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@ diagonalization benchmark from `github.com/r-ccs-cms/sbd`.

## Optional NSYS / NCU profiling

SBD keeps ordinary benchmark timing unprofiled. Set `BK_PROFILER=ncu` or
`SBD_PROFILER_TOOL=ncu` for a separate profiling run; `run.sh` maps that
request to the SBD NCU flow. The profile flow runs rank-0 host
SBD keeps ordinary benchmark timing unprofiled, then collects a separate
profiling run on supported NVIDIA GPU systems. Set `BK_SBD_NCU_PROFILE=false`
or `SBD_PROFILER_TOOL=none` to skip the additional profile flow. The profile
flow runs rank-0 host
`nsys profile --trace=cuda --sample=none`, exports the CUDA kernel summary,
generates `results/sbd_kernel_discovery.json` and `results/sbd_ncu_plan.json`,
and then uses that plan for rank-0 Nsight Compute acquisition when
Expand Down
17 changes: 15 additions & 2 deletions programs/sbd/profile.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,29 @@ sbd_ncu_profile_mode() {
printf '%s\n' "${BK_SBD_NCU_PROFILE_MODE:-discovery}"
}

sbd_supports_ncu_profile() {
case "$1" in
RIKYU|RC_DGXSP|RC_GH200) return 0 ;;
*) return 1 ;;
esac
}

sbd_configure_ncu_profile_from_run_env() {
local system_name="$1"
local profiler_tool
local profiler_level

profiler_tool=$(bk_resolve_profiler_tool none SBD_PROFILER_TOOL) || return 1
if [ -z "$profiler_tool" ]; then
if ! sbd_supports_ncu_profile "$system_name"; then
return 0
fi

profiler_tool=$(bk_resolve_profiler_tool ncu SBD_PROFILER_TOOL) || return 1
if [ -z "$profiler_tool" ]; then
if [ -z "${BK_SBD_NCU_PROFILE:-}" ]; then
export BK_SBD_NCU_PROFILE=false
fi
return 0
fi
if [ "$profiler_tool" != "ncu" ]; then
echo "SBD ${system_name}: only ncu is supported for separate profile acquisition." >&2
return 1
Expand Down
13 changes: 9 additions & 4 deletions scripts/tests/test_sbd_ncu_profile.sh
Original file line number Diff line number Diff line change
Expand Up @@ -142,15 +142,20 @@ chmod +x "${FAKE_BIN}/mpirun" "${FAKE_BIN}/nsys" "${FAKE_BIN}/ncu" "${RUN_DIR}/d
export PATH="${FAKE_BIN}:${PATH}"

(
export BK_PROFILER=ncu
unset BK_PROFILER BK_PROFILER_LEVEL BK_SBD_NCU_PROFILE BK_SBD_NCU_PROFILER_LEVEL SBD_PROFILER_TOOL
sbd_configure_ncu_profile_from_run_env RC_FX700
test -z "${BK_SBD_NCU_PROFILE:-}"
)

(
unset BK_PROFILER BK_PROFILER_LEVEL BK_SBD_NCU_PROFILE BK_SBD_NCU_PROFILER_LEVEL
export SBD_PROFILER_TOOL=none
unset BK_SBD_NCU_PROFILE BK_SBD_NCU_PROFILER_LEVEL
sbd_configure_ncu_profile_from_run_env RIKYU
test -z "${BK_SBD_NCU_PROFILE:-}"
test "${BK_SBD_NCU_PROFILE:-}" = "false"
)

pushd "${RUN_DIR}" >/dev/null
export BK_PROFILER=ncu
unset BK_PROFILER SBD_PROFILER_TOOL BK_SBD_NCU_PROFILE BK_SBD_NCU_PROFILER_LEVEL
export BK_SBD_NCU_PROFILE_MODE=discovery
export BK_SBD_NCU_PLAN_TOP_K=5
export BK_PROFILER_LEVEL=single
Expand Down
Loading