diff --git a/sw/nic/gpuagent/api/gpu.cc b/sw/nic/gpuagent/api/gpu.cc index 40084036..ef3ccad7 100644 --- a/sw/nic/gpuagent/api/gpu.cc +++ b/sw/nic/gpuagent/api/gpu.cc @@ -150,7 +150,8 @@ gpu_entry::update_handler(api_params_base *api_params) { } void -gpu_entry::fill_stats_(aga_gpu_stats_t *stats) { +gpu_entry::fill_stats_(aga_gpu_stats_t *stats, + const aga_gpu_get_filter_t *filter) { // fill stats only for non-parent GPUs if (child_gpus_.size()) { return; @@ -158,11 +159,12 @@ gpu_entry::fill_stats_(aga_gpu_stats_t *stats) { // fetch stats from smi apis smi_gpu_fill_stats(handle_, &key_, is_partitioned_, (partition_id_ == AGA_GPU_INVALID_PARTITION_ID) ? 0 : partition_id_, - first_partition_handle_, stats); + first_partition_handle_, stats, filter); } void -gpu_entry::fill_status_(aga_gpu_spec_t *spec, aga_gpu_status_t *status) { +gpu_entry::fill_status_(aga_gpu_spec_t *spec, aga_gpu_status_t *status, + const aga_gpu_get_filter_t *filter) { if (child_gpus_.size()) { status->num_gpu_partition = child_gpus_.size(); // for parent GPUs get uuids of all children @@ -175,7 +177,7 @@ gpu_entry::fill_status_(aga_gpu_spec_t *spec, aga_gpu_status_t *status) { if (parent_gpu_.valid()) { status->physical_gpu = parent_gpu_; } - smi_gpu_fill_status(handle_, &key_, id_, spec, status); + smi_gpu_fill_status(handle_, &key_, id_, spec, status, filter); } } @@ -201,10 +203,10 @@ gpu_entry::fill_spec_(aga_gpu_spec_t *spec) { } sdk_ret_t -gpu_entry::read(aga_gpu_info_t *info) { +gpu_entry::read(aga_gpu_info_t *info, const aga_gpu_get_filter_t *filter) { fill_spec_(&info->spec); - fill_status_(&info->spec, &info->status); - fill_stats_(&info->stats); + fill_status_(&info->spec, &info->status, filter); + fill_stats_(&info->stats, filter); return SDK_RET_OK; } diff --git a/sw/nic/gpuagent/api/gpu.hpp b/sw/nic/gpuagent/api/gpu.hpp index 04c99705..a59d01a5 100644 --- a/sw/nic/gpuagent/api/gpu.hpp +++ b/sw/nic/gpuagent/api/gpu.hpp @@ -99,9 +99,11 @@ class gpu_entry : public api_base { virtual sdk_ret_t delete_handler(api_params_base *api_params) override; /// \brief read config - /// \param[out] info pointer to the info object + /// \param[out] info pointer to the info object + /// \param[in] filter attributes to skip (NULL fetches all attributes) /// \return SDK_RET_OK on success, failure status code on error - sdk_ret_t read(aga_gpu_info_t *info); + sdk_ret_t read(aga_gpu_info_t *info, + const aga_gpu_get_filter_t *filter = NULL); /// \brief return stringified key of the object (for debugging) virtual string key2str(void) const override { @@ -296,11 +298,15 @@ class gpu_entry : public api_base { /// \brief fill the gpu operational status /// \param[in] spec config specification /// \param[out] status operational status - void fill_status_(aga_gpu_spec_t *spec, aga_gpu_status_t *status); + /// \param[in] filter attributes to skip (NULL fetches all attributes) + void fill_status_(aga_gpu_spec_t *spec, aga_gpu_status_t *status, + const aga_gpu_get_filter_t *filter); /// \brief fill the gpu statistics /// \param[out] stats statistics - void fill_stats_(aga_gpu_stats_t *stats); + /// \param[in] filter attributes to skip (NULL fetches all attributes) + void fill_stats_(aga_gpu_stats_t *stats, + const aga_gpu_get_filter_t *filter); private: /// uuid of the object diff --git a/sw/nic/gpuagent/api/gpu_api.cc b/sw/nic/gpuagent/api/gpu_api.cc index e0a099d8..3f5c0da2 100644 --- a/sw/nic/gpuagent/api/gpu_api.cc +++ b/sw/nic/gpuagent/api/gpu_api.cc @@ -83,7 +83,8 @@ aga_gpu_create (_In_ aga_gpu_spec_t *spec) } sdk_ret_t -aga_gpu_read (_In_ aga_obj_key_t *key, _Out_ aga_gpu_info_t *info) +aga_gpu_read (_In_ aga_obj_key_t *key, _Out_ aga_gpu_info_t *info, + const aga_gpu_get_filter_t *filter) { sdk_ret_t ret; gpu_entry *entry; @@ -95,12 +96,13 @@ aga_gpu_read (_In_ aga_obj_key_t *key, _Out_ aga_gpu_info_t *info) if (unlikely(ret != SDK_RET_OK)) { return ret; } - return entry->read(info); + return entry->read(info, filter); } typedef struct aga_gpu_read_args_s { void *ctxt; gpu_read_cb_t cb; + const aga_gpu_get_filter_t *filter; } aga_gpu_read_args_t; static bool @@ -120,19 +122,21 @@ aga_gpu_info_from_entry (void *entry, void *ctxt) } memset(&info, 0, sizeof(aga_gpu_info_t)); // call entry read - gpu->read(&info); + gpu->read(&info, args->filter); // call cb on info args->cb(&info, args->ctxt); return false; } sdk_ret_t -aga_gpu_read_all (gpu_read_cb_t gpu_read_cb, void *ctxt) +aga_gpu_read_all (gpu_read_cb_t gpu_read_cb, void *ctxt, + const aga_gpu_get_filter_t *filter) { aga_gpu_read_args_t args = { 0 }; args.ctxt = ctxt; args.cb = gpu_read_cb; + args.filter = filter; return gpu_db()->walk(aga_gpu_info_from_entry, &args); } diff --git a/sw/nic/gpuagent/api/include/aga_gpu.hpp b/sw/nic/gpuagent/api/include/aga_gpu.hpp index 3e38df3c..b16e8f78 100644 --- a/sw/nic/gpuagent/api/include/aga_gpu.hpp +++ b/sw/nic/gpuagent/api/include/aga_gpu.hpp @@ -74,6 +74,38 @@ typedef enum aga_gpu_admin_state_e { AGA_GPU_ADMIN_STATE_DOWN = 2, } aga_gpu_admin_state_t; +/// \brief GPU attributes that can be skipped in a get request; +/// default (all false) fetches all attributes +typedef struct aga_gpu_get_filter_s { + /// skip clock status + bool skip_clock_status; + /// skip PCIe status + bool skip_pcie_status; + /// skip XGMI error status + bool skip_xgmi_status; + /// skip process list + bool skip_process_status; + /// skip UALink state + bool skip_ualink_status; + /// skip VRAM usage + bool skip_vram_usage_stats; + /// skip ECC error counts + bool skip_ecc_stats; + /// skip violation stats + bool skip_violation_stats; + /// skip PCIe stats + bool skip_pcie_stats; + /// skip XGMI counters + bool skip_xgmi_stats; + /// skip GPU activity and usage + bool skip_activity_stats; +} aga_gpu_get_filter_t; + +/// true if the given attribute should be skipped; +/// a NULL filter fetches all attributes +#define AGA_GPU_SKIP(_filter_, _field_) \ + (((_filter_) != NULL) && ((_filter_)->_field_)) + /// \brief GPU clock types typedef enum aga_gpu_clock_type_e { AGA_GPU_CLOCK_TYPE_NONE = 0, @@ -996,18 +1028,22 @@ typedef struct aga_cper_info_s { sdk_ret_t aga_gpu_create(_In_ aga_gpu_spec_t *spec); /// \brief read gpu -/// \param[in] key key of the gpu object -/// \param[out] info information +/// \param[in] key key of the gpu object +/// \param[out] info information +/// \param[in] filter attributes to skip (NULL fetches all attributes) /// \return #SDK_RET_OK on success, failure status code on error -sdk_ret_t aga_gpu_read(_In_ aga_obj_key_t *key, _Out_ aga_gpu_info_t *info); +sdk_ret_t aga_gpu_read(_In_ aga_obj_key_t *key, _Out_ aga_gpu_info_t *info, + const aga_gpu_get_filter_t *filter = NULL); typedef void (*gpu_read_cb_t)(aga_gpu_info_t *info, void *ctxt); /// \brief read all gpu information -/// \param[in] cb callback function -/// \param[in] ctxt opaque context passed to cb +/// \param[in] cb callback function +/// \param[in] ctxt opaque context passed to cb +/// \param[in] filter attributes to skip (NULL fetches all attributes) /// \return #SDK_RET_OK on success, failure status code on error -sdk_ret_t aga_gpu_read_all(_In_ gpu_read_cb_t gpu_read_cb, _In_ void *ctxt); +sdk_ret_t aga_gpu_read_all(_In_ gpu_read_cb_t gpu_read_cb, _In_ void *ctxt, + const aga_gpu_get_filter_t *filter = NULL); /// \brief function to get compute partition info of a given physical gpu /// which has been partitioned diff --git a/sw/nic/gpuagent/api/smi/amdsmi/smi_api.cc b/sw/nic/gpuagent/api/smi/amdsmi/smi_api.cc index 06576027..c0bdff15 100644 --- a/sw/nic/gpuagent/api/smi/amdsmi/smi_api.cc +++ b/sw/nic/gpuagent/api/smi/amdsmi/smi_api.cc @@ -827,48 +827,60 @@ sdk_ret_t smi_gpu_fill_status (aga_gpu_handle_t gpu_handle, const aga_obj_key_t *gpu_key, uint32_t gpu_id, - aga_gpu_spec_t *spec, aga_gpu_status_t *status) + aga_gpu_spec_t *spec, aga_gpu_status_t *status, + const aga_gpu_get_filter_t *filter) { amdsmi_status_t amdsmi_ret; amdsmi_xgmi_status_t xgmi_st; amdsmi_gpu_metrics_t metrics_info = { 0 }; - { - std::lock_guard lock(g_gpu_metrics_mutex); - if (g_gpu_metrics.find(gpu_handle) != g_gpu_metrics.end()) { - metrics_info = g_gpu_metrics[gpu_handle]; + // fill clock status + if (!AGA_GPU_SKIP(filter, skip_clock_status)) { + { + std::lock_guard lock(g_gpu_metrics_mutex); + if (g_gpu_metrics.find(gpu_handle) != g_gpu_metrics.end()) { + metrics_info = g_gpu_metrics[gpu_handle]; + } + } + if (metrics_info.common_header.structure_size != 0) { + // fill the clock status with metrics info + smi_fill_clock_status_(gpu_handle, spec, status, &metrics_info); + // fill firmware timestamp + status->fw_timestamp = metrics_info.firmware_timestamp; + if (metrics_info.throttle_status != + std::numeric_limits::max()) { + status->throttling_status = + metrics_info.throttle_status ? + AGA_GPU_THROTTLING_STATUS_ON : + AGA_GPU_THROTTLING_STATUS_OFF; + } + status->xgmi_status.width = metrics_info.xgmi_link_width; + status->xgmi_status.speed = metrics_info.xgmi_link_speed; + status->vram_status.max_bandwidth = metrics_info.vram_max_bandwidth; + } else { + AGA_TRACE_ERR("GPU metrics info not available for GPU {}", + gpu_handle); } - } - if (metrics_info.common_header.structure_size != 0) { - // fill the clock status with metrics info - smi_fill_clock_status_(gpu_handle, spec, status, &metrics_info); - // fill firmware timestamp - status->fw_timestamp = metrics_info.firmware_timestamp; - if (metrics_info.throttle_status != - std::numeric_limits::max()) { - status->throttling_status = - metrics_info.throttle_status ? AGA_GPU_THROTTLING_STATUS_ON : - AGA_GPU_THROTTLING_STATUS_OFF; - } - status->xgmi_status.width = metrics_info.xgmi_link_width; - status->xgmi_status.speed = metrics_info.xgmi_link_speed; - status->vram_status.max_bandwidth = metrics_info.vram_max_bandwidth; - } else { - AGA_TRACE_ERR("Failed to get GPU metrics info for GPU {}, err {}", - gpu_handle, amdsmi_ret); } // fill the PCIe status - smi_fill_pcie_status_(gpu_handle, status); + if (!AGA_GPU_SKIP(filter, skip_pcie_status)) { + smi_fill_pcie_status_(gpu_handle, status); + } // fill the xgmi error count - amdsmi_ret = amdsmi_gpu_xgmi_error_status(gpu_handle, &xgmi_st); - if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) { - AGA_TRACE_ERR("Failed to get xgmi error status for GPU {}, err {}", - gpu_handle, amdsmi_ret); - } else { - status->xgmi_status.error_status = smi_to_aga_gpu_xgmi_error(xgmi_st); + if (!AGA_GPU_SKIP(filter, skip_xgmi_status)) { + amdsmi_ret = amdsmi_gpu_xgmi_error_status(gpu_handle, &xgmi_st); + if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) { + AGA_TRACE_ERR("Failed to get xgmi error status for GPU {}, err {}", + gpu_handle, amdsmi_ret); + } else { + status->xgmi_status.error_status = + smi_to_aga_gpu_xgmi_error(xgmi_st); + } } // fill list of pids using the GPU - smi_fill_gpu_kfd_pid_status_(gpu_handle, gpu_id, status); + if (!AGA_GPU_SKIP(filter, skip_process_status)) { + smi_fill_gpu_kfd_pid_status_(gpu_handle, gpu_id, status); + } // TODO: oper status // TODO: RAS status return SDK_RET_OK; @@ -1282,7 +1294,8 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle, bool is_partitioned, uint32_t partition_id, aga_gpu_handle_t first_partition_handle, - aga_gpu_stats_t *stats) + aga_gpu_stats_t *stats, + const aga_gpu_get_filter_t *filter) { amdsmi_status_t amdsmi_ret; uint64_t sent, received, max_pkt_size; @@ -1297,7 +1310,9 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle, sizeof(aga_gpu_xgmi_link_stats_t) * AGA_GPU_MAX_XGMI_LINKS); // fill VRAM usage - smi_fill_vram_usage_(gpu_handle, &stats->vram_usage); + if (!AGA_GPU_SKIP(filter, skip_vram_usage_stats)) { + smi_fill_vram_usage_(gpu_handle, &stats->vram_usage); + } // fill additional statistics from gpu metrics { std::lock_guard lock(g_gpu_metrics_mutex); @@ -1316,18 +1331,24 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle, // fan speed stats->fan_speed = metrics_info.current_fan_speed; // xgmi link stats - for (uint32_t i = 0; i < AGA_GPU_MAX_XGMI_LINKS; i++) { - stats->xgmi_link_stats[i].data_read = - metrics_info.xgmi_read_data_acc[i]; - stats->xgmi_link_stats[i].data_write = - metrics_info.xgmi_write_data_acc[i]; + if (!AGA_GPU_SKIP(filter, skip_xgmi_stats)) { + for (uint32_t i = 0; i < AGA_GPU_MAX_XGMI_LINKS; i++) { + stats->xgmi_link_stats[i].data_read = + metrics_info.xgmi_read_data_acc[i]; + stats->xgmi_link_stats[i].data_write = + metrics_info.xgmi_write_data_acc[i]; + } } // fill violation statistics only for primary partition if (!partition_id) { - smi_fill_ecc_stats_(gpu_handle, stats); - smi_fill_violation_stats_(gpu_handle, partition_id, - &metrics_info, - &stats->violation_stats); + if (!AGA_GPU_SKIP(filter, skip_ecc_stats)) { + smi_fill_ecc_stats_(gpu_handle, stats); + } + if (!AGA_GPU_SKIP(filter, skip_violation_stats)) { + smi_fill_violation_stats_(gpu_handle, partition_id, + &metrics_info, + &stats->violation_stats); + } } // fill the energy consumed stats->energy_consumed = metrics_info.energy_accumulator * @@ -1344,19 +1365,22 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle, (float)metrics_info.temperature_hbm[i]; } // pcie stats - stats->pcie_stats.replay_count = metrics_info.pcie_replay_count_acc; - stats->pcie_stats.recovery_count = - metrics_info.pcie_l0_to_recov_count_acc; - stats->pcie_stats.replay_rollover_count = - metrics_info.pcie_replay_rover_count_acc; - stats->pcie_stats.nack_sent_count = - metrics_info.pcie_nak_sent_count_acc; - stats->pcie_stats.nack_received_count = - metrics_info.pcie_nak_rcvd_count_acc; - stats->pcie_stats.bidir_bandwidth = - metrics_info.pcie_bandwidth_acc; + if (!AGA_GPU_SKIP(filter, skip_pcie_stats)) { + stats->pcie_stats.replay_count = metrics_info.pcie_replay_count_acc; + stats->pcie_stats.recovery_count = + metrics_info.pcie_l0_to_recov_count_acc; + stats->pcie_stats.replay_rollover_count = + metrics_info.pcie_replay_rover_count_acc; + stats->pcie_stats.nack_sent_count = + metrics_info.pcie_nak_sent_count_acc; + stats->pcie_stats.nack_received_count = + metrics_info.pcie_nak_rcvd_count_acc; + stats->pcie_stats.bidir_bandwidth = + metrics_info.pcie_bandwidth_acc; + } // fill activity and usage information based on partition mode - if (!is_partitioned) { + if (!AGA_GPU_SKIP(filter, skip_activity_stats) && + !is_partitioned) { // non-partitioned mode: use cached metrics_info stats->usage.gfx_activity = metrics_info.average_gfx_activity; stats->usage.umc_activity = metrics_info.average_umc_activity; @@ -1386,46 +1410,54 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle, } } // read PCIe throughput - amdsmi_ret = amdsmi_get_gpu_pci_throughput(gpu_handle, &sent, &received, - &max_pkt_size); - if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) { - AGA_TRACE_ERR("Failed to get PCIe throughput for GPU {}, err {}", - gpu_handle, amdsmi_ret); - } else { - stats->pcie_stats.tx_bytes = received; - stats->pcie_stats.rx_bytes = sent; + if (!AGA_GPU_SKIP(filter, skip_pcie_stats)) { + amdsmi_ret = amdsmi_get_gpu_pci_throughput(gpu_handle, &sent, &received, + &max_pkt_size); + if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) { + AGA_TRACE_ERR("Failed to get PCIe throughput for GPU {}, err {}", + gpu_handle, amdsmi_ret); + } else { + stats->pcie_stats.tx_bytes = received; + stats->pcie_stats.rx_bytes = sent; + } } // read xgmi stats - g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_0_NOP_TX, - &stats->xgmi_neighbor0_tx_nops); - g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_0_REQUEST_TX, - &stats->xgmi_neighbor0_tx_requests); - g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_0_RESPONSE_TX, - &stats->xgmi_neighbor0_tx_responses); - g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_0_BEATS_TX, - &stats->xgmi_neighbor0_tx_beats); - g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_1_NOP_TX, - &stats->xgmi_neighbor1_tx_nops); - g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_1_REQUEST_TX, - &stats->xgmi_neighbor1_tx_requests); - g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_1_RESPONSE_TX, - &stats->xgmi_neighbor1_tx_responses); - g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_1_BEATS_TX, - &stats->xgmi_neighbor1_tx_beats); - g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_DATA_OUT_0, - &stats->xgmi_neighbor0_tx_throughput); - g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_DATA_OUT_1, - &stats->xgmi_neighbor1_tx_throughput); - g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_DATA_OUT_2, - &stats->xgmi_neighbor2_tx_throughput); - g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_DATA_OUT_3, - &stats->xgmi_neighbor3_tx_throughput); - g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_DATA_OUT_4, - &stats->xgmi_neighbor4_tx_throughput); - g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_DATA_OUT_5, - &stats->xgmi_neighbor5_tx_throughput); - // fill activity and usage information based on partition mode - if (is_partitioned) { + if (!AGA_GPU_SKIP(filter, skip_xgmi_stats)) { + g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_0_NOP_TX, + &stats->xgmi_neighbor0_tx_nops); + g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_0_REQUEST_TX, + &stats->xgmi_neighbor0_tx_requests); + g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_0_RESPONSE_TX, + &stats->xgmi_neighbor0_tx_responses); + g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_0_BEATS_TX, + &stats->xgmi_neighbor0_tx_beats); + g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_1_NOP_TX, + &stats->xgmi_neighbor1_tx_nops); + g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_1_REQUEST_TX, + &stats->xgmi_neighbor1_tx_requests); + g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_1_RESPONSE_TX, + &stats->xgmi_neighbor1_tx_responses); + g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_1_BEATS_TX, + &stats->xgmi_neighbor1_tx_beats); + g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_DATA_OUT_0, + &stats->xgmi_neighbor0_tx_throughput); + g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_DATA_OUT_1, + &stats->xgmi_neighbor1_tx_throughput); + g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_DATA_OUT_2, + &stats->xgmi_neighbor2_tx_throughput); + g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_DATA_OUT_3, + &stats->xgmi_neighbor3_tx_throughput); + g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_DATA_OUT_4, + &stats->xgmi_neighbor4_tx_throughput); + g_smi_state.read_counter(gpu_handle, AMDSMI_EVNT_XGMI_DATA_OUT_5, + &stats->xgmi_neighbor5_tx_throughput); + } + // fill activity, usage and violation information based on partition mode; + // both are derived from the partition-specific metrics, so fetch them if + // either one is requested + if (is_partitioned && + (!AGA_GPU_SKIP(filter, skip_activity_stats) || + !AGA_GPU_SKIP(filter, skip_violation_stats))) { // partitioned mode: fetch partition-specific metrics amdsmi_ret = amdsmi_get_gpu_partition_metrics_info(gpu_handle, &metrics_info); @@ -1434,38 +1466,44 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle, "err {}", gpu_handle, amdsmi_ret); // fall through to g_gpu_metrics cache fallback below (partition 0 only) } else { - // activity information - stats->usage.gfx_activity = metrics_info.average_gfx_activity; - stats->usage.umc_activity = metrics_info.average_umc_activity; - stats->usage.mm_activity = metrics_info.average_mm_activity; - stats->gfx_activity_accumulated = metrics_info.gfx_activity_acc; - stats->mem_activity_accumulated = metrics_info.mem_activity_acc; - - // VCN busy stats (activity not available in partition mode) - for (uint16_t i = 0; i < AMDSMI_MAX_NUM_VCN; i++) { - stats->usage.vcn_busy[i] = metrics_info.xcp_stats[0].vcn_busy[i]; - } + if (!AGA_GPU_SKIP(filter, skip_activity_stats)) { + // activity information + stats->usage.gfx_activity = metrics_info.average_gfx_activity; + stats->usage.umc_activity = metrics_info.average_umc_activity; + stats->usage.mm_activity = metrics_info.average_mm_activity; + stats->gfx_activity_accumulated = metrics_info.gfx_activity_acc; + stats->mem_activity_accumulated = metrics_info.mem_activity_acc; + + // VCN busy stats (activity not available in partition mode) + for (uint16_t i = 0; i < AMDSMI_MAX_NUM_VCN; i++) { + stats->usage.vcn_busy[i] = + metrics_info.xcp_stats[0].vcn_busy[i]; + } - // JPEG busy stats (activity not available in partition mode) - for (uint16_t i = 0; i < AMDSMI_MAX_NUM_JPEG_ENG_V1; i++) { - stats->usage.jpeg_busy[i] = metrics_info.xcp_stats[0].jpeg_busy[i]; - } + // JPEG busy stats (activity not available in partition mode) + for (uint16_t i = 0; i < AMDSMI_MAX_NUM_JPEG_ENG_V1; i++) { + stats->usage.jpeg_busy[i] = + metrics_info.xcp_stats[0].jpeg_busy[i]; + } - // GFX busy instances - for (uint16_t i = 0; i < AMDSMI_MAX_NUM_XCC; i++) { - stats->usage.gfx_busy_inst[i] = - metrics_info.xcp_stats[0].gfx_busy_inst[i]; + // GFX busy instances + for (uint16_t i = 0; i < AMDSMI_MAX_NUM_XCC; i++) { + stats->usage.gfx_busy_inst[i] = + metrics_info.xcp_stats[0].gfx_busy_inst[i]; + } } - // fill violation stats for partitioned mode - smi_fill_violation_stats_(gpu_handle, partition_id, - &metrics_info, - &stats->violation_stats); + if (!AGA_GPU_SKIP(filter, skip_violation_stats)) { + smi_fill_violation_stats_(gpu_handle, partition_id, + &metrics_info, + &stats->violation_stats); + } } } // always fill for primary partition with cached metrics info, // as primary partition metrics is not updated in new API. - if (!partition_id) { + if (!partition_id && + !AGA_GPU_SKIP(filter, skip_activity_stats)) { { std::lock_guard lock(g_gpu_metrics_mutex); if (g_gpu_metrics.find(gpu_handle) != g_gpu_metrics.end()) { diff --git a/sw/nic/gpuagent/api/smi/gimamdsmi/smi_api.cc b/sw/nic/gpuagent/api/smi/gimamdsmi/smi_api.cc index 2f422aba..419c9fbc 100644 --- a/sw/nic/gpuagent/api/smi/gimamdsmi/smi_api.cc +++ b/sw/nic/gpuagent/api/smi/gimamdsmi/smi_api.cc @@ -894,7 +894,8 @@ sdk_ret_t smi_gpu_fill_status (aga_gpu_handle_t gpu_handle_in, const aga_obj_key_t *gpu_key, uint32_t gpu_id, - aga_gpu_spec_t *spec, aga_gpu_status_t *status) + aga_gpu_spec_t *spec, aga_gpu_status_t *status, + const aga_gpu_get_filter_t *filter) { AGA_SMI_SESSION_GUARD(gpu_key, gpu_handle_in); @@ -904,9 +905,13 @@ smi_gpu_fill_status (aga_gpu_handle_t gpu_handle_in, status->xgmi_status.error_status = AGA_GPU_XGMI_STATUS_NONE; // fill the clock status without metrics info - smi_fill_clock_status_(gpu_handle, status); + if (!AGA_GPU_SKIP(filter, skip_clock_status)) { + smi_fill_clock_status_(gpu_handle, status); + } // fill the PCIe status - smi_fill_pcie_status_(gpu_handle, status); + if (!AGA_GPU_SKIP(filter, skip_pcie_status)) { + smi_fill_pcie_status_(gpu_handle, status); + } // TODO: oper status // TODO: RAS status return SDK_RET_OK; @@ -1083,7 +1088,8 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle_in, bool is_partitioned, uint32_t partition_id, aga_gpu_handle_t first_partition_handle, - aga_gpu_stats_t *stats) + aga_gpu_stats_t *stats, + const aga_gpu_get_filter_t *filter) { sdk_ret_t ret; int64_t temperature; @@ -1116,33 +1122,41 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle_in, stats->voltage.memory_voltage = power_info.mem_voltage; } // fill the GPU usage - amdsmi_ret = amdsmi_get_gpu_activity(gpu_handle, &usage_info); - if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) { - AGA_TRACE_ERR("Failed to get GPU activity for GPU {}, err {}", - gpu_handle, amdsmi_ret); - } else { - stats->usage.umc_activity = usage_info.umc_activity; - stats->usage.mm_activity = usage_info.mm_activity; - stats->usage.gfx_activity = usage_info.gfx_activity; + if (!AGA_GPU_SKIP(filter, skip_activity_stats)) { + amdsmi_ret = amdsmi_get_gpu_activity(gpu_handle, &usage_info); + if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) { + AGA_TRACE_ERR("Failed to get GPU activity for GPU {}, err {}", + gpu_handle, amdsmi_ret); + } else { + stats->usage.umc_activity = usage_info.umc_activity; + stats->usage.mm_activity = usage_info.mm_activity; + stats->usage.gfx_activity = usage_info.gfx_activity; + } + } + // fill VCN/JPEG/instantaneous activity and violation residency from the + // gpu_metrics stream; walk it if either activity or violation is requested + if (!AGA_GPU_SKIP(filter, skip_activity_stats) || + !AGA_GPU_SKIP(filter, skip_violation_stats)) { + smi_walk_gpu_metrics(gpu_handle, stats); } - // fill VCN/JPEG/instantaneous activity from gpu_metrics stream - smi_walk_gpu_metrics(gpu_handle, stats); // fill the PCIe stats - amdsmi_ret = amdsmi_get_pcie_info(gpu_handle, &pcie_info); - if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) { - AGA_TRACE_ERR("Failed to get PCIe info for GPU {}, err {}", - gpu_handle, amdsmi_ret); - } else { - stats->pcie_stats.replay_count = - pcie_info.pcie_metric.pcie_replay_count; - stats->pcie_stats.recovery_count = - pcie_info.pcie_metric.pcie_l0_to_recovery_count; - stats->pcie_stats.replay_rollover_count = - pcie_info.pcie_metric.pcie_replay_roll_over_count; - stats->pcie_stats.nack_sent_count = - pcie_info.pcie_metric.pcie_nak_sent_count; - stats->pcie_stats.nack_received_count = - pcie_info.pcie_metric.pcie_nak_received_count; + if (!AGA_GPU_SKIP(filter, skip_pcie_stats)) { + amdsmi_ret = amdsmi_get_pcie_info(gpu_handle, &pcie_info); + if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) { + AGA_TRACE_ERR("Failed to get PCIe info for GPU {}, err {}", + gpu_handle, amdsmi_ret); + } else { + stats->pcie_stats.replay_count = + pcie_info.pcie_metric.pcie_replay_count; + stats->pcie_stats.recovery_count = + pcie_info.pcie_metric.pcie_l0_to_recovery_count; + stats->pcie_stats.replay_rollover_count = + pcie_info.pcie_metric.pcie_replay_roll_over_count; + stats->pcie_stats.nack_sent_count = + pcie_info.pcie_metric.pcie_nak_sent_count; + stats->pcie_stats.nack_received_count = + pcie_info.pcie_metric.pcie_nak_received_count; + } } // fill the edge temperature amdsmi_ret = amdsmi_get_temp_metric(gpu_handle, @@ -1215,7 +1229,9 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle_in, stats->temperature.hbm_temperature[3] = (float)temperature; } // fill ECC block stats - smi_fill_ecc_stats_(gpu_handle, stats); + if (!AGA_GPU_SKIP(filter, skip_ecc_stats)) { + smi_fill_ecc_stats_(gpu_handle, stats); + } return SDK_RET_OK; } diff --git a/sw/nic/gpuagent/api/smi/smi_api.hpp b/sw/nic/gpuagent/api/smi/smi_api.hpp index 37930327..f0bd3a47 100644 --- a/sw/nic/gpuagent/api/smi/smi_api.hpp +++ b/sw/nic/gpuagent/api/smi/smi_api.hpp @@ -73,11 +73,13 @@ sdk_ret_t smi_gpu_fill_spec(aga_gpu_handle_t handle, /// \param[in] uuid stable GPU UUID /// \param[in] spec GPU operational spec /// \param[out] status operational status to be filled +/// \param[in] filter attributes to skip (NULL fetches all attributes) /// \return SDK_RET_OK or error code in case of failure sdk_ret_t smi_gpu_fill_status(aga_gpu_handle_t handle, const aga_obj_key_t *gpu_key, uint32_t id, - aga_gpu_spec_t *spec, aga_gpu_status_t *status); + aga_gpu_spec_t *spec, aga_gpu_status_t *status, + const aga_gpu_get_filter_t *filter); /// \brief fill gpu object statistics /// \param[in] handle GPU handle (uuid refreshes inside) @@ -88,13 +90,15 @@ sdk_ret_t smi_gpu_fill_status(aga_gpu_handle_t handle, /// \param[in] main_partition_handle in case of GPU partitions, handle of the /// first partition, else, GPU handle /// \param[out] stats gpu object stats to be filled +/// \param[in] filter attributes to skip (NULL fetches all attributes) /// \return SDK_RET_OK or error code in case of failure sdk_ret_t smi_gpu_fill_stats(aga_gpu_handle_t handle, const aga_obj_key_t *gpu_key, bool is_partitioned, uint32_t partition_id, aga_gpu_handle_t first_partition_handle, - aga_gpu_stats_t *stats); + aga_gpu_stats_t *stats, + const aga_gpu_get_filter_t *filter); /// \brief read all the events and invokve the callback provided for each /// \param[in] cb callback function pointer diff --git a/sw/nic/gpuagent/api/smi/smi_api_mock.cc b/sw/nic/gpuagent/api/smi/smi_api_mock.cc index ccf9a2fc..434dbdf0 100644 --- a/sw/nic/gpuagent/api/smi/smi_api_mock.cc +++ b/sw/nic/gpuagent/api/smi/smi_api_mock.cc @@ -215,7 +215,8 @@ sdk_ret_t smi_gpu_fill_status (aga_gpu_handle_t gpu_handle, const aga_obj_key_t *gpu_key, uint32_t gpu_id, - aga_gpu_spec_t *spec, aga_gpu_status_t *status) + aga_gpu_spec_t *spec, aga_gpu_status_t *status, + const aga_gpu_get_filter_t *filter) { status->index = gpu_id; status->handle = gpu_handle; @@ -246,16 +247,20 @@ smi_gpu_fill_status (aga_gpu_handle_t gpu_handle, fill_gpu_fw_version_(&status->fw_version[9], "VCN", "0x0110101b"); // fill the memory vendor strncpy(status->memory_vendor, "hynix", AGA_MAX_STR_LEN); - smi_fill_clock_status_(gpu_handle, status); + if (!AGA_GPU_SKIP(filter, skip_clock_status)) { + smi_fill_clock_status_(gpu_handle, status); + } // fill the PCIe bus id strncpy(status->pcie_status.pcie_bus_id, g_gpu_map[gpu_handle].bdf.c_str(), AGA_MAX_STR_LEN); - status->pcie_status.slot_type = AGA_PCIE_SLOT_TYPE_OAM; - status->pcie_status.width = 16; - status->pcie_status.max_width = 16; - status->pcie_status.speed = 16; - status->pcie_status.max_speed = 32; - status->pcie_status.bandwidth = 315; + if (!AGA_GPU_SKIP(filter, skip_pcie_status)) { + status->pcie_status.slot_type = AGA_PCIE_SLOT_TYPE_OAM; + status->pcie_status.width = 16; + status->pcie_status.max_width = 16; + status->pcie_status.speed = 16; + status->pcie_status.max_speed = 32; + status->pcie_status.bandwidth = 315; + } // fill VRAM status status->vram_status.type = AGA_VRAM_TYPE_HBM; strcpy(status->vram_status.vendor, "hynix"); @@ -263,10 +268,13 @@ smi_gpu_fill_status (aga_gpu_handle_t gpu_handle, // fill VRAM max bandwidth mock value status->vram_status.max_bandwidth = 3276800; // fill the xgmi error count - status->xgmi_status.error_status = AGA_GPU_XGMI_STATUS_NO_ERROR; - // fill total memory + if (!AGA_GPU_SKIP(filter, skip_xgmi_status)) { + status->xgmi_status.error_status = AGA_GPU_XGMI_STATUS_NO_ERROR; + } // fill kfd pid info - smi_fill_gpu_kfd_pid_status_(gpu_handle, status); + if (!AGA_GPU_SKIP(filter, skip_process_status)) { + smi_fill_gpu_kfd_pid_status_(gpu_handle, status); + } status->partition_id = 0; smi_fill_gpu_enumeration_id_status_(gpu_handle, status); return SDK_RET_OK; @@ -278,7 +286,8 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle, bool is_partitioned, uint32_t partition_id, aga_gpu_handle_t first_partition_handle, - aga_gpu_stats_t *stats) + aga_gpu_stats_t *stats, + const aga_gpu_get_filter_t *filter) { std::random_device rd; // obtain a random number from hardware std::mt19937 gen(rd()); // seed the generator @@ -289,66 +298,76 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle, // fill the current package power stats->package_power = 90 + distr(gen) - distr(gen); // fill the GPU usage - stats->usage.gfx_activity = distr(gen) % 100; + if (!AGA_GPU_SKIP(filter, skip_activity_stats)) { + stats->usage.gfx_activity = distr(gen) % 100; + } // fill VRAM usage - stats->vram_usage.total_vram = 196592; - stats->vram_usage.used_vram = 1273; - stats->vram_usage.free_vram = - stats->vram_usage.total_vram - stats->vram_usage.used_vram; - stats->vram_usage.total_visible_vram = 196592; - stats->vram_usage.used_visible_vram = 1273; - stats->vram_usage.free_visible_vram = - stats->vram_usage.total_visible_vram - - stats->vram_usage.used_visible_vram; - stats->vram_usage.total_gtt = 128716; - stats->vram_usage.used_gtt = 20; - stats->vram_usage.free_gtt = - stats->vram_usage.total_gtt - stats->vram_usage.used_gtt; + if (!AGA_GPU_SKIP(filter, skip_vram_usage_stats)) { + stats->vram_usage.total_vram = 196592; + stats->vram_usage.used_vram = 1273; + stats->vram_usage.free_vram = + stats->vram_usage.total_vram - stats->vram_usage.used_vram; + stats->vram_usage.total_visible_vram = 196592; + stats->vram_usage.used_visible_vram = 1273; + stats->vram_usage.free_visible_vram = + stats->vram_usage.total_visible_vram - + stats->vram_usage.used_visible_vram; + stats->vram_usage.total_gtt = 128716; + stats->vram_usage.used_gtt = 20; + stats->vram_usage.free_gtt = + stats->vram_usage.total_gtt - stats->vram_usage.used_gtt; + } // fill the PCIe stats - ++stats->pcie_stats.replay_count; - ++stats->pcie_stats.tx_bytes; - ++stats->pcie_stats.recovery_count; - ++stats->pcie_stats.replay_rollover_count; - ++stats->pcie_stats.nack_sent_count; - ++stats->pcie_stats.nack_received_count; - ++stats->pcie_stats.rx_bytes; - ++stats->pcie_stats.tx_bytes; - ++stats->pcie_stats.bidir_bandwidth; + if (!AGA_GPU_SKIP(filter, skip_pcie_stats)) { + ++stats->pcie_stats.replay_count; + ++stats->pcie_stats.tx_bytes; + ++stats->pcie_stats.recovery_count; + ++stats->pcie_stats.replay_rollover_count; + ++stats->pcie_stats.nack_sent_count; + ++stats->pcie_stats.nack_received_count; + ++stats->pcie_stats.rx_bytes; + ++stats->pcie_stats.tx_bytes; + ++stats->pcie_stats.bidir_bandwidth; + } // fill the energy consumed stats->energy_consumed = 25293978861568 + distr(gen) - distr(gen); - for (uint16_t i = 0; i < AMDSMI_MAX_NUM_XCC; i++) { - stats->usage.gfx_busy_inst[i] = distr(gen) % 100 ; + if (!AGA_GPU_SKIP(filter, skip_activity_stats)) { + for (uint16_t i = 0; i < AMDSMI_MAX_NUM_XCC; i++) { + stats->usage.gfx_busy_inst[i] = distr(gen) % 100 ; + } } // fill violation stats - stats->violation_stats.current_accumulated_counter = 123456 + distr(gen) - distr(gen); - stats->violation_stats.processor_hot_residency_accumulated = 23456 + distr(gen) - distr(gen); - stats->violation_stats.ppt_residency_accumulated = 34567 + distr(gen) - distr(gen); - stats->violation_stats.socket_thermal_residency_accumulated = 45678 + distr(gen) - distr(gen); - stats->violation_stats.vr_thermal_residency_accumulated = 56789 + distr(gen) - distr(gen); - stats->violation_stats.hbm_thermal_residency_accumulated = 67890 + distr(gen) - distr(gen); - stats->violation_stats.processor_hot_residency_percentage = distr(gen) % 100; - stats->violation_stats.ppt_residency_percentage = distr(gen) % 100; - stats->violation_stats.socket_thermal_residency_percentage = distr(gen) % 100; - stats->violation_stats.vr_thermal_residency_percentage = distr(gen) % 100; - stats->violation_stats.hbm_thermal_residency_percentage = distr(gen) % 100; - - for (uint16_t i = 0; i < AMDSMI_MAX_NUM_XCC; i++) { - stats->violation_stats.gfx_clk_below_host_limit_power_accumulated[i] = - 1234 + distr(gen) - distr(gen); - stats->violation_stats.gfx_clk_below_host_limit_thermal_accumulated[i] = - 2345 + distr(gen) - distr(gen); - stats->violation_stats.gfx_low_utilization_accumulated[i] = - 3456 + distr(gen) - distr(gen); - stats->violation_stats.gfx_clk_below_host_limit_total_accumulated[i] = - 4567 + distr(gen) - distr(gen); - stats->violation_stats.gfx_clk_below_host_limit_power_percentage[i] = - distr(gen) % 100; - stats->violation_stats.gfx_clk_below_host_limit_thermal_percentage[i] = - distr(gen) % 100; - stats->violation_stats.gfx_low_utilization_percentage[i] = - distr(gen) % 100; - stats->violation_stats.gfx_clk_below_host_limit_total_percentage[i] = - distr(gen) % 100; + if (!AGA_GPU_SKIP(filter, skip_violation_stats)) { + stats->violation_stats.current_accumulated_counter = 123456 + distr(gen) - distr(gen); + stats->violation_stats.processor_hot_residency_accumulated = 23456 + distr(gen) - distr(gen); + stats->violation_stats.ppt_residency_accumulated = 34567 + distr(gen) - distr(gen); + stats->violation_stats.socket_thermal_residency_accumulated = 45678 + distr(gen) - distr(gen); + stats->violation_stats.vr_thermal_residency_accumulated = 56789 + distr(gen) - distr(gen); + stats->violation_stats.hbm_thermal_residency_accumulated = 67890 + distr(gen) - distr(gen); + stats->violation_stats.processor_hot_residency_percentage = distr(gen) % 100; + stats->violation_stats.ppt_residency_percentage = distr(gen) % 100; + stats->violation_stats.socket_thermal_residency_percentage = distr(gen) % 100; + stats->violation_stats.vr_thermal_residency_percentage = distr(gen) % 100; + stats->violation_stats.hbm_thermal_residency_percentage = distr(gen) % 100; + + for (uint16_t i = 0; i < AMDSMI_MAX_NUM_XCC; i++) { + stats->violation_stats.gfx_clk_below_host_limit_power_accumulated[i] = + 1234 + distr(gen) - distr(gen); + stats->violation_stats.gfx_clk_below_host_limit_thermal_accumulated[i] = + 2345 + distr(gen) - distr(gen); + stats->violation_stats.gfx_low_utilization_accumulated[i] = + 3456 + distr(gen) - distr(gen); + stats->violation_stats.gfx_clk_below_host_limit_total_accumulated[i] = + 4567 + distr(gen) - distr(gen); + stats->violation_stats.gfx_clk_below_host_limit_power_percentage[i] = + distr(gen) % 100; + stats->violation_stats.gfx_clk_below_host_limit_thermal_percentage[i] = + distr(gen) % 100; + stats->violation_stats.gfx_low_utilization_percentage[i] = + distr(gen) % 100; + stats->violation_stats.gfx_clk_below_host_limit_total_percentage[i] = + distr(gen) % 100; + } } return SDK_RET_OK; } diff --git a/sw/nic/gpuagent/cli/cmd/gpu.go b/sw/nic/gpuagent/cli/cmd/gpu.go index 034f9718..db31f6b1 100644 --- a/sw/nic/gpuagent/cli/cmd/gpu.go +++ b/sw/nic/gpuagent/cli/cmd/gpu.go @@ -63,6 +63,17 @@ var ( memClkFreqHi uint32 printHdr bool severity string + skipClockStatus bool + skipPCIeStatus bool + skipXGMIStatus bool + skipProcessStatus bool + skipUALinkStatus bool + skipVRAMUsageStats bool + skipECCStats bool + skipViolationStats bool + skipPCIeStats bool + skipXGMIStats bool + skipActivityStats bool ) const ( @@ -143,6 +154,7 @@ func init() { gpuShowCmd.AddCommand(gpuAllShowCmd) gpuAllShowCmd.Flags().StringVarP(&gpuID, "id", "i", "", "Specify GPU id") + addGPUSkipFlags(gpuAllShowCmd) gpuShowCmd.AddCommand(gpuStatsShowCmd) gpuStatsShowCmd.Flags().StringVarP(&gpuID, "id", "i", "", "Specify GPU id") @@ -457,6 +469,50 @@ func gpuCPERShowCmdHandler(cmd *cobra.Command, args []string) error { } return nil } + +// addGPUSkipFlags registers the --skip-* flags on a GPU show command +func addGPUSkipFlags(cmd *cobra.Command) { + cmd.Flags().BoolVar(&skipClockStatus, "skip-clock-status", false, + "Skip GPU clock status") + cmd.Flags().BoolVar(&skipPCIeStatus, "skip-pcie-status", false, + "Skip PCIe status") + cmd.Flags().BoolVar(&skipXGMIStatus, "skip-xgmi-status", false, + "Skip XGMI error status") + cmd.Flags().BoolVar(&skipProcessStatus, "skip-process-status", false, + "Skip process list") + cmd.Flags().BoolVar(&skipUALinkStatus, "skip-ualink-status", false, + "Skip UALink state") + cmd.Flags().BoolVar(&skipVRAMUsageStats, "skip-vram-usage-stats", false, + "Skip VRAM usage") + cmd.Flags().BoolVar(&skipECCStats, "skip-ecc-stats", false, + "Skip ECC error counts") + cmd.Flags().BoolVar(&skipViolationStats, "skip-violation-stats", false, + "Skip violation stats") + cmd.Flags().BoolVar(&skipPCIeStats, "skip-pcie-stats", false, + "Skip PCIe stats") + cmd.Flags().BoolVar(&skipXGMIStats, "skip-xgmi-stats", false, + "Skip XGMI counters") + cmd.Flags().BoolVar(&skipActivityStats, "skip-activity-stats", false, + "Skip GPU activity and usage") +} + +// buildGPUGetFilter builds a GPUGetFilter from the --skip-* flag values +func buildGPUGetFilter() *aga.GPUGetFilter { + return &aga.GPUGetFilter{ + SkipClockStatus: skipClockStatus, + SkipPCIeStatus: skipPCIeStatus, + SkipXGMIStatus: skipXGMIStatus, + SkipProcessStatus: skipProcessStatus, + SkipUALinkStatus: skipUALinkStatus, + SkipVRAMUsageStats: skipVRAMUsageStats, + SkipECCStats: skipECCStats, + SkipViolationStats: skipViolationStats, + SkipPCIeStats: skipPCIeStats, + SkipXGMIStats: skipXGMIStats, + SkipActivityStats: skipActivityStats, + } +} + func gpuShowCmdHandler(cmd *cobra.Command, args []string) error { if len(args) > 0 { return fmt.Errorf("Invalid argument") @@ -702,6 +758,7 @@ func gpuAllShowCmdHandler(cmd *cobra.Command, args []string) error { Id: [][]byte{}, } } + req.Filter = buildGPUGetFilter() // connect to GPU agent c, ctxt, cancel, err := utils.CreateNewAGAGRPClient() diff --git a/sw/nic/gpuagent/protos/gpu.proto b/sw/nic/gpuagent/protos/gpu.proto index 86c9cbc3..e932c1f1 100644 --- a/sw/nic/gpuagent/protos/gpu.proto +++ b/sw/nic/gpuagent/protos/gpu.proto @@ -811,10 +811,38 @@ message GPU { GPUStats Stats = 3; } +// filter to control which GPU attributes GPUGet fetches +message GPUGetFilter { + // skip clock status + bool SkipClockStatus = 1; + // skip PCIe status + bool SkipPCIeStatus = 2; + // skip XGMI error status + bool SkipXGMIStatus = 3; + // skip process list + bool SkipProcessStatus = 4; + // skip UALink state + bool SkipUALinkStatus = 5; + // skip VRAM usage + bool SkipVRAMUsageStats = 6; + // skip ECC error counts + bool SkipECCStats = 7; + // skip violation stats + bool SkipViolationStats = 8; + // skip PCIe stats + bool SkipPCIeStats = 9; + // skip XGMI counters + bool SkipXGMIStats = 10; + // skip GPU activity and usage + bool SkipActivityStats = 11; +} + // GPU get request message message GPUGetRequest { // list of GPU uuids - repeated bytes Id = 1; + repeated bytes Id = 1; + // filter of attributes to skip; unset means fetch all + GPUGetFilter Filter = 2; } // response to GPU get request diff --git a/sw/nic/gpuagent/svc/gpu_svc.hpp b/sw/nic/gpuagent/svc/gpu_svc.hpp index e577989a..0074f9ab 100644 --- a/sw/nic/gpuagent/svc/gpu_svc.hpp +++ b/sw/nic/gpuagent/svc/gpu_svc.hpp @@ -40,21 +40,26 @@ aga_svc_gpu_get (const GPUGetRequest *proto_req, GPUGetResponse *proto_rsp) sdk_ret_t ret; aga_obj_key_t key; aga_gpu_info_t info; + aga_gpu_get_filter_t filter = {}; if (proto_req == NULL) { proto_rsp->set_apistatus(types::ApiStatus::API_STATUS_INVALID_ARG); return SDK_RET_INVALID_ARG; } aga_api_trace_verbose("GPU", "Get", proto_req); + // build the get filter from the request + if (proto_req->has_filter()) { + aga_gpu_get_filter_to_spec(proto_req->filter(), &filter); + } if (proto_req->id_size() == 0) { - ret = aga_gpu_read_all(aga_gpu_api_info_to_proto, proto_rsp); + ret = aga_gpu_read_all(aga_gpu_api_info_to_proto, proto_rsp, &filter); proto_rsp->set_apistatus(sdk_ret_to_api_status(ret)); return ret; } for (int i = 0; i < proto_req->id_size(); i ++) { aga_obj_key_proto_to_api_spec(&key, proto_req->id(i)); memset(&info, 0, sizeof(aga_gpu_info_t)); - ret = aga_gpu_read(&key, &info); + ret = aga_gpu_read(&key, &info, &filter); if (unlikely(ret != SDK_RET_OK)) { proto_rsp->set_apistatus(sdk_ret_to_api_status(ret)); break; diff --git a/sw/nic/gpuagent/svc/gpu_to_spec.hpp b/sw/nic/gpuagent/svc/gpu_to_spec.hpp index 93f7ea7a..7845c2f0 100644 --- a/sw/nic/gpuagent/svc/gpu_to_spec.hpp +++ b/sw/nic/gpuagent/svc/gpu_to_spec.hpp @@ -66,6 +66,24 @@ aga_gpu_admin_state_to_spec (amdgpu::GPUAdminState admin_state) return AGA_GPU_ADMIN_STATE_NONE; } +// convert gpu get filter from proto to spec +static inline void +aga_gpu_get_filter_to_spec (const amdgpu::GPUGetFilter& proto_filter, + aga_gpu_get_filter_t *filter) +{ + filter->skip_clock_status = proto_filter.skipclockstatus(); + filter->skip_pcie_status = proto_filter.skippciestatus(); + filter->skip_xgmi_status = proto_filter.skipxgmistatus(); + filter->skip_process_status = proto_filter.skipprocessstatus(); + filter->skip_ualink_status = proto_filter.skipualinkstatus(); + filter->skip_vram_usage_stats = proto_filter.skipvramusagestats(); + filter->skip_ecc_stats = proto_filter.skipeccstats(); + filter->skip_violation_stats = proto_filter.skipviolationstats(); + filter->skip_pcie_stats = proto_filter.skippciestats(); + filter->skip_xgmi_stats = proto_filter.skipxgmistats(); + filter->skip_activity_stats = proto_filter.skipactivitystats(); +} + static inline aga_gpu_compute_partition_type_t aga_gpu_compute_partition_type_to_spec (amdgpu::GPUComputePartitionType type) {