ONNX Runtime: Fallback to CPU if GPU acceleration fails

SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp

@@ -146,6 +146,12 @@ GlobalSettings::GlobalSettings()
         LockMode::UNLOCK_WHILE_RUNNING,
         false
     )
+    , USE_GPU_FOR_ML_INFERENCE(
+        "<b>Use GPU for Machine learning inference:</b><br>"
+        "Use the GPU by default for machine learning. Will fall-back to CPU if using the GPU fails.",
+        LockMode::UNLOCK_WHILE_RUNNING,
+        true
+    )
     , WINDOW_SIZE(
         CONSTRUCT_TOKEN,
         "Window Size/Position:",
@@ -261,6 +267,7 @@ GlobalSettings::GlobalSettings()
     PA_ADD_OPTION(TEMP_FOLDER);
     PA_ADD_OPTION(THEME);
     PA_ADD_OPTION(USE_PADDLE_OCR);
+    PA_ADD_OPTION(USE_GPU_FOR_ML_INFERENCE);
     PA_ADD_OPTION(WINDOW_SIZE);
     PA_ADD_OPTION(LOG_WINDOW_SIZE);
     PA_ADD_OPTION(LOG_WINDOW_STARTUP);

SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.h

@@ -124,6 +124,7 @@ class GlobalSettings : public BatchOption, private ConfigOption::Listener, priva
 
     Pimpl<ThemeSelectorOption> THEME;
     BooleanCheckBoxOption USE_PADDLE_OCR;
+    BooleanCheckBoxOption USE_GPU_FOR_ML_INFERENCE;
     Pimpl<ResolutionOption> WINDOW_SIZE;
     Pimpl<ResolutionOption> LOG_WINDOW_SIZE;
     BooleanCheckBoxOption LOG_WINDOW_STARTUP;

SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp

@@ -28,8 +28,7 @@ namespace ML{
 
 SAMEmbedderSession::SAMEmbedderSession(const std::string& model_path, bool use_gpu)
     : m_env{create_ORT_env()}
-    , m_session_options{create_session_options(ML_MODEL_CACHE_PATH() + "SAMEmbedder/", use_gpu)}
-    , session{create_session(m_env, m_session_options, model_path, ML_MODEL_CACHE_PATH() + "SAMEmbedder/")}
+    , session{create_session(m_env, model_path, ML_MODEL_CACHE_PATH() + "SAMEmbedder/", use_gpu)}
     , memory_info{Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU)}
     , input_names{session.GetInputNames()}
     , output_names{session.GetOutputNames()}
@@ -69,8 +68,7 @@ void SAMEmbedderSession::run(cv::Mat& input_image, std::vector<float>& model_out
 
 SAMSession::SAMSession(const std::string& model_path, bool use_gpu)
     : m_env{create_ORT_env()}
-    , m_session_options{create_session_options(ML_MODEL_CACHE_PATH() + "SAM/", use_gpu)}
-    , session{create_session(m_env, m_session_options, model_path, ML_MODEL_CACHE_PATH() + "SAM/")}
+    , session{create_session(m_env, model_path, ML_MODEL_CACHE_PATH() + "SAM/", use_gpu)}
     , memory_info{Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU)}
     , input_names{session.GetInputNames()}
     , output_names{session.GetOutputNames()}

SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.h

@@ -39,7 +39,6 @@ class SAMEmbedderSession{
 
 private:
     Ort::Env m_env;
-    Ort::SessionOptions m_session_options;
     Ort::Session session;
     Ort::MemoryInfo memory_info;
     Ort::RunOptions run_options;
@@ -74,7 +73,6 @@ class SAMSession{
         std::vector<bool>& output_boolean_mask);
 private:
     Ort::Env m_env;
-    Ort::SessionOptions m_session_options;
     Ort::Session session;
     Ort::MemoryInfo memory_info;
     Ort::RunOptions run_options;

SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp

@@ -10,6 +10,7 @@
 #include <numeric>
 #include <limits>
 #include "CommonFramework/Globals.h"
+#include "CommonFramework/GlobalSettingsPanel.h"
 #include "Common/Cpp/Exceptions.h"
 #include "ML/Models/ML_ONNXRuntimeHelpers.h"
 #include "ML_PaddleOCRPipeline.h"
@@ -53,7 +54,7 @@ PaddleOCRPipeline::PaddleOCRPipeline(Language language)
 PaddleOCRPipeline::PaddleOCRPipeline(Language language, std::string rec_path, std::string dict_path)
     : m_env{create_ORT_env()}
     // , det_session(env, std::wstring(det_path.begin(), det_path.end()).c_str(), Ort::SessionOptions{})
-    , m_rec_session(create_session(m_env, Ort::SessionOptions{}, rec_path, ML_MODEL_CACHE_PATH() + "PaddleOCRPipeline/"))
+    , m_rec_session(create_session(m_env, rec_path, ML_MODEL_CACHE_PATH() + "PaddleOCRPipeline/", GlobalSettings::instance().USE_GPU_FOR_ML_INFERENCE))
     // , memory_info(Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault)) 
     , m_language(language)
     , m_input_name(m_rec_session.GetInputNameAllocated(0, Ort::AllocatorWithDefaultOptions{}).get())

SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp

@@ -13,6 +13,7 @@
 #include "Common/Cpp/PrettyPrint.h"
 #include "Common/Cpp/Filesystem.h"
 #include "Common/Cpp/Concurrency/SpinLock.h"
+#include "CommonFramework/GlobalSettingsPanel.h"
 #include "CommonFramework/ImageTypes/ImageViewRGB32.h"
 #include "CommonFramework/VideoPipeline/VideoOverlay.h"
 #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
@@ -44,7 +45,7 @@ YOLOv5Detector::~YOLOv5Detector() = default;
 
 YOLOv5Detector::YOLOv5Detector(const std::string& model_path)
     : m_model_path(to_resource_filepath(model_path))
-    , m_use_gpu(true)
+    , m_use_gpu(GlobalSettings::instance().USE_GPU_FOR_ML_INFERENCE)
 {
     if (!model_path.ends_with(".onnx")){
         throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, 

SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp

@@ -86,7 +86,7 @@ if (use_gpu){
             std::cout << "Using ROCm execution provider for GPU acceleration" << std::endl;
             rocm_available = true;
         }catch (const Ort::Exception& e){
-            std::cout << "ROCm execution provider not available, falling back to CPU: " << e.what() << std::endl;
+            std::cout << "ROCm execution provider not available: " << e.what() << std::endl;
         }
     }
 
@@ -181,24 +181,58 @@ void write_cache_flag_file(const std::string& model_cache_path, const std::strin
 }
 
 
-Ort::Session create_session(const Ort::Env& env, const Ort::SessionOptions& so,
-    const std::string& model_path, const std::string& model_cache_path)
-{
+Ort::Session create_session(
+    const Ort::Env& env, 
+    const std::string& model_path, 
+    const std::string& model_cache_path,
+    bool try_gpu
+){
     bool write_flag_file = true;
     std::string file_hash;
     std::tie(write_flag_file, file_hash) = clean_up_old_model_cache(model_cache_path, model_path);
 
+    auto onnx_path = str_to_onnx_str(model_path);
+
     auto& logger = global_logger_tagged();
     logger.log("Creating Ort::session from model " + model_path);
-    try{
-        Ort::Session session{env, str_to_onnx_str(model_path).c_str(), so};
-        logger.log("Ort::Session created");
-        // when Ort::Ssssion is created, if possible, it will create a model cache
-        if (write_flag_file){
-            write_cache_flag_file(model_cache_path, file_hash);
+
+    // Attempt 1. using GPU.
+    if (try_gpu){
+        try{
+            logger.log("Attempting to create Ort::Session with GPU acceleration...");
+            Ort::SessionOptions gpu_options = create_session_options(model_cache_path, true);
+            Ort::Session session{env, onnx_path.c_str(), gpu_options};
+            logger.log("Ort::Session created");
+            // when Ort::Ssssion is created, if possible, it will create a model cache
+            if (write_flag_file){
+                write_cache_flag_file(model_cache_path, file_hash);
+            }
+            return session;
+        }catch (const Ort::Exception& e) {
+            logger.log("GPU Session creation failed: " + std::string(e.what()));
+            logger.log("Falling back cleanly to CPU execution...");
+            // Do not throw yet. Fall through to the CPU execution block below
+        } catch (...) {
+            logger.log("Unknown GPU initialization failure. Falling back cleanly to CPU...");
         }
+    }
+
+    // Attempt 2. CPU fallback
+    try {
+        logger.log("Creating dedicated CPU-only session...");
+
+        Ort::SessionOptions cpu_options = create_session_options(model_cache_path, false);;
+
+        Ort::Session session{env, onnx_path.c_str(), cpu_options};
+        logger.log("Ort::Session created");
         return session;
-    }catch (...){
+    }
+    catch (const Ort::Exception& e) {
+        logger.log("CRITICAL: CPU fallback failed completely: " + std::string(e.what()));
+        throw MLModelSessionCreationError(&logger, model_path);
+    }
+    catch (...) {
+        logger.log("CRITICAL: Unknown failure during CPU session fallback.");
         throw MLModelSessionCreationError(&logger, model_path);
     }
 }

SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h

@@ -30,8 +30,12 @@ Ort::SessionOptions create_session_options(const std::string& model_cache_path,
 // model_cache_path: the path to store model caches. This path must be the same path
 //   used in `create_session_options()` to construct the passed-in session options so.
 // NOTE: it may throw `MLModelSessionCreationError` if failed to create session.
-Ort::Session create_session(const Ort::Env& env, const Ort::SessionOptions& so,
-    const std::string& model_path, const std::string& model_cache_path);
+Ort::Session create_session(
+    const Ort::Env& env, 
+    const std::string& model_path, 
+    const std::string& model_cache_path,
+    bool try_gpu
+);
 
 // Handy function to create an ONNX Runtime tensor view class from a vector-like `buffer` object holding
 // the tensor data and an array-like `shape` object that represents the dimension of the tensor.

SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp

@@ -134,8 +134,7 @@ std::tuple<int, int, double, double> resize_image_with_border(
 
 YOLOv5Session::YOLOv5Session(const std::string& model_path, bool use_gpu)
 : m_env{create_ORT_env()}
-, m_session_options(create_session_options(ML_MODEL_CACHE_PATH() + "YOLOv5", use_gpu))
-, m_session{create_session(m_env, m_session_options, model_path, ML_MODEL_CACHE_PATH() + "YOLOv5")}
+, m_session{create_session(m_env, model_path, ML_MODEL_CACHE_PATH() + "YOLOv5", use_gpu)}
 , m_memory_info{Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU)}
 , m_input_names{m_session.GetInputNames()}
 , m_output_names{m_session.GetOutputNames()}

SerialPrograms/Source/ML/Models/ML_YOLOv5Model.h

@@ -44,7 +44,6 @@ class YOLOv5Session{
     std::vector<std::string> m_label_names;
 
     Ort::Env m_env;
-    Ort::SessionOptions m_session_options;
     Ort::Session m_session;
     Ort::MemoryInfo m_memory_info;
     Ort::RunOptions m_run_options;

SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp

@@ -15,6 +15,7 @@
 #include "Common/Cpp/Json/JsonObject.h"
 #include "Common/Cpp/Json/JsonValue.h"
 #include "Common/Cpp/Json/JsonTools.h"
+#include "CommonFramework/GlobalSettingsPanel.h"
 #include "Pokemon/Pokemon_Strings.h"
 #include "ML/DataLabeling/ML_SegmentAnythingModel.h"
 #include "ML/DataLabeling/ML_AnnotationIO.h"
@@ -56,7 +57,7 @@ LabelImages::LabelImages(const LabelImages_Descriptor& descriptor)
     : PanelInstance(descriptor)
     , m_display_session(m_display_option)
     , m_options(LockMode::UNLOCK_WHILE_RUNNING)
-    , m_use_gpu_for_sam_anno(true)
+    , m_use_gpu_for_sam_anno(GlobalSettings::instance().USE_GPU_FOR_ML_INFERENCE)
     , X("<b>X Coordinate:</b>", LockMode::UNLOCK_WHILE_RUNNING, 0.3, 0.0, 1.0)
     , Y("<b>Y Coordinate:</b>", LockMode::UNLOCK_WHILE_RUNNING, 0.3, 0.0, 1.0)
     , WIDTH("<b>Width:</b>", LockMode::UNLOCK_WHILE_RUNNING, 0.4, 0.0, 1.0)
@@ -67,7 +68,6 @@ LabelImages::LabelImages(const LabelImages_Descriptor& descriptor)
     , CUSTOM_LABEL_DATABASE(create_string_select_database({"mc"})) // mc for "main character"
     , CUSTOM_SET_LABEL(CUSTOM_LABEL_DATABASE, LockMode::UNLOCK_WHILE_RUNNING, 0)
     , MANUAL_LABEL(false, LockMode::UNLOCK_WHILE_RUNNING, "", "Custom Label", true)
-    , USE_GPU_FOR_EMBEDDER_SESSION("<b>Enable GPU for Embedder session:</b>", LockMode::LOCK_WHILE_RUNNING, true) 
     , SELECTED_ANNO_COLOR(
         "<b>Color of selected annotation:",
         {
@@ -121,7 +121,6 @@ LabelImages::LabelImages(const LabelImages_Descriptor& descriptor)
     ADD_OPTION(FORM_LABEL);
     ADD_OPTION(CUSTOM_SET_LABEL);
     ADD_OPTION(MANUAL_LABEL);
-    ADD_OPTION(USE_GPU_FOR_EMBEDDER_SESSION);
     ADD_OPTION(SELECTED_ANNO_COLOR);
     ADD_OPTION(UNSELECTED_ANNO_COLOR);
 
@@ -136,7 +135,7 @@ LabelImages::LabelImages(const LabelImages_Descriptor& descriptor)
 
 
 
-    init_sam_session(true);
+    init_sam_session(GlobalSettings::instance().USE_GPU_FOR_ML_INFERENCE);
 
     m_overlay_manager = new LabelImages_OverlayManager(*this);
 }
@@ -511,7 +510,7 @@ void LabelImages::remove_segmentation_exclusion_point(double x, double y){
 void LabelImages::compute_embeddings_for_folder(const std::string& image_folder_path){
     std::string embedding_model_path = RESOURCE_PATH() + "ML/sam_embedder_cpu.onnx";
     std::cout << "Use SAM Embedding model " << embedding_model_path << std::endl;
-    ML::compute_embeddings_for_folder(embedding_model_path, image_folder_path, USE_GPU_FOR_EMBEDDER_SESSION);
+    ML::compute_embeddings_for_folder(embedding_model_path, image_folder_path, GlobalSettings::instance().USE_GPU_FOR_ML_INFERENCE);
 }
 
 void LabelImages::delete_selected_annotation(){

SerialPrograms/Source/ML/Programs/ML_LabelImages.h

@@ -178,8 +178,6 @@ class LabelImages : public PanelInstance, public ConfigOption::Listener {
     // source 3: editable text input
     StringCell MANUAL_LABEL;
 
-    BooleanCheckBoxOption USE_GPU_FOR_EMBEDDER_SESSION;
-
 
     EnumDropdownOption<ColorChoice> SELECTED_ANNO_COLOR;
     EnumDropdownOption<ColorChoice> UNSELECTED_ANNO_COLOR;

SerialPrograms/Source/ML/Programs/ML_LabelImagesWidget.cpp

@@ -129,12 +129,12 @@ LabelImages_Widget::LabelImages_Widget(
     annotation_row->addWidget(load_custom_set_button, 2);
     annotation_row->addWidget(new QLabel(scroll_inner), 10); // an empty label to push other UIs to the left
 
-    // add GPU checkbox row
-    QHBoxLayout* use_gpu_row = new QHBoxLayout();
-    scroll_layout->addLayout(use_gpu_row);
+    // // add GPU checkbox row
+    // QHBoxLayout* use_gpu_row = new QHBoxLayout();
+    // scroll_layout->addLayout(use_gpu_row);
 
-    ConfigWidget* gpu_checkbox_widget = ConfigWidget::make_from_option(program.USE_GPU_FOR_EMBEDDER_SESSION, scroll_inner);
-    use_gpu_row->addWidget(&gpu_checkbox_widget->widget(), 2);    
+    // ConfigWidget* gpu_checkbox_widget = ConfigWidget::make_from_option(program.USE_GPU_FOR_EMBEDDER_SESSION, scroll_inner);
+    // use_gpu_row->addWidget(&gpu_checkbox_widget->widget(), 2);    
 
     // add Color selection dropdown
     QHBoxLayout* selected_color_choice_row = new QHBoxLayout();