Resource Downloading - Settings

SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp

@@ -268,6 +268,8 @@ GlobalSettings::GlobalSettings()
     PA_ADD_OPTION(THEME);
     PA_ADD_OPTION(USE_PADDLE_OCR);
     PA_ADD_OPTION(USE_GPU_FOR_ML_INFERENCE);
+    PA_ADD_OPTION(RESOURCE_DOWNLOAD_TABLE);
+    PA_ADD_OPTION(DOWNLOAD_ERROR);
     PA_ADD_OPTION(WINDOW_SIZE);
     PA_ADD_OPTION(LOG_WINDOW_SIZE);
     PA_ADD_OPTION(LOG_WINDOW_STARTUP);
@@ -451,9 +453,9 @@ void GlobalSettings::on_press(){
     QDesktopServices::openUrl(QUrl::fromLocalFile(QString::fromStdString(RUNTIME_BASE_PATH())));
 }
 
-
-
-
+void GlobalSettings::connect_row_with_download(const std::string& resource_slug, std::shared_ptr<ResourceDownload>& download_ptr){
+    RESOURCE_DOWNLOAD_TABLE.connect_row_with_download(resource_slug, download_ptr);
+}
 
 
 GlobalSettings_Descriptor::GlobalSettings_Descriptor()

SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.h

@@ -17,6 +17,8 @@
 #include "Common/Cpp/Options/ButtonOption.h"
 #include "CommonFramework/Panels/SettingsPanel.h"
 #include "CommonFramework/Panels/PanelTools.h"
+#include "CommonFramework/ResourceDownload/SettingsResourceDownloadOptions.h"
+#include "CommonFramework/ResourceDownload/SettingsResourceDownloadTable.h"
 
 //#include <iostream>
 //using std::cout;
@@ -37,7 +39,7 @@ class PerformanceOptions;
 class AudioPipelineOptions;
 class VideoPipelineOptions;
 class ErrorReportOption;
-
+class ResourceDownload;
 
 
 
@@ -111,6 +113,8 @@ class GlobalSettings : public BatchOption, private ConfigOption::Listener, priva
     virtual void load_json(const JsonValue& json) override;
     virtual JsonValue to_json() const override;
 
+    void connect_row_with_download(const std::string& resource_slug, std::shared_ptr<ResourceDownload>& download_ptr);
+
 private:
     virtual void on_config_value_changed(void* object) override;
     virtual void on_press() override;
@@ -125,6 +129,8 @@ class GlobalSettings : public BatchOption, private ConfigOption::Listener, priva
     Pimpl<ThemeSelectorOption> THEME;
     BooleanCheckBoxOption USE_PADDLE_OCR;
     BooleanCheckBoxOption USE_GPU_FOR_ML_INFERENCE;
+    SettingsResourceDownloadTable RESOURCE_DOWNLOAD_TABLE;
+    SettingsDownloadError DOWNLOAD_ERROR;
     Pimpl<ResolutionOption> WINDOW_SIZE;
     Pimpl<ResolutionOption> LOG_WINDOW_SIZE;
     BooleanCheckBoxOption LOG_WINDOW_STARTUP;

SerialPrograms/Source/CommonFramework/ResourceDownload/DownloadThread.cpp

@@ -57,6 +57,13 @@ void DownloadThread::start_download_thread(){
         });
 
         try {
+
+            // Logger& logger = global_logger_tagged();
+            // throw_and_log<OperationFailedException>(logger, ErrorReport::NO_ERROR_REPORT, 
+            //         "Test");
+            // throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Test."); 
+
+
             // std::this_thread::sleep_for(std::chrono::seconds(7));
 
             run_download(m_remote_metadata);
@@ -72,12 +79,12 @@ void DownloadThread::start_download_thread(){
             success = false;
             m_hooks.report_failed();
         }catch (const std::exception& e) {
-            std::cout << "Standard exception: " << e.what() << std::endl;
+            std::string error_msg = "DownloadThread::start_download_thread: " + std::string(e.what());
             success = false;
-            m_hooks.report_exception_caught("DownloadThread::start_download_thread");
+            m_hooks.report_exception_caught(error_msg.c_str());
         } catch(...){
             success = false;
-            m_hooks.report_exception_caught("DownloadThread::start_download_thread");
+            m_hooks.report_exception_caught("DownloadThread::start_download_thread: Unknown exception. Report this as an error.");
         }
 
     }

SerialPrograms/Source/CommonFramework/ResourceDownload/GlobalResourceDownloadManager.cpp

@@ -0,0 +1,164 @@
+/*  Resource Download Manager
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+
+#include "Common/Cpp/Exceptions.h"
+#include "CommonFramework/GlobalSettingsPanel.h"
+#include "CommonFramework/ResourceDownload/ResourceDownloadHelpers.h"
+#include "CommonFramework/ResourceDownload/ResourceDownload.h"
+#include "GlobalResourceDownloadManager.h"
+
+#include <iostream>
+using std::cout;
+using std::endl;
+
+namespace PokemonAutomation{
+
+
+GlobalResourceDownloadManager& GlobalResourceDownloadManager::instance(){
+    static GlobalResourceDownloadManager manager;
+    return manager;
+}
+
+
+std::shared_ptr<ResourceDownload> GlobalResourceDownloadManager::add_to_download_list(const std::string& resource_slug){
+    std::lock_guard<Mutex> lg(m_queue_lock);
+    auto iter = std::ranges::find_if(m_download_queue, [&](const auto& download_ptr){
+        return download_ptr->get_name() == resource_slug;
+    });
+
+
+    if (iter == m_download_queue.end()){ // given Resource not already within the DownloadQueue
+        DownloadedResourceMetadata resource_metadata = get_remote_resource_metadata_from_resource_slug(resource_slug);
+        auto download_ptr = m_download_queue.emplace_back(std::make_shared<ResourceDownload>(*this, std::move(resource_metadata), m_queue_lock, m_cv));
+
+        download_ptr->add_listener(*this);
+        GlobalSettings::instance().connect_row_with_download(resource_slug, download_ptr);
+        download_ptr->start_download();
+
+        return download_ptr;
+    }
+
+    // Return the existing shared_ptr found in the queue
+    return *iter;
+
+
+}
+
+void GlobalResourceDownloadManager::remove_from_download_list(const std::string& resource_slug){
+    // We need to delete the target ResourceDownloads outside of the lock
+    // (the reason isn't clear to me. I suspect its because GlobalResourceDownloadManager is one of the
+    // listeners within ResourceDownload's ListenerSet, which causes an issue during its destructor)
+    // This temporary vector holds ownership of the target ResourceDownloads
+    // while modifying m_download_queue occurs within the lock.
+    std::vector<std::shared_ptr<ResourceDownload>> to_destroy;
+
+    {
+        std::lock_guard<Mutex> lg(m_queue_lock);
+
+        // std::erase_if modifies the vector in-place
+        std::erase_if(m_download_queue, [&](auto& download_ptr) {
+            if (!download_ptr) return false;
+
+            if (download_ptr->get_name() == resource_slug) {
+                // Move the download_ptr out of the m_download_queue and into our temporary vector
+                // This prevents immediate destruction of download_ptr, while
+                // it's still removed from m_download_queue.
+                to_destroy.push_back(std::move(download_ptr));
+                return true;
+            }
+            return false;
+        });
+
+        m_cv.notify_all();
+    }
+
+    // The 'to_destroy' vector goes out of scope here.
+    // The ResourceDownload destructors run safely out of the lock.
+}
+
+bool GlobalResourceDownloadManager::is_download_ready_to_start(const std::string& resource_slug){
+    // ASSUMES: the calling thread holds the m_lock. therefore, this function doesn't lock the mutex when accessing download_queue.
+    // std::lock_guard<Mutex> lg(m_lock); 
+
+    uint16_t MAX_CONCURRENT_DOWNLOADS = 10;
+
+    auto iter = std::ranges::find_if(m_download_queue, [&](const auto& download_ptr){
+        return download_ptr->get_name() == resource_slug;
+    });
+
+    if (iter == m_download_queue.end()){
+        throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "is_download_ready_to_start: resource_slug not found within download_queue.");
+    }
+
+    uint16_t download_position = (uint16_t)std::distance(m_download_queue.begin(), iter);
+
+    // cout << "download_position: " << std::to_string(download_position) << endl;
+
+    return download_position < MAX_CONCURRENT_DOWNLOADS;
+}
+
+
+void GlobalResourceDownloadManager::cancel_downloads(){
+    std::vector<std::shared_ptr<ResourceDownload>> downloads_to_cancel;
+
+    // Lock briefly to safely snapshot the queue
+    {
+        std::lock_guard<Mutex> lg(m_queue_lock);
+        downloads_to_cancel = m_download_queue; 
+    }
+
+    // We must call cancel_download() outside of the lock.
+    // The reason is that GlobalResourceDownloadManager is a listener to ResourceDownload. 
+    // ResourceDownload::cancel_download() will trigger GlobalResourceDownloadManager::on_download_finished,
+    // which then triggers GlobalResourceDownloadManager::remove_from_download_list(), which is 
+    // done under a lock.
+    // This prevents a deadlock.
+    for (auto& download : downloads_to_cancel) {
+        if (download) {
+            download->cancel_download(); 
+        }
+    }
+
+}
+
+
+//////////////////////////
+// Listener
+//////////////////////////
+void GlobalResourceDownloadManager::add_download_listener(Listener& listener){
+    m_download_listeners.add(listener);
+}
+void GlobalResourceDownloadManager::remove_download_listener(Listener& listener){
+    m_download_listeners.remove(listener);
+}
+
+void GlobalResourceDownloadManager::report_all_downloads_finished(){
+    m_download_listeners.run_method(&Listener::on_all_downloads_finished);
+}
+void GlobalResourceDownloadManager::report_download_failed(const std::string& resource_slug){
+    m_download_listeners.run_method(&Listener::on_download_failed, resource_slug);
+}
+void GlobalResourceDownloadManager::report_unexpected_exception_caught(const std::string& error_msg){
+    m_download_listeners.run_method(&Listener::on_exception_caught, error_msg);
+}
+
+/////////////////////
+// for ResourceDownload::Listener
+/////////////////////
+
+void GlobalResourceDownloadManager::on_download_finished(bool success, const std::string& resource_slug){
+    remove_from_download_list(resource_slug);
+    // check_if_all_downloads_done();
+}
+void GlobalResourceDownloadManager::on_download_failed(const std::string& resource_slug){
+    report_download_failed(resource_slug);
+}
+void GlobalResourceDownloadManager::on_exception_caught(const std::string& error_msg){
+    report_unexpected_exception_caught(error_msg);
+}
+
+}

SerialPrograms/Source/CommonFramework/ResourceDownload/GlobalResourceDownloadManager.h

@@ -0,0 +1,89 @@
+/*  Resource Download Manager
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#ifndef PokemonAutomation_ResourceDownloadManager_H
+#define PokemonAutomation_ResourceDownloadManager_H
+
+#include <vector>
+#include "Common/Cpp/Concurrency/SpinLock.h"
+#include "Common/Cpp/Concurrency/Mutex.h"
+#include "Common/Cpp/Concurrency/ConditionVariable.h"
+#include "Common/Cpp/ListenerSet.h"
+#include "ResourceDownload.h"
+
+namespace PokemonAutomation{
+
+// class ResourceDownload;
+
+
+
+class GlobalResourceDownloadManager : public ResourceDownload::Listener{
+public:
+    struct Listener{
+        virtual void on_all_downloads_finished(){}
+        virtual void on_download_failed(const std::string& resource_slug){}
+        virtual void on_exception_caught(const std::string& error_msg){}
+    };
+
+    void add_download_listener(Listener& listener);
+    void remove_download_listener(Listener& listener);
+
+    void report_all_downloads_finished();
+    void report_download_failed(const std::string& resource_slug);
+    void report_unexpected_exception_caught(const std::string& error_msg);
+
+public:
+    static GlobalResourceDownloadManager& instance();
+
+public:  // ResourceDownload::Listener
+    virtual void on_download_finished(bool success, const std::string& resource_slug) override;
+    virtual void on_download_failed(const std::string& resource_slug) override;
+    virtual void on_exception_caught(const std::string& error_msg) override;
+
+
+public:
+    // check if the download queue already has the resource
+    // if not, add the download to the back of the download queue
+    // and start downloading it
+    // Also, add this manager as a listener to the download
+    std::shared_ptr<ResourceDownload> add_to_download_list(const std::string& resource_slug);
+
+    void remove_from_download_list(const std::string& resource_slug);
+
+    // return true if given resource_slug's position in m_download_queue is less than MAX_CONCURRENT_DOWNLOADS
+    // ASSUMES: the calling thread holds the m_lock. therefore, this function doesn't lock the mutex when accessing m_download_queue.
+    bool is_download_ready_to_start(const std::string& resource_slug);
+
+
+    void cancel_downloads();
+
+
+
+private:
+
+    // queue of downloads
+    // ResourceDownload must be a pointer for several reasons:
+    // - it contains a Mutex/CV, and so can't be moved unless it's a pointer. 
+    // Objects can't be added to a vector if they can't be moved/copied.
+    // - Other objects hold references to ResourceDownload. We will need to
+    // remove items in the middle of the vector. This forces the vector to shift 
+    // all subsequent elements forward in memory to fill the gap. This shifting 
+    // breaks (invalidates) any references or iterators pointing to elements 
+    // at or after the deletion point.
+    std::vector<std::shared_ptr<ResourceDownload>> m_download_queue;
+
+    Mutex m_queue_lock;
+    ConditionVariable m_cv;
+
+    ListenerSet<Listener> m_download_listeners;
+
+
+};
+
+
+
+}
+#endif