Working ONNX Test Branch

Common/Cpp/Filesystem/Filesystem.cpp

@@ -42,6 +42,9 @@ bool create_directories(const Path& path){
     return std::filesystem::create_directories(path.stdpath());
 }
 
+bool remove(const Path& path){
+    return std::filesystem::remove(path);
+}
 std::uintmax_t remove_all(const Path& path){
     return std::filesystem::remove_all(path.stdpath());
 }

Common/Cpp/Filesystem/Filesystem.h

@@ -40,6 +40,7 @@ bool create_directories(const Path& path);
 
 //  Delete the contents of the path (if it is a directory) and the contents of all its subdirectories, recursively.
 //  Then delete the file/directory of path itself. Symlinks are not followed (symlink is removed, not its target).
+bool remove(const Path& path);
 std::uintmax_t remove_all(const Path& path);
 
 //  Copy a file.

SerialPrograms/Source/CommonFramework/ResourceDownload/DownloadThread.cpp

@@ -23,9 +23,6 @@ using std::endl;
 
 namespace PokemonAutomation{
 
-namespace fs = std::filesystem;
-
-
 
 
 DownloadThread::~DownloadThread(){
@@ -104,7 +101,7 @@ void DownloadThread::run_download(DownloadedResourceMetadata resource_metadata){
     try{
 
         // delete directory and the old resource
-        fs::remove_all(Filesystem::Path(resource_directory));
+        Filesystem::remove_all(resource_directory);
 
         // download
         std::string zip_path = resource_directory + "/temp.zip";
@@ -122,7 +119,7 @@ void DownloadThread::run_download(DownloadedResourceMetadata resource_metadata){
         // hash
         std::string hash = 
             hash_file(
-                *this,
+                this,
                 zip_path,
                 [this](uint64_t bytes_done, uint64_t total_bytes){
                     m_hooks.report_hash_progress(bytes_done, total_bytes);
@@ -149,23 +146,23 @@ void DownloadThread::run_download(DownloadedResourceMetadata resource_metadata){
         );
 
         // delete old zip file
-        fs::remove(Filesystem::Path(zip_path));
+        Filesystem::remove(zip_path);
 
         throw_if_cancelled();
 
     }catch(OperationCancelledException&){
         // delete directory and the resource
-        fs::remove_all(Filesystem::Path(resource_directory));
+        Filesystem::remove_all(resource_directory);
 
         throw;
     }catch(OperationFailedException&){
         // delete directory and the resource
-        fs::remove_all(Filesystem::Path(resource_directory));
+        Filesystem::remove_all(resource_directory);
 
         throw;
     }catch(...){
         // delete directory and the resource
-        fs::remove_all(Filesystem::Path(resource_directory));
+        Filesystem::remove_all(resource_directory);
 
         throw;
     }

SerialPrograms/Source/CommonFramework/Tools/FileHash.cpp

@@ -5,9 +5,11 @@
  */
 
 #include "Common/Cpp/Exceptions.h"
-#include <QFile>
-#include <QCryptographicHash>
-#include <QDebug>
+#include "Common/Cpp/ScopeExit.h"
+#include "Common/Cpp/Containers/AlignedMalloc.h"
+#include "Common/Cpp/Filesystem/FileIO.h"
+#include "Common/Cpp/Filesystem/Filesystem.h"
+#include "Common/Cpp/Cryptography/SHA256.h"
 #include "FileHash.h"
 
 //#include <iostream>
@@ -19,35 +21,45 @@ namespace PokemonAutomation{
 
 
 std::string hash_file(
-    CancellableScope& scope,
+    CancellableScope* scope,
     const std::string& file_path,
     std::function<void(uint64_t bytes_done, uint64_t total_bytes)> hash_progress
 ){
-    QFile file(QString::fromStdString(file_path));
-    if (!file.open(QIODevice::ReadOnly)){
-        throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "hash_file: Could not open file.");
+    Filesystem::Path path(file_path);
+    FileIO file(path, FileMode::READ | FileMode::BINARY);
+    if (!file.is_open()){
+        throw InternalProgramError(
+            nullptr,
+            PA_CURRENT_FUNCTION,
+            "hash_file: Could not open file."
+        );
     }
 
-    QCryptographicHash hash(QCryptographicHash::Sha256);
-    qint64 file_size = file.size();
-    qint64 total_bytes_read = 0;
-
-    QByteArray buffer(1024 * 1024, 0); // Pre-allocate 1MB once
-    while (!file.atEnd()){
-        scope.throw_if_cancelled();
-
-        qint64 num_bytes_in_chunk = file.read(buffer.data(), buffer.size());
-        if (num_bytes_in_chunk == -1){
-            throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "hash_file: Read error:" + file.errorString().toStdString());
+    SHA256 hash;
+    uint64_t file_size = Filesystem::file_size(path);
+    uint64_t total_bytes_read = 0;
+
+    constexpr size_t BUFFER_SIZE = 1024 * 1024;
+    void* buffer = aligned_malloc(BUFFER_SIZE, 4096);   // Pre-allocate 1MB once
+    ScopeExit sg([&]{ aligned_free(buffer); });
+
+    size_t bytes_read;
+    do{
+        if (scope != nullptr){
+            scope->throw_if_cancelled();
         }
 
-        hash.addData(QByteArrayView(buffer.data(), num_bytes_in_chunk));
-        total_bytes_read += num_bytes_in_chunk;
+        bytes_read = file.read(buffer, BUFFER_SIZE);
+        hash.push(buffer, bytes_read);
+        total_bytes_read += bytes_read;
 
-        hash_progress(total_bytes_read, file_size);
-    }
+        if (hash_progress != nullptr){
+            hash_progress(total_bytes_read, file_size);
+        }
+    }while (bytes_read == BUFFER_SIZE);
 
-    return hash.result().toHex().toStdString(); 
+    hash.finish();
+    return hash.get_hash_hex();
 }
 
 }

SerialPrograms/Source/CommonFramework/Tools/FileHash.h

@@ -17,10 +17,13 @@ namespace PokemonAutomation{
 
 // uses SHA 256
 std::string hash_file(
-    CancellableScope& scope,
+    CancellableScope* scope,
     const std::string& file_path,
-    std::function<void(uint64_t bytes_done, uint64_t total_bytes)> hash_progress
+    std::function<void(uint64_t bytes_done, uint64_t total_bytes)> hash_progress = nullptr
 );
+inline std::string hash_file(const std::string& file_path){
+    return hash_file(nullptr, file_path, nullptr);
+}
 
 
 }

SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp

@@ -5,11 +5,6 @@
  *  Helper functions to work with ONNX Runtime library
  */
 
-#include <QString>
-#include <QFile>
-#include <QCryptographicHash>
-#include <QByteArray>
-
 #include <iostream>
 #include <string>
 #include <fstream>
@@ -19,27 +14,14 @@
 #include "Common/Cpp/Exceptions.h"
 #include "Common/Cpp/Filesystem/Filesystem.h"
 #include "CommonFramework/Logging/Logger.h"
+#include "CommonFramework/Tools/FileHash.h"
 #include "ML_OrtEnv.h"
 #include "ML_ONNXRuntimeHelpers.h"
 
 namespace PokemonAutomation{
 namespace ML{
 
 
-// Computes the cryptographic hash of a file.
-std::string create_file_hash(const std::string& filepath){
-    QFile file(QString::fromStdString(filepath));
-    if (!file.open(QIODevice::ReadOnly)){
-        return "";
-    }
-
-    QCryptographicHash hash(QCryptographicHash::Sha256);
-    if (hash.addData(&file)){
-        return hash.result().toHex(0).toStdString();
-    }else{
-        return "";
-    }
-}
 
 
 Ort::SessionOptions create_session_options(const std::string& model_cache_path, bool use_gpu){
@@ -143,7 +125,7 @@ Ort::SessionOptions create_session_options(const std::string& model_cache_path,
 // model_path: the model path to load the ML model. This is needed to ensure we delete the old model cache
 //   when a new model
 std::pair<bool, std::string> clean_up_old_model_cache(const std::string& model_cache_path, const std::string& model_path){
-    std::string file_hash = create_file_hash(model_path);
+    std::string file_hash = hash_file(model_path);
     if (file_hash.size() == 0){
         // the model file cannot be loaded
         return {true, ""};

SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h

@@ -16,6 +16,7 @@
 namespace PokemonAutomation{
 namespace ML{
 
+
 // Create an ONNX SessionOptions
 // If on macOS, will use CoreML as the backend.
 // If on Windows, will try CUDA first (NVIDIA GPUs), then DirectML (all GPU vendors).