From e8437ad4fbfb9fb0d1d8e65ed58943e2eab97b73 Mon Sep 17 00:00:00 2001 From: Alexander Yee Date: Sat, 15 Aug 2026 11:57:32 -0700 Subject: [PATCH 1/2] Collapse duplicate file hashing function. --- Common/Cpp/Filesystem/Filesystem.cpp | 3 +++ Common/Cpp/Filesystem/Filesystem.h | 1 + .../ResourceDownload/DownloadThread.cpp | 15 +++++-------- .../Source/CommonFramework/Tools/FileHash.cpp | 10 ++++++--- .../Source/CommonFramework/Tools/FileHash.h | 7 ++++-- .../ML/Models/ML_ONNXRuntimeHelpers.cpp | 22 ++----------------- .../Source/ML/Models/ML_ONNXRuntimeHelpers.h | 1 + 7 files changed, 25 insertions(+), 34 deletions(-) diff --git a/Common/Cpp/Filesystem/Filesystem.cpp b/Common/Cpp/Filesystem/Filesystem.cpp index 8b97e3b1ba..2ab4580515 100644 --- a/Common/Cpp/Filesystem/Filesystem.cpp +++ b/Common/Cpp/Filesystem/Filesystem.cpp @@ -42,6 +42,9 @@ bool create_directories(const Path& path){ return std::filesystem::create_directories(path.stdpath()); } +bool remove(const Path& path){ + return std::filesystem::remove(path); +} std::uintmax_t remove_all(const Path& path){ return std::filesystem::remove_all(path.stdpath()); } diff --git a/Common/Cpp/Filesystem/Filesystem.h b/Common/Cpp/Filesystem/Filesystem.h index 4f6df4db57..08eb322bc7 100644 --- a/Common/Cpp/Filesystem/Filesystem.h +++ b/Common/Cpp/Filesystem/Filesystem.h @@ -40,6 +40,7 @@ bool create_directories(const Path& path); // Delete the contents of the path (if it is a directory) and the contents of all its subdirectories, recursively. // Then delete the file/directory of path itself. Symlinks are not followed (symlink is removed, not its target). +bool remove(const Path& path); std::uintmax_t remove_all(const Path& path); // Copy a file. diff --git a/SerialPrograms/Source/CommonFramework/ResourceDownload/DownloadThread.cpp b/SerialPrograms/Source/CommonFramework/ResourceDownload/DownloadThread.cpp index e6941440ed..0e149ad8f2 100644 --- a/SerialPrograms/Source/CommonFramework/ResourceDownload/DownloadThread.cpp +++ b/SerialPrograms/Source/CommonFramework/ResourceDownload/DownloadThread.cpp @@ -23,9 +23,6 @@ using std::endl; namespace PokemonAutomation{ -namespace fs = std::filesystem; - - DownloadThread::~DownloadThread(){ @@ -104,7 +101,7 @@ void DownloadThread::run_download(DownloadedResourceMetadata resource_metadata){ try{ // delete directory and the old resource - fs::remove_all(Filesystem::Path(resource_directory)); + Filesystem::remove_all(resource_directory); // download std::string zip_path = resource_directory + "/temp.zip"; @@ -122,7 +119,7 @@ void DownloadThread::run_download(DownloadedResourceMetadata resource_metadata){ // hash std::string hash = hash_file( - *this, + this, zip_path, [this](uint64_t bytes_done, uint64_t total_bytes){ m_hooks.report_hash_progress(bytes_done, total_bytes); @@ -149,23 +146,23 @@ void DownloadThread::run_download(DownloadedResourceMetadata resource_metadata){ ); // delete old zip file - fs::remove(Filesystem::Path(zip_path)); + Filesystem::remove(zip_path); throw_if_cancelled(); }catch(OperationCancelledException&){ // delete directory and the resource - fs::remove_all(Filesystem::Path(resource_directory)); + Filesystem::remove_all(resource_directory); throw; }catch(OperationFailedException&){ // delete directory and the resource - fs::remove_all(Filesystem::Path(resource_directory)); + Filesystem::remove_all(resource_directory); throw; }catch(...){ // delete directory and the resource - fs::remove_all(Filesystem::Path(resource_directory)); + Filesystem::remove_all(resource_directory); throw; } diff --git a/SerialPrograms/Source/CommonFramework/Tools/FileHash.cpp b/SerialPrograms/Source/CommonFramework/Tools/FileHash.cpp index ac6d80ba5c..78a89bac8a 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/FileHash.cpp +++ b/SerialPrograms/Source/CommonFramework/Tools/FileHash.cpp @@ -19,7 +19,7 @@ namespace PokemonAutomation{ std::string hash_file( - CancellableScope& scope, + CancellableScope* scope, const std::string& file_path, std::function hash_progress ){ @@ -34,7 +34,9 @@ std::string hash_file( QByteArray buffer(1024 * 1024, 0); // Pre-allocate 1MB once while (!file.atEnd()){ - scope.throw_if_cancelled(); + if (scope != nullptr){ + scope->throw_if_cancelled(); + } qint64 num_bytes_in_chunk = file.read(buffer.data(), buffer.size()); if (num_bytes_in_chunk == -1){ @@ -44,7 +46,9 @@ std::string hash_file( hash.addData(QByteArrayView(buffer.data(), num_bytes_in_chunk)); total_bytes_read += num_bytes_in_chunk; - hash_progress(total_bytes_read, file_size); + if (hash_progress != nullptr){ + hash_progress(total_bytes_read, file_size); + } } return hash.result().toHex().toStdString(); diff --git a/SerialPrograms/Source/CommonFramework/Tools/FileHash.h b/SerialPrograms/Source/CommonFramework/Tools/FileHash.h index c4af6b84e2..ca6eff9b40 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/FileHash.h +++ b/SerialPrograms/Source/CommonFramework/Tools/FileHash.h @@ -17,10 +17,13 @@ namespace PokemonAutomation{ // uses SHA 256 std::string hash_file( - CancellableScope& scope, + CancellableScope* scope, const std::string& file_path, - std::function hash_progress + std::function hash_progress = nullptr ); +inline std::string hash_file(const std::string& file_path){ + return hash_file(nullptr, file_path, nullptr); +} } diff --git a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp index 70215bb4f2..b3b7b31caa 100644 --- a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp +++ b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp @@ -5,11 +5,6 @@ * Helper functions to work with ONNX Runtime library */ -#include -#include -#include -#include - #include #include #include @@ -19,6 +14,7 @@ #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Filesystem/Filesystem.h" #include "CommonFramework/Logging/Logger.h" +#include "CommonFramework/Tools/FileHash.h" #include "ML_OrtEnv.h" #include "ML_ONNXRuntimeHelpers.h" @@ -26,20 +22,6 @@ namespace PokemonAutomation{ namespace ML{ -// Computes the cryptographic hash of a file. -std::string create_file_hash(const std::string& filepath){ - QFile file(QString::fromStdString(filepath)); - if (!file.open(QIODevice::ReadOnly)){ - return ""; - } - - QCryptographicHash hash(QCryptographicHash::Sha256); - if (hash.addData(&file)){ - return hash.result().toHex(0).toStdString(); - }else{ - return ""; - } -} Ort::SessionOptions create_session_options(const std::string& model_cache_path, bool use_gpu){ @@ -143,7 +125,7 @@ Ort::SessionOptions create_session_options(const std::string& model_cache_path, // model_path: the model path to load the ML model. This is needed to ensure we delete the old model cache // when a new model std::pair clean_up_old_model_cache(const std::string& model_cache_path, const std::string& model_path){ - std::string file_hash = create_file_hash(model_path); + std::string file_hash = hash_file(model_path); if (file_hash.size() == 0){ // the model file cannot be loaded return {true, ""}; diff --git a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h index 98767eda83..e847537f49 100644 --- a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h +++ b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h @@ -16,6 +16,7 @@ namespace PokemonAutomation{ namespace ML{ + // Create an ONNX SessionOptions // If on macOS, will use CoreML as the backend. // If on Windows, will try CUDA first (NVIDIA GPUs), then DirectML (all GPU vendors). From 82401cddaac6b543a48ba60f956459013ab63948 Mon Sep 17 00:00:00 2001 From: Alexander Yee Date: Sat, 15 Aug 2026 23:24:30 -0700 Subject: [PATCH 2/2] Remove Qt from FileHash. --- .../Source/CommonFramework/Tools/FileHash.cpp | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Tools/FileHash.cpp b/SerialPrograms/Source/CommonFramework/Tools/FileHash.cpp index 78a89bac8a..3fb1262880 100644 --- a/SerialPrograms/Source/CommonFramework/Tools/FileHash.cpp +++ b/SerialPrograms/Source/CommonFramework/Tools/FileHash.cpp @@ -5,9 +5,11 @@ */ #include "Common/Cpp/Exceptions.h" -#include -#include -#include +#include "Common/Cpp/ScopeExit.h" +#include "Common/Cpp/Containers/AlignedMalloc.h" +#include "Common/Cpp/Filesystem/FileIO.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/Cryptography/SHA256.h" #include "FileHash.h" //#include @@ -23,35 +25,41 @@ std::string hash_file( const std::string& file_path, std::function hash_progress ){ - QFile file(QString::fromStdString(file_path)); - if (!file.open(QIODevice::ReadOnly)){ - throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "hash_file: Could not open file."); + Filesystem::Path path(file_path); + FileIO file(path, FileMode::READ | FileMode::BINARY); + if (!file.is_open()){ + throw InternalProgramError( + nullptr, + PA_CURRENT_FUNCTION, + "hash_file: Could not open file." + ); } - QCryptographicHash hash(QCryptographicHash::Sha256); - qint64 file_size = file.size(); - qint64 total_bytes_read = 0; + SHA256 hash; + uint64_t file_size = Filesystem::file_size(path); + uint64_t total_bytes_read = 0; - QByteArray buffer(1024 * 1024, 0); // Pre-allocate 1MB once - while (!file.atEnd()){ + constexpr size_t BUFFER_SIZE = 1024 * 1024; + void* buffer = aligned_malloc(BUFFER_SIZE, 4096); // Pre-allocate 1MB once + ScopeExit sg([&]{ aligned_free(buffer); }); + + size_t bytes_read; + do{ if (scope != nullptr){ scope->throw_if_cancelled(); } - - qint64 num_bytes_in_chunk = file.read(buffer.data(), buffer.size()); - if (num_bytes_in_chunk == -1){ - throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "hash_file: Read error:" + file.errorString().toStdString()); - } - hash.addData(QByteArrayView(buffer.data(), num_bytes_in_chunk)); - total_bytes_read += num_bytes_in_chunk; + bytes_read = file.read(buffer, BUFFER_SIZE); + hash.push(buffer, bytes_read); + total_bytes_read += bytes_read; if (hash_progress != nullptr){ hash_progress(total_bytes_read, file_size); } - } + }while (bytes_read == BUFFER_SIZE); - return hash.result().toHex().toStdString(); + hash.finish(); + return hash.get_hash_hex(); } }