diff --git a/SerialPrograms/Source/CommonFramework/StaticGlobals.cpp b/SerialPrograms/Source/CommonFramework/StaticGlobals.cpp index 698e4326c9..65bbfd77c2 100644 --- a/SerialPrograms/Source/CommonFramework/StaticGlobals.cpp +++ b/SerialPrograms/Source/CommonFramework/StaticGlobals.cpp @@ -73,6 +73,7 @@ void StaticGlobals::load_json(const JsonValue& json){ debug_obj->read_integer(BOX_SYSTEM_CELL_ROW, "BOX_SYSTEM_CELL_ROW"); debug_obj->read_integer(BOX_SYSTEM_CELL_COL, "BOX_SYSTEM_CELL_COL"); debug_obj->read_boolean(GENERATE_TEST_GOLDEN_FILES, "GENERATE_TEST_GOLDEN_FILES"); + debug_obj->read_boolean(PADDLE_OCR_DEBUG, "PADDLE_OCR_DEBUG"); } } JsonValue StaticGlobals::to_json_debug() const{ @@ -83,6 +84,7 @@ JsonValue StaticGlobals::to_json_debug() const{ debug_obj["BOX_SYSTEM_CELL_ROW"] = BOX_SYSTEM_CELL_ROW; debug_obj["BOX_SYSTEM_CELL_COL"] = BOX_SYSTEM_CELL_COL; debug_obj["GENERATE_TEST_GOLDEN_FILES"] = GENERATE_TEST_GOLDEN_FILES; + debug_obj["PADDLE_OCR_DEBUG"] = PADDLE_OCR_DEBUG; return debug_obj; } diff --git a/SerialPrograms/Source/CommonFramework/StaticGlobals.h b/SerialPrograms/Source/CommonFramework/StaticGlobals.h index 9682176bad..8cdf89a407 100644 --- a/SerialPrograms/Source/CommonFramework/StaticGlobals.h +++ b/SerialPrograms/Source/CommonFramework/StaticGlobals.h @@ -56,6 +56,8 @@ class StaticGlobals{ // then manually inspect to fix any errors. bool GENERATE_TEST_GOLDEN_FILES = false; + bool PADDLE_OCR_DEBUG = false; + }; diff --git a/SerialPrograms/Source/CommonTools/OCR/OCR_Tests.cpp b/SerialPrograms/Source/CommonTools/OCR/OCR_Tests.cpp index d6b0fe694f..7bf4b5cdcb 100644 --- a/SerialPrograms/Source/CommonTools/OCR/OCR_Tests.cpp +++ b/SerialPrograms/Source/CommonTools/OCR/OCR_Tests.cpp @@ -67,6 +67,7 @@ void add_tests_raw_OCR(UnitTestDatabase& database){ database.add("OCR/sentence-1-3.jpg", Language::English, "You hurry to the Pokemon Center, shielding your"); database.add("OCR/sentence-1-3-wide.jpg", Language::English, "You hurry to the Pokemon Center, shielding your"); database.add("OCR/sentence-1-3-tall.jpg", Language::English, "You hurry to the Pokemon Center, shielding your"); + database.add("OCR/german-nature-sanft.png", Language::German, "Wesen: SANFT"); } diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp index 3d2c198501..16f83b84bb 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp @@ -5,12 +5,14 @@ * */ -#include #include #include #include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/Logging/GlobalLogger.h" #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/GlobalSettingsPanel.h" +#include "CommonFramework/StaticGlobals.h" #include "CommonFramework/ImageTypes/ImageRGB32_OpenCV.h" #include "ML/Models/ML_ONNXRuntimeHelpers.h" #include "ML_PaddleOCRPipeline.h" @@ -70,9 +72,9 @@ PaddleOCRPipeline::PaddleOCRPipeline(Language language, std::string rec_path, st , m_language(language) , m_input_name(m_rec_session.GetInputNameAllocated(0, Ort::AllocatorWithDefaultOptions{}).get()) , m_output_name(m_rec_session.GetOutputNameAllocated(0, Ort::AllocatorWithDefaultOptions{}).get()) + , m_logger(global_logger_raw(), "OCR") { - load_dictionary(dict_path); - + load_dictionary(Filesystem::Path(dict_path)); } void PaddleOCRPipeline::run(const std::string& img_path){ @@ -94,20 +96,68 @@ void PaddleOCRPipeline::run(const std::string& img_path){ -void PaddleOCRPipeline::load_dictionary(const std::string& path){ - std::ifstream fs(path); +void PaddleOCRPipeline::load_dictionary(const Filesystem::Path& path){ + + const bool debugging = STATIC_GLOBALS.PADDLE_OCR_DEBUG; + + if (debugging){ + m_logger.log("[OCR-INFO] Loading dictionary from: " + path.string()); + m_logger.log("[OCR-INFO] Current working directory: " + + Filesystem::current_path().string()); + + std::error_code ec; + const auto absolute_path = Filesystem::absolute(path); + + if (!ec) { + m_logger.log("[OCR-INFO] Absolute dictionary path: " + + absolute_path.string()); + + m_logger.log("[OCR-INFO] Dictionary exists: " + + std::string(Filesystem::exists(absolute_path) ? "true" : "false")); + + if (Filesystem::exists(absolute_path)) { + const auto file_size = Filesystem::file_size(absolute_path, ec); + + if (!ec) { + m_logger.log("[OCR-INFO] Dictionary file size: " + + std::to_string(file_size) + " bytes"); + } + } + } + } + + std::ifstream fs_file(path.stdpath()); + + if (!fs_file.is_open()) { + m_logger.log("[OCR-ERROR] Failed to open dictionary: " + path.string()); + throw FileException(nullptr, PA_CURRENT_FUNCTION, "PaddleOCRPipeline::load_dictionary(): Failed to open dictionary.", path.string()); + } + std::string line; - // m_dictionary.push_back("blank"); // CTC blank index - while (std::getline(fs, line)){ + while (std::getline(fs_file, line)) { m_dictionary.push_back(line); } + + if (fs_file.bad()) { + m_logger.log("[OCR-ERROR] I/O error while reading dictionary: " + path.string()); + } + + if (debugging){ + m_logger.log("[OCR-INFO] Loaded " + + std::to_string(m_dictionary.size()) + + " dictionary entries"); + } } + std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ + const bool debugging = STATIC_GLOBALS.PADDLE_OCR_DEBUG; + // 1. Convert Image to OpenCV image (cv::mat) cv::Mat cv_image_rgb = imageviewrgb32_to_cv_mat_rgb(image); if (cv_image_rgb.empty()) { + m_logger.log("[OCR-DEBUG] Input was an empty image."); return ""; } @@ -115,6 +165,7 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ // 2. Crop tightly around the text, with small safety margin cv::Mat cropped_image = crop_to_text_region(cv_image_rgb); if (cropped_image.empty()){ + m_logger.log("[OCR-DEBUG] Crop to text region returned empty image."); return ""; } @@ -132,6 +183,11 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ (int)std::round(target_h * aspect_ratio) ); + if (target_w <= 0 || target_w > 8192){ + m_logger.log("[OCR-ERROR] Abnormally scaled target width calculated: " + std::to_string(target_w)); + return ""; + } + cv::Mat resized; cv::resize( cropped_image, @@ -173,6 +229,40 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ // 6. Define Dynamic Shape std::vector input_shape = {1, 3, target_h, target_w}; + + size_t expected_elements = 1 * 3 * target_h * target_w; + if (debugging){ + m_logger.log("[OCR-DEBUG] Cropped image constraints - Width: " + std::to_string(cropped_image.cols) + + ", Height: " + std::to_string(cropped_image.rows) + + ", Channels: " + std::to_string(cropped_image.channels()) + + ", Total Pixels: " + std::to_string(cropped_image.total())); + + size_t nan_count = 0; + size_t subnormal_count = 0; + for (float val : input_tensor_values) { + if (std::isnan(val)) { + nan_count++; + } else if (val != 0.0f && std::fpclassify(val) == FP_SUBNORMAL) { + subnormal_count++; + } + } + m_logger.log("[OCR-DEBUG] Tensor payload validation - Total Floats: " + std::to_string(input_tensor_values.size()) + + ", NaNs detected: " + std::to_string(nan_count) + + ", Subnormal (denormal) values: " + std::to_string(subnormal_count)); + + // Validate expected payload sizing matches matrix dimensionality + m_logger.log("[OCR-DEBUG] Shape Definition - NCHW: [" + std::to_string(input_shape[0]) + "," + std::to_string(input_shape[1]) + + "," + std::to_string(input_shape[2]) + "," + std::to_string(input_shape[3]) + "]. Expected Elements: " + std::to_string(expected_elements)); + + } + + if (input_tensor_values.size() != static_cast(expected_elements)) { + m_logger.log("[OCR-ERROR] Vector length vs input_shape calculation mismatch!"); + m_logger.log("[OCR-ERROR] Fatal memory stride mismatch. Vector size (" + std::to_string(input_tensor_values.size()) + + ") does not match shape requirement (" + std::to_string(expected_elements)); + return ""; + } + // 7. Create tensor with its own managed memory Ort::AllocatorWithDefaultOptions allocator; auto input_tensor = Ort::Value::CreateTensor( @@ -192,6 +282,10 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ const char* output_names[] = {m_output_name.c_str()}; try{ + if (debugging) { + m_logger.log("[OCR-DEBUG] Calling m_rec_session.Run() now..."); + } + // 8. Run the recognition session auto outputs = m_rec_session.Run( Ort::RunOptions{nullptr}, @@ -329,23 +423,58 @@ cv::Scalar estimate_background_color(const cv::Mat& image) { } - - std::vector preprocess_NCHW(cv::Mat& img){ - std::vector dst(img.rows * img.cols * 3); - for (int c = 0; c < 3; ++c){ - for (int i = 0; i < img.rows * img.cols; ++i){ - dst[c * img.rows * img.cols + i] = ((float*)img.data)[i * 3 + c]; + const int rows = img.rows; + const int cols = img.cols; + const int channels = 3; + + // Allocate a flat memory buffer big enough for all channels + std::vector dst(rows * cols * channels); + + // Define the size of one complete "color plane" (channel) + const int plane_size = rows * cols; + + // Loop through the image row-by-row + for (int y = 0; y < rows; ++y) { + // Safely locate the exact memory address for the start of row 'y' + const float* row_ptr = img.ptr(y); + + // Loop through every pixel column in the current row + for (int x = 0; x < cols; ++x) { + // Calculate the 1D coordinate of the pixel inside a flat 2D plane + int linear_idx = y * cols + x; + + // Extract the interleaved BGR channels explicitly + dst[0 * plane_size + linear_idx] = row_ptr[x * channels + 0]; // Channel 0 + dst[1 * plane_size + linear_idx] = row_ptr[x * channels + 1]; // Channel 1 + dst[2 * plane_size + linear_idx] = row_ptr[x * channels + 2]; // Channel 2 } } return dst; } -std::string decode_CTC(float* data, const std::vector& shape, const std::vector& dict){ + + +std::string PaddleOCRPipeline::decode_CTC(float* data, const std::vector& shape, const std::vector& dict){ + + const bool debugging = STATIC_GLOBALS.PADDLE_OCR_DEBUG; + std::string text = ""; size_t seq_len = static_cast(shape[1]); int64_t num_cls = shape[2]; size_t last_index = 0; + + // Initial boundary logging configuration + if (debugging){ + m_logger.log("[OCR-CTC-DEBUG] Starting decode_CTC. Sequence Length: " + std::to_string(seq_len) + + ", Total Classes: " + std::to_string(num_cls) + + ", Dictionary Size: " + std::to_string(dict.size())); + } + + if (dict.empty()) { + m_logger.log("[OCR-CTC-ERROR] FATAL: Dictionary payload array is empty! Parsing loops will fail to resolve text indicators."); + } + for (size_t i = 0; i < seq_len; ++i){ float* row = data + i * num_cls; // 1. Get the character index with highest probability (Argmax) @@ -358,11 +487,27 @@ std::string decode_CTC(float* data, const std::vector& shape, const std // Index 1 from the model maps to the 1st line of your .txt file (Vector index 0) size_t dict_idx = argmax - 1; if (dict_idx < dict.size()){ + if (debugging) { + m_logger.log("[OCR-CTC-DEBUG] Step " + std::to_string(i) + + ": Predicted Argmax = " + std::to_string(argmax) + + " -> Target Dict Index = " + std::to_string(dict_idx) + + " (Character resolved: '" + dict[dict_idx] + "')"); + } text += dict[dict_idx]; + }else { + m_logger.log("[OCR-CTC-ERROR] Step " + std::to_string(i) + + ": Predicted Argmax = " + std::to_string(argmax) + + " -> Target Dict Index = " + std::to_string(dict_idx) + + " (ERROR: Calculated index is out of bounds for the dictionary memory layout!)"); } } last_index = argmax; } + + if (debugging) { + m_logger.log("[OCR-CTC-DEBUG] Complete loop execution tracking finish. Resulting string extraction: '" + text + "'"); + } + return text; } diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h index a9babd7460..022408fc8a 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h @@ -12,6 +12,7 @@ #include #include #include +#include "Common/Cpp/Logging/TaggedLogger.h" #include "CommonFramework/Language.h" #include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "CommonFramework/ImageTools/ImageBoxes.h" @@ -31,8 +32,10 @@ class PaddleOCRPipeline{ static std::pair get_paths(Language language); + std::string decode_CTC(float* data, const std::vector& shape, const std::vector& dict); + private: - void load_dictionary(const std::string& path); + void load_dictionary(const Filesystem::Path& path); Ort::Env m_env; // Ort::Session det_session; @@ -41,7 +44,8 @@ class PaddleOCRPipeline{ Language m_language; std::string m_input_name; std::string m_output_name; - std::vector m_dictionary; + std::vector m_dictionary; + TaggedLogger m_logger; }; @@ -56,10 +60,11 @@ void add_horizontal_padding(cv::Mat& image); // assumes input image is RGB cv::Scalar estimate_background_color(const cv::Mat& image); -// convert HCW (height, width, channels) to NCHW (batch N, channels C, height H, width W) +// convert HWC (height, width, channels) to NCHW (batch N, channels C, height H, width W) +// HWC: pixels are interleaved. [B,G,R] [B,G,R] [B,G,R] ... +// NCHW: [All Blue Pixels...] [All Green Pixels...] [All Red Pixels...] std::vector preprocess_NCHW(cv::Mat& img); -std::string decode_CTC(float* data, const std::vector& shape, const std::vector& dict); cv::Mat imageviewrgb32_to_cv_mat_rgb(const ImageViewRGB32& image);