diff --git a/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_DigitReader.cpp b/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_DigitReader.cpp index 2e82beaefd..1fb6c407f9 100644 --- a/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_DigitReader.cpp +++ b/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_DigitReader.cpp @@ -14,15 +14,12 @@ #include "Common/Cpp/Logging/AbstractLogger.h" #include "Kernels/Waterfill/Kernels_Waterfill_Session.h" #include "CommonFramework/GlobalAutoPaths.h" -#include "CommonFramework/GlobalSettingsPanel.h" #include "CommonFramework/ImageTools/ImageBoxes.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "CommonFramework/ImageTypes/ImageRGB32_OpenCV.h" #include "CommonTools/ImageMatch/ExactImageMatcher.h" #include "CommonTools/Images/BinaryImage_FilterRgb32.h" -#include "CommonTools/Images/ImageFilter.h" -#include "CommonTools/Images/ImageManip.h" #include "PokemonFRLG_DigitReader.h" @@ -34,128 +31,6 @@ namespace PokemonAutomation{ namespace NintendoSwitch{ namespace PokemonFRLG{ -// Debug counter for unique filenames -static int debug_counter = 0; - -// Full OCR preprocessing pipeline for GBA pixel fonts. -// -// GBA fonts are seven-segment-like with 1-pixel gaps between segments. -// Pipeline: blur at native -> smooth upscale -> BW -> smooth BW -> re-BW -> pad -// -// The native blur connects gaps. Post-BW padding provides margins. -ImageRGB32 preprocess_for_ocr( - const ImageViewRGB32 &image, - const std::string &label, - int blur_kernel_size, int blur_passes, - bool in_range_black, uint32_t bw_min, - uint32_t bw_max, - bool save_debug_images -){ - int id = debug_counter++; - std::string prefix = "DebugDumps/ocr_" + label + "_" + std::to_string(id); - - // Save raw input - if (save_debug_images){ - image.save(prefix + "_0_raw.png"); - } - - cv::Mat src = to_OpenCV_ref(image); - - // Step 0: Rescale the image to match the expected height at 1080p - // so that the Gaussian blur size works across resolutions - constexpr int expected_height = 69; - - double scale_factor = static_cast(expected_height) / static_cast(image.height()); - - int new_w = std::max( - 1, cvRound(static_cast(image.width() * scale_factor)) - ); - - int new_h = std::max( - 1, cvRound(static_cast(image.height() * scale_factor)) - ); - - cv::Mat rescaled; - cv::resize( - src, rescaled, cv::Size(new_w, new_h), 0, 0, - cv::INTER_LINEAR - ); - - // Step 1: Gaussian blur at with 5x5 kernel. - // The 5x5 kernel reaches 2 pixels away (vs 1px for 3x3), bridging - // wider gaps in the seven-segment font. Two passes for heavy smoothing. - cv::Mat blurred_native; - rescaled.copyTo(blurred_native); - if (blur_kernel_size > 0 && blur_passes > 0){ - for (int i = 0; i < blur_passes; i++){ - cv::GaussianBlur( - blurred_native, blurred_native, - cv::Size(blur_kernel_size, blur_kernel_size), 1.5 - ); - } - } - - // Save blurred at rescaled res - ImageRGB32 blurred_native_img(blurred_native.cols, blurred_native.rows); - blurred_native.copyTo(to_OpenCV_ref(blurred_native_img)); - if (save_debug_images){ - blurred_native_img.save(prefix + "_1_blurred_native.png"); - } - - // Step 2: Smooth upscale 4x with bilinear interpolation. - scale_factor = 4; - new_w = static_cast(image.width()) * scale_factor; - new_h = static_cast(image.height()) * scale_factor; - cv::Mat resized; - cv::resize( - blurred_native, resized, cv::Size(new_w, new_h), 0, 0, - cv::INTER_LINEAR - ); - - // Save upscaled - ImageRGB32 resized_img(resized.cols, resized.rows); - resized.copyTo(to_OpenCV_ref(resized_img)); - if (save_debug_images){ - resized_img.save(prefix + "_2_upscaled.png"); - } - - // Step 3: BW threshold on the smooth upscaled image. - ImageRGB32 bw = - to_blackwhite_rgb32_range(resized_img, in_range_black, bw_min, bw_max); - if (save_debug_images){ - bw.save(prefix + "_3_bw.png"); - } - - // Step 4: Post-BW smoothing -> re-threshold. - // The BW image has angular seven-segment shapes. GaussianBlur on the - // binary image creates gray anti-aliased edges. Re-thresholding at 128 - // rounds the corners into natural smooth digit shapes that Tesseract - // recognizes much better. This is equivalent to morphological closing. - cv::Mat bw_mat = to_OpenCV_ref(bw); - cv::Mat smoothed; - cv::GaussianBlur(bw_mat, smoothed, cv::Size(7, 7), 2.0); - - // Re-threshold: convert smoothed back to ImageRGB32 and BW threshold. - // After blur on BW: text areas are dark gray (~0-64), bg areas are - // light gray (~192-255), edge zones are mid-gray (~64-192). - // Threshold at [0..128] captures text + expanded edges -> BLACK. - ImageRGB32 smoothed_img(smoothed.cols, smoothed.rows); - smoothed.copyTo(to_OpenCV_ref(smoothed_img)); - ImageRGB32 smooth_bw = to_blackwhite_rgb32_range( - smoothed_img, true, combine_rgb(0, 0, 0), combine_rgb(128, 128, 128)); - if (save_debug_images){ - smooth_bw.save(prefix + "_4_smooth_bw.png"); - } - - // Step 5: Pad with white border (Tesseract needs margins). - ImageRGB32 padded = pad_image(smooth_bw, smooth_bw.height() / 2, 0xffffffff); - if (save_debug_images){ - padded.save(prefix + "_5_padded.png"); - } - - return padded; -} - // --------------------------------------------------------------------------- // Template store: loads 10 digit matchers from a resource sub-directory. // Results are cached in a static map keyed by template type. diff --git a/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_DigitReader.h b/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_DigitReader.h index 0fbb5eeee2..ebccb7ff25 100644 --- a/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_DigitReader.h +++ b/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_DigitReader.h @@ -11,6 +11,8 @@ */ #ifndef PokemonAutomation_PokemonFRLG_DigitReader_H +#define PokemonAutomation_PokemonFRLG_DigitReader_H + #include #include @@ -27,21 +29,6 @@ enum class DigitTemplateType{ LevelBox, // Lilac level box: PokemonFRLG/LevelDigits/ }; -// Full OCR preprocessing pipeline for GBA pixel fonts. -// -// GBA fonts are seven-segment-like with 1-pixel gaps between segments. -// Pipeline: blur at native -> smooth upscale -> BW -> smooth BW -> re-BW -> pad -// -// The native blur connects gaps. Post-BW padding provides margins. -ImageRGB32 preprocess_for_ocr( - const ImageViewRGB32 &image, - const std::string &label, - int blur_kernel_size, int blur_passes, - bool in_range_black, uint32_t bw_min, - uint32_t bw_max, - bool save_debug_images = false -); - // Read a string of decimal digits from `stat_region`. // // template_type Which template set to use (StatBox or LevelBox). diff --git a/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_OcrPreprocessing.cpp b/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_OcrPreprocessing.cpp new file mode 100644 index 0000000000..c75e34a7a8 --- /dev/null +++ b/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_OcrPreprocessing.cpp @@ -0,0 +1,85 @@ +/* FRLG OCR Preprocessing + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/ImageTypes/ImageRGB32_OpenCV.h" + +#include "PokemonFRLG_OcrPreprocessing.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonFRLG{ + + +ImageRGB32 preprocess_for_ocr( + const ImageViewRGB32& image, + int blur_kernel_size, int blur_passes +){ + if (!image || image.width() == 0 || image.height() == 0){ + return ImageRGB32(); + } + + // Rescale the image to match the expected height at 1080p + // so that the Gaussian blur size works across resolutions + constexpr int expected_height = 69; + + double scale_factor = static_cast(expected_height) / static_cast(image.height()); + + int new_w = std::max( + 1, cvRound(static_cast(image.width() * scale_factor)) + ); + + int new_h = std::max( + 1, cvRound(static_cast(image.height() * scale_factor)) + ); + + ImageRGB32 blurred = image.scale_to(new_w, new_h); + + // The 5x5 kernel reaches 2 pixels away (vs 1px for 3x3), bridging + // wider gaps in the seven-segment font. Two passes for heavy smoothing. + cv::Mat mat = to_OpenCV_ref(blurred); + if (blur_kernel_size > 0 && blur_passes > 0){ + for (int i = 0; i < blur_passes; i++){ + cv::GaussianBlur( + mat, mat, + cv::Size(blur_kernel_size, blur_kernel_size), 1.5 + ); + } + } + + return blurred; +} + + +const std::vector& DARK_TEXT_FILTERS(){ + static std::vector filters{ + {combine_rgb(0, 0, 0), combine_rgb(96, 96, 96)}, + {combine_rgb(0, 0, 0), combine_rgb(128, 128, 128)}, + {combine_rgb(0, 0, 0), combine_rgb(160, 160, 160)}, + {combine_rgb(0, 0, 0), combine_rgb(190, 190, 190)}, + }; + return filters; +} + +const std::vector& BRIGHT_TEXT_FILTERS(){ + static std::vector filters{ + {combine_rgb(180, 180, 180), combine_rgb(255, 255, 255)}, + {combine_rgb(200, 200, 200), combine_rgb(255, 255, 255)}, + {combine_rgb(220, 220, 220), combine_rgb(255, 255, 255)}, + {combine_rgb(230, 230, 230), combine_rgb(255, 255, 255)}, + {combine_rgb(240, 240, 240), combine_rgb(255, 255, 255)}, + }; + return filters; +} + + +} // namespace PokemonFRLG +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_OcrPreprocessing.h b/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_OcrPreprocessing.h new file mode 100644 index 0000000000..d5db06085c --- /dev/null +++ b/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_OcrPreprocessing.h @@ -0,0 +1,40 @@ +/* FRLG OCR Preprocessing + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonFRLG_OcrPreprocessing_H +#define PokemonAutomation_PokemonFRLG_OcrPreprocessing_H + +#include +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonTools/OCR/OCR_Routines.h" + +namespace PokemonAutomation{ + +class ImageViewRGB32; + +namespace NintendoSwitch{ +namespace PokemonFRLG{ + + +// Gaussian-blur `image` so the gaps in the GBA font close up. +// The blur is adjusted from the passed size to match the size of the image +ImageRGB32 preprocess_for_ocr( + const ImageViewRGB32& image, + int blur_kernel_size = 7, int blur_passes = 2 +); + +// For dark GBA text against a light background +const std::vector& DARK_TEXT_FILTERS(); + +// For white GBA text against a dark background +const std::vector& BRIGHT_TEXT_FILTERS(); + + +} // namespace PokemonFRLG +} // namespace NintendoSwitch +} // namespace PokemonAutomation + +#endif diff --git a/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_StatsReader.cpp b/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_StatsReader.cpp index b1c3a7b97e..1f3081a396 100644 --- a/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_StatsReader.cpp +++ b/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_StatsReader.cpp @@ -20,6 +20,7 @@ #include "Pokemon/Inference/Pokemon_NatureReader.h" #include "PokemonFRLG/PokemonFRLG_Settings.h" #include "PokemonFRLG_DigitReader.h" +#include "PokemonFRLG_OcrPreprocessing.h" #include namespace PokemonAutomation { @@ -85,72 +86,24 @@ bool StatsReader::read_name( ImageViewRGB32 name_box = extract_box_reference(game_screen, jpn ? m_box_name_jpn : m_box_name); - static const std::vector WHITE_THRESHOLDS = { 180, 200, 220, 230, 240 }; - OCR::StringMatchResult best_result; - bool initialized = false; - std::vector debug_images; - if (save_debug_images){ - debug_images.reserve(WHITE_THRESHOLDS.size()); - } - for (int thresh : WHITE_THRESHOLDS){ - ImageRGB32 name_filtered(name_box.width(), name_box.height()); - for (size_t r = 0; r < name_box.height(); r++){ - for (size_t c = 0; c < name_box.width(); c++){ - Color pixel(name_box.pixel(c, r)); - if (pixel.red() > thresh && pixel.green() > thresh && pixel.blue() > thresh){ - name_filtered.pixel(c, r) = (uint32_t)0xff000000; // Black - }else{ - name_filtered.pixel(c, r) = (uint32_t)0xffffffff; // White - } - } - } - ImageRGB32 name_ready = preprocess_for_ocr( - name_filtered, "name", 7, 2, true, - combine_rgb(0, 0, 0), jpn ? combine_rgb(160, 160, 160) : combine_rgb(140, 140, 140) + ImageRGB32 name_ready = preprocess_for_ocr(name_box); + + OCR::StringMatchResult result = subset.empty() + ? Pokemon::PokemonNameReader::instance().read_substring( + logger, language, name_ready, BRIGHT_TEXT_FILTERS() + ) + : Pokemon::PokemonNameReader(subset).read_substring( + logger, language, name_ready, BRIGHT_TEXT_FILTERS() ); - const std::vector name_text_color_ranges{ - {combine_rgb(0, 0, 0), combine_rgb(120, 120, 120)} - }; - - OCR::StringMatchResult result; - if (subset.size() > 0){ - auto name_result = Pokemon::PokemonNameReader(subset).read_substring( - logger, language, name_ready, name_text_color_ranges - ); - if (!name_result.results.empty()){ - result = name_result; - } - }else{ - auto name_result = Pokemon::PokemonNameReader::instance().read_substring( - logger, language, name_ready, name_text_color_ranges - ); - if (!name_result.results.empty()){ - result = name_result; - } - } - if (!result.results.empty()){ - if (!initialized || result.results.begin()->first < best_result.results.begin()->first){ - best_result = result; - initialized = true; - } - } - if (save_debug_images){ - debug_images.emplace_back(std::move(name_ready)); - } - } - if (initialized && !best_result.results.empty()){ - stats.name = best_result.results.begin()->second.token; + if (!result.results.empty()){ + stats.name = result.results.begin()->second.token; return true; } logger.log("Failed to read species name.", COLOR_RED); if (save_debug_images){ name_box.save("DebugDumps/ocr_name_box.png"); - for (size_t c = 0; c < debug_images.size(); c++){ - debug_images[c].save( - "DebugDumps/ocr_name_ready_" + std::to_string(WHITE_THRESHOLDS[c]) + ".png" - ); - } + name_ready.save("DebugDumps/ocr_name_ready.png"); } return false; } @@ -297,19 +250,13 @@ bool StatsReader::read_nature( const static Pokemon::NatureReader reader("PokemonFRLG/NatureCheckerOCR.json"); ImageViewRGB32 nature_raw = extract_box_reference(game_screen, jpn ? m_box_nature_jpn : m_box_nature); - ImageRGB32 nature_ready = preprocess_for_ocr( - nature_raw, "nature", 7, 2, true, - combine_rgb(0, 0, 0), combine_rgb(190, 190, 190) - ); + ImageRGB32 nature_ready = preprocess_for_ocr(nature_raw); - OCR::StringMatchResult nature_result = reader.match_substring_from_image( - &logger, language, nature_ready, - Pokemon::NatureReader::MAX_LOG10P, - Pokemon::NatureReader::MAX_LOG10P_SPREAD, - OCR::PageSegMode::SINGLE_LINE); + OCR::StringMatchResult nature_result = reader.read_substring( + logger, language, nature_ready, DARK_TEXT_FILTERS() + ); if (!nature_result.results.empty()){ - nature_result.log(logger, Pokemon::NatureReader::MAX_LOG10P, "Nature Final"); stats.nature = nature_result.results.begin()->second.token; return true; } diff --git a/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_TrainerIdReader.cpp b/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_TrainerIdReader.cpp index c8e4c17cba..97050c215b 100644 --- a/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_TrainerIdReader.cpp +++ b/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_TrainerIdReader.cpp @@ -43,23 +43,9 @@ uint16_t TrainerIdReader::read_tid( ImageViewRGB32 tid_region = extract_box_reference(game_screen, language == Language::Japanese ? m_box_tid_jpn : m_box_tid); - if (GlobalSettings::instance().OCR_LIBRARY != OcrLibrary::PADDLE_OCR){ - // Tesseract-free path: waterfill segmentation + template matching - // against the PokemonFRLG/Digits/0-9.png templates. - return uint16_t(read_digits_waterfill_template(logger, tid_region)); - } - - // PaddleOCR path (original): preprocess then per-digit waterfill OCR. - ImageRGB32 ocr_ready = preprocess_for_ocr( - tid_region, "TID", 7, 2, true, - combine_rgb(0, 0, 0), combine_rgb(190, 190, 190) - ); - - // Waterfill isolates each digit -> per-char SINGLE_CHAR OCR. - return uint16_t(OCR::read_number_waterfill( - logger, ocr_ready, 0xff000000, - 0xff808080 - )); + // waterfill segmentation + template matching + // against the PokemonFRLG/Digits/0-9.png templates. + return uint16_t(read_digits_waterfill_template(logger, tid_region)); } } // namespace PokemonFRLG diff --git a/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_WildEncounterReader.cpp b/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_WildEncounterReader.cpp index f9d8dfeebb..901c107664 100644 --- a/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_WildEncounterReader.cpp +++ b/SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_WildEncounterReader.cpp @@ -17,7 +17,7 @@ #include "CommonTools/OCR/OCR_Routines.h" #include "Pokemon/Inference/Pokemon_NameReader.h" #include "PokemonFRLG/PokemonFRLG_Settings.h" -#include "PokemonFRLG_DigitReader.h" +#include "PokemonFRLG_OcrPreprocessing.h" #include namespace PokemonAutomation { @@ -52,22 +52,23 @@ PokemonFRLG_WildEncounter WildEncounterReader::read_encounter( ImageViewRGB32 name_box = extract_box_reference(game_screen, jpn ? m_box_name_jpn : m_box_name); - ImageRGB32 name_ready = preprocess_for_ocr( - name_box, "name", 7, 2, true, - combine_rgb(0, 0, 0), combine_rgb(190, 190, 190) - ); + // Dark text on the light battle HP bar. Blur to close the gaps in the GBA + // font, then let the matcher try each threshold in turn. + ImageRGB32 name_ready = preprocess_for_ocr(name_box); - const std::vector name_text_color_ranges{ - {combine_rgb(0, 0, 0), combine_rgb(120, 120, 120)} - }; - - // auto name_result = Pokemon::PokemonNameReader::instance().read_substring( auto name_result = Pokemon::PokemonNameReader(subset).read_substring( logger, language, name_ready, - name_text_color_ranges, + DARK_TEXT_FILTERS(), 0.01, 0.50, max_log10p); if (!name_result.results.empty()){ encounter.name = name_result.results.begin()->second.token; + }else{ + logger.log("Failed to read species name.", COLOR_RED); + if (GlobalSettings::instance().SAVE_DEBUG_IMAGES){ + name_box.save("DebugDumps/ocr_encounter_box.png"); + name_ready.save("DebugDumps/ocr_encounter_ready.png"); + game_screen.save("DebugDumps/ocr_encounter_frame.png"); + } } return encounter; } diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index dc4fb970d3..8ea4a3c331 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1599,6 +1599,8 @@ file(GLOB LIBRARY_SOURCES Source/PokemonFRLG/Inference/PokemonFRLG_DaycareManDetector.h Source/PokemonFRLG/Inference/PokemonFRLG_DigitReader.cpp Source/PokemonFRLG/Inference/PokemonFRLG_DigitReader.h + Source/PokemonFRLG/Inference/PokemonFRLG_OcrPreprocessing.cpp + Source/PokemonFRLG/Inference/PokemonFRLG_OcrPreprocessing.h Source/PokemonFRLG/Inference/PokemonFRLG_StatsReader.cpp Source/PokemonFRLG/Inference/PokemonFRLG_StatsReader.h Source/PokemonFRLG/Inference/PokemonFRLG_PokemonSpriteReader.cpp