FRLG digit OCR improvements

SerialPrograms/Source/PokemonFRLG/Inference/Menus/PokemonFRLG_TrainerCardDetector.cpp

@@ -20,7 +20,7 @@ namespace NintendoSwitch{
 namespace PokemonFRLG{
 
 TrainerCardDetector::TrainerCardDetector(Color color)
-    : m_box_tid_top(0.581250, 0.108413, 0.332692, 0.007212) // light blue (227, 255, 255)
+    : m_box_tid_top(0.68125, 0.108413, 0.232692, 0.007212) // light blue (227, 255, 255)
     , m_box_tid_right(0.914904, 0.117067, 0.005769, 0.064904) 
     , m_box_stripe_top(0.038942, 0.090384, 0.918269, 0.002885) // blue  (94, 177, 255)
     , m_box_stripe_right(0.954808, 0.095432, 0.002885, 0.109615)

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_BattleLevelUpReader.cpp

@@ -59,7 +59,7 @@ StatReads BattleLevelUpReader::read_stats(Logger &logger, const ImageViewRGB32&
 
     auto read_stat = [&](const ImageFloatBox &box, const std::string &name){
         ImageViewRGB32 stat_region = extract_box_reference(game_screen, box);
-        return read_digits_waterfill_template(logger, stat_region);
+        return read_digits_waterfill_template(logger, stat_region, DigitTemplateType::DialogBox);
     };
 
     StatReads stats;
@@ -105,7 +105,11 @@ class Test_BattleLevelUpReader : public UnitTest{
 
 
 void add_tests_BattleLevelUpReader(UnitTestDatabase& database){
-    // to do
+    database.add<Test_BattleLevelUpReader>("PokemonFRLG/BattleLevelUpReader/charizard_1.png");
+    database.add<Test_BattleLevelUpReader>("PokemonFRLG/BattleLevelUpReader/horsea_1.png");
+    database.add<Test_BattleLevelUpReader>("PokemonFRLG/BattleLevelUpReader/horsea_2.png");
+    database.add<Test_BattleLevelUpReader>("PokemonFRLG/BattleLevelUpReader/horsea_3.png");
+    database.add<Test_BattleLevelUpReader>("PokemonFRLG/BattleLevelUpReader/deoxys_1_jpn.jpg");
 }
 
 

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_DigitReader.cpp

@@ -4,6 +4,7 @@
  *
  */
 
+#include <algorithm>
 #include <array>
 #include <map>
 #include <memory>
@@ -34,9 +35,10 @@ namespace PokemonFRLG{
 // ---------------------------------------------------------------------------
 // Template store: loads 10 digit matchers from a resource sub-directory.
 // Results are cached in a static map keyed by template type.
-// Supports both:
+// Supports:
 // - StatBox (yellow stat boxes): PokemonFRLG/Digits/
 // - LevelBox (lilac level box): PokemonFRLG/LevelDigits/
+// - DialogBox (white dialog box): PokemonFRLG/DialogDigits/
 // ---------------------------------------------------------------------------
 
 static std::string get_template_path(DigitTemplateType type){
@@ -45,6 +47,8 @@ static std::string get_template_path(DigitTemplateType type){
         return "PokemonFRLG/Digits/";
     case DigitTemplateType::LevelBox:
         return "PokemonFRLG/LevelDigits/";
+    case DigitTemplateType::DialogBox:
+        return "PokemonFRLG/DialogDigits/";
     default:
         return "PokemonFRLG/Digits/";
     }
@@ -89,14 +93,65 @@ struct DigitTemplates{
     }
 };
 
+// Shrink a digit's bounding box down to the glyph itself.
+static ImagePixelBox tighten_to_glyph(const ImageViewRGB32& region, const ImagePixelBox& box){
+    const size_t PADDING = 1;
+
+    auto brightness = [&](size_t x, size_t y) -> uint32_t{
+        uint32_t p = region.pixel(x, y);
+        return (p & 0xff) + ((p >> 8) & 0xff) + ((p >> 16) & 0xff);
+    };
+
+    uint32_t min_brightness = 0xffffffff;
+    uint32_t max_brightness = 0;
+    for (size_t y = box.min_y; y < box.max_y; ++y){
+        for (size_t x = box.min_x; x < box.max_x; ++x){
+            uint32_t b = brightness(x, y);
+            min_brightness = std::min(min_brightness, b);
+            max_brightness = std::max(max_brightness, b);
+        }
+    }
+    if (min_brightness > max_brightness){
+        return box;     //  Empty box.
+    }
+    uint32_t glyph_max = (min_brightness + max_brightness) / 2;
+
+    size_t glyph_min_x = box.max_x;
+    size_t glyph_min_y = box.max_y;
+    size_t glyph_max_x = box.min_x;
+    size_t glyph_max_y = box.min_y;
+    for (size_t y = box.min_y; y < box.max_y; ++y){
+        for (size_t x = box.min_x; x < box.max_x; ++x){
+            if (brightness(x, y) > glyph_max){
+                continue;
+            }
+            glyph_min_x = std::min(glyph_min_x, x);
+            glyph_min_y = std::min(glyph_min_y, y);
+            glyph_max_x = std::max(glyph_max_x, x);
+            glyph_max_y = std::max(glyph_max_y, y);
+        }
+    }
+    if (glyph_min_x > glyph_max_x || glyph_min_y > glyph_max_y){
+        return box;     //  Nothing dark enough to be a glyph.
+    }
+
+    //  max_x/max_y are exclusive, so the last glyph pixel needs a +1 of its own.
+    return ImagePixelBox(
+        glyph_min_x >= box.min_x + PADDING ? glyph_min_x - PADDING : box.min_x,
+        glyph_min_y >= box.min_y + PADDING ? glyph_min_y - PADDING : box.min_y,
+        std::min(glyph_max_x + 1 + PADDING, box.max_x),
+        std::min(glyph_max_y + 1 + PADDING, box.max_y)
+    );
+}
+
 // ---------------------------------------------------------------------------
 // Main function
 // ---------------------------------------------------------------------------
 int read_digits_waterfill_template(
     Logger& logger,
     const ImageViewRGB32& stat_region,
-    double rmsd_threshold,
     DigitTemplateType template_type,
+    double rmsd_threshold,
     const std::string& dump_prefix,
     uint8_t binarize_high,
     bool save_debug_images
@@ -197,8 +252,11 @@ int read_digits_waterfill_template(
             size_t min_x = obj.min_x + i * split_w;
             size_t max_x = (i == expected_digits - 1) ? obj.max_x : obj.min_x + (i + 1) * split_w;
 
-            // Crop original (unblurred) region to the split bounding box.
-            ImagePixelBox bbox(min_x, obj.min_y, max_x, obj.max_y);
+            // Crop the unblurred region to the split bounding box,
+            // shrink to the glyph so a captured drop shadow doesn't cause an offset
+            ImagePixelBox bbox = tighten_to_glyph(
+                stat_region, ImagePixelBox(min_x, obj.min_y, max_x, obj.max_y)
+            );
             ImageViewRGB32 crop = extract_box_reference(stat_region, bbox);
 
             if (save_debug_images && dump_prefix == "levelDigit"){

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_DigitReader.h

@@ -27,6 +27,7 @@ namespace PokemonFRLG{
 enum class DigitTemplateType{
     StatBox,      // Yellow stat boxes (default): PokemonFRLG/Digits/
     LevelBox,     // Lilac level box: PokemonFRLG/LevelDigits/
+    DialogBox,    // White dialog box: PokemonFRLG/DialogDigits/
 };
 
 // Read a string of decimal digits from `stat_region`.
@@ -38,8 +39,8 @@ enum class DigitTemplateType{
 int read_digits_waterfill_template(
     Logger& logger,
     const ImageViewRGB32& stat_region,
-    double rmsd_threshold = 175.0,
     DigitTemplateType template_type = DigitTemplateType::StatBox,
+    double rmsd_threshold = 175.0,
     const std::string& dump_prefix = "digit",
     uint8_t binarize_high = 0xBE,  // 0xBE=190 for yellow stat boxes;
     bool save_debug_images = false

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_PartyLevelUpReader.cpp

@@ -59,7 +59,7 @@ StatReads PartyLevelUpReader::read_stats(Logger &logger, const ImageViewRGB32& f
 
     auto read_stat = [&](const ImageFloatBox &box, const std::string &name){
         ImageViewRGB32 stat_region = extract_box_reference(game_screen, box);
-        return read_digits_waterfill_template(logger, stat_region);
+        return read_digits_waterfill_template(logger, stat_region, DigitTemplateType::DialogBox);
     };
 
     StatReads stats;
@@ -105,7 +105,20 @@ class Test_PartyLevelUpReader : public UnitTest{
 
 
 void add_tests_PartyLevelUpReader(UnitTestDatabase& database){
-    // to do
+        database.add<Test_PartyLevelUpReader>("PokemonFRLG/PartyLevelUpReader/raikou_1.png");
+        database.add<Test_PartyLevelUpReader>("PokemonFRLG/PartyLevelUpReader/raikou_2.png");
+        database.add<Test_PartyLevelUpReader>("PokemonFRLG/PartyLevelUpReader/raikou_3.png");
+        database.add<Test_PartyLevelUpReader>("PokemonFRLG/PartyLevelUpReader/raikou_4.png");
+        database.add<Test_PartyLevelUpReader>("PokemonFRLG/PartyLevelUpReader/raikou_5.png");
+        database.add<Test_PartyLevelUpReader>("PokemonFRLG/PartyLevelUpReader/raikou_6.png");
+        database.add<Test_PartyLevelUpReader>("PokemonFRLG/PartyLevelUpReader/deoxys_1.png");
+        database.add<Test_PartyLevelUpReader>("PokemonFRLG/PartyLevelUpReader/deoxys_2.png");
+        database.add<Test_PartyLevelUpReader>("PokemonFRLG/PartyLevelUpReader/deoxys_3.png");
+        database.add<Test_PartyLevelUpReader>("PokemonFRLG/PartyLevelUpReader/dratini_1.png");
+        database.add<Test_PartyLevelUpReader>("PokemonFRLG/PartyLevelUpReader/dratini_2.png");
+        database.add<Test_PartyLevelUpReader>("PokemonFRLG/PartyLevelUpReader/dratini_3.png");
+        database.add<Test_PartyLevelUpReader>("PokemonFRLG/PartyLevelUpReader/dratini_4.png");
+        database.add<Test_PartyLevelUpReader>("PokemonFRLG/PartyLevelUpReader/dratini_5.png");
 }
 
 

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_StatsReader.cpp

@@ -35,7 +35,7 @@ namespace PokemonFRLG {
 StatsReader::StatsReader(Color color)
     : m_color(color), 
     m_box_nature(0.028976, 0.729610, 0.502487, 0.066639),
-    m_box_level(0.052000, 0.120140, 0.099000, 0.069416),
+    m_box_level(0.058333, 0.120140, 0.092308, 0.069416),
     m_box_name(0.163158, 0.122917, 0.262811, 0.066639),
     m_box_gender(0.430769, 0.114423, 0.034615, 0.081731),
     m_box_hp(0.805274, 0.131247, 0.183790, 0.066639),
@@ -53,7 +53,8 @@ StatsReader::StatsReader(Color color)
     m_box_defense_jpn(0.859615, 0.323792, 0.121795, 0.068269),
     m_box_sp_attack_jpn(0.859615, 0.404315, 0.121795, 0.068269),
     m_box_sp_defense_jpn(0.859615, 0.484838, 0.121795, 0.068269),
-    m_box_speed_jpn(0.859615, 0.565361, 0.121795, 0.068269)   
+    m_box_speed_jpn(0.859615, 0.565361, 0.121795, 0.068269),
+    m_box_level_spa(0.062179, 0.120140, 0.088462, 0.069416)
 {}
 
 void StatsReader::make_overlays(VideoOverlaySet &items) const {
@@ -78,6 +79,7 @@ void StatsReader::make_overlays(VideoOverlaySet &items) const {
     items.add(m_color, GAME_BOX.inner_to_outer(m_box_sp_attack_jpn));
     items.add(m_color, GAME_BOX.inner_to_outer(m_box_sp_defense_jpn));
     items.add(m_color, GAME_BOX.inner_to_outer(m_box_speed_jpn));
+    items.add(m_color, GAME_BOX.inner_to_outer(m_box_level_spa));
 }
 
 bool StatsReader::read_name(
@@ -158,9 +160,11 @@ bool StatsReader::read_level(
     const std::set<std::string>& subset,
     bool save_debug_images
 ){
-    const bool jpn = language == Language::Japanese;
-
-    ImageViewRGB32 level_box = extract_box_reference(game_screen, jpn ? m_box_level_jpn : m_box_level);
+    ImageViewRGB32 level_box = extract_box_reference(game_screen, 
+        language == Language::Japanese ? m_box_level_jpn :
+        language == Language::Spanish  ? m_box_level_spa :
+                                         m_box_level
+    );
 
     // The level uses white text with dark shadow on a lilac background.
     // The digit reader's binarizer captures dark pixels (<=190 on all channels)
@@ -182,26 +186,16 @@ bool StatsReader::read_level(
             }
         }
     }
-    // Trim left 7% to exclude the "L" glyph blob (always at x~0).
-    // The actual level digits start at ~13%+ of the box width.
-    size_t lv_skip = preprocessed.width() * 7 / 100;
-    ImagePixelBox digits_bbox(
-        lv_skip, 0, preprocessed.width(),
-        preprocessed.height()
-    );
-    ImageViewRGB32 level_digit_view =
-            extract_box_reference(preprocessed, digits_bbox);
     // Use threshold 230 (not 175): lilac-background blob crops inherently
     // give higher RMSD than yellow stat-box crops due to background colour.
     stats.level = read_digits_waterfill_template(
-            logger, level_digit_view, 230.0, DigitTemplateType::LevelBox,
+            logger, preprocessed, DigitTemplateType::LevelBox, 230,
             "levelDigit", 0x7F);
     // log if it's an obviously bad read
     if (!stats.level.has_value() || stats.level.value_or(-1) < 2 || stats.level.value_or(-1) > 100){
         logger.log("Level OCR result out of range", COLOR_RED);
         if (save_debug_images){
             preprocessed.save("DebugDumps/ocr_level_preprocessed.png");
-            level_digit_view.save("DebugDumps/ocr_level_digits_trimmed.png");
         }
         return false;
     }
@@ -307,7 +301,7 @@ void StatsReader::read_page2(
         ImageViewRGB32 stat_region = extract_box_reference(game_screen, box);
 
         // waterfill segmentation + template matching against the PokemonFRLG/Digits/0-9.png templates.
-        int stat = read_digits_waterfill_template(logger, stat_region);
+        int stat = read_digits_waterfill_template(logger, stat_region, DigitTemplateType::StatBox);
         // log impossible values or failed reads
         if (name != "hp" && (stat < 1 || stat > 614)){ // 614 comes from a max Def Shuckle
             logger.log("OCR result for " + name + " out of range: " + std::to_string(stat), COLOR_RED);
@@ -533,6 +527,7 @@ void add_tests_StatsReader(UnitTestDatabase& database){
     database.add<Test_StatsReaderPage2>("PokemonFRLG/StatsReader/Page2/mewtwo_1_spa.png");
     database.add<Test_StatsReaderPage2>("PokemonFRLG/StatsReader/Page2/nidoranf_1_eng.jpg");
     database.add<Test_StatsReaderPage2>("PokemonFRLG/StatsReader/Page2/raikou_1_eng.png");
+    database.add<Test_StatsReaderPage2>("PokemonFRLG/StatsReader/Page2/raikou_2_eng.png");
     database.add<Test_StatsReaderPage2>("PokemonFRLG/StatsReader/Page2/venonat_1_eng.jpg");
 }
 

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_StatsReader.h

@@ -85,6 +85,10 @@ class StatsReader {
     ImageFloatBox m_box_sp_attack_jpn;
     ImageFloatBox m_box_sp_defense_jpn;
     ImageFloatBox m_box_speed_jpn;
+    //  Spanish labels the level "Nv." where the other Latin-script languages
+    //  use "Lv". The trailing period pushes the digits ~6px right (at 1080p),
+    //  so Spanish needs its own left edge. Right edge matches m_box_level.
+    ImageFloatBox m_box_level_spa;
 
     // return false if the read failed
 

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_TrainerIdReader.cpp

@@ -32,12 +32,15 @@ namespace PokemonFRLG {
 TrainerIdReader::TrainerIdReader(Color color)
     : m_color(color)
     , m_box_tid(0.742683, 0.117314, 0.129734, 0.076006)
+    , m_box_tid_spa(0.766667, 0.112286, 0.129345, 0.076068)
     , m_box_tid_jpn(0.712981, 0.118836, 0.207212, 0.077373)
 {}
 
 void TrainerIdReader::make_overlays(VideoOverlaySet &items) const {
     const BoxOption &GAME_BOX = GameSettings::instance().GAME_BOX;
     items.add(m_color, GAME_BOX.inner_to_outer(m_box_tid));
+    items.add(m_color, GAME_BOX.inner_to_outer(m_box_tid_spa));
+    items.add(m_color, GAME_BOX.inner_to_outer(m_box_tid_jpn));
 }
 
 uint16_t TrainerIdReader::read_tid(
@@ -47,11 +50,13 @@ uint16_t TrainerIdReader::read_tid(
             extract_box_reference(frame, GameSettings::instance().GAME_BOX);
 
 
-    ImageViewRGB32 tid_region = extract_box_reference(game_screen, language == Language::Japanese ? m_box_tid_jpn : m_box_tid);
+    ImageViewRGB32 tid_region = extract_box_reference(game_screen, language == Language::Japanese ? m_box_tid_jpn :
+                                                                   language == Language::Spanish  ? m_box_tid_spa :
+                                                                   m_box_tid);
 
     // waterfill segmentation + template matching
     // against the PokemonFRLG/Digits/0-9.png templates.
-    return uint16_t(read_digits_waterfill_template(logger, tid_region));
+    return uint16_t(read_digits_waterfill_template(logger, tid_region, DigitTemplateType::DialogBox));
 }
 
 
@@ -95,6 +100,7 @@ class Test_TrainerIdReader : public UnitTest{
 void add_tests_TrainerIdReader(UnitTestDatabase& database){
     database.add<Test_TrainerIdReader>("PokemonFRLG/TrainerIdReader/tom_eng_60895.jpg");
     database.add<Test_TrainerIdReader>("PokemonFRLG/TrainerIdReader/nyash_jpn_45345.png");
+    database.add<Test_TrainerIdReader>("PokemonFRLG/TrainerIdReader/alberto_spa_65385.png");
 }
 
 

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_TrainerIdReader.h

@@ -38,6 +38,7 @@ class TrainerIdReader {
 private:
     Color m_color;
     ImageFloatBox m_box_tid;
+    ImageFloatBox m_box_tid_spa;
     ImageFloatBox m_box_tid_jpn;
 };
 

SerialPrograms/Source/PokemonFRLG/Programs/TestPrograms/PokemonFRLG_ReadTrainerId.cpp

@@ -65,18 +65,23 @@ void ReadTrainerId::program(
     TrainerCardDetector detector(COLOR_RED);
     TrainerIdReader reader;
     VideoOverlaySet overlays(env.console.overlay());
+    detector.make_overlays(overlays);
     reader.make_overlays(overlays);
 
     VideoSnapshot screen = env.console.video().snapshot();
     bool trainercard = detector.detect(screen);
     if (trainercard){
-        env.log("Trainer Card detected.");
-        env.log("Reading TID...");
-        uint16_t tid = reader.read_tid(env.logger(), LANGUAGE, screen);
-        env.log("TID: " + std::to_string(tid));
+        env.log("Trainer Card detected.", COLOR_BLUE);
     }else{
-        env.log("Trainer Card not detected!");
+        env.log("Trainer Card not detected!", COLOR_RED);
     }
+    env.log("Reading TID...");
+    uint16_t tid = reader.read_tid(env.logger(), LANGUAGE, screen);
+    env.log("TID: " + std::to_string(tid));
+
+    env.log("Finished Reading TID. Verification boxes are on overlay.",
+                    COLOR_BLUE);
+    scope.wait_for(10s);
 }
 
 } // namespace PokemonFRLG