Skip to content

Commit eef0871

Browse files
authored
A few fixes for FRLG RNG programs (#1282)
* switch to is_grey for distinguishing dark/light colors * fix color-related issues * fix 0 rare candies calibration issue * multi-filter white text detection for first step of name OCR * handle empty results * also handle if all results are empty * add negative background detection * better calibration with recoverable errors
1 parent 669fe77 commit eef0871

8 files changed

Lines changed: 150 additions & 81 deletions

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_BattlePokemonDetector.cpp

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,11 @@ namespace PokemonFRLG{
2525

2626
BattlePokemonDetector::BattlePokemonDetector(Color color)
2727
// be warned: the name/level/hp box moves up and down a little bit
28-
: m_left_box(0.577404, 0.504327, 0.001442, 0.121875) // off-white (255, 255, 236), can be interrupted by a status condition
29-
, m_right_box(0.948558, 0.585096, 0.000481, 0.063462) // dark teal (74, 111, 102)
28+
: m_left_box(0.577404, 0.504327, 0.001442, 0.121875) // off-white rgb(255, 255, 235), can be interrupted by a status condition
29+
, m_right_box(0.948558, 0.585096, 0.000481, 0.063462) // dark teal rgb(72, 106, 98)
3030
, m_top_box(0.594712, 0.481971, 0.325962, 0.002163) // off-white, movement makes this unreliable
3131
, m_bottom_box(0.554808, 0.674519, 0.034615, 0.002163) // dark teal
32+
, m_background_box(0.552564, 0.583653, 0.414103, 0.009615) // depends on in-game location. white or rgb(215, 169, 96)
3233
{}
3334
void BattlePokemonDetector::make_overlays(VideoOverlaySet& items) const{
3435
const BoxOption& GAME_BOX = GameSettings::instance().GAME_BOX;
@@ -44,21 +45,25 @@ bool BattlePokemonDetector::detect(const ImageViewRGB32& screen){
4445
ImageViewRGB32 right_image = extract_box_reference(game_screen, m_right_box);
4546
ImageViewRGB32 top_image = extract_box_reference(game_screen, m_top_box);
4647
ImageViewRGB32 bottom_image = extract_box_reference(game_screen, m_bottom_box);
47-
if (is_solid(left_image, { 0.3418, 0.3418, 0.3164 })
48-
&& is_solid(right_image, { 0.2578, 0.3868, 0.3554 }, 0.075, 5)
49-
&& is_solid(top_image, { 0.3418, 0.3418, 0.3164 })
50-
&& is_solid(bottom_image, { 0.2578, 0.3868, 0.3554 }, 0.075, 5)
48+
ImageViewRGB32 background_image = extract_box_reference(game_screen, m_background_box);
49+
50+
if (is_grey(left_image, 680, 770, 20)
51+
&& is_grey(right_image, 220, 320, 20)
52+
&& is_grey(top_image, 680, 770, 20)
53+
&& is_grey(bottom_image, 220, 320, 20)
54+
&& !(is_white(background_image) || is_solid(background_image, {0.4468, 0.3528, 0.2004}, 0.25, 20))
5155
){
5256
return true;
5357
}
5458
return false;
5559
}
5660

5761
BattleOpponentDetector::BattleOpponentDetector(Color color)
58-
: m_left_box(0.067308, 0.140865, 0.001442, 0.090144) // off-white (255, 255, 236)
62+
: m_left_box(0.067308, 0.140865, 0.001442, 0.090144) // off-white rgb(255, 255, 235)
5963
, m_right_box(0.422596, 0.132211, 0.000481, 0.054808) // off-white
6064
, m_top_box(0.085577, 0.119952, 0.329327, 0.000721) // off-white
61-
, m_bottom_box(0.109615, 0.264182, 0.328365, 0.001442) // dark teal (74, 111, 102)
65+
, m_bottom_box(0.109615, 0.264182, 0.328365, 0.001442) // dark teal rgb(72, 106, 98)
66+
, m_background_box(0.050641, 0.126923, 0.388462, 0.063462) // depends on in-game location. white or rgb(187, 149, 65)
6267
{}
6368
void BattleOpponentDetector::make_overlays(VideoOverlaySet& items) const{
6469
const BoxOption& GAME_BOX = GameSettings::instance().GAME_BOX;
@@ -74,11 +79,13 @@ bool BattleOpponentDetector::detect(const ImageViewRGB32& screen){
7479
ImageViewRGB32 right_image = extract_box_reference(game_screen, m_right_box);
7580
ImageViewRGB32 top_image = extract_box_reference(game_screen, m_top_box);
7681
ImageViewRGB32 bottom_image = extract_box_reference(game_screen, m_bottom_box);
82+
ImageViewRGB32 background_image = extract_box_reference(game_screen, m_background_box);
7783

78-
if (is_solid(left_image, { 0.3418, 0.3418, 0.3164 })
79-
&& is_solid(right_image, { 0.3418, 0.3418, 0.3164 })
80-
&& is_solid(top_image, { 0.3418, 0.3418, 0.3164 })
81-
&& is_solid(bottom_image, { 0.2578, 0.3868, 0.3554 }, 0.075, 5)
84+
if (is_grey(left_image, 680, 770, 20)
85+
&& is_grey(right_image, 680, 770, 20)
86+
&& is_grey(top_image, 680, 770, 20)
87+
&& is_grey(bottom_image, 220, 320, 20)
88+
&& !(is_white(background_image) || is_solid(background_image, {0.4663, 0.3716, 0.1621}, 0.25, 20))
8289
){
8390
return true;
8491
}

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_BattlePokemonDetector.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ class BattlePokemonDetector : public StaticScreenDetector{
3333
ImageFloatBox m_right_box;
3434
ImageFloatBox m_top_box;
3535
ImageFloatBox m_bottom_box;
36+
ImageFloatBox m_background_box;
3637
};
3738

3839
// Watches for the player's Pokemon to disappear
@@ -56,6 +57,7 @@ class BattleOpponentDetector : public StaticScreenDetector{
5657
ImageFloatBox m_right_box;
5758
ImageFloatBox m_top_box;
5859
ImageFloatBox m_bottom_box;
60+
ImageFloatBox m_background_box;
5961
};
6062

6163
// Watches for the opponent to disappear

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_StatsReader.cpp

Lines changed: 93 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -88,38 +88,58 @@ void StatsReader::read_page1(
8888

8989
ImageViewRGB32 name_box = extract_box_reference(game_screen, jpn ? m_box_name_jpn : m_box_name);
9090

91-
// remove shadow
92-
ImageRGB32 name_filtered = filter_rgb32_range(
93-
name_box, 0xff000000, 0xffc7c7c7, Color(0xffc3c3c3), true
94-
);
95-
// make text black
96-
name_filtered = filter_rgb32_range(
97-
name_filtered, 0xffc8c8c8, 0xffffffff, Color(0xff000000), true
98-
);
99-
100-
ImageRGB32 name_ready = preprocess_for_ocr(
101-
name_filtered, "name", 7, 2, true,
102-
combine_rgb(0, 0, 0), jpn ? combine_rgb(160, 160, 160) : combine_rgb(120, 120, 120)
103-
);
104-
105-
const std::vector<OCR::TextColorRange> name_text_color_ranges{
106-
{combine_rgb(0, 0, 0), combine_rgb(120, 120, 120)}
107-
};
10891

109-
if (subset.size() > 0){
110-
auto name_result = Pokemon::PokemonNameReader(subset).read_substring(
111-
logger, language, name_ready, name_text_color_ranges
112-
);
113-
if (!name_result.results.empty()){
114-
stats.name = name_result.results.begin()->second.token;
92+
static const std::vector<int> WHITE_THRESHOLDS = { 180, 200, 220, 230, 240 };
93+
OCR::StringMatchResult best_result;
94+
bool initialized = false;
95+
for (int thresh : WHITE_THRESHOLDS){
96+
ImageRGB32 name_filtered(name_box.width(), name_box.height());
97+
for (size_t r = 0; r < name_box.height(); r++){
98+
for (size_t c = 0; c < name_box.width(); c++){
99+
Color pixel(name_box.pixel(c, r));
100+
if (pixel.red() > thresh && pixel.green() > thresh && pixel.blue() > thresh){
101+
name_filtered.pixel(c, r) = (uint32_t)0xff000000; // Black
102+
}else{
103+
name_filtered.pixel(c, r) = (uint32_t)0xffffffff; // White
104+
}
105+
}
115106
}
116-
}else{
117-
auto name_result = Pokemon::PokemonNameReader::instance().read_substring(
118-
logger, language, name_ready, name_text_color_ranges
107+
ImageRGB32 name_ready = preprocess_for_ocr(
108+
name_filtered, "name", 7, 2, true,
109+
combine_rgb(0, 0, 0), jpn ? combine_rgb(160, 160, 160) : combine_rgb(140, 140, 140)
119110
);
120-
if (!name_result.results.empty()){
121-
stats.name = name_result.results.begin()->second.token;
111+
112+
const std::vector<OCR::TextColorRange> name_text_color_ranges{
113+
{combine_rgb(0, 0, 0), combine_rgb(120, 120, 120)}
114+
};
115+
116+
OCR::StringMatchResult result;
117+
if (subset.size() > 0){
118+
auto name_result = Pokemon::PokemonNameReader(subset).read_substring(
119+
logger, language, name_ready, name_text_color_ranges
120+
);
121+
if (!name_result.results.empty()){
122+
result = name_result;
123+
}
124+
}else{
125+
auto name_result = Pokemon::PokemonNameReader::instance().read_substring(
126+
logger, language, name_ready, name_text_color_ranges
127+
);
128+
if (!name_result.results.empty()){
129+
result = name_result;
130+
}
122131
}
132+
if (!result.results.empty()){
133+
if (!initialized){
134+
best_result = result;
135+
initialized = true;
136+
}else if (result.results.begin()->first < best_result.results.begin()->first){
137+
best_result = result;
138+
}
139+
}
140+
}
141+
if (initialized && !best_result.results.empty()){
142+
stats.name = best_result.results.begin()->second.token;
123143
}
124144

125145
// Detect gender by comparing red vs blue pixels
@@ -148,52 +168,29 @@ void StatsReader::read_page1(
148168
stats.gender = SummaryGender::Genderless;
149169
}
150170

151-
152-
153171
ImageViewRGB32 level_box = extract_box_reference(game_screen, jpn ? m_box_level_jpn : m_box_level);
154172

155-
ImageRGB32 level_upscaled =
156-
level_box.scale_to(level_box.width() * 4, level_box.height() * 4);
157-
if (save_debug_images){
158-
level_upscaled.save("DebugDumps/ocr_level_upscaled.png");
159-
}
160-
161-
// The level has a colored (lilac) background. The text is white, with a
162-
// gray/black shadow. To bridge the gaps and make a solid black character on a
163-
// white background: We want to turn BOTH the bright white text AND the dark
164-
// shadow into BLACK pixels, and turn the mid-tone lilac background into
165-
// WHITE. We can do this by keeping pixels that are very bright (text) or very
166-
// dark (shadow).
167-
168-
ImageRGB32 level_ready(level_upscaled.width(), level_upscaled.height());
169-
for (size_t r = 0; r < level_upscaled.height(); r++){
170-
for (size_t c = 0; c < level_upscaled.width(); c++){
171-
Color pixel(level_upscaled.pixel(c, r));
172-
// If it's very bright (white text) OR very dark (shadow), it becomes
173-
// black text. Otherwise (lilac background), it becomes white background.
174-
if ((pixel.red() > 200 && pixel.green() > 200 && pixel.blue() > 200) ||
175-
(pixel.red() < 100 && pixel.green() < 100 && pixel.blue() < 100)){
176-
level_ready.pixel(c, r) = (uint32_t)0xff000000; // Black
177-
}else{
178-
level_ready.pixel(c, r) = (uint32_t)0xffffffff; // White
179-
}
180-
}
181-
}
182-
183-
if (save_debug_images){
184-
level_ready.save("DebugDumps/ocr_level_ready.png");
185-
}
186-
187173
if (!GlobalSettings::instance().USE_PADDLE_OCR){
188174
// The level uses white text with dark shadow on a lilac background.
189175
// The digit reader's binarizer captures dark pixels (<=190 on all channels)
190176
// but NOT the white text (all channels 255 -> excluded). This leaves the
191177
// shadow outline fragmented into many small disconnected blobs.
192178
// Preprocess: convert bright-white text pixels to black so the binarizer
193179
// merges text + shadow into one solid connected blob per digit.
194-
ImageRGB32 preprocessed = filter_rgb32_range(
195-
level_box, 0xffc8c8c8, 0xffffffff, Color(0xff000000), true
196-
);
180+
ImageRGB32 preprocessed = level_box.scale_to(level_box.width(), level_box.height());
181+
for (size_t r = 0; r < level_box.height(); r++){
182+
for (size_t c = 0; c < level_box.width(); c++){
183+
Color pixel(level_box.pixel(c, r));
184+
// Try to detect lilac background first based on low green channel,
185+
// replacing it with a darker lilac color (for matching the template)
186+
// For other pixels, if it's bright it becomes black text.
187+
if ((pixel.blue() > pixel.green() + 25) && (pixel.red() > pixel.green() + 15)){
188+
preprocessed.pixel(c, r) = (uint32_t)0xffd1b0f0; // from template
189+
}else if (pixel.red() > 200 && pixel.green() > 200 && pixel.blue() > 200){
190+
preprocessed.pixel(c, r) = (uint32_t)0xff000000; // Black
191+
}
192+
}
193+
}
197194
if (save_debug_images){
198195
preprocessed.save("DebugDumps/ocr_level_preprocessed.png");
199196
}
@@ -215,6 +212,37 @@ void StatsReader::read_page1(
215212
logger, level_digit_view, 230.0, DigitTemplateType::LevelBox,
216213
"levelDigit", 0x7F);
217214
}else{
215+
// The level has a colored (lilac) background. The text is white, with a
216+
// gray/black shadow. To bridge the gaps and make a solid black character on a
217+
// white background: We want to turn BOTH the bright white text AND the dark
218+
// shadow into BLACK pixels, and turn the mid-tone lilac background into
219+
// WHITE. We can do this by keeping pixels that are very bright (text) or very
220+
// dark (shadow).
221+
ImageRGB32 level_upscaled =
222+
level_box.scale_to(level_box.width() * 4, level_box.height() * 4);
223+
if (save_debug_images){
224+
level_upscaled.save("DebugDumps/ocr_level_upscaled.png");
225+
}
226+
ImageRGB32 level_ready(level_upscaled.width(), level_upscaled.height());
227+
for (size_t r = 0; r < level_upscaled.height(); r++){
228+
for (size_t c = 0; c < level_upscaled.width(); c++){
229+
Color pixel(level_upscaled.pixel(c, r));
230+
// Try to detect lilac background first based on low green channel.
231+
// For other pixels, if it's very bright (white text) OR very dark (shadow),
232+
// it becomes black text. Otherwise, it becomes white background.
233+
if ((pixel.blue() > pixel.green() + 25) && (pixel.red() > pixel.green() + 15)){
234+
level_ready.pixel(c, r) = (uint32_t)0xffffffff; // White
235+
}else if ((pixel.red() > 200 && pixel.green() > 200 && pixel.blue() > 200) ||
236+
(pixel.red() < 100 && pixel.green() < 100 && pixel.blue() < 100)){
237+
level_ready.pixel(c, r) = (uint32_t)0xff000000; // Black
238+
}else{
239+
level_ready.pixel(c, r) = (uint32_t)0xffffffff; // White
240+
}
241+
}
242+
}
243+
if (save_debug_images){
244+
level_ready.save("DebugDumps/ocr_level_ready.png");
245+
}
218246
// Pass the binarized image to PaddleOCR
219247
stats.level = OCR::read_number(logger, level_ready, language);
220248
}

SerialPrograms/Source/PokemonFRLG/Programs/RngManipulation/PokemonFRLG_GiftRng.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -463,7 +463,7 @@ void GiftRng::program(SingleSwitchProgramEnvironment& env, ProControllerContext&
463463
RNG_CALIBRATION.set_hits(search_hits);
464464
bool finished = update_history(
465465
env.console, uncertain_history, calibration_history, MAX_HISTORY_LENGTH,
466-
calibrations, search_hits, 1
466+
calibrations, search_hits, 1, 2, MAX_RARE_CANDIES == 0
467467
);
468468

469469
for (uint64_t i=0; i<MAX_RARE_CANDIES; i++){
@@ -473,6 +473,10 @@ void GiftRng::program(SingleSwitchProgramEnvironment& env, ProControllerContext&
473473

474474
bool failed = use_rare_candy(env.console, context, LANGUAGE, pokemon, filters, BASE_STATS, AdvRngMethod::Method1, false, i == 0);
475475
if (failed){
476+
update_history(
477+
env.console, uncertain_history, calibration_history,
478+
MAX_HISTORY_LENGTH, calibrations, search_hits, 1, 2, true
479+
);
476480
stats.errors++;
477481
send_program_recoverable_error_notification(
478482
env, NOTIFICATION_ERROR_RECOVERABLE,

SerialPrograms/Source/PokemonFRLG/Programs/RngManipulation/PokemonFRLG_RoamingLegendaryRng.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -434,7 +434,7 @@ void RoamingLegendaryRng::program(SingleSwitchProgramEnvironment& env, ProContro
434434
RNG_CALIBRATION.set_hits(search_hits);
435435
bool finished = update_history(
436436
env.console, uncertain_history, calibration_history, MAX_HISTORY_LENGTH,
437-
calibrations, search_hits, 1
437+
calibrations, search_hits, 1, 2, MAX_RARE_CANDIES == 0
438438
);
439439
finished = finished || all_indistinguishable(search_hits, searcher, GENDER_THRESHOLD);
440440

@@ -445,6 +445,10 @@ void RoamingLegendaryRng::program(SingleSwitchProgramEnvironment& env, ProContro
445445

446446
bool failed = use_rare_candy(env.console, context, LANGUAGE, pokemon, filters, BASE_STATS, AdvRngMethod::Method1, false, i == 0);
447447
if (failed){
448+
update_history(
449+
env.console, uncertain_history, calibration_history,
450+
MAX_HISTORY_LENGTH, calibrations, search_hits, 1, 2, true
451+
);
448452
stats.errors++;
449453
send_program_recoverable_error_notification(
450454
env, NOTIFICATION_ERROR_RECOVERABLE,

SerialPrograms/Source/PokemonFRLG/Programs/RngManipulation/PokemonFRLG_StarterRng.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -756,12 +756,20 @@ void StarterRng::program(SingleSwitchProgramEnvironment& env, ProControllerConte
756756
// Stage 2: first search update -- post-rival-battle
757757
bool failed = walk_to_rival_battle(env, context);
758758
if (failed){
759+
update_history(
760+
env.console, uncertain_history, calibration_history,
761+
MAX_HISTORY_LENGTH, calibrations, search_hits, 1, 2, true
762+
);
759763
stats.errors++;
760764
continue; // reset game
761765
}
762766

763767
failed = auto_battle_rival(env, context, pokemon, filters, BASE_STATS);
764768
if (failed){
769+
update_history(
770+
env.console, uncertain_history, calibration_history,
771+
MAX_HISTORY_LENGTH, calibrations, search_hits, 1, 2, true
772+
);
765773
stats.errors++;
766774
continue; // reset game
767775
}
@@ -787,6 +795,10 @@ void StarterRng::program(SingleSwitchProgramEnvironment& env, ProControllerConte
787795
// Stage 3: subsequent search updates -- leveling up from wild encounters
788796
failed = walk_to_route1_from_lab(env, context);
789797
if (failed){
798+
update_history(
799+
env.console, uncertain_history, calibration_history,
800+
MAX_HISTORY_LENGTH, calibrations, search_hits, 1, 2, true
801+
);
790802
stats.errors++;
791803
continue; // reset game
792804
}
@@ -808,6 +820,10 @@ void StarterRng::program(SingleSwitchProgramEnvironment& env, ProControllerConte
808820

809821
int ret2 = autolevel_on_route1(env, context, pokemon, filters, BASE_STATS);
810822
if (ret2 < 0){
823+
update_history(
824+
env.console, uncertain_history, calibration_history,
825+
MAX_HISTORY_LENGTH, calibrations, search_hits, 1, 2, true
826+
);
811827
stats.errors++;
812828
break;
813829
}else if(ret2 == 1){

SerialPrograms/Source/PokemonFRLG/Programs/RngManipulation/PokemonFRLG_StaticRng.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -457,7 +457,7 @@ void StaticRng::program(SingleSwitchProgramEnvironment& env, ProControllerContex
457457
RNG_CALIBRATION.set_hits(search_hits);
458458
bool finished = update_history(
459459
env.console, uncertain_history, calibration_history, MAX_HISTORY_LENGTH,
460-
calibrations, search_hits, 1
460+
calibrations, search_hits, 1, 2, MAX_RARE_CANDIES == 0
461461
);
462462

463463
for (uint64_t i=0; i<MAX_RARE_CANDIES; i++){
@@ -466,6 +466,10 @@ void StaticRng::program(SingleSwitchProgramEnvironment& env, ProControllerContex
466466
}
467467
bool failed = use_rare_candy(env.console, context, LANGUAGE, pokemon, filters, BASE_STATS, AdvRngMethod::Method1, false, i == 0);
468468
if (failed) {
469+
update_history(
470+
env.console, uncertain_history, calibration_history,
471+
MAX_HISTORY_LENGTH, calibrations, search_hits, 1, 2, true
472+
);
469473
stats.errors++;
470474
send_program_recoverable_error_notification(
471475
env, NOTIFICATION_ERROR_RECOVERABLE,

0 commit comments

Comments
 (0)