diff --git a/SerialPrograms/Source/Pokemon/Inference/Pokemon_TypeReader.cpp b/SerialPrograms/Source/Pokemon/Inference/Pokemon_TypeReader.cpp new file mode 100644 index 0000000000..eb4e0b322a --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Inference/Pokemon_TypeReader.cpp @@ -0,0 +1,426 @@ +/* Type Reader + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "Common/Cpp/CancellableScope.h" +#include "CommonFramework/Globals.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/ImageMatch/ExactImageMatcher.h" +#include "CommonTools/Images/ImageFilter.h" +#include "CommonTools/Images/BinaryImage_FilterRgb32.h" +#include "Kernels/Waterfill/Kernels_Waterfill.h" +#include "Pokemon_TypeReader.h" + +#include + +namespace PokemonAutomation { +namespace Pokemon { + +using namespace Kernels; +using namespace Kernels::Waterfill; + + +class TypeSprite { +public: + TypeSprite(const std::string& slug, PokemonTypeGeneration generation) + : m_slug(slug) + { + ImageRGB32 sprite; + switch (generation) + { + case PokemonAutomation::Pokemon::PokemonTypeGeneration::GEN8: + sprite = ImageRGB32(RESOURCE_PATH() + "Pokemon/Types/Gen8/" + slug + ".png"); + break; + case PokemonAutomation::Pokemon::PokemonTypeGeneration::GEN9: + sprite = ImageRGB32(RESOURCE_PATH() + "Pokemon/Types/Gen9/" + slug + ".png"); + break; + default: + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Invalid enum."); + } + + sprite = filter_rgb32_range( + sprite, + 0xff000000, 0xffffffff, + Color(0), false + ); + + PackedBinaryMatrix matrix = compress_rgb32_to_binary_min(sprite, 224, 224, 224); + std::vector objects = find_objects_inplace(matrix, 10); + + WaterfillObject object; + for (const WaterfillObject& item : objects) { + object.merge_assume_no_overlap(item); + } + + m_aspect_ratio = object.aspect_ratio(); + m_matcher = std::make_unique( + sprite.sub_image(object.min_x, object.min_y, object.width(), object.height()).copy(), + ImageMatch::WeightedExactImageMatcher::InverseStddevWeight{ 1, 64 } + ); + } + + const std::string& slug() const { return m_slug; } + double aspect_ratio() const { return m_aspect_ratio; } + const ImageMatch::WeightedExactImageMatcher& matcher() const { return *m_matcher; } + +private: + std::string m_slug; + double m_aspect_ratio; + std::unique_ptr m_matcher; +}; + +struct Gen8TypeSpriteDatabase { + std::map m_type_map; + + static Gen8TypeSpriteDatabase& instance() { + static Gen8TypeSpriteDatabase data; + return data; + } + + Gen8TypeSpriteDatabase() { + for (const auto& item : POKEMON_TYPE_SLUGS()) { + if (item.first == PokemonType::NONE) { + continue; + } + m_type_map.emplace(item.first, TypeSprite(item.second, PokemonTypeGeneration::GEN8)); + } + } +}; + +struct Gen9TypeSpriteDatabase { + std::map m_type_map; + + static Gen9TypeSpriteDatabase& instance() { + static Gen9TypeSpriteDatabase data; + return data; + } + + Gen9TypeSpriteDatabase() { + for (const auto& item : POKEMON_TYPE_SLUGS()) { + if (item.first == PokemonType::NONE) { + continue; + } + m_type_map.emplace(item.first, TypeSprite(item.second, PokemonTypeGeneration::GEN9)); + } + } +}; + +size_t distance_sqr(const ImagePixelBox& a, const ImagePixelBox& b) { + bool overlap_x = a.min_x <= b.max_x && b.min_x <= a.max_x; + bool overlap_y = a.min_y <= b.max_y && b.min_y <= a.max_y; + if (overlap_x && overlap_y) { + return 0; + } + + size_t dist_x = 0; + if (!overlap_x) { + dist_x = a.max_x < b.min_x + ? b.min_x - a.max_x + : a.min_x - b.max_x; + } + + size_t dist_y = 0; + if (!overlap_y) { + dist_y = a.max_y < b.min_y + ? b.min_y - a.max_y + : a.min_y - b.max_y; + } + + return dist_x * dist_x + dist_y * dist_y; +} + +std::pair match_type_symbol(const ImageViewRGB32& image, PokemonTypeGeneration generation) { + size_t width = image.width(); + size_t height = image.height(); + if (width * height < 100) { + return { 1.0, PokemonType::NONE }; + } + if (width > 2 * height) { + return { 1.0, PokemonType::NONE }; + } + if (height > 2 * width) { + return { 1.0, PokemonType::NONE }; + } + ImageStats stats = image_stats(image); + if (stats.stddev.sum() < 50) { + // if (print){ + // cout << "stats.stddev.sum() = " << stats.stddev.sum() << endl; + // } + return { 1.0, PokemonType::NONE }; + } + + double aspect_ratio = (double)width / height; + + // static int c = 0; + // image.save("test-" + std::to_string(threshold) + "-" + std::to_string(c++) + ".png"); + + // std::map rank; + + const std::map* type_sprite_map; + + switch (generation) { + case PokemonAutomation::Pokemon::PokemonTypeGeneration::GEN8: + type_sprite_map = &Gen8TypeSpriteDatabase::instance().m_type_map; + break; + case PokemonAutomation::Pokemon::PokemonTypeGeneration::GEN9: + type_sprite_map = &Gen9TypeSpriteDatabase::instance().m_type_map; + break; + default: + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Invalid enum."); + } + + double best_score = 0.45; + PokemonType best_type = PokemonType::NONE; + for (const auto& item : *type_sprite_map) { + // if (threshold != 700 || id != 55){ + // continue; + // } + + double expected_aspect_ratio = item.second.aspect_ratio(); + double ratio = aspect_ratio / expected_aspect_ratio; +#if 0 + if (print) { + cout << item.second.slug() + << " : expected = " << expected_aspect_ratio + << ", actual = " << aspect_ratio + << ", ratio = " << ratio << endl; + } +#endif + if (std::abs(ratio - 1) > 0.2) { + continue; + } + + double rmsd_alpha = item.second.matcher().diff(image); + + // item.second.matcher().m_image.save("sprite.png"); + // if (print){ + // cout << item.second.slug() << ": " << rmsd_alpha << endl; + // } + +#if 0 + // Handicap fairy due to white and pink being too similar in color and + // false positiving on the background. + if (item.first == PokemonType::FAIRY) { + rmsd_ratio *= 1.5; + } + + // Bonus for dark because or large contrast. + if (item.first == PokemonType::DARK) { + rmsd_ratio *= 0.8; + } +#endif + + if (best_score > rmsd_alpha) { + best_score = rmsd_alpha; + best_type = item.first; + // cout << item.second.slug() << ": " << stats.stddev << endl; + } + } + + // if (best_type != PokemonType::NONE){ + // cout << get_type_slug(best_type) << ": " << best_score << endl; + // } + return { best_score, best_type }; +} + +void find_type_symbol_candidates( + std::multimap>& candidates, + const ImageViewPlanar32& original_screen, + const ImageViewRGB32& image, + PackedBinaryMatrix& matrix, + double max_area_ratio, + PokemonTypeGeneration generation +){ + size_t max_area = (size_t)(image.width() * image.height() * max_area_ratio); + std::vector objects = find_objects_inplace( + matrix, + (size_t)(20. * original_screen.total_pixels() / (1920*1080)) + ); + +// static int index = 0; + + std::map objmap; + for (size_t c = 0; c < objects.size(); c++){ + if (objects[c].area > max_area){ + continue; + } + objmap[c] = objects[c]; + +#if 0 + extract_box_reference(image, ImagePixelBox(objects[c])).save("test-" + std::to_string(index++) + ".png"); +#endif + } + +// cout << "begin = " << objmap.size() << endl; + + // Merge nearby objects. + bool changed; + do{ + changed = false; + for (auto iter0 = objmap.begin(); iter0 != objmap.end(); ++iter0){ + for (auto iter1 = objmap.begin(); iter1 != objmap.end();){ + if (iter0->first >= iter1->first){ + ++iter1; + continue; + } + const WaterfillObject& obj0 = iter0->second; + const WaterfillObject& obj1 = iter1->second; + size_t distance = distance_sqr( + ImagePixelBox(obj0.min_x, obj0.min_y, obj0.max_x, obj0.max_y), + ImagePixelBox(obj1.min_x, obj1.min_y, obj1.max_x, obj1.max_y) + ); + if (distance < 5*5){ + iter0->second.merge_assume_no_overlap(iter1->second); + iter1 = objmap.erase(iter1); + changed = true; + }else{ + ++iter1; + } + } + } + }while (changed); + +// cout << "merged = " << objmap.size() << endl; + + // Identify objects. + for (const auto& item : objmap){ + ImageViewRGB32 img = extract_box_reference(image, item.second); + +// print = index == 137; +// img.save("test-" + std::to_string(index++) + ".png"); + + std::pair result = match_type_symbol(img, generation); +// cout << "result = " << POKEMON_TYPE_SLUGS().get_string(result.second) << ": " << result.first << endl; + if (result.second != PokemonType::NONE){ + const WaterfillObject& obj = item.second; + candidates.emplace( + result.first, + std::pair( + result.second, + ImagePixelBox(obj.min_x, obj.min_y, obj.max_x, obj.max_y) + ) + ); + } + } + +// cout << "candidates = " << candidates.size() << endl; +} + + +std::multimap> find_type_symbols( + const ImageViewPlanar32& original_screen, + const ImageViewRGB32& image, + double max_area_ratio, + PokemonTypeGeneration generation +) { + std::multimap> candidates; + + { + std::vector matrices = compress_rgb32_to_binary_range( + image, + { + {0xff808060, 0xffffffff}, + {0xffa0a060, 0xffffffff}, + {0xff606060, 0xffffffff}, + {0xff707070, 0xffffffff}, + {0xff808080, 0xffffffff}, + {0xff909090, 0xffffffff}, + {0xffa0a0a0, 0xffffffff}, + {0xffb0b0b0, 0xffffffff}, + {0xffc0c0c0, 0xffffffff}, + {0xffd0d0d0, 0xffffffff}, + {0xffe0e0e0, 0xffffffff}, + } + ); + for (PackedBinaryMatrix& matrix : matrices) { + find_type_symbol_candidates(candidates, original_screen, image, matrix, max_area_ratio, generation); + } + } + + // cout << "-------------" << endl; + + std::multimap> filtered; + for (const auto& candidate : candidates) { + // cout << POKEMON_TYPE_SLUGS().get_string(candidate.second.first) << ": " << candidate.first << endl; + // hits.emplace_back(overlay, translate_to_parent(screen, box, candidate.second.second.box), COLOR_GREEN); + + bool is_dupe = false; + for (const auto& item : filtered) { + if (distance_sqr(candidate.second.second, item.second.second) == 0) { + is_dupe = true; + break; + } + } + if (!is_dupe) { + filtered.emplace(candidate); + } + } + +#if 0 + static int c = 0; + for (const auto& item : filtered) { + // cout << get_type_slug(item.second.first) << ": " << item.first << " - [" << item.second.second.center_x() << "," << item.second.second.center_y() << "]" << endl; + const ImagePixelBox& box = item.second.second; + ImageViewRGB32 img = image.sub_image( + box.min_x, box.min_y, + box.width(), box.height() + ); + img.save("test-" + std::to_string(c++) + ".png"); + } +#endif + + return filtered; +} + +std::pair read_pokemon_types( + const ImageViewRGB32& original_screen, + const ImageFloatBox& box, + PokemonTypeGeneration generation +) { + + ImageViewRGB32 image = extract_box_reference(original_screen, box); + + std::multimap> filtered = find_type_symbols( + original_screen, + image, + 0.20, + generation + ); + + // Re-sort by screen position (left-to-right, top-to-bottom) so that primary/secondary + // assignment is stable regardless of which type was detected with higher confidence. + std::vector> sorted; + sorted.reserve(filtered.size()); + for (const auto& item : filtered) { + sorted.emplace_back(item.second); + } + std::sort(sorted.begin(), sorted.end(), [](const auto& a, const auto& b) { + size_t height = a.second.max_y - a.second.min_y; + size_t diff_y = a.second.min_y > b.second.min_y + ? a.second.min_y - b.second.min_y + : b.second.min_y - a.second.min_y; + if (diff_y > height) { + return a.second.min_y < b.second.min_y; + } + return a.second.min_x < b.second.min_x; + }); + + std::pair result{ PokemonType::NONE, PokemonType::NONE }; + for (const auto& item : sorted) { + PokemonType type = item.first; + if (result.first == PokemonType::NONE) { + result.first = type; + } + else if (type != result.first) { + result.second = type; + break; + } + } + return result; +} + +} +} \ No newline at end of file diff --git a/SerialPrograms/Source/Pokemon/Inference/Pokemon_TypeReader.h b/SerialPrograms/Source/Pokemon/Inference/Pokemon_TypeReader.h new file mode 100644 index 0000000000..bdc48e00e1 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Inference/Pokemon_TypeReader.h @@ -0,0 +1,43 @@ +/* Type Reader + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_Pokemon_TypeReader_H +#define PokemonAutomation_Pokemon_TypeReader_H + +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "Pokemon/Pokemon_Types.h" + +namespace PokemonAutomation { +namespace Pokemon { + +// The style of type symbols. +// GEN8: SWSH, Needs testing: LGPE +// GEN9: SV, LZA, HOME(Currently), Needs testing: Champions, LA, BDSP +enum class PokemonTypeGeneration{ + GEN8, + GEN9, +}; + +// Find all type symbols inside the image. +std::multimap> find_type_symbols( + const ImageViewPlanar32& original_screen, + const ImageViewRGB32& image, + double max_area_ratio, + PokemonTypeGeneration generation +); + +// Reads the types of a Pokemon. Second type will be PokemonType::NONE if the Pokemon is single-type +std::pair read_pokemon_types( + const ImageViewRGB32& original_screen, + const ImageFloatBox& box, + PokemonTypeGeneration generation +); + +} +} +#endif \ No newline at end of file diff --git a/SerialPrograms/Source/Pokemon/Options/Pokemon_BoxSortingTable.cpp b/SerialPrograms/Source/Pokemon/Options/Pokemon_BoxSortingTable.cpp index ef3b381c1e..7a06bddb33 100644 --- a/SerialPrograms/Source/Pokemon/Options/Pokemon_BoxSortingTable.cpp +++ b/SerialPrograms/Source/Pokemon/Options/Pokemon_BoxSortingTable.cpp @@ -18,6 +18,7 @@ const EnumDropdownDatabase& SortingRuleType_Database(){ {SortingRuleType::Alpha, "alpha", "Alpha"}, {SortingRuleType::Ball_Slug, "ball_slug", "Ball Type"}, {SortingRuleType::Gender, "gender", "Gender (Male, Female, Genderless)"}, + //{SortingRuleType::Type, "type", "Types"}, Mainly used for testing. The order of the PokemonType Enum is not the most intuitive for sorting. }); return database; } diff --git a/SerialPrograms/Source/Pokemon/Options/Pokemon_BoxSortingTable.h b/SerialPrograms/Source/Pokemon/Options/Pokemon_BoxSortingTable.h index a0c736adc1..a8a5320169 100644 --- a/SerialPrograms/Source/Pokemon/Options/Pokemon_BoxSortingTable.h +++ b/SerialPrograms/Source/Pokemon/Options/Pokemon_BoxSortingTable.h @@ -23,6 +23,7 @@ enum class SortingRuleType Alpha, Ball_Slug, Gender, + Type }; struct SortingRule diff --git a/SerialPrograms/Source/Pokemon/Pokemon_CollectedPokemonInfo.cpp b/SerialPrograms/Source/Pokemon/Pokemon_CollectedPokemonInfo.cpp index a7fd926038..b2dc725640 100644 --- a/SerialPrograms/Source/Pokemon/Pokemon_CollectedPokemonInfo.cpp +++ b/SerialPrograms/Source/Pokemon/Pokemon_CollectedPokemonInfo.cpp @@ -11,7 +11,6 @@ #include "Pokemon/Pokemon_Strings.h" #include "Pokemon/Pokemon_BoxCursor.h" #include "Pokemon/Resources/Pokemon_PokemonNames.h" -#include "Pokemon/Resources/Pokemon_PokemonSlugs.h" #include "Pokemon_CollectedPokemonInfo.h" namespace PokemonAutomation{ @@ -27,7 +26,9 @@ bool operator==(const CollectedPokemonInfo& lhs, const CollectedPokemonInfo& rhs lhs.alpha == rhs.alpha && lhs.ball_slug == rhs.ball_slug && lhs.gender == rhs.gender && - lhs.ot_id == rhs.ot_id; + lhs.ot_id == rhs.ot_id && + lhs.primaryType == rhs.primaryType && + lhs.secondaryType == rhs.secondaryType; } @@ -73,6 +74,14 @@ bool operator<(const std::optional& lhs, const std::option return (lhs->gender < rhs->gender) != preference.reverse; } break; + case SortingRuleType::Type: + if (lhs->primaryType != rhs->primaryType){ + return (lhs->primaryType < rhs->primaryType) != preference.reverse; + } + if (lhs->secondaryType != rhs->secondaryType){ + return (lhs->secondaryType < rhs->secondaryType) != preference.reverse; + } + break; default: throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "unknown SortingRuleType"); } // end switch @@ -94,6 +103,8 @@ std::ostream& operator<<(std::ostream& os, const std::optionalball_slug << " "; os << "gender:" << gender_to_string(pokemon->gender) << " "; os << "ot_id:" << pokemon->ot_id << " "; + os << "primaryType:" << POKEMON_TYPE_SLUGS().get_string(pokemon->primaryType) << " "; + os << "secondaryType:" << POKEMON_TYPE_SLUGS().get_string(pokemon->secondaryType) << " "; os << ")"; }else{ os << "(empty)"; @@ -144,6 +155,8 @@ void save_boxes_data_to_json(const std::vectorball_slug; pokemon["gender"] = gender_to_string(current_pokemon->gender); pokemon["ot_id"] = current_pokemon->ot_id; + pokemon["primaryType"] = POKEMON_TYPE_SLUGS().get_string(current_pokemon->primaryType); + pokemon["secondaryType"] = POKEMON_TYPE_SLUGS().get_string(current_pokemon->secondaryType); } pokemon_data.push_back(std::move(pokemon)); } diff --git a/SerialPrograms/Source/Pokemon/Pokemon_CollectedPokemonInfo.h b/SerialPrograms/Source/Pokemon/Pokemon_CollectedPokemonInfo.h index e430a7f5c8..7bd6117f0c 100644 --- a/SerialPrograms/Source/Pokemon/Pokemon_CollectedPokemonInfo.h +++ b/SerialPrograms/Source/Pokemon/Pokemon_CollectedPokemonInfo.h @@ -14,6 +14,7 @@ #include #include "Pokemon/Options/Pokemon_StatsHuntFilter.h" #include "Pokemon/Options/Pokemon_BoxSortingTable.h" +#include "Pokemon_Types.h" namespace PokemonAutomation{ namespace Pokemon{ @@ -31,6 +32,8 @@ struct CollectedPokemonInfo{ std::string ball_slug = ""; StatsHuntGenderFilter gender = StatsHuntGenderFilter::Genderless; uint32_t ot_id = 0; // original trainer ID + PokemonType primaryType = PokemonType::NONE; + PokemonType secondaryType = PokemonType::NONE; }; bool operator==(const CollectedPokemonInfo& lhs, const CollectedPokemonInfo& rhs); diff --git a/SerialPrograms/Source/PokemonHome/Programs/PokemonHome_BoxSorter.cpp b/SerialPrograms/Source/PokemonHome/Programs/PokemonHome_BoxSorter.cpp index 6385f43d94..ec2e679715 100644 --- a/SerialPrograms/Source/PokemonHome/Programs/PokemonHome_BoxSorter.cpp +++ b/SerialPrograms/Source/PokemonHome/Programs/PokemonHome_BoxSorter.cpp @@ -41,6 +41,7 @@ language #include "CommonTools/OCR/OCR_NumberReader.h" #include "CommonTools/StartupChecks/StartProgramChecks.h" #include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "Pokemon/Inference/Pokemon_TypeReader.h" #include "Pokemon/Pokemon_Strings.h" #include "Pokemon/Resources/Pokemon_PokemonNames.h" #include "Pokemon/Resources/Pokemon_PokemonSlugs.h" @@ -401,6 +402,8 @@ void read_summary_screen( ImageFloatBox nature_box(0.157, 0.783, 0.212, 0.042); // Nature ImageFloatBox ability_box(0.158, 0.838, 0.213, 0.042); // Ability ImageFloatBox alpha_box(0.787, 0.095, 0.024, 0.046); // Alpha symbol + ImageFloatBox type_box(0.615, 0.240, 0.071, 0.057); // Type symbols + video_overlay_set.add(COLOR_WHITE, national_dex_number_box); video_overlay_set.add(COLOR_BLUE, shiny_symbol_box); @@ -461,7 +464,12 @@ void read_summary_screen( dump_image(env.console, ProgramInfo(), "ReadSummary_OT", screen); } cur_pokemon_info.ot_id = ot_id; - + + auto [primaryType, secondaryType] = read_pokemon_types(screen, type_box, PokemonTypeGeneration::GEN9); + + cur_pokemon_info.primaryType = primaryType; + cur_pokemon_info.secondaryType = secondaryType; + env.add_overlay_log(create_overlay_info(cur_pokemon_info)); video_overlay_set.clear(); diff --git a/SerialPrograms/Source/PokemonLZA/Programs/PokemonLZA_BoxSorter.cpp b/SerialPrograms/Source/PokemonLZA/Programs/PokemonLZA_BoxSorter.cpp index 3d564f80e2..86c29ee9b9 100644 --- a/SerialPrograms/Source/PokemonLZA/Programs/PokemonLZA_BoxSorter.cpp +++ b/SerialPrograms/Source/PokemonLZA/Programs/PokemonLZA_BoxSorter.cpp @@ -22,6 +22,7 @@ #include "CommonTools/VisualDetectors/FrozenImageDetector.h" #include "CommonTools/StartupChecks/StartProgramChecks.h" #include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "Pokemon/Inference/Pokemon_TypeReader.h" #include "Pokemon/Pokemon_Strings.h" #include "Pokemon/Resources/Pokemon_PokemonNames.h" #include "Pokemon/Pokemon_BoxCursor.h" @@ -319,6 +320,7 @@ void BoxSorter::program(SingleSwitchProgramEnvironment& env, ProControllerContex name_slug = LUMIOSE_DEX_SLUGS()[dex_number-1]; } + auto [primaryType, secondaryType] = read_pokemon_types(screen, ImageFloatBox(0.467, 0.256, 0.110, 0.041), PokemonTypeGeneration::GEN9); boxes_data.push_back( CollectedPokemonInfo{ .preferences = &sort_preferences, @@ -326,6 +328,8 @@ void BoxSorter::program(SingleSwitchProgramEnvironment& env, ProControllerContex .name_slug = name_slug, .shiny = shiny_detector.detect(screen), .alpha = alpha_detector.detect(screen), + .primaryType = primaryType, + .secondaryType = secondaryType, } ); ss << "\u2705 " ; // checkbox diff --git a/SerialPrograms/Source/PokemonSwSh/Inference/PokemonSwSh_TypeSymbolFinder.cpp b/SerialPrograms/Source/PokemonSwSh/Inference/PokemonSwSh_TypeSymbolFinder.cpp deleted file mode 100644 index e5b188f982..0000000000 --- a/SerialPrograms/Source/PokemonSwSh/Inference/PokemonSwSh_TypeSymbolFinder.cpp +++ /dev/null @@ -1,312 +0,0 @@ -/* Type Symbol Finder - * - * From: https://github.com/PokemonAutomation/ - * - */ - -#include "Common/Cpp/CancellableScope.h" -#include "Kernels/Waterfill/Kernels_Waterfill.h" -#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" -#include "CommonFramework/ImageTools/ImageStats.h" -#include "CommonTools/Images/BinaryImage_FilterRgb32.h" -#include "PokemonSwSh/Resources/PokemonSwSh_TypeSprites.h" -#include "PokemonSwSh_TypeSymbolFinder.h" - -#include -using std::cout; -using std::endl; - -namespace PokemonAutomation{ -namespace NintendoSwitch{ -namespace PokemonSwSh{ - -using namespace Kernels; -using namespace Kernels::Waterfill; - - - -size_t distance_sqr(const ImagePixelBox& a, const ImagePixelBox& b){ - bool overlap_x = a.min_x <= b.max_x && b.min_x <= a.max_x; - bool overlap_y = a.min_y <= b.max_y && b.min_y <= a.max_y; - if (overlap_x && overlap_y){ - return 0; - } - - size_t dist_x = 0; - if (!overlap_x){ - dist_x = a.max_x < b.min_x - ? b.min_x - a.max_x - : a.min_x - b.max_x; - } - - size_t dist_y = 0; - if (!overlap_y){ - dist_y = a.max_y < b.min_y - ? b.min_y - a.max_y - : a.min_y - b.max_y; - } - - return dist_x*dist_x + dist_y*dist_y; -} - -//bool print = false; - -std::pair match_type_symbol(const ImageViewRGB32& image){ - size_t width = image.width(); - size_t height = image.height(); - if (width * height < 100){ - return {1.0, PokemonType::NONE}; - } - if (width > 2 * height){ - return {1.0, PokemonType::NONE}; - } - if (height > 2 * width){ - return {1.0, PokemonType::NONE}; - } - ImageStats stats = image_stats(image); - if (stats.stddev.sum() < 50){ -// if (print){ -// cout << "stats.stddev.sum() = " << stats.stddev.sum() << endl; -// } - return {1.0, PokemonType::NONE}; - } - - double aspect_ratio = (double)width / height; - -// static int c = 0; -// image.save("test-" + std::to_string(threshold) + "-" + std::to_string(c++) + ".png"); - -// std::map rank; - double best_score = 0.45; - PokemonType best_type = PokemonType::NONE; - for (const auto& item : all_type_sprites()){ -// if (threshold != 700 || id != 55){ -// continue; -// } - - double expected_aspect_ratio = item.second.aspect_ratio(); - double ratio = aspect_ratio / expected_aspect_ratio; -#if 0 - if (print){ - cout << item.second.slug() - << " : expected = " << expected_aspect_ratio - << ", actual = " << aspect_ratio - << ", ratio = " << ratio << endl; - } -#endif - if (std::abs(ratio - 1) > 0.2){ - continue; - } - - double rmsd_alpha = item.second.matcher().diff(image); - -// item.second.matcher().m_image.save("sprite.png"); -// if (print){ -// cout << item.second.slug() << ": " << rmsd_alpha << endl; -// } - -#if 0 - // Handicap fairy due to white and pink being too similar in color and - // false positiving on the background. - if (item.first == PokemonType::FAIRY){ - rmsd_ratio *= 1.5; - } - - // Bonus for dark because or large contrast. - if (item.first == PokemonType::DARK){ - rmsd_ratio *= 0.8; - } -#endif - - if (best_score > rmsd_alpha){ - best_score = rmsd_alpha; - best_type = item.first; -// cout << item.second.slug() << ": " << stats.stddev << endl; - } - } - -// if (best_type != PokemonType::NONE){ -// cout << get_type_slug(best_type) << ": " << best_score << endl; -// } - return {best_score, best_type}; -} - -void find_type_symbol_candidates( - std::multimap>& candidates, - const ImageViewPlanar32& original_screen, - const ImageViewRGB32& image, - PackedBinaryMatrix& matrix, double max_area_ratio -){ - size_t max_area = (size_t)(image.width() * image.height() * max_area_ratio); - std::vector objects = find_objects_inplace( - matrix, - (size_t)(20. * original_screen.total_pixels() / (1920*1080)) - ); - -// static int index = 0; - - std::map objmap; - for (size_t c = 0; c < objects.size(); c++){ - if (objects[c].area > max_area){ - continue; - } - objmap[c] = objects[c]; - -#if 0 - extract_box_reference(image, ImagePixelBox(objects[c])).save("test-" + std::to_string(index++) + ".png"); -#endif - } - -// cout << "begin = " << objmap.size() << endl; - - // Merge nearby objects. - bool changed; - do{ - changed = false; - for (auto iter0 = objmap.begin(); iter0 != objmap.end(); ++iter0){ - for (auto iter1 = objmap.begin(); iter1 != objmap.end();){ - if (iter0->first >= iter1->first){ - ++iter1; - continue; - } - const WaterfillObject& obj0 = iter0->second; - const WaterfillObject& obj1 = iter1->second; - size_t distance = distance_sqr( - ImagePixelBox(obj0.min_x, obj0.min_y, obj0.max_x, obj0.max_y), - ImagePixelBox(obj1.min_x, obj1.min_y, obj1.max_x, obj1.max_y) - ); - if (distance < 5*5){ - iter0->second.merge_assume_no_overlap(iter1->second); - iter1 = objmap.erase(iter1); - changed = true; - }else{ - ++iter1; - } - } - } - }while (changed); - -// cout << "merged = " << objmap.size() << endl; - - // Identify objects. - for (const auto& item : objmap){ - ImageViewRGB32 img = extract_box_reference(image, item.second); - -// print = index == 137; -// img.save("test-" + std::to_string(index++) + ".png"); - - std::pair result = match_type_symbol(img); -// cout << "result = " << POKEMON_TYPE_SLUGS().get_string(result.second) << ": " << result.first << endl; - if (result.second != PokemonType::NONE){ - const WaterfillObject& obj = item.second; - candidates.emplace( - result.first, - std::pair( - result.second, - ImagePixelBox(obj.min_x, obj.min_y, obj.max_x, obj.max_y) - ) - ); - } - } - -// cout << "candidates = " << candidates.size() << endl; -} - - - -std::multimap> find_type_symbols( - const ImageViewPlanar32& original_screen, - const ImageViewRGB32& image, double max_area_ratio -){ - std::multimap> candidates; - - { - std::vector matrices = compress_rgb32_to_binary_range( - image, - { - {0xff808060, 0xffffffff}, - {0xffa0a060, 0xffffffff}, - {0xff606060, 0xffffffff}, - {0xff707070, 0xffffffff}, - {0xff808080, 0xffffffff}, - {0xff909090, 0xffffffff}, - {0xffa0a0a0, 0xffffffff}, - {0xffb0b0b0, 0xffffffff}, - {0xffc0c0c0, 0xffffffff}, - {0xffd0d0d0, 0xffffffff}, - {0xffe0e0e0, 0xffffffff}, - } - ); - for (PackedBinaryMatrix& matrix : matrices){ - find_type_symbol_candidates(candidates, original_screen,image, matrix, max_area_ratio); - } - } - -// cout << "-------------" << endl; - - std::multimap> filtered; - for (const auto& candidate : candidates){ -// cout << POKEMON_TYPE_SLUGS().get_string(candidate.second.first) << ": " << candidate.first << endl; -// hits.emplace_back(overlay, translate_to_parent(screen, box, candidate.second.second.box), COLOR_GREEN); - - bool is_dupe = false; - for (const auto& item : filtered){ - if (distance_sqr(candidate.second.second, item.second.second) == 0){ - is_dupe = true; - break; - } - } - if (!is_dupe){ - filtered.emplace(candidate); - } - } - -#if 0 - static int c = 0; - for (const auto& item : filtered){ -// cout << get_type_slug(item.second.first) << ": " << item.first << " - [" << item.second.second.center_x() << "," << item.second.second.center_y() << "]" << endl; - const ImagePixelBox& box = item.second.second; - ImageViewRGB32 img = image.sub_image( - box.min_x, box.min_y, - box.width(), box.height() - ); - img.save("test-" + std::to_string(c++) + ".png"); - } -#endif - - return filtered; -} - - - - - -void test_find_type_symbols( - CancellableScope& scope, - VideoOverlay& overlay, - const ImageFloatBox& box, - const ImageViewRGB32& screen, double max_area_ratio -){ - ImageViewRGB32 image = extract_box_reference(screen, box); - - std::multimap> candidates = find_type_symbols(screen, image, max_area_ratio); - - - std::deque hits; -// hits.clear(); - cout << "---------------" << endl; - for (const auto& item : candidates){ - cout << POKEMON_TYPE_SLUGS().get_string(item.second.first) << ": " << item.first << endl; - hits.emplace_back(overlay, translate_to_parent(screen, box, item.second.second), COLOR_GREEN); - } - - scope.wait_for(std::chrono::seconds(10)); -} - - - - - -} -} -} diff --git a/SerialPrograms/Source/PokemonSwSh/Inference/PokemonSwSh_TypeSymbolFinder.h b/SerialPrograms/Source/PokemonSwSh/Inference/PokemonSwSh_TypeSymbolFinder.h deleted file mode 100644 index d945e05ce2..0000000000 --- a/SerialPrograms/Source/PokemonSwSh/Inference/PokemonSwSh_TypeSymbolFinder.h +++ /dev/null @@ -1,44 +0,0 @@ -/* Type Symbol Finder - * - * From: https://github.com/PokemonAutomation/ - * - */ - -#ifndef PokemonAutomation_PokemonSwSh_TypeSymbolFinder_H -#define PokemonAutomation_PokemonSwSh_TypeSymbolFinder_H - -#include -#include "CommonFramework/ImageTypes/ImageViewPlanar32.h" -#include "CommonFramework/ImageTools/ImageBoxes.h" -#include "Pokemon/Pokemon_Types.h" -//#include "PokemonSwSh/Resources/PokemonSwSh_TypeSprites.h" - -namespace PokemonAutomation{ - class ProgramEnvironment; - class VideoOverlay; -namespace NintendoSwitch{ -namespace PokemonSwSh{ - -using namespace Pokemon; - - -// Find all type symbols inside the image. -std::multimap> find_type_symbols( - const ImageViewPlanar32& original_screen, - const ImageViewRGB32& image, double max_area_ratio -); - - - -void test_find_type_symbols( - ProgramEnvironment& env, - VideoOverlay& overlay, - const ImageFloatBox& box, - const ImageViewRGB32& screen, double max_area_ratio = 0.20 -); - - -} -} -} -#endif diff --git a/SerialPrograms/Source/PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_BattleMenu.cpp b/SerialPrograms/Source/PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_BattleMenu.cpp index e7813ecce3..5219bbe7c9 100644 --- a/SerialPrograms/Source/PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_BattleMenu.cpp +++ b/SerialPrograms/Source/PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_BattleMenu.cpp @@ -13,8 +13,8 @@ #include "CommonTools/Images/SolidColorTest.h" #include "CommonTools/Images/ColorClustering.h" #include "Pokemon/Inference/Pokemon_ReadHpBar.h" +#include "Pokemon/Inference/Pokemon_TypeReader.h" #include "PokemonSwSh/Resources/PokemonSwSh_MaxLairDatabase.h" -#include "PokemonSwSh/Inference/PokemonSwSh_TypeSymbolFinder.h" #include "PokemonSwSh/MaxLair/Options/PokemonSwSh_MaxLair_Options.h" #include "PokemonSwSh_MaxLair_Detect_PokemonReader.h" #include "PokemonSwSh_MaxLair_Detect_HPPP.h" @@ -263,7 +263,12 @@ std::set BattleMenuReader::read_opponent_in_summary(Logger& logger, PokemonType type0, type1; { ImageViewRGB32 types = extract_box_reference(screen, m_summary_opponent_types); - std::multimap> candidates = find_type_symbols(screen, types, 0.2); + std::multimap> candidates = find_type_symbols( + screen, + types, + 0.2, + PokemonTypeGeneration::GEN8 + ); std::string type_str = "Type Read Result:\n"; for (const auto& item : candidates){ diff --git a/SerialPrograms/Source/PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_PathMap.cpp b/SerialPrograms/Source/PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_PathMap.cpp index 6a1e07fa4f..0cab6940e7 100644 --- a/SerialPrograms/Source/PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_PathMap.cpp +++ b/SerialPrograms/Source/PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_PathMap.cpp @@ -9,7 +9,7 @@ #include "CommonFramework/Tools/ErrorDumper.h" #include "CommonFramework/Tools/ProgramEnvironment.h" #include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" -#include "PokemonSwSh/Inference/PokemonSwSh_TypeSymbolFinder.h" +#include "Pokemon/Inference/Pokemon_TypeReader.h" //#include "PokemonSwSh/MaxLair/Options/PokemonSwSh_MaxLair_Options.h" #include "PokemonSwSh_MaxLair_Detect_PathMap.h" @@ -36,7 +36,12 @@ bool read_type_array( type[c] = PokemonType::NONE; } - std::multimap> candidates = find_type_symbols(screen, image, 0.20); + std::multimap> candidates = find_type_symbols( + screen, + image, + 0.20, + PokemonTypeGeneration::GEN8 + ); // cout << candidates.size() << endl; #if 0 diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index f0a89d4520..39431ef121 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1239,6 +1239,8 @@ file(GLOB LIBRARY_SOURCES Source/Pokemon/Inference/Pokemon_TrainIVCheckerOCR.h Source/Pokemon/Inference/Pokemon_TrainPokemonOCR.cpp Source/Pokemon/Inference/Pokemon_TrainPokemonOCR.h + Source/Pokemon/Inference/Pokemon_TypeReader.cpp + Source/Pokemon/Inference/Pokemon_TypeReader.h Source/Pokemon/Options/Pokemon_BoxSortingTable.cpp Source/Pokemon/Options/Pokemon_BoxSortingTable.h Source/Pokemon/Options/Pokemon_EncounterBotOptions.h @@ -2382,8 +2384,6 @@ file(GLOB LIBRARY_SOURCES Source/PokemonSwSh/Inference/PokemonSwSh_SelectionArrowFinder.h Source/PokemonSwSh/Inference/PokemonSwSh_SummaryShinySymbolDetector.cpp Source/PokemonSwSh/Inference/PokemonSwSh_SummaryShinySymbolDetector.h - Source/PokemonSwSh/Inference/PokemonSwSh_TypeSymbolFinder.cpp - Source/PokemonSwSh/Inference/PokemonSwSh_TypeSymbolFinder.h Source/PokemonSwSh/Inference/PokemonSwSh_YCommDetector.cpp Source/PokemonSwSh/Inference/PokemonSwSh_YCommDetector.h Source/PokemonSwSh/Inference/RNG/PokemonSwSh_OrbeetleAttackAnimationDetector.cpp