From 2d06e6c6173f397a89ec9fc245a0b9117125f2d2 Mon Sep 17 00:00:00 2001 From: theAstrogoth Date: Tue, 11 Aug 2026 08:23:50 -0500 Subject: [PATCH 1/3] add BDSP blink detection --- .../Inference/Rng/PokemonBDSP_BlinkScenes.cpp | 58 ++++++ .../Inference/Rng/PokemonBDSP_BlinkScenes.h | 29 +++ .../Rng/PokemonBDSP_EyeBlinkDetector.cpp | 176 ++++++++++++++++++ .../Rng/PokemonBDSP_EyeBlinkDetector.h | 93 +++++++++ .../Options/PokemonBDSP_PlayerModelOption.cpp | 57 ++++++ .../Options/PokemonBDSP_PlayerModelOption.h | 32 ++++ SerialPrograms/cmake/SourceFiles.cmake | 6 + 7 files changed, 451 insertions(+) create mode 100644 SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp new file mode 100644 index 0000000000..3412cfb9ab --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp @@ -0,0 +1,58 @@ +/* BDSP Blink Scenes + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "Common/Cpp/Exceptions.h" +#include "PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h" +#include "PokemonBDSP_BlinkScenes.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +std::vector lake_eye_templates(uint8_t player_model){ + if (player_model < 1 || player_model > BDSP_PLAYER_MODEL_COUNT){ + throw InternalProgramError( + nullptr, PA_CURRENT_FUNCTION, + "Unknown player model: " + std::to_string(player_model) + ); + } + return { + BdspEyeTemplate{ + "lake_templates/model" + std::to_string(player_model) + ".png", + {0.5026, 0.4472, 0.0240, 0.0426}, + "Player" + }, + BdspEyeTemplate{ + "lake_templates/barry.png", + {0.5521, 0.4528, 0.0214, 0.0380}, + "Barry" + }, + }; +} + + +std::vector bedroom_eye_templates(uint8_t player_model){ + if (player_model < 1 || player_model > BDSP_PLAYER_MODEL_COUNT){ + throw InternalProgramError( + nullptr, PA_CURRENT_FUNCTION, + "Unknown player model: " + std::to_string(player_model) + ); + } + return { + BdspEyeTemplate{ + "bedroom_templates/model" + std::to_string(player_model) + ".png", + {0.4708, 0.4500, 0.0214, 0.0491}, + "Player" + }, + }; +} + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.h b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.h new file mode 100644 index 0000000000..b1c49bb7fd --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.h @@ -0,0 +1,29 @@ +/* BDSP Blink Scenes + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_BlinkScenes_H +#define PokemonAutomation_PokemonBDSP_BlinkScenes_H + +#include +#include +#include "PokemonBDSP_EyeBlinkDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Which eyes to watch, and where they sit, in each scene the RNG programs use. + +std::vector lake_eye_templates(uint8_t player_model); + +std::vector bedroom_eye_templates(uint8_t player_model); + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.cpp new file mode 100644 index 0000000000..8b3a314848 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.cpp @@ -0,0 +1,176 @@ +/* BDSP Eye Blink Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonBDSP_EyeBlinkDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +static float to_grey(uint32_t pixel){ + float r = (float)((pixel >> 16) & 0xff); + float g = (float)((pixel >> 8) & 0xff); + float b = (float)(pixel & 0xff); + return 0.299f * r + 0.587f * g + 0.114f * b; +} + + +EyeBlinkDetector::EyeBlinkDetector(std::shared_ptr open_eye, ImageFloatBox search_box) + : m_template(std::move(open_eye)) + , m_width(0) + , m_height(0) + , m_norm(0) + , m_search_box(search_box) +{ + if (m_template == nullptr || !*m_template){ + throw InternalProgramError( + nullptr, PA_CURRENT_FUNCTION, + "EyeBlinkDetector: No eye template was supplied." + ); + } + m_width = m_template->width(); + m_height = m_template->height(); + + m_centered.resize(m_width * m_height); + double sum = 0; + for (size_t y = 0; y < m_height; y++){ + for (size_t x = 0; x < m_width; x++){ + float value = to_grey(m_template->pixel(x, y)); + m_centered[y * m_width + x] = value; + sum += value; + } + } + double mean = sum / (double)m_centered.size(); + for (float& value : m_centered){ + value = (float)(value - mean); + m_norm += (double)value * value; + } + + if (m_norm <= 0){ + // A flat template correlates with everything equally and can never see a blink + throw InternalProgramError( + nullptr, PA_CURRENT_FUNCTION, + "EyeBlinkDetector: The eye template is a single flat colour." + ); + } +} + +double EyeBlinkDetector::match(const ImageViewRGB32& frame) const{ + ImageViewRGB32 region = extract_box_reference(frame, m_search_box); + size_t region_width = region.width(); + size_t region_height = region.height(); + if (region_width < m_width || region_height < m_height){ + return 1.0; + } + + std::vector grey(region_width * region_height); + for (size_t y = 0; y < region_height; y++){ + for (size_t x = 0; x < region_width; x++){ + grey[y * region_width + x] = to_grey(region.pixel(x, y)); + } + } + + const double count = (double)(m_width * m_height); + double best = -1.0; + + for (size_t dy = 0; dy + m_height <= region_height; dy++){ + for (size_t dx = 0; dx + m_width <= region_width; dx++){ + double sum = 0; + double sum_squares = 0; + double cross = 0; + for (size_t y = 0; y < m_height; y++){ + const float* row = grey.data() + (dy + y) * region_width + dx; + const float* tpl = m_centered.data() + y * m_width; + for (size_t x = 0; x < m_width; x++){ + double value = row[x]; + sum += value; + sum_squares += value * value; + cross += tpl[x] * value; + } + } + // The template already has its mean removed, so the window's mean + // drops out of the numerator and only its variance is needed. + double variance = sum_squares - sum * sum / count; + if (variance <= 0){ + continue; + } + double score = cross / std::sqrt(variance * m_norm); + if (score > best){ + best = score; + } + } + } + + return best; +} + + + +EyeBlinkWatcher::EyeBlinkWatcher( + std::string label, + std::shared_ptr open_eye, + ImageFloatBox search_box, + Color color +) + : VisualInferenceCallback(label) + , m_label(std::move(label)) + , m_color(color) + , m_detector(std::move(open_eye), search_box) + , m_last_match(1.0) +{} + +void EyeBlinkWatcher::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_detector.search_box(), m_label); +} + +bool EyeBlinkWatcher::process_frame(const ImageViewRGB32& frame, WallClock timestamp){ + double value = m_detector.match(frame); + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + m_samples.emplace_back(BlinkMatchSample{timestamp, value}); + m_last_match = value; + // Never stops the session + return false; +} + +std::vector EyeBlinkWatcher::samples() const{ + ReadSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + return m_samples; +} +size_t EyeBlinkWatcher::sample_count() const{ + ReadSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + return m_samples.size(); +} +double EyeBlinkWatcher::last_match() const{ + ReadSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + return m_last_match; +} + +void EyeBlinkWatcher::discard_before(WallClock keep_from){ + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + auto keep = std::lower_bound( + m_samples.begin(), m_samples.end(), keep_from, + [](const BlinkMatchSample& sample, WallClock cutoff){ + return sample.timestamp < cutoff; + } + ); + if (keep == m_samples.begin()){ + return; + } + m_samples.erase(m_samples.begin(), keep); +} + + + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.h b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.h new file mode 100644 index 0000000000..b4f93a45de --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.h @@ -0,0 +1,93 @@ +/* BDSP Eye Blink Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_EyeBlinkDetector_H +#define PokemonAutomation_PokemonBDSP_EyeBlinkDetector_H + +#include +#include +#include +#include +#include "Common/Cpp/Color.h" +#include "Common/Cpp/Concurrency/SpinLock.h" +#include "Common/Cpp/Time.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +class EyeBlinkDetector{ +public: + EyeBlinkDetector(std::shared_ptr open_eye, ImageFloatBox search_box); + + double match(const ImageViewRGB32& frame) const; + + const ImageFloatBox& search_box() const{ return m_search_box; } + size_t template_width() const{ return m_width; } + size_t template_height() const{ return m_height; } + +private: + std::shared_ptr m_template; + size_t m_width; + size_t m_height; + std::vector m_centered; + double m_norm; + ImageFloatBox m_search_box; +}; + + +struct BlinkMatchSample{ + WallClock timestamp; + double match; +}; + +struct BdspEyeTemplate{ + // Path under the PokemonBDSP/Rng resource folder, scene subfolder included. + std::string asset; + ImageFloatBox box; + std::string label; +}; + + +class EyeBlinkWatcher : public VisualInferenceCallback{ +public: + EyeBlinkWatcher( + std::string label, + std::shared_ptr open_eye, + ImageFloatBox search_box, + Color color = COLOR_RED + ); + + const std::string& label() const{ return m_label; } + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool process_frame(const ImageViewRGB32& frame, WallClock timestamp) override; + + std::vector samples() const; + size_t sample_count() const; + double last_match() const; + + void discard_before(WallClock keep_from); + +private: + std::string m_label; + Color m_color; + EyeBlinkDetector m_detector; + + mutable SpinLock m_lock; + std::vector m_samples; + double m_last_match; +}; + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp new file mode 100644 index 0000000000..df8d64ce5b --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp @@ -0,0 +1,57 @@ +/* Player Model Select + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "PokemonBDSP_PlayerModelOption.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ + +struct PlayerModelIcons{ + std::vector icons; + StringSelectDatabase database; + + PlayerModelIcons(){ + icons.reserve(BDSP_PLAYER_MODEL_COUNT); + for (uint8_t model = 1; model <= BDSP_PLAYER_MODEL_COUNT; model++){ + std::string number = std::to_string(model); + icons.emplace_back( + RESOURCE_PATH() + "PokemonBDSP/Rng/model_icons/model" + number + ".png" + ); + database.add_entry(StringSelectEntry( + "model-" + number, "Model " + number, icons.back() + )); + } + } +}; +const PlayerModelIcons& PLAYER_MODEL_ICONS(){ + static PlayerModelIcons icons; + return icons; +} +} + + +PlayerModelOption::PlayerModelOption() + : StringSelectOption( + "Character model:
The appearance chosen for the player character. " + "Each one is watched through a different eye template.", + PLAYER_MODEL_ICONS().database, + LockMode::LOCK_WHILE_RUNNING, + "model-1" + ) +{} + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h new file mode 100644 index 0000000000..c6316924b0 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h @@ -0,0 +1,32 @@ +/* Player Model Select + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_PlayerModelOption_H +#define PokemonAutomation_PokemonBDSP_PlayerModelOption_H + +#include +#include "CommonTools/Options/StringSelectOption.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// The eight appearances the player character can be given at the start of the game. +const uint8_t BDSP_PLAYER_MODEL_COUNT = 8; + + +class PlayerModelOption : public StringSelectOption{ +public: + PlayerModelOption(); + uint8_t model_number() const{ return (uint8_t)(index() + 1); } +}; + + +} +} +} +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 985274f34a..e9e388d0cb 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1413,6 +1413,10 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Inference/BoxSystem/PokemonBDSP_BoxShinyDetector.h Source/PokemonBDSP/Inference/BoxSystem/PokemonBDSP_IvJudgeReader.cpp Source/PokemonBDSP/Inference/BoxSystem/PokemonBDSP_IvJudgeReader.h + Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp + Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.h + Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.cpp + Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.h Source/PokemonBDSP/Inference/Rng/PokemonBDSP_SummaryReader.cpp Source/PokemonBDSP/Inference/Rng/PokemonBDSP_SummaryReader.h Source/PokemonBDSP/Inference/PokemonBDSP_DialogDetector.cpp @@ -1453,6 +1457,8 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Options/PokemonBDSP_EggStepOption.h Source/PokemonBDSP/Options/PokemonBDSP_EncounterBotCommon.h Source/PokemonBDSP/Options/PokemonBDSP_LearnMove.h + Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp + Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h Source/PokemonBDSP/Options/PokemonBDSP_ShortcutDirection.cpp Source/PokemonBDSP/Options/PokemonBDSP_ShortcutDirection.h Source/PokemonBDSP/Panels_PokemonBDSP.cpp From 84216dd36dd1f05b875557b6481907bfb63697a9 Mon Sep 17 00:00:00 2001 From: theAstrogoth Date: Wed, 12 Aug 2026 19:09:10 -0500 Subject: [PATCH 2/3] use OpenCV's matchTemplate() --- .../Rng/PokemonBDSP_EyeBlinkDetector.cpp | 116 +++++++----------- .../Rng/PokemonBDSP_EyeBlinkDetector.h | 24 ++-- 2 files changed, 58 insertions(+), 82 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.cpp index 8b3a314848..2e22a64c1a 100644 --- a/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.cpp @@ -6,8 +6,10 @@ #include #include +#include #include "Common/Cpp/Exceptions.h" -#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/ImageTypes/ImageRGB32_OpenCV.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" #include "PokemonBDSP_EyeBlinkDetector.h" @@ -16,46 +18,27 @@ namespace NintendoSwitch{ namespace PokemonBDSP{ -static float to_grey(uint32_t pixel){ - float r = (float)((pixel >> 16) & 0xff); - float g = (float)((pixel >> 8) & 0xff); - float b = (float)(pixel & 0xff); - return 0.299f * r + 0.587f * g + 0.114f * b; -} +// Returned when the frame cannot be matched at all. Callers read a high score +// as "eye open", so a frame we can't measure never gets counted as a blink. +const double NO_MATCH = 1.0; -EyeBlinkDetector::EyeBlinkDetector(std::shared_ptr open_eye, ImageFloatBox search_box) - : m_template(std::move(open_eye)) - , m_width(0) - , m_height(0) - , m_norm(0) - , m_search_box(search_box) +EyeBlinkDetector::EyeBlinkDetector(const ImageViewRGB32& open_eye, ImageFloatBox search_box) + : m_search_box(search_box) { - if (m_template == nullptr || !*m_template){ + if (!open_eye){ throw InternalProgramError( nullptr, PA_CURRENT_FUNCTION, "EyeBlinkDetector: No eye template was supplied." ); } - m_width = m_template->width(); - m_height = m_template->height(); - - m_centered.resize(m_width * m_height); - double sum = 0; - for (size_t y = 0; y < m_height; y++){ - for (size_t x = 0; x < m_width; x++){ - float value = to_grey(m_template->pixel(x, y)); - m_centered[y * m_width + x] = value; - sum += value; - } - } - double mean = sum / (double)m_centered.size(); - for (float& value : m_centered){ - value = (float)(value - mean); - m_norm += (double)value * value; - } - if (m_norm <= 0){ + cv::cvtColor(to_OpenCV_ref(open_eye), m_template, cv::COLOR_BGRA2GRAY); + + cv::Scalar mean; + cv::Scalar stddev; + cv::meanStdDev(m_template, mean, stddev); + if (stddev[0] <= 0){ // A flat template correlates with everything equally and can never see a blink throw InternalProgramError( nullptr, PA_CURRENT_FUNCTION, @@ -65,51 +48,42 @@ EyeBlinkDetector::EyeBlinkDetector(std::shared_ptr open_eye, I } double EyeBlinkDetector::match(const ImageViewRGB32& frame) const{ + if (!frame || frame.height() == 0){ + return NO_MATCH; + } + ImageViewRGB32 region = extract_box_reference(frame, m_search_box); - size_t region_width = region.width(); - size_t region_height = region.height(); - if (region_width < m_width || region_height < m_height){ - return 1.0; + if (!region){ + return NO_MATCH; } - std::vector grey(region_width * region_height); - for (size_t y = 0; y < region_height; y++){ - for (size_t x = 0; x < region_width; x++){ - grey[y * region_width + x] = to_grey(region.pixel(x, y)); + // Rescale the region to the resolution the template was cropped at + // based on the capture height + ImageRGB32 rescaled; + if (frame.height() != BDSP_TEMPLATE_REFERENCE_HEIGHT){ + double scale = (double)BDSP_TEMPLATE_REFERENCE_HEIGHT / (double)frame.height(); + size_t width = (size_t)std::llround(region.width() * scale); + size_t height = (size_t)std::llround(region.height() * scale); + if (width == 0 || height == 0){ + return NO_MATCH; } + rescaled = region.scale_to(width, height); + region = rescaled; } - const double count = (double)(m_width * m_height); - double best = -1.0; - - for (size_t dy = 0; dy + m_height <= region_height; dy++){ - for (size_t dx = 0; dx + m_width <= region_width; dx++){ - double sum = 0; - double sum_squares = 0; - double cross = 0; - for (size_t y = 0; y < m_height; y++){ - const float* row = grey.data() + (dy + y) * region_width + dx; - const float* tpl = m_centered.data() + y * m_width; - for (size_t x = 0; x < m_width; x++){ - double value = row[x]; - sum += value; - sum_squares += value * value; - cross += tpl[x] * value; - } - } - // The template already has its mean removed, so the window's mean - // drops out of the numerator and only its variance is needed. - double variance = sum_squares - sum * sum / count; - if (variance <= 0){ - continue; - } - double score = cross / std::sqrt(variance * m_norm); - if (score > best){ - best = score; - } - } + if (region.width() < template_width() || region.height() < template_height()){ + return NO_MATCH; } + // Match the greyscale image to the template by zero-normalized cross correlation + cv::Mat grey; + cv::cvtColor(to_OpenCV_ref(region), grey, cv::COLOR_BGRA2GRAY); + + cv::Mat scores; + cv::matchTemplate(grey, m_template, scores, cv::TM_CCOEFF_NORMED); + + double best = NO_MATCH; + cv::minMaxLoc(scores, nullptr, &best); return best; } @@ -117,14 +91,14 @@ double EyeBlinkDetector::match(const ImageViewRGB32& frame) const{ EyeBlinkWatcher::EyeBlinkWatcher( std::string label, - std::shared_ptr open_eye, + const ImageViewRGB32& open_eye, ImageFloatBox search_box, Color color ) : VisualInferenceCallback(label) , m_label(std::move(label)) , m_color(color) - , m_detector(std::move(open_eye), search_box) + , m_detector(open_eye, search_box) , m_last_match(1.0) {} diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.h b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.h index b4f93a45de..32c599e5ed 100644 --- a/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.h +++ b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.h @@ -8,14 +8,14 @@ #define PokemonAutomation_PokemonBDSP_EyeBlinkDetector_H #include -#include #include #include +#include #include "Common/Cpp/Color.h" #include "Common/Cpp/Concurrency/SpinLock.h" #include "Common/Cpp/Time.h" #include "CommonFramework/ImageTools/ImageBoxes.h" -#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h" namespace PokemonAutomation{ @@ -23,22 +23,24 @@ namespace NintendoSwitch{ namespace PokemonBDSP{ +// Rescale a search region to the equivalent from a 1080p capture +const size_t BDSP_TEMPLATE_REFERENCE_HEIGHT = 1080; + + class EyeBlinkDetector{ public: - EyeBlinkDetector(std::shared_ptr open_eye, ImageFloatBox search_box); + EyeBlinkDetector(const ImageViewRGB32& open_eye, ImageFloatBox search_box); + // Best normalized cross-correlation of the eye template + // https://en.wikipedia.org/wiki/Cross-correlation#Terminology_in_image_processing double match(const ImageViewRGB32& frame) const; const ImageFloatBox& search_box() const{ return m_search_box; } - size_t template_width() const{ return m_width; } - size_t template_height() const{ return m_height; } + size_t template_width() const{ return (size_t)m_template.cols; } + size_t template_height() const{ return (size_t)m_template.rows; } private: - std::shared_ptr m_template; - size_t m_width; - size_t m_height; - std::vector m_centered; - double m_norm; + cv::Mat m_template; ImageFloatBox m_search_box; }; @@ -60,7 +62,7 @@ class EyeBlinkWatcher : public VisualInferenceCallback{ public: EyeBlinkWatcher( std::string label, - std::shared_ptr open_eye, + const ImageViewRGB32& open_eye, ImageFloatBox search_box, Color color = COLOR_RED ); From 97f2054795569e84d3b3744c7701d786c19d2165 Mon Sep 17 00:00:00 2001 From: theAstrogoth Date: Wed, 12 Aug 2026 19:09:24 -0500 Subject: [PATCH 3/3] use title case for resources --- .../PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp | 6 +++--- .../PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp index 3412cfb9ab..93c7807fb6 100644 --- a/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp @@ -23,12 +23,12 @@ std::vector lake_eye_templates(uint8_t player_model){ } return { BdspEyeTemplate{ - "lake_templates/model" + std::to_string(player_model) + ".png", + "LakeTemplates/Model-" + std::to_string(player_model) + ".png", {0.5026, 0.4472, 0.0240, 0.0426}, "Player" }, BdspEyeTemplate{ - "lake_templates/barry.png", + "LakeTemplates/Barry.png", {0.5521, 0.4528, 0.0214, 0.0380}, "Barry" }, @@ -45,7 +45,7 @@ std::vector bedroom_eye_templates(uint8_t player_model){ } return { BdspEyeTemplate{ - "bedroom_templates/model" + std::to_string(player_model) + ".png", + "BedroomTemplates/Model-" + std::to_string(player_model) + ".png", {0.4708, 0.4500, 0.0214, 0.0491}, "Player" }, diff --git a/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp index df8d64ce5b..370d4854a3 100644 --- a/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp @@ -26,7 +26,7 @@ struct PlayerModelIcons{ for (uint8_t model = 1; model <= BDSP_PLAYER_MODEL_COUNT; model++){ std::string number = std::to_string(model); icons.emplace_back( - RESOURCE_PATH() + "PokemonBDSP/Rng/model_icons/model" + number + ".png" + RESOURCE_PATH() + "PokemonBDSP/Rng/ModelIcons/Model-" + number + ".png" ); database.add_entry(StringSelectEntry( "model-" + number, "Model " + number, icons.back()