diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp new file mode 100644 index 0000000000..93c7807fb6 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp @@ -0,0 +1,58 @@ +/* BDSP Blink Scenes + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "Common/Cpp/Exceptions.h" +#include "PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h" +#include "PokemonBDSP_BlinkScenes.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +std::vector lake_eye_templates(uint8_t player_model){ + if (player_model < 1 || player_model > BDSP_PLAYER_MODEL_COUNT){ + throw InternalProgramError( + nullptr, PA_CURRENT_FUNCTION, + "Unknown player model: " + std::to_string(player_model) + ); + } + return { + BdspEyeTemplate{ + "LakeTemplates/Model-" + std::to_string(player_model) + ".png", + {0.5026, 0.4472, 0.0240, 0.0426}, + "Player" + }, + BdspEyeTemplate{ + "LakeTemplates/Barry.png", + {0.5521, 0.4528, 0.0214, 0.0380}, + "Barry" + }, + }; +} + + +std::vector bedroom_eye_templates(uint8_t player_model){ + if (player_model < 1 || player_model > BDSP_PLAYER_MODEL_COUNT){ + throw InternalProgramError( + nullptr, PA_CURRENT_FUNCTION, + "Unknown player model: " + std::to_string(player_model) + ); + } + return { + BdspEyeTemplate{ + "BedroomTemplates/Model-" + std::to_string(player_model) + ".png", + {0.4708, 0.4500, 0.0214, 0.0491}, + "Player" + }, + }; +} + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.h b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.h new file mode 100644 index 0000000000..b1c49bb7fd --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.h @@ -0,0 +1,29 @@ +/* BDSP Blink Scenes + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_BlinkScenes_H +#define PokemonAutomation_PokemonBDSP_BlinkScenes_H + +#include +#include +#include "PokemonBDSP_EyeBlinkDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Which eyes to watch, and where they sit, in each scene the RNG programs use. + +std::vector lake_eye_templates(uint8_t player_model); + +std::vector bedroom_eye_templates(uint8_t player_model); + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.cpp new file mode 100644 index 0000000000..2e22a64c1a --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.cpp @@ -0,0 +1,150 @@ +/* BDSP Eye Blink Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/ImageTypes/ImageRGB32_OpenCV.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonBDSP_EyeBlinkDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Returned when the frame cannot be matched at all. Callers read a high score +// as "eye open", so a frame we can't measure never gets counted as a blink. +const double NO_MATCH = 1.0; + + +EyeBlinkDetector::EyeBlinkDetector(const ImageViewRGB32& open_eye, ImageFloatBox search_box) + : m_search_box(search_box) +{ + if (!open_eye){ + throw InternalProgramError( + nullptr, PA_CURRENT_FUNCTION, + "EyeBlinkDetector: No eye template was supplied." + ); + } + + cv::cvtColor(to_OpenCV_ref(open_eye), m_template, cv::COLOR_BGRA2GRAY); + + cv::Scalar mean; + cv::Scalar stddev; + cv::meanStdDev(m_template, mean, stddev); + if (stddev[0] <= 0){ + // A flat template correlates with everything equally and can never see a blink + throw InternalProgramError( + nullptr, PA_CURRENT_FUNCTION, + "EyeBlinkDetector: The eye template is a single flat colour." + ); + } +} + +double EyeBlinkDetector::match(const ImageViewRGB32& frame) const{ + if (!frame || frame.height() == 0){ + return NO_MATCH; + } + + ImageViewRGB32 region = extract_box_reference(frame, m_search_box); + if (!region){ + return NO_MATCH; + } + + // Rescale the region to the resolution the template was cropped at + // based on the capture height + ImageRGB32 rescaled; + if (frame.height() != BDSP_TEMPLATE_REFERENCE_HEIGHT){ + double scale = (double)BDSP_TEMPLATE_REFERENCE_HEIGHT / (double)frame.height(); + size_t width = (size_t)std::llround(region.width() * scale); + size_t height = (size_t)std::llround(region.height() * scale); + if (width == 0 || height == 0){ + return NO_MATCH; + } + rescaled = region.scale_to(width, height); + region = rescaled; + } + + if (region.width() < template_width() || region.height() < template_height()){ + return NO_MATCH; + } + + // Match the greyscale image to the template by zero-normalized cross correlation + cv::Mat grey; + cv::cvtColor(to_OpenCV_ref(region), grey, cv::COLOR_BGRA2GRAY); + + cv::Mat scores; + cv::matchTemplate(grey, m_template, scores, cv::TM_CCOEFF_NORMED); + + double best = NO_MATCH; + cv::minMaxLoc(scores, nullptr, &best); + return best; +} + + + +EyeBlinkWatcher::EyeBlinkWatcher( + std::string label, + const ImageViewRGB32& open_eye, + ImageFloatBox search_box, + Color color +) + : VisualInferenceCallback(label) + , m_label(std::move(label)) + , m_color(color) + , m_detector(open_eye, search_box) + , m_last_match(1.0) +{} + +void EyeBlinkWatcher::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_detector.search_box(), m_label); +} + +bool EyeBlinkWatcher::process_frame(const ImageViewRGB32& frame, WallClock timestamp){ + double value = m_detector.match(frame); + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + m_samples.emplace_back(BlinkMatchSample{timestamp, value}); + m_last_match = value; + // Never stops the session + return false; +} + +std::vector EyeBlinkWatcher::samples() const{ + ReadSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + return m_samples; +} +size_t EyeBlinkWatcher::sample_count() const{ + ReadSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + return m_samples.size(); +} +double EyeBlinkWatcher::last_match() const{ + ReadSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + return m_last_match; +} + +void EyeBlinkWatcher::discard_before(WallClock keep_from){ + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + auto keep = std::lower_bound( + m_samples.begin(), m_samples.end(), keep_from, + [](const BlinkMatchSample& sample, WallClock cutoff){ + return sample.timestamp < cutoff; + } + ); + if (keep == m_samples.begin()){ + return; + } + m_samples.erase(m_samples.begin(), keep); +} + + + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.h b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.h new file mode 100644 index 0000000000..32c599e5ed --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.h @@ -0,0 +1,95 @@ +/* BDSP Eye Blink Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_EyeBlinkDetector_H +#define PokemonAutomation_PokemonBDSP_EyeBlinkDetector_H + +#include +#include +#include +#include +#include "Common/Cpp/Color.h" +#include "Common/Cpp/Concurrency/SpinLock.h" +#include "Common/Cpp/Time.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Rescale a search region to the equivalent from a 1080p capture +const size_t BDSP_TEMPLATE_REFERENCE_HEIGHT = 1080; + + +class EyeBlinkDetector{ +public: + EyeBlinkDetector(const ImageViewRGB32& open_eye, ImageFloatBox search_box); + + // Best normalized cross-correlation of the eye template + // https://en.wikipedia.org/wiki/Cross-correlation#Terminology_in_image_processing + double match(const ImageViewRGB32& frame) const; + + const ImageFloatBox& search_box() const{ return m_search_box; } + size_t template_width() const{ return (size_t)m_template.cols; } + size_t template_height() const{ return (size_t)m_template.rows; } + +private: + cv::Mat m_template; + ImageFloatBox m_search_box; +}; + + +struct BlinkMatchSample{ + WallClock timestamp; + double match; +}; + +struct BdspEyeTemplate{ + // Path under the PokemonBDSP/Rng resource folder, scene subfolder included. + std::string asset; + ImageFloatBox box; + std::string label; +}; + + +class EyeBlinkWatcher : public VisualInferenceCallback{ +public: + EyeBlinkWatcher( + std::string label, + const ImageViewRGB32& open_eye, + ImageFloatBox search_box, + Color color = COLOR_RED + ); + + const std::string& label() const{ return m_label; } + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool process_frame(const ImageViewRGB32& frame, WallClock timestamp) override; + + std::vector samples() const; + size_t sample_count() const; + double last_match() const; + + void discard_before(WallClock keep_from); + +private: + std::string m_label; + Color m_color; + EyeBlinkDetector m_detector; + + mutable SpinLock m_lock; + std::vector m_samples; + double m_last_match; +}; + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp new file mode 100644 index 0000000000..370d4854a3 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp @@ -0,0 +1,57 @@ +/* Player Model Select + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "PokemonBDSP_PlayerModelOption.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ + +struct PlayerModelIcons{ + std::vector icons; + StringSelectDatabase database; + + PlayerModelIcons(){ + icons.reserve(BDSP_PLAYER_MODEL_COUNT); + for (uint8_t model = 1; model <= BDSP_PLAYER_MODEL_COUNT; model++){ + std::string number = std::to_string(model); + icons.emplace_back( + RESOURCE_PATH() + "PokemonBDSP/Rng/ModelIcons/Model-" + number + ".png" + ); + database.add_entry(StringSelectEntry( + "model-" + number, "Model " + number, icons.back() + )); + } + } +}; +const PlayerModelIcons& PLAYER_MODEL_ICONS(){ + static PlayerModelIcons icons; + return icons; +} +} + + +PlayerModelOption::PlayerModelOption() + : StringSelectOption( + "Character model:
The appearance chosen for the player character. " + "Each one is watched through a different eye template.", + PLAYER_MODEL_ICONS().database, + LockMode::LOCK_WHILE_RUNNING, + "model-1" + ) +{} + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h new file mode 100644 index 0000000000..c6316924b0 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h @@ -0,0 +1,32 @@ +/* Player Model Select + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_PlayerModelOption_H +#define PokemonAutomation_PokemonBDSP_PlayerModelOption_H + +#include +#include "CommonTools/Options/StringSelectOption.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// The eight appearances the player character can be given at the start of the game. +const uint8_t BDSP_PLAYER_MODEL_COUNT = 8; + + +class PlayerModelOption : public StringSelectOption{ +public: + PlayerModelOption(); + uint8_t model_number() const{ return (uint8_t)(index() + 1); } +}; + + +} +} +} +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 985274f34a..e9e388d0cb 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1413,6 +1413,10 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Inference/BoxSystem/PokemonBDSP_BoxShinyDetector.h Source/PokemonBDSP/Inference/BoxSystem/PokemonBDSP_IvJudgeReader.cpp Source/PokemonBDSP/Inference/BoxSystem/PokemonBDSP_IvJudgeReader.h + Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp + Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.h + Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.cpp + Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.h Source/PokemonBDSP/Inference/Rng/PokemonBDSP_SummaryReader.cpp Source/PokemonBDSP/Inference/Rng/PokemonBDSP_SummaryReader.h Source/PokemonBDSP/Inference/PokemonBDSP_DialogDetector.cpp @@ -1453,6 +1457,8 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Options/PokemonBDSP_EggStepOption.h Source/PokemonBDSP/Options/PokemonBDSP_EncounterBotCommon.h Source/PokemonBDSP/Options/PokemonBDSP_LearnMove.h + Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp + Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h Source/PokemonBDSP/Options/PokemonBDSP_ShortcutDirection.cpp Source/PokemonBDSP/Options/PokemonBDSP_ShortcutDirection.h Source/PokemonBDSP/Panels_PokemonBDSP.cpp