add BDSP blink detection

SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.cpp

@@ -0,0 +1,58 @@
+/*  BDSP Blink Scenes
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include <string>
+#include "Common/Cpp/Exceptions.h"
+#include "PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.h"
+#include "PokemonBDSP_BlinkScenes.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonBDSP{
+
+
+std::vector<BdspEyeTemplate> lake_eye_templates(uint8_t player_model){
+    if (player_model < 1 || player_model > BDSP_PLAYER_MODEL_COUNT){
+        throw InternalProgramError(
+            nullptr, PA_CURRENT_FUNCTION,
+            "Unknown player model: " + std::to_string(player_model)
+        );
+    }
+    return {
+        BdspEyeTemplate{
+            "LakeTemplates/Model-" + std::to_string(player_model) + ".png",
+            {0.5026, 0.4472, 0.0240, 0.0426},
+            "Player"
+        },
+        BdspEyeTemplate{
+            "LakeTemplates/Barry.png",
+            {0.5521, 0.4528, 0.0214, 0.0380},
+            "Barry"
+        },
+    };
+}
+
+
+std::vector<BdspEyeTemplate> bedroom_eye_templates(uint8_t player_model){
+    if (player_model < 1 || player_model > BDSP_PLAYER_MODEL_COUNT){
+        throw InternalProgramError(
+            nullptr, PA_CURRENT_FUNCTION,
+            "Unknown player model: " + std::to_string(player_model)
+        );
+    }
+    return {
+        BdspEyeTemplate{
+            "BedroomTemplates/Model-" + std::to_string(player_model) + ".png",
+            {0.4708, 0.4500, 0.0214, 0.0491},
+            "Player"
+        },
+    };
+}
+
+
+}
+}
+}

SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_BlinkScenes.h

@@ -0,0 +1,29 @@
+/*  BDSP Blink Scenes
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#ifndef PokemonAutomation_PokemonBDSP_BlinkScenes_H
+#define PokemonAutomation_PokemonBDSP_BlinkScenes_H
+
+#include <stdint.h>
+#include <vector>
+#include "PokemonBDSP_EyeBlinkDetector.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonBDSP{
+
+
+//  Which eyes to watch, and where they sit, in each scene the RNG programs use.
+
+std::vector<BdspEyeTemplate> lake_eye_templates(uint8_t player_model);
+
+std::vector<BdspEyeTemplate> bedroom_eye_templates(uint8_t player_model);
+
+
+}
+}
+}
+#endif

SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.cpp

@@ -0,0 +1,150 @@
+/*  BDSP Eye Blink Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include <algorithm>
+#include <cmath>
+#include <opencv2/imgproc.hpp>
+#include "Common/Cpp/Exceptions.h"
+#include "CommonFramework/ImageTypes/ImageRGB32.h"
+#include "CommonFramework/ImageTypes/ImageRGB32_OpenCV.h"
+#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
+#include "PokemonBDSP_EyeBlinkDetector.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonBDSP{
+
+
+//  Returned when the frame cannot be matched at all. Callers read a high score
+//  as "eye open", so a frame we can't measure never gets counted as a blink.
+const double NO_MATCH = 1.0;
+
+
+EyeBlinkDetector::EyeBlinkDetector(const ImageViewRGB32& open_eye, ImageFloatBox search_box)
+    : m_search_box(search_box)
+{
+    if (!open_eye){
+        throw InternalProgramError(
+            nullptr, PA_CURRENT_FUNCTION,
+            "EyeBlinkDetector: No eye template was supplied."
+        );
+    }
+
+    cv::cvtColor(to_OpenCV_ref(open_eye), m_template, cv::COLOR_BGRA2GRAY);
+
+    cv::Scalar mean;
+    cv::Scalar stddev;
+    cv::meanStdDev(m_template, mean, stddev);
+    if (stddev[0] <= 0){
+        //  A flat template correlates with everything equally and can never see a blink
+        throw InternalProgramError(
+            nullptr, PA_CURRENT_FUNCTION,
+            "EyeBlinkDetector: The eye template is a single flat colour."
+        );
+    }
+}
+
+double EyeBlinkDetector::match(const ImageViewRGB32& frame) const{
+    if (!frame || frame.height() == 0){
+        return NO_MATCH;
+    }
+
+    ImageViewRGB32 region = extract_box_reference(frame, m_search_box);
+    if (!region){
+        return NO_MATCH;
+    }
+
+    //  Rescale the region to the resolution the template was cropped at
+    //  based on the capture height
+    ImageRGB32 rescaled;
+    if (frame.height() != BDSP_TEMPLATE_REFERENCE_HEIGHT){
+        double scale = (double)BDSP_TEMPLATE_REFERENCE_HEIGHT / (double)frame.height();
+        size_t width = (size_t)std::llround(region.width() * scale);
+        size_t height = (size_t)std::llround(region.height() * scale);
+        if (width == 0 || height == 0){
+            return NO_MATCH;
+        }
+        rescaled = region.scale_to(width, height);
+        region = rescaled;
+    }
+
+    if (region.width() < template_width() || region.height() < template_height()){
+        return NO_MATCH;
+    }
+
+    //  Match the greyscale image to the template by zero-normalized cross correlation
+    cv::Mat grey;
+    cv::cvtColor(to_OpenCV_ref(region), grey, cv::COLOR_BGRA2GRAY);
+
+    cv::Mat scores;
+    cv::matchTemplate(grey, m_template, scores, cv::TM_CCOEFF_NORMED);
+
+    double best = NO_MATCH;
+    cv::minMaxLoc(scores, nullptr, &best);
+    return best;
+}
+
+
+
+EyeBlinkWatcher::EyeBlinkWatcher(
+    std::string label,
+    const ImageViewRGB32& open_eye,
+    ImageFloatBox search_box,
+    Color color
+)
+    : VisualInferenceCallback(label)
+    , m_label(std::move(label))
+    , m_color(color)
+    , m_detector(open_eye, search_box)
+    , m_last_match(1.0)
+{}
+
+void EyeBlinkWatcher::make_overlays(VideoOverlaySet& items) const{
+    items.add(m_color, m_detector.search_box(), m_label);
+}
+
+bool EyeBlinkWatcher::process_frame(const ImageViewRGB32& frame, WallClock timestamp){
+    double value = m_detector.match(frame);
+    WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION);
+    m_samples.emplace_back(BlinkMatchSample{timestamp, value});
+    m_last_match = value;
+    //  Never stops the session
+    return false;
+}
+
+std::vector<BlinkMatchSample> EyeBlinkWatcher::samples() const{
+    ReadSpinLock lg(m_lock, PA_CURRENT_FUNCTION);
+    return m_samples;
+}
+size_t EyeBlinkWatcher::sample_count() const{
+    ReadSpinLock lg(m_lock, PA_CURRENT_FUNCTION);
+    return m_samples.size();
+}
+double EyeBlinkWatcher::last_match() const{
+    ReadSpinLock lg(m_lock, PA_CURRENT_FUNCTION);
+    return m_last_match;
+}
+
+void EyeBlinkWatcher::discard_before(WallClock keep_from){
+    WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION);
+    auto keep = std::lower_bound(
+        m_samples.begin(), m_samples.end(), keep_from,
+        [](const BlinkMatchSample& sample, WallClock cutoff){
+            return sample.timestamp < cutoff;
+        }
+    );
+    if (keep == m_samples.begin()){
+        return;
+    }
+    m_samples.erase(m_samples.begin(), keep);
+}
+
+
+
+
+}
+}
+}

SerialPrograms/Source/PokemonBDSP/Inference/Rng/PokemonBDSP_EyeBlinkDetector.h

@@ -0,0 +1,95 @@
+/*  BDSP Eye Blink Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#ifndef PokemonAutomation_PokemonBDSP_EyeBlinkDetector_H
+#define PokemonAutomation_PokemonBDSP_EyeBlinkDetector_H
+
+#include <stddef.h>
+#include <string>
+#include <vector>
+#include <opencv2/core/mat.hpp>
+#include "Common/Cpp/Color.h"
+#include "Common/Cpp/Concurrency/SpinLock.h"
+#include "Common/Cpp/Time.h"
+#include "CommonFramework/ImageTools/ImageBoxes.h"
+#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
+#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonBDSP{
+
+
+//  Rescale a search region to the equivalent from a 1080p capture
+const size_t BDSP_TEMPLATE_REFERENCE_HEIGHT = 1080;
+
+
+class EyeBlinkDetector{
+public:
+    EyeBlinkDetector(const ImageViewRGB32& open_eye, ImageFloatBox search_box);
+
+    //  Best normalized cross-correlation of the eye template
+    //  https://en.wikipedia.org/wiki/Cross-correlation#Terminology_in_image_processing
+    double match(const ImageViewRGB32& frame) const;
+
+    const ImageFloatBox& search_box() const{ return m_search_box; }
+    size_t template_width() const{ return (size_t)m_template.cols; }
+    size_t template_height() const{ return (size_t)m_template.rows; }
+
+private:
+    cv::Mat m_template;
+    ImageFloatBox m_search_box;
+};
+
+
+struct BlinkMatchSample{
+    WallClock timestamp;
+    double match;
+};
+
+struct BdspEyeTemplate{
+    //  Path under the PokemonBDSP/Rng resource folder, scene subfolder included.
+    std::string asset;
+    ImageFloatBox box;
+    std::string label;
+};
+
+
+class EyeBlinkWatcher : public VisualInferenceCallback{
+public:
+    EyeBlinkWatcher(
+        std::string label,
+        const ImageViewRGB32& open_eye,
+        ImageFloatBox search_box,
+        Color color = COLOR_RED
+    );
+
+    const std::string& label() const{ return m_label; }
+
+    virtual void make_overlays(VideoOverlaySet& items) const override;
+    virtual bool process_frame(const ImageViewRGB32& frame, WallClock timestamp) override;
+
+    std::vector<BlinkMatchSample> samples() const;
+    size_t sample_count() const;
+    double last_match() const;
+
+    void discard_before(WallClock keep_from);
+
+private:
+    std::string m_label;
+    Color m_color;
+    EyeBlinkDetector m_detector;
+
+    mutable SpinLock m_lock;
+    std::vector<BlinkMatchSample> m_samples;
+    double m_last_match;
+};
+
+
+}
+}
+}
+#endif

SerialPrograms/Source/PokemonBDSP/Options/PokemonBDSP_PlayerModelOption.cpp

@@ -0,0 +1,57 @@
+/*  Player Model Select
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include <string>
+#include <vector>
+#include "CommonFramework/GlobalAutoPaths.h"
+#include "CommonFramework/ImageTypes/ImageRGB32.h"
+#include "PokemonBDSP_PlayerModelOption.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonBDSP{
+
+
+namespace{
+
+struct PlayerModelIcons{
+    std::vector<ImageRGB32> icons;
+    StringSelectDatabase database;
+
+    PlayerModelIcons(){
+        icons.reserve(BDSP_PLAYER_MODEL_COUNT);
+        for (uint8_t model = 1; model <= BDSP_PLAYER_MODEL_COUNT; model++){
+            std::string number = std::to_string(model);
+            icons.emplace_back(
+                RESOURCE_PATH() + "PokemonBDSP/Rng/ModelIcons/Model-" + number + ".png"
+            );
+            database.add_entry(StringSelectEntry(
+                "model-" + number, "Model " + number, icons.back()
+            ));
+        }
+    }
+};
+const PlayerModelIcons& PLAYER_MODEL_ICONS(){
+    static PlayerModelIcons icons;
+    return icons;
+}
+}
+
+
+PlayerModelOption::PlayerModelOption()
+    : StringSelectOption(
+        "<b>Character model:</b><br>The appearance chosen for the player character. "
+        "Each one is watched through a different eye template.",
+        PLAYER_MODEL_ICONS().database,
+        LockMode::LOCK_WHILE_RUNNING,
+        "model-1"
+    )
+{}
+
+
+}
+}
+}