From 508765fc26522b2b936aac9a4b7f5c37be8cde94 Mon Sep 17 00:00:00 2001 From: denvoros Date: Thu, 23 Jul 2026 09:50:01 -0600 Subject: [PATCH 1/3] feat(video): add GPU backend as a start --- Common/Cpp/Options/EnumDropdownDatabase.h | 3 +- SerialPrograms/CMakeLists.txt | 15 +- .../Source/CommonFramework/Main.cpp | 13 +- .../Backends/CameraImplementations.cpp | 12 +- .../Backends/CameraImplementations.h | 4 +- .../Backends/CameraWidgetQt6.5.cpp | 361 ++++++++---------- .../Backends/CameraWidgetQt6_QML.cpp | 222 +++++++++++ .../Backends/CameraWidgetQt6_QML.h | 108 ++++++ .../Backends/CameraWidgetQt6_QVideoWidget.cpp | 201 ++++++++++ .../Backends/CameraWidgetQt6_QVideoWidget.h | 119 ++++++ .../CommonTools/Images/WaterfillUtilities.h | 1 + .../DiscordIntegrationSettings.cpp | 4 + .../ML/Inference/ML_PaddleOCRPipeline.h | 3 + .../Map/PokemonLZA_DirectionArrowDetector.cpp | 3 + SerialPrograms/cmake/SourceFiles.cmake | 3 + 15 files changed, 846 insertions(+), 226 deletions(-) create mode 100644 SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.cpp create mode 100644 SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.h create mode 100644 SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.cpp create mode 100644 SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.h diff --git a/Common/Cpp/Options/EnumDropdownDatabase.h b/Common/Cpp/Options/EnumDropdownDatabase.h index 84f27cbbf9..5a293c5642 100644 --- a/Common/Cpp/Options/EnumDropdownDatabase.h +++ b/Common/Cpp/Options/EnumDropdownDatabase.h @@ -92,8 +92,7 @@ class EnumDropdownDatabase : public IntegerEnumDropdownDatabase{ public: EnumDropdownDatabase() : IntegerEnumDropdownDatabase() {} EnumDropdownDatabase(std::initializer_list list){ - size_t index = 0; - for (auto iter = list.begin(); iter != list.end(); ++iter, index++){ + for (auto iter = list.begin(); iter != list.end(); ++iter){ add( iter->value, std::move(iter->slug), diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index 0880ad7a99..44302c8898 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -78,10 +78,10 @@ if(WIN32 AND QT_DEPLOY_FILES) if(QT_CANDIDATE_DIR AND EXISTS "${QT_CANDIDATE_DIR}") message(STATUS "Using preferred Qt directory for Qt${QT_MAJOR} ${PREFERRED_QT_VER}: ${QT_CANDIDATE_DIR}") list(APPEND CMAKE_PREFIX_PATH "${QT_CANDIDATE_DIR}") - find_package(Qt${QT_MAJOR} ${PREFERRED_QT_VER} COMPONENTS Widgets SerialPort Multimedia MultimediaWidgets REQUIRED) + find_package(Qt${QT_MAJOR} ${PREFERRED_QT_VER} COMPONENTS Widgets SerialPort Multimedia MultimediaWidgets OpenGLWidgets Qml Quick QuickWidgets REQUIRED) else() # Find all subdirectories in the Qt base directory - find_package(Qt${QT_MAJOR} COMPONENTS Widgets SerialPort Multimedia MultimediaWidgets REQUIRED) + find_package(Qt${QT_MAJOR} COMPONENTS Widgets SerialPort Multimedia MultimediaWidgets OpenGLWidgets Qml Quick QuickWidgets REQUIRED) file(GLOB QT_SUBDIRS LIST_DIRECTORIES true "${QT_BASE_DIR}/${QT_MAJOR}*") # Filter and sort the directories to find the latest version @@ -109,7 +109,7 @@ if(WIN32 AND QT_DEPLOY_FILES) REQUIRED ) else() - find_package(Qt${QT_MAJOR} COMPONENTS Widgets SerialPort Multimedia MultimediaWidgets REQUIRED) + find_package(Qt${QT_MAJOR} COMPONENTS Widgets SerialPort Multimedia MultimediaWidgets OpenGLWidgets Qml Quick QuickWidgets REQUIRED) endif() # disable deprecated Qt APIs @@ -188,7 +188,7 @@ endif() # Function to apply common properties to both library and executable targets function(apply_common_target_properties target_name) set_target_properties(${target_name} PROPERTIES LINKER_LANGUAGE CXX) - target_link_libraries(${target_name} PRIVATE Qt${QT_MAJOR}::Widgets Qt${QT_MAJOR}::SerialPort Qt${QT_MAJOR}::Multimedia Qt${QT_MAJOR}::MultimediaWidgets) + target_link_libraries(${target_name} PRIVATE Qt${QT_MAJOR}::Widgets Qt${QT_MAJOR}::SerialPort Qt${QT_MAJOR}::Multimedia Qt${QT_MAJOR}::MultimediaWidgets Qt${QT_MAJOR}::OpenGLWidgets Qt${QT_MAJOR}::Qml Qt${QT_MAJOR}::Quick Qt${QT_MAJOR}::QuickWidgets) target_link_libraries(${target_name} PRIVATE Threads::Threads) #add defines @@ -468,10 +468,9 @@ else() # macOS and Linux endif() # end Linux # Find OpenCV - pkg_search_module(OpenCV REQUIRED opencv4 opencv) - target_include_directories(SerialProgramsLib SYSTEM PRIVATE ${OpenCV_INCLUDE_DIRS}) # "SYSTEM" to suppress warnings - target_link_directories(SerialProgramsLib PUBLIC ${OpenCV_LIBRARY_DIRS}) - target_link_libraries(SerialProgramsLib PUBLIC ${OpenCV_LINK_LIBRARIES}) + find_package(OpenCV REQUIRED) + target_include_directories(SerialProgramsLib SYSTEM PUBLIC ${OpenCV_INCLUDE_DIRS}) # "SYSTEM" to suppress warnings + target_link_libraries(SerialProgramsLib PUBLIC ${OpenCV_LIBS}) #we hope to use our own Tesseract build in future so we can rid that dependency #but right now to run on Linux and Mac we need to use external Tesseract library diff --git a/SerialPrograms/Source/CommonFramework/Main.cpp b/SerialPrograms/Source/CommonFramework/Main.cpp index 0ff2e130d5..6e92eefe1b 100644 --- a/SerialPrograms/Source/CommonFramework/Main.cpp +++ b/SerialPrograms/Source/CommonFramework/Main.cpp @@ -227,11 +227,18 @@ int main(int argc, char *argv[]){ #endif #if defined(__linux__) - // Qt multimedia, default to gstreamer to prevent flickering - // Easier than the alternative which is compiling qt6multimedia with QT_DEFAULT_MEDIA_BACKEND + // Enable libv4l2 for GStreamer by default to fix issues with cheap capture cards + // This helps prevent "Invalid argument (22)" buffer allocation errors on v4l2src. + if (qEnvironmentVariableIsEmpty("GST_V4L2_USE_LIBV4L2")) { + qputenv("GST_V4L2_USE_LIBV4L2", "1"); + } + + // Qt multimedia, default to ffmpeg to prevent GStreamer capture drops. + // GStreamer used to be preferred to prevent flickering but causes fatal + // crashes (buffer allocation errors) with cheap capture cards on Linux. // See: https://doc.qt.io/qt-6.5/qtmultimedia-index.html if (qEnvironmentVariableIsEmpty("QT_MEDIA_BACKEND")) - qputenv("QT_MEDIA_BACKEND", "gstreamer"); + qputenv("QT_MEDIA_BACKEND", "ffmpeg"); #endif // So far, this is only needed on Mac where static initialization is fucked up. diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp index 446a722af3..1b7ac15b63 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp @@ -17,9 +17,9 @@ #if 0 #elif QT_VERSION_MAJOR == 6 #include "CameraWidgetQt6.h" -#if QT_VERSION_MINOR >= 5 #include "CameraWidgetQt6.5.h" -#endif +#include "CameraWidgetQt6_QVideoWidget.h" +#include "CameraWidgetQt6_QML.h" #endif @@ -55,6 +55,14 @@ struct CameraBackends{ "qt6-QVideoSink", "Qt6: QVideoSink", std::make_unique() ); + m_backends.emplace_back( + "qt6-qvideowidget", "Qt6: GPU Offloaded (QOpenGLWidget)", + std::make_unique() + ); + m_backends.emplace_back( + "qt6-qml", "Qt6: GPU Offloaded (QML QQuickWidget)", + std::make_unique() + ); #endif #if QT_VERSION_MAJOR == 6 && QT_VERSION_MINOR >= 5 m_backends.emplace_back( diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.h index b7cac8f497..430a21a107 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.h @@ -23,7 +23,7 @@ class VideoBackendOption : public IntegerEnumDropdownOption{ // Abstract base class for camera backend implementation. // Call `get_camera_backend()` to get the camera backend singleton. -// Current camera backend is implemented in CameraWidgetQt6.h/cpp. +// Qt6 camera backends are implemented in CameraWidgetQt6*.h/cpp. class CameraBackend{ public: virtual ~CameraBackend() = default; @@ -47,7 +47,7 @@ class CameraBackend{ // Get the camera backend singleton. -// Current camera backend is implemented in CameraWidgetQt6.h/cpp. +// Qt6 camera backends are implemented in CameraWidgetQt6*.h/cpp. const CameraBackend& get_camera_backend(); // Call `get_camera_backend()` to get camera backend reference and get all cameras' info. diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp index 9bdab15789..fa411502b5 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp @@ -1,4 +1,4 @@ -/* Camera Widget (Qt6.5) +/* Camera Widget (Qt6.5) * * From: https://github.com/PokemonAutomation/ * @@ -7,163 +7,135 @@ #include #if QT_VERSION_MAJOR == 6 && QT_VERSION_MINOR >= 5 -//#include +// #include #include -#include +#include #include +#include +#include #include #include -#include -//#include "Common/Cpp/Exceptions.h" -//#include "Common/Cpp/Time.h" -//#include "Common/Cpp/PrettyPrint.h" +// #include "Common/Cpp/Exceptions.h" +// #include "Common/Cpp/Time.h" +// #include "Common/Cpp/PrettyPrint.h" +#include "CameraWidgetQt6.5.h" +#include "CameraWidgetQt6.h" #include "Common/Qt/Redispatch.h" #include "CommonFramework/Logging/Logger.h" -#include "VideoFrameQt.h" #include "MediaServicesQt6.h" -#include "CameraWidgetQt6.h" -#include "CameraWidgetQt6.5.h" - -//#include -//using std::cout; -//using std::endl; - -namespace PokemonAutomation{ -namespace CameraQt65QMediaCaptureSession{ - - - +#include "VideoFrameQt.h" +// #include +// using std::cout; +// using std::endl; +namespace PokemonAutomation { +namespace CameraQt65QMediaCaptureSession { -std::vector CameraBackend::get_all_cameras() const{ +std::vector CameraBackend::get_all_cameras() const { #if 1 - const auto cameras = GlobalMediaServices::instance().get_all_cameras(); + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); #else - const auto cameras = QMediaDevices::videoInputs(); + const auto cameras = QMediaDevices::videoInputs(); #endif - std::vector ret; - for (const auto& info : cameras){ - ret.emplace_back(info.id().toStdString()); - } - return ret; + std::vector ret; + for (const auto &info : cameras) { + ret.emplace_back(info.id().toStdString()); + } + return ret; } -std::string CameraBackend::get_camera_name(const CameraInfo& info) const{ +std::string CameraBackend::get_camera_name(const CameraInfo &info) const { #if 1 - const auto cameras = GlobalMediaServices::instance().get_all_cameras(); + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); #else - const auto cameras = QMediaDevices::videoInputs(); + const auto cameras = QMediaDevices::videoInputs(); #endif - for (const auto& camera : cameras){ - if (camera.id().toStdString() == info.device_name()){ - return camera.description().toStdString(); - } + for (const auto &camera : cameras) { + if (camera.id().toStdString() == info.device_name()) { + return camera.description().toStdString(); } - global_logger_tagged().log( - "Error: No such camera for CameraInfo: " + info.device_name(), - COLOR_RED - ); - return ""; + } + global_logger_tagged().log( + "Error: No such camera for CameraInfo: " + info.device_name(), COLOR_RED); + return ""; } -std::unique_ptr CameraBackend::make_video_source( - Logger& logger, - const CameraInfo& info, - Resolution resolution, - VideoFormat format, - FramesPerSecond fps -) const{ - return std::make_unique(logger, info, resolution, format, fps); +std::unique_ptr +CameraBackend::make_video_source(Logger &logger, const CameraInfo &info, + Resolution resolution, VideoFormat format, + FramesPerSecond fps) const { + return std::make_unique(logger, info, resolution, format, + fps); } - - - - - - -CameraVideoSource::~CameraVideoSource(){ - if (!m_capture_session){ - return; - } - try{ - m_logger.log("Stopping Camera..."); - }catch (...){} - - run_on_main_thread_and_wait([&]{ -// m_camera->stop(); - m_capture_session.reset(); - m_camera.reset(); - }); +CameraVideoSource::~CameraVideoSource() { + if (!m_capture_session) { + return; + } + try { + m_logger.log("Stopping Camera..."); + } catch (...) { + } + + run_on_main_thread_and_wait([&] { + // m_camera->stop(); + m_capture_session.reset(); + m_camera.reset(); + }); } -CameraVideoSource::CameraVideoSource( - Logger& logger, - const CameraInfo& info, - Resolution desired_resolution, - VideoFormat desired_format, - FramesPerSecond desired_fps -) - : VideoSource(logger, true) - , m_logger(logger) - , m_last_frame(logger) - , m_snapshot_manager(logger, m_last_frame) -{ -// cout << "desired_resolution = " << desired_resolution.width << " x " << desired_resolution.height << endl; -// cout << "desired_fps = " << desired_fps << endl; - - if (!info){ - return; - } - m_logger.log("Starting Camera: Backend = CameraQt65QMediaCaptureSession"); - - run_on_main_thread_and_wait([&]{ - init(info, desired_resolution, desired_format, desired_fps); - }); +CameraVideoSource::CameraVideoSource(Logger &logger, const CameraInfo &info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps) + : VideoSource(logger, true), m_logger(logger), m_last_frame(logger), + m_snapshot_manager(logger, m_last_frame) { + // cout << "desired_resolution = " << desired_resolution.width << " x " << + // desired_resolution.height << endl; cout << "desired_fps = " << + // desired_fps << endl; + + if (!info) { + return; + } + m_logger.log("Starting Camera: Backend = CameraQt65QMediaCaptureSession"); + + run_on_main_thread_and_wait( + [&] { init(info, desired_resolution, desired_format, desired_fps); }); } -void CameraVideoSource::init( - const CameraInfo& info, - Resolution desired_resolution, - VideoFormat desired_format, - FramesPerSecond desired_fps -){ - m_metaobject.reset(new QObject()); - - auto cameras = QMediaDevices::videoInputs(); - const QCameraDevice* device = nullptr; - for (const auto& camera : cameras){ - if (camera.id().toStdString() == info.device_name()){ - device = &camera; - break; - } - } - if (device == nullptr){ - m_logger.log("Camera not found: " + info.device_name(), COLOR_RED); - return; +void CameraVideoSource::init(const CameraInfo &info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps) { + m_metaobject.reset(new QObject()); + + auto cameras = QMediaDevices::videoInputs(); + const QCameraDevice *device = nullptr; + for (const auto &camera : cameras) { + if (camera.id().toStdString() == info.device_name()) { + device = &camera; + break; } - m_logger.log("Camera: " + device->description().toStdString()); - - QCameraFormat format = CameraQt6QVideoSink::build_format_set( - m_logger, - m_formats, - *device, - desired_resolution, - desired_format, - desired_fps - ); - if (format.isNull()){ - return; - } - - CameraQt6QVideoSink::get_format(format, m_resolution, m_format, m_fps); - m_logger.log( - "Resolution: " + m_resolution.to_string() + - ", Format: " + VideoFormat_database().find(m_format)->display + - ", FPS: " + std::to_string(m_fps) - ); - - m_camera.reset(new QCameraThread(m_logger, *device, format)); - - m_capture_session.reset(new QMediaCaptureSession()); - m_capture_session->setCamera(&m_camera->camera()); + } + if (device == nullptr) { + m_logger.log("Camera not found: " + info.device_name(), COLOR_RED); + return; + } + m_logger.log("Camera: " + device->description().toStdString()); + + QCameraFormat format = CameraQt6QVideoSink::build_format_set( + m_logger, m_formats, *device, desired_resolution, desired_format, + desired_fps); + if (format.isNull()) { + return; + } + + CameraQt6QVideoSink::get_format(format, m_resolution, m_format, m_fps); + m_logger.log("Resolution: " + m_resolution.to_string() + + ", Format: " + VideoFormat_database().find(m_format)->display + + ", FPS: " + std::to_string(m_fps)); + + m_camera.reset(new QCameraThread(m_logger, *device, format)); + + m_capture_session.reset(new QMediaCaptureSession()); + m_capture_session->setCamera(&m_camera->camera()); #if 0 connect(m_camera.get(), &QCamera::errorOccurred, this, [&](){ @@ -177,89 +149,60 @@ void CameraVideoSource::init( #endif } - -void CameraVideoSource::set_video_output(QGraphicsVideoItem& item){ - if (m_capture_session == nullptr){ - return; - } - if (m_capture_session->videoSink() == item.videoSink()){ - return; - } - m_capture_session->setVideoOutput(&item); - - m_metaobject->connect( - item.videoSink(), &QVideoSink::videoFrameChanged, - &m_camera->camera(), [&](const QVideoFrame& frame){ - // This runs on the QCamera's thread. So it is off the critical path. - - WallClock now = current_time(); - if (!m_last_frame.push_frame(frame, now)){ - return; - } - report_source_frame(std::make_shared(now, frame)); - } - ); +void CameraVideoSource::set_video_output(QGraphicsVideoItem &item) { + if (m_capture_session == nullptr) { + return; + } + if (m_capture_session->videoSink() == item.videoSink()) { + return; + } + m_capture_session->setVideoOutput(&item); + + m_metaobject->connect(item.videoSink(), &QVideoSink::videoFrameChanged, + &m_camera->camera(), [&](const QVideoFrame &frame) { + // This runs on the QCamera's thread. So it is off + // the critical path. + + WallClock now = current_time(); + if (!m_last_frame.push_frame(frame, now)) { + return; + } + report_source_frame( + std::make_shared(now, frame)); + }); } - - - - - -QWidget* CameraVideoSource::make_display_QtWidget(QWidget* parent){ - return new CameraVideoDisplay(parent, *this); +QWidget *CameraVideoSource::make_display_QtWidget(QWidget *parent) { + return new CameraVideoDisplay(parent, *this); } +CameraVideoDisplay::CameraVideoDisplay(QWidget *parent, + CameraVideoSource &source) + : QWidget(parent), m_source(source), m_view(new StaticQGraphicsView(this)), + m_sanitizer("CameraVideoDisplay") { + this->setMinimumSize(80, 45); + // m_view->setViewport(new QOpenGLWidget()); + m_view->setFixedSize(this->size()); + m_view->setScene(&m_scene); + m_video.setSize(this->size()); + m_scene.setSceneRect(QRectF(QPointF(0, 0), this->size())); + m_scene.addItem(&m_video); + source.set_video_output(m_video); - - - - -CameraVideoDisplay::CameraVideoDisplay(QWidget* parent, CameraVideoSource& source) - : QWidget(parent) - , m_source(source) - , m_view(new StaticQGraphicsView(this)) - , m_sanitizer("CameraVideoDisplay") -{ - this->setMinimumSize(80, 45); - m_view->setFixedSize(this->size()); - m_view->setScene(&m_scene); - m_video.setSize(this->size()); - m_scene.setSceneRect(QRectF(QPointF(0, 0), this->size())); - m_scene.addItem(&m_video); - source.set_video_output(m_video); - - connect( - &m_scene, &QGraphicsScene::changed, - this, [&](const QList&){ - auto scope_check = m_sanitizer.check_scope(); - m_source.report_rendered_frame(current_time()); - } - ); -} -void CameraVideoDisplay::resizeEvent(QResizeEvent* event){ + connect(&m_scene, &QGraphicsScene::changed, this, [&](const QList &) { auto scope_check = m_sanitizer.check_scope(); - m_view->setFixedSize(this->size()); - m_scene.setSceneRect(QRectF(QPointF(0, 0), this->size())); - m_video.setSize(this->size()); -} - - - - - - - - - - - - - - - + m_source.report_rendered_frame(current_time()); + }); } +void CameraVideoDisplay::resizeEvent(QResizeEvent *event) { + auto scope_check = m_sanitizer.check_scope(); + m_view->setFixedSize(this->size()); + m_scene.setSceneRect(QRectF(QPointF(0, 0), this->size())); + m_video.setSize(this->size()); } + +} // namespace CameraQt65QMediaCaptureSession +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.cpp new file mode 100644 index 0000000000..32b331c124 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.cpp @@ -0,0 +1,222 @@ +#include +#if QT_VERSION_MAJOR == 6 + +#include +#include +#include +#include +#include +#include "Common/Qt/Redispatch.h" +#include "CommonFramework/Logging/Logger.h" +#include "CameraWidgetQt6.h" +#include "CameraWidgetQt6_QML.h" +#include "MediaServicesQt6.h" +#include "VideoFrameQt.h" + +namespace PokemonAutomation{ +namespace CameraQt6QML{ + +std::vector CameraBackend::get_all_cameras() const{ +#if 1 + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); +#else + const auto cameras = QMediaDevices::videoInputs(); +#endif + std::vector ret; + for (const auto& info : cameras){ + ret.emplace_back(info.id().toStdString()); + } + return ret; +} +std::string CameraBackend::get_camera_name(const CameraInfo& info) const{ +#if 1 + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); +#else + const auto cameras = QMediaDevices::videoInputs(); +#endif + for (const auto& camera : cameras){ + if (camera.id().toStdString() == info.device_name()){ + return camera.description().toStdString(); + } + } + global_logger_tagged().log("Error: No such camera for CameraInfo: " + info.device_name(), COLOR_RED); + return ""; +} + +std::unique_ptr CameraBackend::make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps +) const{ + return std::make_unique(logger, info, resolution, format, fps); +} + + +CameraVideoSource::~CameraVideoSource(){ + if (!m_capture){ + return; + } + try{ + m_logger.log("Stopping Camera..."); + }catch (...){} + + run_on_main_thread_and_wait([&]{ + m_metaobject.reset(); + m_capture.reset(); + m_camera.reset(); + }); +} +CameraVideoSource::CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps +) + : VideoSource(logger, true) + , m_logger(logger) + , m_last_frame(logger) + , m_snapshot_manager(logger, m_last_frame) +{ + if (!info){ + return; + } + m_logger.log("Starting Camera: Backend = CameraQt6QML"); + + run_on_main_thread_and_wait([&]{ + init(info, desired_resolution, desired_format, desired_fps); + }); +} +void CameraVideoSource::init( + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps +){ + m_metaobject.reset(new QObject()); + + auto cameras = QMediaDevices::videoInputs(); + const QCameraDevice* device = nullptr; + for (const auto& camera : cameras){ + if (camera.id().toStdString() == info.device_name()){ + device = &camera; + break; + } + } + if (device == nullptr){ + m_logger.log("Camera not found: " + info.device_name(), COLOR_RED); + return; + } + m_logger.log("Camera: " + device->description().toStdString()); + + QCameraFormat format = CameraQt6QVideoSink::build_format_set( + m_logger, + m_formats, + *device, + desired_resolution, + desired_format, + desired_fps + ); + if (format.isNull()){ + return; + } + + CameraQt6QVideoSink::get_format(format, m_resolution, m_format, m_fps); + m_logger.log( + "Resolution: " + m_resolution.to_string() + + ", Format: " + VideoFormat_database().find(m_format)->display + + ", FPS: " + std::to_string(m_fps) + ); + + m_camera.reset(new QCameraThread(m_logger, *device, format)); + m_video_sink.reset(new QVideoSink()); + m_capture.reset(new QMediaCaptureSession()); + m_capture->setCamera(&m_camera->camera()); + m_capture->setVideoSink(m_video_sink.get()); + + // Auto-recovery: If the capture card glitches and the pipeline breaks, try to automatically restart it. + m_metaobject->connect(&m_camera->camera(), &QCamera::errorOccurred, + &m_camera->camera(), [this](QCamera::Error, const QString& errorString){ + m_logger.log("Camera error detected: " + errorString.toStdString() + ". Attempting auto-restart...", COLOR_RED); + QMetaObject::invokeMethod(&m_camera->camera(), "start", Qt::QueuedConnection); + }); +} + +QWidget* CameraVideoSource::make_display_QtWidget(QWidget* parent){ + CameraVideoDisplay* display = new CameraVideoDisplay(parent, *this); + + QObject* root = display->rootObject(); + if (root){ + QObject* vo = root->findChild("videoOutput"); + if (vo){ + QVideoSink* qml_sink = vo->property("videoSink").value(); + if (qml_sink){ + m_metaobject->disconnect(); + m_metaobject->connect( + m_video_sink.get(), &QVideoSink::videoFrameChanged, + qml_sink, [this, qml_sink](const QVideoFrame& frame){ + WallClock now = current_time(); + if (!m_last_frame.push_frame(frame, now)){ + return; + } + + report_source_frame(std::make_shared(now, frame)); + + QMetaObject::invokeMethod(qml_sink, [qml_sink, frame]() { + qml_sink->setVideoFrame(frame); + }, Qt::QueuedConnection); + + report_rendered_frame(now); + } + ); + } + } + } + + return display; +} + +CameraVideoDisplay::~CameraVideoDisplay(){ + m_source.remove_source_frame_listener(*this); +} +CameraVideoDisplay::CameraVideoDisplay(QWidget* parent, CameraVideoSource& source) + : QQuickWidget(parent) + , m_source(source) + , m_sanitizer("CameraVideoDisplay") +{ + this->setMinimumSize(80, 45); + this->setResizeMode(QQuickWidget::SizeRootObjectToView); + + QString qml = QStringLiteral( + "import QtQuick\n" + "import QtMultimedia\n" + "Rectangle {\n" + " color: \"black\"\n" + " VideoOutput {\n" + " id: videoOutput\n" + " anchors.fill: parent\n" + " objectName: \"videoOutput\"\n" + " fillMode: VideoOutput.PreserveAspectFit\n" + " }\n" + "}\n" + ); + + QTemporaryFile file; + if (file.open()){ + file.write(qml.toUtf8()); + file.close(); + this->setSource(QUrl::fromLocalFile(file.fileName())); + } + + source.add_source_frame_listener(*this); +} + +void CameraVideoDisplay::on_frame(std::shared_ptr frame){ + // Not needed, QML handles the rendering directly via SceneGraph +} + +} +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.h new file mode 100644 index 0000000000..77cf6b88e5 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.h @@ -0,0 +1,108 @@ +#ifndef PokemonAutomation_VideoPipeline_Qt6QML_H +#define PokemonAutomation_VideoPipeline_Qt6QML_H + +#include +#if QT_VERSION_MAJOR == 6 + +#include +#include +#include +#include +#include "CommonFramework/Tools/StatAccumulator.h" +#include "CommonFramework/VideoPipeline/VideoSource.h" +#include "CommonFramework/VideoPipeline/CameraInfo.h" +#include "QCameraThread.h" +#include "QVideoFrameCache.h" +#include "SnapshotManager.h" +#include "CameraImplementations.h" + +class QCamera; +class QVideoSink; + +namespace PokemonAutomation{ +namespace CameraQt6QML{ + +class CameraBackend : public PokemonAutomation::CameraBackend{ +public: + virtual std::vector get_all_cameras() const override; + virtual std::string get_camera_name(const CameraInfo& info) const override; + + virtual std::unique_ptr make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps + ) const override; +}; + +class CameraVideoSource : public VideoSource{ +public: + virtual ~CameraVideoSource(); + CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps + ); + + virtual Resolution current_resolution() const override{ return m_resolution; } + virtual VideoFormat current_format() const override{ return m_format; } + virtual FramesPerSecond current_fps() const override{ return m_fps; } + virtual const VideoFormatSet& supported_formats() const override{ return m_formats; } + + virtual VideoSnapshot snapshot_latest_blocking() override{ + return m_snapshot_manager.snapshot_latest_blocking(); + } + virtual VideoSnapshot snapshot_recent_nonblocking(WallClock min_time) override{ + return m_snapshot_manager.snapshot_recent_nonblocking(min_time); + } + + virtual QWidget* make_display_QtWidget(QWidget* parent) override; + +private: + void init( + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps + ); + +private: + friend class CameraVideoDisplay; + + std::unique_ptr m_metaobject; + + Logger& m_logger; + Resolution m_resolution; + VideoFormat m_format; + FramesPerSecond m_fps; + + std::unique_ptr m_camera; + std::unique_ptr m_video_sink; + std::unique_ptr m_capture; + + VideoFormatSet m_formats; + +private: + QVideoFrameCache m_last_frame; + SnapshotManager m_snapshot_manager; +}; + +class CameraVideoDisplay : public QQuickWidget, public VideoFrameListener{ +public: + virtual ~CameraVideoDisplay(); + CameraVideoDisplay(QWidget* parent, CameraVideoSource& source); + + virtual void on_frame(std::shared_ptr frame) override; + +private: + CameraVideoSource& m_source; + LifetimeSanitizer m_sanitizer; +}; + +} +} +#endif +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.cpp new file mode 100644 index 0000000000..2738474ce2 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.cpp @@ -0,0 +1,201 @@ +/* Camera Widget (Qt6 QVideoWidget) + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#if QT_VERSION_MAJOR == 6 + +#include +#include +#include +#include "Common/Qt/Redispatch.h" +#include "CommonFramework/Logging/Logger.h" +#include "CameraWidgetQt6.h" +#include "CameraWidgetQt6_QVideoWidget.h" +#include "MediaServicesQt6.h" +#include "VideoFrameQt.h" + +namespace PokemonAutomation{ +namespace CameraQt6QVideoWidget{ + +std::vector CameraBackend::get_all_cameras() const{ +#if 1 + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); +#else + const auto cameras = QMediaDevices::videoInputs(); +#endif + std::vector ret; + for (const auto& info : cameras){ + ret.emplace_back(info.id().toStdString()); + } + return ret; +} +std::string CameraBackend::get_camera_name(const CameraInfo& info) const{ +#if 1 + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); +#else + const auto cameras = QMediaDevices::videoInputs(); +#endif + for (const auto& camera : cameras){ + if (camera.id().toStdString() == info.device_name()){ + return camera.description().toStdString(); + } + } + global_logger_tagged().log("Error: No such camera for CameraInfo: " + info.device_name(), COLOR_RED); + return ""; +} + +std::unique_ptr CameraBackend::make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps +) const{ + return std::make_unique(logger, info, resolution, format, fps); +} + + +CameraVideoSource::~CameraVideoSource(){ + if (!m_capture){ + return; + } + try{ + m_logger.log("Stopping Camera..."); + }catch (...){} + + run_on_main_thread_and_wait([&]{ + m_metaobject.reset(); + m_capture.reset(); + m_video_sink.reset(); + m_camera.reset(); + }); +} +CameraVideoSource::CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps +) + : VideoSource(logger, true) + , m_logger(logger) + , m_last_frame(logger) + , m_snapshot_manager(logger, m_last_frame) +{ + if (!info){ + return; + } + m_logger.log("Starting Camera: Backend = CameraQt6QVideoWidget"); + + run_on_main_thread_and_wait([&]{ + init(info, desired_resolution, desired_format, desired_fps); + }); +} +void CameraVideoSource::init( + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps +){ + m_metaobject.reset(new QObject()); + + auto cameras = QMediaDevices::videoInputs(); + const QCameraDevice* device = nullptr; + for (const auto& camera : cameras){ + if (camera.id().toStdString() == info.device_name()){ + device = &camera; + break; + } + } + if (device == nullptr){ + m_logger.log("Camera not found: " + info.device_name(), COLOR_RED); + return; + } + m_logger.log("Camera: " + device->description().toStdString()); + + QCameraFormat format = CameraQt6QVideoSink::build_format_set( + m_logger, + m_formats, + *device, + desired_resolution, + desired_format, + desired_fps + ); + if (format.isNull()){ + return; + } + + CameraQt6QVideoSink::get_format(format, m_resolution, m_format, m_fps); + m_logger.log( + "Resolution: " + m_resolution.to_string() + + ", Format: " + VideoFormat_database().find(m_format)->display + + ", FPS: " + std::to_string(m_fps) + ); + + m_camera.reset(new QCameraThread(m_logger, *device, format)); + m_video_sink.reset(new QVideoSink()); + m_capture.reset(new QMediaCaptureSession()); + m_capture->setCamera(&m_camera->camera()); + m_capture->setVideoSink(m_video_sink.get()); + + m_metaobject->connect( + m_video_sink.get(), &QVideoSink::videoFrameChanged, + &m_camera->camera(), [&](const QVideoFrame& frame){ + WallClock now = current_time(); + if (!m_last_frame.push_frame(frame, now)){ + return; + } + report_source_frame(std::make_shared(now, frame)); + } + ); +} + +QWidget* CameraVideoSource::make_display_QtWidget(QWidget* parent){ + return new CameraVideoDisplay(parent, *this); +} + +CameraVideoDisplay::~CameraVideoDisplay(){ + m_source.remove_source_frame_listener(*this); +} +CameraVideoDisplay::CameraVideoDisplay(QWidget* parent, CameraVideoSource& source) + : QOpenGLWidget(parent) + , m_source(source) + , m_sanitizer("CameraVideoDisplay") +{ + this->setMinimumSize(80, 45); + source.add_source_frame_listener(*this); +} + +void CameraVideoDisplay::on_frame(std::shared_ptr frame){ + QMetaObject::invokeMethod(this, [this, frame = std::move(frame)]() mutable { + m_last_frame = std::move(frame); + this->update(); + }, Qt::QueuedConnection); +} + +void CameraVideoDisplay::paintEvent(QPaintEvent* event){ + QOpenGLWidget::paintEvent(event); + + if (!m_last_frame){ + return; + } + + QVideoFrame frame = m_last_frame->frame; + if (!frame.isValid()){ + return; + } + + QRect rect(0, 0, this->width(), this->height()); + QVideoFrame::PaintOptions options; + QPainter painter(this); + + frame.paint(&painter, rect, options); + m_source.report_rendered_frame(current_time()); +} + +} +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.h new file mode 100644 index 0000000000..e86df68d75 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.h @@ -0,0 +1,119 @@ +/* Camera Widget (Qt6 QVideoWidget) + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_VideoPipeline_Qt6QVideoWidget_H +#define PokemonAutomation_VideoPipeline_Qt6QVideoWidget_H + +#include +#if QT_VERSION_MAJOR == 6 + +// NOTE: Maintainers must ensure QT += multimediawidgets (or find_package(Qt6 COMPONENTS MultimediaWidgets REQUIRED)) is included in the build system. + +#include +#include +#include +#include +#include "CommonFramework/Tools/StatAccumulator.h" +#include "CommonFramework/VideoPipeline/VideoSource.h" +#include "CommonFramework/VideoPipeline/CameraInfo.h" +#include "QCameraThread.h" +#include "QVideoFrameCache.h" +#include "SnapshotManager.h" +#include "CameraImplementations.h" + +class QCamera; +class QVideoSink; + +namespace PokemonAutomation{ +namespace CameraQt6QVideoWidget{ + +class CameraBackend : public PokemonAutomation::CameraBackend{ +public: + virtual std::vector get_all_cameras() const override; + virtual std::string get_camera_name(const CameraInfo& info) const override; + + virtual std::unique_ptr make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps + ) const override; +}; + +class CameraVideoSource : public VideoSource{ +public: + virtual ~CameraVideoSource(); + CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps + ); + + virtual Resolution current_resolution() const override{ return m_resolution; } + virtual VideoFormat current_format() const override{ return m_format; } + virtual FramesPerSecond current_fps() const override{ return m_fps; } + virtual const VideoFormatSet& supported_formats() const override{ return m_formats; } + + virtual VideoSnapshot snapshot_latest_blocking() override{ + return m_snapshot_manager.snapshot_latest_blocking(); + } + virtual VideoSnapshot snapshot_recent_nonblocking(WallClock min_time) override{ + return m_snapshot_manager.snapshot_recent_nonblocking(min_time); + } + + virtual QWidget* make_display_QtWidget(QWidget* parent) override; + +private: + void init( + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps + ); + +private: + friend class CameraVideoDisplay; + + std::unique_ptr m_metaobject; + + Logger& m_logger; + Resolution m_resolution; + VideoFormat m_format; + FramesPerSecond m_fps; + + std::unique_ptr m_camera; + std::unique_ptr m_video_sink; + std::unique_ptr m_capture; + + VideoFormatSet m_formats; + +private: + QVideoFrameCache m_last_frame; + SnapshotManager m_snapshot_manager; +}; + +class CameraVideoDisplay : public QOpenGLWidget, private VideoFrameListener{ +public: + ~CameraVideoDisplay(); + CameraVideoDisplay(QWidget* parent, CameraVideoSource& source); + +private: + virtual void on_frame(std::shared_ptr frame) override; + virtual void paintEvent(QPaintEvent* event) override; + +private: + CameraVideoSource& m_source; + std::shared_ptr m_last_frame; + LifetimeSanitizer m_sanitizer; +}; + +} +} +#endif +#endif diff --git a/SerialPrograms/Source/CommonTools/Images/WaterfillUtilities.h b/SerialPrograms/Source/CommonTools/Images/WaterfillUtilities.h index 5aaaeee06a..ce669622e4 100644 --- a/SerialPrograms/Source/CommonTools/Images/WaterfillUtilities.h +++ b/SerialPrograms/Source/CommonTools/Images/WaterfillUtilities.h @@ -8,6 +8,7 @@ #ifndef PokemonAutomation_CommonTools_WaterfillUtilities_H #define PokemonAutomation_CommonTools_WaterfillUtilities_H +#include #include #include #include "Common/Cpp/ImageResolution.h" diff --git a/SerialPrograms/Source/Integrations/DiscordIntegrationSettings.cpp b/SerialPrograms/Source/Integrations/DiscordIntegrationSettings.cpp index ef338fc022..ec1fe03cd2 100644 --- a/SerialPrograms/Source/Integrations/DiscordIntegrationSettings.cpp +++ b/SerialPrograms/Source/Integrations/DiscordIntegrationSettings.cpp @@ -122,6 +122,7 @@ void DiscordIntegrationSettingsOption::on_config_value_changed([[maybe_unused]] } void DiscordIntegrationSettingsOption::on_press(ButtonCell& button){ +#ifdef PA_DPP if (&button == &m_start_button){ DppClient::Client::instance().connect(); on_config_value_changed(this); @@ -132,6 +133,9 @@ void DiscordIntegrationSettingsOption::on_press(ButtonCell& button){ on_config_value_changed(this); return; } +#else + (void)button; +#endif } diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h index 9fe2bbbdf2..6e4ab45111 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h @@ -13,6 +13,9 @@ #include #include #include +#if __has_include() +#include +#endif #include "CommonFramework/Language.h" #include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "CommonFramework/ImageTools/ImageBoxes.h" diff --git a/SerialPrograms/Source/PokemonLZA/Inference/Map/PokemonLZA_DirectionArrowDetector.cpp b/SerialPrograms/Source/PokemonLZA/Inference/Map/PokemonLZA_DirectionArrowDetector.cpp index 80d245fe55..c637a9b0cf 100644 --- a/SerialPrograms/Source/PokemonLZA/Inference/Map/PokemonLZA_DirectionArrowDetector.cpp +++ b/SerialPrograms/Source/PokemonLZA/Inference/Map/PokemonLZA_DirectionArrowDetector.cpp @@ -8,6 +8,9 @@ #include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" #include +#if __has_include() +#include +#endif #include // #define DEBUG_DIRECTION_ARROW diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index cb6c92197a..bf37dfdd63 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -545,6 +545,9 @@ file(GLOB LIBRARY_SOURCES Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.h Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.h + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.cpp + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QVideoWidget.h + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.cpp Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.h Source/CommonFramework/VideoPipeline/Backends/MediaServicesQt6.cpp From 20e1da71719efaf82ea82fc96395901b9541942e Mon Sep 17 00:00:00 2001 From: denvoros Date: Thu, 23 Jul 2026 10:25:03 -0600 Subject: [PATCH 2/3] feat(video): get opencv as an actual working backend (linux/maybe mac) --- .../Backends/CameraImplementations.cpp | 10 + .../Backends/CameraWidgetOpenCV.cpp | 192 ++++++++++++++++++ .../Backends/CameraWidgetOpenCV.h | 119 +++++++++++ SerialPrograms/cmake/SourceFiles.cmake | 2 + 4 files changed, 323 insertions(+) create mode 100644 SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.cpp create mode 100644 SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.h diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp index 1b7ac15b63..fc200f8cc4 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp @@ -22,6 +22,10 @@ #include "CameraWidgetQt6_QML.h" #endif +#if defined(__linux__) || defined(__APPLE__) +#include "CameraWidgetOpenCV.h" +#endif + namespace PokemonAutomation{ @@ -50,6 +54,12 @@ struct CameraBackends{ } CameraBackends(){ +#if defined(__linux__) || defined(__APPLE__) + m_backends.emplace_back( + "opencv-v4l2", "Linux: OpenCV V4L2", + std::make_unique() + ); +#endif #if QT_VERSION_MAJOR == 6 m_backends.emplace_back( "qt6-QVideoSink", "Qt6: QVideoSink", diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.cpp new file mode 100644 index 0000000000..81c3c7f8da --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.cpp @@ -0,0 +1,192 @@ +/* Camera Widget (OpenCV V4L2) + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#if defined(__linux__) || defined(__APPLE__) + +#include +#include +#include +#include +#include "Common/Qt/Redispatch.h" +#include "CommonFramework/Logging/Logger.h" +#include "VideoFrameQt.h" +#include "MediaServicesQt6.h" +#include "CameraWidgetOpenCV.h" + +namespace PokemonAutomation{ +namespace CameraOpenCV{ + +std::vector CameraBackend::get_all_cameras() const{ + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); + std::vector ret; + for (const auto& info : cameras){ + ret.emplace_back(info.id().toStdString()); + } + return ret; +} + +std::string CameraBackend::get_camera_name(const CameraInfo& info) const{ + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); + for (const auto& camera : cameras){ + if (camera.id().toStdString() == info.device_name()){ + return camera.description().toStdString(); + } + } + return ""; +} + +std::unique_ptr CameraBackend::make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps +) const{ + return std::make_unique(logger, info, resolution, format, fps); +} + + +CameraVideoSource::~CameraVideoSource(){ + m_stop.store(true, std::memory_order_release); + if (m_thread.joinable()){ + m_thread.join(); + } +} + +CameraVideoSource::CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps +) + : VideoSource(logger, true) + , m_logger(logger) + , m_stop(false) + , m_last_frame(logger) + , m_snapshot_manager(logger, m_last_frame) +{ + if (!info){ + return; + } + m_logger.log("Starting Camera: Backend = OpenCV V4L2"); + + // Populate fake format set so UI has some options if we don't query it. + // In a real implementation we would probe, but here we just pass the desired through + m_formats[{1920, 1080}][VideoFormat::MJPEG] = {60, 30}; + m_formats[{1280, 720}][VideoFormat::MJPEG] = {60, 30}; + m_formats[{1920, 1080}][VideoFormat::YUYV] = {10, 5}; + m_formats[{desired_resolution.width, desired_resolution.height}][desired_format] = {desired_fps}; + + m_resolution = desired_resolution; + m_format = desired_format; + m_fps = desired_fps; + + std::string device_path = info.device_name(); +#if defined(__linux__) + int device_index = 0; + if (device_path.find("/dev/video") == 0) { + try { + device_index = std::stoi(device_path.substr(10)); + } catch (...) {} + } + m_cap = std::make_unique(device_index, cv::CAP_V4L2); +#else + m_cap = std::make_unique(device_path); +#endif + + if (!m_cap->isOpened()){ + m_logger.log("Camera failed to open via OpenCV.", COLOR_RED); + return; + } + + if (desired_format == VideoFormat::MJPEG){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); + } else if (desired_format == VideoFormat::YUYV){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('Y', 'U', 'Y', 'V')); + } + + m_cap->set(cv::CAP_PROP_FRAME_WIDTH, desired_resolution.width); + m_cap->set(cv::CAP_PROP_FRAME_HEIGHT, desired_resolution.height); + if (desired_fps > 0) { + m_cap->set(cv::CAP_PROP_FPS, desired_fps); + } + + m_logger.log("Resolution: " + m_resolution.to_string() + ", FPS: " + std::to_string(m_fps)); + + m_thread = std::thread(&CameraVideoSource::thread_body, this); +} + +void CameraVideoSource::thread_body(){ + cv::Mat bgr, bgra; + while (!m_stop.load(std::memory_order_acquire)){ + if (!m_cap->read(bgr)){ + continue; // Could sleep or break on disconnect + } + + WallClock now = current_time(); + + cv::cvtColor(bgr, bgra, cv::COLOR_BGR2BGRA); + + QVideoFrameFormat format(QSize(bgra.cols, bgra.rows), QVideoFrameFormat::Format_BGRA8888); + QVideoFrame frame(format); + + if (frame.map(QVideoFrame::WriteOnly)){ + memcpy(frame.bits(0), bgra.data, bgra.total() * bgra.elemSize()); + frame.unmap(); + + if (!m_last_frame.push_frame(frame, now)){ + continue; + } + report_source_frame(std::make_shared(now, frame)); + } + } +} + +QWidget* CameraVideoSource::make_display_QtWidget(QWidget* parent){ + return new CameraVideoDisplay(parent, *this); +} + +CameraVideoDisplay::~CameraVideoDisplay(){ + m_source.remove_source_frame_listener(*this); +} +CameraVideoDisplay::CameraVideoDisplay(QWidget* parent, CameraVideoSource& source) + : QWidget(parent) + , m_source(source) +{ + source.add_source_frame_listener(*this); +} +void CameraVideoDisplay::on_frame(std::shared_ptr frame){ + m_last_frame = std::move(frame); + queue_on_main_thread([this]{ + this->update(); + }); +} +void CameraVideoDisplay::paintEvent(QPaintEvent* event){ + QWidget::paintEvent(event); + + if (!m_last_frame){ + return; + } + + QVideoFrame frame = m_last_frame->frame; + if (!frame.isValid()){ + return; + } + + QRect rect(0, 0, this->width(), this->height()); + QVideoFrame::PaintOptions options; + QPainter painter(this); + + frame.paint(&painter, rect, options); + m_source.report_rendered_frame(current_time()); +} + + +} +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.h new file mode 100644 index 0000000000..15d2afb87c --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.h @@ -0,0 +1,119 @@ +/* Camera Widget (OpenCV V4L2) + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_VideoPipeline_OpenCV_H +#define PokemonAutomation_VideoPipeline_OpenCV_H + +#include +#if defined(__linux__) || defined(__APPLE__) + +#include +#include +#include +#include "Common/Cpp/Concurrency/Mutex.h" +#include "CommonFramework/VideoPipeline/VideoSource.h" +#include "CommonFramework/VideoPipeline/CameraInfo.h" +#include "CameraImplementations.h" +#include "QVideoFrameCache.h" +#include "SnapshotManager.h" + +// Forward declaration for OpenCV +namespace cv { + class VideoCapture; + class Mat; +} + +namespace PokemonAutomation{ +namespace CameraOpenCV{ + +class CameraBackend : public PokemonAutomation::CameraBackend{ +public: + virtual std::vector get_all_cameras() const override; + virtual std::string get_camera_name(const CameraInfo& info) const override; + + virtual std::unique_ptr make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps + ) const override; +}; + +class CameraVideoSource : public VideoSource{ +public: + virtual ~CameraVideoSource(); + CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps + ); + + virtual Resolution current_resolution() const override{ + return m_resolution; + } + virtual VideoFormat current_format() const override{ + return m_format; + } + virtual FramesPerSecond current_fps() const override{ + return m_fps; + } + virtual const VideoFormatSet& supported_formats() const override{ + return m_formats; + } + + virtual VideoSnapshot snapshot_latest_blocking() override{ + return m_snapshot_manager.snapshot_latest_blocking(); + } + virtual VideoSnapshot snapshot_recent_nonblocking(WallClock min_time) override{ + return m_snapshot_manager.snapshot_recent_nonblocking(min_time); + } + + virtual QWidget* make_display_QtWidget(QWidget* parent) override; + +private: + void thread_body(); + +private: + friend class CameraVideoDisplay; + + Logger& m_logger; + Resolution m_resolution; + VideoFormat m_format; + FramesPerSecond m_fps; + + VideoFormatSet m_formats; + + std::unique_ptr m_cap; + std::thread m_thread; + std::atomic m_stop; + + QVideoFrameCache m_last_frame; + SnapshotManager m_snapshot_manager; +}; + +class CameraVideoDisplay : public QWidget, private VideoFrameListener{ +public: + ~CameraVideoDisplay(); + CameraVideoDisplay(QWidget* parent, CameraVideoSource& source); + +private: + virtual void on_frame(std::shared_ptr frame) override; + virtual void paintEvent(QPaintEvent* event) override; + +private: + CameraVideoSource& m_source; + std::shared_ptr m_last_frame; + + LifetimeSanitizer m_sanitizer; +}; + +} +} +#endif +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index bf37dfdd63..32375b4336 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -550,6 +550,8 @@ file(GLOB LIBRARY_SOURCES Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QML.cpp Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.h + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.cpp + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.h Source/CommonFramework/VideoPipeline/Backends/MediaServicesQt6.cpp Source/CommonFramework/VideoPipeline/Backends/MediaServicesQt6.h Source/CommonFramework/VideoPipeline/Backends/QCameraThread.h From 8d44dc72f1c97938fbda409875126630b31a9b10 Mon Sep 17 00:00:00 2001 From: denvoros Date: Thu, 20 Aug 2026 13:17:33 -0600 Subject: [PATCH 3/3] feat(video): additional GPU code for various backend ideas --- .../Backends/CameraImplementations.cpp | 5 + .../Backends/CameraWidgetOpenCV.cpp | 124 ++++++- .../Backends/CameraWidgetOpenCV.h | 2 + .../CameraWidgetOpenCV_QOpenGLWidget.cpp | 326 ++++++++++++++++++ .../CameraWidgetOpenCV_QOpenGLWidget.h | 128 +++++++ SerialPrograms/cmake/SourceFiles.cmake | 2 + 6 files changed, 578 insertions(+), 9 deletions(-) create mode 100644 SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.cpp create mode 100644 SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.h diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp index fc200f8cc4..578a31d177 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraImplementations.cpp @@ -24,6 +24,7 @@ #if defined(__linux__) || defined(__APPLE__) #include "CameraWidgetOpenCV.h" +#include "CameraWidgetOpenCV_QOpenGLWidget.h" #endif @@ -59,6 +60,10 @@ struct CameraBackends{ "opencv-v4l2", "Linux: OpenCV V4L2", std::make_unique() ); + m_backends.emplace_back( + "opencv-v4l2-gl", "Linux: OpenCV V4L2 (GPU Offloaded)", + std::make_unique() + ); #endif #if QT_VERSION_MAJOR == 6 m_backends.emplace_back( diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.cpp index 81c3c7f8da..f2a08fc10e 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.cpp @@ -11,6 +11,12 @@ #include #include #include +#if defined(__linux__) +#include +#include +#include +#include +#endif #include "Common/Qt/Redispatch.h" #include "CommonFramework/Logging/Logger.h" #include "VideoFrameQt.h" @@ -75,18 +81,71 @@ CameraVideoSource::CameraVideoSource( } m_logger.log("Starting Camera: Backend = OpenCV V4L2"); - // Populate fake format set so UI has some options if we don't query it. - // In a real implementation we would probe, but here we just pass the desired through - m_formats[{1920, 1080}][VideoFormat::MJPEG] = {60, 30}; - m_formats[{1280, 720}][VideoFormat::MJPEG] = {60, 30}; - m_formats[{1920, 1080}][VideoFormat::YUYV] = {10, 5}; - m_formats[{desired_resolution.width, desired_resolution.height}][desired_format] = {desired_fps}; +#if defined(__linux__) + int fd = open(info.device_name().c_str(), O_RDWR); + if (fd >= 0) { + struct v4l2_fmtdesc fmtdesc = {}; + fmtdesc.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + while (ioctl(fd, VIDIOC_ENUM_FMT, &fmtdesc) == 0) { + VideoFormat fmt = VideoFormat::OTHER; + bool valid_fmt = false; + if (fmtdesc.pixelformat == V4L2_PIX_FMT_MJPEG) { fmt = VideoFormat::MJPEG; valid_fmt = true; } + else if (fmtdesc.pixelformat == V4L2_PIX_FMT_YUYV) { fmt = VideoFormat::YUYV; valid_fmt = true; } + + if (valid_fmt) { + struct v4l2_frmsizeenum frmsize = {}; + frmsize.pixel_format = fmtdesc.pixelformat; + while (ioctl(fd, VIDIOC_ENUM_FRAMESIZES, &frmsize) == 0) { + if (frmsize.type == V4L2_FRMSIZE_TYPE_DISCRETE) { + uint32_t w = frmsize.discrete.width; + uint32_t h = frmsize.discrete.height; + + int ival_index = 0; + while (true) { + struct v4l2_frmivalenum frmival = {}; + frmival.index = ival_index; + frmival.pixel_format = fmtdesc.pixelformat; + frmival.width = w; + frmival.height = h; + + if (ioctl(fd, VIDIOC_ENUM_FRAMEINTERVALS, &frmival) != 0) { + break; + } + + if (frmival.type == V4L2_FRMIVAL_TYPE_DISCRETE) { + double fps = (double)frmival.discrete.denominator / frmival.discrete.numerator; + m_formats[{w, h}][fmt].insert((size_t)std::round(fps)); + } + ival_index++; + } + } + frmsize.index++; + } + } + fmtdesc.index++; + } + close(fd); + } +#endif + + // Ensure the desired format is at least present in case probing failed + if (m_formats[{desired_resolution.width, desired_resolution.height}][desired_format].empty()) { + m_formats[{desired_resolution.width, desired_resolution.height}][desired_format].insert(desired_fps); + } m_resolution = desired_resolution; m_format = desired_format; m_fps = desired_fps; std::string device_path = info.device_name(); + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); + for (const auto& camera : cameras){ + if (camera.id().toStdString() == device_path){ + m_camera_description = camera.description().toStdString(); + break; + } + } + #if defined(__linux__) int device_index = 0; if (device_path.find("/dev/video") == 0) { @@ -94,7 +153,8 @@ CameraVideoSource::CameraVideoSource( device_index = std::stoi(device_path.substr(10)); } catch (...) {} } - m_cap = std::make_unique(device_index, cv::CAP_V4L2); + m_device_index = device_index; + m_cap = std::make_unique(m_device_index, cv::CAP_V4L2); #else m_cap = std::make_unique(device_path); #endif @@ -123,10 +183,56 @@ CameraVideoSource::CameraVideoSource( void CameraVideoSource::thread_body(){ cv::Mat bgr, bgra; + int consecutive_failures = 0; while (!m_stop.load(std::memory_order_acquire)){ - if (!m_cap->read(bgr)){ - continue; // Could sleep or break on disconnect + if (!m_cap->read(bgr) || bgr.empty()){ + consecutive_failures++; + if (consecutive_failures % 30 == 1) { + m_logger.log("Failed to read frame from OpenCV. (Failures: " + std::to_string(consecutive_failures) + ")", COLOR_RED); + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + if (consecutive_failures > 40) { + m_logger.log("Too many consecutive read failures. Attempting to restart capture device...", COLOR_RED); + m_cap->release(); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); +#if defined(__linux__) + m_cap->open(m_device_index, cv::CAP_V4L2); + if (!m_cap->isOpened() && !m_camera_description.empty()) { + m_logger.log("Failed to reopen device index " + std::to_string(m_device_index) + ". Searching for it by name...", COLOR_RED); + const auto new_cameras = GlobalMediaServices::instance().get_all_cameras(); + for (const auto& camera : new_cameras) { + if (camera.description().toStdString() == m_camera_description) { + std::string new_path = camera.id().toStdString(); + if (new_path.find("/dev/video") == 0) { + try { + m_device_index = std::stoi(new_path.substr(10)); + m_logger.log("Found device at new index " + std::to_string(m_device_index), COLOR_RED); + m_cap->open(m_device_index, cv::CAP_V4L2); + if (m_cap->isOpened()) break; + } catch (...) {} + } + } + } + } +#endif + if (m_cap->isOpened()) { + if (m_format == VideoFormat::MJPEG){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); + } else if (m_format == VideoFormat::YUYV){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('Y', 'U', 'Y', 'V')); + } + m_cap->set(cv::CAP_PROP_FRAME_WIDTH, m_resolution.width); + m_cap->set(cv::CAP_PROP_FRAME_HEIGHT, m_resolution.height); + if (m_fps > 0) { + m_cap->set(cv::CAP_PROP_FPS, m_fps); + } + consecutive_failures = 0; + } + } + continue; } + consecutive_failures = 0; WallClock now = current_time(); diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.h index 15d2afb87c..b0ad47db95 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.h @@ -89,6 +89,8 @@ class CameraVideoSource : public VideoSource{ VideoFormatSet m_formats; + int m_device_index = 0; + std::string m_camera_description; std::unique_ptr m_cap; std::thread m_thread; std::atomic m_stop; diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.cpp new file mode 100644 index 0000000000..272baab968 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.cpp @@ -0,0 +1,326 @@ +/* Camera Widget (OpenCV V4L2) + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#if defined(__linux__) || defined(__APPLE__) + +#include +#include +#include +#include +#include +#include +#if defined(__linux__) +#include +#include +#include +#include +#endif +#include "Common/Qt/Redispatch.h" +#include "CommonFramework/Logging/Logger.h" +#include "VideoFrameQt.h" +#include "MediaServicesQt6.h" +#include "CameraWidgetOpenCV_QOpenGLWidget.h" + +namespace PokemonAutomation{ +namespace CameraOpenCV_QOpenGLWidget{ + +std::vector CameraBackend::get_all_cameras() const{ + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); + std::vector ret; + for (const auto& info : cameras){ + ret.emplace_back(info.id().toStdString()); + } + return ret; +} + +std::string CameraBackend::get_camera_name(const CameraInfo& info) const{ + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); + for (const auto& camera : cameras){ + if (camera.id().toStdString() == info.device_name()){ + return camera.description().toStdString(); + } + } + return ""; +} + +std::unique_ptr CameraBackend::make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps +) const{ + return std::make_unique(logger, info, resolution, format, fps); +} + + +CameraVideoSource::~CameraVideoSource(){ + m_stop.store(true, std::memory_order_release); + if (m_thread.joinable()){ + m_thread.join(); + } +} + +CameraVideoSource::CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps +) + : VideoSource(logger, true) + , m_logger(logger) + , m_stop(false) + , m_last_frame(logger) + , m_snapshot_manager(logger, m_last_frame) +{ + if (!info){ + return; + } + m_logger.log("Starting Camera: Backend = OpenCV V4L2"); + +#if defined(__linux__) + int fd = open(info.device_name().c_str(), O_RDWR); + if (fd >= 0) { + struct v4l2_fmtdesc fmtdesc = {}; + fmtdesc.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + while (ioctl(fd, VIDIOC_ENUM_FMT, &fmtdesc) == 0) { + VideoFormat fmt = VideoFormat::OTHER; + bool valid_fmt = false; + if (fmtdesc.pixelformat == V4L2_PIX_FMT_MJPEG) { fmt = VideoFormat::MJPEG; valid_fmt = true; } + else if (fmtdesc.pixelformat == V4L2_PIX_FMT_YUYV) { fmt = VideoFormat::YUYV; valid_fmt = true; } + + if (valid_fmt) { + struct v4l2_frmsizeenum frmsize = {}; + frmsize.pixel_format = fmtdesc.pixelformat; + while (ioctl(fd, VIDIOC_ENUM_FRAMESIZES, &frmsize) == 0) { + if (frmsize.type == V4L2_FRMSIZE_TYPE_DISCRETE) { + uint32_t w = frmsize.discrete.width; + uint32_t h = frmsize.discrete.height; + + int ival_index = 0; + while (true) { + struct v4l2_frmivalenum frmival = {}; + frmival.index = ival_index; + frmival.pixel_format = fmtdesc.pixelformat; + frmival.width = w; + frmival.height = h; + + if (ioctl(fd, VIDIOC_ENUM_FRAMEINTERVALS, &frmival) != 0) { + break; + } + + if (frmival.type == V4L2_FRMIVAL_TYPE_DISCRETE) { + double fps = (double)frmival.discrete.denominator / frmival.discrete.numerator; + m_formats[{w, h}][fmt].insert((size_t)std::round(fps)); + } + ival_index++; + } + } + frmsize.index++; + } + } + fmtdesc.index++; + } + close(fd); + } +#endif + + // Ensure the desired format is at least present in case probing failed + if (m_formats[{desired_resolution.width, desired_resolution.height}][desired_format].empty()) { + m_formats[{desired_resolution.width, desired_resolution.height}][desired_format].insert(desired_fps); + } + + m_resolution = desired_resolution; + m_format = desired_format; + m_fps = desired_fps; + + std::string device_path = info.device_name(); + const auto cameras = GlobalMediaServices::instance().get_all_cameras(); + for (const auto& camera : cameras){ + if (camera.id().toStdString() == device_path){ + m_camera_description = camera.description().toStdString(); + break; + } + } + +#if defined(__linux__) + int device_index = 0; + if (device_path.find("/dev/video") == 0) { + try { + device_index = std::stoi(device_path.substr(10)); + } catch (...) {} + } + m_device_index = device_index; + m_cap = std::make_unique(m_device_index, cv::CAP_V4L2); +#else + m_cap = std::make_unique(device_path); +#endif + + if (!m_cap->isOpened()){ + m_logger.log("Camera failed to open via OpenCV.", COLOR_RED); + return; + } + + if (desired_format == VideoFormat::MJPEG){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); + } else if (desired_format == VideoFormat::YUYV){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('Y', 'U', 'Y', 'V')); + } + + m_cap->set(cv::CAP_PROP_FRAME_WIDTH, desired_resolution.width); + m_cap->set(cv::CAP_PROP_FRAME_HEIGHT, desired_resolution.height); + if (desired_fps > 0) { + m_cap->set(cv::CAP_PROP_FPS, desired_fps); + } + + m_logger.log("Resolution: " + m_resolution.to_string() + ", FPS: " + std::to_string(m_fps)); + + m_thread = std::thread(&CameraVideoSource::thread_body, this); +} + +void CameraVideoSource::thread_body(){ + cv::Mat bgr, bgra; + int consecutive_failures = 0; + while (!m_stop.load(std::memory_order_acquire)){ + if (!m_cap->read(bgr) || bgr.empty()){ + consecutive_failures++; + if (consecutive_failures % 30 == 1) { + m_logger.log("Failed to read frame from OpenCV. (Failures: " + std::to_string(consecutive_failures) + ")", COLOR_RED); + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + if (consecutive_failures > 40) { + m_logger.log("Too many consecutive read failures. Attempting to restart capture device...", COLOR_RED); + m_cap->release(); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); +#if defined(__linux__) + m_cap->open(m_device_index, cv::CAP_V4L2); + if (!m_cap->isOpened() && !m_camera_description.empty()) { + m_logger.log("Failed to reopen device index " + std::to_string(m_device_index) + ". Searching for it by name...", COLOR_RED); + const auto new_cameras = GlobalMediaServices::instance().get_all_cameras(); + for (const auto& camera : new_cameras) { + if (camera.description().toStdString() == m_camera_description) { + std::string new_path = camera.id().toStdString(); + if (new_path.find("/dev/video") == 0) { + try { + m_device_index = std::stoi(new_path.substr(10)); + m_logger.log("Found device at new index " + std::to_string(m_device_index), COLOR_RED); + m_cap->open(m_device_index, cv::CAP_V4L2); + if (m_cap->isOpened()) break; + } catch (...) {} + } + } + } + } +#endif + if (m_cap->isOpened()) { + if (m_format == VideoFormat::MJPEG){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); + } else if (m_format == VideoFormat::YUYV){ + m_cap->set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('Y', 'U', 'Y', 'V')); + } + m_cap->set(cv::CAP_PROP_FRAME_WIDTH, m_resolution.width); + m_cap->set(cv::CAP_PROP_FRAME_HEIGHT, m_resolution.height); + if (m_fps > 0) { + m_cap->set(cv::CAP_PROP_FPS, m_fps); + } + consecutive_failures = 0; + } + } + continue; + } + consecutive_failures = 0; + + WallClock now = current_time(); + + cv::cvtColor(bgr, bgra, cv::COLOR_BGR2BGRA); + + QVideoFrameFormat format(QSize(bgra.cols, bgra.rows), QVideoFrameFormat::Format_BGRA8888); + QVideoFrame frame(format); + + if (frame.map(QVideoFrame::WriteOnly)){ + memcpy(frame.bits(0), bgra.data, bgra.total() * bgra.elemSize()); + frame.unmap(); + + if (!m_last_frame.push_frame(frame, now)){ + continue; + } + report_source_frame(std::make_shared(now, frame)); + } + } +} + +QWidget* CameraVideoSource::make_display_QtWidget(QWidget* parent){ + return new CameraVideoDisplay(parent, *this); +} + +CameraVideoDisplay::~CameraVideoDisplay(){ + m_source.remove_source_frame_listener(*this); +} +CameraVideoDisplay::CameraVideoDisplay(QWidget* parent, CameraVideoSource& source) + : QOpenGLWidget(parent) + , m_source(source) + , m_width(80) + , m_height(45) + , m_sanitizer("CameraVideoDisplay") +{ + this->setMinimumSize(80, 45); + source.add_source_frame_listener(*this); +} +void CameraVideoDisplay::resizeEvent(QResizeEvent* event){ + QOpenGLWidget::resizeEvent(event); + m_width.store(event->size().width(), std::memory_order_relaxed); + m_height.store(event->size().height(), std::memory_order_relaxed); +} + +void CameraVideoDisplay::on_frame(std::shared_ptr frame){ + int w = m_width.load(std::memory_order_relaxed); + int h = m_height.load(std::memory_order_relaxed); + + std::shared_ptr scaled_frame = frame; + + if (w > 0 && h > 0 && frame && frame->frame.isValid()){ + QSize target_size(w, h); + if (frame->frame.size() != target_size){ + QImage img = frame->frame.toImage(); + QImage scaled = img.scaled(target_size, Qt::IgnoreAspectRatio, Qt::FastTransformation); + QVideoFrame out(scaled); + scaled_frame = std::make_shared(frame->timestamp, out); + } + } + + QMetaObject::invokeMethod(this, [this, frame = std::move(scaled_frame)]() mutable { + m_last_frame = std::move(frame); + this->update(); + }, Qt::QueuedConnection); +} + +void CameraVideoDisplay::paintEvent(QPaintEvent* event){ + QOpenGLWidget::paintEvent(event); + + if (!m_last_frame){ + return; + } + + QVideoFrame frame = m_last_frame->frame; + if (!frame.isValid()){ + return; + } + + QRect rect(0, 0, this->width(), this->height()); + QVideoFrame::PaintOptions options; + QPainter painter(this); + + frame.paint(&painter, rect, options); + m_source.report_rendered_frame(current_time()); +} + + +} +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.h new file mode 100644 index 0000000000..27e78833c2 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.h @@ -0,0 +1,128 @@ +/* Camera Widget (OpenCV V4L2) + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_CameraWidgetOpenCV_QOpenGLWidget_H +#define PokemonAutomation_CameraWidgetOpenCV_QOpenGLWidget_H + +#include +#if defined(__linux__) || defined(__APPLE__) + +#include +#include +#include +#include +#include "Common/Cpp/Concurrency/Mutex.h" +#include "CommonFramework/VideoPipeline/VideoSource.h" +#include "CommonFramework/VideoPipeline/CameraInfo.h" +#include "CameraImplementations.h" +#include "QVideoFrameCache.h" +#include "SnapshotManager.h" + +// Forward declaration for OpenCV +namespace cv { + class VideoCapture; + class Mat; +} + +namespace PokemonAutomation{ +namespace CameraOpenCV_QOpenGLWidget{ + +class CameraBackend : public PokemonAutomation::CameraBackend{ +public: + virtual std::vector get_all_cameras() const override; + virtual std::string get_camera_name(const CameraInfo& info) const override; + + virtual std::unique_ptr make_video_source( + Logger& logger, + const CameraInfo& info, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps + ) const override; +}; + +class CameraVideoSource : public VideoSource{ +public: + virtual ~CameraVideoSource(); + CameraVideoSource( + Logger& logger, + const CameraInfo& info, + Resolution desired_resolution, + VideoFormat desired_format, + FramesPerSecond desired_fps + ); + + virtual Resolution current_resolution() const override{ + return m_resolution; + } + virtual VideoFormat current_format() const override{ + return m_format; + } + virtual FramesPerSecond current_fps() const override{ + return m_fps; + } + virtual const VideoFormatSet& supported_formats() const override{ + return m_formats; + } + + virtual VideoSnapshot snapshot_latest_blocking() override{ + return m_snapshot_manager.snapshot_latest_blocking(); + } + virtual VideoSnapshot snapshot_recent_nonblocking(WallClock min_time) override{ + return m_snapshot_manager.snapshot_recent_nonblocking(min_time); + } + + virtual QWidget* make_display_QtWidget(QWidget* parent) override; + +private: + void thread_body(); + +private: + friend class CameraVideoDisplay; + + Logger& m_logger; + Resolution m_resolution; + VideoFormat m_format; + FramesPerSecond m_fps; + + VideoFormatSet m_formats; + + int m_device_index = 0; + std::string m_camera_description; + std::unique_ptr m_cap; + std::thread m_thread; + std::atomic m_stop; + + QVideoFrameCache m_last_frame; + SnapshotManager m_snapshot_manager; +}; + +class CameraVideoDisplay : public QOpenGLWidget, private VideoFrameListener{ +public: + ~CameraVideoDisplay(); + CameraVideoDisplay(QWidget* parent, CameraVideoSource& source); + +private: + virtual void on_frame(std::shared_ptr frame) override; + +protected: + virtual void resizeEvent(QResizeEvent* event) override; + virtual void paintEvent(QPaintEvent* event) override; + +private: + CameraVideoSource& m_source; + std::shared_ptr m_last_frame; + + std::atomic m_width; + std::atomic m_height; + + LifetimeSanitizer m_sanitizer; +}; + +} +} +#endif +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 32375b4336..83b230d95a 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -552,6 +552,8 @@ file(GLOB LIBRARY_SOURCES Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.h Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.cpp Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV.h + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.cpp + Source/CommonFramework/VideoPipeline/Backends/CameraWidgetOpenCV_QOpenGLWidget.h Source/CommonFramework/VideoPipeline/Backends/MediaServicesQt6.cpp Source/CommonFramework/VideoPipeline/Backends/MediaServicesQt6.h Source/CommonFramework/VideoPipeline/Backends/QCameraThread.h