Merge pull request #818 from maximkulkin/opencv-grabber

Added HDMI frame grabber ability using OpenCV

CMakeLists.txt

@@ -68,6 +68,9 @@ message(STATUS "ENABLE_TINKERFORGE = " ${ENABLE_TINKERFORGE})
 option(ENABLE_V4L2 "Enable the V4L2 grabber" ON)
 message(STATUS "ENABLE_V4L2 = " ${ENABLE_V4L2})
 
+option(ENABLE_OPENCV "Enable the OpenCV grabber" ON)
+message(STATUS "ENABLE_OPENCV = " ${ENABLE_OPENCV})
+
 option(ENABLE_WS2812BPWM   "Enable the WS2812b-PWM device" ${DEFAULT_WS2812BPWM} )
 message(STATUS "ENABLE_WS2812BPWM = " ${ENABLE_WS2812BPWM})
 

CompileHowto.txt

@@ -1,6 +1,6 @@
 # Install the required tools and dependencies
 sudo apt-get update
-sudo apt-get install git cmake build-essential libqt4-dev libusb-1.0-0-dev python-dev libxrender-dev python
+sudo apt-get install git cmake build-essential libqt4-dev libusb-1.0-0-dev python-dev libxrender-dev python 
 
 # OSMC dependencies
 sudo apt-get install rbp-userland-dev-osmc
@@ -15,6 +15,9 @@ export FIRMWARE_DIR="raspberrypi-firmware"
 git clone --depth 1 https://github.com/raspberrypi/firmware.git "$FIRMWARE_DIR"
 sudo cp -R "$FIRMWARE_DIR/hardfp/opt/" /opt
 
+# OpenCV HDMI grabber dependencies (must be version 2.x) (check with: pkg-config --modversion opencv)
+sudo apt-get install libopencv*
+
 # create hyperion directory and checkout the code from github
 # You might want to add "--depth 1" if you only want to recompile the current source or don't want to use git any further
 export HYPERION_DIR="hyperion"
@@ -35,6 +38,8 @@ cmake -DCMAKE_BUILD_TYPE=Release -DPLATFORM="rpi" -Wno-dev ..
 cmake -DCMAKE_BUILD_TYPE=Release  -DPLATFORM="rpi-pwm" -Wno-dev ..
 # or if you are not compiling on the raspberry pi (e.g. OrangePi) and need to disable the Dispmanx grabber and support for spi devices
 cmake -DENABLE_DISPMANX=OFF -DENABLE_SPIDEV=OFF -DENABLE_X11=ON -DCMAKE_BUILD_TYPE=Release -Wno-dev ..
+# to enable the HDMI OpenCV grabber append cmake parameter
+ENABLE_OPENCV=ON
 # on amlogic platforms
 cmake -DENABLE_DISPMANX=OFF -DENABLE_SPIDEV=OFF -DENABLE_AMLOGIC=ON -DCMAKE_BUILD_TYPE=Release -Wno-dev ..
 # as an alternative for the dispmanx grabber on non-rpi devices (e.g. cubox-i) you could try the framebuffer grabber

HyperionConfig.h.in

@@ -27,6 +27,9 @@
 // Define to enable the osx grabber
 #cmakedefine ENABLE_OSX
 
+// Define to enable the opencv grabber
+#cmakedefine ENABLE_OPENCV
+
 // Define to enable the bonjour/zeroconf publishing
 #cmakedefine ENABLE_ZEROCONF
 

config/hyperion.config.json.example

@@ -300,6 +300,21 @@
 		"blueSignalThreshold"  : 0.0
 	},
 
+	/// Configuration for the embedded OpenCV grabber
+	///  * input		: integer id of input device [default= 0]
+	///  * width		: optional resolution (note: will force OpenCV to capture in that resolution) [default=0]
+	///  * height		: optional resolution (note: will force OpenCV to capture in that resolution) [default=0]
+	///  * frequency_Hz	: optional capture frequency [defaults=10]
+	///  * priority		: optional hyperion priority channel [default=900]
+	"opencvgrabber" :
+	{
+		"input" : 0,
+		"width" : 0,
+		"height" : 0,
+		"frequency_Hz" : 10,
+		"priority" : 900
+	},
+
 	///  The configuration for each individual led. This contains the specification of the area 
 	///  averaged of an input image for each led to determine its color. Each item in the list 
 	///  contains the following fields:

include/grabber/OpenCVGrabber.h

@@ -0,0 +1,27 @@
+#pragma once
+
+// Qt includes
+#include <QObject>
+
+// util includes
+#include <utils/Image.h>
+#include <utils/ColorRgb.h>
+
+// grabber includes
+#include <opencv2/opencv.hpp>
+
+/// Capture class for OpenCV
+class OpenCVGrabber : public QObject
+{
+    Q_OBJECT
+
+public:
+    OpenCVGrabber(int input, int width, int height);
+    virtual ~OpenCVGrabber();
+
+public slots:
+    void grabFrame(Image<ColorRgb> & image);
+
+private:
+    cv::VideoCapture _capture;
+};

include/grabber/OpenCVWrapper.h

@@ -0,0 +1,65 @@
+#pragma once
+
+// Qt includes
+#include <QTimer>
+
+// Hyperion includes
+#include <hyperion/Hyperion.h>
+#include <hyperion/ImageProcessor.h>
+
+// Grabber includes
+#include <grabber/OpenCVGrabber.h>
+
+class OpenCVWrapper : public QObject
+{
+    Q_OBJECT
+
+public:
+    OpenCVWrapper(int input,
+                  int width,
+                  int height,
+                  int frequency,
+                  int hyperionPriority,
+                  Hyperion * hyperion);
+    virtual ~OpenCVWrapper();
+
+public slots:
+    void start();
+
+    void stop();
+
+signals:
+    void emitColors(int priority, const std::vector<ColorRgb> &ledColors, const int timeout_ms);
+    void emitImage(int priority, const Image<ColorRgb> & image, const int timeout_ms);
+
+private slots:
+    void grabFrame();
+    void checkSources();
+
+private:
+    /// The timeout of the led colors [ms]
+    const int _timeout_ms;
+
+    /// The priority of the led colors
+    const int _priority;
+
+    /// Grab frequency [Hz]
+    const int _frequency;
+
+    /// The OpenCV grabber
+    OpenCVGrabber _grabber;
+
+    /// The processor for transforming images to led colors
+    ImageProcessor * _processor;
+
+    /// The Hyperion instance
+    Hyperion * _hyperion;
+
+    /// The list with computed led colors
+    std::vector<ColorRgb> _ledColors;
+
+    /// Timer which tests if a higher priority source is active
+    QTimer _priority_check_timer;
+
+    QTimer _grab_timer;
+};

libsrc/grabber/CMakeLists.txt

@@ -18,6 +18,10 @@ if (ENABLE_V4L2)
 	add_subdirectory(v4l2)
 endif (ENABLE_V4L2)
 
+if (ENABLE_OPENCV)
+	add_subdirectory(opencv)
+endif (ENABLE_OPENCV)
+
 if (ENABLE_X11)
 	add_subdirectory(x11)
 endif()

libsrc/grabber/opencv/CMakeLists.txt

@@ -0,0 +1,38 @@
+find_package(OpenCV REQUIRED)
+
+# Define the current source locations
+SET(CURRENT_HEADER_DIR ${CMAKE_SOURCE_DIR}/include/grabber)
+SET(CURRENT_SOURCE_DIR ${CMAKE_SOURCE_DIR}/libsrc/grabber/opencv)
+
+SET(OPENCV_GRABBER_QT_HEADERS
+    ${CURRENT_HEADER_DIR}/OpenCVGrabber.h
+    ${CURRENT_HEADER_DIR}/OpenCVWrapper.h
+)
+
+SET(OPENCV_GRABBER_SOURCES
+    ${CURRENT_SOURCE_DIR}/OpenCVGrabber.cpp
+    ${CURRENT_SOURCE_DIR}/OpenCVWrapper.cpp
+)
+
+if(ENABLE_QT5)
+    QT5_WRAP_CPP(OPENCV_GRABBER_HEADERS_MOC ${OPENCV_GRABBER_QT_HEADERS})
+else()
+    QT4_WRAP_CPP(OPENCV_GRABBER_HEADERS_MOC ${OPENCV_GRABBER_QT_HEADERS})
+endif()
+
+add_library(opencv-grabber
+    ${OPENCV_GRABBER_HEADERS}
+    ${OPENCV_GRABBER_SOURCES}
+    ${OPENCV_GRABBER_QT_HEADERS}
+    ${OPENCV_GRABBER_HEADERS_MOC}
+)
+
+if(ENABLE_QT5)
+qt5_use_modules(opencv-grabber Widgets)
+endif(ENABLE_QT5)
+
+target_link_libraries(opencv-grabber
+    hyperion
+    ${QT_LIBRARIES}
+    ${OpenCV_LIBS}
+)

libsrc/grabber/opencv/OpenCVGrabber.cpp

@@ -0,0 +1,34 @@
+#include <iostream>
+#include <sstream>
+#include <stdexcept>
+#include <opencv2/opencv.hpp>
+
+#include "grabber/OpenCVGrabber.h"
+
+
+OpenCVGrabber::OpenCVGrabber(int input, int width, int height)
+    : _capture(input)
+{
+    if (width && height) {
+        _capture.set(CV_CAP_PROP_FRAME_WIDTH, width);
+        _capture.set(CV_CAP_PROP_FRAME_HEIGHT, height);
+    }
+}
+
+OpenCVGrabber::~OpenCVGrabber()
+{
+}
+
+void OpenCVGrabber::grabFrame(Image<ColorRgb> & image)
+{
+    cv::Mat frame;
+    _capture >> frame;
+
+    const int width = frame.cols, height = frame.rows;
+
+    cv::Mat rgbFrame(width, height, CV_8UC3);
+    cvtColor(frame, rgbFrame, cv::COLOR_BGR2RGB);
+
+    image.resize(width, height);
+    memcpy(image.memptr(), rgbFrame.ptr(), width * height * 3);
+}

libsrc/grabber/opencv/OpenCVWrapper.cpp

@@ -0,0 +1,102 @@
+#include <QMetaType>
+
+#include <grabber/OpenCVWrapper.h>
+
+#include <hyperion/ImageProcessorFactory.h>
+
+OpenCVWrapper::OpenCVWrapper(
+        int input,
+        int width,
+        int height,
+        int frequency,
+        int hyperionPriority,
+        Hyperion *hyperion) :
+    _timeout_ms(1000),
+    _priority(hyperionPriority),
+    _frequency(frequency),
+    _grabber(input, width, height),
+    _processor(ImageProcessorFactory::getInstance().newImageProcessor()),
+    _hyperion(hyperion),
+    _ledColors(hyperion->getLedCount(), ColorRgb{0,0,0}),
+    _priority_check_timer(),
+    _grab_timer()
+{
+    // register the image type
+    qRegisterMetaType<Image<ColorRgb>>("Image<ColorRgb>");
+    qRegisterMetaType<std::vector<ColorRgb>>("std::vector<ColorRgb>");
+
+    // send color data to Hyperion using a queued connection to handle the data over to the main event loop
+    QObject::connect(
+                this, SIGNAL(emitColors(int,std::vector<ColorRgb>,int)),
+                _hyperion, SLOT(setColors(int,std::vector<ColorRgb>,int)),
+                Qt::QueuedConnection);
+
+    // setup the higher prio source checker
+    // this will disable the grabber when a source with hisher priority is active
+    _priority_check_timer.setInterval(500);
+    _priority_check_timer.setSingleShot(false);
+    QObject::connect(&_priority_check_timer, SIGNAL(timeout()), this, SLOT(checkSources()));
+    _priority_check_timer.start();
+
+    _grab_timer.setInterval(1000 / _frequency);
+    _grab_timer.setSingleShot(false);
+
+    QObject::connect(&_grab_timer, SIGNAL(timeout()), this, SLOT(grabFrame()));
+}
+
+OpenCVWrapper::~OpenCVWrapper()
+{
+    delete _processor;
+}
+
+void OpenCVWrapper::start()
+{
+    if (_grab_timer.isActive())
+        return;
+
+    _grab_timer.start();
+    std::cout << "OPENCVGRABBER INFO: started" << std::endl;
+}
+
+void OpenCVWrapper::stop()
+{
+    if (!_grab_timer.isActive())
+        return;
+
+    _grab_timer.stop();
+    std::cout << "OPENCVGRABBER INFO: stopped" << std::endl;
+}
+
+void OpenCVWrapper::grabFrame()
+{
+    Image<ColorRgb> image;
+
+    _grabber.grabFrame(image);
+
+    // process the new image
+    _processor->process(image, _ledColors);
+
+    // forward to other hyperions
+    emit emitImage(_priority, image, _timeout_ms);
+
+    // send colors to Hyperion
+    emit emitColors(_priority, _ledColors, _timeout_ms);
+}
+
+void OpenCVWrapper::checkSources()
+{
+    QList<int> activePriorities = _hyperion->getActivePriorities();
+
+    for (int x : activePriorities)
+    {
+        if (x < _priority)
+        {
+            // found a higher priority source: grabber should be disabled
+            stop();
+            return;
+        }
+    }
+
+    // no higher priority source was found: grabber should be enabled
+    start();
+}

src/hyperiond/CMakeLists.txt

@@ -31,6 +31,10 @@ if (ENABLE_V4L2)
 	target_link_libraries(hyperiond v4l2-grabber)
 endif ()
 
+if (ENABLE_OPENCV)
+	target_link_libraries(hyperiond opencv-grabber)
+endif ()
+
 if (ENABLE_AMLOGIC)
 	target_link_libraries(hyperiond amlogic-grabber)
 endif ()