diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 0b33d04fb..f21049c36 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -469,6 +469,7 @@ dai_add_example(replay_video_meta RVC2/Replay/replay_video_meta.cpp ON OFF) dai_add_example(camera_multiple_outputs RVC4/Camera/camera_multiple_outputs.cpp ON OFF) # Host nodes +dai_add_example(emotions RVC2/Emotions/emotions.cpp ON OFF) dai_add_example(rgb_video RVC2/ColorCamera/rgb_video.cpp ON OFF) dai_add_example(host_node HostNodes/host_node.cpp ON OFF) dai_add_example(threaded_host_node HostNodes/threaded_host_node.cpp ON OFF) diff --git a/examples/cpp/RVC2/Emotions/emotions.cpp b/examples/cpp/RVC2/Emotions/emotions.cpp new file mode 100644 index 000000000..e66dd29cd --- /dev/null +++ b/examples/cpp/RVC2/Emotions/emotions.cpp @@ -0,0 +1,171 @@ +// Includes common necessary includes for development using depthai library +#include "depthai/depthai.hpp" +#include "xtensor/xsort.hpp" + +class Face2ImageManipConfig : public dai::NodeCRTP { +public: + Input inputDetections = dai::Node::Input{*this, {}}; + Output outputManipulators = dai::Node::Output{*this, {}}; + + void run() override { + while(isRunning()) { + std::shared_ptr inDet; + inDet = inputDetections.get(); + + if(!inDet) + { + continue; + } + + for(auto& detection : inDet->detections) { + std::shared_ptr manipulator = std::make_shared(); + manipulator->setCropRect(detection.xmin, + detection.ymin, + detection.xmax, + detection.ymax); + manipulator->setResize(64,64); + outputManipulators.send(manipulator); + } + + } + } +}; + +std::array emotions = {"neutral", "happy", "sad", "surprise", "anger"}; + +void displayFrame(cv::Mat& frame, std::vector& detections, std::vector>& resultLayers) +{ + auto color = cv::Scalar(255, 0, 0); + for(int i = 0; i < detections.size(); i++) { + auto& detection = detections[i]; + int x1 = detection.xmin * frame.cols; + int y1 = detection.ymin * frame.rows; + int x2 = detection.xmax * frame.cols; + int y2 = detection.ymax * frame.rows; + + std::stringstream confStr; + if(i < resultLayers.size()) { + auto emotionIndex = xt::argmax(resultLayers[i])(0); + if(emotionIndex < emotions.size()) { + confStr << emotions[emotionIndex]; + } else { + confStr << "Err index: " << emotionIndex; + } + } else { + confStr << "NA"; + } + + cv::putText(frame, confStr.str(), cv::Point(x1 + 10, y1 + 40), cv::FONT_HERSHEY_TRIPLEX, 0.5, color); + cv::rectangle(frame, cv::Rect(cv::Point(x1, y1), cv::Point(x2, y2)), color, cv::FONT_HERSHEY_SIMPLEX); + } + cv::imshow("video", frame); +} + +int main(int argc, char** argv) { + std::string nnPath; + std::string nnEmoPath; + if(argc > 2) { + nnPath = std::string(argv[1]); + nnEmoPath = std::string(argv[2]); + } else { + std::cout << "call with arguments: {detection blob} {emotion blob}" << std::endl; + return 1; + } + + + // Create pipeline + auto device = std::make_shared(dai::OpenVINO::VERSION_UNIVERSAL, dai::UsbSpeed::HIGH); + dai::Pipeline pipeline(device); + // Define source and output + auto camRgb = pipeline.create(); + auto nn = pipeline.create(); + auto nnEmo = pipeline.create(); + auto det = pipeline.create(); + auto manipConf = pipeline.create(); + auto manip = pipeline.create(); + + // Camera props + camRgb->setBoardSocket(dai::CameraBoardSocket::CAM_A); + camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P); + camRgb->setPreviewSize(300, 300); + camRgb->setInterleaved(false); + + // Face detection NN props + nn->setNumInferenceThreads(2); + nn->input.setBlocking(false); + dai::OpenVINO::Blob blob1(nnPath); + nn->setBlob(blob1); + + // Face detection NN parser props + det->setBlob(blob1); + det->setNNFamily(DetectionNetworkType::MOBILENET); + det->setConfidenceThreshold(0.5); + + // Emotion detection NN props + nnEmo->setNumInferenceThreads(2); + nnEmo->input.setBlocking(false); + dai::OpenVINO::Blob blob2(nnEmoPath); + nnEmo->setBlob(blob2); + + // ImageManip props + manip->initialConfig.setResize(64,64); + + // Linking + /* + rgb -> nn -> det -> manipConf -> manip -> nnEmo + ---------------------------> + */ + camRgb->preview.link(nn->input); + nn->out.link(det->input); + det->out.link(manipConf->inputDetections); + manipConf->outputManipulators.link(manip->inputConfig); + camRgb->preview.link(manip->inputImage); + manip->out.link(nnEmo->input); + + auto outPassthrough = nn->passthrough.createOutputQueue(); + auto outDet = det->out.createOutputQueue(); + auto outNNEmo = nnEmo->out.createOutputQueue(); + + pipeline.start(); + while(pipeline.isRunning()) { + std::shared_ptr inRgb; + std::shared_ptr inDet; + std::vector> inNNEmos; + + inRgb = outPassthrough->get(); + inDet = outDet->get(); + inNNEmos = outNNEmo->getAll(); + + cv::Mat frame; + std::vector detections; + std::vector> resultLayers; + + if(inRgb) { + frame = inRgb->getCvFrame(); + } + + if(inDet) { + detections = inDet->detections; + } + + for(auto& inNN : inNNEmos) { + if(inNN && !inNN->tensors.empty()) { + resultLayers.push_back(inNN->getTensor(inNN->tensors.back().name, false)); + } else { + resultLayers.push_back({}); + } + } + + + + if(!frame.empty()) { + displayFrame(frame, detections, resultLayers); + } + + int key = cv::waitKey(1); + if(key == 'q' || key == 'Q') { + return 0; + } + } + return 0; +}