Merge pull request #940 from luxonis/release_v2.24.0

Release v2.24.0

CMakeLists.txt

@@ -46,7 +46,7 @@ if(WIN32)
 endif()
 
 # Create depthai project
-project(depthai VERSION "2.23.0" LANGUAGES CXX C)
+project(depthai VERSION "2.24.0" LANGUAGES CXX C)
 get_directory_property(has_parent PARENT_DIRECTORY)
 if(has_parent)
     set(DEPTHAI_VERSION ${PROJECT_VERSION} PARENT_SCOPE)
@@ -208,8 +208,10 @@ add_library(${TARGET_CORE_NAME}
     src/pipeline/node/ColorCamera.cpp
     src/pipeline/node/Camera.cpp
     src/pipeline/node/ToF.cpp
+    src/pipeline/node/MessageDemux.cpp
     src/pipeline/node/MonoCamera.cpp
     src/pipeline/node/StereoDepth.cpp
+    src/pipeline/node/Sync.cpp
     src/pipeline/node/NeuralNetwork.cpp
     src/pipeline/node/ImageManip.cpp
     src/pipeline/node/Warp.cpp
@@ -229,6 +231,7 @@ add_library(${TARGET_CORE_NAME}
     src/pipeline/node/UVC.cpp
     src/pipeline/datatype/Buffer.cpp
     src/pipeline/datatype/ImgFrame.cpp
+    src/pipeline/datatype/EncodedFrame.cpp
     src/pipeline/datatype/ImageManipConfig.cpp
     src/pipeline/datatype/CameraControl.cpp
     src/pipeline/datatype/NNData.cpp
@@ -247,6 +250,8 @@ add_library(${TARGET_CORE_NAME}
     src/pipeline/datatype/TrackedFeatures.cpp
     src/pipeline/datatype/FeatureTrackerConfig.cpp
     src/pipeline/datatype/ToFConfig.cpp
+    src/pipeline/datatype/MessageGroup.cpp
+    src/utility/H26xParsers.cpp
     src/utility/Initialization.cpp
     src/utility/Resources.cpp
     src/utility/Path.cpp

README.md

@@ -186,6 +186,9 @@ The following environment variables can be set to alter default behavior of the
 | DEPTHAI_BOOTLOADER_BINARY_USB | Overrides device USB Bootloader binary. Mostly for internal debugging purposes. |
 | DEPTHAI_BOOTLOADER_BINARY_ETH | Overrides device Network Bootloader binary. Mostly for internal debugging purposes. |
 | DEPTHAI_ALLOW_FACTORY_FLASHING | Internal use only |
+| DEPTHAI_LIBUSB_ANDROID_JAVAVM | JavaVM pointer that is passed to libusb for rootless Android interaction with devices. Interpreted as decimal value of uintptr_t |
+| DEPTHAI_CRASHDUMP | Directory in which to save the crash dump. |
+| DEPTHAI_CRASHDUMP_TIMEOUT | Specifies the duration in seconds to wait for device reboot when obtaining a crash dump. Crash dump retrieval disabled if 0. |
 
 ## Running tests
 

cmake/Depthai/DepthaiDeviceSideConfig.cmake

@@ -2,7 +2,7 @@
 set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot")
 
 # "full commit hash of device side binary"
-set(DEPTHAI_DEVICE_SIDE_COMMIT "39a9d271a9ed0172f6481877723fca96d41b54c6")
+set(DEPTHAI_DEVICE_SIDE_COMMIT "a95f582a61ec9bdbd0f72dec84822455872ffaf7")
 
 # "version if applicable"
 set(DEPTHAI_DEVICE_SIDE_VERSION "")

cmake/Hunter/config.cmake

@@ -7,9 +7,9 @@ hunter_config(
 
 hunter_config(
     XLink
-    VERSION "luxonis-2021.4.2-develop"
-    URL "https://github.com/luxonis/XLink/archive/c940feaf9321f06a7d9660f28e686a9718135f38.tar.gz"
-    SHA1 "52935b6ceb470ee632de3348b9d2aaa2c6c24ac0"
+    VERSION "luxonis-2021.4.2-xlink-linkid-race-fix"
+    URL "https://github.com/luxonis/XLink/archive/e9eb1ef38030176ad70cddd3b545d5e6c509f1e1.tar.gz"
+    SHA1 "b1e4ded41cd7b9c37189468e2aaddbb10cbda9f6"
     CMAKE_ARGS
         XLINK_ENABLE_LIBUSB=${DEPTHAI_ENABLE_LIBUSB}
 )
@@ -101,9 +101,9 @@ hunter_config(
 # Specific Catch2 version
 hunter_config(
     Catch2
-    VERSION "2.13.7"
-    URL "https://github.com/catchorg/Catch2/archive/refs/tags/v3.2.1.tar.gz"
-    SHA1 "acfba7f71cbbbbf60bc1bc4c0e3efca4a9c70df7"
+    VERSION "3.4.0"
+    URL "https://github.com/catchorg/Catch2/archive/refs/tags/v3.4.0.tar.gz"
+    SHA1 "4c308576c856a43dc88949a8f64ef90ebf94ae1b"
 )
 
 # ZLib - Luxonis fix for alias on imported target for old CMake versions

examples/CMakeLists.txt

@@ -369,3 +369,9 @@ target_compile_definitions(detection_parser PRIVATE BLOB_PATH="${mobilenet_blob}
 
 # DetectionParser
 dai_add_example(crash_report CrashReport/crash_report.cpp OFF)
+
+# Sync
+dai_add_example(sync_scripts Sync/sync_scripts.cpp ON)
+dai_add_example(demux_message_group Sync/demux_message_group.cpp ON)
+dai_add_example(depth_video_synced Sync/depth_video_synced.cpp ON)
+dai_add_example(imu_video_synced Sync/imu_video_synced.cpp ON)

examples/SpatialDetection/spatial_calculator_multi_roi.cpp

@@ -61,7 +61,7 @@ int main() {
 
     // Connect to device and start pipeline
     dai::Device device(pipeline);
-    device.setIrLaserDotProjectorBrightness(1000);
+    device.setIrLaserDotProjectorIntensity(0.7f);
 
     // Output queue will be used to get the depth frames from the outputs defined above
     auto depthQueue = device.getOutputQueue("depth", 4, false);
@@ -106,4 +106,4 @@ int main() {
         }
     }
     return 0;
-}
+}

examples/Sync/demux_message_group.cpp

@@ -0,0 +1,63 @@
+#include <chrono>
+#include <iostream>
+
+#include "depthai/depthai.hpp"
+
+int main() {
+    dai::Pipeline pipeline;
+
+    auto script1 = pipeline.create<dai::node::Script>();
+    script1->setScript(
+        R"SCRPT(
+from time import sleep
+
+while True:
+    sleep(1)
+    b = Buffer(512)
+    b.setData(bytes(4 * [i for i in range(0, 128)]))
+    b.setTimestamp(Clock.now())
+    node.io['out'].send(b)
+)SCRPT");
+
+    auto script2 = pipeline.create<dai::node::Script>();
+    script2->setScript(
+        R"SCRPT(
+from time import sleep
+
+while True:
+    sleep(0.3)
+    b = Buffer(512)
+    b.setData(bytes(4 * [i for i in range(128, 256)]))
+    b.setTimestamp(Clock.now())
+    node.io['out'].send(b)
+)SCRPT");
+
+    auto sync = pipeline.create<dai::node::Sync>();
+    sync->setSyncThreshold(std::chrono::milliseconds(100));
+
+    auto demux = pipeline.create<dai::node::MessageDemux>();
+
+    auto xout1 = pipeline.create<dai::node::XLinkOut>();
+    xout1->setStreamName("xout1");
+    auto xout2 = pipeline.create<dai::node::XLinkOut>();
+    xout2->setStreamName("xout2");
+
+    script1->outputs["out"].link(sync->inputs["s1"]);
+    script2->outputs["out"].link(sync->inputs["s2"]);
+    sync->out.link(demux->input);
+    demux->outputs["s1"].link(xout1->input);
+    demux->outputs["s2"].link(xout2->input);
+
+    dai::Device device(pipeline);
+    std::cout << "Start" << std::endl;
+    auto queue1 = device.getOutputQueue("xout1", 10, true);
+    auto queue2 = device.getOutputQueue("xout2", 10, true);
+    while(true) {
+        auto bufS1 = queue1->get<dai::Buffer>();
+        auto bufS2 = queue2->get<dai::Buffer>();
+        std::cout << "Buffer 1 timestamp: " << bufS1->getTimestamp().time_since_epoch().count() << std::endl;
+        std::cout << "Buffer 2 timestamp: " << bufS2->getTimestamp().time_since_epoch().count() << std::endl;
+        std::cout << "----------" << std::endl;
+        std::this_thread::sleep_for(std::chrono::milliseconds(200));
+    }
+}

examples/Sync/depth_video_synced.cpp

@@ -0,0 +1,68 @@
+#include <iostream>
+
+// Includes common necessary includes for development using depthai library
+#include "depthai/depthai.hpp"
+
+int main() {
+    // Create pipeline
+    dai::Pipeline pipeline;
+
+    // Define sources and outputs
+    auto monoLeft = pipeline.create<dai::node::MonoCamera>();
+    auto monoRight = pipeline.create<dai::node::MonoCamera>();
+    auto color = pipeline.create<dai::node::ColorCamera>();
+    auto stereo = pipeline.create<dai::node::StereoDepth>();
+    auto sync = pipeline.create<dai::node::Sync>();
+
+    auto xoutGrp = pipeline.create<dai::node::XLinkOut>();
+
+    // XLinkOut
+    xoutGrp->setStreamName("xout");
+
+    // Properties
+    monoLeft->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P);
+    monoLeft->setCamera("left");
+    monoRight->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P);
+    monoRight->setCamera("right");
+
+    stereo->setDefaultProfilePreset(dai::node::StereoDepth::PresetMode::HIGH_ACCURACY);
+
+    color->setCamera("color");
+
+    sync->setSyncThreshold(std::chrono::milliseconds(100));
+
+    // Linking
+    monoLeft->out.link(stereo->left);
+    monoRight->out.link(stereo->right);
+
+    stereo->disparity.link(sync->inputs["disparity"]);
+    color->video.link(sync->inputs["video"]);
+
+    sync->out.link(xoutGrp->input);
+
+    // Connect to device and start pipeline
+    dai::Device device(pipeline);
+
+    auto queue = device.getOutputQueue("xout", 10, true);
+
+    float disparityMultiplier = 255 / stereo->initialConfig.getMaxDisparity();
+
+    while(true) {
+        auto msgGrp = queue->get<dai::MessageGroup>();
+        for(auto& frm : *msgGrp) {
+            auto imgFrm = std::dynamic_pointer_cast<dai::ImgFrame>(frm.second);
+            cv::Mat img = imgFrm->getCvFrame();
+            if(frm.first == "disparity") {
+                img.convertTo(img, CV_8UC1, disparityMultiplier);  // Extend disparity range
+                cv::applyColorMap(img, img, cv::COLORMAP_JET);
+            }
+            cv::imshow(frm.first, img);
+        }
+
+        int key = cv::waitKey(1);
+        if(key == 'q' || key == 'Q') {
+            return 0;
+        }
+    }
+    return 0;
+}

examples/Sync/imu_video_synced.cpp

@@ -0,0 +1,74 @@
+#include <iostream>
+#include <opencv2/opencv.hpp>
+
+#include "depthai/depthai.hpp"
+
+int main() {
+    dai::Device device;
+
+    auto imuType = device.getConnectedIMU();
+    auto imuFirmwareVersion = device.getIMUFirmwareVersion();
+    std::cout << "IMU type: " << imuType << ", firmware version: " << imuFirmwareVersion << std::endl;
+
+    if(imuType != "BNO086") {
+        std::cout << "Rotation vector output is supported only by BNO086!" << std::endl;
+        return 1;
+    }
+
+    dai::Pipeline pipeline;
+
+    auto colorCamera = pipeline.create<dai::node::ColorCamera>();
+    auto imu = pipeline.create<dai::node::IMU>();
+    auto sync = pipeline.create<dai::node::Sync>();
+    auto xoutGroup = pipeline.create<dai::node::XLinkOut>();
+
+    xoutGroup->setStreamName("xout");
+
+    colorCamera->setCamera("color");
+
+    imu->enableIMUSensor(dai::IMUSensor::ROTATION_VECTOR, 120);
+    imu->setBatchReportThreshold(1);
+    imu->setMaxBatchReports(10);
+
+    sync->setSyncThreshold(std::chrono::milliseconds(10));
+    sync->setSyncAttempts(-1);  // Infinite attempts
+
+    colorCamera->video.link(sync->inputs["video"]);
+    imu->out.link(sync->inputs["imu"]);
+
+    sync->out.link(xoutGroup->input);
+
+    device.startPipeline(pipeline);
+
+    auto groupQueue = device.getOutputQueue("xout", 3, false);
+
+    while(true) {
+        auto groupMessage = groupQueue->get<dai::MessageGroup>();
+        auto imuData = groupMessage->get<dai::IMUData>("imu");
+        auto colorData = groupMessage->get<dai::ImgFrame>("video");
+        auto timeDifference = imuData->getTimestampDevice() - colorData->getTimestampDevice();
+        auto timeDifferenceUs = std::chrono::duration_cast<std::chrono::microseconds>(timeDifference).count();
+
+        std::cout << "Time difference between messages is: " << std::abs(timeDifferenceUs / 1000.0) << " ms" << std::endl;
+
+        for(auto& packet : imuData->packets) {
+            auto& rv = packet.rotationVector;
+
+            printf(
+                "Quaternion: i: %.3f j: %.3f k: %.3f real: %.3f\n"
+                "Accuracy (rad): %.3f \n",
+                rv.i,
+                rv.j,
+                rv.k,
+                rv.real,
+                rv.rotationVectorAccuracy);
+        }
+
+        cv::imshow("Color", colorData->getCvFrame());
+        if(cv::waitKey(1) == 'q') {
+            break;
+        }
+    }
+
+    return 0;
+}

examples/Sync/sync_scripts.cpp

@@ -0,0 +1,56 @@
+#include <chrono>
+#include <iostream>
+
+#include "depthai/depthai.hpp"
+
+int main() {
+    dai::Pipeline pipeline;
+
+    auto script1 = pipeline.create<dai::node::Script>();
+    script1->setScript(
+        R"SCRPT(
+from time import sleep
+
+while True:
+    sleep(1)
+    b = Buffer(512)
+    b.setData(bytes(4 * [i for i in range(0, 128)]))
+    b.setTimestamp(Clock.now())
+    node.io['out'].send(b)
+)SCRPT");
+
+    auto script2 = pipeline.create<dai::node::Script>();
+    script2->setScript(
+        R"SCRPT(
+from time import sleep
+
+while True:
+    sleep(0.3)
+    b = Buffer(512)
+    b.setData(bytes(4 * [i for i in range(128, 256)]))
+    b.setTimestamp(Clock.now())
+    node.io['out'].send(b)
+)SCRPT");
+
+    auto sync = pipeline.create<dai::node::Sync>();
+    sync->setSyncThreshold(std::chrono::milliseconds(100));
+
+    auto xout = pipeline.create<dai::node::XLinkOut>();
+    xout->setStreamName("xout");
+
+    sync->out.link(xout->input);
+    script1->outputs["out"].link(sync->inputs["s1"]);
+    script2->outputs["out"].link(sync->inputs["s2"]);
+
+    dai::Device device(pipeline);
+    std::cout << "Start" << std::endl;
+    auto queue = device.getOutputQueue("xout", 10, true);
+    while(true) {
+        auto grp = queue->get<dai::MessageGroup>();
+        std::cout << "Buffer 1 timestamp: " << grp->get<dai::Buffer>("s1")->getTimestamp().time_since_epoch().count() << std::endl;
+        std::cout << "Buffer 2 timestamp: " << grp->get<dai::Buffer>("s2")->getTimestamp().time_since_epoch().count() << std::endl;
+        std::cout << "Time interval between messages: " << static_cast<double>(grp->getIntervalNs()) / 1e6 << "ms" << std::endl;
+        std::cout << "----------" << std::endl;
+        std::this_thread::sleep_for(std::chrono::milliseconds(200));
+    }
+}

include/depthai/device/CalibrationHandler.hpp

@@ -166,7 +166,10 @@ class CalibrationHandler {
      * Get the Distortion Coefficients object
      *
      * @param cameraId Uses the cameraId to identify which distortion Coefficients to return.
-     * @return the distortion coefficients of the requested camera in this order: [k1,k2,p1,p2,k3,k4,k5,k6,s1,s2,s3,s4,τx,τy]
+     * @return the distortion coefficients of the requested camera in this order: [k1,k2,p1,p2,k3,k4,k5,k6,s1,s2,s3,s4,τx,τy] for CameraModel::Perspective
+     * or [k1, k2, k3, k4] for CameraModel::Fisheye
+     * see https://docs.opencv.org/4.5.4/d9/d0c/group__calib3d.html for Perspective model (Rational Polynomial Model)
+     * see https://docs.opencv.org/4.5.4/db/d58/group__calib3d__fisheye.html for Fisheye model
      */
     std::vector<float> getDistortionCoefficients(CameraBoardSocket cameraId) const;