From 9f5e2293490ff19b8c1469ffc1c9ffe52f9cc1d9 Mon Sep 17 00:00:00 2001 From: Danilo Pejovic Date: Mon, 25 Nov 2024 10:18:24 +0100 Subject: [PATCH] Adding testbed implementation of CI tests; adding option to tests rvc4 against different versions of os; --- .github/workflows/stability.workflow.yml | 15 ++-- .github/workflows/test.workflow.yml | 80 +++++++++++-------- scripts/hil/powercycle_rvc2.py | 31 +++++++ scripts/hil/prepare_hil_framework.sh | 5 ++ scripts/hil/run_hil_stability.sh | 20 +++++ scripts/hil/run_hil_tests.sh | 60 ++++++++++++++ tests/run_tests.py | 31 +++++-- .../ondevice_tests/device_usbspeed_test.cpp | 13 +-- tests/src/ondevice_tests/filesystem_test.cpp | 7 +- 9 files changed, 206 insertions(+), 56 deletions(-) create mode 100644 scripts/hil/powercycle_rvc2.py create mode 100755 scripts/hil/prepare_hil_framework.sh create mode 100755 scripts/hil/run_hil_stability.sh create mode 100755 scripts/hil/run_hil_tests.sh diff --git a/.github/workflows/stability.workflow.yml b/.github/workflows/stability.workflow.yml index cfecdc134..82d598647 100644 --- a/.github/workflows/stability.workflow.yml +++ b/.github/workflows/stability.workflow.yml @@ -22,17 +22,14 @@ jobs: # Testing test: - runs-on: ['self-hosted', 'hil-stability', 'linux'] + runs-on: ['self-hosted', 'testbed-runner', 'linux'] timeout-minutes: 1450 # 24h & 10minutes steps: - - name: Cache .hunter folder - uses: actions/cache@v3 - with: - path: $HOME/.hun_vanilla - key: hunter-linux-stability-vanilla - uses: actions/checkout@v3 with: submodules: 'recursive' + - name: Prepare HIL Framework + run: source scripts/hil/prepare_hil_framework.sh # TODO also modify above hunter key to 'asan' # - name: Specify ASAN toolchain path @@ -48,7 +45,5 @@ jobs: # Release build - name: Configure, Build and Test run: | - cmake -S . -B build -D CMAKE_BUILD_TYPE=Release -D HUNTER_ROOT=$HOME/.hun_vanilla -D DEPTHAI_BUILD_TESTS=ON - cmake --build build --parallel 8 --config Release --target stability_stress_test - cd build - ../ci/stability_stress_test_combined.sh 86400 \ No newline at end of file + export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/$GITHUB_RUN_ID+stability" + exec hil --capabilities depthai-core-hil --reservation-name $RESERVATION_NAME -w -s --commands 'cd /tmp/depthai-core|| exit' 'scripts/hil/run_hil_stability.sh' diff --git a/.github/workflows/test.workflow.yml b/.github/workflows/test.workflow.yml index e48512e4c..02da9bb17 100644 --- a/.github/workflows/test.workflow.yml +++ b/.github/workflows/test.workflow.yml @@ -19,7 +19,7 @@ concurrency: jobs: # Testing - test: + rvc2_test: env: VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" strategy: @@ -33,14 +33,9 @@ jobs: arch: 'x64' flavor: 'tsan' fail-fast: false - runs-on: ['self-hosted', 'hil-test-v3', '${{ matrix.os }}', '${{ matrix.arch }}'] + runs-on: ['self-hosted', 'testbed-runner'] steps: - - name: Cache .hunter folder - uses: actions/cache@v3 - with: - path: $HOME/.hun2_${{ matrix.flavor }} - key: hunter-${{ matrix.os }}-${{ matrix.cmake }} - uses: actions/checkout@v3 with: submodules: 'recursive' @@ -51,37 +46,56 @@ jobs: core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - - name: Create virtual environment - run: | - python -m venv venv - source venv/bin/activate - python -m pip install --upgrade pip - python -m pip install numpy - - - name: Specify toolchain path - if: matrix.flavor == 'vanilla' - run: echo "CMAKE_TOOLCHAIN_PATH=" >> $GITHUB_ENV - - name: Specify toolchain path - if: matrix.flavor != 'vanilla' - run: echo "CMAKE_TOOLCHAIN_PATH=$PWD/cmake/toolchain/${{ matrix.flavor }}.cmake" >> $GITHUB_ENV + - name: Prepare HIL Framework + run: source scripts/hil/prepare_hil_framework.sh - name: Configure, Build and Test if: matrix.os != 'linux' run: | - source venv/bin/activate # Activate virtual environment - cmake -S . -B build -D CMAKE_BUILD_TYPE=Release -D HUNTER_ROOT=$HOME/.hun2_${{ matrix.flavor }} -D DEPTHAI_BUILD_EXAMPLES=ON -D DEPTHAI_BUILD_TESTS=ON -D DEPTHAI_TEST_EXAMPLES=ON -D DEPTHAI_BUILD_PYTHON=ON -D DEPTHAI_PYTHON_TEST_EXAMPLES=ON -D DEPTHAI_PYTHON_ENABLE_EXAMPLES=ON - cmake --build build --parallel 4 --config Release - cd tests - python3 run_tests.py + export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#rvc2-${{ matrix.os }}-${{ matrix.flavor }}" + exec hil --capabilities depthai-core-hil --reservation-name $RESERVATION_NAME --wait --sync-workspace --exclude-build --commands 'cd /tmp/depthai-core|| exit' 'scripts/hil/run_hil_tests.sh ${{ matrix.flavor }} --rvc2' - name: Configure, Build and Test if: matrix.os == 'linux' run: | - export DISPLAY=:99 - xdpyinfo -display $DISPLAY >/dev/null 2>&1 || (Xvfb $DISPLAY &) - source venv/bin/activate # Activate virtual environment - python3 -m pip install jinja2 - cmake -S . -B build -D CMAKE_BUILD_TYPE=Release -D HUNTER_ROOT=$HOME/.hun2_${{ matrix.flavor }} -D DEPTHAI_BUILD_EXAMPLES=ON -D DEPTHAI_BUILD_TESTS=ON -D DEPTHAI_TEST_EXAMPLES=ON -D DEPTHAI_BUILD_PYTHON=ON -D DEPTHAI_PYTHON_TEST_EXAMPLES=ON -D DEPTHAI_PYTHON_ENABLE_EXAMPLES=ON - cmake --build build --parallel 4 --config Release - cd tests - python3 run_tests.py + export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#rvc2-${{ matrix.os }}-${{ matrix.flavor }}" + exec hil --capabilities depthai-core-hil --reservation-name $RESERVATION_NAME --wait --sync-workspace --exclude-build --commands 'cd /tmp/depthai-core|| exit' 'scripts/hil/run_hil_tests.sh ${{ matrix.flavor }} --rvc2' + + # Testing + rvc4_test: + env: + VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" + strategy: + matrix: + # os: ['windows', 'macos', 'linux'] + os: ['linux'] + arch: ['x64'] + flavor: ['vanilla', 'asan-ubsan', 'tsan'] + rvc4os: ['r851.1.2'] + exclude: + - os: 'windows' + arch: 'x64' + flavor: 'tsan' + fail-fast: false + runs-on: ['self-hosted', 'testbed-runner'] + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Prepare HIL Framework + run: source scripts/hil/prepare_hil_framework.sh + + - name: Configure, Build and Test + if: matrix.os != 'linux' + run: | + export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#rvc4-${{ matrix.os }}-${{ matrix.rvc4os }}-${{ matrix.flavor }}"" + exec hil --capabilities depthai-core-hil --models oak4_pro --reservation-name $RESERVATION_NAME --wait --sync-workspace --exclude-build --commands 'cd /tmp/depthai-core|| exit' 'scripts/hil/run_hil_tests.sh ${{ matrix.flavor }} --rvc4 ' + + - name: Configure, Build and Test + if: matrix.os == 'linux' + run: | + export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#rvc4-${{ matrix.os }}-${{ matrix.rvc4os }}-${{ matrix.flavor }}" + exec hil --capabilities depthai-core-hil --models oak4_pro --reservation-name $RESERVATION_NAME --wait --sync-workspace --exclude-build --rvc4-os-version ${{ matrix.rvc4os }} --commands 'cd /tmp/depthai-core|| exit' 'scripts/hil/run_hil_tests.sh ${{ matrix.flavor }} --rvc4' + diff --git a/scripts/hil/powercycle_rvc2.py b/scripts/hil/powercycle_rvc2.py new file mode 100644 index 000000000..7357c7c60 --- /dev/null +++ b/scripts/hil/powercycle_rvc2.py @@ -0,0 +1,31 @@ +from lib_testbed.power.PowerFactory import PowerFactory +from lib_testbed.config.Config import Config +from lib_testbed.utils.ssh_util import * +from time import sleep + +testbed_name=os.getenv("SET_HIL_TESTBED") +config=Config(testbed_name) +print(config.cameras) + +for camera in config.cameras: + if camera.platform == "rvc2": + print(camera.name) + power_control=PowerFactory.get_power_object_for_device(config, camera.name) + power_control[0].off() + sleep(2) + power_control[0].on() +for camera in config.cameras: + if camera.platform == "rvc4": + power_control=PowerFactory.get_power_object_for_device(config, camera.name) + if len(power_control) == 2: + power_control[1].off() + power_control[0].off() + sleep(2) + power_control[0].on() + power_control[1].on() + elif len(power_control) == 1: + power_control[0].off() + sleep(2) + power_control[0].on() + else: + print("No devices available in power_control.") diff --git a/scripts/hil/prepare_hil_framework.sh b/scripts/hil/prepare_hil_framework.sh new file mode 100755 index 000000000..b6f61ff00 --- /dev/null +++ b/scripts/hil/prepare_hil_framework.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +echo "/home/$USER/hil_framework/lib_testbed/tools" >> $GITHUB_PATH +echo "PYTHONPATH="$PYTHONPATH:/home/$USER/hil_framework"" >> $GITHUB_ENV +echo "HIL_FRAMEWORK_PATH="/home/$USER/hil_framework"" >> $GITHUB_ENV diff --git a/scripts/hil/run_hil_stability.sh b/scripts/hil/run_hil_stability.sh new file mode 100755 index 000000000..57171697c --- /dev/null +++ b/scripts/hil/run_hil_stability.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Set up a Python virtual environment +rm -rf venv +python3 -m venv venv +source venv/bin/activate +rm -rf build/ +export PATH="$PATH:/home/hil/hil_framework/lib_testbed/tools" +export PYTHONPATH="$PYTHONPATH:/home/hil/hil_framework" +export HIL_FRAMEWORK_PATH="/home/hil/hil_framework" +source /home/hil/.SETUP_CONFIG_VARS + +# Install required Python packages +pip install numpy pytest pytest-html > /dev/null 2>&1 +pushd /home/$USER/hil_framework/ > /dev/null 2>&1 && pip install -r requirements.txt > /dev/null 2>&1 && popd > /dev/null 2>&1 + +cmake -S . -B build -D CMAKE_BUILD_TYPE=Release -D HUNTER_ROOT=$HOME/.hun_vanilla -D DEPTHAI_BUILD_TESTS=ON +cmake --build build --parallel 8 --config Release --target stability_stress_test +cd build +../ci/stability_stress_test_combined.sh 86400 diff --git a/scripts/hil/run_hil_tests.sh b/scripts/hil/run_hil_tests.sh new file mode 100755 index 000000000..9643a16d1 --- /dev/null +++ b/scripts/hil/run_hil_tests.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Check if the argument is provided +if [ $# -lt 1 ]; then + echo "Usage: $0 [--rvc2 | --rvc4]" + echo " test_type Specify test flavor: vanilla, asan-ubsan, or tsan." + echo " --rvc2 Optional: Run tests with RVC2 configuration." + echo " --rvc4 Optional: Run tests with RVC4 configuration." + exit 1 +fi +# Get the test type from the argument +TEST_FLAVOR=$1 +TEST_ARGS=$2 + +# Set up a Python virtual environment +rm -rf venv +python3 -m venv venv +source venv/bin/activate +export LC_ALL=en_US.UTF-8 +locale + +if [ "$TEST_FLAVOR" == "vanilla" ]; then + echo $CMAKE_TOOLCHAIN_PATH +else + export CMAKE_TOOLCHAIN_PATH=$PWD/cmake/toolchain/${FLAVOR}.cmake +fi + +export PATH="$PATH:/home/hil/hil_framework/lib_testbed/tools" +export PYTHONPATH="$PYTHONPATH:/home/hil/hil_framework" +export HIL_FRAMEWORK_PATH="/home/hil/hil_framework" +source /home/hil/.SETUP_CONFIG_VARS + +# Install required Python packages +pip install numpy pytest pytest-html > /dev/null 2>&1 +pushd /home/$USER/hil_framework/ > /dev/null 2>&1 && git pull && git submodule update --init --recursive > /dev/null 2>&1 && popd > /dev/null 2>&1 +pushd /home/$USER/hil_framework/ > /dev/null 2>&1 && pip install -r requirements.txt > /dev/null 2>&1 && popd > /dev/null 2>&1 + +# Check for optional RVC arguments +if [[ "$TEST_ARGS" == "--rvc4" ]]; then + echo "Running RVC4 configuration commands..." + adb root + adb shell systemctl stop agentconfd setup + adb shell systemctl disable agentconfd setup + adb shell mkdir -p /persist/factory + adb shell touch /persist/factory/enabled + adb shell reboot + echo "Device reboot initiated for RVC4. Factory mode enabled." +elif [[ "$TEST_ARGS" == "--rvc2" ]]; then + python scripts/hil/powercycle_rvc2.py + sleep 10 + echo "RVC2 configuration specified. Continuing with test setup..." +fi + +cmake -S . -B build -D CMAKE_BUILD_TYPE=Release -D HUNTER_ROOT=$HOME/.hun2_$TEST_FLAVOR -D DEPTHAI_BUILD_EXAMPLES=ON -D DEPTHAI_BUILD_TESTS=ON -D DEPTHAI_TEST_EXAMPLES=ON -D DEPTHAI_BUILD_PYTHON=ON -D DEPTHAI_PYTHON_TEST_EXAMPLES=ON -D DEPTHAI_PYTHON_ENABLE_EXAMPLES=ON +cmake --build build --parallel 2 --config Release + +export DISPLAY=:99 +xdpyinfo -display $DISPLAY >/dev/null 2>&1 || (Xvfb $DISPLAY &) +cd tests +python3 run_tests.py $TEST_ARGS diff --git a/tests/run_tests.py b/tests/run_tests.py index efcfe2eba..430a86b34 100644 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -77,17 +77,29 @@ def compute_or_result(results): default=default_path, ) + parser.add_argument( + "--rvc4", + action="store_true", + required=False, + ) + + parser.add_argument( + "--rvc2", + action="store_true", + required=False, + ) + args = parser.parse_args() test_dir = args.test_dir print("Going to run tests in directory:", test_dir) # cd to the test directory os.chdir(pathlib.Path(test_dir).resolve()) # Example test configurations - test_configs = [ + all_configs = [ { "name": "Host", "env": {}, - "label": ["onhost"], + "labels": ["onhost"], }, { "name": "RVC4", @@ -97,22 +109,31 @@ def compute_or_result(results): { "name": "RVC2 - USB", "env": {"DEPTHAI_PLATFORM": "rvc2", "DEPTHAI_PROTOCOL": "usb"}, - "label": ["rvc2", "usb"], + "labels": ["rvc2", "usb"], }, { "name": "RVC2 - POE", "env": {"DEPTHAI_PLATFORM": "rvc2", "DEPTHAI_PROTOCOL": "tcpip"}, - "label": ["rvc2", "poe"], + "labels": ["rvc2", "poe"], }, ] # List to keep track of results resultThreads = [] + # Filter configurations based on command-line arguments + if args.rvc4==args.rvc2: + test_configs=all_configs + elif args.rvc4: + test_configs = [config for config in all_configs if "rvc4" in config.get("labels", []) or "onhost" in config.get("labels", [])] + elif args.rvc2: + test_configs = [config for config in all_configs if "rvc2" in config.get("labels", []) or "onhost" in config.get("labels", [])] + + for config in test_configs: name = config["name"] env_vars = config["env"] - labels = config.get("labels") or config.get("label") + labels = config.get("labels") print(f"Running tests for configuration: {name}") resultThread = run_ctest(env_vars, labels, blocking=False, name=name) diff --git a/tests/src/ondevice_tests/device_usbspeed_test.cpp b/tests/src/ondevice_tests/device_usbspeed_test.cpp index 9ecb06140..e6cea9a8c 100644 --- a/tests/src/ondevice_tests/device_usbspeed_test.cpp +++ b/tests/src/ondevice_tests/device_usbspeed_test.cpp @@ -70,12 +70,13 @@ TEST_CASE("Usb config modes") { p.setBoardConfig(cfg.board); } - SECTION("UsbSpeed::SUPER_PLUS") { - dai::DeviceBase::Config cfg; - cfg.board.usb.maxSpeed = dai::UsbSpeed::SUPER_PLUS; - speed = dai::UsbSpeed::SUPER_PLUS; - p.setBoardConfig(cfg.board); - } + // SECTION("UsbSpeed::SUPER_PLUS") { + // dai::DeviceBase::Config cfg; + // cfg.board.usb.maxSpeed = dai::UsbSpeed::SUPER_PLUS; + // speed = dai::UsbSpeed::SUPER_PLUS; + // p.setBoardConfig(cfg.board); + // } + // dai::Device d(p); REQUIRE(d.getUsbSpeed() == speed); diff --git a/tests/src/ondevice_tests/filesystem_test.cpp b/tests/src/ondevice_tests/filesystem_test.cpp index 1860a10c8..4996d76c1 100644 --- a/tests/src/ondevice_tests/filesystem_test.cpp +++ b/tests/src/ondevice_tests/filesystem_test.cpp @@ -246,11 +246,14 @@ TEST_CASE("dai::Path with DeviceBootloader") { const std::wstring wstrBadfile(LPATH5); const dai::Path diaBadWide(LPATH5); #endif - + std::this_thread::sleep_for(std::chrono::seconds(10)); bool found = false; dai::DeviceInfo deviceInfo; std::tie(found, deviceInfo) = dai::DeviceBootloader::getFirstAvailableDevice(); if(found) { + if(deviceInfo.state == X_LINK_BOOTLOADER) { + std::cout << "Device is already booted into bootloader mode. Booting tests will be skipped." << std::endl; + } else { REQUIRE_NOTHROW([&]() { dai::DeviceBootloader bl(deviceInfo); auto currentBlType = bl.getType(); @@ -310,7 +313,7 @@ TEST_CASE("dai::Path with DeviceBootloader") { }(), ContainsSubstring("doesn't exist")); #endif - } else { + }} else { std::cout << "No devices found" << std::endl; } }