Initialize v2 branch

.ci/Dockerfile

@@ -0,0 +1,67 @@
+#########################################################
+## Python Environment with CUDA
+#########################################################
+ARG ver_cuda="11.7.1"
+
+FROM nvidia/cuda:${ver_cuda}-devel-ubuntu20.04 AS python_base_cuda
+LABEL maintainer="OpenVINO Training Extensions Development Team"
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+ARG action_runner_url
+ARG uid
+ARG gid
+
+# Setup proxies
+ENV http_proxy=$HTTP_PROXY
+ENV https_proxy=$HTTPS_PROXY
+ENV no_proxy=$NO_PROXY
+ENV DEBIAN_FRONTEND="noninteractive"
+
+# added ninja-build for using CppExtension in the torch
+# hadolint ignore=DL3008
+RUN apt-get update && apt-get install --no-install-recommends -y \
+        software-properties-common \
+        wget \
+        ffmpeg \
+        libpython3.10 \
+        curl \
+        nodejs \
+        unzip \
+        npm \
+        ruby-dev \
+        git \
+        cron \
+        ninja-build \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# set /tmp folder cleaning schedule at 7PM every day which is older than a day
+RUN echo "0 19 * * * find /tmp/* -mtime +1 -exec rm -rf {} \;" >> ./cron_clean_tmp.txt && \
+    crontab cron_clean_tmp.txt && \
+    # Create a non-root user with having given UID & GID
+    groupadd -r -g ${gid} validation && \
+    useradd -l -r -m validation -g ${gid} -u ${uid} && \
+    echo "${gid}:${uid}"
+
+USER validation
+
+WORKDIR /home/validation
+
+
+#########################################################
+## OTX Development Env
+#########################################################
+
+FROM python_base_cuda as otx_development_env
+
+RUN mkdir actions-runner
+WORKDIR /home/validation/actions-runner
+ARG action_runner_url
+# download actions-runner and extract it
+RUN curl -o actions-runner-linux-x64.tar.gz -L ${action_runner_url} && \
+    tar xzf ./actions-runner-linux-x64.tar.gz && \
+    rm actions-runner-linux-x64.tar.gz
+
+WORKDIR /home/validation

.ci/README.md

@@ -0,0 +1,102 @@
+# Guide to Setting up the CI using the Docker images
+
+1. Build the docker image using the `build.sh` in the .ci directory.
+   Make sure you are in the root directory of `training_extensions`.
+
+   ```bash
+   training_extensions$ .ci/build.sh --help
+      USAGE: .ci/build.sh <tag> [Options]
+      Positional args
+         <tag>               Tag name to be tagged to newly built image
+      Options
+         -p|--push           Push built image(s) to registry
+         -u|--url            url to get Github actions-runner package
+         -c|--cuda           Specify CUDA version
+         -r|--reg            Specify docker registry URL <default: local>
+         -h|--help           Print this message
+   ```
+
+   Below example builds an image using actions-runner v2.305.0 based on `NVIDIA CUDA 11.7.1` image and tag it as `2.305.0`.
+
+   ```bash
+   training_extensions$ .ci/build.sh 2.305.0 -u https://github.com/actions/runner/releases/download/v2.305.0/actions-runner-linux-x64-2.305.0.tar.gz -c 11.7.1
+   ```
+
+   > **_Note_**: While building an image, script will use your system's environment variables `http_proxy`, `https_proxy`, and `no_proxy`. If you need to use proxy to access external entity, please check those settings before using this script.
+
+   <!-- -->
+
+   > **_Note_**: The docker image name will be `<DOCKER_REG_ADDR>/ote/ci/cu<VER_CUDA>/runner:<TAG>`
+
+   <!-- -->
+
+   > **_Note_**: You can get the latest version of Github actions-runner package downloading URL from [here](https://github.com/actions/runner/releases).
+
+1. Create a container and start runner
+
+   ```bash
+   training_extensions$ .ci/start-runner.sh --help
+      USAGE: $0 <container-prefix> <github-token> [Options]
+      Positional args
+         <container-prefix>  Prefix to the ci container and actions-runner
+         <github-token>      Github token string
+      Options
+         -g|--gpu-ids        GPU ID or IDs (comma separated) for runner or 'all'
+         -c|--cuda           Specify CUDA version
+         -t|--tag            Specify TAG for the CI container
+         -l|--labels         Additional label string to set the actions-runner
+         -m|--mount          Dataset root path to be mounted to the started container (absolute path)
+         -r|--reg            Specify docker registry URL <default: local>
+         -d|--debug          Flag to start debugging CI container
+         -a|--attach-cache   Attach host path to the .cache on the container
+         -f|--fix-cpus       Specify the number of CPUs to set for the CI container
+         -h|--help           Print this message
+   ```
+
+   Below example starts a runner named as `<container-prefix>-0` with GPU ID 0 (GPU ID will be attached to both container and runner name)
+
+   ```bash
+   training_extensions$ .ci/start-runner.sh <container-prefix> <github-token> -g 0
+   ```
+
+   If there exist the container named as same, it will be stopped before starting a new container.
+
+   All configurations were configured and the runner is started successfully, you can see the messages below.
+
+   ```
+   v Settings Saved.
+
+   Successfully started actions runner
+   ```
+
+   > **_Note_**: About to getting tokens that used in the command above, you need to have proper permission to this repository. Please contact the repo admin to discuss futher.
+
+   <!-- -->
+
+   > **_Note_**: If there is no docker image for the OpenVINO™ Training Extensions CI on the host machine, this script will pull it from the registry and that will take some time to complete pull operation. It can lead an error on starting runner instance because of the expiring of the given token's validity. In this case, you should re-run the start-runner script again with refreshed token.
+
+1. Stop the runner and running container
+
+   ```bash
+   training_extensions$ .ci/stop-runner.sh
+     USAGE: .ci/stop-runner.sh <container-name> <github-token> [Options]
+     Options
+         -h|--help           Print this message
+   ```
+
+   Below example stops a runner named as `otx-ci-container`
+
+   ```bash
+   training_extensions$ .ci/stop-runner.sh otx-ci-container <github-token>
+   ```
+
+   > **_Note_**: If there is an action in progress on the actions-runner which you want to stop, this script will be resulted with an error. To perform force stopping the runner, you can stop the docker container using `docker stop` command on the host machine.
+
+1. Monitor the running runner
+   ```bash
+   training_extensions$ .ci/check-runner.sh --help
+   USAGE: .ci/check-runner.sh <container-name> [Options]
+   Options
+       -r|--runner         Check runner's log instead of Job one
+       -h|--help           Print this message
+   ```

.ci/build.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+VER_CUDA="11.7.1"
+ACTIONS_RUNNER_URL="https://github.com/actions/runner/releases/download/v2.305.0/actions-runner-linux-x64-2.305.0.tar.gz"
+DOCKER_REG_ADDR="local"
+POSITIONAL=()
+while [[ $# -gt 0 ]]; do
+  key="$1"
+
+  case $key in
+    -p|--push)
+      PUSH="yes"
+      shift # past argument
+      ;;
+    -u|--url)
+      ACTIONS_RUNNER_URL="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -c|--cuda)
+      VER_CUDA="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -r|--reg)
+      DOCKER_REG_ADDR="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -h|--help)
+      DEFAULT="yes"
+      break
+      shift # past argument
+      ;;
+    *)    # unknown option
+      POSITIONAL+=("$1") # save it in an array for later
+      shift # past argument
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL[@]}" # restore positional parameters
+
+if [ "$#" -lt 1 ] || [ "$DEFAULT" == "yes" ]; then
+cat << EndofMessage
+    USAGE: $0 <tag> [Options]
+    Positional args
+        <tag>               Tag name to be tagged to newly built image
+    Options
+        -p|--push           Push built image(s) to registry
+        -u|--url            url to get Github actions-runner package
+        -c|--cuda           Specify CUDA version
+        -r|--reg            Specify docker registry URL <default: local>
+        -h|--help           Print this message
+EndofMessage
+exit 0
+fi
+
+TAG=$1
+
+docker build -f .ci/Dockerfile \
+--build-arg HTTP_PROXY="${http_proxy:?}" \
+--build-arg HTTPS_PROXY="${https_proxy:?}" \
+--build-arg NO_PROXY="${no_proxy:?}" \
+--build-arg ver_cuda="$VER_CUDA" \
+--build-arg action_runner_url="$ACTIONS_RUNNER_URL" \
+--build-arg gid="$(id -g)" \
+--build-arg uid="$UID" \
+--tag "$DOCKER_REG_ADDR"/ote/ci/cu"$VER_CUDA"/runner:"$TAG" \
+--tag "$DOCKER_REG_ADDR"/ote/ci/cu"$VER_CUDA"/runner:latest .; RET=$?
+
+
+if [ $RET -ne 0 ]; then
+    echo "failed to build a 'ote/ci/cu$VER_CUDA/runner' image. $RET"
+    exit 1
+fi
+
+echo "Successfully built docker image."
+
+if [ "$PUSH" == "yes" ]; then
+    docker push "$DOCKER_REG_ADDR"/ote/ci/cu"$VER_CUDA"/runner:"$TAG"; RET=$?
+    if [ $RET -ne 0 ]; then
+        echo "failed to push a docker image to registry. $RET"
+        exit 1
+    fi
+    docker push "$DOCKER_REG_ADDR"/ote/ci/cu"$VER_CUDA"/runner:latest; RET=$?
+    if [ $RET -ne 0 ]; then
+        echo "failed to push a docker image to registry. $RET"
+        exit 1
+    fi
+else
+    echo "Newly built image was not pushed to the registry. use '-p|--push' option to push image."
+fi
+
+

.ci/check-runner.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+RUNNER="0"
+
+POSITIONAL=()
+while [[ $# -gt 0 ]]; do
+  key="$1"
+
+  case $key in
+    -r|--runner)
+      RUNNER="1"
+      shift # past argument
+      ;;
+    -h|--help)
+      DEFAULT="yes"
+      break
+      shift # past argument
+      ;;
+    *)    # unknown option
+      POSITIONAL+=("$1") # save it in an array for later
+      shift # past argument
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL[@]}" # restore positional parameters
+
+if [ "$#" -lt 1 ] || [ "$DEFAULT" == "yes" ]; then
+cat << EndofMessage
+    USAGE: $0 <container-name> [Options]
+    Options
+        -r|--runner         Check runner's log instead of Job one
+        -h|--help           Print this message
+EndofMessage
+exit 0
+fi
+
+CONTAINER_NAME=$1
+
+docker inspect "$CONTAINER_NAME"; RET=$?
+
+if [ $RET -eq 0 ]; then
+    if [ $RUNNER -eq "0" ]; then
+        docker exec -it "$CONTAINER_NAME" bash -c \
+            'logfile=$(find . -type f -name "Worker_*" | tail -1); tail -f $logfile'
+    else
+        docker exec -it "$CONTAINER_NAME" bash -c \
+            'logfile=$(find . -type f -name "Runner_*" | tail -1); tail -f $logfile'
+    fi
+else
+    echo "cannot find running container $CONTAINER_NAME"
+    exit 1
+fi

.ci/csv.tmpl

@@ -0,0 +1,10 @@
+{{ range . }}
+Trivy Vulnerability Scan Results ({{ .Target }})
+VulnerabilityID,Severity,CVSS Score,Title,Library,Vulnerable Version,Fixed Version,Information URL,Triage Information
+{{ range .Vulnerabilities }}{{ .VulnerabilityID }},{{ .Severity }},{{ range $key, $value := .CVSS }}{{ if (eq $key "nvd") }}{{ .V3Score }}{{ end }}{{ end }},"{{ .Title }}","{{ .PkgName }}","{{ .InstalledVersion }}","{{ .FixedVersion }}",{{ .PrimaryURL }}
+{{ end }}
+Trivy Dependency Scan Results ({{ .Target }})
+ID,Name,Version,Notes
+{{ range .Packages }}{{ .ID }},{{ .Name }},{{ .Version }}
+{{ end }}
+{{ end }}