add ovs forwarder specific config scripts

main_test.go

@@ -1,5 +1,7 @@
 // Copyright (c) 2020-2021 Doc.ai and/or its affiliates.
 //
+// Copyright (c) 2021 Nordix Foundation.
+//
 // SPDX-License-Identifier: Apache-2.0
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -35,6 +37,10 @@ func TestSRIOV(t *testing.T) {
 	suite.Run(t, new(sriov.Suite))
 }
 
+func TestOVS(t *testing.T) {
+	suite.Run(t, new(ovs.Suite))
+}
+
 func TestMultiForwarder(t *testing.T) {
 	suite.Run(t, new(multiforwarder.Suite))
 }

scripts/create-kubernetes-cluster.sh

@@ -27,6 +27,12 @@ pids=""
 pids+=" $!"
 wait_pids "${pids}" "SR-IOV config failed" || exit 1
 
+# Setup SmartNIC
+pids=""
+/bin/bash scripts/smartnic/setup-SmartNIC.sh "${master_ip}" "${worker_ip}" "${SSH_OPTS}" &
+pids+=" $!"
+wait_pids "${pids}" "SmartNIC config failed" || exit 1
+
 # Create k8s scripts directory on nodes
 ssh ${SSH_OPTS} root@${master_ip} mkdir k8s
 ssh ${SSH_OPTS} root@${worker_ip} mkdir k8s

scripts/smartnic/config-SmartNIC.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# shellcheck disable=SC2064,SC2129
+
+CONFIG_DIRECTORY="/var/lib/networkservicemesh"
+CONFIG_FILE="${CONFIG_DIRECTORY}/smartnic.config"
+
+function softlink_target() {
+  softlink="$1"
+
+  raw_target="$(stat -c %N "${softlink}")"
+  test $? -eq 0 || return 1
+
+  target=$(echo "${raw_target}" | sed -E "s/(.*\/)(.*)'/\2/g")
+  test $? -eq 0 || return 2
+
+  echo "${target}"
+  return 0
+}
+
+function config_link() {
+  device="/sys/class/net/$1/device"
+  IFS=","; read -ra domains <<< "$2"; unset IFS
+
+  pci_addr="$(softlink_target "${device}")"
+  test $? -eq 0 || return 1
+
+  echo "  ${pci_addr}:" >> "${CONFIG_FILE}"
+  echo "    pfKernelDriver: mlx5_core" >> "${CONFIG_FILE}"
+  echo "    vfKernelDriver: mlx5_core" >> "${CONFIG_FILE}"
+  echo "    capabilities:" >> "${CONFIG_FILE}"
+  echo "      - 100G" >> "${CONFIG_FILE}"
+  echo "    serviceDomains:" >> "${CONFIG_FILE}"
+  for domain in "${domains[@]}"; do
+    echo "      - ${domain}" >> "${CONFIG_FILE}"
+  done
+
+  return 0
+}
+
+mkdir -p "${CONFIG_DIRECTORY}"
+
+echo "---" > "${CONFIG_FILE}"
+echo "physicalFunctions:" >> "${CONFIG_FILE}"
+
+for link_domains in "$@"; do
+  IFS="="; read -ra args <<< "${link_domains}"; unset IFS
+  config_link "${args[0]}" "${args[1]}"
+  test $? -eq 0 || exit 1
+done

scripts/smartnic/enable-SmartNIC.sh

@@ -0,0 +1,31 @@
+#!/bin/bash -x
+# shellcheck disable=SC2086
+
+device="/sys/class/net/$1/device"
+
+# modprobe mlx5_core driver
+MLX5_CORE_DRIVER_DIR="/sys/bus/pci/drivers/mlx5_core"
+ls -l "${MLX5_CORE_DRIVER_DIR}" || modprobe mlx5_core || exit 1
+
+# Don't forget to remove VFs for the link
+trap "echo 0 >'${device}/sriov_numvfs'" err exit
+
+# Add 2 Smart VFs for the link
+echo 2 > "${device}/sriov_numvfs" || exit 2
+
+# Change PF to appropriate modes
+echo legacy > "${device}/compat/devlink/vport_match_mode" || exit 2
+echo dmfs > "${device}/compat/devlink/steering_mode" || exit 2
+echo switchdev > "${device}/compat/devlink/mode" || exit 2
+
+# Enable mlx5_core driver for the VFs
+for i in `seq 0 2`; do pci_id=$(cat "${device}/virtfn$i/uevent" | grep "PCI_ID" | sed -E "s/PCI_ID=(.*):(.*)/\1 \2/g"); test $? -eq 0 || exit 3; echo "${pci_id}" > "${MLX5_CORE_DRIVER_DIR}/new_id" || exit 4; done
+
+# Waiting for the SmartVF devices to be up again
+while [ `ip link | grep -c smartvf` != "2" ]; do sleep 1; done
+
+# Assign the representor MAC addresses to the VF MAC adresses
+for i in `seq 0 2`; do mac=`ip l show smartvf0_$i | grep -o "link/ether [^ ]*" | cut -d' ' -f2`; echo "vf $i: $mac"; ip l set $1 vf $i mac $mac; done
+
+# Representor devices manually have to be set to "up"
+for p in `ip l | grep -o "ens5f0[^:]*"`; do echo $p; ip link set $p up; done

scripts/smartnic/setup-SmartNIC.sh

@@ -0,0 +1,53 @@
+#!/bin/bash -x
+# shellcheck disable=SC2086
+
+master_ip="$1"
+worker_ip="$2"
+SSH_OPTS="$3"
+
+function wait_pids() {
+  pids="$1"
+  message="$2"
+  for pid in ${pids}; do
+    echo "waiting for PID ${pid}"
+    wait ${pid}
+    code=$?
+    if test $code -ne 0; then
+      echo "${message}: process exited with code $code, aborting..."
+      return 1
+    fi
+  done
+  return 0
+}
+
+SMARTNIC_DIR=$(dirname "$0")
+
+# Create SmartNIC scripts directory on nodes
+ssh ${SSH_OPTS} root@${master_ip} mkdir smartnic
+ssh ${SSH_OPTS} root@${worker_ip} mkdir smartnic
+
+# TODO: change snic0 into packet's MLX5 interface name
+
+# Configure SmartNIC on the nodes
+scp ${SSH_OPTS} ${SMARTNIC_DIR}/enable-SmartNIC.sh root@${master_ip}:smartnic/enable-SmartNIC.sh snic0 || exit 1
+scp ${SSH_OPTS} ${SMARTNIC_DIR}/enable-SmartNIC.sh root@${worker_ip}:smartnic/enable-SmartNIC.sh snic0 || exit 2
+
+pids=""
+ssh ${SSH_OPTS} root@${master_ip} ./smartnic/enable-SmartNIC.sh &
+pids+=" $!"
+ssh ${SSH_OPTS} root@${worker_ip} ./smartnic/enable-SmartNIC.sh &
+pids+=" $!"
+wait_pids "${pids}" "SmartNIC setup failed" || exit 3
+
+sleep 5
+
+# Create SmartNIC config
+scp ${SSH_OPTS} ${SMARTNIC_DIR}/config-SmartNIC.sh root@${master_ip}:smartnic/config-SmartNIC.sh || exit 4
+scp ${SSH_OPTS} ${SMARTNIC_DIR}/config-SmartNIC.sh root@${worker_ip}:smartnic/config-SmartNIC.sh || exit 5
+
+pids=""
+ssh ${SSH_OPTS} root@${master_ip} ./smartnic/config-SmartNIC.sh snic0=worker.domain &
+pids+=" $!"
+ssh ${SSH_OPTS} root@${worker_ip} ./smartnic/config-SmartNIC.sh snic0=worker.domain &
+pids+=" $!"
+wait_pids "${pids}" "NSM SmartNIC config failed" || exit 6