From fb920231add9e457302943c6a346b381a6852845 Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy Date: Tue, 14 Sep 2021 13:17:33 +0200 Subject: [PATCH] add smartnic config scripts Signed-off-by: Periyasamy Palanisamy --- main_test.go | 6 ++++ scripts/create-kubernetes-cluster.sh | 6 ++++ scripts/smartnic/config-SmartNIC.sh | 49 +++++++++++++++++++++++++ scripts/smartnic/enable-SmartNIC.sh | 31 ++++++++++++++++ scripts/smartnic/setup-SmartNIC.sh | 53 ++++++++++++++++++++++++++++ 5 files changed, 145 insertions(+) create mode 100644 scripts/smartnic/config-SmartNIC.sh create mode 100644 scripts/smartnic/enable-SmartNIC.sh create mode 100644 scripts/smartnic/setup-SmartNIC.sh diff --git a/main_test.go b/main_test.go index 790e2266..1395726f 100644 --- a/main_test.go +++ b/main_test.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2021 Doc.ai and/or its affiliates. // +// Copyright (c) 2021 Nordix Foundation. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -35,6 +37,10 @@ func TestSRIOV(t *testing.T) { suite.Run(t, new(sriov.Suite)) } +func TestOVS(t *testing.T) { + suite.Run(t, new(ovs.Suite)) +} + func TestMultiForwarder(t *testing.T) { suite.Run(t, new(multiforwarder.Suite)) } diff --git a/scripts/create-kubernetes-cluster.sh b/scripts/create-kubernetes-cluster.sh index 5206862f..fe61561f 100755 --- a/scripts/create-kubernetes-cluster.sh +++ b/scripts/create-kubernetes-cluster.sh @@ -27,6 +27,12 @@ pids="" pids+=" $!" wait_pids "${pids}" "SR-IOV config failed" || exit 1 +# Setup SmartNIC +pids="" +/bin/bash scripts/smartnic/setup-SmartNIC.sh "${master_ip}" "${worker_ip}" "${SSH_OPTS}" & +pids+=" $!" +wait_pids "${pids}" "SmartNIC config failed" || exit 1 + # Create k8s scripts directory on nodes ssh ${SSH_OPTS} root@${master_ip} mkdir k8s ssh ${SSH_OPTS} root@${worker_ip} mkdir k8s diff --git a/scripts/smartnic/config-SmartNIC.sh b/scripts/smartnic/config-SmartNIC.sh new file mode 100644 index 00000000..6c0a1b97 --- /dev/null +++ b/scripts/smartnic/config-SmartNIC.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# shellcheck disable=SC2064,SC2129 + +CONFIG_DIRECTORY="/var/lib/networkservicemesh" +CONFIG_FILE="${CONFIG_DIRECTORY}/smartnic.config" + +function softlink_target() { + softlink="$1" + + raw_target="$(stat -c %N "${softlink}")" + test $? -eq 0 || return 1 + + target=$(echo "${raw_target}" | sed -E "s/(.*\/)(.*)'/\2/g") + test $? -eq 0 || return 2 + + echo "${target}" + return 0 +} + +function config_link() { + device="/sys/class/net/$1/device" + IFS=","; read -ra domains <<< "$2"; unset IFS + + pci_addr="$(softlink_target "${device}")" + test $? -eq 0 || return 1 + + echo " ${pci_addr}:" >> "${CONFIG_FILE}" + echo " pfKernelDriver: mlx5_core" >> "${CONFIG_FILE}" + echo " vfKernelDriver: mlx5_core" >> "${CONFIG_FILE}" + echo " capabilities:" >> "${CONFIG_FILE}" + echo " - 100G" >> "${CONFIG_FILE}" + echo " serviceDomains:" >> "${CONFIG_FILE}" + for domain in "${domains[@]}"; do + echo " - ${domain}" >> "${CONFIG_FILE}" + done + + return 0 +} + +mkdir -p "${CONFIG_DIRECTORY}" + +echo "---" > "${CONFIG_FILE}" +echo "physicalFunctions:" >> "${CONFIG_FILE}" + +for link_domains in "$@"; do + IFS="="; read -ra args <<< "${link_domains}"; unset IFS + config_link "${args[0]}" "${args[1]}" + test $? -eq 0 || exit 1 +done diff --git a/scripts/smartnic/enable-SmartNIC.sh b/scripts/smartnic/enable-SmartNIC.sh new file mode 100644 index 00000000..b659a981 --- /dev/null +++ b/scripts/smartnic/enable-SmartNIC.sh @@ -0,0 +1,31 @@ +#!/bin/bash -x +# shellcheck disable=SC2086 + +device="/sys/class/net/$1/device" + +# modprobe mlx5_core driver +MLX5_CORE_DRIVER_DIR="/sys/bus/pci/drivers/mlx5_core" +ls -l "${MLX5_CORE_DRIVER_DIR}" || modprobe mlx5_core || exit 1 + +# Don't forget to remove VFs for the link +trap "echo 0 >'${device}/sriov_numvfs'" err exit + +# Add 2 Smart VFs for the link +echo 2 > "${device}/sriov_numvfs" || exit 2 + +# Change PF to appropriate modes +echo legacy > "${device}/compat/devlink/vport_match_mode" || exit 2 +echo dmfs > "${device}/compat/devlink/steering_mode" || exit 2 +echo switchdev > "${device}/compat/devlink/mode" || exit 2 + +# Enable mlx5_core driver for the VFs +for i in `seq 0 2`; do pci_id=$(cat "${device}/virtfn$i/uevent" | grep "PCI_ID" | sed -E "s/PCI_ID=(.*):(.*)/\1 \2/g"); test $? -eq 0 || exit 3; echo "${pci_id}" > "${MLX5_CORE_DRIVER_DIR}/new_id" || exit 4; done + +# Waiting for the SmartVF devices to be up again +while [ `ip link | grep -c smartvf` != "2" ]; do sleep 1; done + +# Assign the representor MAC addresses to the VF MAC adresses +for i in `seq 0 2`; do mac=`ip l show smartvf0_$i | grep -o "link/ether [^ ]*" | cut -d' ' -f2`; echo "vf $i: $mac"; ip l set $1 vf $i mac $mac; done + +# Representor devices manually have to be set to "up" +for p in `ip l | grep -o "ens5f0[^:]*"`; do echo $p; ip link set $p up; done diff --git a/scripts/smartnic/setup-SmartNIC.sh b/scripts/smartnic/setup-SmartNIC.sh new file mode 100644 index 00000000..76b77d05 --- /dev/null +++ b/scripts/smartnic/setup-SmartNIC.sh @@ -0,0 +1,53 @@ +#!/bin/bash -x +# shellcheck disable=SC2086 + +master_ip="$1" +worker_ip="$2" +SSH_OPTS="$3" + +function wait_pids() { + pids="$1" + message="$2" + for pid in ${pids}; do + echo "waiting for PID ${pid}" + wait ${pid} + code=$? + if test $code -ne 0; then + echo "${message}: process exited with code $code, aborting..." + return 1 + fi + done + return 0 +} + +SMARTNIC_DIR=$(dirname "$0") + +# Create SmartNIC scripts directory on nodes +ssh ${SSH_OPTS} root@${master_ip} mkdir smartnic +ssh ${SSH_OPTS} root@${worker_ip} mkdir smartnic + +# TODO: change snic0 into packet's MLX5 interface name + +# Configure SmartNIC on the nodes +scp ${SSH_OPTS} ${SMARTNIC_DIR}/enable-SmartNIC.sh root@${master_ip}:smartnic/enable-SmartNIC.sh snic0 || exit 1 +scp ${SSH_OPTS} ${SMARTNIC_DIR}/enable-SmartNIC.sh root@${worker_ip}:smartnic/enable-SmartNIC.sh snic0 || exit 2 + +pids="" +ssh ${SSH_OPTS} root@${master_ip} ./smartnic/enable-SmartNIC.sh & +pids+=" $!" +ssh ${SSH_OPTS} root@${worker_ip} ./smartnic/enable-SmartNIC.sh & +pids+=" $!" +wait_pids "${pids}" "SmartNIC setup failed" || exit 3 + +sleep 5 + +# Create SmartNIC config +scp ${SSH_OPTS} ${SMARTNIC_DIR}/config-SmartNIC.sh root@${master_ip}:smartnic/config-SmartNIC.sh || exit 4 +scp ${SSH_OPTS} ${SMARTNIC_DIR}/config-SmartNIC.sh root@${worker_ip}:smartnic/config-SmartNIC.sh || exit 5 + +pids="" +ssh ${SSH_OPTS} root@${master_ip} ./smartnic/config-SmartNIC.sh snic0=worker.domain & +pids+=" $!" +ssh ${SSH_OPTS} root@${worker_ip} ./smartnic/config-SmartNIC.sh snic0=worker.domain & +pids+=" $!" +wait_pids "${pids}" "NSM SmartNIC config failed" || exit 6 \ No newline at end of file