From 28e7d51b5f0ebcabdb6b17aeae200cc01f9f8119 Mon Sep 17 00:00:00 2001 From: Laszlo Kiraly Date: Thu, 22 Feb 2024 17:32:01 +0100 Subject: [PATCH] run ovs forwarder with pure SRIOV config instead smartnic and printout devices Signed-off-by: Laszlo Kiraly --- .github/workflows/ci.yaml | 7 ++-- scripts/setup-cluster.sh | 4 +-- scripts/smartnic/config-SmartNIC.sh | 51 ----------------------------- scripts/smartnic/enable-SmartNIC.sh | 30 ++--------------- scripts/smartnic/setup-SmartNIC.sh | 13 ++------ tests_calico-vpp/main_test.go | 5 +-- tests_default/main_test.go | 19 +++++++---- 7 files changed, 22 insertions(+), 107 deletions(-) delete mode 100755 scripts/smartnic/config-SmartNIC.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a04da15..398147c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -126,11 +126,8 @@ jobs: - name: Integration tests run: | export KUBECONFIG=$HOME/.kube/config_packet - if [[ "${{ matrix.CNI }}" == "calico-vpp" ]]; then - go test -count 1 -timeout 1h50m -race -v ./tests_${{ matrix.CNI }} -run TestRunOVS -parallel 4 || true - else - go test -count 1 -timeout 1h50m -race -v ./tests_${{ matrix.CNI }} -run TestRunOVS -parallel 4 - fi + go test -count 1 -timeout 50m -race -v ./tests_${{ matrix.CNI }} -run TestRunOVS + false env: ARTIFACTS_DIR: logs working-directory: ${{ github.repository }} diff --git a/scripts/setup-cluster.sh b/scripts/setup-cluster.sh index fa2a4d8..88b780a 100755 --- a/scripts/setup-cluster.sh +++ b/scripts/setup-cluster.sh @@ -101,11 +101,9 @@ done ## Setup ovs /bin/bash scripts/ovs/setup-ovs.sh "${master_ip}" "${worker_ip}" "${SSH_OPTS}" || exit 13 -## Setup SmartNIC +## Setup ovs if [[ "$CNI" == "calico-vpp" ]]; then # calico /bin/bash scripts/smartnic/setup-SmartNIC.sh "${master_ip}" "${worker_ip}" "${SSH_OPTS}" "${CALICO_INTERFACE}" || exit 14 -else - /bin/bash scripts/smartnic/setup-SmartNIC.sh "${master_ip}" "${worker_ip}" "${SSH_OPTS}" "${SRIOV_INTERFACE}" || exit 14 fi ## Remove master label from the control-plane node to be able to use it as worker node diff --git a/scripts/smartnic/config-SmartNIC.sh b/scripts/smartnic/config-SmartNIC.sh deleted file mode 100755 index e85c007..0000000 --- a/scripts/smartnic/config-SmartNIC.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -x -# shellcheck disable=SC2064,SC2129 - -set -e - -CONFIG_DIRECTORY="/var/lib/networkservicemesh" -CONFIG_FILE="${CONFIG_DIRECTORY}/smartnic.config" - -function softlink_target() { - softlink="$1" - - raw_target="$(stat -c %N "${softlink}")" - test $? -eq 0 || return 1 - - target=$(echo "${raw_target}" | sed -E "s/(.*\/)(.*)'/\2/g") - test $? -eq 0 || return 2 - - echo "${target}" - return 0 -} - -function config_link() { - device="/sys/class/net/$1/device" - IFS=","; read -ra domains <<< "$2"; unset IFS - - pci_addr="$(softlink_target "${device}")" - test $? -eq 0 || return 1 - - echo " ${pci_addr}:" >> "${CONFIG_FILE}" - echo " pfKernelDriver: mlx5_core" >> "${CONFIG_FILE}" - echo " vfKernelDriver: mlx5_core" >> "${CONFIG_FILE}" - echo " capabilities:" >> "${CONFIG_FILE}" - echo " - 100G" >> "${CONFIG_FILE}" - echo " serviceDomains:" >> "${CONFIG_FILE}" - for domain in "${domains[@]}"; do - echo " - ${domain}" >> "${CONFIG_FILE}" - done - - return 0 -} - -mkdir -p "${CONFIG_DIRECTORY}" - -echo "---" > "${CONFIG_FILE}" -echo "physicalFunctions:" >> "${CONFIG_FILE}" - -for link_domains in "$@"; do - IFS="="; read -ra args <<< "${link_domains}"; unset IFS - config_link "${args[0]}" "${args[1]}" - test $? -eq 0 || exit 1 -done \ No newline at end of file diff --git a/scripts/smartnic/enable-SmartNIC.sh b/scripts/smartnic/enable-SmartNIC.sh index 2c63af8..7fb1a17 100755 --- a/scripts/smartnic/enable-SmartNIC.sh +++ b/scripts/smartnic/enable-SmartNIC.sh @@ -4,31 +4,5 @@ set -e ls /sys/class/net -device="/sys/class/net/$1/device" - -# modprobe mlx5_core driver -MLX5_CORE_DRIVER_DIR="/sys/bus/pci/drivers/mlx5_core" -ls -l "${MLX5_CORE_DRIVER_DIR}" || modprobe mlx5_core || exit 1 - -# Don't forget to remove VFs for the link -trap "echo 0 >'${device}/sriov_numvfs'" err exit - -# Add 2 Smart VFs for the link -echo 2 > "${device}/sriov_numvfs" || exit 2 - -# Change PF to appropriate modes -echo legacy > "${device}/compat/devlink/vport_match_mode" || exit 2 -echo dmfs > "${device}/compat/devlink/steering_mode" || exit 2 -echo switchdev > "${device}/compat/devlink/mode" || exit 2 - -# Enable mlx5_core driver for the VFs -for i in $(seq 0 2); do pci_id=$(grep "PCI_ID" "${device}/virtfn$i/uevent" | sed -E "s/PCI_ID=(.*):(.*)/\1 \2/g"); test $? -eq 0 || exit 3; echo "${pci_id}" > "${MLX5_CORE_DRIVER_DIR}/new_id" || exit 4; done - -# Waiting for the SmartVF devices to be up again -while [ "$(ip link | grep -c smartvf)" != "2" ]; do sleep 1; done - -# Assign the representor MAC addresses to the VF MAC adresses -for i in $(seq 0 2); do mac=$(ip l show smartvf0_$i | grep -o "link/ether [^ ]*" | cut -d' ' -f2); echo "vf $i: $mac"; ip l set $1 vf $i mac $mac; done - -# Representor devices manually have to be set to "up" -for p in $(ip l | grep -o "ens5f0[^:]*"); do echo $p; ip link set $p up; done \ No newline at end of file +ip -details link show +lspci | egrep -i 'network|ethernet' \ No newline at end of file diff --git a/scripts/smartnic/setup-SmartNIC.sh b/scripts/smartnic/setup-SmartNIC.sh index c58e592..3dcb7e2 100755 --- a/scripts/smartnic/setup-SmartNIC.sh +++ b/scripts/smartnic/setup-SmartNIC.sh @@ -43,13 +43,6 @@ wait_pids "${pids}" "SmartNIC setup failed" || exit 3 sleep 5 -# Create SmartNIC config -scp ${SSH_OPTS} ${SMARTNIC_DIR}/config-SmartNIC.sh root@${master_ip}:smartnic/config-SmartNIC.sh || exit 4 -scp ${SSH_OPTS} ${SMARTNIC_DIR}/config-SmartNIC.sh root@${worker_ip}:smartnic/config-SmartNIC.sh || exit 5 - -pids="" -ssh ${SSH_OPTS} root@${master_ip} ./smartnic/config-SmartNIC.sh ${snic}=worker.domain & -pids+=" $!" -ssh ${SSH_OPTS} root@${worker_ip} ./smartnic/config-SmartNIC.sh ${snic}=worker.domain & -pids+=" $!" -wait_pids "${pids}" "NSM SmartNIC config failed" || exit 6 \ No newline at end of file +# Copy SRIOV config as SmartNIC config +ssh ${SSH_OPTS} root@${master_ip} cp /var/lib/networkservicemesh/sriov.config /var/lib/networkservicemesh/smartnic.config +ssh ${SSH_OPTS} root@${worker_ip} cp /var/lib/networkservicemesh/sriov.config /var/lib/networkservicemesh/smartnic.config \ No newline at end of file diff --git a/tests_calico-vpp/main_test.go b/tests_calico-vpp/main_test.go index 164269a..6e1f9ee 100644 --- a/tests_calico-vpp/main_test.go +++ b/tests_calico-vpp/main_test.go @@ -76,10 +76,7 @@ type ovsSuite struct { } func (s *ovsSuite) BeforeTest(suiteName, testName string) { - switch testName { - case - "TestSmartVF2SmartVF", - "TestKernel2KernelVLAN": + if testName == "TestKernel2KernelVLAN" { s.T().Skip() } } diff --git a/tests_default/main_test.go b/tests_default/main_test.go index 3280669..328332c 100644 --- a/tests_default/main_test.go +++ b/tests_default/main_test.go @@ -21,10 +21,8 @@ package main_test import ( - "flag" "testing" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/networkservicemesh/integration-tests/extensions/parallel" @@ -61,9 +59,18 @@ func TestFeatureSuite(t *testing.T) { parallel.Run(t, new(features.Suite), "TestVl3_basic", "TestVl3_dns", "TestScale_from_zero", "TestVl3_scale_from_zero", "TestSelect_forwarder") } +type ovsSuite struct { + ovs.Suite +} + +func (s *ovsSuite) BeforeTest(suiteName, testName string) { + switch testName { + case + "TestSmartVF2SmartVF", + "TestKernel2KernelVLAN": + s.T().Skip() + } +} func TestRunOVS(t *testing.T) { - f := flag.Lookup("testify.m") - require.NoError(t, flag.Set("testify.m", "TestKernel2Kernel")) - defer func() { _ = flag.Set("testify.m", f.Value.String()) }() - suite.Run(t, new(ovs.Suite)) + suite.Run(t, new(ovsSuite)) }