Skip to content

Commit

Permalink
Changes for Host VFs and num of APFs increased to 48, to
Browse files Browse the repository at this point in the history 
accomodate 16 Host VFs.
Pre-defined range of ACC APF interface indexes allocated
for various usages like(Host VFs, NF PRs, NFs(for ingress/egress)).
Added changes to use AccApfInfo type for mac-list and intf-name
list. Basic sanity test for Init->SetupAccApfs and GetDevices works as
expected.
Changes to run devmem commands to add 48 ACC APFs to forwarding group one.
By default only 16 ACC APFs are added to forwarding Group one.
Function->AddAccApfsToGroupOne added to ipuplugin's Run function.

Changes tested for init, getDevices, CreateBridgePort, DeleteBridgePort,
host-VF-to-host-VF(ping test). Basic sanity test for CreateNetworkFunction, and DeleteNetworkFunction works,
and verified resource allocation. But due to probably setup/config issue, issue with
testing for:->Ping from ext-client to D4 or NF(ingress) or test using NF.
  • Loading branch information
@sudhar-krishnakumar
sudhar-krishnakumar committed Dec 23, 2024
1 parent f01e94e commit a05e5fb
Show file tree
Hide file tree
Showing 7 changed files with 291 additions and 119 deletions.
128 changes: 67 additions & 61 deletions ipu-plugin/pkg/ipuplugin/bridgeport.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,65 +28,82 @@ import (
"google.golang.org/protobuf/types/known/emptypb"
)

const (
outerVlanId = 0 // hardcoded s-tag
)

/*
interfaces slice will be populated with PortIds on ACC(port representators->PRs on ACC, for Host VFs),
for example, if ACC interface name is enp0s1f0d4, PortId(vportId) will be 4.
Will also include port representators->PRs needed for Network Functions.
*/
var interfaces []uint
var intfMap map[uint]bool
var intfMapInit bool = false

// Note: 3 reserved(last digit of interface name, for example, enp0s1f0d8, is 8) in exlude list in deviceplugin.
var interfaces [3]string = [3]string{"6", "7", "8"}
var intfMap map[string]bool

func initMap() error {
if intfMapInit == false {
intfMap = make(map[string]bool)
var index uint
if !intfMapInit {
for index = HOST_VF_START_ID; index <= HOST_VF_END_ID; index = index + 1 {
interfaces = append(interfaces, index)
}
for index = NF_PR_START_ID; index <= NF_PR_END_ID; index = index + 1 {
interfaces = append(interfaces, index)
}
intfMap = make(map[uint]bool)
for _, intf := range interfaces {
intfMap[intf] = false
}
if len(interfaces) != len(intfMap) {
log.Errorf("initMap setup error\n")
return fmt.Errorf("initMap setup error\n")
log.Errorf("initMap setup error")
return fmt.Errorf("initMap setup error")
}
intfMapInit = true
}
return nil
}

// in-order(sorted by interface name->interfaces) allocation, based on available ACC interfaces(for Host VF)
func allocateAccInterface() (error, string) {
var intfName string = ""
log.Debugf("allocateAccInterface\n")
if intfMapInit == false {
func AllocateAccInterface(allocPr string) (uint, error) {
var intfId uint = 0
start, end := 0, 0

found := false
log.Debugf("AllocateAccInterface\n")
if !intfMapInit {
initMap()
}
for _, key := range interfaces {
log.Debugf("intfName->%v\n", key)
if allocPr == types.HostVfPr {
start = 0
end = HOST_VF_END_ID - HOST_VF_START_ID
} else {
start = HOST_VF_END_ID - HOST_VF_START_ID + 1
end = start + NF_PR_END_ID - NF_PR_START_ID
}
for i := start; i <= end; i++ {
key := interfaces[i]
value, present := intfMap[key]
if present == true && value == false {
if present && !value {
log.Debugf("Found avail Intf->%v: \n", key)
intfMap[key] = true
intfName = key
intfId = key
found = true
break
}
}
if intfName != "" {
return nil, intfName
if found {
return intfId, nil
}
log.Errorf("Interface not available\n")
return fmt.Errorf("Interface not available\n"), intfName
log.Errorf("AllocateAccInterface: Interface not available")
return intfId, fmt.Errorf("AllocateAccInterface: interface not available")
}

func freeAccInterface(intfName string) error {
log.Debugf("freeAccInterface\n")
value, present := intfMap[intfName]
if present == true && value == true {
log.Debugf("Found allocated Intf->%v: \n", intfName)
intfMap[intfName] = false
func FreeAccInterface(intfId uint) error {
log.Debugf("FreeAccInterface\n")
value, present := intfMap[intfId]
if present && value {
log.Debugf("Found allocated Intf->%v: \n", intfId)
intfMap[intfId] = false
return nil
}
log.Errorf("Interface->%s not found in freeAccInterface\n", intfName)
return fmt.Errorf("Interface->%s not found in freeAccInterface\n", intfName)
log.Errorf("Interface->%v not found in FreeAccInterface", intfId)
return fmt.Errorf("interface->%v not found in FreeAccInterface", intfId)
}

// CreateBridgePort executes the creation of the port
Expand Down Expand Up @@ -127,32 +144,26 @@ func (s *server) CreateBridgePort(_ context.Context, in *pb.CreateBridgePortRequ

CheckAndAddPeerToPeerP4Rules(s.p4rtbin)

err, intfName := allocateAccInterface()
intfId, err := AllocateAccInterface(types.HostVfPr)
if err != nil {
return nil, fmt.Errorf("error from allocateAccInterface->%v", err)
return nil, fmt.Errorf("error from AllocateAccInterface->%v", err)
}

intIndex, err := strconv.Atoi(string(intfName))
if err != nil {
log.Errorf("error->%v converting, intfName->%v", err, intfName)
return nil, fmt.Errorf("error->%v converting, intfName->%v", err, intfName)
} else {
log.Infof("intIndex->%v, fullIntfName->%v", intIndex, AccIntfNames[intIndex])
}
log.Infof("intfId->%v, fullIntfName->%v", intfId, AccApfInfo[intfId].Name)

if err := s.bridgeCtlr.AddPort(AccIntfNames[intIndex]); err != nil {
log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
freeAccInterface(intfName)
return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
if err := s.bridgeCtlr.AddPort(AccApfInfo[intfId].Name); err != nil {
log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
FreeAccInterface(intfId)
return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
}

// Add FXP rules
log.Infof("AddHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfMacList[intIndex])
p4rtclient.AddHostVfP4Rules(s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfMacList[intIndex])
log.Infof("AddHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfInfo[intfId].Mac)
p4rtclient.AddHostVfP4Rules(s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfInfo[intfId].Mac)

resp := proto.Clone(in.BridgePort).(*pb.BridgePort)
resp.Status = &pb.BridgePortStatus{OperStatus: pb.BPOperStatus_BP_OPER_STATUS_UP}
pbBridgePortInfo := &types.BridgePortInfo{PbBrPort: resp, PortInterface: intfName}
pbBridgePortInfo := &types.BridgePortInfo{PbBrPort: resp, PortId: intfId}
s.Ports[in.BridgePort.Name] = pbBridgePortInfo
return resp, nil
}
Expand Down Expand Up @@ -180,22 +191,17 @@ func (s *server) DeleteBridgePort(_ context.Context, in *pb.DeleteBridgePortRequ
}
portInfo = brPortInfo.PbBrPort

intIndex, err := strconv.Atoi(string(brPortInfo.PortInterface))
if err != nil {
log.Errorf("error->%v converting, intfName->%v", err, brPortInfo.PortInterface)
return nil, fmt.Errorf("error->%v converting, intfName->%v", err, brPortInfo.PortInterface)
} else {
log.Infof("intIndex->%v, fullIntfName->%v", intIndex, AccIntfNames[intIndex])
}
intfId := brPortInfo.PortId
log.Infof("intfIndex->%v, fullIntfName->%v", intfId, AccApfInfo[intfId].Name)

if err := s.bridgeCtlr.DeletePort(AccIntfNames[intIndex]); err != nil {
log.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
return nil, fmt.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
if err := s.bridgeCtlr.DeletePort(AccApfInfo[intfId].Name); err != nil {
log.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
return nil, fmt.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
}
freeAccInterface(brPortInfo.PortInterface)
FreeAccInterface(brPortInfo.PortId)
// Delete FXP rules
log.Infof("DeleteHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, portInfo.Spec.MacAddress, AccApfMacList[intIndex])
p4rtclient.DeleteHostVfP4Rules(s.p4rtbin, portInfo.Spec.MacAddress, AccApfMacList[intIndex])
log.Infof("DeleteHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, portInfo.Spec.MacAddress, AccApfInfo[intfId].Mac)
p4rtclient.DeleteHostVfP4Rules(s.p4rtbin, portInfo.Spec.MacAddress, AccApfInfo[intfId].Mac)

delete(s.Ports, in.Name)
return &emptypb.Empty{}, nil
Expand Down
15 changes: 10 additions & 5 deletions ipu-plugin/pkg/ipuplugin/deviceplugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,6 @@ type DevicePluginService struct {
}

var (
//TODO: Use (GetFilteredPfs), to find interface names to be excluded.
//excluding d3(host-acc), reserving D4-D5(QSPF ports) D6-D8(for max 3 host VFs), D9-D10(for single NF)
exclude = []string{"enp0s1f0", "enp0s1f0d1", "enp0s1f0d2", "enp0s1f0d3", "enp0s1f0d4", "enp0s1f0d5", "enp0s1f0d6",
"enp0s1f0d7", "enp0s1f0d8", "enp0s1f0d9", "enp0s1f0d10"}
sysClassNet = "/sys/class/net"
sysBusPciDevices = "/sys/bus/pci/devices"
deviceCode = "0x1452"
Expand Down Expand Up @@ -235,6 +231,15 @@ func (s *DevicePluginService) SetNumVfs(ctx context.Context, vfCountReq *pb.VfCo

func discoverHostDevices(mode string) (map[string]*pb.Device, error) {

//Note: It is expected that ACC-Init happens prior to GetDevices,
//this check is meant to catch any anomalies.
if mode == types.IpuMode {
if len(AccApfsAvailForCNI) == 0 {
log.Errorf("discoverHostDevices: Error, AccApfsAvailForCNI not setup")
return make(map[string]*pb.Device), fmt.Errorf("discoverHostDevices: Error, AccApfsAvailForCNI not setup")
}
}

devices := make(map[string]*pb.Device)

files, err := os.ReadDir(sysClassNet)
Expand All @@ -253,7 +258,7 @@ func discoverHostDevices(mode string) (map[string]*pb.Device, error) {
device_code := strings.TrimSpace(string(deviceCodeByte))
if mode == types.IpuMode {
if device_code == deviceCode {
if !slices.Contains(exclude, file.Name()) {
if slices.Contains(AccApfsAvailForCNI, file.Name()) {
devices[file.Name()] = &pb.Device{ID: file.Name(), Health: pluginapi.Healthy}
}
}
Expand Down
63 changes: 61 additions & 2 deletions ipu-plugin/pkg/ipuplugin/ipuplugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"os"
"os/signal"
"path/filepath"
"strconv"
"syscall"
"time"

Expand Down Expand Up @@ -124,6 +125,59 @@ func waitForInfraP4d() (string, error) {
return "", nil
}

func AddAccApfsToGroupOne() error {
vsiList, err := utils.GetAvailableAccVsiList()
if err != nil {
log.Errorf("AddAccApfsToGroupOne: unable to reach the IMC %v", err)
return fmt.Errorf("AddAccApfsToGroupOne: unable to reach the IMC %v", err)
}
if len(vsiList) == 0 {
log.Errorf("no APFs initialized on ACC")
return fmt.Errorf("no APFs initialized on ACC")
}
log.Infof("AddAccApfsToGroupOne, vsiList->%v", vsiList)
/* Steps from script(for reference)
VSI_GROUP_INIT=$(printf "0x%x" $((0x8000050000000000 + IDPF_VPORT_VSI_HEX)))
VSI_GROUP_WRITE=$(printf "0x%x" $((0xA000050000000000 + IDPF_VPORT_VSI_HEX)))
devmem 0x20292002a0 64 ${VSI_GROUP_INIT}
devmem 0x2029200388 64 0x1
devmem 0x20292002a0 64 ${VSI_GROUP_WRITE}
*/
for i := 0; i < len(vsiList); i++ {
log.Infof("Add to VSI Group 1, vsi->%v", vsiList[i])
hexStr := vsiList[i]
// skip "0x" prefix
hexStr = hexStr[2:]

// Convert to hex value
hexVal, err := strconv.ParseInt(hexStr, 16, 64)
if err != nil {
log.Errorf("error decoding hex: %v", err)
return fmt.Errorf("error decoding hex: %v", err)
}

vsiGroupInit := 0x8000050000000000 + uint(hexVal)

Check failure

Code scanning / CodeQL

Incorrect conversion between integer types High

Incorrect conversion of a signed 64-bit integer from
strconv.ParseInt
Loading
to a lower bit size type uint without an upper bound check.
vsiGroupWrite := 0xA000050000000000 + uint(hexVal)

Check failure

Code scanning / CodeQL

Incorrect conversion between integer types High

Incorrect conversion of a signed 64-bit integer from
strconv.ParseInt
Loading
to a lower bit size type uint without an upper bound check.

vsiGroupInitString := fmt.Sprintf("0x%X", vsiGroupInit)
vsiGroupWriteString := fmt.Sprintf("0x%X", vsiGroupWrite)

devMemCmd1 := "devmem 0x20292002a0 64 " + vsiGroupInitString
devMemCmd2 := "devmem 0x2029200388 64 0x1"
devMemCmd3 := "devmem 0x20292002a0 64 " + vsiGroupWriteString

devMemCmd := devMemCmd1 + "; " + devMemCmd2 + "; " + devMemCmd3 + "; "
log.Infof("devMemCmd->%v", devMemCmd)

_, err = utils.ExecuteScript(fmt.Sprintf(`ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 [email protected] "%s"`, devMemCmd))
if err != nil {
log.Errorf("err exec devMemCmd->%v", err)
return fmt.Errorf("err exec devMemCmd->%v", err)
}
}
return nil
}

func (s *server) Run() error {
var err error
signalChannel := make(chan os.Signal, 2)
Expand All @@ -139,6 +193,11 @@ func (s *server) Run() error {
if _, err := waitForInfraP4d(); err != nil {
return err
}
if err := AddAccApfsToGroupOne(); err != nil {
log.Fatalf("error from->AddAccApfsToGroupOne: %v", err)
return fmt.Errorf("error from->AddAccApfsToGroupOne: %v", err)
}

// Create bridge if it doesn't exist
if err := s.bridgeCtlr.EnsureBridgeExists(); err != nil {
log.Fatalf("error while checking host bridge existence: %v", err)
Expand Down Expand Up @@ -174,8 +233,8 @@ func (s *server) Stop() {
s.bridgeCtlr.DeleteBridges()
}

log.Infof("DeletePhyPortRules, path->%s, 1->%v, 2->%v", s.p4rtbin, AccApfMacList[PHY_PORT0_INTF_INDEX], AccApfMacList[PHY_PORT1_INTF_INDEX])
p4rtclient.DeletePhyPortRules(s.p4rtbin, AccApfMacList[PHY_PORT0_INTF_INDEX], AccApfMacList[PHY_PORT1_INTF_INDEX])
log.Infof("DeletePhyPortRules, path->%s, 1->%v, 2->%v", s.p4rtbin, AccApfInfo[PHY_PORT0_INTF_INDEX].Mac, AccApfInfo[PHY_PORT1_INTF_INDEX].Mac)
p4rtclient.DeletePhyPortRules(s.p4rtbin, AccApfInfo[PHY_PORT0_INTF_INDEX].Mac, AccApfInfo[PHY_PORT1_INTF_INDEX].Mac)

vfMacList, err := utils.GetVfMacList()
if err != nil {
Expand Down
Loading

0 comments on commit a05e5fb

Please sign in to comment.