From a05e5fb3460e3e5993ae48eca8d5266f3e303b60 Mon Sep 17 00:00:00 2001 From: Sudhar Krishnakumar Date: Thu, 12 Dec 2024 14:49:50 -0500 Subject: [PATCH] Changes for Host VFs and num of APFs increased to 48, to accomodate 16 Host VFs. Pre-defined range of ACC APF interface indexes allocated for various usages like(Host VFs, NF PRs, NFs(for ingress/egress)). Added changes to use AccApfInfo type for mac-list and intf-name list. Basic sanity test for Init->SetupAccApfs and GetDevices works as expected. Changes to run devmem commands to add 48 ACC APFs to forwarding group one. By default only 16 ACC APFs are added to forwarding Group one. Function->AddAccApfsToGroupOne added to ipuplugin's Run function. Changes tested for init, getDevices, CreateBridgePort, DeleteBridgePort, host-VF-to-host-VF(ping test). Basic sanity test for CreateNetworkFunction, and DeleteNetworkFunction works, and verified resource allocation. But due to probably setup/config issue, issue with testing for:->Ping from ext-client to D4 or NF(ingress) or test using NF. --- ipu-plugin/pkg/ipuplugin/bridgeport.go | 128 +++++++++--------- ipu-plugin/pkg/ipuplugin/deviceplugin.go | 15 +- ipu-plugin/pkg/ipuplugin/ipuplugin.go | 63 ++++++++- ipu-plugin/pkg/ipuplugin/lifecycleservice.go | 82 ++++++----- .../pkg/ipuplugin/networkfunctionservice.go | 103 +++++++++++--- ipu-plugin/pkg/types/types.go | 6 +- ipu-plugin/pkg/utils/utils.go | 13 ++ 7 files changed, 291 insertions(+), 119 deletions(-) diff --git a/ipu-plugin/pkg/ipuplugin/bridgeport.go b/ipu-plugin/pkg/ipuplugin/bridgeport.go index a8ab89cf..aa343e24 100644 --- a/ipu-plugin/pkg/ipuplugin/bridgeport.go +++ b/ipu-plugin/pkg/ipuplugin/bridgeport.go @@ -28,25 +28,31 @@ import ( "google.golang.org/protobuf/types/known/emptypb" ) -const ( - outerVlanId = 0 // hardcoded s-tag -) - +/* +interfaces slice will be populated with PortIds on ACC(port representators->PRs on ACC, for Host VFs), +for example, if ACC interface name is enp0s1f0d4, PortId(vportId) will be 4. +Will also include port representators->PRs needed for Network Functions. +*/ +var interfaces []uint +var intfMap map[uint]bool var intfMapInit bool = false -// Note: 3 reserved(last digit of interface name, for example, enp0s1f0d8, is 8) in exlude list in deviceplugin. -var interfaces [3]string = [3]string{"6", "7", "8"} -var intfMap map[string]bool - func initMap() error { - if intfMapInit == false { - intfMap = make(map[string]bool) + var index uint + if !intfMapInit { + for index = HOST_VF_START_ID; index <= HOST_VF_END_ID; index = index + 1 { + interfaces = append(interfaces, index) + } + for index = NF_PR_START_ID; index <= NF_PR_END_ID; index = index + 1 { + interfaces = append(interfaces, index) + } + intfMap = make(map[uint]bool) for _, intf := range interfaces { intfMap[intf] = false } if len(interfaces) != len(intfMap) { - log.Errorf("initMap setup error\n") - return fmt.Errorf("initMap setup error\n") + log.Errorf("initMap setup error") + return fmt.Errorf("initMap setup error") } intfMapInit = true } @@ -54,39 +60,50 @@ func initMap() error { } // in-order(sorted by interface name->interfaces) allocation, based on available ACC interfaces(for Host VF) -func allocateAccInterface() (error, string) { - var intfName string = "" - log.Debugf("allocateAccInterface\n") - if intfMapInit == false { +func AllocateAccInterface(allocPr string) (uint, error) { + var intfId uint = 0 + start, end := 0, 0 + + found := false + log.Debugf("AllocateAccInterface\n") + if !intfMapInit { initMap() } - for _, key := range interfaces { - log.Debugf("intfName->%v\n", key) + if allocPr == types.HostVfPr { + start = 0 + end = HOST_VF_END_ID - HOST_VF_START_ID + } else { + start = HOST_VF_END_ID - HOST_VF_START_ID + 1 + end = start + NF_PR_END_ID - NF_PR_START_ID + } + for i := start; i <= end; i++ { + key := interfaces[i] value, present := intfMap[key] - if present == true && value == false { + if present && !value { log.Debugf("Found avail Intf->%v: \n", key) intfMap[key] = true - intfName = key + intfId = key + found = true break } } - if intfName != "" { - return nil, intfName + if found { + return intfId, nil } - log.Errorf("Interface not available\n") - return fmt.Errorf("Interface not available\n"), intfName + log.Errorf("AllocateAccInterface: Interface not available") + return intfId, fmt.Errorf("AllocateAccInterface: interface not available") } -func freeAccInterface(intfName string) error { - log.Debugf("freeAccInterface\n") - value, present := intfMap[intfName] - if present == true && value == true { - log.Debugf("Found allocated Intf->%v: \n", intfName) - intfMap[intfName] = false +func FreeAccInterface(intfId uint) error { + log.Debugf("FreeAccInterface\n") + value, present := intfMap[intfId] + if present && value { + log.Debugf("Found allocated Intf->%v: \n", intfId) + intfMap[intfId] = false return nil } - log.Errorf("Interface->%s not found in freeAccInterface\n", intfName) - return fmt.Errorf("Interface->%s not found in freeAccInterface\n", intfName) + log.Errorf("Interface->%v not found in FreeAccInterface", intfId) + return fmt.Errorf("interface->%v not found in FreeAccInterface", intfId) } // CreateBridgePort executes the creation of the port @@ -127,32 +144,26 @@ func (s *server) CreateBridgePort(_ context.Context, in *pb.CreateBridgePortRequ CheckAndAddPeerToPeerP4Rules(s.p4rtbin) - err, intfName := allocateAccInterface() + intfId, err := AllocateAccInterface(types.HostVfPr) if err != nil { - return nil, fmt.Errorf("error from allocateAccInterface->%v", err) + return nil, fmt.Errorf("error from AllocateAccInterface->%v", err) } - intIndex, err := strconv.Atoi(string(intfName)) - if err != nil { - log.Errorf("error->%v converting, intfName->%v", err, intfName) - return nil, fmt.Errorf("error->%v converting, intfName->%v", err, intfName) - } else { - log.Infof("intIndex->%v, fullIntfName->%v", intIndex, AccIntfNames[intIndex]) - } + log.Infof("intfId->%v, fullIntfName->%v", intfId, AccApfInfo[intfId].Name) - if err := s.bridgeCtlr.AddPort(AccIntfNames[intIndex]); err != nil { - log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[intIndex]) - freeAccInterface(intfName) - return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[intIndex]) + if err := s.bridgeCtlr.AddPort(AccApfInfo[intfId].Name); err != nil { + log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name) + FreeAccInterface(intfId) + return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name) } // Add FXP rules - log.Infof("AddHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfMacList[intIndex]) - p4rtclient.AddHostVfP4Rules(s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfMacList[intIndex]) + log.Infof("AddHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfInfo[intfId].Mac) + p4rtclient.AddHostVfP4Rules(s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfInfo[intfId].Mac) resp := proto.Clone(in.BridgePort).(*pb.BridgePort) resp.Status = &pb.BridgePortStatus{OperStatus: pb.BPOperStatus_BP_OPER_STATUS_UP} - pbBridgePortInfo := &types.BridgePortInfo{PbBrPort: resp, PortInterface: intfName} + pbBridgePortInfo := &types.BridgePortInfo{PbBrPort: resp, PortId: intfId} s.Ports[in.BridgePort.Name] = pbBridgePortInfo return resp, nil } @@ -180,22 +191,17 @@ func (s *server) DeleteBridgePort(_ context.Context, in *pb.DeleteBridgePortRequ } portInfo = brPortInfo.PbBrPort - intIndex, err := strconv.Atoi(string(brPortInfo.PortInterface)) - if err != nil { - log.Errorf("error->%v converting, intfName->%v", err, brPortInfo.PortInterface) - return nil, fmt.Errorf("error->%v converting, intfName->%v", err, brPortInfo.PortInterface) - } else { - log.Infof("intIndex->%v, fullIntfName->%v", intIndex, AccIntfNames[intIndex]) - } + intfId := brPortInfo.PortId + log.Infof("intfIndex->%v, fullIntfName->%v", intfId, AccApfInfo[intfId].Name) - if err := s.bridgeCtlr.DeletePort(AccIntfNames[intIndex]); err != nil { - log.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccIntfNames[intIndex]) - return nil, fmt.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccIntfNames[intIndex]) + if err := s.bridgeCtlr.DeletePort(AccApfInfo[intfId].Name); err != nil { + log.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name) + return nil, fmt.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name) } - freeAccInterface(brPortInfo.PortInterface) + FreeAccInterface(brPortInfo.PortId) // Delete FXP rules - log.Infof("DeleteHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, portInfo.Spec.MacAddress, AccApfMacList[intIndex]) - p4rtclient.DeleteHostVfP4Rules(s.p4rtbin, portInfo.Spec.MacAddress, AccApfMacList[intIndex]) + log.Infof("DeleteHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, portInfo.Spec.MacAddress, AccApfInfo[intfId].Mac) + p4rtclient.DeleteHostVfP4Rules(s.p4rtbin, portInfo.Spec.MacAddress, AccApfInfo[intfId].Mac) delete(s.Ports, in.Name) return &emptypb.Empty{}, nil diff --git a/ipu-plugin/pkg/ipuplugin/deviceplugin.go b/ipu-plugin/pkg/ipuplugin/deviceplugin.go index 2ebd2203..2ef2a90c 100644 --- a/ipu-plugin/pkg/ipuplugin/deviceplugin.go +++ b/ipu-plugin/pkg/ipuplugin/deviceplugin.go @@ -23,10 +23,6 @@ type DevicePluginService struct { } var ( - //TODO: Use (GetFilteredPfs), to find interface names to be excluded. - //excluding d3(host-acc), reserving D4-D5(QSPF ports) D6-D8(for max 3 host VFs), D9-D10(for single NF) - exclude = []string{"enp0s1f0", "enp0s1f0d1", "enp0s1f0d2", "enp0s1f0d3", "enp0s1f0d4", "enp0s1f0d5", "enp0s1f0d6", - "enp0s1f0d7", "enp0s1f0d8", "enp0s1f0d9", "enp0s1f0d10"} sysClassNet = "/sys/class/net" sysBusPciDevices = "/sys/bus/pci/devices" deviceCode = "0x1452" @@ -235,6 +231,15 @@ func (s *DevicePluginService) SetNumVfs(ctx context.Context, vfCountReq *pb.VfCo func discoverHostDevices(mode string) (map[string]*pb.Device, error) { + //Note: It is expected that ACC-Init happens prior to GetDevices, + //this check is meant to catch any anomalies. + if mode == types.IpuMode { + if len(AccApfsAvailForCNI) == 0 { + log.Errorf("discoverHostDevices: Error, AccApfsAvailForCNI not setup") + return make(map[string]*pb.Device), fmt.Errorf("discoverHostDevices: Error, AccApfsAvailForCNI not setup") + } + } + devices := make(map[string]*pb.Device) files, err := os.ReadDir(sysClassNet) @@ -253,7 +258,7 @@ func discoverHostDevices(mode string) (map[string]*pb.Device, error) { device_code := strings.TrimSpace(string(deviceCodeByte)) if mode == types.IpuMode { if device_code == deviceCode { - if !slices.Contains(exclude, file.Name()) { + if slices.Contains(AccApfsAvailForCNI, file.Name()) { devices[file.Name()] = &pb.Device{ID: file.Name(), Health: pluginapi.Healthy} } } diff --git a/ipu-plugin/pkg/ipuplugin/ipuplugin.go b/ipu-plugin/pkg/ipuplugin/ipuplugin.go index b00296ce..125f6196 100644 --- a/ipu-plugin/pkg/ipuplugin/ipuplugin.go +++ b/ipu-plugin/pkg/ipuplugin/ipuplugin.go @@ -21,6 +21,7 @@ import ( "os" "os/signal" "path/filepath" + "strconv" "syscall" "time" @@ -124,6 +125,59 @@ func waitForInfraP4d() (string, error) { return "", nil } +func AddAccApfsToGroupOne() error { + vsiList, err := utils.GetAvailableAccVsiList() + if err != nil { + log.Errorf("AddAccApfsToGroupOne: unable to reach the IMC %v", err) + return fmt.Errorf("AddAccApfsToGroupOne: unable to reach the IMC %v", err) + } + if len(vsiList) == 0 { + log.Errorf("no APFs initialized on ACC") + return fmt.Errorf("no APFs initialized on ACC") + } + log.Infof("AddAccApfsToGroupOne, vsiList->%v", vsiList) + /* Steps from script(for reference) + VSI_GROUP_INIT=$(printf "0x%x" $((0x8000050000000000 + IDPF_VPORT_VSI_HEX))) + VSI_GROUP_WRITE=$(printf "0x%x" $((0xA000050000000000 + IDPF_VPORT_VSI_HEX))) + devmem 0x20292002a0 64 ${VSI_GROUP_INIT} + devmem 0x2029200388 64 0x1 + devmem 0x20292002a0 64 ${VSI_GROUP_WRITE} + */ + for i := 0; i < len(vsiList); i++ { + log.Infof("Add to VSI Group 1, vsi->%v", vsiList[i]) + hexStr := vsiList[i] + // skip "0x" prefix + hexStr = hexStr[2:] + + // Convert to hex value + hexVal, err := strconv.ParseInt(hexStr, 16, 64) + if err != nil { + log.Errorf("error decoding hex: %v", err) + return fmt.Errorf("error decoding hex: %v", err) + } + + vsiGroupInit := 0x8000050000000000 + uint(hexVal) + vsiGroupWrite := 0xA000050000000000 + uint(hexVal) + + vsiGroupInitString := fmt.Sprintf("0x%X", vsiGroupInit) + vsiGroupWriteString := fmt.Sprintf("0x%X", vsiGroupWrite) + + devMemCmd1 := "devmem 0x20292002a0 64 " + vsiGroupInitString + devMemCmd2 := "devmem 0x2029200388 64 0x1" + devMemCmd3 := "devmem 0x20292002a0 64 " + vsiGroupWriteString + + devMemCmd := devMemCmd1 + "; " + devMemCmd2 + "; " + devMemCmd3 + "; " + log.Infof("devMemCmd->%v", devMemCmd) + + _, err = utils.ExecuteScript(fmt.Sprintf(`ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@192.168.0.1 "%s"`, devMemCmd)) + if err != nil { + log.Errorf("err exec devMemCmd->%v", err) + return fmt.Errorf("err exec devMemCmd->%v", err) + } + } + return nil +} + func (s *server) Run() error { var err error signalChannel := make(chan os.Signal, 2) @@ -139,6 +193,11 @@ func (s *server) Run() error { if _, err := waitForInfraP4d(); err != nil { return err } + if err := AddAccApfsToGroupOne(); err != nil { + log.Fatalf("error from->AddAccApfsToGroupOne: %v", err) + return fmt.Errorf("error from->AddAccApfsToGroupOne: %v", err) + } + // Create bridge if it doesn't exist if err := s.bridgeCtlr.EnsureBridgeExists(); err != nil { log.Fatalf("error while checking host bridge existence: %v", err) @@ -174,8 +233,8 @@ func (s *server) Stop() { s.bridgeCtlr.DeleteBridges() } - log.Infof("DeletePhyPortRules, path->%s, 1->%v, 2->%v", s.p4rtbin, AccApfMacList[PHY_PORT0_INTF_INDEX], AccApfMacList[PHY_PORT1_INTF_INDEX]) - p4rtclient.DeletePhyPortRules(s.p4rtbin, AccApfMacList[PHY_PORT0_INTF_INDEX], AccApfMacList[PHY_PORT1_INTF_INDEX]) + log.Infof("DeletePhyPortRules, path->%s, 1->%v, 2->%v", s.p4rtbin, AccApfInfo[PHY_PORT0_INTF_INDEX].Mac, AccApfInfo[PHY_PORT1_INTF_INDEX].Mac) + p4rtclient.DeletePhyPortRules(s.p4rtbin, AccApfInfo[PHY_PORT0_INTF_INDEX].Mac, AccApfInfo[PHY_PORT1_INTF_INDEX].Mac) vfMacList, err := utils.GetVfMacList() if err != nil { diff --git a/ipu-plugin/pkg/ipuplugin/lifecycleservice.go b/ipu-plugin/pkg/ipuplugin/lifecycleservice.go index 0d1e7df8..9df40dfc 100644 --- a/ipu-plugin/pkg/ipuplugin/lifecycleservice.go +++ b/ipu-plugin/pkg/ipuplugin/lifecycleservice.go @@ -57,26 +57,38 @@ const ( deviceId = "0x1452" vendorId = "0x8086" imcAddress = "192.168.0.1:22" - ApfNumber = 16 + ApfNumber = 48 last_byte_mac_range = 239 ) +type AccApfInfoType struct { + Mac string + Name string +} + +var AccApfInfo []AccApfInfoType +var AccApfsAvailForCNI []string + var InitAccApfMacs = false -var AccApfMacList []string var PeerToPeerP4RulesAdded = false -// Reserved ACC interfaces(using vport_id or last digit of interface name, like 4 represents-> enp0s1f0d4) +// Reserved ACC interfaces(using vport_id or last digit of interface name, for example, index 4 represents-> enp0s1f0d4) const ( - PHY_PORT0_INTF_INDEX = 4 - PHY_PORT1_INTF_INDEX = 5 - NF_IN_PR_INTF_INDEX = 9 - NF_OUT_PR_INTF_INDEX = 10 + START_ID = 0 + RSVD_INIT_LEN = 4 + PHY_PORT0_INTF_INDEX = (START_ID + RSVD_INIT_LEN) + PHY_PORT1_INTF_INDEX = (PHY_PORT0_INTF_INDEX + 1) + MAX_NF_CNT = 4 + NF_PR_START_ID = (PHY_PORT1_INTF_INDEX + 1) + NF_PR_LEN = (MAX_NF_CNT * 2) + NF_PR_END_ID = (NF_PR_START_ID + NF_PR_LEN - 1) + NF_AVAIL_START_ID = NF_PR_END_ID + 1 + NF_AVAIL_END_ID = (NF_AVAIL_START_ID + NF_PR_LEN - 1) + HOST_VF_START_ID = (NF_AVAIL_END_ID + 1) + MAX_HOST_VF_CNT = (16) + HOST_VF_END_ID = (HOST_VF_START_ID + MAX_HOST_VF_CNT - 1) ) -// TODO: GetFilteredPFs can be used to fill the array. -var AccIntfNames = [ApfNumber]string{"enp0s1f0", "enp0s1f0d1", "enp0s1f0d2", "enp0s1f0d3", "enp0s1f0d4", "enp0s1f0d5", "enp0s1f0d6", - "enp0s1f0d7", "enp0s1f0d8", "enp0s1f0d9", "enp0s1f0d10", "enp0s1f0d11", "enp0s1f0d12", "enp0s1f0d13", "enp0s1f0d14", "enp0s1f0d15"} - func NewLifeCycleService(daemonHostIp, daemonIpuIp string, daemonPort int, mode string, p4rtbin string, brCtlr types.BridgeController) *LifeCycleServiceServer { return &LifeCycleServiceServer{ daemonHostIp: daemonHostIp, @@ -943,6 +955,7 @@ func (e *ExecutableHandlerImpl) validate() bool { log.Errorf("Not enough APFs %v, expected->%v", numAPFs, ApfNumber) return false } + if noReboot, infoStr := skipIMCReboot(); !noReboot { fmt.Printf("IMC reboot required : %v\n", infoStr) return false @@ -952,26 +965,33 @@ func (e *ExecutableHandlerImpl) validate() bool { } func (e *ExecutableHandlerImpl) SetupAccApfs() error { - var err error - if !InitAccApfMacs { - AccApfMacList, err = utils.GetAccApfMacList() - - if err != nil { - log.Errorf("unable to reach the IMC %v", err) - return fmt.Errorf("unable to reach the IMC %v", err) + var pfList []netlink.Link + InitHandlers() + if err := GetFilteredPFs(&pfList); err != nil { + log.Errorf("SetupAccApfs: err->%v from GetFilteredPFs", err) + return fmt.Errorf("SetupAccApfs: err->%v from GetFilteredPFs", err) } - - if len(AccApfMacList) != ApfNumber { - log.Errorf("not enough APFs initialized on ACC, total APFs->%d, APFs->%v", len(AccApfMacList), AccApfMacList) - return fmt.Errorf("not enough APFs initialized on ACC, total APFs->%d", len(AccApfMacList)) + if len(pfList) != ApfNumber { + log.Errorf("not enough APFs initialized on ACC, total APFs->%d, APFs->%v", len(pfList), pfList) + return fmt.Errorf("not enough APFs initialized on ACC, total APFs->%d, APFs->%v", len(pfList), pfList) } - log.Infof("On ACC, total APFs->%d", len(AccApfMacList)) - for i := 0; i < len(AccApfMacList); i++ { - log.Infof("index->%d, mac->%s", i, AccApfMacList[i]) + + for i := 0; i < len(pfList); i++ { + accApf := AccApfInfoType{ + Mac: pfList[i].Attrs().HardwareAddr.String(), + Name: pfList[i].Attrs().Name, + } + AccApfInfo = append(AccApfInfo, accApf) } - InitAccApfMacs = true } + log.Infof("AccApfInfo->%v", AccApfInfo) + for i := NF_AVAIL_START_ID; i <= NF_AVAIL_END_ID; i = i + 1 { + AccApfsAvailForCNI = append(AccApfsAvailForCNI, AccApfInfo[i].Name) + } + log.Infof("AccApfsAvailForCNI->%v", AccApfsAvailForCNI) + + InitAccApfMacs = true return nil } @@ -1003,13 +1023,13 @@ func (s *FXPHandlerImpl) configureFXP(p4rtbin string, brCtlr types.BridgeControl } //Add Phy Port0 to ovs bridge //Note: Per current design, Phy Port1 is added to a different bridge(through P4 rules). - if err := brCtlr.AddPort(AccIntfNames[PHY_PORT0_INTF_INDEX]); err != nil { - log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[PHY_PORT0_INTF_INDEX]) - return fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[PHY_PORT0_INTF_INDEX]) + if err := brCtlr.AddPort(AccApfInfo[PHY_PORT0_INTF_INDEX].Name); err != nil { + log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[PHY_PORT0_INTF_INDEX].Name) + return fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[PHY_PORT0_INTF_INDEX].Name) } //Add P4 rules for phy ports - log.Infof("AddPhyPortRules, path->%s, 1->%v, 2->%v", p4rtbin, AccApfMacList[PHY_PORT0_INTF_INDEX], AccApfMacList[PHY_PORT1_INTF_INDEX]) - p4rtclient.AddPhyPortRules(p4rtbin, AccApfMacList[PHY_PORT0_INTF_INDEX], AccApfMacList[PHY_PORT1_INTF_INDEX]) + log.Infof("AddPhyPortRules, path->%s, 1->%v, 2->%v", p4rtbin, AccApfInfo[PHY_PORT0_INTF_INDEX].Mac, AccApfInfo[PHY_PORT1_INTF_INDEX].Mac) + p4rtclient.AddPhyPortRules(p4rtbin, AccApfInfo[PHY_PORT0_INTF_INDEX].Mac, AccApfInfo[PHY_PORT1_INTF_INDEX].Mac) CheckAndAddPeerToPeerP4Rules(p4rtbin) diff --git a/ipu-plugin/pkg/ipuplugin/networkfunctionservice.go b/ipu-plugin/pkg/ipuplugin/networkfunctionservice.go index 1b5dad05..2f987051 100644 --- a/ipu-plugin/pkg/ipuplugin/networkfunctionservice.go +++ b/ipu-plugin/pkg/ipuplugin/networkfunctionservice.go @@ -16,6 +16,8 @@ package ipuplugin import ( "context" + "crypto/md5" + "encoding/hex" "fmt" "github.com/intel/ipu-opi-plugins/ipu-plugin/pkg/p4rtclient" @@ -27,12 +29,17 @@ import ( "google.golang.org/grpc/status" ) +const ( + PRS_LEN = 2 +) + type NetworkFunctionServiceServer struct { pb.UnimplementedNetworkFunctionServiceServer Ports map[string]*types.BridgePortInfo bridgeCtlr types.BridgeController p4RtClient types.P4RTClient p4rtbin string + nfReqMap map[string][PRS_LEN]uint } func NewNetworkFunctionService(ports map[string]*types.BridgePortInfo, brCtlr types.BridgeController, p4Client types.P4RTClient, p4rtbin string) *NetworkFunctionServiceServer { @@ -41,7 +48,42 @@ func NewNetworkFunctionService(ports map[string]*types.BridgePortInfo, brCtlr ty bridgeCtlr: brCtlr, p4RtClient: p4Client, p4rtbin: p4rtbin, + nfReqMap: make(map[string][PRS_LEN]uint), + } +} + +func AllocateAccInterfaceForNF() ([PRS_LEN]uint, error) { + var intfIds [PRS_LEN]uint + + log.Debugf("AllocateAccInterfaceForNF\n") + for i := 0; i < PRS_LEN; i++ { + intfId, err := AllocateAccInterface(types.NfPr) + if err != nil { + return intfIds, fmt.Errorf("error from AllocateAccInterface->%v", err) + } + intfIds[i] = intfId + } + log.Infof("AllocateAccInterfaceForNF: Interfaces allocated->%v\n", intfIds) + return intfIds, nil +} + +func FreeAccInterfaceForNF(intfIds [PRS_LEN]uint) error { + + log.Debugf("FreeAccInterfaceForNF, intfIds->%v\n", intfIds) + for i := 0; i < PRS_LEN; i++ { + err := FreeAccInterface(intfIds[i]) + if err != nil { + log.Errorf("error from AllocateAccInterface->%v", err) + } } + return nil +} + +func deriveKey(in *pb.NFRequest) string { + nfReqHash := md5.Sum([]byte(in.Input + in.Output)) + nfReqHashStr := hex.EncodeToString(nfReqHash[:]) + log.Infof("deriveKey->%s", nfReqHashStr) + return nfReqHashStr } func (s *NetworkFunctionServiceServer) CreateNetworkFunction(ctx context.Context, in *pb.NFRequest) (*pb.Empty, error) { @@ -56,21 +98,34 @@ func (s *NetworkFunctionServiceServer) CreateNetworkFunction(ctx context.Context CheckAndAddPeerToPeerP4Rules(s.p4rtbin) - if err := s.bridgeCtlr.AddPort(AccIntfNames[NF_IN_PR_INTF_INDEX]); err != nil { - log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[NF_IN_PR_INTF_INDEX]) - return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[NF_IN_PR_INTF_INDEX]) + intfIds, err := AllocateAccInterfaceForNF() + if err != nil { + log.Errorf("error from AllocateAccInterfaceForNF: %v, intfIds->%v", err, intfIds) + return nil, fmt.Errorf("error from AllocateAccInterfaceForNF: %v, intfIds->%v", err, intfIds) + } + NF_IN_PR := intfIds[0] + NF_OUT_PR := intfIds[1] + log.Infof("CNF: allocated NF PRs index (IN)->%v, OUT->%v", NF_IN_PR, NF_OUT_PR) + + if err := s.bridgeCtlr.AddPort(AccApfInfo[NF_IN_PR].Name); err != nil { + log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[NF_IN_PR].Name) + FreeAccInterfaceForNF(intfIds) + return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[NF_IN_PR].Name) } - if err := s.bridgeCtlr.AddPort(AccIntfNames[NF_OUT_PR_INTF_INDEX]); err != nil { - log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[NF_OUT_PR_INTF_INDEX]) - return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[NF_OUT_PR_INTF_INDEX]) + if err := s.bridgeCtlr.AddPort(AccApfInfo[NF_OUT_PR].Name); err != nil { + log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[NF_OUT_PR].Name) + FreeAccInterfaceForNF(intfIds) + return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[NF_OUT_PR].Name) } - log.Infof("added interfaces:inPR->%s, outPR->%s", AccIntfNames[NF_IN_PR_INTF_INDEX], AccIntfNames[NF_OUT_PR_INTF_INDEX]) + log.Infof("added interfaces:inPR->%s, outPR->%s", AccApfInfo[NF_IN_PR].Name, AccApfInfo[NF_OUT_PR].Name) /*Note: Currently this API does not have host-VF info, since there is no reference to what was passed by DPU in CreateBridgePort. As a work-around, we take full vfMacList, and write P4 rules, to connect all host VFs to NF. */ // Generate the P4 rules and program the FXP with NF comms log.Infof("AddNFP4Rules, path->%s, 1-%v, 2-%v, 3-%v, 4-%v, 5-%v", - s.p4rtbin, vfMacList, in.Input, in.Output, AccApfMacList[NF_IN_PR_INTF_INDEX], AccApfMacList[NF_OUT_PR_INTF_INDEX]) - p4rtclient.AddNFP4Rules(s.p4rtbin, vfMacList, in.Input, in.Output, AccApfMacList[NF_IN_PR_INTF_INDEX], AccApfMacList[NF_OUT_PR_INTF_INDEX]) + s.p4rtbin, vfMacList, in.Input, in.Output, AccApfInfo[NF_IN_PR].Mac, AccApfInfo[NF_OUT_PR].Mac) + p4rtclient.AddNFP4Rules(s.p4rtbin, vfMacList, in.Input, in.Output, AccApfInfo[NF_IN_PR].Mac, AccApfInfo[NF_OUT_PR].Mac) + + s.nfReqMap[deriveKey(in)] = intfIds return &pb.Empty{}, nil } @@ -86,19 +141,31 @@ func (s *NetworkFunctionServiceServer) DeleteNetworkFunction(ctx context.Context if len(vfMacList) == 0 { return nil, status.Error(codes.Internal, "No NFs initialized on the host") } - if err := s.bridgeCtlr.DeletePort(AccIntfNames[NF_IN_PR_INTF_INDEX]); err != nil { - log.Errorf("failed to delete port to bridge: %v, for interface->%v", err, AccIntfNames[NF_IN_PR_INTF_INDEX]) - return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[NF_IN_PR_INTF_INDEX]) + mapKey := deriveKey(in) + intfIds, ok := s.nfReqMap[mapKey] + if !ok { + log.Errorf("DNF:in->%s, out->%s, key->%v, not found in map", in.Input, in.Output, mapKey) + return nil, fmt.Errorf("DNF:in->%s, out->%s, key->%v, not found in map", in.Input, in.Output, mapKey) + } + NF_IN_PR := intfIds[0] + NF_OUT_PR := intfIds[1] + log.Infof("DNF: NF PRs index (IN)->%v, OUT->%v", NF_IN_PR, NF_OUT_PR) + + if err := s.bridgeCtlr.DeletePort(AccApfInfo[NF_IN_PR].Name); err != nil { + log.Errorf("failed to delete port to bridge: %v, for interface->%v", err, AccApfInfo[NF_IN_PR].Name) + return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[NF_IN_PR].Name) } - if err := s.bridgeCtlr.DeletePort(AccIntfNames[NF_OUT_PR_INTF_INDEX]); err != nil { - log.Errorf("failed to delete port to bridge: %v, for interface->%v", err, AccIntfNames[NF_OUT_PR_INTF_INDEX]) - return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[NF_OUT_PR_INTF_INDEX]) + if err := s.bridgeCtlr.DeletePort(AccApfInfo[NF_OUT_PR].Name); err != nil { + log.Errorf("failed to delete port to bridge: %v, for interface->%v", err, AccApfInfo[NF_OUT_PR].Name) + return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[NF_OUT_PR].Name) } - log.Infof("deleted interfaces:inPR->%s, outPR->%s", AccIntfNames[NF_IN_PR_INTF_INDEX], AccIntfNames[NF_OUT_PR_INTF_INDEX]) + log.Infof("deleted interfaces:inPR->%s, outPR->%s", AccApfInfo[NF_IN_PR].Name, AccApfInfo[NF_OUT_PR].Name) log.Infof("DeleteNFP4Rules, path->%s, 1-%v, 2-%v, 3-%v, 4-%v, 5-%v", - s.p4rtbin, vfMacList, in.Input, in.Output, AccApfMacList[NF_IN_PR_INTF_INDEX], AccApfMacList[NF_OUT_PR_INTF_INDEX]) - p4rtclient.DeleteNFP4Rules(s.p4rtbin, vfMacList, in.Input, in.Output, AccApfMacList[NF_IN_PR_INTF_INDEX], AccApfMacList[NF_OUT_PR_INTF_INDEX]) + s.p4rtbin, vfMacList, in.Input, in.Output, AccApfInfo[NF_IN_PR].Mac, AccApfInfo[NF_OUT_PR].Mac) + p4rtclient.DeleteNFP4Rules(s.p4rtbin, vfMacList, in.Input, in.Output, AccApfInfo[NF_IN_PR].Mac, AccApfInfo[NF_OUT_PR].Mac) + + delete(s.nfReqMap, mapKey) return &pb.Empty{}, nil } diff --git a/ipu-plugin/pkg/types/types.go b/ipu-plugin/pkg/types/types.go index afb604a6..d4d5179d 100644 --- a/ipu-plugin/pkg/types/types.go +++ b/ipu-plugin/pkg/types/types.go @@ -21,8 +21,8 @@ import ( type BridgeType int type BridgePortInfo struct { - PbBrPort *pb.BridgePort - PortInterface string + PbBrPort *pb.BridgePort + PortId uint //For example, if ACC interface name is enp0s1f0d4, PortId(vportId) will be 4. } const ( @@ -30,6 +30,8 @@ const ( LinuxBridge HostMode = "host" IpuMode = "ipu" + HostVfPr = "HostVfPr" // PR(port representator on ACC) for Host VF + NfPr = "NfPr" // PR(port representator on ACC) for Network Function ) func (b BridgeType) String() string { diff --git a/ipu-plugin/pkg/utils/utils.go b/ipu-plugin/pkg/utils/utils.go index dded187a..1195bbff 100644 --- a/ipu-plugin/pkg/utils/utils.go +++ b/ipu-plugin/pkg/utils/utils.go @@ -139,6 +139,19 @@ func ImcQueryfindVsiGivenMacAddr(mode string, mac string) (string, error) { return output, nil } +// skips ACC interfaces D0 to D3, which are used internally. So, not available for other usages. +func GetAvailableAccVsiList() ([]string, error) { + // reach out to the IMC + vsiList, err := ExecuteScript(`ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@192.168.0.1 "/usr/bin/cli_client -cq" \ + | awk '{if(($2 == "0x4") && ($10 != "0x0") && ($10 != "0x1") && ($10 != "0x2") && ($10 != "0x3")) {print $8}}'`) + + if err != nil { + return nil, fmt.Errorf("unable to reach the IMC %v", err) + } + + return strings.Split(strings.TrimSpace(vsiList), "\n"), nil +} + func GetVfMacList() ([]string, error) { // reach out to the IMC to get the mac addresses of the VFs output, err := ExecuteScript(`ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@192.168.0.1 "/usr/bin/cli_client -cq" \