Skip to content

Commit

Permalink
Changes to allow upto 16 Host VFs and num of APFs increased to 48. (#306
Browse files Browse the repository at this point in the history 
)

* Changes for Host VFs and num of APFs increased to 48, to accomodate 16 Host VFs.
Pre-defined range of ACC APF interface indexes allocated for various usages like(Host VFs, NF PRs, NFs(for ingress/egress)). Added changes to use AccApfInfo type for mac-list and intf-name list. Basic sanity test for Init->SetupAccApfs and GetDevices works as expected.
Changes to run devmem commands to add 48 ACC APFs to forwarding group one. By default only 16 ACC APFs are added to forwarding Group one. Function->AddAccApfsToGroupOne added to run devmem commands for all APFs on ACC.

Changes tested for init, getDevices, CreateBridgePort, DeleteBridgePort, host-VF-to-host-VF(ping test). With CreateNetworkFunction, and DeleteNetworkFunction works, and verified ping testing between external-client(CVL NIC) to NF's ingressMAC, and NF-egress to Host-VF, and between Host-VF to external-client.

* Added changes to query IMC config(/etc/dpcp/cp_init.cfg) to
determine acc_apf, and then wait for those APFs to get
initialized. Also, old redhat P4 package code to check
for D6 interface, was causing issue, when default P4 package,
was used, made that check strictly for redhat P4 package.
  • Loading branch information
@sudhar-krishnakumar
sudhar-krishnakumar authored Jan 3, 2025
1 parent a34fefe commit 10d134f
Show file tree
Hide file tree
Showing 10 changed files with 427 additions and 122 deletions.
10 changes: 9 additions & 1 deletion ipu-plugin/ipuplugin/cmd/rootcmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,16 @@ var (

log.Info("Initializing IPU plugin")
if mode == types.IpuMode {
//Note::For linux-P4 package, we dont need to
//update portMuxVsi(not used). For redhat P4
//we would need this check. In case, where default
//P4 package is loaded with 4 APFs, this check,
//will fail.
//As a quick fix, restricting check to redhat P4 package only.
//TODO:: Cleanup what is not needed for linux-P4, and
//perhaps we can move this check else-where for redhat-P4.
vsi, err := findVsiForPfInterface(mode, intf)
if err != nil {
if (err != nil) && (p4pkg == "redhat") {
log.Errorf("Not able to find VSI->%d, for bridge interface->%v\n", vsi, intf)
exitWithError(err, 5)
} else {
Expand Down
133 changes: 71 additions & 62 deletions ipu-plugin/pkg/ipuplugin/bridgeport.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,65 +28,85 @@ import (
"google.golang.org/protobuf/types/known/emptypb"
)

const (
outerVlanId = 0 // hardcoded s-tag
)

/*
interfaces slice will be populated with PortIds on ACC(port representators->PRs on ACC, for Host VFs),
for example, if ACC interface name is enp0s1f0d4, PortId(vportId) will be 4.
Will also include port representators->PRs needed for Network Functions.
intfMap is a map, that has key:value, between interfaceId and whether it is available(true or false) for use.
*/
var interfaces []uint
var intfMap map[uint]bool
var intfMapInit bool = false

// Note: 3 reserved(last digit of interface name, for example, enp0s1f0d8, is 8) in exlude list in deviceplugin.
var interfaces [3]string = [3]string{"6", "7", "8"}
var intfMap map[string]bool

func initMap() error {
if intfMapInit == false {
intfMap = make(map[string]bool)
var index uint
if !intfMapInit {
for index = HOST_VF_START_ID; index <= HOST_VF_END_ID; index = index + 1 {
interfaces = append(interfaces, index)
}
for index = NF_PR_START_ID; index <= NF_PR_END_ID; index = index + 1 {
interfaces = append(interfaces, index)
}
intfMap = make(map[uint]bool)
for _, intf := range interfaces {
intfMap[intf] = false
}
if len(interfaces) != len(intfMap) {
log.Errorf("initMap setup error\n")
return fmt.Errorf("initMap setup error\n")
log.Errorf("initMap setup error")
return fmt.Errorf("initMap setup error")
}
intfMapInit = true
}
return nil
}

// in-order(sorted by interface name->interfaces) allocation, based on available ACC interfaces(for Host VF)
func allocateAccInterface() (error, string) {
var intfName string = ""
log.Debugf("allocateAccInterface\n")
if intfMapInit == false {
// in-order(sorted by interface IDs) allocation. Based on available ACC interfaces(for Host VF
// and NF PRs). Currently there are 2 ranges, first range(sorted) is for available Host-VF interface IDs
// (HOST_VF_START_ID to HOST_VF_END_ID) and second range(sorted) for NF PRs(NF_PR_START_ID to NF_PR_END_ID)
func AllocateAccInterface(allocPr string) (uint, error) {
var intfId uint = 0
start, end := 0, 0

found := false
log.Debugf("AllocateAccInterface\n")
if !intfMapInit {
initMap()
}
for _, key := range interfaces {
log.Debugf("intfName->%v\n", key)
if allocPr == types.HostVfPr {
start = 0
end = HOST_VF_END_ID - HOST_VF_START_ID
} else {
start = HOST_VF_END_ID - HOST_VF_START_ID + 1
end = start + NF_PR_END_ID - NF_PR_START_ID
}
for i := start; i <= end; i++ {
key := interfaces[i]
value, present := intfMap[key]
if present == true && value == false {
if present && !value {
log.Debugf("Found avail Intf->%v: \n", key)
intfMap[key] = true
intfName = key
intfId = key
found = true
break
}
}
if intfName != "" {
return nil, intfName
if found {
return intfId, nil
}
log.Errorf("Interface not available\n")
return fmt.Errorf("Interface not available\n"), intfName
log.Errorf("AllocateAccInterface: Interface not available")
return intfId, fmt.Errorf("AllocateAccInterface: interface not available")
}

func freeAccInterface(intfName string) error {
log.Debugf("freeAccInterface\n")
value, present := intfMap[intfName]
if present == true && value == true {
log.Debugf("Found allocated Intf->%v: \n", intfName)
intfMap[intfName] = false
func FreeAccInterface(intfId uint) error {
log.Debugf("FreeAccInterface\n")
value, present := intfMap[intfId]
if present && value {
log.Debugf("Found allocated Intf->%v: \n", intfId)
intfMap[intfId] = false
return nil
}
log.Errorf("Interface->%s not found in freeAccInterface\n", intfName)
return fmt.Errorf("Interface->%s not found in freeAccInterface\n", intfName)
log.Errorf("Interface->%v not found in FreeAccInterface", intfId)
return fmt.Errorf("interface->%v not found in FreeAccInterface", intfId)
}

// CreateBridgePort executes the creation of the port
Expand Down Expand Up @@ -127,32 +147,26 @@ func (s *server) CreateBridgePort(_ context.Context, in *pb.CreateBridgePortRequ

CheckAndAddPeerToPeerP4Rules(s.p4rtbin)

err, intfName := allocateAccInterface()
intfId, err := AllocateAccInterface(types.HostVfPr)
if err != nil {
return nil, fmt.Errorf("error from allocateAccInterface->%v", err)
return nil, fmt.Errorf("error from AllocateAccInterface->%v", err)
}

intIndex, err := strconv.Atoi(string(intfName))
if err != nil {
log.Errorf("error->%v converting, intfName->%v", err, intfName)
return nil, fmt.Errorf("error->%v converting, intfName->%v", err, intfName)
} else {
log.Infof("intIndex->%v, fullIntfName->%v", intIndex, AccIntfNames[intIndex])
}
log.Infof("intfId->%v, fullIntfName->%v", intfId, AccApfInfo[intfId].Name)

if err := s.bridgeCtlr.AddPort(AccIntfNames[intIndex]); err != nil {
log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
freeAccInterface(intfName)
return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
if err := s.bridgeCtlr.AddPort(AccApfInfo[intfId].Name); err != nil {
log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
FreeAccInterface(intfId)
return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
}

// Add FXP rules
log.Infof("AddHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfMacList[intIndex])
p4rtclient.AddHostVfP4Rules(s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfMacList[intIndex])
log.Infof("AddHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfInfo[intfId].Mac)
p4rtclient.AddHostVfP4Rules(s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfInfo[intfId].Mac)

resp := proto.Clone(in.BridgePort).(*pb.BridgePort)
resp.Status = &pb.BridgePortStatus{OperStatus: pb.BPOperStatus_BP_OPER_STATUS_UP}
pbBridgePortInfo := &types.BridgePortInfo{PbBrPort: resp, PortInterface: intfName}
pbBridgePortInfo := &types.BridgePortInfo{PbBrPort: resp, PortId: intfId}
s.Ports[in.BridgePort.Name] = pbBridgePortInfo
return resp, nil
}
Expand Down Expand Up @@ -180,22 +194,17 @@ func (s *server) DeleteBridgePort(_ context.Context, in *pb.DeleteBridgePortRequ
}
portInfo = brPortInfo.PbBrPort

intIndex, err := strconv.Atoi(string(brPortInfo.PortInterface))
if err != nil {
log.Errorf("error->%v converting, intfName->%v", err, brPortInfo.PortInterface)
return nil, fmt.Errorf("error->%v converting, intfName->%v", err, brPortInfo.PortInterface)
} else {
log.Infof("intIndex->%v, fullIntfName->%v", intIndex, AccIntfNames[intIndex])
}
intfId := brPortInfo.PortId
log.Infof("intfIndex->%v, fullIntfName->%v", intfId, AccApfInfo[intfId].Name)

if err := s.bridgeCtlr.DeletePort(AccIntfNames[intIndex]); err != nil {
log.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
return nil, fmt.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
if err := s.bridgeCtlr.DeletePort(AccApfInfo[intfId].Name); err != nil {
log.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
return nil, fmt.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
}
freeAccInterface(brPortInfo.PortInterface)
FreeAccInterface(brPortInfo.PortId)
// Delete FXP rules
log.Infof("DeleteHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, portInfo.Spec.MacAddress, AccApfMacList[intIndex])
p4rtclient.DeleteHostVfP4Rules(s.p4rtbin, portInfo.Spec.MacAddress, AccApfMacList[intIndex])
log.Infof("DeleteHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, portInfo.Spec.MacAddress, AccApfInfo[intfId].Mac)
p4rtclient.DeleteHostVfP4Rules(s.p4rtbin, portInfo.Spec.MacAddress, AccApfInfo[intfId].Mac)

delete(s.Ports, in.Name)
return &emptypb.Empty{}, nil
Expand Down
4 changes: 2 additions & 2 deletions ipu-plugin/pkg/ipuplugin/bridgeport_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ var _ = Describe("bridgeport", Serial, func() {
},
}
fakePortBridgeInfo := &types.BridgePortInfo{
fakePort, "fakePort1",
fakePort, 10,
}

fakeReq := &pb.CreateBridgePortRequest{BridgePort: fakePort}
Expand All @@ -131,7 +131,7 @@ var _ = Describe("bridgeport", Serial, func() {
},
}
fakePortBridgeInfo := &types.BridgePortInfo{
fakePort, "fakePort1",
fakePort, 10,
}
ipuServer.Ports["fakePort"] = fakePortBridgeInfo // fakePort already exists in internal Map

Expand Down
15 changes: 10 additions & 5 deletions ipu-plugin/pkg/ipuplugin/deviceplugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,6 @@ type DevicePluginService struct {
}

var (
//TODO: Use (GetFilteredPfs), to find interface names to be excluded.
//excluding d3(host-acc), reserving D4-D5(QSPF ports) D6-D8(for max 3 host VFs), D9-D10(for single NF)
exclude = []string{"enp0s1f0", "enp0s1f0d1", "enp0s1f0d2", "enp0s1f0d3", "enp0s1f0d4", "enp0s1f0d5", "enp0s1f0d6",
"enp0s1f0d7", "enp0s1f0d8", "enp0s1f0d9", "enp0s1f0d10"}
sysClassNet = "/sys/class/net"
sysBusPciDevices = "/sys/bus/pci/devices"
deviceCode = "0x1452"
Expand Down Expand Up @@ -235,6 +231,15 @@ func (s *DevicePluginService) SetNumVfs(ctx context.Context, vfCountReq *pb.VfCo

func discoverHostDevices(mode string) (map[string]*pb.Device, error) {

//Note: It is expected that ACC-Init happens prior to GetDevices,
//this check is meant to catch any anomalies.
if mode == types.IpuMode {
if len(AccApfsAvailForCNI) == 0 {
log.Errorf("discoverHostDevices: Error, AccApfsAvailForCNI not setup")
return make(map[string]*pb.Device), fmt.Errorf("discoverHostDevices: Error, AccApfsAvailForCNI not setup")
}
}

devices := make(map[string]*pb.Device)

files, err := os.ReadDir(sysClassNet)
Expand All @@ -253,7 +258,7 @@ func discoverHostDevices(mode string) (map[string]*pb.Device, error) {
device_code := strings.TrimSpace(string(deviceCodeByte))
if mode == types.IpuMode {
if device_code == deviceCode {
if !slices.Contains(exclude, file.Name()) {
if slices.Contains(AccApfsAvailForCNI, file.Name()) {
devices[file.Name()] = &pb.Device{ID: file.Name(), Health: pluginapi.Healthy}
}
}
Expand Down
4 changes: 2 additions & 2 deletions ipu-plugin/pkg/ipuplugin/ipuplugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,8 @@ func (s *server) Stop() {
s.bridgeCtlr.DeleteBridges()
}

log.Infof("DeletePhyPortRules, path->%s, 1->%v, 2->%v", s.p4rtbin, AccApfMacList[PHY_PORT0_INTF_INDEX], AccApfMacList[PHY_PORT1_INTF_INDEX])
p4rtclient.DeletePhyPortRules(s.p4rtbin, AccApfMacList[PHY_PORT0_INTF_INDEX], AccApfMacList[PHY_PORT1_INTF_INDEX])
log.Infof("DeletePhyPortRules, path->%s, 1->%v, 2->%v", s.p4rtbin, AccApfInfo[PHY_PORT0_INTF_INDEX].Mac, AccApfInfo[PHY_PORT1_INTF_INDEX].Mac)
p4rtclient.DeletePhyPortRules(s.p4rtbin, AccApfInfo[PHY_PORT0_INTF_INDEX].Mac, AccApfInfo[PHY_PORT1_INTF_INDEX].Mac)

vfMacList, err := utils.GetVfMacList()
if err != nil {
Expand Down
Loading

0 comments on commit 10d134f

Please sign in to comment.