Skip to content

Commit

Permalink
WIP. Changes for Host VFs and num of APFs increase.
Browse files Browse the repository at this point in the history 
Added changes to use AccApfInfo type for mac-list and intf-name
list. Basic sanity test for Init->SetupAccApfs and GetDevices works as
expected. Added host-vf res allocation. Pending ->NF PR res alloc, and
ctrl-map changes.
  • Loading branch information
@sudhar-krishnakumar
sudhar-krishnakumar committed Dec 12, 2024
1 parent f01e94e commit 8b3316a
Show file tree
Hide file tree
Showing 6 changed files with 136 additions and 107 deletions.
100 changes: 47 additions & 53 deletions ipu-plugin/pkg/ipuplugin/bridgeport.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,22 @@ import (
"google.golang.org/protobuf/types/known/emptypb"
)

const (
outerVlanId = 0 // hardcoded s-tag
)

var intfMapInit bool = false

// Note: 3 reserved(last digit of interface name, for example, enp0s1f0d8, is 8) in exlude list in deviceplugin.
var interfaces [3]string = [3]string{"6", "7", "8"}
var intfMap map[string]bool
/*
interfaces slice will be populated with PortIds on ACC(port representators on ACC, for Host VFs), for example,
if ACC interface name is enp0s1f0d4, PortId(vportId) will be 4.
*/
var interfaces []uint
var intfMap map[uint]bool

func initMap() error {
if intfMapInit == false {
intfMap = make(map[string]bool)
var index uint
if !intfMapInit {
for index = HOST_VF_START_ID; index <= HOST_VF_END_ID; index = index + 1 {
interfaces = append(interfaces, index)
}
intfMap = make(map[uint]bool)
for _, intf := range interfaces {
intfMap[intf] = false
}
Expand All @@ -54,39 +57,41 @@ func initMap() error {
}

// in-order(sorted by interface name->interfaces) allocation, based on available ACC interfaces(for Host VF)
func allocateAccInterface() (error, string) {
var intfName string = ""
func allocateAccInterface() (error, uint) {
var intfId uint = 0
found := false
log.Debugf("allocateAccInterface\n")
if intfMapInit == false {
if !intfMapInit {
initMap()
}
for _, key := range interfaces {
log.Debugf("intfName->%v\n", key)
log.Debugf("intfId->%v\n", key)
value, present := intfMap[key]
if present == true && value == false {
if present && !value {
log.Debugf("Found avail Intf->%v: \n", key)
intfMap[key] = true
intfName = key
intfId = key
found = true
break
}
}
if intfName != "" {
return nil, intfName
if found {
return nil, intfId
}
log.Errorf("Interface not available\n")
return fmt.Errorf("Interface not available\n"), intfName
return fmt.Errorf("interface not available"), intfId
}

func freeAccInterface(intfName string) error {
func freeAccInterface(intfId uint) error {
log.Debugf("freeAccInterface\n")
value, present := intfMap[intfName]
if present == true && value == true {
log.Debugf("Found allocated Intf->%v: \n", intfName)
intfMap[intfName] = false
value, present := intfMap[intfId]
if present && value {
log.Debugf("Found allocated Intf->%v: \n", intfId)
intfMap[intfId] = false
return nil
}
log.Errorf("Interface->%s not found in freeAccInterface\n", intfName)
return fmt.Errorf("Interface->%s not found in freeAccInterface\n", intfName)
log.Errorf("Interface->%v not found in freeAccInterface\n", intfId)
return fmt.Errorf("interface->%v not found in freeAccInterface", intfId)
}

// CreateBridgePort executes the creation of the port
Expand Down Expand Up @@ -127,32 +132,26 @@ func (s *server) CreateBridgePort(_ context.Context, in *pb.CreateBridgePortRequ

CheckAndAddPeerToPeerP4Rules(s.p4rtbin)

err, intfName := allocateAccInterface()
err, intfId := allocateAccInterface()
if err != nil {
return nil, fmt.Errorf("error from allocateAccInterface->%v", err)
}

intIndex, err := strconv.Atoi(string(intfName))
if err != nil {
log.Errorf("error->%v converting, intfName->%v", err, intfName)
return nil, fmt.Errorf("error->%v converting, intfName->%v", err, intfName)
} else {
log.Infof("intIndex->%v, fullIntfName->%v", intIndex, AccIntfNames[intIndex])
}
log.Infof("intfId->%v, fullIntfName->%v", intfId, AccApfInfo[intfId].Name)

if err := s.bridgeCtlr.AddPort(AccIntfNames[intIndex]); err != nil {
log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
freeAccInterface(intfName)
return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
if err := s.bridgeCtlr.AddPort(AccApfInfo[intfId].Name); err != nil {
log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
freeAccInterface(intfId)
return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
}

// Add FXP rules
log.Infof("AddHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfMacList[intIndex])
p4rtclient.AddHostVfP4Rules(s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfMacList[intIndex])
log.Infof("AddHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfInfo[intfId].Mac)
p4rtclient.AddHostVfP4Rules(s.p4rtbin, in.BridgePort.Spec.MacAddress, AccApfInfo[intfId].Mac)

resp := proto.Clone(in.BridgePort).(*pb.BridgePort)
resp.Status = &pb.BridgePortStatus{OperStatus: pb.BPOperStatus_BP_OPER_STATUS_UP}
pbBridgePortInfo := &types.BridgePortInfo{PbBrPort: resp, PortInterface: intfName}
pbBridgePortInfo := &types.BridgePortInfo{PbBrPort: resp, PortId: intfId}
s.Ports[in.BridgePort.Name] = pbBridgePortInfo
return resp, nil
}
Expand Down Expand Up @@ -180,22 +179,17 @@ func (s *server) DeleteBridgePort(_ context.Context, in *pb.DeleteBridgePortRequ
}
portInfo = brPortInfo.PbBrPort

intIndex, err := strconv.Atoi(string(brPortInfo.PortInterface))
if err != nil {
log.Errorf("error->%v converting, intfName->%v", err, brPortInfo.PortInterface)
return nil, fmt.Errorf("error->%v converting, intfName->%v", err, brPortInfo.PortInterface)
} else {
log.Infof("intIndex->%v, fullIntfName->%v", intIndex, AccIntfNames[intIndex])
}
intfId := brPortInfo.PortId
log.Infof("intfIndex->%v, fullIntfName->%v", intfId, AccApfInfo[intfId].Name)

if err := s.bridgeCtlr.DeletePort(AccIntfNames[intIndex]); err != nil {
log.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
return nil, fmt.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
if err := s.bridgeCtlr.DeletePort(AccApfInfo[intfId].Name); err != nil {
log.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
return nil, fmt.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
}
freeAccInterface(brPortInfo.PortInterface)
freeAccInterface(brPortInfo.PortId)
// Delete FXP rules
log.Infof("DeleteHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, portInfo.Spec.MacAddress, AccApfMacList[intIndex])
p4rtclient.DeleteHostVfP4Rules(s.p4rtbin, portInfo.Spec.MacAddress, AccApfMacList[intIndex])
log.Infof("DeleteHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtbin, portInfo.Spec.MacAddress, AccApfInfo[intfId].Mac)
p4rtclient.DeleteHostVfP4Rules(s.p4rtbin, portInfo.Spec.MacAddress, AccApfInfo[intfId].Mac)

delete(s.Ports, in.Name)
return &emptypb.Empty{}, nil
Expand Down
15 changes: 10 additions & 5 deletions ipu-plugin/pkg/ipuplugin/deviceplugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,6 @@ type DevicePluginService struct {
}

var (
//TODO: Use (GetFilteredPfs), to find interface names to be excluded.
//excluding d3(host-acc), reserving D4-D5(QSPF ports) D6-D8(for max 3 host VFs), D9-D10(for single NF)
exclude = []string{"enp0s1f0", "enp0s1f0d1", "enp0s1f0d2", "enp0s1f0d3", "enp0s1f0d4", "enp0s1f0d5", "enp0s1f0d6",
"enp0s1f0d7", "enp0s1f0d8", "enp0s1f0d9", "enp0s1f0d10"}
sysClassNet = "/sys/class/net"
sysBusPciDevices = "/sys/bus/pci/devices"
deviceCode = "0x1452"
Expand Down Expand Up @@ -235,6 +231,15 @@ func (s *DevicePluginService) SetNumVfs(ctx context.Context, vfCountReq *pb.VfCo

func discoverHostDevices(mode string) (map[string]*pb.Device, error) {

//Note: It is expected that ACC-Init happens prior to GetDevices,
//this check is meant to catch any anomalies.
if mode == types.IpuMode {
if len(AccApfsAvailForCNI) == 0 {
log.Errorf("discoverHostDevices: Error, AccApfsAvailForCNI not setup")
return make(map[string]*pb.Device), fmt.Errorf("discoverHostDevices: Error, AccApfsAvailForCNI not setup")
}
}

devices := make(map[string]*pb.Device)

files, err := os.ReadDir(sysClassNet)
Expand All @@ -253,7 +258,7 @@ func discoverHostDevices(mode string) (map[string]*pb.Device, error) {
device_code := strings.TrimSpace(string(deviceCodeByte))
if mode == types.IpuMode {
if device_code == deviceCode {
if !slices.Contains(exclude, file.Name()) {
if slices.Contains(AccApfsAvailForCNI, file.Name()) {
devices[file.Name()] = &pb.Device{ID: file.Name(), Health: pluginapi.Healthy}
}
}
Expand Down
4 changes: 2 additions & 2 deletions ipu-plugin/pkg/ipuplugin/ipuplugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,8 @@ func (s *server) Stop() {
s.bridgeCtlr.DeleteBridges()
}

log.Infof("DeletePhyPortRules, path->%s, 1->%v, 2->%v", s.p4rtbin, AccApfMacList[PHY_PORT0_INTF_INDEX], AccApfMacList[PHY_PORT1_INTF_INDEX])
p4rtclient.DeletePhyPortRules(s.p4rtbin, AccApfMacList[PHY_PORT0_INTF_INDEX], AccApfMacList[PHY_PORT1_INTF_INDEX])
log.Infof("DeletePhyPortRules, path->%s, 1->%v, 2->%v", s.p4rtbin, AccApfInfo[PHY_PORT0_INTF_INDEX].Mac, AccApfInfo[PHY_PORT1_INTF_INDEX].Mac)
p4rtclient.DeletePhyPortRules(s.p4rtbin, AccApfInfo[PHY_PORT0_INTF_INDEX].Mac, AccApfInfo[PHY_PORT1_INTF_INDEX].Mac)

vfMacList, err := utils.GetVfMacList()
if err != nil {
Expand Down
84 changes: 57 additions & 27 deletions ipu-plugin/pkg/ipuplugin/lifecycleservice.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,25 +57,47 @@ const (
deviceId = "0x1452"
vendorId = "0x8086"
imcAddress = "192.168.0.1:22"
ApfNumber = 16
ApfNumber = 48
last_byte_mac_range = 239
)

type AccApfInfoType struct {
Mac string
Name string
}

var AccApfInfo []AccApfInfoType
var AccApfsAvailForCNI []string

var InitAccApfMacs = false
var AccApfMacList []string
var PeerToPeerP4RulesAdded = false

// Reserved ACC interfaces(using vport_id or last digit of interface name, like 4 represents-> enp0s1f0d4)
const (
/*const (
PHY_PORT0_INTF_INDEX = 4
PHY_PORT1_INTF_INDEX = 5
NF_IN_PR_INTF_INDEX = 9
NF_OUT_PR_INTF_INDEX = 10
)
)*/

// TODO: GetFilteredPFs can be used to fill the array.
var AccIntfNames = [ApfNumber]string{"enp0s1f0", "enp0s1f0d1", "enp0s1f0d2", "enp0s1f0d3", "enp0s1f0d4", "enp0s1f0d5", "enp0s1f0d6",
"enp0s1f0d7", "enp0s1f0d8", "enp0s1f0d9", "enp0s1f0d10", "enp0s1f0d11", "enp0s1f0d12", "enp0s1f0d13", "enp0s1f0d14", "enp0s1f0d15"}
const (
START_ID = 0
RSVD_INIT_LEN = 4
PHY_PORT0_INTF_INDEX = (START_ID + RSVD_INIT_LEN)
PHY_PORT1_INTF_INDEX = (PHY_PORT0_INTF_INDEX + 1)
MAX_NF_CNT = 4
NF_PR_START_ID = (PHY_PORT1_INTF_INDEX + 1)
NF_PR_LEN = (MAX_NF_CNT * 2)
NF_PR_END_ID = (NF_PR_START_ID + NF_PR_LEN - 1)
NF_AVAIL_START_ID = NF_PR_END_ID + 1
NF_AVAIL_END_ID = (NF_AVAIL_START_ID + NF_PR_LEN - 1)
//TODO: Remove fixed NF_IN_PR_INTF_INDEX, NF_OUT_PR_INTF_INDEX, once NF_PR APF resource alloc changes are done.
NF_IN_PR_INTF_INDEX = NF_PR_START_ID
NF_OUT_PR_INTF_INDEX = NF_IN_PR_INTF_INDEX + 1
HOST_VF_START_ID = (NF_AVAIL_END_ID + 1)
MAX_HOST_VF_CNT = (16)
HOST_VF_END_ID = (HOST_VF_START_ID + MAX_HOST_VF_CNT - 1)
)

func NewLifeCycleService(daemonHostIp, daemonIpuIp string, daemonPort int, mode string, p4rtbin string, brCtlr types.BridgeController) *LifeCycleServiceServer {
return &LifeCycleServiceServer{
Expand Down Expand Up @@ -943,6 +965,7 @@ func (e *ExecutableHandlerImpl) validate() bool {
log.Errorf("Not enough APFs %v, expected->%v", numAPFs, ApfNumber)
return false
}

if noReboot, infoStr := skipIMCReboot(); !noReboot {
fmt.Printf("IMC reboot required : %v\n", infoStr)
return false
Expand All @@ -952,26 +975,33 @@ func (e *ExecutableHandlerImpl) validate() bool {
}

func (e *ExecutableHandlerImpl) SetupAccApfs() error {
var err error

if !InitAccApfMacs {
AccApfMacList, err = utils.GetAccApfMacList()

if err != nil {
log.Errorf("unable to reach the IMC %v", err)
return fmt.Errorf("unable to reach the IMC %v", err)
var pfList []netlink.Link
InitHandlers()
if err := GetFilteredPFs(&pfList); err != nil {
log.Errorf("SetupAccApfs: err->%v from GetFilteredPFs", err)
return fmt.Errorf("SetupAccApfs: err->%v from GetFilteredPFs", err)
}

if len(AccApfMacList) != ApfNumber {
log.Errorf("not enough APFs initialized on ACC, total APFs->%d, APFs->%v", len(AccApfMacList), AccApfMacList)
return fmt.Errorf("not enough APFs initialized on ACC, total APFs->%d", len(AccApfMacList))
if len(pfList) != ApfNumber {
log.Errorf("not enough APFs initialized on ACC, total APFs->%d, APFs->%v", len(pfList), pfList)
return fmt.Errorf("not enough APFs initialized on ACC, total APFs->%d, APFs->%v", len(pfList), pfList)
}
log.Infof("On ACC, total APFs->%d", len(AccApfMacList))
for i := 0; i < len(AccApfMacList); i++ {
log.Infof("index->%d, mac->%s", i, AccApfMacList[i])

for i := 0; i < len(pfList); i++ {
accApf := AccApfInfoType{
Mac: pfList[i].Attrs().HardwareAddr.String(),
Name: pfList[i].Attrs().Name,
}
AccApfInfo = append(AccApfInfo, accApf)
}
InitAccApfMacs = true
}
log.Infof("AccApfInfo->%v", AccApfInfo)
for i := NF_AVAIL_START_ID; i <= NF_AVAIL_END_ID; i = i + 1 {
AccApfsAvailForCNI = append(AccApfsAvailForCNI, AccApfInfo[i].Name)
}
log.Infof("AccApfsAvailForCNI->%v", AccApfsAvailForCNI)

InitAccApfMacs = true
return nil
}

Expand Down Expand Up @@ -1003,13 +1033,13 @@ func (s *FXPHandlerImpl) configureFXP(p4rtbin string, brCtlr types.BridgeControl
}
//Add Phy Port0 to ovs bridge
//Note: Per current design, Phy Port1 is added to a different bridge(through P4 rules).
if err := brCtlr.AddPort(AccIntfNames[PHY_PORT0_INTF_INDEX]); err != nil {
log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[PHY_PORT0_INTF_INDEX])
return fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[PHY_PORT0_INTF_INDEX])
if err := brCtlr.AddPort(AccApfInfo[PHY_PORT0_INTF_INDEX].Name); err != nil {
log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[PHY_PORT0_INTF_INDEX].Name)
return fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[PHY_PORT0_INTF_INDEX].Name)
}
//Add P4 rules for phy ports
log.Infof("AddPhyPortRules, path->%s, 1->%v, 2->%v", p4rtbin, AccApfMacList[PHY_PORT0_INTF_INDEX], AccApfMacList[PHY_PORT1_INTF_INDEX])
p4rtclient.AddPhyPortRules(p4rtbin, AccApfMacList[PHY_PORT0_INTF_INDEX], AccApfMacList[PHY_PORT1_INTF_INDEX])
log.Infof("AddPhyPortRules, path->%s, 1->%v, 2->%v", p4rtbin, AccApfInfo[PHY_PORT0_INTF_INDEX].Mac, AccApfInfo[PHY_PORT1_INTF_INDEX].Mac)
p4rtclient.AddPhyPortRules(p4rtbin, AccApfInfo[PHY_PORT0_INTF_INDEX].Mac, AccApfInfo[PHY_PORT1_INTF_INDEX].Mac)

CheckAndAddPeerToPeerP4Rules(p4rtbin)

Expand Down
Loading

0 comments on commit 8b3316a

Please sign in to comment.