debugging and default flags when mh used

Dockerfile

@@ -13,12 +13,9 @@ RUN ls -al
 FROM golang:1.13-alpine
 LABEL MAINTAINER='RDEI Team <[email protected]>'
 RUN echo '@edgemain http://dl-3.alpinelinux.org/alpine/edge/main' >> /etc/apk/repositories
-RUN apk add libpcap iptables haproxy iproute2 ipvsadm@edgemain gcc libc-dev git libpcap-dev && rm -rf /var/cache/apk/*
-RUN rm -rf /var/cache/apk/*
-
+RUN apk add libpcap iptables haproxy iproute2 ipvsadm@edgemain gcc libc-dev git libpcap-dev && rm -rf /var/cache/apk/*; rm -rf /var/cache/apk/*
 COPY --from=0 /app/src/cmd/ravel/ravel /bin/
 COPY --from=0 /app/src/cmd/ravel/gobgp /bin/
 COPY --from=0 /app/src/cmd/ravel/gobgpd /bin/
 RUN chmod 750 /bin/gobgp /bin/gobgpd /bin/ravel
-
 ENTRYPOINT ["/bin/ravel"]

Makefile

@@ -1,4 +1,6 @@
-TAG=v2.5.0-rc4
+TAG=v2.5.0-proto53
+
+# rc6: hub.comcast.net/k8s-eng/ravel:v2.5.0-proto45
 
 # Not a complicated makefile, just a place to ensure
 # that we don't forget how to build and push to a registry.
@@ -21,5 +23,4 @@ push-gobgp:
 	docker push hub.comcast.net/k8s-eng/gobgpd:v2.2.0
 
 m1:
-	docker buildx build --platform linux/amd64 --push -t hub.comcast.net/k8s-eng/ravel:${TAG} -f Dockerfile .
-	#docker buildx build --platform linux/amd64,linux/arm64 --push -t membermatters/membermatters .
+	docker buildx build --platform linux/amd64 --load -t hub.comcast.net/k8s-eng/ravel:${TAG} -f Dockerfile .

pkg/bgp/worker.go

@@ -19,6 +19,11 @@ const (
 	addrKindIPV6 = "ipv6"
 )
 
+// TODO - remove when not pinning to debug
+func init() {
+	log.SetLevel(log.DebugLevel)
+}
+
 // BGPWorker describes a BGP worker that can advertise BGP routes and communities
 type BGPWorker interface {
 	Start() error
@@ -280,7 +285,7 @@ func (b *bgpserver) periodic() {
 	queueDepthTicker := time.NewTicker(60 * time.Second)
 	defer queueDepthTicker.Stop()
 
-	bgpInterval := 2000 * time.Millisecond
+	bgpInterval := time.Second * 2
 	bgpTicker := time.NewTicker(bgpInterval)
 	defer bgpTicker.Stop()
 
@@ -528,14 +533,15 @@ func (b *bgpserver) performReconfigure() {
 	addressesV4, addressesV6, err := b.ipDevices.Get()
 	if err != nil {
 		b.metrics.Reconfigure("error", time.Now().Sub(start))
-		log.Infof("bgp: unable to compare configurations with error %v\n", err)
+		log.Errorf("bgp: unable to compare configurations with error %v\n", err)
 		return
 	}
 
 	// splice together to compare against the internal state of configs
 	// addresses is sorted within the CheckConfigParity function
 	addresses := append(addressesV4, addressesV6...)
 
+	log.Debugln("CheckConfigParity: bgpserver passing in these addresses:", addresses)
 	// compare configurations and apply new IPVS rules if they're different
 	same, err := b.ipvs.CheckConfigParity(b.nodes, b.config, addresses, b.configReady())
 	if err != nil {

pkg/director/director.go

@@ -21,6 +21,11 @@ const (
 	colocationModeIPVS     = "ipvs"
 )
 
+// TODO - remove when not pinning to debug
+func init() {
+	logrus.SetLevel(logrus.DebugLevel)
+}
+
 // TODO: instant startup
 
 // A director is the control flow for kube2ipvs. It can only be started once, and it can only be stopped once.
@@ -293,7 +298,8 @@ func (d *director) arps() {
 func (d *director) periodic() {
 
 	// reconfig ipvs
-	checkInterval := 100 * time.Millisecond
+	checkInterval := time.Second // reduced by eg 11/9/21
+	// checkInterval := 100 * time.Millisecond
 	t := time.NewTicker(checkInterval)
 	d.logger.Infof("starting periodic ticker. config check %v", checkInterval)
 
@@ -372,6 +378,7 @@ func (d *director) applyConf(force bool) error {
 		// splice together to compare against the internal state of configs
 		// addresses is sorted within the CheckConfigParity function
 		addresses := append(addressesV4, addressesV6...)
+		log.Debugln("CheckConfigParity: director passing in these addresses:", addresses)
 
 		same, err := d.ipvs.CheckConfigParity(d.nodes, d.config, addresses, d.configReady())
 		if err != nil {

pkg/realserver/configure.go

@@ -26,6 +26,11 @@ type RealServer interface {
 	Stop() error
 }
 
+// TODO - remove
+func init() {
+	log.SetLevel(log.DebugLevel)
+}
+
 // realserver is responsible for managing iptables
 type realserver struct {
 	sync.Mutex

pkg/system/ip.go

@@ -255,7 +255,7 @@ func (i *ipManager) get() ([]string, []string, error) {
 		// return nil, nil, fmt.Errorf("ipManager: error running shell command ip -details link show | grep -B 2 dummy: %+v", err)
 		return nil, nil, fmt.Errorf("ipManager: error running shell command ip link show | grep -B 2 dummy: %+v", err)
 	}
-	log.Debugln("ipManager: get() done fetching dummy interfaces. parsing address data")
+	log.Debugln("ipManager: get() done fetching dummy interfaces. parsing address data:", iFaces)
 
 	// split them into v4 or v6 addresses
 	return i.parseAddressData(iFaces)
@@ -370,11 +370,6 @@ func (i *ipManager) retrieveDummyIFaces() ([]string, error) {
 		log.Infoln("retrieveDummyIFaces took", runDuration)
 	}()
 
-	// DEBUG
-	// DEBUG
-	// DEBUG
-	log.SetLevel(log.DebugLevel) // TODO - remove
-
 	log.Debugln("ipManager: Retrieving dummy interfaces. Waiting to lock interfaceMu...")
 
 	// mutex this operation to prevent overlapping queries
@@ -452,7 +447,7 @@ func (i *ipManager) retrieveDummyIFaces() ([]string, error) {
 			// it errs exit 1. Return no ifaces
 			return []string{}, err
 		}
-		log.Infoln("ipManager: found no dummy interfaces")
+		log.Infoln("ipManager: found no dummy interfaces because of error while waiting for c2 to complete", err)
 		return []string{}, nil
 	}
 	log.Debugln("ipManager: process 2 completed")
@@ -464,7 +459,7 @@ func (i *ipManager) retrieveDummyIFaces() ([]string, error) {
 
 	// list over the interfaces parsed from CLI output and glob them up
 	iFaces := []string{}
-	b2SplFromLines := strings.Split(string(b2.Bytes()), "\n")
+	b2SplFromLines := strings.Split(b2.String(), "\n")
 	for _, l := range b2SplFromLines {
 		if strings.Contains(l, "mtu") {
 			awked := strings.Split(l, " ")

pkg/system/ipvs.go

@@ -23,6 +23,11 @@ const (
 	colocationModeIPVS     = "ipvs"
 )
 
+// TODO - remove when not pinning to debug
+func init() {
+	log.SetLevel(log.DebugLevel)
+}
+
 // IPVS is an interface for getting and setting IPVS configurations
 type IPVS interface {
 	Get() ([]string, error)
@@ -183,6 +188,17 @@ func (i *ipvs) generateRules(nodes types.NodesList, config *types.ClusterConfig)
 
 		// Add rules for Frontend ipvsadm
 		for port, serviceConfig := range ports {
+
+			log.Debugln("ipvs: The scheduler for service", serviceConfig.Service, serviceConfig.PortName, "is set to", serviceConfig.IPVSOptions.Scheduler())
+			log.Debugln("ipvs: The raw scheduler for service", serviceConfig.Service, serviceConfig.PortName, "is set to", serviceConfig.IPVSOptions.RawScheduler)
+
+			// If we have the scheduler set to `mh`, and flags are blank, then set flag-1,flag-2.
+			// This prevents dropped packets when maglev is used.
+			if serviceConfig.IPVSOptions.Scheduler() == "mh" && serviceConfig.IPVSOptions.Flags == "" {
+				log.Infoln("ipvs: Assuming flag-1,flag-2 for mh scheduler without flags.  Name:", serviceConfig.Service)
+				serviceConfig.IPVSOptions.Flags = "flag-1,flag-2"
+			}
+
 			log.Debugln("ipvs: generating ipvs rule for", port, serviceConfig)
 			// set rules for tcp / udp
 			if serviceConfig.TCPEnabled {
@@ -230,7 +246,7 @@ func (i *ipvs) generateRules(nodes types.NodesList, config *types.ClusterConfig)
 	for _, node := range nodes {
 		eligible, reason := node.IsEligibleBackendV4(config.NodeLabels, i.nodeIP, i.ignoreCordon)
 		if !eligible {
-			log.Debugln("ipvs: node %s deemed ineligible. %v", i.nodeIP, reason)
+			log.Debugf("ipvs: node %s deemed ineligible. %v", i.nodeIP, reason)
 			continue
 		}
 		eligibleNodes = append(eligibleNodes, node)
@@ -300,6 +316,15 @@ func (i *ipvs) generateRulesV6(nodes types.NodesList, config *types.ClusterConfi
 	for vip, ports := range config.Config6 {
 		// Add rules for Frontend ipvsadm as tcp / udp
 		for port, serviceConfig := range ports {
+
+			// If we have the scheduler set to `mh`, and flags are blank, then set flag-1,flag-2.
+			// This prevents dropped packets when maglev is used.
+			if serviceConfig.IPVSOptions.Scheduler() == "mh" && serviceConfig.IPVSOptions.Flags == "" {
+				serviceConfig.IPVSOptions.Flags = "flag-1,flag-2"
+				log.Debugln("v6 ipvs: The scheduler for service", serviceConfig.Service, serviceConfig.PortName, "is set to", serviceConfig.IPVSOptions.Scheduler())
+				log.Debugln("v6 ipvs: The raw scheduler for service", serviceConfig.Service, serviceConfig.PortName, "is set to", serviceConfig.IPVSOptions.RawScheduler)
+			}
+
 			// set rules for tcp / udp
 			if serviceConfig.TCPEnabled {
 				rule := fmt.Sprintf(
@@ -635,23 +660,32 @@ func (i *ipvs) CheckConfigParity(nodes types.NodesList, config *types.ClusterCon
 	// pull existing ipvs configurations
 	ipvsConfigured, err := i.Get()
 	if err != nil {
+		log.Debugln("CheckConfigParity: ipvsConfigured had an error.  not equal")
 		return false, err
 	}
 
 	// generate desired ipvs configurations
 	ipvsGenerated, err := i.generateRules(nodes, config)
 	if err != nil {
+		log.Debugln("CheckConfigParity: error when generating rules.  not equal")
 		return false, fmt.Errorf("generating IPVS rules: %v", err)
 	}
 
 	// compare and return
 	// XXX this might not be platform-independent...
 	sort.Sort(sort.StringSlice(addresses))
 	if !reflect.DeepEqual(vips, addresses) {
+		log.Debugln("CheckConfigParity: deep equal between vips and addresses NOT EQUAL")
+		log.Debugln("CheckConfigParity: VIPS values:", vips)
+		log.Debugln("CheckConfigParity: Addresses values:", addresses)
 		return false, nil
 	}
 
-	return ipvsEquality(ipvsConfigured, ipvsGenerated, newConfig), nil
+	isEqual, err := ipvsEquality(ipvsConfigured, ipvsGenerated, newConfig), nil
+	if !isEqual {
+		log.Debugln("CheckConfigParity: ivsEquality was not equal")
+	}
+	return isEqual, err
 }
 
 // Equality for the IPVS IP addresses currently existing (ipvsConfigured)
@@ -663,11 +697,15 @@ func (i *ipvs) CheckConfigParity(nodes types.NodesList, config *types.ClusterCon
 // don't appear the same way in each array.
 func ipvsEquality(ipvsConfigured []string, ipvsGenerated []string, newConfig bool) bool {
 	if len(ipvsConfigured) != len(ipvsGenerated) {
+		log.Debugln("ipvsEquality: evaluated FALSE due to number of generated vs configured rules")
 		return false
 	}
 	for _, existing := range ipvsConfigured {
 		found := false
+		var desiredDebug string
 		for i, desired := range ipvsGenerated {
+			desiredDebug = desired
+			log.Debugf("ipvsEquality: comparing existing %s with genreated %s \r\n", existing, desired)
 			// If it's a brand new configuration, weight don't matter, otherwise, they do
 			// weights only appear on "-a" rules
 			if newConfig && strings.HasPrefix(desired, "-a") {
@@ -686,13 +724,16 @@ func ipvsEquality(ipvsConfigured []string, ipvsGenerated []string, newConfig boo
 		}
 		if !found {
 			// the IP address represented by value of "existing" isn't in desired IPs
+			log.Debugln("ipvsEquality: evaluated false after looping over desired (generated) ipvs rules.  the not found desired item was", desiredDebug)
 			return false
 		}
 	}
 	if len(ipvsGenerated) > 0 {
 		// There's a new IP address desired that isn't configured
+		log.Debugln("ipvsEquality: ipvsGenerated count is more than 0")
 		return false
 	}
+	log.Debugln("ipvsEquality: generated and configured rules evaluated as equal")
 	return true
 }
 

pkg/types/clusterconfig.go

@@ -131,7 +131,7 @@ type IPVSOptions struct {
 // Scheduler returns a scheduler
 func (i *IPVSOptions) Scheduler() string {
 	var scheduler string
-	switch i.RawScheduler {
+	switch strings.TrimSpace(strings.ToLower(i.RawScheduler)) {
 	case "rr":
 		scheduler = "rr"
 	case "wrr":
@@ -149,7 +149,7 @@ func (i *IPVSOptions) Scheduler() string {
 	default:
 		// not supported:  lblc, lblcr, sed, nq
 		if len(i.RawScheduler) > 0 {
-			log.Errorln("Invalid scheduler specified: %s.  Using weighted round robin...", i.RawScheduler)
+			log.Errorln("ipvs: Invalid scheduler specified in IPVSOptions: %s.  Using weighted round robin...", i.RawScheduler)
 		}
 		scheduler = "wrr"
 	}