diff --git a/pkg/globalnet/controllers/base_controllers.go b/pkg/globalnet/controllers/base_controllers.go index 4dab497ae..e9fe9b35d 100644 --- a/pkg/globalnet/controllers/base_controllers.go +++ b/pkg/globalnet/controllers/base_controllers.go @@ -29,7 +29,7 @@ import ( "github.com/submariner-io/admiral/pkg/federate" "github.com/submariner-io/admiral/pkg/util" submarinerv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1" - iptiface "github.com/submariner-io/submariner/pkg/globalnet/controllers/iptables" + pfiface "github.com/submariner-io/submariner/pkg/globalnet/controllers/packetfilter" "github.com/submariner-io/submariner/pkg/ipam" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -60,11 +60,11 @@ func newBaseSyncerController() *baseSyncerController { } } -func newBaseIPAllocationController(pool *ipam.IPPool, iptIface iptiface.Interface) *baseIPAllocationController { +func newBaseIPAllocationController(pool *ipam.IPPool, pfIface pfiface.Interface) *baseIPAllocationController { return &baseIPAllocationController{ baseSyncerController: newBaseSyncerController(), pool: pool, - iptIface: iptIface, + pfIface: pfIface, } } diff --git a/pkg/globalnet/controllers/cluster_egressip_controller.go b/pkg/globalnet/controllers/cluster_egressip_controller.go index 1497f75bb..911668d0a 100644 --- a/pkg/globalnet/controllers/cluster_egressip_controller.go +++ b/pkg/globalnet/controllers/cluster_egressip_controller.go @@ -29,7 +29,7 @@ import ( "github.com/submariner-io/admiral/pkg/util" submarinerv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1" "github.com/submariner-io/submariner/pkg/globalnet/constants" - "github.com/submariner-io/submariner/pkg/globalnet/controllers/iptables" + "github.com/submariner-io/submariner/pkg/globalnet/controllers/packetfilter" "github.com/submariner-io/submariner/pkg/globalnet/metrics" "github.com/submariner-io/submariner/pkg/ipam" corev1 "k8s.io/api/core/v1" @@ -49,13 +49,13 @@ func NewClusterGlobalEgressIPController(config *syncer.ResourceSyncerConfig, loc logger.Info("Creating ClusterGlobalEgressIP controller") - iptIface, err := iptables.New() + pfIface, err := packetfilter.New() if err != nil { return nil, errors.WithMessage(err, "error creating the IPTablesInterface handler") } controller := &clusterGlobalEgressIPController{ - baseIPAllocationController: newBaseIPAllocationController(pool, iptIface), + baseIPAllocationController: newBaseIPAllocationController(pool, pfIface), localSubnets: localSubnets, } @@ -215,7 +215,7 @@ func (c *clusterGlobalEgressIPController) flushClusterGlobalEgressRules(allocate func (c *clusterGlobalEgressIPController) deleteClusterGlobalEgressRules(srcIPList []string, snatIP string) error { for _, srcIP := range srcIPList { - if err := c.iptIface.RemoveClusterEgressRules(srcIP, snatIP, globalNetIPTableMark); err != nil { + if err := c.pfIface.RemoveClusterEgressRules(srcIP, snatIP, globalNetIPTableMark); err != nil { return err //nolint:wrapcheck // Let the caller wrap it } } @@ -228,7 +228,7 @@ func (c *clusterGlobalEgressIPController) programClusterGlobalEgressRules(alloca egressRulesProgrammed := []string{} for _, srcIP := range c.localSubnets { - if err := c.iptIface.AddClusterEgressRules(srcIP, snatIP, globalNetIPTableMark); err != nil { + if err := c.pfIface.AddClusterEgressRules(srcIP, snatIP, globalNetIPTableMark); err != nil { _ = c.deleteClusterGlobalEgressRules(egressRulesProgrammed, snatIP) return err //nolint:wrapcheck // Let the caller wrap it diff --git a/pkg/globalnet/controllers/cluster_egressip_controller_test.go b/pkg/globalnet/controllers/cluster_egressip_controller_test.go index 22e2f1e16..3655e7627 100644 --- a/pkg/globalnet/controllers/cluster_egressip_controller_test.go +++ b/pkg/globalnet/controllers/cluster_egressip_controller_test.go @@ -30,6 +30,7 @@ import ( "github.com/submariner-io/submariner/pkg/globalnet/constants" "github.com/submariner-io/submariner/pkg/globalnet/controllers" "github.com/submariner-io/submariner/pkg/ipam" + "github.com/submariner-io/submariner/pkg/packetfilter" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -39,7 +40,7 @@ var _ = Describe("ClusterGlobalEgressIP controller", func() { When("the well-known ClusterGlobalEgressIP does not exist on startup", func() { It("should create it and allocate the default number of global IPs", func() { t.awaitClusterGlobalEgressIPStatusAllocated(controllers.DefaultNumberOfClusterEgressIPs) - t.awaitIPTableRules(getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs...) + t.awaitPacketFilterRules(getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs...) }) }) @@ -74,7 +75,7 @@ var _ = Describe("ClusterGlobalEgressIP controller", func() { }) It("should program the necessary IP table rules for the allocated IPs", func() { - t.awaitIPTableRules(existing.Status.AllocatedIPs...) + t.awaitPacketFilterRules(existing.Status.AllocatedIPs...) }) }) @@ -87,12 +88,12 @@ var _ = Describe("ClusterGlobalEgressIP controller", func() { It("should reallocate the global IPs", func() { t.awaitClusterGlobalEgressIPStatusAllocated(*existing.Spec.NumberOfIPs) - t.awaitIPTableRules(getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs...) + t.awaitPacketFilterRules(getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs...) }) It("should release the previously allocated IPs", func() { t.awaitIPsReleasedFromPool(existing.Status.AllocatedIPs...) - t.awaitNoIPTableRules(existing.Status.AllocatedIPs...) + t.awaitNoPacketFilterRules(existing.Status.AllocatedIPs...) }) }) @@ -121,14 +122,14 @@ var _ = Describe("ClusterGlobalEgressIP controller", func() { Status: metav1.ConditionTrue, }) - t.awaitIPTableRules(getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs...) + t.awaitPacketFilterRules(getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs...) }) }) Context("and programming the IP table rules fails", func() { BeforeEach(func() { t.createClusterGlobalEgressIP(existing) - t.ipt.AddFailOnAppendRuleMatcher(ContainSubstring(existing.Status.AllocatedIPs[0])) + t.pFilter.AddFailOnAppendRuleMatcher(ContainSubstring(existing.Status.AllocatedIPs[0])) }) It("should reallocate the global IPs", func() { @@ -142,7 +143,7 @@ var _ = Describe("ClusterGlobalEgressIP controller", func() { }) allocatedIPs := getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs - t.awaitIPTableRules(allocatedIPs...) + t.awaitPacketFilterRules(allocatedIPs...) t.awaitIPsReleasedFromPool(existing.Status.AllocatedIPs...) }) }) @@ -225,12 +226,12 @@ var _ = Describe("ClusterGlobalEgressIP controller", func() { It("should reallocate the global IPs", func() { t.awaitClusterGlobalEgressIPStatusAllocated(numberOfIPs) - t.awaitIPTableRules(getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs...) + t.awaitPacketFilterRules(getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs...) }) It("should release the previously allocated IPs", func() { t.awaitIPsReleasedFromPool(existing.Status.AllocatedIPs...) - t.awaitNoIPTableRules(existing.Status.AllocatedIPs...) + t.awaitNoPacketFilterRules(existing.Status.AllocatedIPs...) }) }) @@ -241,12 +242,12 @@ var _ = Describe("ClusterGlobalEgressIP controller", func() { It("should reallocate the global IPs", func() { t.awaitClusterGlobalEgressIPStatusAllocated(numberOfIPs) - t.awaitIPTableRules(getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs...) + t.awaitPacketFilterRules(getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs...) }) It("should release the previously allocated IPs", func() { t.awaitIPsReleasedFromPool(existing.Status.AllocatedIPs...) - t.awaitNoIPTableRules(existing.Status.AllocatedIPs...) + t.awaitNoPacketFilterRules(existing.Status.AllocatedIPs...) }) }) @@ -265,31 +266,31 @@ var _ = Describe("ClusterGlobalEgressIP controller", func() { It("should release the previously allocated IPs", func() { t.awaitIPsReleasedFromPool(existing.Status.AllocatedIPs...) - t.awaitNoIPTableRules(existing.Status.AllocatedIPs...) + t.awaitNoPacketFilterRules(existing.Status.AllocatedIPs...) }) }) Context("and IP tables cleanup of previously allocated IPs initially fails", func() { BeforeEach(func() { numberOfIPs = *existing.Spec.NumberOfIPs + 1 - t.ipt.AddFailOnDeleteRuleMatcher(ContainSubstring(existing.Status.AllocatedIPs[0])) + t.pFilter.AddFailOnDeleteRuleMatcher(ContainSubstring(existing.Status.AllocatedIPs[0])) }) It("should eventually cleanup the IP tables and reallocate", func() { - t.awaitNoIPTableRules(existing.Status.AllocatedIPs...) + t.awaitNoPacketFilterRules(existing.Status.AllocatedIPs...) t.awaitClusterGlobalEgressIPStatusAllocated(numberOfIPs) - t.awaitIPTableRules(getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs...) + t.awaitPacketFilterRules(getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs...) }) }) Context("and programming of IP tables initially fails", func() { BeforeEach(func() { numberOfIPs = *existing.Spec.NumberOfIPs + 1 - t.ipt.AddFailOnAppendRuleMatcher(Not(ContainSubstring(existing.Status.AllocatedIPs[0]))) + t.pFilter.AddFailOnAppendRuleMatcher(Not(ContainSubstring(existing.Status.AllocatedIPs[0]))) }) It("should eventually reallocate the global IPs", func() { - t.awaitNoIPTableRules(existing.Status.AllocatedIPs...) + t.awaitNoPacketFilterRules(existing.Status.AllocatedIPs...) t.awaitEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName, numberOfIPs, metav1.Condition{ Type: string(submarinerv1.GlobalEgressIPAllocated), Status: metav1.ConditionFalse, @@ -298,7 +299,7 @@ var _ = Describe("ClusterGlobalEgressIP controller", func() { Type: string(submarinerv1.GlobalEgressIPAllocated), Status: metav1.ConditionTrue, }) - t.awaitIPTableRules(getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs...) + t.awaitPacketFilterRules(getGlobalEgressIPStatus(t.clusterGlobalEgressIPs, constants.ClusterGlobalEgressIPName).AllocatedIPs...) }) }) @@ -393,14 +394,14 @@ func (t *clusterGlobalEgressIPControllerTestDriver) start() { Expect(t.controller.Start()).To(Succeed()) } -func (t *clusterGlobalEgressIPControllerTestDriver) awaitIPTableRules(ips ...string) { - t.ipt.AwaitRule("nat", constants.SmGlobalnetEgressChainForCluster, ContainSubstring(getSNATAddress(ips...))) +func (t *clusterGlobalEgressIPControllerTestDriver) awaitPacketFilterRules(ips ...string) { + t.pFilter.AwaitRule(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChainForCluster, ContainSubstring(getSNATAddress(ips...))) for _, localSubnet := range t.localSubnets { - t.ipt.AwaitRule("nat", constants.SmGlobalnetEgressChainForCluster, ContainSubstring(localSubnet)) + t.pFilter.AwaitRule(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChainForCluster, ContainSubstring(localSubnet)) } } -func (t *clusterGlobalEgressIPControllerTestDriver) awaitNoIPTableRules(ips ...string) { - t.ipt.AwaitNoRule("nat", constants.SmGlobalnetEgressChainForCluster, ContainSubstring(getSNATAddress(ips...))) +func (t *clusterGlobalEgressIPControllerTestDriver) awaitNoPacketFilterRules(ips ...string) { + t.pFilter.AwaitNoRule(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChainForCluster, ContainSubstring(getSNATAddress(ips...))) } diff --git a/pkg/globalnet/controllers/controllers_suite_test.go b/pkg/globalnet/controllers/controllers_suite_test.go index d260b51f9..af3b0df08 100644 --- a/pkg/globalnet/controllers/controllers_suite_test.go +++ b/pkg/globalnet/controllers/controllers_suite_test.go @@ -38,8 +38,8 @@ import ( "github.com/submariner-io/submariner/pkg/ipam" "github.com/submariner-io/submariner/pkg/ipset" fakeIPSet "github.com/submariner-io/submariner/pkg/ipset/fake" - "github.com/submariner-io/submariner/pkg/iptables" - fakeIPT "github.com/submariner-io/submariner/pkg/iptables/fake" + "github.com/submariner-io/submariner/pkg/packetfilter" + fakePF "github.com/submariner-io/submariner/pkg/packetfilter/fake" routeAgent "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -88,7 +88,7 @@ type testDriverBase struct { restMapper meta.RESTMapper dynClient *dynamicfake.FakeDynamicClient scheme *runtime.Scheme - ipt *fakeIPT.IPTables + pFilter *fakePF.PacketFilter ipSet *fakeIPSet.IPSet pool *ipam.IPPool localSubnets []string @@ -109,12 +109,11 @@ func newTestDriverBase() *testDriverBase { restMapper: test.GetRESTMapperFor(&submarinerv1.Endpoint{}, &corev1.Service{}, &corev1.Node{}, &corev1.Pod{}, &corev1.Endpoints{}, &submarinerv1.GlobalEgressIP{}, &submarinerv1.ClusterGlobalEgressIP{}, &submarinerv1.GlobalIngressIP{}, &mcsv1a1.ServiceExport{}), scheme: runtime.NewScheme(), - ipt: fakeIPT.New(), + pFilter: fakePF.New(), ipSet: fakeIPSet.New(), globalCIDR: localCIDR, localSubnets: []string{}, } - Expect(mcsv1a1.AddToScheme(t.scheme)).To(Succeed()) Expect(submarinerv1.AddToScheme(t.scheme)).To(Succeed()) Expect(corev1.AddToScheme(t.scheme)).To(Succeed()) @@ -140,10 +139,6 @@ func newTestDriverBase() *testDriverBase { t.nodes = t.dynClient.Resource(*test.GetGroupVersionResourceFor(t.restMapper, &corev1.Node{})) - iptables.NewFunc = func() (iptables.Interface, error) { - return t.ipt, nil - } - ipset.NewFunc = func() ipset.Interface { return t.ipSet } @@ -153,8 +148,6 @@ func newTestDriverBase() *testDriverBase { func (t *testDriverBase) afterEach() { t.controller.Stop() - - iptables.NewFunc = nil } func (t *testDriverBase) verifyIPsReservedInPool(ips ...string) { @@ -231,8 +224,10 @@ func (t *testDriverBase) createServiceExport(s *corev1.Service) { }) } -func (t *testDriverBase) createIPTableChain(table, chain string) { - _ = t.ipt.NewChain(table, chain) +func (t *testDriverBase) createPFilterChain(table packetfilter.TableType, chain string) { + _ = t.pFilter.CreateChainIfNotExists(table, &packetfilter.Chain{ + Name: chain, + }) } func (t *testDriverBase) getGlobalIngressIPStatus(name string) *submarinerv1.GlobalIngressIPStatus { diff --git a/pkg/globalnet/controllers/gateway_monitor.go b/pkg/globalnet/controllers/gateway_monitor.go index b35dc5ff6..ab5421780 100644 --- a/pkg/globalnet/controllers/gateway_monitor.go +++ b/pkg/globalnet/controllers/gateway_monitor.go @@ -21,7 +21,6 @@ package controllers import ( "context" "os" - "strings" "sync/atomic" "time" @@ -35,8 +34,8 @@ import ( "github.com/submariner-io/submariner/pkg/event/controller" "github.com/submariner-io/submariner/pkg/globalnet/constants" "github.com/submariner-io/submariner/pkg/ipam" - "github.com/submariner-io/submariner/pkg/iptables" "github.com/submariner-io/submariner/pkg/netlink" + "github.com/submariner-io/submariner/pkg/packetfilter" routeAgent "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -86,9 +85,9 @@ func NewGatewayMonitor(config *GatewayMonitorConfig) (Interface, error) { var err error - gatewayMonitor.ipt, err = iptables.New() + gatewayMonitor.pFilter, err = packetfilter.New() if err != nil { - return nil, errors.Wrap(err, "error creating IP tables") + return nil, errors.Wrap(err, "error creating packetfilter") } nodeName, ok := os.LookupEnv("NODE_NAME") @@ -441,8 +440,11 @@ func (g *gatewayMonitor) stopControllers(ctx context.Context, clearGlobalnetChai func (g *gatewayMonitor) createGlobalNetMarkingChain() error { logger.V(log.DEBUG).Infof("Install/ensure %s chain exists", constants.SmGlobalnetMarkChain) - if err := g.ipt.CreateChainIfNotExists("nat", constants.SmGlobalnetMarkChain); err != nil { - return errors.Wrapf(err, "error creating iptables chain %s", constants.SmGlobalnetMarkChain) + if err := g.pFilter.CreateChainIfNotExists(packetfilter.TableTypeNAT, + &packetfilter.Chain{ + Name: constants.SmGlobalnetMarkChain, + }); err != nil { + return errors.Wrapf(err, "error creating packetfilter chain %s", constants.SmGlobalnetMarkChain) } return nil @@ -452,94 +454,142 @@ func (g *gatewayMonitor) createGlobalNetMarkingChain() error { func (g *gatewayMonitor) createGlobalnetChains() error { logger.V(log.DEBUG).Infof("Install/ensure %s chain exists", constants.SmGlobalnetIngressChain) - if err := g.ipt.CreateChainIfNotExists("nat", constants.SmGlobalnetIngressChain); err != nil { - return errors.Wrapf(err, "error creating iptables chain %s", constants.SmGlobalnetIngressChain) + chain := packetfilter.ChainIPHook{ + Name: constants.SmGlobalnetIngressChain, + Type: packetfilter.ChainTypeNAT, + Hook: packetfilter.ChainHookPrerouting, + Priority: packetfilter.ChainPriorityFirst, } + if err := g.pFilter.CreateIPHookChainIfNotExists(&chain); err != nil { + return errors.Wrapf(err, "error creating IPHook chain %s", constants.SmGlobalnetIngressChain) + } + + logger.V(log.DEBUG).Infof("Install/ensure %s chain exists", routeAgent.SmPostRoutingChain) - forwardToSubGlobalNetChain := []string{"-j", constants.SmGlobalnetIngressChain} - if err := g.ipt.PrependUnique("nat", "PREROUTING", forwardToSubGlobalNetChain); err != nil { - logger.Errorf(err, "Error appending iptables rule %q", strings.Join(forwardToSubGlobalNetChain, " ")) + chain = packetfilter.ChainIPHook{ + Name: routeAgent.SmPostRoutingChain, + Type: packetfilter.ChainTypeNAT, + Hook: packetfilter.ChainHookPostrouting, + Priority: packetfilter.ChainPriorityFirst, + } + if err := g.pFilter.CreateIPHookChainIfNotExists(&chain); err != nil { + return errors.Wrap(err, "error creating IPHook chain") } logger.V(log.DEBUG).Infof("Install/ensure %s chain exists", constants.SmGlobalnetEgressChain) - if err := g.ipt.CreateChainIfNotExists("nat", constants.SmGlobalnetEgressChain); err != nil { - return errors.Wrapf(err, "error creating iptables chain %s", constants.SmGlobalnetEgressChain) + if err := g.pFilter.CreateChainIfNotExists(packetfilter.TableTypeNAT, + &packetfilter.Chain{ + Name: constants.SmGlobalnetEgressChain, + }); err != nil { + return errors.Wrapf(err, "error creating packetfilter chain %s", constants.SmGlobalnetEgressChain) } - logger.V(log.DEBUG).Infof("Install/ensure %s chain exists", routeAgent.SmPostRoutingChain) - - if err := g.ipt.CreateChainIfNotExists("nat", routeAgent.SmPostRoutingChain); err != nil { - return errors.Wrapf(err, "error creating iptables chain %s", routeAgent.SmPostRoutingChain) + ruleSpec := packetfilter.Rule{ + TargetChain: constants.SmGlobalnetEgressChain, + Action: packetfilter.RuleActionJump, } - forwardToSubGlobalNetChain = []string{"-j", constants.SmGlobalnetEgressChain} - if err := g.ipt.PrependUnique("nat", routeAgent.SmPostRoutingChain, forwardToSubGlobalNetChain); err != nil { - logger.Errorf(err, "Error inserting iptables rule %q", strings.Join(forwardToSubGlobalNetChain, " ")) + if err := g.pFilter.PrependUnique(packetfilter.TableTypeNAT, routeAgent.SmPostRoutingChain, &ruleSpec); err != nil { + return errors.Wrapf(err, "Error prepending rule %+v", ruleSpec) } if err := g.createGlobalNetMarkingChain(); err != nil { return err } - forwardToSubGlobalNetChain = []string{"-j", constants.SmGlobalnetMarkChain} - if err := g.ipt.PrependUnique("nat", constants.SmGlobalnetEgressChain, forwardToSubGlobalNetChain); err != nil { - logger.Errorf(err, "Error inserting iptables rule %q", strings.Join(forwardToSubGlobalNetChain, " ")) + ruleSpec = packetfilter.Rule{ + TargetChain: constants.SmGlobalnetMarkChain, + Action: packetfilter.RuleActionJump, + } + + if err := g.pFilter.PrependUnique(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChain, &ruleSpec); err != nil { + logger.Errorf(err, "Error inserting iptables rule %+v", ruleSpec) } logger.V(log.DEBUG).Infof("Install/ensure %s chain exists", constants.SmGlobalnetEgressChainForPods) - if err := g.ipt.CreateChainIfNotExists("nat", constants.SmGlobalnetEgressChainForPods); err != nil { - return errors.Wrapf(err, "error creating iptables chain %s", constants.SmGlobalnetEgressChainForPods) + if err := g.pFilter.CreateChainIfNotExists(packetfilter.TableTypeNAT, + &packetfilter.Chain{ + Name: constants.SmGlobalnetEgressChainForPods, + }); err != nil { + return errors.Wrapf(err, "error creating packetfilter chain %s", constants.SmGlobalnetEgressChainForPods) } logger.V(log.DEBUG).Infof("Install/ensure %s chain exists", constants.SmGlobalnetEgressChainForHeadlessSvcPods) - if err := g.ipt.CreateChainIfNotExists("nat", constants.SmGlobalnetEgressChainForHeadlessSvcPods); err != nil { - return errors.Wrapf(err, "error creating iptables chain %s", constants.SmGlobalnetEgressChainForHeadlessSvcPods) + if err := g.pFilter.CreateChainIfNotExists(packetfilter.TableTypeNAT, + &packetfilter.Chain{ + Name: constants.SmGlobalnetEgressChainForHeadlessSvcPods, + }); err != nil { + return errors.Wrapf(err, "error creating packetfilter chain %s", constants.SmGlobalnetEgressChainForHeadlessSvcPods) } logger.V(log.DEBUG).Infof("Install/ensure %s chain exists", constants.SmGlobalnetEgressChainForHeadlessSvcEPs) - if err := g.ipt.CreateChainIfNotExists("nat", constants.SmGlobalnetEgressChainForHeadlessSvcEPs); err != nil { - return errors.Wrapf(err, "error creating iptables chain %s", constants.SmGlobalnetEgressChainForHeadlessSvcEPs) + if err := g.pFilter.CreateChainIfNotExists(packetfilter.TableTypeNAT, + &packetfilter.Chain{ + Name: constants.SmGlobalnetEgressChainForHeadlessSvcEPs, + }); err != nil { + return errors.Wrapf(err, "error creating packetfilter chain %s", constants.SmGlobalnetEgressChainForHeadlessSvcEPs) } logger.V(log.DEBUG).Infof("Install/ensure %s chain exists", constants.SmGlobalnetEgressChainForNamespace) - if err := g.ipt.CreateChainIfNotExists("nat", constants.SmGlobalnetEgressChainForNamespace); err != nil { - return errors.Wrapf(err, "error creating iptables chain %s", constants.SmGlobalnetEgressChainForNamespace) + if err := g.pFilter.CreateChainIfNotExists(packetfilter.TableTypeNAT, + &packetfilter.Chain{ + Name: constants.SmGlobalnetEgressChainForNamespace, + }); err != nil { + return errors.Wrapf(err, "error creating packetfilter chain %s", constants.SmGlobalnetEgressChainForNamespace) } logger.V(log.DEBUG).Infof("Install/ensure %s chain exists", constants.SmGlobalnetEgressChainForCluster) - if err := g.ipt.CreateChainIfNotExists("nat", constants.SmGlobalnetEgressChainForCluster); err != nil { - return errors.Wrapf(err, "error creating iptables chain %s", constants.SmGlobalnetEgressChainForCluster) + if err := g.pFilter.CreateChainIfNotExists(packetfilter.TableTypeNAT, + &packetfilter.Chain{ + Name: constants.SmGlobalnetEgressChainForCluster, + }); err != nil { + return errors.Wrapf(err, "error creating packetfilter chain %s", constants.SmGlobalnetEgressChainForCluster) } - forwardToSubGlobalNetChain = []string{"-j", constants.SmGlobalnetEgressChainForPods} - if err := g.ipt.InsertUnique("nat", constants.SmGlobalnetEgressChain, 2, forwardToSubGlobalNetChain); err != nil { - logger.Errorf(err, "Error inserting iptables rule %q", strings.Join(forwardToSubGlobalNetChain, " ")) + ruleSpec = packetfilter.Rule{ + TargetChain: constants.SmGlobalnetEgressChainForPods, + Action: packetfilter.RuleActionJump, + } + if err := g.pFilter.InsertUnique(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChain, 2, &ruleSpec); err != nil { + logger.Errorf(err, "Error inserting packetfilter rule %+v", ruleSpec) } - forwardToSubGlobalNetChain = []string{"-j", constants.SmGlobalnetEgressChainForHeadlessSvcPods} - if err := g.ipt.InsertUnique("nat", constants.SmGlobalnetEgressChain, 3, forwardToSubGlobalNetChain); err != nil { - logger.Errorf(err, "Error inserting iptables rule %q", strings.Join(forwardToSubGlobalNetChain, " ")) + ruleSpec = packetfilter.Rule{ + TargetChain: constants.SmGlobalnetEgressChainForHeadlessSvcPods, + Action: packetfilter.RuleActionJump, + } + if err := g.pFilter.InsertUnique(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChain, 3, &ruleSpec); err != nil { + logger.Errorf(err, "Error inserting packetfilter rule %+v", ruleSpec) } - forwardToSubGlobalNetChain = []string{"-j", constants.SmGlobalnetEgressChainForHeadlessSvcEPs} - if err := g.ipt.InsertUnique("nat", constants.SmGlobalnetEgressChain, 4, forwardToSubGlobalNetChain); err != nil { - logger.Errorf(err, "Error inserting iptables rule %q", strings.Join(forwardToSubGlobalNetChain, " ")) + ruleSpec = packetfilter.Rule{ + TargetChain: constants.SmGlobalnetEgressChainForHeadlessSvcEPs, + Action: packetfilter.RuleActionJump, + } + if err := g.pFilter.InsertUnique(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChain, 4, &ruleSpec); err != nil { + logger.Errorf(err, "Error inserting packetfilter rule %+v", ruleSpec) } - forwardToSubGlobalNetChain = []string{"-j", constants.SmGlobalnetEgressChainForNamespace} - if err := g.ipt.InsertUnique("nat", constants.SmGlobalnetEgressChain, 5, forwardToSubGlobalNetChain); err != nil { - logger.Errorf(err, "Error inserting iptables rule %q", strings.Join(forwardToSubGlobalNetChain, " ")) + ruleSpec = packetfilter.Rule{ + TargetChain: constants.SmGlobalnetEgressChainForNamespace, + Action: packetfilter.RuleActionJump, + } + if err := g.pFilter.InsertUnique(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChain, 5, &ruleSpec); err != nil { + logger.Errorf(err, "Error inserting packetfilter rule %+v", ruleSpec) } - forwardToSubGlobalNetChain = []string{"-j", constants.SmGlobalnetEgressChainForCluster} - if err := g.ipt.InsertUnique("nat", constants.SmGlobalnetEgressChain, 6, forwardToSubGlobalNetChain); err != nil { - logger.Errorf(err, "Error inserting iptables rule %q", strings.Join(forwardToSubGlobalNetChain, " ")) + ruleSpec = packetfilter.Rule{ + TargetChain: constants.SmGlobalnetEgressChainForCluster, + Action: packetfilter.RuleActionJump, + } + if err := g.pFilter.InsertUnique(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChain, 6, &ruleSpec); err != nil { + logger.Errorf(err, "Error inserting packetfilter rule %+v", ruleSpec) } return nil @@ -548,15 +598,15 @@ func (g *gatewayMonitor) createGlobalnetChains() error { func (g *gatewayMonitor) clearGlobalnetChains() { logger.Info("Active gateway migrated, flushing Globalnet chains.") - if err := g.ipt.ClearChain("nat", constants.SmGlobalnetIngressChain); err != nil { + if err := g.pFilter.ClearChain(packetfilter.TableTypeNAT, constants.SmGlobalnetIngressChain); err != nil { logger.Errorf(err, "Error while flushing rules in %s chain", constants.SmGlobalnetIngressChain) } - if err := g.ipt.ClearChain("nat", constants.SmGlobalnetEgressChain); err != nil { + if err := g.pFilter.ClearChain(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChain); err != nil { logger.Errorf(err, "Error while flushing rules in %s chain", constants.SmGlobalnetEgressChain) } - if err := g.ipt.ClearChain("nat", constants.SmGlobalnetMarkChain); err != nil { + if err := g.pFilter.ClearChain(packetfilter.TableTypeNAT, constants.SmGlobalnetMarkChain); err != nil { logger.Errorf(err, "Error while flushing rules in %s chain", constants.SmGlobalnetMarkChain) } } @@ -567,18 +617,22 @@ func (g *gatewayMonitor) markRemoteClusterTraffic(addRules bool, subnets ...stri continue } - ruleSpec := []string{"-d", subnet, "-j", "MARK", "--set-mark", globalNetIPTableMark} + ruleSpec := packetfilter.Rule{ + DestCIDR: subnet, + MarkValue: globalNetIPTableMark, + Action: packetfilter.RuleActionMark, + } if addRules { - logger.V(log.DEBUG).Infof("Marking traffic destined to remote cluster: %s", strings.Join(ruleSpec, " ")) + logger.V(log.DEBUG).Infof("Marking traffic destined to remote cluster: %+v", &ruleSpec) - if err := g.ipt.AppendUnique("nat", constants.SmGlobalnetMarkChain, ruleSpec...); err != nil { - logger.Errorf(err, "Error appending iptables rule \"%s\"", strings.Join(ruleSpec, " ")) + if err := g.pFilter.AppendUnique(packetfilter.TableTypeNAT, constants.SmGlobalnetMarkChain, &ruleSpec); err != nil { + logger.Errorf(err, "Error appending packetfilter rule %+v", &ruleSpec) } } else { - logger.V(log.DEBUG).Infof("Deleting rule that marks remote cluster traffic: %s", strings.Join(ruleSpec, " ")) - if err := g.ipt.Delete("nat", constants.SmGlobalnetMarkChain, ruleSpec...); err != nil { - logger.Errorf(err, "Error deleting iptables rule \"%s\"", strings.Join(ruleSpec, " ")) + logger.V(log.DEBUG).Infof("Deleting rule that marks remote cluster traffic: %+v", &ruleSpec) + if err := g.pFilter.Delete(packetfilter.TableTypeNAT, constants.SmGlobalnetMarkChain, &ruleSpec); err != nil { + logger.Errorf(err, "Error deleting iptables rule %+v", &ruleSpec) } } } diff --git a/pkg/globalnet/controllers/gateway_monitor_test.go b/pkg/globalnet/controllers/gateway_monitor_test.go index e57af4f42..e8e9876d1 100644 --- a/pkg/globalnet/controllers/gateway_monitor_test.go +++ b/pkg/globalnet/controllers/gateway_monitor_test.go @@ -33,6 +33,7 @@ import ( "github.com/submariner-io/submariner/pkg/globalnet/controllers" netlinkAPI "github.com/submariner-io/submariner/pkg/netlink" fakeNetlink "github.com/submariner-io/submariner/pkg/netlink/fake" + "github.com/submariner-io/submariner/pkg/packetfilter" routeAgent "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/dynamic" @@ -55,7 +56,7 @@ var _ = Describe("Endpoint monitoring", func() { JustBeforeEach(func() { t.createNode(nodeName, "", "") endpoint = t.createEndpoint(newEndpointSpec(clusterID, t.hostName, localCIDR)) - t.createIPTableChain("nat", kubeProxyIPTableChainName) + t.createPFilterChain(packetfilter.TableTypeNAT, kubeProxyIPTableChainName) }) It("should start the controllers", func() { @@ -188,10 +189,10 @@ var _ = Describe("Endpoint monitoring", func() { When("a remote Endpoint with non-overlapping CIDRs is created then removed", func() { It("should add/remove appropriate IP table rule(s)", func() { endpoint := t.createEndpoint(newEndpointSpec(remoteClusterID, t.hostName, remoteCIDR)) - t.ipt.AwaitRule("nat", constants.SmGlobalnetMarkChain, ContainSubstring(remoteCIDR)) + t.pFilter.AwaitRule(packetfilter.TableTypeNAT, constants.SmGlobalnetMarkChain, ContainSubstring(remoteCIDR)) Expect(t.endpoints.Delete(context.TODO(), endpoint.Name, metav1.DeleteOptions{})).To(Succeed()) - t.ipt.AwaitNoRule("nat", constants.SmGlobalnetMarkChain, ContainSubstring(remoteCIDR)) + t.pFilter.AwaitNoRule(packetfilter.TableTypeNAT, constants.SmGlobalnetMarkChain, ContainSubstring(remoteCIDR)) }) }) @@ -199,7 +200,7 @@ var _ = Describe("Endpoint monitoring", func() { It("should not add expected IP table rule(s)", func() { t.createEndpoint(newEndpointSpec(remoteClusterID, t.hostName, localCIDR)) time.Sleep(500 * time.Millisecond) - t.ipt.AwaitNoRule("nat", constants.SmGlobalnetMarkChain, ContainSubstring(localCIDR)) + t.pFilter.AwaitNoRule(packetfilter.TableTypeNAT, constants.SmGlobalnetMarkChain, ContainSubstring(localCIDR)) }) }) }) @@ -271,7 +272,7 @@ func (t *gatewayMonitorTestDriver) start() { Expect(err).To(Succeed()) Expect(t.controller.Start()).To(Succeed()) - t.ipt.AwaitChain("nat", constants.SmGlobalnetMarkChain) + t.pFilter.AwaitChain(packetfilter.TableTypeNAT, constants.SmGlobalnetMarkChain) } func (t *gatewayMonitorTestDriver) createEndpoint(spec *submarinerv1.EndpointSpec) *submarinerv1.Endpoint { @@ -301,16 +302,16 @@ func (t *gatewayMonitorTestDriver) ensureControllersStopped() { } func (t *gatewayMonitorTestDriver) awaitGlobalnetChains() { - t.ipt.AwaitChain("nat", constants.SmGlobalnetIngressChain) - t.ipt.AwaitChain("nat", constants.SmGlobalnetEgressChain) - t.ipt.AwaitChain("nat", routeAgent.SmPostRoutingChain) - t.ipt.AwaitChain("nat", constants.SmGlobalnetMarkChain) + t.pFilter.AwaitChain(packetfilter.TableTypeNAT, constants.SmGlobalnetIngressChain) + t.pFilter.AwaitChain(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChain) + t.pFilter.AwaitChain(packetfilter.TableTypeNAT, routeAgent.SmPostRoutingChain) + t.pFilter.AwaitChain(packetfilter.TableTypeNAT, constants.SmGlobalnetMarkChain) } func (t *gatewayMonitorTestDriver) awaitNoGlobalnetChains() { - t.ipt.AwaitNoChain("nat", constants.SmGlobalnetIngressChain) - t.ipt.AwaitNoChain("nat", constants.SmGlobalnetEgressChain) - t.ipt.AwaitNoChain("nat", constants.SmGlobalnetMarkChain) + t.pFilter.AwaitNoChain(packetfilter.TableTypeNAT, constants.SmGlobalnetIngressChain) + t.pFilter.AwaitNoChain(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChain) + t.pFilter.AwaitNoChain(packetfilter.TableTypeNAT, constants.SmGlobalnetMarkChain) } func newEndpointSpec(clusterID, hostname, subnet string) *submarinerv1.EndpointSpec { diff --git a/pkg/globalnet/controllers/global_egressip_controller.go b/pkg/globalnet/controllers/global_egressip_controller.go index f50b3feea..2e31234e7 100644 --- a/pkg/globalnet/controllers/global_egressip_controller.go +++ b/pkg/globalnet/controllers/global_egressip_controller.go @@ -30,7 +30,7 @@ import ( "github.com/submariner-io/admiral/pkg/util" "github.com/submariner-io/admiral/pkg/watcher" submarinerv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1" - "github.com/submariner-io/submariner/pkg/globalnet/controllers/iptables" + "github.com/submariner-io/submariner/pkg/globalnet/controllers/packetfilter" "github.com/submariner-io/submariner/pkg/globalnet/metrics" "github.com/submariner-io/submariner/pkg/ipam" "github.com/submariner-io/submariner/pkg/ipset" @@ -49,13 +49,13 @@ func NewGlobalEgressIPController(config *syncer.ResourceSyncerConfig, pool *ipam logger.Info("Creating GlobalEgressIP controller") - iptIface, err := iptables.New() + pfIface, err := packetfilter.New() if err != nil { - return nil, errors.WithMessage(err, "error creating the IPTablesInterface handler") + return nil, errors.WithMessage(err, "error creating the packetfilter Interface handler") } controller := &globalEgressIPController{ - baseIPAllocationController: newBaseIPAllocationController(pool, iptIface), + baseIPAllocationController: newBaseIPAllocationController(pool, pfIface), podWatchers: map[string]*egressPodWatcher{}, watcherConfig: watcher.Config{ RestMapper: config.RestMapper, @@ -182,13 +182,13 @@ func (c *globalEgressIPController) programGlobalEgressRules(key string, allocate snatIP := getTargetSNATIPaddress(allocatedIPs) if podSelector != nil { - if err := c.iptIface.AddEgressRulesForPods(key, namedIPSet.Name(), snatIP, globalNetIPTableMark); err != nil { - _ = c.iptIface.RemoveEgressRulesForPods(key, namedIPSet.Name(), snatIP, globalNetIPTableMark) + if err := c.pfIface.AddEgressRulesForPods(key, namedIPSet.Name(), snatIP, globalNetIPTableMark); err != nil { + _ = c.pfIface.RemoveEgressRulesForPods(key, namedIPSet.Name(), snatIP, globalNetIPTableMark) return err } } else { - if err := c.iptIface.AddEgressRulesForNamespace(key, namedIPSet.Name(), snatIP, globalNetIPTableMark); err != nil { - _ = c.iptIface.RemoveEgressRulesForNamespace(key, namedIPSet.Name(), snatIP, globalNetIPTableMark) + if err := c.pfIface.AddEgressRulesForNamespace(key, namedIPSet.Name(), snatIP, globalNetIPTableMark); err != nil { + _ = c.pfIface.RemoveEgressRulesForNamespace(key, namedIPSet.Name(), snatIP, globalNetIPTableMark) return err } } @@ -378,11 +378,11 @@ func (c *globalEgressIPController) flushGlobalEgressRulesAndReleaseIPs(key, ipSe return c.flushRulesAndReleaseIPs(key, numRequeues, func(allocatedIPs []string) error { metrics.RecordDeallocateGlobalEgressIPs(c.pool.GetCIDR(), len(allocatedIPs)) if globalEgressIP.Spec.PodSelector != nil { - return c.iptIface.RemoveEgressRulesForPods(key, ipSetName, + return c.pfIface.RemoveEgressRulesForPods(key, ipSetName, getTargetSNATIPaddress(allocatedIPs), globalNetIPTableMark) } - return c.iptIface.RemoveEgressRulesForNamespace(key, ipSetName, getTargetSNATIPaddress(allocatedIPs), globalNetIPTableMark) + return c.pfIface.RemoveEgressRulesForNamespace(key, ipSetName, getTargetSNATIPaddress(allocatedIPs), globalNetIPTableMark) }, globalEgressIP.Status.AllocatedIPs...) } diff --git a/pkg/globalnet/controllers/global_egressip_controller_test.go b/pkg/globalnet/controllers/global_egressip_controller_test.go index 45e9442b1..3c5bf8af4 100644 --- a/pkg/globalnet/controllers/global_egressip_controller_test.go +++ b/pkg/globalnet/controllers/global_egressip_controller_test.go @@ -31,6 +31,7 @@ import ( "github.com/submariner-io/submariner/pkg/globalnet/constants" "github.com/submariner-io/submariner/pkg/globalnet/controllers" "github.com/submariner-io/submariner/pkg/ipam" + "github.com/submariner-io/submariner/pkg/packetfilter" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -89,7 +90,7 @@ func testGlobalEgressIPCreated(t *globalEgressIPControllerTestDriver, podSelecto Context("with the NumberOfIPs unspecified", func() { It("should allocate one global IP and program the necessary IP table rules", func() { t.awaitGlobalEgressIPStatusAllocated(globalEgressIPName, 1) - t.awaitIPTableRules(egressChain, getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs...) + t.awaitPacketFilterRules(egressChain, getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs...) }) It("should start a Pod watcher", func() { @@ -105,7 +106,7 @@ func testGlobalEgressIPCreated(t *globalEgressIPControllerTestDriver, podSelecto It("should allocate the specified number of global IPs and program the necessary IP table rules", func() { t.awaitGlobalEgressIPStatusAllocated(globalEgressIPName, *numberOfIPs) - t.awaitIPTableRules(egressChain, getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs...) + t.awaitPacketFilterRules(egressChain, getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs...) }) }) @@ -145,7 +146,7 @@ func testGlobalEgressIPCreated(t *globalEgressIPControllerTestDriver, podSelecto Context("and programming the IP table rules initially fails", func() { BeforeEach(func() { - t.ipt.AddFailOnAppendRuleMatcher(Not(BeEmpty())) + t.pFilter.AddFailOnAppendRuleMatcher(Not(BeEmpty())) t.ipSet.AddFailOnCreateSetMatchers(Not(BeEmpty())) }) @@ -159,7 +160,7 @@ func testGlobalEgressIPCreated(t *globalEgressIPControllerTestDriver, podSelecto Status: metav1.ConditionTrue, }) - t.awaitIPTableRules(egressChain, getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs...) + t.awaitPacketFilterRules(egressChain, getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs...) t.watches.AwaitWatchStarted("pods") }) }) @@ -202,20 +203,20 @@ func testGlobalEgressIPCreated(t *globalEgressIPControllerTestDriver, podSelecto JustBeforeEach(func() { t.awaitGlobalEgressIPStatusAllocated(globalEgressIPName, 1) allocatedIPs = getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs - ipSetName = t.awaitIPTableRules(egressChain, allocatedIPs...) + ipSetName = t.awaitPacketFilterRules(egressChain, allocatedIPs...) }) It("should release the allocated global IPs and clean up the IP tables", func() { Expect(t.globalEgressIPs.Delete(context.TODO(), globalEgressIPName, metav1.DeleteOptions{})).To(Succeed()) t.awaitIPsReleasedFromPool(allocatedIPs...) t.ipSet.AwaitSetDeleted(ipSetName) - t.awaitNoIPTableRules(egressChain, allocatedIPs...) + t.awaitNoPacketFilterRules(egressChain, allocatedIPs...) t.watches.AwaitWatchStopped("pods") }) Context("and cleanup of the IP tables initially fails", func() { JustBeforeEach(func() { - t.ipt.AddFailOnDeleteRuleMatcher(ContainSubstring(ipSetName)) + t.pFilter.AddFailOnDeleteRuleMatcher(ContainSubstring(ipSetName)) t.ipSet.AddFailOnDestroySetMatchers(Equal(ipSetName)) }) @@ -223,7 +224,7 @@ func testGlobalEgressIPCreated(t *globalEgressIPControllerTestDriver, podSelecto Expect(t.globalEgressIPs.Delete(context.TODO(), globalEgressIPName, metav1.DeleteOptions{})).To(Succeed()) t.awaitIPsReleasedFromPool(allocatedIPs...) t.ipSet.AwaitSetDeleted(ipSetName) - t.awaitNoIPTableRules(egressChain, allocatedIPs...) + t.awaitNoPacketFilterRules(egressChain, allocatedIPs...) }) }) }) @@ -267,7 +268,7 @@ func testExistingGlobalEgressIP(t *globalEgressIPControllerTestDriver, podSelect }) It("should program the necessary IP table rules for the allocated IPs", func() { - t.awaitIPTableRules(egressChain, existing.Status.AllocatedIPs...) + t.awaitPacketFilterRules(egressChain, existing.Status.AllocatedIPs...) }) It("should start a Pod watcher", func() { @@ -284,12 +285,12 @@ func testExistingGlobalEgressIP(t *globalEgressIPControllerTestDriver, podSelect It("should reallocate the global IPs", func() { t.awaitGlobalEgressIPStatusAllocated(globalEgressIPName, *existing.Spec.NumberOfIPs) - t.awaitIPTableRules(egressChain, getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs...) + t.awaitPacketFilterRules(egressChain, getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs...) }) It("should release the previously allocated IPs", func() { t.awaitIPsReleasedFromPool(existing.Status.AllocatedIPs...) - t.awaitNoIPTableRules(egressChain, existing.Status.AllocatedIPs...) + t.awaitNoPacketFilterRules(egressChain, existing.Status.AllocatedIPs...) }) It("should start a Pod watcher", func() { @@ -322,14 +323,14 @@ func testExistingGlobalEgressIP(t *globalEgressIPControllerTestDriver, podSelect Status: metav1.ConditionTrue, }) - t.awaitIPTableRules(egressChain, getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs...) + t.awaitPacketFilterRules(egressChain, getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs...) }) }) Context("and programming the IP table rules fails", func() { BeforeEach(func() { t.createGlobalEgressIP(existing) - t.ipt.AddFailOnAppendRuleMatcher(ContainSubstring(existing.Status.AllocatedIPs[0])) + t.pFilter.AddFailOnAppendRuleMatcher(ContainSubstring(existing.Status.AllocatedIPs[0])) }) It("should reallocate the global IPs", func() { @@ -343,7 +344,7 @@ func testExistingGlobalEgressIP(t *globalEgressIPControllerTestDriver, podSelect }) allocatedIPs := getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs - t.awaitIPTableRules(egressChain, allocatedIPs...) + t.awaitPacketFilterRules(egressChain, allocatedIPs...) t.awaitIPsReleasedFromPool(existing.Status.AllocatedIPs...) }) }) @@ -407,10 +408,10 @@ func testGlobalEgressIPUpdated(t *globalEgressIPControllerTestDriver, podSelecto testReallocated := func() { t.awaitIPsReleasedFromPool(existing.Status.AllocatedIPs...) - t.awaitNoIPTableRules(egressChain, existing.Status.AllocatedIPs...) + t.awaitNoPacketFilterRules(egressChain, existing.Status.AllocatedIPs...) t.awaitGlobalEgressIPStatusAllocated(globalEgressIPName, *existing.Spec.NumberOfIPs) - t.awaitIPTableRules(egressChain, getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs...) + t.awaitPacketFilterRules(egressChain, getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs...) t.watches.AwaitNoWatchStopped("pods") } @@ -438,7 +439,7 @@ func testGlobalEgressIPUpdated(t *globalEgressIPControllerTestDriver, podSelecto It("should release the previously allocated IPs and update the status", func() { t.awaitIPsReleasedFromPool(existing.Status.AllocatedIPs...) - t.awaitNoIPTableRules(egressChain, existing.Status.AllocatedIPs...) + t.awaitNoPacketFilterRules(egressChain, existing.Status.AllocatedIPs...) t.awaitEgressIPStatus(t.globalEgressIPs, globalEgressIPName, 0, metav1.Condition{ Type: string(submarinerv1.GlobalEgressIPAllocated), @@ -486,7 +487,7 @@ func testEgressPodEvents(t *globalEgressIPControllerTestDriver) { JustBeforeEach(func() { t.createGlobalEgressIP(egressIP) t.awaitGlobalEgressIPStatusAllocated(globalEgressIPName, 1) - ipSet = t.awaitIPTableRules(egressChain, getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs...) + ipSet = t.awaitPacketFilterRules(egressChain, getGlobalEgressIPStatus(t.globalEgressIPs, globalEgressIPName).AllocatedIPs...) t.createPod(pod) }) @@ -621,13 +622,13 @@ func (t *globalEgressIPControllerTestDriver) start() { Expect(t.controller.Start()).To(Succeed()) } -func (t *globalEgressIPControllerTestDriver) awaitIPTableRules(chain string, ips ...string) string { +func (t *globalEgressIPControllerTestDriver) awaitPacketFilterRules(chain string, ips ...string) string { set := t.ipSet.AwaitOneSet(HavePrefix(controllers.IPSetPrefix)) - t.ipt.AwaitRule("nat", chain, And(ContainSubstring(set), ContainSubstring(getSNATAddress(ips...)))) + t.pFilter.AwaitRule(packetfilter.TableTypeNAT, chain, And(ContainSubstring(set), ContainSubstring(getSNATAddress(ips...)))) return set } -func (t *globalEgressIPControllerTestDriver) awaitNoIPTableRules(chain string, ips ...string) { - t.ipt.AwaitNoRule("nat", chain, ContainSubstring(getSNATAddress(ips...))) +func (t *globalEgressIPControllerTestDriver) awaitNoPacketFilterRules(chain string, ips ...string) { + t.pFilter.AwaitNoRule(packetfilter.TableTypeNAT, chain, ContainSubstring(getSNATAddress(ips...))) } diff --git a/pkg/globalnet/controllers/global_ingressip_controller.go b/pkg/globalnet/controllers/global_ingressip_controller.go index b84df53bf..9c8c1a16b 100644 --- a/pkg/globalnet/controllers/global_ingressip_controller.go +++ b/pkg/globalnet/controllers/global_ingressip_controller.go @@ -29,7 +29,7 @@ import ( "github.com/submariner-io/admiral/pkg/syncer" "github.com/submariner-io/admiral/pkg/util" submarinerv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1" - "github.com/submariner-io/submariner/pkg/globalnet/controllers/iptables" + pfiface "github.com/submariner-io/submariner/pkg/globalnet/controllers/packetfilter" "github.com/submariner-io/submariner/pkg/globalnet/metrics" "github.com/submariner-io/submariner/pkg/ipam" corev1 "k8s.io/api/core/v1" @@ -47,9 +47,9 @@ func NewGlobalIngressIPController(config *syncer.ResourceSyncerConfig, pool *ipa logger.Info("Creating GlobalIngressIP controller") - iptIface, err := iptables.New() + pfIface, err := pfiface.New() if err != nil { - return nil, errors.Wrap(err, "error creating the IPTablesInterface handler") + return nil, errors.Wrap(err, "error creating the PacketFilter Interface handler") } _, gvr, err := util.ToUnstructuredResource(&corev1.Service{}, config.RestMapper) @@ -58,7 +58,7 @@ func NewGlobalIngressIPController(config *syncer.ResourceSyncerConfig, pool *ipa } controller := &globalIngressIPController{ - baseIPAllocationController: newBaseIPAllocationController(pool, iptIface), + baseIPAllocationController: newBaseIPAllocationController(pool, pfIface), services: config.SourceClient.Resource(*gvr), scheme: config.Scheme, } @@ -85,7 +85,7 @@ func NewGlobalIngressIPController(config *syncer.ResourceSyncerConfig, pool *ipa //nolint:wrapcheck // No need to wrap these errors. err = controller.reserveAllocatedIPs(federator, obj, func(reservedIPs []string) error { var target string - var tType iptables.TargetType + var tType pfiface.TargetType metrics.RecordAllocateGlobalIngressIPs(pool.GetCIDR(), len(reservedIPs)) @@ -93,21 +93,21 @@ func NewGlobalIngressIPController(config *syncer.ResourceSyncerConfig, pool *ipa return controller.ensureInternalServiceExists(gip) } else if gip.Spec.Target == submarinerv1.HeadlessServicePod { target = gip.GetAnnotations()[headlessSvcPodIP] - tType = iptables.PodTarget + tType = pfiface.PodTarget } else if gip.Spec.Target == submarinerv1.HeadlessServiceEndpoints { target = gip.GetAnnotations()[headlessSvcEndpointsIP] - tType = iptables.EndpointsTarget + tType = pfiface.EndpointsTarget } else { return nil } - err := controller.iptIface.AddIngressRulesForHeadlessSvc(reservedIPs[0], target, tType) + err := controller.pfIface.AddIngressRulesForHeadlessSvc(reservedIPs[0], target, tType) if err != nil { return err } key, _ := cache.MetaNamespaceKeyFunc(obj) - return controller.iptIface.AddEgressRulesForHeadlessSvc(key, target, reservedIPs[0], globalNetIPTableMark, tType) + return controller.pfIface.AddEgressRulesForHeadlessSvc(key, target, reservedIPs[0], globalNetIPTableMark, tType) }) if err != nil { @@ -220,14 +220,14 @@ func (c *globalIngressIPController) onCreate(ingressIP *submarinerv1.GlobalIngre } } else { var annotationKey string - var tType iptables.TargetType + var tType pfiface.TargetType if ingressIP.Spec.Target == submarinerv1.HeadlessServicePod { annotationKey = headlessSvcPodIP - tType = iptables.PodTarget + tType = pfiface.PodTarget } else if ingressIP.Spec.Target == submarinerv1.HeadlessServiceEndpoints { annotationKey = headlessSvcEndpointsIP - tType = iptables.EndpointsTarget + tType = pfiface.EndpointsTarget } target := ingressIP.GetAnnotations()[annotationKey] @@ -239,14 +239,14 @@ func (c *globalIngressIPController) onCreate(ingressIP *submarinerv1.GlobalIngre return true } - err = c.iptIface.AddIngressRulesForHeadlessSvc(ips[0], target, tType) + err = c.pfIface.AddIngressRulesForHeadlessSvc(ips[0], target, tType) if err != nil { logger.Errorf(err, "Error while programming Service %q ingress rules for %v", key, tType) err = errors.WithMessage(err, "Error programming ingress rules") } else { - err = c.iptIface.AddEgressRulesForHeadlessSvc(key, target, ips[0], globalNetIPTableMark, tType) + err = c.pfIface.AddEgressRulesForHeadlessSvc(key, target, ips[0], globalNetIPTableMark, tType) if err != nil { - _ = c.iptIface.RemoveIngressRulesForHeadlessSvc(ips[0], target, tType) + _ = c.pfIface.RemoveIngressRulesForHeadlessSvc(ips[0], target, tType) err = errors.WithMessage(err, "Error programming egress rules") } } @@ -373,24 +373,24 @@ func (c *globalIngressIPController) onDelete(ingressIP *submarinerv1.GlobalIngre return c.flushRulesAndReleaseIPs(key, numRequeues, func(allocatedIPs []string) error { var target string - var tType iptables.TargetType + var tType pfiface.TargetType metrics.RecordDeallocateGlobalIngressIPs(c.pool.GetCIDR(), len(allocatedIPs)) if ingressIP.Spec.Target == submarinerv1.HeadlessServicePod { target = ingressIP.GetAnnotations()[headlessSvcPodIP] - tType = iptables.PodTarget + tType = pfiface.PodTarget } else if ingressIP.Spec.Target == submarinerv1.HeadlessServiceEndpoints { target = ingressIP.GetAnnotations()[headlessSvcEndpointsIP] - tType = iptables.EndpointsTarget + tType = pfiface.EndpointsTarget } if target != "" { - if err := c.iptIface.RemoveIngressRulesForHeadlessSvc(ingressIP.Status.AllocatedIP, target, tType); err != nil { + if err := c.pfIface.RemoveIngressRulesForHeadlessSvc(ingressIP.Status.AllocatedIP, target, tType); err != nil { return err } - return c.iptIface.RemoveEgressRulesForHeadlessSvc(key, target, ingressIP.Status.AllocatedIP, globalNetIPTableMark, tType) + return c.pfIface.RemoveEgressRulesForHeadlessSvc(key, target, ingressIP.Status.AllocatedIP, globalNetIPTableMark, tType) } return nil diff --git a/pkg/globalnet/controllers/global_ingressip_controller_test.go b/pkg/globalnet/controllers/global_ingressip_controller_test.go index 32ecd658a..f094e7647 100644 --- a/pkg/globalnet/controllers/global_ingressip_controller_test.go +++ b/pkg/globalnet/controllers/global_ingressip_controller_test.go @@ -29,6 +29,7 @@ import ( "github.com/submariner-io/submariner/pkg/globalnet/constants" "github.com/submariner-io/submariner/pkg/globalnet/controllers" "github.com/submariner-io/submariner/pkg/ipam" + "github.com/submariner-io/submariner/pkg/packetfilter" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -190,7 +191,7 @@ func testGlobalIngressIPCreatedClusterIPSvc(t *globalIngressIPControllerTestDriv } func testGlobalIngressIPCreatedHeadlessSvc(t *globalIngressIPControllerTestDriver, ingressIP *submarinerv1.GlobalIngressIP, - awaitIPTableRules, awaitNoIPTableRules func(string), ruleMatch string, + awaitPacketFilterRules, awaitNoPacketFilterRules func(string), ruleMatch string, ) { JustBeforeEach(func() { service := newClusterIPService() @@ -202,7 +203,7 @@ func testGlobalIngressIPCreatedHeadlessSvc(t *globalIngressIPControllerTestDrive t.awaitIngressIPStatusAllocated(globalIngressIPName) allocatedIP := t.getGlobalIngressIPStatus(globalIngressIPName).AllocatedIP Expect(allocatedIP).ToNot(BeEmpty()) - awaitIPTableRules(allocatedIP) + awaitPacketFilterRules(allocatedIP) }) Context("with the IP pool exhausted", func() { @@ -222,7 +223,7 @@ func testGlobalIngressIPCreatedHeadlessSvc(t *globalIngressIPControllerTestDrive Context("and programming of IP tables initially fails", func() { BeforeEach(func() { - t.ipt.AddFailOnAppendRuleMatcher(ContainSubstring(ruleMatch)) + t.pFilter.AddFailOnAppendRuleMatcher(ContainSubstring(ruleMatch)) }) It("should eventually allocate a global IP", func() { @@ -235,7 +236,7 @@ func testGlobalIngressIPCreatedHeadlessSvc(t *globalIngressIPControllerTestDrive Status: metav1.ConditionTrue, }) - awaitIPTableRules(t.getGlobalIngressIPStatus(globalIngressIPName).AllocatedIP) + awaitPacketFilterRules(t.getGlobalIngressIPStatus(globalIngressIPName).AllocatedIP) }) }) @@ -251,17 +252,17 @@ func testGlobalIngressIPCreatedHeadlessSvc(t *globalIngressIPControllerTestDrive It("should release the allocated global IP", func() { t.awaitIPsReleasedFromPool(allocatedIP) - awaitNoIPTableRules(allocatedIP) + awaitNoPacketFilterRules(allocatedIP) }) Context("and cleanup of IP tables initially fails", func() { BeforeEach(func() { - t.ipt.AddFailOnDeleteRuleMatcher(ContainSubstring(ruleMatch)) + t.pFilter.AddFailOnDeleteRuleMatcher(ContainSubstring(ruleMatch)) }) It("should eventually cleanup the IP tables and reallocate", func() { t.awaitIPsReleasedFromPool(allocatedIP) - awaitNoIPTableRules(allocatedIP) + awaitNoPacketFilterRules(allocatedIP) }) }) }) @@ -408,7 +409,7 @@ func testExistingGlobalIngressIPClusterIPSvc(t *globalIngressIPControllerTestDri } func testExistingGlobalIngressIPHeadlessSvc(t *globalIngressIPControllerTestDriver, ingressIP *submarinerv1.GlobalIngressIP, - awaitIPTableRules func(string), + awaitPacketFilterRules func(string), ) { var existing *submarinerv1.GlobalIngressIP @@ -439,7 +440,7 @@ func testExistingGlobalIngressIPHeadlessSvc(t *globalIngressIPControllerTestDriv }) It("should program the relevant IP table rules", func() { - awaitIPTableRules(existing.Status.AllocatedIP) + awaitPacketFilterRules(existing.Status.AllocatedIP) }) Context("and it's already reserved", func() { @@ -457,13 +458,13 @@ func testExistingGlobalIngressIPHeadlessSvc(t *globalIngressIPControllerTestDriv Status: metav1.ConditionTrue, }) - awaitIPTableRules(t.getGlobalIngressIPStatus(globalIngressIPName).AllocatedIP) + awaitPacketFilterRules(t.getGlobalIngressIPStatus(globalIngressIPName).AllocatedIP) }) }) Context("and programming the IP table rules fails", func() { BeforeEach(func() { - t.ipt.AddFailOnAppendRuleMatcher(ContainSubstring(existing.Status.AllocatedIP)) + t.pFilter.AddFailOnAppendRuleMatcher(ContainSubstring(existing.Status.AllocatedIP)) }) It("should reallocate the global IP", func() { @@ -477,7 +478,7 @@ func testExistingGlobalIngressIPHeadlessSvc(t *globalIngressIPControllerTestDriv }) allocatedIP := t.getGlobalIngressIPStatus(globalIngressIPName).AllocatedIP - awaitIPTableRules(allocatedIP) + awaitPacketFilterRules(allocatedIP) t.awaitIPsReleasedFromPool(existing.Status.AllocatedIP) }) }) @@ -490,7 +491,7 @@ func testExistingGlobalIngressIPHeadlessSvc(t *globalIngressIPControllerTestDriv It("should allocate it and program the relevant IP table rules", func() { t.awaitIngressIPStatusAllocated(globalIngressIPName) - awaitIPTableRules(existing.Status.AllocatedIP) + awaitPacketFilterRules(existing.Status.AllocatedIP) }) }) } @@ -537,33 +538,39 @@ func (t *globalIngressIPControllerTestDriver) start() syncer.Interface { } func (t *globalIngressIPControllerTestDriver) awaitPodEgressRules(podIP, snatIP string) { - t.ipt.AwaitRule("nat", constants.SmGlobalnetEgressChainForHeadlessSvcPods, And(ContainSubstring(podIP), ContainSubstring(snatIP))) + t.pFilter.AwaitRule(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChainForHeadlessSvcPods, And(ContainSubstring(podIP), + ContainSubstring(snatIP))) } func (t *globalIngressIPControllerTestDriver) awaitNoPodEgressRules(podIP, snatIP string) { - t.ipt.AwaitNoRule("nat", constants.SmGlobalnetEgressChainForHeadlessSvcPods, Or(ContainSubstring(podIP), ContainSubstring(snatIP))) + t.pFilter.AwaitNoRule(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChainForHeadlessSvcPods, Or(ContainSubstring(podIP), + ContainSubstring(snatIP))) } func (t *globalIngressIPControllerTestDriver) awaitPodIngressRules(podIP, snatIP string) { - t.ipt.AwaitRule("nat", constants.SmGlobalnetIngressChain, And(ContainSubstring(podIP), ContainSubstring(snatIP))) + t.pFilter.AwaitRule(packetfilter.TableTypeNAT, constants.SmGlobalnetIngressChain, And(ContainSubstring(podIP), ContainSubstring(snatIP))) } func (t *globalIngressIPControllerTestDriver) awaitNoPodIngressRules(podIP, snatIP string) { - t.ipt.AwaitNoRule("nat", constants.SmGlobalnetIngressChain, Or(ContainSubstring(podIP), ContainSubstring(snatIP))) + t.pFilter.AwaitNoRule(packetfilter.TableTypeNAT, constants.SmGlobalnetIngressChain, Or(ContainSubstring(podIP), ContainSubstring(snatIP))) } func (t *globalIngressIPControllerTestDriver) awaitEndpointsEgressRules(endpointsIP, snatIP string) { - t.ipt.AwaitRule("nat", constants.SmGlobalnetEgressChainForHeadlessSvcEPs, And(ContainSubstring(endpointsIP), ContainSubstring(snatIP))) + t.pFilter.AwaitRule(packetfilter.TableTypeNAT, + constants.SmGlobalnetEgressChainForHeadlessSvcEPs, And(ContainSubstring(endpointsIP), ContainSubstring(snatIP))) } func (t *globalIngressIPControllerTestDriver) awaitNoEndpointsEgressRules(endpointsIP, snatIP string) { - t.ipt.AwaitNoRule("nat", constants.SmGlobalnetEgressChainForHeadlessSvcEPs, Or(ContainSubstring(endpointsIP), ContainSubstring(snatIP))) + t.pFilter.AwaitNoRule(packetfilter.TableTypeNAT, + constants.SmGlobalnetEgressChainForHeadlessSvcEPs, Or(ContainSubstring(endpointsIP), ContainSubstring(snatIP))) } func (t *globalIngressIPControllerTestDriver) awaitEndpointsIngressRules(endpointsIP, snatIP string) { - t.ipt.AwaitRule("nat", constants.SmGlobalnetIngressChain, And(ContainSubstring(endpointsIP), ContainSubstring(snatIP))) + t.pFilter.AwaitRule(packetfilter.TableTypeNAT, + constants.SmGlobalnetIngressChain, And(ContainSubstring(endpointsIP), ContainSubstring(snatIP))) } func (t *globalIngressIPControllerTestDriver) awaitNoEndpointsIngressRules(endpointsIP, snatIP string) { - t.ipt.AwaitNoRule("nat", constants.SmGlobalnetIngressChain, Or(ContainSubstring(endpointsIP), ContainSubstring(snatIP))) + t.pFilter.AwaitNoRule(packetfilter.TableTypeNAT, + constants.SmGlobalnetIngressChain, Or(ContainSubstring(endpointsIP), ContainSubstring(snatIP))) } diff --git a/pkg/globalnet/controllers/iptables/iface.go b/pkg/globalnet/controllers/iptables/iface.go deleted file mode 100644 index 7cf36c357..000000000 --- a/pkg/globalnet/controllers/iptables/iface.go +++ /dev/null @@ -1,310 +0,0 @@ -/* -SPDX-License-Identifier: Apache-2.0 - -Copyright Contributors to the Submariner project. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package iptables - -import ( - "crypto/sha256" - "encoding/base32" - "fmt" - "strings" - - "github.com/pkg/errors" - "github.com/submariner-io/admiral/pkg/log" - "github.com/submariner-io/submariner/pkg/globalnet/constants" - "github.com/submariner-io/submariner/pkg/iptables" - corev1 "k8s.io/api/core/v1" - logf "sigs.k8s.io/controller-runtime/pkg/log" -) - -type Interface interface { - AddClusterEgressRules(sourceIP, snatIP, globalNetIPTableMark string) error - RemoveClusterEgressRules(sourceIP, snatIP, globalNetIPTableMark string) error - AddIngressRulesForHeadlessSvc(globalIP, podIP string, targetType TargetType) error - RemoveIngressRulesForHeadlessSvc(globalIP, podIP string, targetType TargetType) error - GetKubeProxyClusterIPServiceChainName(service *corev1.Service, kubeProxyServiceChainPrefix string) (string, bool, error) - AddIngressRulesForHealthCheck(cniIfaceIP, globalIP string) error - RemoveIngressRulesForHealthCheck(cniIfaceIP, globalIP string) error - AddEgressRulesForHeadlessSvc(key, sourceIP, snatIP, globalNetIPTableMark string, targetType TargetType) error - RemoveEgressRulesForHeadlessSvc(key, sourceIP, snatIP, globalNetIPTableMark string, targetType TargetType) error - AddEgressRulesForPods(namespace, ipSetName, snatIP, globalNetIPTableMark string) error - RemoveEgressRulesForPods(namespace, ipSetName, snatIP, globalNetIPTableMark string) error - AddEgressRulesForNamespace(namespace, ipSetName, snatIP, globalNetIPTableMark string) error - RemoveEgressRulesForNamespace(namespace, ipSetName, snatIP, globalNetIPTableMark string) error - FlushIPTableChain(table, chainName string) error - DeleteIPTableChain(table, chainName string) error - DeleteIPTableRule(table, chainName, jumpTarget string) error -} - -type ipTables struct { - ipt iptables.Interface -} - -type TargetType string - -const ( - PodTarget TargetType = "Pod" - EndpointsTarget TargetType = "Endpoints" -) - -var logger = log.Logger{Logger: logf.Log.WithName("IPTables")} - -func New() (Interface, error) { - iptableHandler, err := iptables.New() - if err != nil { - return nil, err //nolint:wrapcheck // Let the caller wrap it - } - - iptableIface := &ipTables{ - ipt: iptableHandler, - } - - return iptableIface, nil -} - -func (i *ipTables) AddClusterEgressRules(subnet, snatIP, globalNetIPTableMark string) error { - ruleSpec := []string{"-p", "all", "-s", subnet, "-m", "mark", "--mark", globalNetIPTableMark, "-j", "SNAT", "--to", snatIP} - logger.V(log.DEBUG).Infof("Installing iptable egress rules for Cluster: %s", strings.Join(ruleSpec, " ")) - - if err := i.ipt.AppendUnique("nat", constants.SmGlobalnetEgressChainForCluster, ruleSpec...); err != nil { - return errors.Wrapf(err, "error appending iptables rule \"%s\"", strings.Join(ruleSpec, " ")) - } - - return nil -} - -func (i *ipTables) RemoveClusterEgressRules(subnet, snatIP, globalNetIPTableMark string) error { - ruleSpec := []string{"-p", "all", "-s", subnet, "-m", "mark", "--mark", globalNetIPTableMark, "-j", "SNAT", "--to", snatIP} - logger.V(log.DEBUG).Infof("Deleting iptable egress rules for Cluster: %s", strings.Join(ruleSpec, " ")) - - if err := i.ipt.Delete("nat", constants.SmGlobalnetEgressChainForCluster, ruleSpec...); err != nil { - return errors.Wrapf(err, "error deleting iptables rule \"%s\"", strings.Join(ruleSpec, " ")) - } - - return nil -} - -func (i *ipTables) AddIngressRulesForHeadlessSvc(globalIP, ip string, targetType TargetType) error { - if globalIP == "" || ip == "" { - return fmt.Errorf("globalIP %q or %s IP %q cannot be empty", globalIP, targetType, ip) - } - - ruleSpec := []string{"-d", globalIP, "-j", "DNAT", "--to", ip} - logger.V(log.DEBUG).Infof("Installing iptables rule for Headless SVC %s for %s", strings.Join(ruleSpec, " "), targetType) - - if err := i.ipt.AppendUnique("nat", constants.SmGlobalnetIngressChain, ruleSpec...); err != nil { - return errors.Wrapf(err, "error appending iptables rule \"%s\"", strings.Join(ruleSpec, " ")) - } - - return nil -} - -func (i *ipTables) RemoveIngressRulesForHeadlessSvc(globalIP, ip string, targetType TargetType) error { - if globalIP == "" || ip == "" { - return fmt.Errorf("globalIP %q or %s IP %q cannot be empty", globalIP, targetType, ip) - } - - ruleSpec := []string{"-d", globalIP, "-j", "DNAT", "--to", ip} - - logger.V(log.DEBUG).Infof("Deleting iptables rule for Headless SVC %s for %s", strings.Join(ruleSpec, " "), targetType) - - if err := i.ipt.Delete("nat", constants.SmGlobalnetIngressChain, ruleSpec...); err != nil { - return errors.Wrapf(err, "error deleting iptables rule \"%s\"", strings.Join(ruleSpec, " ")) - } - - return nil -} - -func (i *ipTables) GetKubeProxyClusterIPServiceChainName(service *corev1.Service, - kubeProxyServiceChainPrefix string, -) (string, bool, error) { - // CNIs that use kube-proxy with iptables for loadbalancing create an iptables chain for each service - // and incoming traffic to the clusterIP Service is directed into the respective chain. - // Reference: https://bit.ly/2OPhlwk - prefix := service.GetNamespace() + "/" + service.GetName() - serviceNames := []string{prefix + ":" + service.Spec.Ports[0].Name} - - if service.Spec.Ports[0].Name == "" { - // In newer k8s versions (v1.19+), they omit the ":" if the port name is empty so we need to handle both formats (see - // https://github.com/kubernetes/kubernetes/pull/90031). - serviceNames = append(serviceNames, prefix) - } - - for _, serviceName := range serviceNames { - protocol := strings.ToLower(string(service.Spec.Ports[0].Protocol)) - hash := sha256.Sum256([]byte(serviceName + protocol)) - encoded := base32.StdEncoding.EncodeToString(hash[:]) - chainName := kubeProxyServiceChainPrefix + encoded[:16] - - chainExists, err := i.ipt.ChainExists("nat", chainName) - if err != nil { - return "", false, errors.Wrapf(err, "error checking if chain %s exists", chainName) - } - - if chainExists { - return chainName, true, nil - } - } - - return "", false, nil -} - -func (i *ipTables) AddIngressRulesForHealthCheck(cniIfaceIP, globalIP string) error { - ruleSpec := []string{"-p", "icmp", "-d", globalIP, "-j", "DNAT", "--to", cniIfaceIP} - logger.V(log.DEBUG).Infof("Installing iptable ingress rules for Node: %s", strings.Join(ruleSpec, " ")) - - if err := i.ipt.AppendUnique("nat", constants.SmGlobalnetIngressChain, ruleSpec...); err != nil { - return errors.Wrapf(err, "error appending iptables rule \"%s\"", strings.Join(ruleSpec, " ")) - } - - return nil -} - -func (i *ipTables) RemoveIngressRulesForHealthCheck(cniIfaceIP, globalIP string) error { - ruleSpec := []string{"-p", "icmp", "-d", globalIP, "-j", "DNAT", "--to", cniIfaceIP} - logger.V(log.DEBUG).Infof("Deleting iptable ingress rules for Node: %s", strings.Join(ruleSpec, " ")) - - if err := i.ipt.Delete("nat", constants.SmGlobalnetIngressChain, ruleSpec...); err != nil { - return errors.Wrapf(err, "error deleting iptables rule \"%s\"", strings.Join(ruleSpec, " ")) - } - - return nil -} - -func (i *ipTables) AddEgressRulesForHeadlessSvc(key, sourceIP, snatIP, globalNetIPTableMark string, targetType TargetType) error { - ruleSpec := []string{"-p", "all", "-s", sourceIP, "-m", "mark", "--mark", globalNetIPTableMark, "-j", "SNAT", "--to", snatIP} - logger.V(log.DEBUG).Infof("Installing iptable egress rules for HDLS SVC %q for %s: %s", key, targetType, strings.Join(ruleSpec, " ")) - - var chain string - - if targetType == PodTarget { - chain = constants.SmGlobalnetEgressChainForHeadlessSvcPods - } else if targetType == EndpointsTarget { - chain = constants.SmGlobalnetEgressChainForHeadlessSvcEPs - } - - if err := i.ipt.AppendUnique("nat", chain, ruleSpec...); err != nil { - return errors.Wrapf(err, "error appending iptables rule \"%s\"", strings.Join(ruleSpec, " ")) - } - - return nil -} - -func (i *ipTables) RemoveEgressRulesForHeadlessSvc(key, sourceIP, snatIP, globalNetIPTableMark string, targetType TargetType) error { - ruleSpec := []string{"-p", "all", "-s", sourceIP, "-m", "mark", "--mark", globalNetIPTableMark, "-j", "SNAT", "--to", snatIP} - logger.V(log.DEBUG).Infof("Deleting iptable egress rules for HDLS SVC %q for %s: %s", key, targetType, strings.Join(ruleSpec, " ")) - - var chain string - - if targetType == PodTarget { - chain = constants.SmGlobalnetEgressChainForHeadlessSvcPods - } else if targetType == EndpointsTarget { - chain = constants.SmGlobalnetEgressChainForHeadlessSvcEPs - } - - if err := i.ipt.Delete("nat", chain, ruleSpec...); err != nil { - return errors.Wrapf(err, "error deleting iptables rule \"%s\"", strings.Join(ruleSpec, " ")) - } - - return nil -} - -func (i *ipTables) AddEgressRulesForPods(key, ipSetName, snatIP, globalNetIPTableMark string) error { - ruleSpec := []string{ - "-p", "all", "-m", "set", "--match-set", ipSetName, "src", "-m", "mark", - "--mark", globalNetIPTableMark, "-j", "SNAT", "--to", snatIP, - } - logger.V(log.DEBUG).Infof("Installing iptable egress rules for Pods %q: %s", key, strings.Join(ruleSpec, " ")) - - if err := i.ipt.AppendUnique("nat", constants.SmGlobalnetEgressChainForPods, ruleSpec...); err != nil { - return errors.Wrapf(err, "error appending iptables rule \"%s\"", strings.Join(ruleSpec, " ")) - } - - return nil -} - -func (i *ipTables) RemoveEgressRulesForPods(key, ipSetName, snatIP, globalNetIPTableMark string) error { - ruleSpec := []string{ - "-p", "all", "-m", "set", "--match-set", ipSetName, "src", "-m", "mark", - "--mark", globalNetIPTableMark, "-j", "SNAT", "--to", snatIP, - } - logger.V(log.DEBUG).Infof("Deleting iptable egress rules for Pods %q: %s", key, strings.Join(ruleSpec, " ")) - - if err := i.ipt.Delete("nat", constants.SmGlobalnetEgressChainForPods, ruleSpec...); err != nil { - return errors.Wrapf(err, "error deleting iptables rule \"%s\"", strings.Join(ruleSpec, " ")) - } - - return nil -} - -func (i *ipTables) AddEgressRulesForNamespace(namespace, ipSetName, snatIP, globalNetIPTableMark string) error { - ruleSpec := []string{ - "-p", "all", "-m", "set", "--match-set", ipSetName, "src", "-m", "mark", - "--mark", globalNetIPTableMark, "-j", "SNAT", "--to", snatIP, - } - logger.V(log.DEBUG).Infof("Installing iptable egress rules for Namespace %q: %s", namespace, strings.Join(ruleSpec, " ")) - - if err := i.ipt.AppendUnique("nat", constants.SmGlobalnetEgressChainForNamespace, ruleSpec...); err != nil { - return errors.Wrapf(err, "error appending iptables rule \"%s\"", strings.Join(ruleSpec, " ")) - } - - return nil -} - -func (i *ipTables) RemoveEgressRulesForNamespace(namespace, ipSetName, snatIP, globalNetIPTableMark string) error { - ruleSpec := []string{ - "-p", "all", "-m", "set", "--match-set", ipSetName, "src", "-m", "mark", - "--mark", globalNetIPTableMark, "-j", "SNAT", "--to", snatIP, - } - logger.V(log.DEBUG).Infof("Deleting iptable egress rules for Namespace %q: %s", namespace, strings.Join(ruleSpec, " ")) - - if err := i.ipt.Delete("nat", constants.SmGlobalnetEgressChainForNamespace, ruleSpec...); err != nil { - return errors.Wrapf(err, "error deleting iptables rule \"%s\"", strings.Join(ruleSpec, " ")) - } - - return nil -} - -func (i *ipTables) FlushIPTableChain(table, chainName string) error { - logger.Infof("Flushing iptable rules in %q chain of table %q", chainName, table) - - if err := i.ipt.ClearChain(table, chainName); err != nil { - return errors.Wrapf(err, "error flushing iptables rules in %q chain of table %q", chainName, table) - } - - return nil -} - -func (i *ipTables) DeleteIPTableChain(table, chainName string) error { - logger.Infof("Deleting iptable chain %q of table %q", chainName, table) - - if err := i.ipt.DeleteChain(table, chainName); err != nil { - return errors.Wrapf(err, "error deleting iptable chain %q of table %q", chainName, table) - } - - return nil -} - -func (i *ipTables) DeleteIPTableRule(table, chainName, jumpTarget string) error { - ruleSpec := []string{"-j", jumpTarget} - if err := i.ipt.Delete(table, chainName, ruleSpec...); err != nil { - return errors.Wrapf(err, "error deleting iptables rule \"%s\"", strings.Join(ruleSpec, " ")) - } - - return nil -} diff --git a/pkg/globalnet/controllers/node_controller.go b/pkg/globalnet/controllers/node_controller.go index ce6e17c39..c715fba0d 100644 --- a/pkg/globalnet/controllers/node_controller.go +++ b/pkg/globalnet/controllers/node_controller.go @@ -26,7 +26,7 @@ import ( "github.com/submariner-io/admiral/pkg/syncer" admUtil "github.com/submariner-io/admiral/pkg/util" "github.com/submariner-io/submariner/pkg/globalnet/constants" - "github.com/submariner-io/submariner/pkg/globalnet/controllers/iptables" + packetfilter "github.com/submariner-io/submariner/pkg/globalnet/controllers/packetfilter" "github.com/submariner-io/submariner/pkg/ipam" routeAgent "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" corev1 "k8s.io/api/core/v1" @@ -42,13 +42,13 @@ func NewNodeController(config *syncer.ResourceSyncerConfig, pool *ipam.IPPool, n logger.Info("Creating Node controller") - iptIface, err := iptables.New() + pfIface, err := packetfilter.New() if err != nil { - return nil, errors.Wrap(err, "error creating the IPTablesInterface handler") + return nil, errors.Wrap(err, "error creating the PacketFilter Interface handler") } controller := &nodeController{ - baseIPAllocationController: newBaseIPAllocationController(pool, iptIface), + baseIPAllocationController: newBaseIPAllocationController(pool, pfIface), nodeName: nodeName, } @@ -149,7 +149,7 @@ func (n *nodeController) allocateIP(node *corev1.Node, op syncer.Operation) (run logger.Infof("Adding ingress rules for node %q with global IP %s, CNI IP %s", node.Name, globalIP, cniIfaceIP) - if err := n.iptIface.AddIngressRulesForHealthCheck(cniIfaceIP, globalIP); err != nil { + if err := n.pfIface.AddIngressRulesForHealthCheck(cniIfaceIP, globalIP); err != nil { logger.Errorf(err, "Error programming rules for Gateway healthcheck on node %q", node.Name) _ = n.pool.Release(globalIP) @@ -178,7 +178,7 @@ func (n *nodeController) reserveAllocatedIP(federator federate.Federator, obj *u err := n.pool.Reserve(existingGlobalIP) if err == nil { - err = n.iptIface.AddIngressRulesForHealthCheck(cniIfaceIP, existingGlobalIP) + err = n.pfIface.AddIngressRulesForHealthCheck(cniIfaceIP, existingGlobalIP) if err != nil { _ = n.pool.Release(existingGlobalIP) } @@ -233,7 +233,7 @@ func (n *nodeController) onNodeUpdated(oldObj, newObj *unstructured.Unstructured } if oldCNIIfaceIPOnNode != "" && oldGlobalIPOnNode != "" { - if err := n.iptIface.RemoveIngressRulesForHealthCheck(oldCNIIfaceIPOnNode, oldGlobalIPOnNode); err != nil { + if err := n.pfIface.RemoveIngressRulesForHealthCheck(oldCNIIfaceIPOnNode, oldGlobalIPOnNode); err != nil { logger.Errorf(err, "Error deleting rules for Gateway healthcheck on node %q", n.nodeName) } } diff --git a/pkg/globalnet/controllers/node_controller_test.go b/pkg/globalnet/controllers/node_controller_test.go index 6b5405878..081401e78 100644 --- a/pkg/globalnet/controllers/node_controller_test.go +++ b/pkg/globalnet/controllers/node_controller_test.go @@ -29,6 +29,7 @@ import ( "github.com/submariner-io/submariner/pkg/globalnet/constants" "github.com/submariner-io/submariner/pkg/globalnet/controllers" "github.com/submariner-io/submariner/pkg/ipam" + "github.com/submariner-io/submariner/pkg/packetfilter" routeAgent "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,7 +47,7 @@ var _ = Describe("Node controller", func() { }) It("should allocate it and program the relevant iptable rules", func() { - t.awaitIPTableRules(t.awaitNodeGlobalIP("")) + t.awaitPacketFilterRules(t.awaitNodeGlobalIP("")) }) Context("and the IP pool is initially exhausted", func() { @@ -80,7 +81,7 @@ var _ = Describe("Node controller", func() { }) It("should program the relevant iptable rules", func() { - t.awaitIPTableRules(node.GetAnnotations()[constants.SmGlobalIP]) + t.awaitPacketFilterRules(node.GetAnnotations()[constants.SmGlobalIP]) }) It("should reserve the global IP", func() { @@ -94,7 +95,7 @@ var _ = Describe("Node controller", func() { It("should reallocate the global IP", func() { globalIP := t.awaitNodeGlobalIP(node.GetAnnotations()[constants.SmGlobalIP]) - t.awaitIPTableRules(globalIP) + t.awaitPacketFilterRules(globalIP) }) }) }) @@ -114,16 +115,16 @@ var _ = Describe("Node controller", func() { }) It("should allocate a global IP and program the relevant iptable rules", func() { - t.awaitIPTableRules(t.awaitNodeGlobalIP("")) + t.awaitPacketFilterRules(t.awaitNodeGlobalIP("")) }) Context("and programming of IP tables initially fails", func() { BeforeEach(func() { - t.ipt.AddFailOnAppendRuleMatcher(ContainSubstring(cniInterfaceIP)) + t.pFilter.AddFailOnAppendRuleMatcher(ContainSubstring(cniInterfaceIP)) }) It("should eventually allocate a global IP and program the relevant iptable rules", func() { - t.awaitIPTableRules(t.awaitNodeGlobalIP("")) + t.awaitPacketFilterRules(t.awaitNodeGlobalIP("")) }) }) }) @@ -140,8 +141,8 @@ var _ = Describe("Node controller", func() { addAnnotation(node, routeAgent.CNIInterfaceIP, cniInterfaceIP) test.UpdateResource(t.nodes, node) - t.ipt.AwaitNoRule("nat", constants.SmGlobalnetIngressChain, ContainSubstring(oldCNIIfaceIP)) - t.awaitIPTableRules(node.GetAnnotations()[constants.SmGlobalIP]) + t.pFilter.AwaitNoRule(packetfilter.TableTypeNAT, constants.SmGlobalnetIngressChain, ContainSubstring(oldCNIIfaceIP)) + t.awaitPacketFilterRules(node.GetAnnotations()[constants.SmGlobalIP]) t.verifyIPsReservedInPool(node.GetAnnotations()[constants.SmGlobalIP]) }) }) @@ -204,6 +205,7 @@ func (t *nodeControllerTestDriver) start() { Expect(t.controller.Start()).To(Succeed()) } -func (t *nodeControllerTestDriver) awaitIPTableRules(globalIP string) { - t.ipt.AwaitRule("nat", constants.SmGlobalnetIngressChain, And(ContainSubstring(globalIP), ContainSubstring(cniInterfaceIP))) +func (t *nodeControllerTestDriver) awaitPacketFilterRules(globalIP string) { + t.pFilter.AwaitRule(packetfilter.TableTypeNAT, + constants.SmGlobalnetIngressChain, And(ContainSubstring(globalIP), ContainSubstring(cniInterfaceIP))) } diff --git a/pkg/globalnet/controllers/packetfilter/iface.go b/pkg/globalnet/controllers/packetfilter/iface.go new file mode 100644 index 000000000..8f103d464 --- /dev/null +++ b/pkg/globalnet/controllers/packetfilter/iface.go @@ -0,0 +1,317 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright Contributors to the Submariner project. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package packetfilter + +import ( + "fmt" + + "github.com/pkg/errors" + "github.com/submariner-io/admiral/pkg/log" + "github.com/submariner-io/submariner/pkg/globalnet/constants" + "github.com/submariner-io/submariner/pkg/packetfilter" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +type Interface interface { + AddClusterEgressRules(sourceIP, snatIP, globalNetIPTableMark string) error + RemoveClusterEgressRules(sourceIP, snatIP, globalNetIPTableMark string) error + AddIngressRulesForHeadlessSvc(globalIP, podIP string, targetType TargetType) error + RemoveIngressRulesForHeadlessSvc(globalIP, podIP string, targetType TargetType) error + AddIngressRulesForHealthCheck(cniIfaceIP, globalIP string) error + RemoveIngressRulesForHealthCheck(cniIfaceIP, globalIP string) error + AddEgressRulesForHeadlessSvc(key, sourceIP, snatIP, globalNetIPTableMark string, targetType TargetType) error + RemoveEgressRulesForHeadlessSvc(key, sourceIP, snatIP, globalNetIPTableMark string, targetType TargetType) error + + AddEgressRulesForPods(key, ipSetName, snatIP, globalNetIPTableMark string) error + RemoveEgressRulesForPods(key, ipSetName, snatIP, globalNetIPTableMark string) error + AddEgressRulesForNamespace(namespace, ipSetName, snatIP, globalNetIPTableMark string) error + RemoveEgressRulesForNamespace(namespace, ipSetName, snatIP, globalNetIPTableMark string) error + FlushNatChain(chainName string) error + DeleteNatChain(chainName string) error +} + +type pfilter struct { + pFilter packetfilter.Interface +} + +type TargetType string + +const ( + PodTarget TargetType = "Pod" + EndpointsTarget TargetType = "Endpoints" +) + +var logger = log.Logger{Logger: logf.Log.WithName("PacketFilter")} + +func New() (Interface, error) { + pFilterHandler, err := packetfilter.New() + if err != nil { + return nil, err //nolint:wrapcheck // Let the caller wrap it + } + + pFilterIface := &pfilter{ + pFilter: pFilterHandler, + } + + return pFilterIface, nil +} + +func (i *pfilter) AddClusterEgressRules(subnet, snatIP, globalNetIPTableMark string) error { + ruleSpec := packetfilter.Rule{ + Proto: packetfilter.RuleProtoAll, + SrcCIDR: subnet, + MarkValue: globalNetIPTableMark, + SnatCIDR: snatIP, + Action: packetfilter.RuleActionSNAT, + } + + if err := i.pFilter.AppendUnique(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChainForCluster, &ruleSpec); err != nil { + return errors.Wrapf(err, "unable to append rule %+v", &ruleSpec) + } + + return nil +} + +func (i *pfilter) RemoveClusterEgressRules(subnet, snatIP, globalNetIPTableMark string) error { + ruleSpec := packetfilter.Rule{ + Proto: packetfilter.RuleProtoAll, + SrcCIDR: subnet, + MarkValue: globalNetIPTableMark, + SnatCIDR: snatIP, + Action: packetfilter.RuleActionSNAT, + } + + if err := i.pFilter.Delete(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChainForCluster, &ruleSpec); err != nil { + return errors.Wrapf(err, "error deleting packetfilter rule %+v", &ruleSpec) + } + + return nil +} + +func (i *pfilter) AddIngressRulesForHeadlessSvc(globalIP, ip string, targetType TargetType) error { + if globalIP == "" || ip == "" { + return fmt.Errorf("globalIP %q or %s IP %q cannot be empty", globalIP, targetType, ip) + } + + ruleSpec := packetfilter.Rule{ + DestCIDR: globalIP, + DnatCIDR: ip, + Action: packetfilter.RuleActionDNAT, + } + logger.V(log.DEBUG).Infof("Installing packetfilter rule for Headless SVC %+v for %s", &ruleSpec, targetType) + + if err := i.pFilter.AppendUnique(packetfilter.TableTypeNAT, constants.SmGlobalnetIngressChain, &ruleSpec); err != nil { + return errors.Wrapf(err, "unable to append rule %+v", &ruleSpec) + } + + return nil +} + +func (i *pfilter) RemoveIngressRulesForHeadlessSvc(globalIP, ip string, targetType TargetType) error { + if globalIP == "" || ip == "" { + return fmt.Errorf("globalIP %q or %s IP %q cannot be empty", globalIP, targetType, ip) + } + + ruleSpec := packetfilter.Rule{ + DestCIDR: globalIP, + DnatCIDR: ip, + Action: packetfilter.RuleActionDNAT, + } + logger.V(log.DEBUG).Infof("Deleting iptables rule for Headless SVC %+v for %s", &ruleSpec, targetType) + + if err := i.pFilter.Delete(packetfilter.TableTypeNAT, constants.SmGlobalnetIngressChain, &ruleSpec); err != nil { + return errors.Wrapf(err, "error deleting packetfilter rule %+v", &ruleSpec) + } + + return nil +} + +func (i *pfilter) AddIngressRulesForHealthCheck(cniIfaceIP, globalIP string) error { + ruleSpec := packetfilter.Rule{ + Proto: packetfilter.RuleProtoICMP, + DestCIDR: globalIP, + DnatCIDR: cniIfaceIP, + Action: packetfilter.RuleActionDNAT, + } + logger.V(log.DEBUG).Infof("Installing packetfilter ingress rules for Node: %+v", &ruleSpec) + + if err := i.pFilter.AppendUnique(packetfilter.TableTypeNAT, constants.SmGlobalnetIngressChain, &ruleSpec); err != nil { + return errors.Wrapf(err, "unable to append rule %+v", &ruleSpec) + } + + return nil +} + +func (i *pfilter) RemoveIngressRulesForHealthCheck(cniIfaceIP, globalIP string) error { + ruleSpec := packetfilter.Rule{ + Proto: packetfilter.RuleProtoICMP, + DestCIDR: globalIP, + DnatCIDR: cniIfaceIP, + Action: packetfilter.RuleActionDNAT, + } + logger.V(log.DEBUG).Infof("Deleting packetfilter ingress rules for Node: %+v", &ruleSpec) + + if err := i.pFilter.Delete(packetfilter.TableTypeNAT, constants.SmGlobalnetIngressChain, &ruleSpec); err != nil { + return errors.Wrapf(err, "error deleting packetfilter rule %+v", &ruleSpec) + } + + return nil +} + +func (i *pfilter) AddEgressRulesForHeadlessSvc(key, sourceIP, snatIP, globalNetIPTableMark string, targetType TargetType) error { + ruleSpec := packetfilter.Rule{ + Proto: packetfilter.RuleProtoAll, + SrcCIDR: sourceIP, + MarkValue: globalNetIPTableMark, + SnatCIDR: snatIP, + Action: packetfilter.RuleActionSNAT, + } + logger.V(log.DEBUG).Infof("Installing packetfilter egress rules for HDLS SVC %q for %s: %+v", key, targetType, &ruleSpec) + + var chain string + + if targetType == PodTarget { + chain = constants.SmGlobalnetEgressChainForHeadlessSvcPods + } else if targetType == EndpointsTarget { + chain = constants.SmGlobalnetEgressChainForHeadlessSvcEPs + } + + if err := i.pFilter.AppendUnique(packetfilter.TableTypeNAT, chain, &ruleSpec); err != nil { + return errors.Wrapf(err, "error appending packetfilter rule %+v", &ruleSpec) + } + + return nil +} + +func (i *pfilter) RemoveEgressRulesForHeadlessSvc(key, sourceIP, snatIP, globalNetIPTableMark string, targetType TargetType) error { + ruleSpec := packetfilter.Rule{ + Proto: packetfilter.RuleProtoAll, + SrcCIDR: sourceIP, + MarkValue: globalNetIPTableMark, + SnatCIDR: snatIP, + Action: packetfilter.RuleActionSNAT, + } + logger.V(log.DEBUG).Infof("Deleting iptable egress rules for HDLS SVC %q for %s: %+v", key, targetType, &ruleSpec) + + var chain string + + if targetType == PodTarget { + chain = constants.SmGlobalnetEgressChainForHeadlessSvcPods + } else if targetType == EndpointsTarget { + chain = constants.SmGlobalnetEgressChainForHeadlessSvcEPs + } + + if err := i.pFilter.Delete(packetfilter.TableTypeNAT, chain, &ruleSpec); err != nil { + return errors.Wrapf(err, "error deleting packetfilter rule %+v", &ruleSpec) + } + + return nil +} + +func (i *pfilter) AddEgressRulesForPods(key, ipSetName, snatIP, globalNetIPTableMark string) error { + ruleSpec := &packetfilter.Rule{ + Proto: packetfilter.RuleProtoAll, + SrcSetName: ipSetName, + SnatCIDR: snatIP, + MarkValue: globalNetIPTableMark, + Action: packetfilter.RuleActionSNAT, + } + + logger.V(log.DEBUG).Infof("Installing egress rules for pods %q: %q", key, ruleSpec) + + if err := i.pFilter.AppendUnique(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChainForPods, ruleSpec); err != nil { + return errors.Wrapf(err, "error appending rule \"%q\" chain:%s", ruleSpec, constants.SmGlobalnetEgressChainForPods) + } + + return nil +} + +func (i *pfilter) RemoveEgressRulesForPods(key, ipSetName, snatIP, globalNetIPTableMark string) error { + ruleSpec := &packetfilter.Rule{ + Proto: packetfilter.RuleProtoAll, + SrcSetName: ipSetName, + SnatCIDR: snatIP, + MarkValue: globalNetIPTableMark, + Action: packetfilter.RuleActionSNAT, + } + + logger.V(log.DEBUG).Infof("Deleting egress rules for Pods %q: %q", key, ruleSpec) + + if err := i.pFilter.Delete(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChainForPods, ruleSpec); err != nil { + return errors.Wrapf(err, "error rule \"%q\" chain:%s", ruleSpec, constants.SmGlobalnetEgressChainForPods) + } + + return nil +} + +func (i *pfilter) AddEgressRulesForNamespace(namespace, ipSetName, snatIP, globalNetIPTableMark string) error { + ruleSpec := &packetfilter.Rule{ + Proto: packetfilter.RuleProtoAll, + SrcSetName: ipSetName, + SnatCIDR: snatIP, + MarkValue: globalNetIPTableMark, + Action: packetfilter.RuleActionSNAT, + } + + logger.V(log.DEBUG).Infof("Installing egress rules for Namespace %q: %q", namespace, ruleSpec) + + if err := i.pFilter.AppendUnique(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChainForNamespace, ruleSpec); err != nil { + return errors.Wrapf(err, "error appending rule \"%q\" chain:%s", ruleSpec, constants.SmGlobalnetEgressChainForNamespace) + } + + return nil +} + +func (i *pfilter) RemoveEgressRulesForNamespace(namespace, ipSetName, snatIP, globalNetIPTableMark string) error { + ruleSpec := &packetfilter.Rule{ + Proto: packetfilter.RuleProtoAll, + SrcSetName: ipSetName, + SnatCIDR: snatIP, + MarkValue: globalNetIPTableMark, + Action: packetfilter.RuleActionSNAT, + } + + logger.V(log.DEBUG).Infof("Deleting egress rules for Namespace %q: %q", namespace, ruleSpec) + + if err := i.pFilter.Delete(packetfilter.TableTypeNAT, constants.SmGlobalnetEgressChainForNamespace, ruleSpec); err != nil { + return errors.Wrapf(err, "error rule \"%q\" chain:%s", ruleSpec, constants.SmGlobalnetEgressChainForNamespace) + } + + return nil +} + +func (i *pfilter) FlushNatChain(chainName string) error { + logger.Infof("Flushing packetfilter rules in %q chain of table NAT", chainName) + + if err := i.pFilter.ClearChain(packetfilter.TableTypeNAT, chainName); err != nil { + return errors.Wrapf(err, "error flushing packetfilter rules in %q chain of table NAT", chainName) + } + + return nil +} + +func (i *pfilter) DeleteNatChain(chainName string) error { + logger.Infof("Deleting packetfilter chain %q of table NAT", chainName) + + if err := i.pFilter.DeleteChain(packetfilter.TableTypeNAT, chainName); err != nil { + return errors.Wrapf(err, "error deleting packetfilter chain %q of table NAT", chainName) + } + + return nil +} diff --git a/pkg/globalnet/controllers/service_export_controller.go b/pkg/globalnet/controllers/service_export_controller.go index a2f6a04f8..6fffa0e16 100644 --- a/pkg/globalnet/controllers/service_export_controller.go +++ b/pkg/globalnet/controllers/service_export_controller.go @@ -24,7 +24,7 @@ import ( "github.com/submariner-io/admiral/pkg/syncer" "github.com/submariner-io/admiral/pkg/util" submarinerv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1" - "github.com/submariner-io/submariner/pkg/globalnet/controllers/iptables" + gnpacketfilter "github.com/submariner-io/submariner/pkg/globalnet/controllers/packetfilter" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -72,12 +72,12 @@ func NewServiceExportController(config *syncer.ResourceSyncerConfig, podControll return nil, errors.Wrap(err, "error creating the syncer") } - iptIface, err := iptables.New() + pfIface, err := gnpacketfilter.New() if err != nil { - return nil, errors.Wrap(err, "error creating the IPTablesInterface handler") + return nil, errors.Wrap(err, "error creating the Packetfilter Interface handler") } - controller.iptIface = iptIface + controller.pfIface = pfIface _, gvr, err = util.ToUnstructuredResource(&submarinerv1.GlobalIngressIP{}, config.RestMapper) if err != nil { diff --git a/pkg/globalnet/controllers/service_export_controller_test.go b/pkg/globalnet/controllers/service_export_controller_test.go index 44f38bcd2..ecff4156b 100644 --- a/pkg/globalnet/controllers/service_export_controller_test.go +++ b/pkg/globalnet/controllers/service_export_controller_test.go @@ -29,6 +29,7 @@ import ( submarinerv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1" "github.com/submariner-io/submariner/pkg/globalnet/controllers" "github.com/submariner-io/submariner/pkg/ipam" + "github.com/submariner-io/submariner/pkg/packetfilter" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -48,7 +49,7 @@ func testClusterIPService() { BeforeEach(func() { service = newClusterIPService() - t.createIPTableChain("nat", kubeProxyIPTableChainName) + t.createPFilterChain(packetfilter.TableTypeNAT, kubeProxyIPTableChainName) }) When("an existing Service is exported", func() { diff --git a/pkg/globalnet/controllers/types.go b/pkg/globalnet/controllers/types.go index 858222420..261635c9e 100644 --- a/pkg/globalnet/controllers/types.go +++ b/pkg/globalnet/controllers/types.go @@ -28,10 +28,10 @@ import ( "github.com/submariner-io/admiral/pkg/syncer" "github.com/submariner-io/admiral/pkg/watcher" "github.com/submariner-io/submariner/pkg/event" - iptiface "github.com/submariner-io/submariner/pkg/globalnet/controllers/iptables" + pfIface "github.com/submariner-io/submariner/pkg/globalnet/controllers/packetfilter" "github.com/submariner-io/submariner/pkg/ipam" "github.com/submariner-io/submariner/pkg/ipset" - "github.com/submariner-io/submariner/pkg/iptables" + "github.com/submariner-io/submariner/pkg/packetfilter" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -128,7 +128,7 @@ type gatewayMonitor struct { GatewayMonitorConfig syncerConfig *syncer.ResourceSyncerConfig remoteEndpointTimeStamp map[string]metav1.Time - ipt iptables.Interface + pFilter packetfilter.Interface shuttingDown atomic.Bool leaderElectionInfo atomic.Pointer[LeaderElectionInfo] nodeName string @@ -143,8 +143,8 @@ type baseSyncerController struct { type baseIPAllocationController struct { *baseSyncerController - pool *ipam.IPPool - iptIface iptiface.Interface + pool *ipam.IPPool + pfIface pfIface.Interface } type globalEgressIPController struct { @@ -178,7 +178,7 @@ type serviceExportController struct { *baseSyncerController services dynamic.NamespaceableResourceInterface ingressIPs dynamic.ResourceInterface - iptIface iptiface.Interface + pfIface pfIface.Interface podControllers *IngressPodControllers endpointsControllers *ServiceExportEndpointsControllers ingressEndpointsControllers *IngressEndpointsControllers diff --git a/pkg/globalnet/controllers/uninstall.go b/pkg/globalnet/controllers/uninstall.go index 838749116..551c95732 100644 --- a/pkg/globalnet/controllers/uninstall.go +++ b/pkg/globalnet/controllers/uninstall.go @@ -32,8 +32,9 @@ import ( submarinerv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1" versioned "github.com/submariner-io/submariner/pkg/client/clientset/versioned" "github.com/submariner-io/submariner/pkg/globalnet/constants" - "github.com/submariner-io/submariner/pkg/globalnet/controllers/iptables" + pfiface "github.com/submariner-io/submariner/pkg/globalnet/controllers/packetfilter" "github.com/submariner-io/submariner/pkg/ipset" + "github.com/submariner-io/submariner/pkg/packetfilter" routeAgent "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -48,8 +49,11 @@ import ( ) func UninstallDataPath() { - ipt, err := iptables.New() - logger.FatalOnError(err, "Error initializing IP tables") + gnpFilter, err := pfiface.New() + logger.FatalOnError(err, "Error initializing GN PacketFilter") + + pFilter, err := packetfilter.New() + logger.FatalOnError(err, "Error initializing PacketFilter") natTableChains := []string{ // The chains have to be deleted in a specific order. @@ -64,25 +68,35 @@ func UninstallDataPath() { } for _, chain := range natTableChains { - err = ipt.FlushIPTableChain(constants.NATTable, chain) + err = gnpFilter.FlushNatChain(chain) if err != nil { // Just log an error as this is part of uninstallation. - logger.Errorf(err, "Error flushing iptables chain %q", chain) + logger.Errorf(err, "Error flushing packetfilter chain %q", chain) } } - err = ipt.FlushIPTableChain(constants.NATTable, routeAgent.SmPostRoutingChain) + err = gnpFilter.FlushNatChain(routeAgent.SmPostRoutingChain) if err != nil { - logger.Errorf(err, "Error flushing iptables chain %q", routeAgent.SmPostRoutingChain) + logger.Errorf(err, "Error flushing packetfilter chain %q", routeAgent.SmPostRoutingChain) + } + + chain := packetfilter.ChainIPHook{ + Name: constants.SmGlobalnetIngressChain, + Type: packetfilter.ChainTypeNAT, + Hook: packetfilter.ChainHookPrerouting, + Priority: packetfilter.ChainPriorityFirst, } - if err := ipt.DeleteIPTableRule(constants.NATTable, "PREROUTING", constants.SmGlobalnetIngressChain); err != nil { - logger.Errorf(err, "Error deleting iptables rule for %q in PREROUTING chain", constants.SmGlobalnetIngressChain) + if err := pFilter.DeleteIPHookChain(&chain); err != nil { + logger.Errorf(err, "error creating IPHook chain %s", constants.SmGlobalnetIngressChain) } for _, chain := range natTableChains { - err = ipt.DeleteIPTableChain(constants.NATTable, chain) - if err != nil { + if chain == constants.SmGlobalnetIngressChain { + continue + } + + if err = gnpFilter.DeleteNatChain(chain); err != nil { logger.Errorf(err, "Error deleting iptables chain %q", chain) } } diff --git a/pkg/globalnet/main.go b/pkg/globalnet/main.go index 97002fa43..f909f8f25 100644 --- a/pkg/globalnet/main.go +++ b/pkg/globalnet/main.go @@ -38,6 +38,8 @@ import ( "github.com/submariner-io/submariner/pkg/cidr" submarinerClientset "github.com/submariner-io/submariner/pkg/client/clientset/versioned" "github.com/submariner-io/submariner/pkg/globalnet/controllers" + packetfilter "github.com/submariner-io/submariner/pkg/packetfilter" + iptables "github.com/submariner-io/submariner/pkg/packetfilter/iptables" "github.com/submariner-io/submariner/pkg/versions" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/dynamic" @@ -92,6 +94,10 @@ func main() { logger.Info("Starting submariner-globalnet", spec) + // set packetfilter driver to iptables + // TODO: check which driver is supported on platform + packetfilter.SetNewDriverFn(iptables.New) + // set up signals so we handle the first shutdown signal gracefully stopCh := signals.SetupSignalHandler().Done() diff --git a/pkg/iptables/adapter.go b/pkg/iptables/adapter.go deleted file mode 100644 index 5b699110a..000000000 --- a/pkg/iptables/adapter.go +++ /dev/null @@ -1,145 +0,0 @@ -/* -SPDX-License-Identifier: Apache-2.0 - -Copyright Contributors to the Submariner project. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package iptables - -import ( - "strings" - - "github.com/pkg/errors" - level "github.com/submariner-io/admiral/pkg/log" - "k8s.io/utils/set" -) - -type Adapter struct { - Basic -} - -func (a *Adapter) CreateChainIfNotExists(table, chain string) error { - exists, err := a.ChainExists(table, chain) - if err == nil && exists { - return nil - } - - if err != nil { - return errors.Wrapf(err, "error finding IP table chain %q in table %q", chain, table) - } - - return errors.Wrap(a.NewChain(table, chain), "error creating IP table chain") -} - -func (a *Adapter) InsertUnique(table, chain string, position int, ruleSpec []string) error { - rules, err := a.List(table, chain) - if err != nil { - return errors.Wrapf(err, "error listing the rules in %s chain", chain) - } - - isPresentAtRequiredPosition := false - numOccurrences := 0 - - for index, rule := range rules { - if strings.Contains(rule, strings.Join(ruleSpec, " ")) { - logger.V(level.DEBUG).Infof("In %s table, iptables rule \"%s\", exists at index %d.", table, strings.Join(ruleSpec, " "), index) - numOccurrences++ - - if index == position { - isPresentAtRequiredPosition = true - } - } - } - - // The required rule is present in the Chain, but either there are multiple occurrences or its - // not at the desired location - if numOccurrences > 1 || !isPresentAtRequiredPosition { - for i := 0; i < numOccurrences; i++ { - if err = a.Delete(table, chain, ruleSpec...); err != nil { - return errors.Wrapf(err, "error deleting stale IP table rule %q", strings.Join(ruleSpec, " ")) - } - } - } - - // The required rule is present only once and is at the desired location - if numOccurrences == 1 && isPresentAtRequiredPosition { - logger.V(level.DEBUG).Infof("In %s table, iptables rule \"%s\", already exists.", table, strings.Join(ruleSpec, " ")) - return nil - } else if err := a.Insert(table, chain, position, ruleSpec...); err != nil { - return errors.Wrapf(err, "error inserting IP table rule %q", strings.Join(ruleSpec, " ")) - } - - return nil -} - -func (a *Adapter) PrependUnique(table, chain string, ruleSpec []string) error { - // Submariner requires certain iptable rules to be programmed at the beginning of an iptables Chain - // so that we can preserve the sourceIP for inter-cluster traffic and avoid K8s SDN making changes - // to the traffic. - // In this API, we check if the required iptable rule is present at the beginning of the chain. - // If the rule is already present and there are no stale[1] flows, we simply return. If not, we create one. - // [1] Sometimes after we program the rule at the beginning of the chain, K8s SDN might insert some - // new rules ahead of the rule that we programmed. In such cases, the rule that we programmed will - // not be the first rule to hit and Submariner behavior might get affected. So, we query the rules - // in the chain to see if the rule slipped its position, and if so, delete all such occurrences. - // We then re-program a new rule at the beginning of the chain as required. - return a.InsertUnique(table, chain, 1, ruleSpec) -} - -func (a *Adapter) UpdateChainRules(table, chain string, rules [][]string) error { - existingRules, err := a.List(table, chain) - if err != nil { - return errors.Wrapf(err, "error listing the rules in table %q, chain %q", table, chain) - } - - ruleStrings := set.New[string]() - - for _, existingRule := range existingRules { - ruleSpec := strings.Split(existingRule, " ") - if ruleSpec[0] == "-A" { - ruleSpec = ruleSpec[2:] // remove "-A", "$chain" - ruleStrings.Insert(strings.Trim(strings.Join(ruleSpec, " "), " ")) - } - } - - for _, ruleSpec := range rules { - ruleString := strings.Join(ruleSpec, " ") - - if ruleStrings.Has(ruleString) { - ruleStrings.Delete(ruleString) - } else { - logger.V(level.DEBUG).Infof("Adding iptables rule in %q, %q: %q", table, chain, ruleSpec) - - if err := a.Append(table, chain, ruleSpec...); err != nil { - return errors.Wrapf(err, "error adding rule to %v to %q, %q", ruleSpec, table, chain) - } - } - } - - // remaining elements should not be there, remove them - for _, rule := range ruleStrings.UnsortedList() { - logger.V(level.DEBUG).Infof("Deleting stale iptables rule in %q, %q: %q", table, chain, rule) - ruleSpec := strings.Split(rule, " ") - - if err := a.Delete(table, chain, ruleSpec...); err != nil { - // Log and let go, as this is not a fatal error, or something that will make real harm, - // it's more harmful to keep retrying. At this point on next update deletion of stale rules - // will happen again - logger.Warningf("Unable to delete iptables entry from table %q, chain %q: %q", table, chain, rule) - } - } - - return nil -} diff --git a/pkg/iptables/fake/iptables.go b/pkg/iptables/fake/iptables.go deleted file mode 100644 index b261a3e26..000000000 --- a/pkg/iptables/fake/iptables.go +++ /dev/null @@ -1,243 +0,0 @@ -/* -SPDX-License-Identifier: Apache-2.0 - -Copyright Contributors to the Submariner project. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package fake - -import ( - "errors" - "strings" - "sync" - - . "github.com/onsi/gomega" - "github.com/submariner-io/submariner/pkg/iptables" - "k8s.io/utils/set" -) - -type basicType struct { - mutex sync.Mutex - chainRules map[string]set.Set[string] - tableChains map[string]set.Set[string] - failOnAppendRuleMatchers []interface{} - failOnDeleteRuleMatchers []interface{} -} - -type IPTables struct { - iptables.Adapter -} - -func New() *IPTables { - return &IPTables{ - Adapter: iptables.Adapter{ - Basic: &basicType{ - chainRules: map[string]set.Set[string]{}, - tableChains: map[string]set.Set[string]{}, - }, - }, - } -} - -func (i *basicType) Append(table, chain string, rulespec ...string) error { - return i.addRule(table, chain, rulespec...) -} - -func (i *basicType) AppendUnique(table, chain string, rulespec ...string) error { - return i.addRule(table, chain, rulespec...) -} - -func (i *basicType) Insert(table, chain string, _ int, rulespec ...string) error { - return i.addRule(table, chain, rulespec...) -} - -func (i *basicType) Delete(table, chain string, rulespec ...string) error { - i.mutex.Lock() - defer i.mutex.Unlock() - - err := matchRuleForError(&i.failOnDeleteRuleMatchers, rulespec...) - if err != nil { - return err - } - - ruleSet := i.chainRules[table+"/"+chain] - if ruleSet != nil { - ruleSet.Delete(strings.Join(rulespec, " ")) - } - - return nil -} - -func (i *basicType) addRule(table, chain string, rulespec ...string) error { - i.mutex.Lock() - defer i.mutex.Unlock() - - err := matchRuleForError(&i.failOnAppendRuleMatchers, rulespec...) - if err != nil { - return err - } - - ruleSet := i.chainRules[table+"/"+chain] - if ruleSet == nil { - ruleSet = set.New[string]() - i.chainRules[table+"/"+chain] = ruleSet - } - - ruleSet.Insert(strings.Join(rulespec, " ")) - - return nil -} - -func matchRuleForError(matchers *[]interface{}, rulespec ...string) error { - for i, m := range *matchers { - matches, err := ContainElement(m).Match([]string{strings.Join(rulespec, " ")}) - Expect(err).To(Succeed()) - - if matches { - *matchers = (*matchers)[i+1:] - return errors.New("mock IP table rule error") - } - } - - return nil -} - -func (i *basicType) List(table, chain string) ([]string, error) { - return i.listRules(table, chain), nil -} - -func (i *basicType) listRules(table, chain string) []string { - i.mutex.Lock() - defer i.mutex.Unlock() - - rules := i.chainRules[table+"/"+chain] - if rules != nil { - return rules.UnsortedList() - } - - return []string{} -} - -func (i *basicType) ListChains(table string) ([]string, error) { - return i.listChains(table), nil -} - -func (i *basicType) listChains(table string) []string { - i.mutex.Lock() - defer i.mutex.Unlock() - - chains := i.tableChains[table] - if chains != nil { - return chains.UnsortedList() - } - - return []string{} -} - -func (i *basicType) NewChain(table, chain string) error { - i.addChainsFor(table, chain) - return nil -} - -func (i *basicType) ClearChain(table, chain string) error { - i.mutex.Lock() - defer i.mutex.Unlock() - - chainSet := i.tableChains[table] - if chainSet != nil { - chainSet.Delete(chain) - } - - return nil -} - -func (i *basicType) ChainExists(table, chain string) (bool, error) { - i.mutex.Lock() - defer i.mutex.Unlock() - - chainSet := i.tableChains[table] - if chainSet != nil { - return chainSet.Has(chain), nil - } - - return false, nil -} - -func (i *basicType) DeleteChain(table, chain string) error { - i.mutex.Lock() - defer i.mutex.Unlock() - - chainSet := i.tableChains[table] - if chainSet != nil { - chainSet.Delete(chain) - } - - return nil -} - -func (i *basicType) addChainsFor(table string, chains ...string) { - i.mutex.Lock() - defer i.mutex.Unlock() - - chainSet := i.tableChains[table] - if chainSet == nil { - chainSet = set.New[string]() - i.tableChains[table] = chainSet - } - - chainSet.Insert(chains...) -} - -func (i *IPTables) AwaitChain(table string, stringOrMatcher interface{}) { - Eventually(func() []string { - return i.basic().listChains(table) - }, 5).Should(ContainElement(stringOrMatcher), "IP table %q chains", table) -} - -func (i *IPTables) AwaitNoChain(table string, stringOrMatcher interface{}) { - Eventually(func() []string { - return i.basic().listChains(table) - }, 5).ShouldNot(ContainElement(stringOrMatcher), "IP table %q chains", table) -} - -func (i *IPTables) AwaitRule(table, chain string, stringOrMatcher interface{}) { - Eventually(func() []string { - return i.basic().listRules(table, chain) - }, 5).Should(ContainElement(stringOrMatcher), "Rules for IP table %q, chain %q", table, chain) -} - -func (i *IPTables) AwaitNoRule(table, chain string, stringOrMatcher interface{}) { - Eventually(func() []string { - return i.basic().listRules(table, chain) - }, 5).ShouldNot(ContainElement(stringOrMatcher), "Rules for IP table %q, chain %q", table, chain) -} - -func (i *IPTables) AddFailOnAppendRuleMatcher(stringOrMatcher interface{}) { - i.basic().mutex.Lock() - defer i.basic().mutex.Unlock() - - i.basic().failOnAppendRuleMatchers = append(i.basic().failOnAppendRuleMatchers, stringOrMatcher) -} - -func (i *IPTables) AddFailOnDeleteRuleMatcher(stringOrMatcher interface{}) { - i.basic().mutex.Lock() - defer i.basic().mutex.Unlock() - - i.basic().failOnDeleteRuleMatchers = append(i.basic().failOnDeleteRuleMatchers, stringOrMatcher) -} - -func (i *IPTables) basic() *basicType { - return i.Adapter.Basic.(*basicType) -} diff --git a/pkg/iptables/iptables.go b/pkg/iptables/iptables.go deleted file mode 100644 index c479d0029..000000000 --- a/pkg/iptables/iptables.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -SPDX-License-Identifier: Apache-2.0 - -Copyright Contributors to the Submariner project. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package iptables - -import ( - "github.com/coreos/go-iptables/iptables" - "github.com/pkg/errors" - "github.com/submariner-io/admiral/pkg/log" - logf "sigs.k8s.io/controller-runtime/pkg/log" -) - -type Basic interface { - Append(table, chain string, rulespec ...string) error - AppendUnique(table, chain string, rulespec ...string) error - Delete(table, chain string, rulespec ...string) error - Insert(table, chain string, pos int, rulespec ...string) error - List(table, chain string) ([]string, error) - ListChains(table string) ([]string, error) - NewChain(table, chain string) error - ChainExists(table, chain string) (bool, error) - ClearChain(table, chain string) error - DeleteChain(table, chain string) error -} - -type Interface interface { - Basic - CreateChainIfNotExists(table, chain string) error - InsertUnique(table, chain string, position int, ruleSpec []string) error - PrependUnique(table, chain string, ruleSpec []string) error - // UpdateChainRules ensures that the rules in the list are the ones in rules, without any preference for the order, - // any stale rules will be removed from the chain, and any missing rules will be added. - UpdateChainRules(table, chain string, rules [][]string) error -} - -type iptablesWrapper struct { - *iptables.IPTables -} - -var logger = log.Logger{Logger: logf.Log.WithName("IPTables")} - -var NewFunc func() (Interface, error) - -func New() (Interface, error) { - if NewFunc != nil { - return NewFunc() - } - - ipt, err := iptables.New(iptables.IPFamily(iptables.ProtocolIPv4), iptables.Timeout(5)) - if err != nil { - return nil, errors.Wrap(err, "error creating IP tables") - } - - return &Adapter{Basic: &iptablesWrapper{IPTables: ipt}}, nil -} - -func (i *iptablesWrapper) Delete(table, chain string, rulespec ...string) error { - err := i.IPTables.Delete(table, chain, rulespec...) - - var iptError *iptables.Error - - ok := errors.As(err, &iptError) - if ok && iptError.IsNotExist() { - return nil - } - - return errors.Wrap(err, "error deleting IP table rule") -} diff --git a/pkg/packetfilter/adapter.go b/pkg/packetfilter/adapter.go new file mode 100644 index 000000000..dc0f6991a --- /dev/null +++ b/pkg/packetfilter/adapter.go @@ -0,0 +1,114 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright Contributors to the Submariner project. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package packetfilter + +import ( + "fmt" + + "github.com/pkg/errors" + level "github.com/submariner-io/admiral/pkg/log" +) + +func (a *Adapter) PrependUnique(table TableType, chain string, ruleSpec *Rule) error { + return a.InsertUnique(table, chain, 1, ruleSpec) +} + +func (a *Adapter) UpdateChainRules(table TableType, chain string, rules []*Rule) error { + currentRules, err := a.List(table, chain) + if err != nil { + return errors.Wrapf(err, "error listing the rules in table %q, chain %q", table, chain) + } + + existingRules := make(map[string]*Rule) + for _, existingRule := range currentRules { + existingRules[fmt.Sprintf("%+v", existingRule)] = existingRule + } + + for _, rule := range rules { + ruleString := fmt.Sprintf("%+v", rule) + _, ok := existingRules[ruleString] + + if ok { + delete(existingRules, ruleString) + } else { + logger.V(level.DEBUG).Infof("Adding packetfilter rule in %q, %q: %q", table, chain, ruleString) + + if err := a.Append(table, chain, rule); err != nil { + return errors.Wrapf(err, "error adding rule %q to %q, %q", ruleString, table, chain) + } + } + } + + // remaining elements should not be there, remove them + for ruleStr, rule := range existingRules { + logger.V(level.DEBUG).Infof("Deleting stale packetfilter rule in %q, %q: %q", table, chain, ruleStr) + + if err := a.Delete(table, chain, rule); err != nil { + // Log and let go, as this is not a fatal error, or something that will make real harm, + // it's more harmful to keep retrying. At this point on next update deletion of stale rules + // will happen again + logger.Warningf("Unable to delete packetfilter entry from table %q, chain %q: %q", table, chain, ruleStr) + } + } + + return nil +} + +func (a *Adapter) InsertUnique(table TableType, chain string, position int, rule *Rule) error { + existingRules, err := a.List(table, chain) + if err != nil { + return errors.Wrapf(err, "error listing the rules in table %q, chain %q", table, chain) + } + + isPresentAtRequiredPosition := false + numOccurrences := 0 + + ruleString := fmt.Sprintf("%+v", rule) + + for index, rule := range existingRules { + if ruleString == fmt.Sprintf("%+v", rule) { + logger.V(level.DEBUG).Infof("In %q table, rule \"%s\", exists at index %d.", table, ruleString, index) + numOccurrences++ + + if index == position { + isPresentAtRequiredPosition = true + } + } + } + + // The required rule is present in the Chain, but either there are multiple occurrences or its + // not at the desired location + if numOccurrences > 1 || !isPresentAtRequiredPosition { + for i := 0; i < numOccurrences; i++ { + if err = a.Delete(table, chain, rule); err != nil { + return errors.Wrapf(err, "error deleting stale rule %q", ruleString) + } + } + } + + // The required rule is present only once and is at the desired location + if numOccurrences == 1 && isPresentAtRequiredPosition { + logger.V(level.DEBUG).Infof("In %q table, rule \"%s\", already exists.", table, ruleString) + return nil + } else if err := a.Insert(table, chain, position, rule); err != nil { + return errors.Wrapf(err, "error inserting IP rule %q", ruleString) + } + + return nil +} diff --git a/pkg/packetfilter/fake/packetfilter.go b/pkg/packetfilter/fake/packetfilter.go new file mode 100644 index 000000000..e2488a750 --- /dev/null +++ b/pkg/packetfilter/fake/packetfilter.go @@ -0,0 +1,344 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright Contributors to the Submariner project. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "encoding/json" + "fmt" + "sync" + + . "github.com/onsi/gomega" + "github.com/pkg/errors" + "github.com/submariner-io/submariner/pkg/packetfilter" + "k8s.io/utils/set" +) + +type PacketFilter struct { + mutex sync.Mutex + chainRules map[string]set.Set[string] + tableChains map[packetfilter.TableType]set.Set[string] + failOnAppendRuleMatchers []interface{} + failOnDeleteRuleMatchers []interface{} +} + +var ( + iphookChainTypeToTableType = [packetfilter.ChainTypeMAX]packetfilter.TableType{ + packetfilter.TableTypeFilter, + packetfilter.TableTypeRoute, + packetfilter.TableTypeNAT, + } + chainHookToStr = [packetfilter.ChainHookMAX]string{"PREROUTING", "INPUT", "FORWARD", "OUTPUT", "POSTROUTING"} +) + +func New() *PacketFilter { + pf := &PacketFilter{ + chainRules: map[string]set.Set[string]{}, + tableChains: map[packetfilter.TableType]set.Set[string]{}, + } + + packetfilter.SetNewDriverFn(func() (packetfilter.Driver, error) { + return pf, nil + }) + + return pf +} + +func (i *PacketFilter) ChainExists(table packetfilter.TableType, chain string) (bool, error) { + return i.chainExists(table, chain) +} + +func (i *PacketFilter) AppendUnique(table packetfilter.TableType, chain string, rule *packetfilter.Rule) error { + ruleSpecStr, err := json.Marshal(*rule) + if err != nil { + return errors.Wrap(err, "AppendUnique failed") + } + + return i.addRule(table, chain, string(ruleSpecStr)) +} + +func (i *PacketFilter) CreateIPHookChainIfNotExists(chain *packetfilter.ChainIPHook) error { + if err := i.createChainIfNotExists(iphookChainTypeToTableType[chain.Type], chain.Name); err != nil { + return errors.Wrapf(err, "error creating IP tables %v:%s chain", iphookChainTypeToTableType[chain.Type], chain.Name) + } + + return nil +} + +func (i *PacketFilter) CreateChainIfNotExists(table packetfilter.TableType, chain *packetfilter.Chain) error { + if err := i.createChainIfNotExists(table, chain.Name); err != nil { + return errors.Wrapf(err, "error creating IP tables %s chain", chain.Name) + } + + return nil +} + +func (i *PacketFilter) DeleteIPHookChain(chain *packetfilter.ChainIPHook) error { + tableType := iphookChainTypeToTableType[chain.Type] + jumpRule := chain.JumpRule + + if jumpRule == nil { + jumpRule = &packetfilter.Rule{ + TargetChain: chain.Name, + Action: packetfilter.RuleActionJump, + } + } + + if err := i.Delete(tableType, chainHookToStr[chain.Hook], jumpRule); err != nil { + return errors.Wrap(err, "error deleting Jump Rule") + } + + i.deleteChain(tableType, chain.Name) + + return nil +} + +func (i *PacketFilter) DeleteChain(table packetfilter.TableType, chain string) error { + i.deleteChain(table, chain) + + return nil +} + +func (i *PacketFilter) ClearChain(table packetfilter.TableType, chain string) error { + i.clearChain(table, chain) + + return nil +} + +func (i *PacketFilter) Delete(table packetfilter.TableType, chain string, rule *packetfilter.Rule) error { + ruleSpecStr, err := json.Marshal(*rule) + if err != nil { + return errors.Wrap(err, "Delete failed") + } + + return i.delete(table, chain, string(ruleSpecStr)) +} + +func fromRuleSpec(spec string) *packetfilter.Rule { + var rule packetfilter.Rule + + err := json.Unmarshal([]byte(spec), &rule) + if err != nil { + return nil + } + + return &rule +} + +func (i *PacketFilter) List(table packetfilter.TableType, chain string) ([]*packetfilter.Rule, error) { + existingRules := i.listRules(table, chain) + + rules := []*packetfilter.Rule{} + + for _, existingRule := range existingRules { + rules = append(rules, fromRuleSpec(existingRule)) + } + + return rules, nil +} + +func (i *PacketFilter) Append(table packetfilter.TableType, chain string, rule *packetfilter.Rule) error { + ruleSpecStr, err := json.Marshal(*rule) + if err != nil { + return errors.Wrap(err, "Append failed") + } + + return i.addRule(table, chain, string(ruleSpecStr)) +} + +func (i *PacketFilter) Insert(table packetfilter.TableType, chain string, _ int, ruleSpec *packetfilter.Rule) error { + ruleSpecStr, err := json.Marshal(*ruleSpec) + if err != nil { + return errors.Wrap(err, "Insert failed") + } + + return i.addRule(table, chain, string(ruleSpecStr)) +} + +func (i *PacketFilter) createChainIfNotExists(table packetfilter.TableType, chain string) error { + exists, err := i.chainExists(table, chain) + if err == nil && exists { + return nil + } + + if err != nil { + return errors.Wrapf(err, "error finding IP table chain %q in table %q", chain, table) + } + + i.addChainsFor(table, chain) + + return nil +} + +func (i *PacketFilter) delete(table packetfilter.TableType, chain, rulespec string) error { + i.mutex.Lock() + defer i.mutex.Unlock() + + err := matchRuleForError(&i.failOnDeleteRuleMatchers, rulespec) + if err != nil { + return err + } + + ruleSet := i.chainRules[fmt.Sprintf("%v/%s", table, chain)] + if ruleSet != nil { + ruleSet.Delete(rulespec) + } + + return nil +} + +func (i *PacketFilter) deleteChain(table packetfilter.TableType, chain string) { + i.mutex.Lock() + defer i.mutex.Unlock() + + chainSet := i.tableChains[table] + if chainSet != nil { + chainSet.Delete(chain) + } +} + +func (i *PacketFilter) clearChain(table packetfilter.TableType, chain string) { + i.mutex.Lock() + defer i.mutex.Unlock() + + chainSet := i.tableChains[table] + if chainSet != nil { + chainSet.Delete(chain) + } +} + +func (i *PacketFilter) addChainsFor(table packetfilter.TableType, chains ...string) { + i.mutex.Lock() + defer i.mutex.Unlock() + + chainSet := i.tableChains[table] + if chainSet == nil { + chainSet = set.New[string]() + i.tableChains[table] = chainSet + } + + chainSet.Insert(chains...) +} + +func (i *PacketFilter) addRule(table packetfilter.TableType, chain, rulespec string) error { + i.mutex.Lock() + defer i.mutex.Unlock() + + err := matchRuleForError(&i.failOnAppendRuleMatchers, rulespec) + if err != nil { + return err + } + + ruleSet := i.chainRules[fmt.Sprintf("%v/%s", table, chain)] + if ruleSet == nil { + ruleSet = set.New[string]() + i.chainRules[fmt.Sprintf("%v/%s", table, chain)] = ruleSet + } + + ruleSet.Insert(rulespec) + + return nil +} + +func (i *PacketFilter) listRules(table packetfilter.TableType, chain string) []string { + i.mutex.Lock() + defer i.mutex.Unlock() + + rules := i.chainRules[fmt.Sprintf("%v/%s", table, chain)] + if rules != nil { + return rules.UnsortedList() + } + + return []string{} +} + +func (i *PacketFilter) listChains(table packetfilter.TableType) []string { + i.mutex.Lock() + defer i.mutex.Unlock() + + chains := i.tableChains[table] + if chains != nil { + return chains.UnsortedList() + } + + return []string{} +} + +func (i *PacketFilter) chainExists(table packetfilter.TableType, chain string) (bool, error) { + i.mutex.Lock() + defer i.mutex.Unlock() + + chainSet := i.tableChains[table] + if chainSet != nil { + return chainSet.Has(chain), nil + } + + return false, nil +} + +func matchRuleForError(matchers *[]interface{}, rulespec string) error { + for i, m := range *matchers { + matches, err := ContainElement(m).Match([]string{rulespec}) + Expect(err).To(Succeed()) + + if matches { + *matchers = (*matchers)[i+1:] + return errors.New("mock IP table rule error") + } + } + + return nil +} + +func (i *PacketFilter) AwaitChain(table packetfilter.TableType, stringOrMatcher interface{}) { + Eventually(func() []string { + return i.listChains(table) + }, 5).Should(ContainElement(stringOrMatcher), "IP table %v chains", table) +} + +func (i *PacketFilter) AwaitNoChain(table packetfilter.TableType, stringOrMatcher interface{}) { + Eventually(func() []string { + return i.listChains(table) + }, 5).ShouldNot(ContainElement(stringOrMatcher), "IP table %v chains", table) +} + +func (i *PacketFilter) AwaitRule(table packetfilter.TableType, chain string, stringOrMatcher interface{}) { + Eventually(func() []string { + return i.listRules(table, chain) + }, 5).Should(ContainElement(stringOrMatcher), "Rules for IP table %v, chain %q", table, chain) +} + +func (i *PacketFilter) AwaitNoRule(table packetfilter.TableType, chain string, stringOrMatcher interface{}) { + Eventually(func() []string { + return i.listRules(table, chain) + }, 5).ShouldNot(ContainElement(stringOrMatcher), "Rules for IP table %v, chain %q", table, chain) +} + +func (i *PacketFilter) AddFailOnAppendRuleMatcher(stringOrMatcher interface{}) { + i.mutex.Lock() + defer i.mutex.Unlock() + + i.failOnAppendRuleMatchers = append(i.failOnAppendRuleMatchers, stringOrMatcher) +} + +func (i *PacketFilter) AddFailOnDeleteRuleMatcher(stringOrMatcher interface{}) { + i.mutex.Lock() + defer i.mutex.Unlock() + + i.failOnDeleteRuleMatchers = append(i.failOnDeleteRuleMatchers, stringOrMatcher) +} diff --git a/pkg/packetfilter/iptables/iptables.go b/pkg/packetfilter/iptables/iptables.go new file mode 100644 index 000000000..b16af35ad --- /dev/null +++ b/pkg/packetfilter/iptables/iptables.go @@ -0,0 +1,463 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright Contributors to the Submariner project. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package iptables + +import ( + "strings" + + "github.com/coreos/go-iptables/iptables" + "github.com/pkg/errors" + "github.com/submariner-io/admiral/pkg/log" + "github.com/submariner-io/submariner/pkg/packetfilter" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + remoteCIDRIPSet = "SUBMARINER-REMOTECIDRS" + localCIDRIPSet = "SUBMARINER-LOCALCIDRS" + smPostRoutingChain = "SUBMARINER-POSTROUTING" + smGlobalnetEgressChainForPods = "SM-GN-EGRESS-PODS" + smGlobalnetEgressChainForNamespace = "SM-GN-EGRESS-NS" + gnIPSetPrefix = "SM-GN-" + DriverName = "IPTables" +) + +var ( + tableTypeToStr = [packetfilter.TableTypeMAX]string{"filter", "mangle", "nat"} + iphookChainTypeToStr = [packetfilter.ChainTypeMAX]string{"filter", "mangle", "nat"} + iphookChainTypeToTableType = [packetfilter.ChainTypeMAX]packetfilter.TableType{ + packetfilter.TableTypeFilter, + packetfilter.TableTypeRoute, + packetfilter.TableTypeNAT, + } + chainHookToStr = [packetfilter.ChainHookMAX]string{"PREROUTING", "INPUT", "FORWARD", "OUTPUT", "POSTROUTING"} + ruleActiontoStr = [packetfilter.RuleActionMAX]string{"", "ACCEPT", "TCPMSS", "MARK", "SNAT", "DNAT"} + logger = log.Logger{Logger: logf.Log.WithName(DriverName)} +) + +type PacketFilter struct { + ipt *iptables.IPTables +} + +func New() (packetfilter.Driver, error) { + ipt, err := iptables.New(iptables.IPFamily(iptables.ProtocolIPv4), iptables.Timeout(5)) + if err != nil { + return nil, errors.Wrap(err, "error creating IP tables") + } + + return &PacketFilter{ + ipt: ipt, + }, nil +} + +func (a *PacketFilter) ChainExists(table packetfilter.TableType, chain string) (bool, error) { + ok, err := a.ipt.ChainExists(tableTypeToStr[table], chain) + return ok, errors.Wrap(err, "ChainExists failed") +} + +func (a *PacketFilter) AppendUnique(table packetfilter.TableType, chain string, rule *packetfilter.Rule) error { + ruleSpecStr, err := ToRuleSpec(rule) + if err != nil { + return errors.Wrap(err, "AppendUnique failed") + } + + return errors.Wrap(a.ipt.AppendUnique(tableTypeToStr[table], chain, ruleSpecStr...), "error AppendUnique rule") +} + +func (a *PacketFilter) CreateIPHookChainIfNotExists(chain *packetfilter.ChainIPHook) error { + if err := a.createChainIfNotExists(iphookChainTypeToStr[chain.Type], chain.Name); err != nil { + return errors.Wrapf(err, "error creating IP tables %s:%s chain", iphookChainTypeToStr[chain.Type], chain.Name) + } + + jumpRule := chain.JumpRule + if jumpRule == nil { + jumpRule = &packetfilter.Rule{ + TargetChain: chain.Name, + Action: packetfilter.RuleActionJump, + } + } + + if chain.Priority == packetfilter.ChainPriorityFirst { + ruleSpecStr, err := ToRuleSpec(jumpRule) + if err != nil { + return errors.Wrap(err, "Failed to translate packetfilter Rule to str") + } + + table := tableTypeToStr[iphookChainTypeToTableType[chain.Type]] + + if err := a.ipt.InsertUnique(table, chainHookToStr[chain.Hook], 1, ruleSpecStr...); err != nil { + return errors.Wrap(err, "error InsertUnique rule") + } + } else { + if err := a.AppendUnique(iphookChainTypeToTableType[chain.Type], chainHookToStr[chain.Hook], jumpRule); err != nil { + return errors.Wrap(err, "error AppendUnique rule") + } + } + + return nil +} + +func (a *PacketFilter) CreateChainIfNotExists(table packetfilter.TableType, chain *packetfilter.Chain) error { + if err := a.createChainIfNotExists(tableTypeToStr[table], chain.Name); err != nil { + return errors.Wrapf(err, "error creating IP tables %s chain", chain.Name) + } + + return nil +} + +func (a *PacketFilter) DeleteIPHookChain(chain *packetfilter.ChainIPHook) error { + tableType := iphookChainTypeToTableType[chain.Type] + jumpRule := chain.JumpRule + + if jumpRule == nil { + jumpRule = &packetfilter.Rule{ + TargetChain: chain.Name, + Action: packetfilter.RuleActionJump, + } + } + + if err := a.Delete(tableType, chainHookToStr[chain.Hook], jumpRule); err != nil { + return errors.Wrap(err, "error deleting Jump Rule") + } + + if err := a.ipt.DeleteChain(tableTypeToStr[tableType], chain.Name); err != nil { + return errors.Wrap(err, "error deleting chain") + } + + return nil +} + +func (a *PacketFilter) DeleteChain(table packetfilter.TableType, chain string) error { + if err := a.ipt.DeleteChain(tableTypeToStr[table], chain); err != nil { + return errors.Wrap(err, "error deleting chain") + } + + return nil +} + +func (a *PacketFilter) ClearChain(table packetfilter.TableType, chain string) error { + if err := a.ipt.ClearChain(tableTypeToStr[table], chain); err != nil { + return errors.Wrap(err, "error clearing chain") + } + + return nil +} + +func (a *PacketFilter) Delete(table packetfilter.TableType, chain string, rule *packetfilter.Rule) error { + ruleSpecStr, err := ToRuleSpec(rule) + if err != nil { + return errors.Wrap(err, "error translating ruleSpec to str") + } + + return a.delete(tableTypeToStr[table], chain, ruleSpecStr...) +} + +func (a *PacketFilter) List(table packetfilter.TableType, chain string) ([]*packetfilter.Rule, error) { + existingRules, err := a.ipt.List(tableTypeToStr[table], chain) + if err != nil { + return []*packetfilter.Rule{}, errors.Wrapf(err, "error listing the rules in table %q, chain %q", tableTypeToStr[table], chain) + } + + rules := []*packetfilter.Rule{} + + for _, existingRule := range existingRules { + ruleSpec := strings.Split(existingRule, " ") + if ruleSpec[0] == "-A" { + ruleSpec = ruleSpec[2:] // remove "-A", "$chain" + } + + rules = append(rules, FromRuleSpec(ruleSpec)) + } + + return rules, nil +} + +func (a *PacketFilter) Insert(table packetfilter.TableType, chain string, pos int, rule *packetfilter.Rule) error { + ruleSpecStr, err := ToRuleSpec(rule) + if err != nil { + return errors.Wrap(err, "Insert failed") + } + + return errors.Wrap(a.ipt.Insert(tableTypeToStr[table], chain, pos, ruleSpecStr...), "error inserting ruleSpec to str") +} + +func (a *PacketFilter) Append(table packetfilter.TableType, chain string, rule *packetfilter.Rule) error { + ruleSpecStr, err := ToRuleSpec(rule) + if err != nil { + return errors.Wrap(err, "Append failed") + } + + return errors.Wrap(a.ipt.Append(tableTypeToStr[table], chain, ruleSpecStr...), "error appending ruleSpec to str") +} + +func (a *PacketFilter) createChainIfNotExists(table, chain string) error { + exists, err := a.ipt.ChainExists(table, chain) + if err == nil && exists { + return nil + } + + if err != nil { + return errors.Wrapf(err, "error finding IP table chain %q in table %q", chain, table) + } + + return errors.Wrap(a.ipt.NewChain(table, chain), "error creating IP table chain") +} + +func (a *PacketFilter) delete(table, chain string, rulespec ...string) error { + err := a.ipt.Delete(table, chain, rulespec...) + + var iptError *iptables.Error + + ok := errors.As(err, &iptError) + if ok && iptError.IsNotExist() { + return nil + } + + return errors.Wrap(err, "error deleting IP table rule") +} + +func protoToRuleSpec(ruleSpec *[]string, proto packetfilter.RuleProto) { + switch proto { + case packetfilter.RuleProtoUDP: + *ruleSpec = append(*ruleSpec, "-p", "udp", "-m", "udp") + case packetfilter.RuleProtoTCP: + *ruleSpec = append(*ruleSpec, "-p", "tcp", "-m", "tcp") + case packetfilter.RuleProtoICMP: + *ruleSpec = append(*ruleSpec, "-p", "icmp") + case packetfilter.RuleProtoAll: + *ruleSpec = append(*ruleSpec, "-p", "all") + case packetfilter.RuleProtoUndefined: + } +} + +func mssClampToRuleSpec(ruleSpec *[]string, clampType packetfilter.MssClampType, mssValue string) { + switch clampType { + case packetfilter.UndefinedMSS: + case packetfilter.ToPMTU: + *ruleSpec = append(*ruleSpec, "-p", "tcp", "-m", "tcp", "--tcp-flags", "SYN,RST", "SYN", "--clamp-mss-to-pmtu") + case packetfilter.ToValue: + *ruleSpec = append(*ruleSpec, "-p", "tcp", "-m", "tcp", "--tcp-flags", "SYN,RST", "SYN", "--set-mss", mssValue) + } +} + +func setToRuleSpec(ruleSpec *[]string, srcSetName, destSetName string) { + if srcSetName != "" { + *ruleSpec = append(*ruleSpec, "-m", "set", "--match-set", srcSetName, "src") + } + + if destSetName != "" { + *ruleSpec = append(*ruleSpec, "-m", "set", "--match-set", destSetName, "dst") + } +} + +func ToRuleSpec(rule *packetfilter.Rule) ([]string, error) { + var ruleSpec []string + protoToRuleSpec(&ruleSpec, rule.Proto) + + if rule.SrcCIDR != "" { + ruleSpec = append(ruleSpec, "-s", rule.SrcCIDR) + } + + if rule.DestCIDR != "" { + ruleSpec = append(ruleSpec, "-d", rule.DestCIDR) + } + + if rule.MarkValue != "" && rule.Action != packetfilter.RuleActionMark { + ruleSpec = append(ruleSpec, "-m", "mark", "--mark", rule.MarkValue) + } + + setToRuleSpec(&ruleSpec, rule.SrcSetName, rule.DestSetName) + + if rule.OutInterface != "" { + ruleSpec = append(ruleSpec, "-o", rule.OutInterface) + } + + if rule.InInterface != "" { + ruleSpec = append(ruleSpec, "-i", rule.InInterface) + } + + if rule.DPort != "" { + ruleSpec = append(ruleSpec, "--dport", rule.DPort) + } + + switch rule.Action { + case packetfilter.RuleActionMAX: + case packetfilter.RuleActionJump: + ruleSpec = append(ruleSpec, "-j", rule.TargetChain) + case packetfilter.RuleActionAccept, packetfilter.RuleActionMss, + packetfilter.RuleActionMark, packetfilter.RuleActionSNAT, packetfilter.RuleActionDNAT: + ruleSpec = append(ruleSpec, "-j", ruleActiontoStr[rule.Action]) + default: + return ruleSpec, errors.Errorf(" rule.Action %d is invalid", rule.Action) + } + + if rule.SnatCIDR != "" { + ruleSpec = append(ruleSpec, "--to-source", rule.SnatCIDR) + } + + if rule.DnatCIDR != "" { + ruleSpec = append(ruleSpec, "--to-destination", rule.DnatCIDR) + } + + mssClampToRuleSpec(&ruleSpec, rule.ClampType, rule.MssValue) + + if rule.MarkValue != "" && rule.Action == packetfilter.RuleActionMark { + ruleSpec = append(ruleSpec, "--set-mark", rule.MarkValue) + } + + logger.Infof("ToRuleSpec: %s", strings.Join(ruleSpec, " ")) + + return ruleSpec, nil +} + +func FromRuleSpec(spec []string) *packetfilter.Rule { + rule := &packetfilter.Rule{} + + length := len(spec) + i := 0 + + for i < length { + switch spec[i] { + case "-p": + rule.Proto, i = parseNextTerm(spec, i, parseProtocol) + case "-s": + rule.SrcCIDR, i = parseNextTerm(spec, i, noopParse) + case "-d": + rule.DestCIDR, i = parseNextTerm(spec, i, noopParse) + case "-m": + i = parseRuleMatch(spec, i, rule) + case "--to-destination": + rule.DnatCIDR, i = parseNextTerm(spec, i, noopParse) + case "--to-source": + rule.SnatCIDR, i = parseNextTerm(spec, i, noopParse) + case "-i": + rule.InInterface, i = parseNextTerm(spec, i, noopParse) + case "-o": + rule.OutInterface, i = parseNextTerm(spec, i, noopParse) + case "--dport": + rule.DPort, i = parseNextTerm(spec, i, noopParse) + case "--set-mark": + rule.MarkValue, i = parseNextTerm(spec, i, noopParse) + case "-j": + rule.Action, i = parseNextTerm(spec, i, parseAction) + if rule.Action == packetfilter.RuleActionJump { + rule.TargetChain = spec[i] + } + } + + i++ + } + + if rule.Action == packetfilter.RuleActionMss { + rule.Proto = packetfilter.RuleProtoUndefined + } + + return rule +} + +func parseNextTerm[T any](spec []string, i int, parse func(s string) T) (T, int) { + if i+1 >= len(spec) { + return *new(T), i + } + + i++ + + return parse(spec[i]), i +} + +func parseRuleMatch(spec []string, i int, rule *packetfilter.Rule) int { + if i+1 >= len(spec) { + return i + } + + i++ + + switch spec[i] { + case "mark": + if i+2 < len(spec) && spec[i+1] == "--mark" { + rule.MarkValue = spec[i+2] + i += 2 + } + case "set": + if i+3 < len(spec) && spec[i+1] == "--match-set" { + if spec[i+3] == "src" { + rule.SrcSetName = spec[i+2] + } else if spec[i+3] == "dst" { + rule.DestSetName = spec[i+2] + } + + i += 3 + } + case "tcp": + // Parses the form: "-m", "tcp", "--tcp-flags", "SYN,RST", "SYN", "--clamp-mss-to-pmtu" + i = parseTCPSpec(spec, i, rule) + } + + return i +} + +func parseTCPSpec(spec []string, i int, rule *packetfilter.Rule) int { + i++ + for i < len(spec) { + if spec[i] == "--clamp-mss-to-pmtu" { + rule.ClampType = packetfilter.ToPMTU + break + } else if spec[i] == "--set-mss" { + rule.MssValue, i = parseNextTerm(spec, i, noopParse) + rule.ClampType = packetfilter.ToValue + break + } else if !strings.HasPrefix(spec[i], "--") && strings.HasPrefix(spec[i], "-") { + i-- + break + } + + i++ + } + + return i +} + +func parseAction(s string) packetfilter.RuleAction { + for i := 1; i < len(ruleActiontoStr); i++ { + if s == ruleActiontoStr[i] { + return packetfilter.RuleAction(i) + } + } + + return packetfilter.RuleActionJump +} + +func parseProtocol(s string) packetfilter.RuleProto { + switch s { + case "udp": + return packetfilter.RuleProtoUDP + case "tcp": + return packetfilter.RuleProtoTCP + case "icmp": + return packetfilter.RuleProtoICMP + case "all": + return packetfilter.RuleProtoAll + default: + return packetfilter.RuleProtoUndefined + } +} + +func noopParse(s string) string { + return s +} diff --git a/pkg/packetfilter/iptables/iptables_suite_test.go b/pkg/packetfilter/iptables/iptables_suite_test.go new file mode 100644 index 000000000..65d37bc22 --- /dev/null +++ b/pkg/packetfilter/iptables/iptables_suite_test.go @@ -0,0 +1,36 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright Contributors to the Submariner project. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package iptables_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/submariner-io/admiral/pkg/log/kzerolog" +) + +var _ = BeforeSuite(func() { + kzerolog.InitK8sLogging() +}) + +func TestIptables(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "IPTables Suite") +} diff --git a/pkg/packetfilter/iptables/iptables_test.go b/pkg/packetfilter/iptables/iptables_test.go new file mode 100644 index 000000000..603ff3ade --- /dev/null +++ b/pkg/packetfilter/iptables/iptables_test.go @@ -0,0 +1,106 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright Contributors to the Submariner project. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package iptables_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/submariner-io/submariner/pkg/packetfilter" + "github.com/submariner-io/submariner/pkg/packetfilter/iptables" +) + +var _ = Describe("Rule conversion", func() { + Specify("should correctly convert to and from a rule spec string", func() { + // -m set --match-set src-set src -m set --match-set dest-set dst -j TCPMSS -p tcp -m tcp --tcp-flags SYN,RST SYN --clamp-mss-to-pmtu + testRuleConversion(&packetfilter.Rule{ + SrcSetName: "src-set", + DestSetName: "dest-set", + Action: packetfilter.RuleActionMss, + ClampType: packetfilter.ToPMTU, + }) + + // -m set --match-set src-set src -m set --match-set dest-set dst -j TCPMSS -p tcp -m tcp --tcp-flags SYN,RST SYN --set-mss mss-value + testRuleConversion(&packetfilter.Rule{ + SrcSetName: "src-set", + DestSetName: "dest-set", + MssValue: "mss-value", + Action: packetfilter.RuleActionMss, + ClampType: packetfilter.ToValue, + }) + + // -p udp -m udp -s 171.254.1.0/24 -d 170.254.1.0/24 -o out-iface -i in-iface --dport d-port -j ACCEPT + testRuleConversion(&packetfilter.Rule{ + Proto: packetfilter.RuleProtoUDP, + DestCIDR: "170.254.1.0/24", + SrcCIDR: "171.254.1.0/24", + OutInterface: "out-iface", + InInterface: "in-iface", + DPort: "d-port", + Action: packetfilter.RuleActionAccept, + }) + + // -p all -m mark --mark mark-value -m set --match-set src-set src -j SNAT --to-source 172.254.1.0/24 + testRuleConversion(&packetfilter.Rule{ + Proto: packetfilter.RuleProtoAll, + SrcSetName: "src-set", + MarkValue: "mark-value", + SnatCIDR: "172.254.1.0/24", + Action: packetfilter.RuleActionSNAT, + }) + + // -p all -s 171.254.1.0/24 -m mark --mark mark-value -j SNAT --to-source 172.254.1.0/24 + testRuleConversion(&packetfilter.Rule{ + Proto: packetfilter.RuleProtoTCP, + SrcCIDR: "171.254.1.0/24", + MarkValue: "mark-value", + SnatCIDR: "172.254.1.0/24", + Action: packetfilter.RuleActionSNAT, + }) + + // -p icmp -d 171.254.1.0/24 -j DNAT --to-destination 172.254.1.0/24 + testRuleConversion(&packetfilter.Rule{ + Proto: packetfilter.RuleProtoICMP, + DestCIDR: "171.254.1.0/24", + DnatCIDR: "172.254.1.0/24", + Action: packetfilter.RuleActionDNAT, + }) + + // -d 171.254.1.0/24 -j MARK --set-mark mark-value + testRuleConversion(&packetfilter.Rule{ + DestCIDR: "171.254.1.0/24", + MarkValue: "mark-value", + Action: packetfilter.RuleActionMark, + }) + + // -p udp -m udp -j target-chain + testRuleConversion(&packetfilter.Rule{ + Proto: packetfilter.RuleProtoUDP, + TargetChain: "target-chain", + Action: packetfilter.RuleActionJump, + }) + }) +}) + +func testRuleConversion(rule *packetfilter.Rule) { + spec, err := iptables.ToRuleSpec(rule) + Expect(err).To(Succeed()) + + parsed := iptables.FromRuleSpec(spec) + Expect(parsed).To(Equal(rule)) +} diff --git a/pkg/packetfilter/packetfilter.go b/pkg/packetfilter/packetfilter.go new file mode 100644 index 000000000..052d815a6 --- /dev/null +++ b/pkg/packetfilter/packetfilter.go @@ -0,0 +1,186 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright Contributors to the Submariner project. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package packetfilter + +import ( + "github.com/pkg/errors" + "github.com/submariner-io/admiral/pkg/log" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +var logger = log.Logger{Logger: logf.Log.WithName("Packetfilter")} + +const ( + TableTypeFilter TableType = iota + TableTypeRoute // mangle + TableTypeNAT + TableTypeMAX +) + +type TableType uint32 + +type GlobalEgressHandle string + +type RuleAction uint32 + +const ( + RuleActionJump RuleAction = iota + RuleActionAccept + RuleActionMss + RuleActionMark + RuleActionSNAT + RuleActionDNAT + RuleActionMAX +) + +type RuleProto uint32 + +const ( + RuleProtoUndefined RuleProto = iota + RuleProtoAll + RuleProtoTCP + RuleProtoUDP + RuleProtoICMP +) + +type MssClampType uint32 + +const ( + UndefinedMSS MssClampType = iota + ToPMTU + ToValue +) + +type ChainHook uint32 + +const ( + ChainHookPrerouting ChainHook = iota + ChainHookInput + ChainHookForward + ChainHookOutput + ChainHookPostrouting + ChainHookMAX +) + +type ChainPriority uint32 + +const ( + ChainPriorityFirst ChainPriority = iota + ChainPriorityLast +) + +type ChainType uint32 + +const ( + ChainTypeFilter ChainType = iota + ChainTypeRoute // mangle + ChainTypeNAT + ChainTypeMAX +) + +type ChainPolicy uint32 + +const ( + ChainPolicyAccept ChainType = iota + ChainPolicyDrop +) + +type Rule struct { + DestCIDR string + SrcCIDR string + + SrcSetName string + DestSetName string + + SnatCIDR string + DnatCIDR string + + OutInterface string + InInterface string + TargetChain string + MssValue string + + DPort string + MarkValue string + Action RuleAction + Proto RuleProto + ClampType MssClampType +} + +// Supported policy values are accept (which is the default) or drop. +type Chain struct { + Name string + Policy ChainPolicy +} + +type ChainIPHook struct { + Name string + Type ChainType + Hook ChainHook + Priority ChainPriority + Policy ChainPolicy + JumpRule *Rule +} + +type Driver interface { + // Chains + ChainExists(table TableType, chain string) (bool, error) + CreateIPHookChainIfNotExists(chain *ChainIPHook) error + CreateChainIfNotExists(table TableType, chain *Chain) error + DeleteIPHookChain(chain *ChainIPHook) error + DeleteChain(table TableType, chain string) error + ClearChain(table TableType, chain string) error + + // rules + Delete(table TableType, chain string, rule *Rule) error + AppendUnique(table TableType, chain string, rule *Rule) error + List(table TableType, chain string) ([]*Rule, error) + Append(table TableType, chain string, rule *Rule) error + Insert(table TableType, chain string, pos int, rule *Rule) error +} + +type Interface interface { + Driver + InsertUnique(table TableType, chain string, position int, ruleSpec *Rule) error + PrependUnique(table TableType, chain string, ruleSpec *Rule) error + UpdateChainRules(table TableType, chain string, rules []*Rule) error +} + +var newDriverFn func() (Driver, error) + +func SetNewDriverFn(f func() (Driver, error)) { + newDriverFn = f +} + +type Adapter struct { + Driver +} + +func New() (Interface, error) { + if newDriverFn == nil { + return nil, errors.New("no driver registered") + } + + driver, err := newDriverFn() + if err != nil { + return nil, errors.Wrap(err, "error creating packet filter Driver") + } + + return &Adapter{Driver: driver}, nil +} diff --git a/pkg/routeagent_driver/constants/constants.go b/pkg/routeagent_driver/constants/constants.go index 7c0a9951a..5309f8bbd 100644 --- a/pkg/routeagent_driver/constants/constants.go +++ b/pkg/routeagent_driver/constants/constants.go @@ -22,6 +22,7 @@ const ( // IPTable chains used by RouteAgent. SmPostRoutingChain = "SUBMARINER-POSTROUTING" SmInputChain = "SUBMARINER-INPUT" + SmForwardChain = "SUBMARINER-FORWARD" PostRoutingChain = "POSTROUTING" InputChain = "INPUT" ForwardChain = "FORWARD" diff --git a/pkg/routeagent_driver/handlers/kubeproxy/iptables_iface.go b/pkg/routeagent_driver/handlers/kubeproxy/iptables_iface.go deleted file mode 100644 index e5ce647a0..000000000 --- a/pkg/routeagent_driver/handlers/kubeproxy/iptables_iface.go +++ /dev/null @@ -1,126 +0,0 @@ -/* -SPDX-License-Identifier: Apache-2.0 - -Copyright Contributors to the Submariner project. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeproxy - -import ( - "strconv" - "strings" - - "github.com/pkg/errors" - "github.com/submariner-io/admiral/pkg/log" - "github.com/submariner-io/submariner/pkg/iptables" - "github.com/submariner-io/submariner/pkg/port" - "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" - iptcommon "github.com/submariner-io/submariner/pkg/routeagent_driver/iptables" -) - -func (kp *SyncHandler) createIPTableChains() error { - ipt, err := iptables.New() - if err != nil { - return errors.Wrap(err, "error initializing iptables") - } - - if err := iptcommon.InitSubmarinerPostRoutingChain(ipt); err != nil { - return errors.Wrap(err, "error initializing POST routing chain") - } - - logger.V(log.DEBUG).Infof("Install/ensure %q chain exists", constants.SmInputChain) - - if err = ipt.CreateChainIfNotExists(constants.FilterTable, constants.SmInputChain); err != nil { - return errors.Wrap(err, "unable to create SUBMARINER-INPUT chain in iptables") - } - - forwardToSubInputRuleSpec := []string{"-p", "udp", "-m", "udp", "-j", constants.SmInputChain} - if err = ipt.AppendUnique(constants.FilterTable, constants.InputChain, forwardToSubInputRuleSpec...); err != nil { - return errors.Wrapf(err, "unable to append iptables rule %q", strings.Join(forwardToSubInputRuleSpec, " ")) - } - - logger.V(log.DEBUG).Infof("Allow VxLAN incoming traffic in %q Chain", constants.SmInputChain) - - ruleSpec := []string{"-p", "udp", "-m", "udp", "--dport", strconv.Itoa(port.IntraClusterVxLAN), "-j", "ACCEPT"} - - if err = ipt.AppendUnique(constants.FilterTable, constants.SmInputChain, ruleSpec...); err != nil { - return errors.Wrapf(err, "unable to append iptables rule %q", strings.Join(ruleSpec, " ")) - } - - logger.V(log.DEBUG).Infof("Insert rule to allow traffic over %s interface in FORWARDing Chain", VxLANIface) - - ruleSpec = []string{"-o", VxLANIface, "-j", "ACCEPT"} - - if err = ipt.PrependUnique(constants.FilterTable, "FORWARD", ruleSpec); err != nil { - return errors.Wrap(err, "unable to insert iptable rule in filter table to allow vxlan traffic") - } - - if kp.cniIface != nil { - // Program rules to support communication from HostNetwork to remoteCluster - sourceAddress := strconv.Itoa(VxLANVTepNetworkPrefix) + ".0.0.0/8" - ruleSpec = []string{"-s", sourceAddress, "-o", VxLANIface, "-j", "SNAT", "--to", kp.cniIface.IPAddress} - logger.V(log.DEBUG).Infof("Installing rule for host network to remote cluster communication: %s", strings.Join(ruleSpec, " ")) - - if err = ipt.AppendUnique(constants.NATTable, constants.SmPostRoutingChain, ruleSpec...); err != nil { - return errors.Wrapf(err, "error appending iptables rule %q", strings.Join(ruleSpec, " ")) - } - } - - return nil -} - -func (kp *SyncHandler) updateIptableRulesForInterClusterTraffic(inputCidrBlocks []string, operation Operation) { - for _, inputCidrBlock := range inputCidrBlocks { - err := kp.programIptableRulesForInterClusterTraffic(inputCidrBlock, operation) - if err != nil { - logger.Errorf(err, "Failed to program iptable rules") - } - } -} - -func (kp *SyncHandler) programIptableRulesForInterClusterTraffic(remoteCidrBlock string, operation Operation) error { - for _, localClusterCidr := range kp.localClusterCidr { - outboundRuleSpec := []string{"-s", localClusterCidr, "-d", remoteCidrBlock, "-j", "ACCEPT"} - incomingRuleSpec := []string{"-s", remoteCidrBlock, "-d", localClusterCidr, "-j", "ACCEPT"} - - if operation == Add { - logger.V(log.DEBUG).Infof("Installing iptables rule for outgoing traffic: %s", strings.Join(outboundRuleSpec, " ")) - - if err := kp.ipTables.AppendUnique(constants.NATTable, constants.SmPostRoutingChain, outboundRuleSpec...); err != nil { - return errors.Wrapf(err, "error appending iptables rule %q", strings.Join(outboundRuleSpec, " ")) - } - - logger.V(log.DEBUG).Infof("Installing iptables rule for incoming traffic: %s", strings.Join(incomingRuleSpec, " ")) - - if err := kp.ipTables.AppendUnique(constants.NATTable, constants.SmPostRoutingChain, incomingRuleSpec...); err != nil { - return errors.Wrapf(err, "error appending iptables rule %q", strings.Join(incomingRuleSpec, " ")) - } - } else if operation == Delete { - logger.V(log.DEBUG).Infof("Deleting iptables rule for outgoing traffic: %s", strings.Join(outboundRuleSpec, " ")) - - if err := kp.ipTables.Delete(constants.NATTable, constants.SmPostRoutingChain, outboundRuleSpec...); err != nil { - return errors.Wrapf(err, "error deleting iptables rule %q", strings.Join(outboundRuleSpec, " ")) - } - - logger.V(log.DEBUG).Infof("Deleting iptables rule for incoming traffic: %s", strings.Join(incomingRuleSpec, " ")) - - if err := kp.ipTables.Delete(constants.NATTable, constants.SmPostRoutingChain, incomingRuleSpec...); err != nil { - return errors.Wrapf(err, "error deleting iptables rule %q", strings.Join(incomingRuleSpec, " ")) - } - } - } - - return nil -} diff --git a/pkg/routeagent_driver/handlers/kubeproxy/kp_iptables.go b/pkg/routeagent_driver/handlers/kubeproxy/kp_packetfilter.go similarity index 93% rename from pkg/routeagent_driver/handlers/kubeproxy/kp_iptables.go rename to pkg/routeagent_driver/handlers/kubeproxy/kp_packetfilter.go index b1d14520d..667dd2cae 100644 --- a/pkg/routeagent_driver/handlers/kubeproxy/kp_iptables.go +++ b/pkg/routeagent_driver/handlers/kubeproxy/kp_packetfilter.go @@ -27,8 +27,8 @@ import ( "github.com/submariner-io/submariner/pkg/cidr" cni "github.com/submariner-io/submariner/pkg/cni" "github.com/submariner-io/submariner/pkg/event" - "github.com/submariner-io/submariner/pkg/iptables" "github.com/submariner-io/submariner/pkg/netlink" + "github.com/submariner-io/submariner/pkg/packetfilter" cniapi "github.com/submariner-io/submariner/pkg/routeagent_driver/cni" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/utils/set" @@ -45,8 +45,7 @@ type SyncHandler struct { remoteSubnetGw map[string]net.IP remoteVTEPs set.Set[string] routeCacheGWNode set.Set[string] - - ipTables iptables.Interface + pFilter packetfilter.Interface netLink netlink.Interface vxlanDevice *vxLanIface vxlanGwIP *net.IP @@ -58,7 +57,7 @@ type SyncHandler struct { var logger = log.Logger{Logger: logf.Log.WithName("KubeProxy")} func NewSyncHandler(localClusterCidr, localServiceCidr []string) *SyncHandler { - ipTables, err := iptables.New() + pFilter, err := packetfilter.New() utilruntime.Must(err) return &SyncHandler{ @@ -70,7 +69,7 @@ func NewSyncHandler(localClusterCidr, localServiceCidr []string) *SyncHandler { remoteVTEPs: set.New[string](), routeCacheGWNode: set.New[string](), netLink: netlink.New(), - ipTables: ipTables, + pFilter: pFilter, } } @@ -122,9 +121,9 @@ func (kp *SyncHandler) Init() error { } // Create the necessary IPTable chains in the filter and nat tables. - err = kp.createIPTableChains() + err = kp.createPFilterChains() if err != nil { - return errors.Wrapf(err, "createIPTableChains returned error") + return errors.Wrapf(err, "createPFilterChains returned error") } return nil diff --git a/pkg/routeagent_driver/handlers/kubeproxy/packetfilter_iface.go b/pkg/routeagent_driver/handlers/kubeproxy/packetfilter_iface.go new file mode 100644 index 000000000..0b85321b4 --- /dev/null +++ b/pkg/routeagent_driver/handlers/kubeproxy/packetfilter_iface.go @@ -0,0 +1,159 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright Contributors to the Submariner project. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeproxy + +import ( + "strconv" + + "github.com/pkg/errors" + "github.com/submariner-io/admiral/pkg/log" + "github.com/submariner-io/submariner/pkg/packetfilter" + "github.com/submariner-io/submariner/pkg/port" + "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" +) + +func (kp *SyncHandler) createPFilterChains() error { + logger.V(log.DEBUG).Infof("Install/ensure %q/%s IPHook chain exists", constants.SmPostRoutingChain, "NAT") + + if err := kp.pFilter.CreateIPHookChainIfNotExists(&packetfilter.ChainIPHook{ + Name: constants.SmPostRoutingChain, + Type: packetfilter.ChainTypeNAT, + Hook: packetfilter.ChainHookPostrouting, + Priority: packetfilter.ChainPriorityFirst, + }); err != nil { + return errors.Wrap(err, "error installing IPHook chain") + } + + logger.V(log.DEBUG).Infof("Install/ensure %q/%s IPHook chain exists", constants.SmInputChain, "Filter") + + if err := kp.pFilter.CreateIPHookChainIfNotExists(&packetfilter.ChainIPHook{ + Name: constants.SmInputChain, + Type: packetfilter.ChainTypeFilter, + Hook: packetfilter.ChainHookInput, + Priority: packetfilter.ChainPriorityLast, + JumpRule: &packetfilter.Rule{ + Proto: packetfilter.RuleProtoUDP, + Action: packetfilter.RuleActionJump, + TargetChain: constants.SmInputChain, + }, + }); err != nil { + return errors.Wrap(err, "error installing IPHook chain") + } + + logger.V(log.DEBUG).Infof("Allow VxLAN incoming traffic in %q Chain", constants.SmInputChain) + + ruleSpec := packetfilter.Rule{ + Proto: packetfilter.RuleProtoUDP, + DPort: strconv.Itoa(port.IntraClusterVxLAN), + Action: packetfilter.RuleActionAccept, + } + + if err := kp.pFilter.AppendUnique(packetfilter.TableTypeFilter, constants.SmInputChain, &ruleSpec); err != nil { + return errors.Wrapf(err, "unable to append rule %+v", &ruleSpec) + } + + if err := kp.pFilter.CreateIPHookChainIfNotExists(&packetfilter.ChainIPHook{ + Name: constants.SmForwardChain, + Type: packetfilter.ChainTypeFilter, + Hook: packetfilter.ChainHookForward, + Priority: packetfilter.ChainPriorityFirst, + }); err != nil { + return errors.Wrap(err, "error installing IPHook chain") + } + + logger.V(log.DEBUG).Infof("Insert rule to allow traffic over %s interface in %s Chain", VxLANIface, constants.SmForwardChain) + + ruleSpec = packetfilter.Rule{ + OutInterface: VxLANIface, + Action: packetfilter.RuleActionAccept, + } + if err := kp.pFilter.PrependUnique(packetfilter.TableTypeFilter, constants.SmForwardChain, &ruleSpec); err != nil { + return errors.Wrapf(err, "unable to append rule %+v to allow vxlan traffic", &ruleSpec) + } + + if kp.cniIface != nil { + // Program rules to support communication from HostNetwork to remoteCluster + ruleSpec = packetfilter.Rule{ + OutInterface: VxLANIface, + SrcCIDR: strconv.Itoa(VxLANVTepNetworkPrefix) + ".0.0.0/8", + SnatCIDR: kp.cniIface.IPAddress, + Action: packetfilter.RuleActionSNAT, + } + + logger.V(log.DEBUG).Infof("Installing rule for host network to remote cluster communication: %+v", ruleSpec) + + if err := kp.pFilter.AppendUnique(packetfilter.TableTypeNAT, constants.SmPostRoutingChain, &ruleSpec); err != nil { + return errors.Wrapf(err, "unable to append rule %+v", &ruleSpec) + } + } + + return nil +} + +func (kp *SyncHandler) updateIptableRulesForInterClusterTraffic(inputCidrBlocks []string, operation Operation) { + for _, inputCidrBlock := range inputCidrBlocks { + err := kp.programIptableRulesForInterClusterTraffic(inputCidrBlock, operation) + if err != nil { + logger.Errorf(err, "Failed to program iptable rules") + } + } +} + +func (kp *SyncHandler) programIptableRulesForInterClusterTraffic(remoteCidrBlock string, operation Operation) error { + for _, localClusterCidr := range kp.localClusterCidr { + outboundRule := packetfilter.Rule{ + Action: packetfilter.RuleActionAccept, + SrcCIDR: localClusterCidr, + DestCIDR: remoteCidrBlock, + } + incomingRule := packetfilter.Rule{ + Action: packetfilter.RuleActionAccept, + SrcCIDR: remoteCidrBlock, + DestCIDR: localClusterCidr, + } + + if operation == Add { + logger.V(log.DEBUG).Infof("Installing packetfilter rule for outgoing traffic: %+v", outboundRule) + + if err := kp.pFilter.AppendUnique(packetfilter.TableTypeNAT, constants.SmPostRoutingChain, &outboundRule); err != nil { + return errors.Wrapf(err, "error appending packetfilter rule %+v", outboundRule) + } + + logger.V(log.DEBUG).Infof("Installing packetfilter rule for incoming traffic: %+v", incomingRule) + + if err := kp.pFilter.AppendUnique(packetfilter.TableTypeNAT, constants.SmPostRoutingChain, &incomingRule); err != nil { + return errors.Wrapf(err, "error appending packetfilter rule %+v", incomingRule) + } + } else if operation == Delete { + logger.V(log.DEBUG).Infof("Deleting packetfilter rule for outgoing traffic: %+v", outboundRule) + + if err := kp.pFilter.Delete(packetfilter.TableTypeNAT, constants.SmPostRoutingChain, &outboundRule); err != nil { + return errors.Wrapf(err, "error deleting packetfilter rule %+v", outboundRule) + } + + logger.V(log.DEBUG).Infof("Deleting packetfilter rule for incoming traffic: %+v", incomingRule) + + if err := kp.pFilter.Delete(packetfilter.TableTypeNAT, constants.SmPostRoutingChain, &incomingRule); err != nil { + return errors.Wrapf(err, "error deleting packetfilter rule %+v", incomingRule) + } + } + } + + return nil +} diff --git a/pkg/routeagent_driver/handlers/kubeproxy/sync_handler_test.go b/pkg/routeagent_driver/handlers/kubeproxy/sync_handler_test.go index 5c75bd7a5..513f93609 100644 --- a/pkg/routeagent_driver/handlers/kubeproxy/sync_handler_test.go +++ b/pkg/routeagent_driver/handlers/kubeproxy/sync_handler_test.go @@ -26,10 +26,10 @@ import ( . "github.com/onsi/gomega" submarinerv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1" "github.com/submariner-io/submariner/pkg/event/testing" - "github.com/submariner-io/submariner/pkg/iptables" - fakeIPT "github.com/submariner-io/submariner/pkg/iptables/fake" netlinkAPI "github.com/submariner-io/submariner/pkg/netlink" fakeNetlink "github.com/submariner-io/submariner/pkg/netlink/fake" + "github.com/submariner-io/submariner/pkg/packetfilter" + fakePF "github.com/submariner-io/submariner/pkg/packetfilter/fake" "github.com/submariner-io/submariner/pkg/routeagent_driver/cni" "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" "github.com/submariner-io/submariner/pkg/routeagent_driver/handlers/kubeproxy" @@ -353,7 +353,7 @@ func testUninstall() { type testDriver struct { *testing.ControllerSupport handler *kubeproxy.SyncHandler - ipTables *fakeIPT.IPTables + pFilter *fakePF.PacketFilter netLink *fakeNetlink.NetLink localEndpoint *submarinerv1.Endpoint remoteEndpoint *submarinerv1.Endpoint @@ -379,11 +379,7 @@ func newTestDriver() *testDriver { netlinkAPI.NewFunc = func() netlinkAPI.Interface { return t.netLink } - - t.ipTables = fakeIPT.New() - iptables.NewFunc = func() (iptables.Interface, error) { - return t.ipTables, nil - } + t.pFilter = fakePF.New() cni.DiscoverFunc = func(clusterCIDR string) (*cni.Interface, error) { return &cni.Interface{ @@ -401,7 +397,6 @@ func newTestDriver() *testDriver { }) AfterEach(func() { - iptables.NewFunc = nil netlinkAPI.NewFunc = nil cni.DiscoverFunc = nil }) @@ -429,7 +424,7 @@ func (t *testDriver) verifyNoHostNetworkingRoutes() { func (t *testDriver) verifyRemoteSubnetIPTableRules() { for _, remoteCIDR := range t.remoteEndpoint.Spec.Subnets { - t.ipTables.AwaitRule("nat", constants.SmPostRoutingChain, + t.pFilter.AwaitRule(packetfilter.TableTypeNAT, constants.SmPostRoutingChain, And(ContainSubstring(localClusterCIDR), ContainSubstring(remoteCIDR))) } } diff --git a/pkg/routeagent_driver/handlers/kubeproxy/uninstall.go b/pkg/routeagent_driver/handlers/kubeproxy/uninstall.go index b549f95c3..9dc68fca5 100644 --- a/pkg/routeagent_driver/handlers/kubeproxy/uninstall.go +++ b/pkg/routeagent_driver/handlers/kubeproxy/uninstall.go @@ -22,8 +22,8 @@ import ( "net" "github.com/submariner-io/admiral/pkg/log" - "github.com/submariner-io/submariner/pkg/iptables" netlinkAPI "github.com/submariner-io/submariner/pkg/netlink" + "github.com/submariner-io/submariner/pkg/packetfilter" "github.com/submariner-io/submariner/pkg/port" "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" "github.com/vishvananda/netlink" @@ -72,51 +72,54 @@ func deleteVxLANInterface() { } func deleteIPTableChains() { - ipt, err := iptables.New() + pFilter, err := packetfilter.New() if err != nil { - logger.Errorf(err, "Failed to initialize IPTable interface") + logger.Errorf(err, "Failed to initialize packetfilter interface") return } - logger.Infof("Flushing iptable entries in %q chain of %q table", constants.SmPostRoutingChain, constants.NATTable) + logger.Infof("Flushing packetfilter entries in %q chain of %q table", constants.SmPostRoutingChain, constants.NATTable) - if err := ipt.ClearChain(constants.NATTable, constants.SmPostRoutingChain); err != nil { - logger.Errorf(err, "Error flushing iptables chain %q of %q table", constants.SmPostRoutingChain, + if err := pFilter.ClearChain(packetfilter.TableTypeNAT, constants.SmPostRoutingChain); err != nil { + logger.Errorf(err, "Error flushing packetfilter chain %q of %q table", constants.SmPostRoutingChain, constants.NATTable) } logger.Infof("Deleting iptable entry in %q chain of %q table", constants.PostRoutingChain, constants.NATTable) - ruleSpec := []string{"-j", constants.SmPostRoutingChain} - if err := ipt.Delete(constants.NATTable, constants.PostRoutingChain, ruleSpec...); err != nil { - logger.Errorf(err, "Error deleting iptables rule from %q chain", constants.PostRoutingChain) + chain := packetfilter.ChainIPHook{ + Name: constants.SmPostRoutingChain, + Type: packetfilter.ChainTypeNAT, + Hook: packetfilter.ChainHookPostrouting, + Priority: packetfilter.ChainPriorityFirst, } - - logger.Infof("Deleting iptable %q chain of %q table", constants.SmPostRoutingChain, constants.NATTable) - - if err := ipt.DeleteChain(constants.NATTable, constants.SmPostRoutingChain); err != nil { - logger.Errorf(err, "Error deleting iptable chain %q of table %q", constants.SmPostRoutingChain, + if err := pFilter.DeleteIPHookChain(&chain); err != nil { + logger.Errorf(err, "Error deleting IPHook chain %q of %q table", constants.SmPostRoutingChain, constants.NATTable) } logger.Infof("Flushing iptable entries in %q chain of %q table", constants.SmInputChain, constants.FilterTable) - if err := ipt.ClearChain(constants.FilterTable, constants.SmInputChain); err != nil { - logger.Errorf(err, "Error flushing iptables chain %q of %q table", constants.SmInputChain, + if err := pFilter.ClearChain(packetfilter.TableTypeFilter, constants.SmInputChain); err != nil { + logger.Errorf(err, "Error flushing packetfilter chain %q of %q table", constants.SmInputChain, constants.FilterTable) } logger.Infof("Deleting iptable entry in %q chain of %q table", constants.InputChain, constants.FilterTable) - ruleSpec = []string{"-p", "udp", "-m", "udp", "-j", constants.SmInputChain} - if err := ipt.Delete(constants.FilterTable, constants.InputChain, ruleSpec...); err != nil { - logger.Errorf(err, "Error deleting iptables rule from %q chain", constants.InputChain) + chain = packetfilter.ChainIPHook{ + Name: constants.SmInputChain, + Type: packetfilter.ChainTypeFilter, + Hook: packetfilter.ChainHookInput, + Priority: packetfilter.ChainPriorityLast, + JumpRule: &packetfilter.Rule{ + Proto: packetfilter.RuleProtoUDP, + Action: packetfilter.RuleActionJump, + TargetChain: constants.SmInputChain, + }, } - - logger.Infof("Deleting iptable %q chain of %q table", constants.SmInputChain, constants.FilterTable) - - if err := ipt.DeleteChain(constants.FilterTable, constants.SmInputChain); err != nil { - logger.Errorf(err, "Error deleting iptable chain %q of table %q", constants.SmInputChain, + if err := pFilter.DeleteIPHookChain(&chain); err != nil { + logger.Errorf(err, "Error deleting IPHook chain %q of %q table", constants.InputChain, constants.FilterTable) } } diff --git a/pkg/routeagent_driver/handlers/mtu/mtuhandler.go b/pkg/routeagent_driver/handlers/mtu/mtuhandler.go index 80ca084f7..dfe59d3a4 100644 --- a/pkg/routeagent_driver/handlers/mtu/mtuhandler.go +++ b/pkg/routeagent_driver/handlers/mtu/mtuhandler.go @@ -20,7 +20,6 @@ package mtu import ( "strconv" - "strings" "github.com/pkg/errors" "github.com/submariner-io/admiral/pkg/log" @@ -29,8 +28,8 @@ import ( "github.com/submariner-io/submariner/pkg/cidr" "github.com/submariner-io/submariner/pkg/event" "github.com/submariner-io/submariner/pkg/ipset" - "github.com/submariner-io/submariner/pkg/iptables" netlinkAPI "github.com/submariner-io/submariner/pkg/netlink" + "github.com/submariner-io/submariner/pkg/packetfilter" "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" utilexec "k8s.io/utils/exec" k8snet "k8s.io/utils/net" @@ -53,7 +52,7 @@ const ( type mtuHandler struct { event.HandlerBase localClusterCidr []string - ipt iptables.Interface + pFilter packetfilter.Interface remoteIPSet ipset.Named localIPSet ipset.Named forceMss forceMssSts @@ -86,19 +85,22 @@ func (h *mtuHandler) GetName() string { func (h *mtuHandler) Init() error { var err error - h.ipt, err = iptables.New() + h.pFilter, err = packetfilter.New() if err != nil { return errors.Wrap(err, "error initializing iptables") } ipSetIface := ipset.New(utilexec.New()) - if err := h.ipt.CreateChainIfNotExists(constants.MangleTable, constants.SmPostRoutingChain); err != nil { - return errors.Wrapf(err, "error creating iptables chain %s", constants.SmPostRoutingChain) + if err := h.pFilter.CreateIPHookChainIfNotExists(&packetfilter.ChainIPHook{ + Name: constants.SmPostRoutingChain, + Type: packetfilter.ChainTypeRoute, + Hook: packetfilter.ChainHookPostrouting, + Priority: packetfilter.ChainPriorityFirst, + }); err != nil { + return errors.Wrapf(err, "error creating IPHookChain chain %s", constants.SmPostRoutingChain) } - forwardToSubMarinerPostRoutingChain := []string{"-j", constants.SmPostRoutingChain} - h.remoteIPSet = h.newNamedIPSet(constants.RemoteCIDRIPSet, ipSetIface) if err := h.remoteIPSet.Create(true); err != nil { return errors.Wrapf(err, "error creating ipset %q", constants.RemoteCIDRIPSet) @@ -109,36 +111,33 @@ func (h *mtuHandler) Init() error { return errors.Wrapf(err, "error creating ipset %q", constants.LocalCIDRIPSet) } - if err := h.ipt.PrependUnique(constants.MangleTable, constants.PostRoutingChain, - forwardToSubMarinerPostRoutingChain); err != nil { - return errors.Wrapf(err, "error inserting iptables rule %q", - strings.Join(forwardToSubMarinerPostRoutingChain, " ")) - } - - // iptable rules to clamp TCP MSS to a fixed value will be programmed when the local endpoint is created + // packetfilter rules to clamp TCP MSS to a fixed value will be programmed when the local endpoint is created if h.forceMss == needed { return nil } - logger.Info("Creating iptables clamp-mss-to-pmtu rules") + logger.Info("Creating packetfilter clamp-mss-to-pmtu rules") - ruleSpecSource := []string{ - "-m", "set", "--match-set", constants.LocalCIDRIPSet, "src", "-m", "set", "--match-set", - constants.RemoteCIDRIPSet, "dst", "-p", "tcp", "-m", "tcp", "--tcp-flags", "SYN,RST", "SYN", "-j", "TCPMSS", - "--clamp-mss-to-pmtu", + ruleSpecSource := &packetfilter.Rule{ + SrcSetName: constants.LocalCIDRIPSet, + DestSetName: constants.RemoteCIDRIPSet, + Action: packetfilter.RuleActionMss, + ClampType: packetfilter.ToPMTU, } - ruleSpecDest := []string{ - "-m", "set", "--match-set", constants.RemoteCIDRIPSet, "src", "-m", "set", "--match-set", - constants.LocalCIDRIPSet, "dst", "-p", "tcp", "-m", "tcp", "--tcp-flags", "SYN,RST", "SYN", "-j", "TCPMSS", - "--clamp-mss-to-pmtu", + + ruleSpecDest := &packetfilter.Rule{ + SrcSetName: constants.RemoteCIDRIPSet, + DestSetName: constants.LocalCIDRIPSet, + Action: packetfilter.RuleActionMss, + ClampType: packetfilter.ToPMTU, } - if err := h.ipt.AppendUnique(constants.MangleTable, constants.SmPostRoutingChain, ruleSpecSource...); err != nil { - return errors.Wrapf(err, "error appending iptables rule %q", strings.Join(ruleSpecSource, " ")) + if err := h.pFilter.AppendUnique(packetfilter.TableTypeRoute, constants.SmPostRoutingChain, ruleSpecSource); err != nil { + return errors.Wrapf(err, "error appending packetfilter rule %q", ruleSpecSource) } - if err := h.ipt.AppendUnique(constants.MangleTable, constants.SmPostRoutingChain, ruleSpecDest...); err != nil { - return errors.Wrapf(err, "error appending iptables rule %q", strings.Join(ruleSpecSource, " ")) + if err := h.pFilter.AppendUnique(packetfilter.TableTypeRoute, constants.SmPostRoutingChain, ruleSpecDest); err != nil { + return errors.Wrapf(err, "error appending packetfilter rule %q", ruleSpecDest) } return nil @@ -161,7 +160,7 @@ func (h *mtuHandler) LocalEndpointCreated(endpoint *submV1.Endpoint) error { } if h.forceMss == needed { - logger.Info("Creating iptables set-mss rules") + logger.Info("Creating packetfilter set-mss rules") err := h.forceMssClamping(endpoint) if err != nil { @@ -239,25 +238,21 @@ func (h *mtuHandler) newNamedIPSet(key string, ipSetIface ipset.Interface) ipset } func (h *mtuHandler) Uninstall() error { - logger.Infof("Flushing iptable entries in %q chain of %q table", constants.SmPostRoutingChain, constants.MangleTable) + logger.Infof("Flushing packetfilter entries in %q chain of table type Route", constants.SmPostRoutingChain) - if err := h.ipt.ClearChain(constants.MangleTable, constants.SmPostRoutingChain); err != nil { - logger.Errorf(err, "Error flushing iptables chain %q of %q table", constants.SmPostRoutingChain, - constants.MangleTable) + if err := h.pFilter.ClearChain(packetfilter.TableTypeRoute, constants.SmPostRoutingChain); err != nil { + logger.Errorf(err, "Error flushing chain %q of table type Route", constants.SmPostRoutingChain) } - logger.Infof("Deleting iptable entry in %q chain of %q table", constants.PostRoutingChain, constants.MangleTable) + logger.Infof("Deleting IPHook chain %q of table type Route", constants.SmPostRoutingChain) - ruleSpec := []string{"-j", constants.SmPostRoutingChain} - if err := h.ipt.Delete(constants.MangleTable, constants.PostRoutingChain, ruleSpec...); err != nil { - logger.Errorf(err, "Error deleting iptables rule from %q chain", constants.PostRoutingChain) - } - - logger.Infof("Deleting iptable %q chain of %q table", constants.SmPostRoutingChain, constants.MangleTable) - - if err := h.ipt.DeleteChain(constants.MangleTable, constants.SmPostRoutingChain); err != nil { - logger.Errorf(err, "Error deleting iptable chain %q of table %q", constants.SmPostRoutingChain, - constants.MangleTable) + if err := h.pFilter.DeleteIPHookChain(&packetfilter.ChainIPHook{ + Name: constants.SmPostRoutingChain, + Type: packetfilter.ChainTypeRoute, + Hook: packetfilter.ChainHookPostrouting, + Priority: packetfilter.ChainPriorityFirst, + }); err != nil { + logger.Errorf(err, "Error deleting IP hook chain %q of table type Route", constants.SmPostRoutingChain) } if err := h.localIPSet.Flush(); err != nil { @@ -299,20 +294,26 @@ func (h *mtuHandler) forceMssClamping(endpoint *submV1.Endpoint) error { } logger.Infof("forceMssClamping to: %d (%s) ", tcpMssValue, tcpMssSrc) - ruleSpecSource := []string{ - "-m", "set", "--match-set", constants.LocalCIDRIPSet, "src", "-m", "set", "--match-set", - constants.RemoteCIDRIPSet, "dst", "-p", "tcp", "-m", "tcp", "--tcp-flags", "SYN,RST", "SYN", "-j", "TCPMSS", - "--set-mss", strconv.Itoa(tcpMssValue), - } - ruleSpecDest := []string{ - "-m", "set", "--match-set", constants.RemoteCIDRIPSet, "src", "-m", "set", "--match-set", - constants.LocalCIDRIPSet, "dst", "-p", "tcp", "-m", "tcp", "--tcp-flags", "SYN,RST", "SYN", "-j", "TCPMSS", - "--set-mss", strconv.Itoa(tcpMssValue), - } - if err := h.ipt.UpdateChainRules(constants.MangleTable, constants.SmPostRoutingChain, - [][]string{ruleSpecSource, ruleSpecDest}); err != nil { - return errors.Wrapf(err, "error updating chain %s table %s rules", constants.SmPostRoutingChain, constants.MangleTable) + rules := []*packetfilter.Rule{} + + rules = append(rules, &packetfilter.Rule{ + SrcSetName: constants.LocalCIDRIPSet, + DestSetName: constants.RemoteCIDRIPSet, + Action: packetfilter.RuleActionMss, + ClampType: packetfilter.ToValue, + MssValue: strconv.Itoa(tcpMssValue), + }, &packetfilter.Rule{ + DestSetName: constants.LocalCIDRIPSet, + SrcSetName: constants.RemoteCIDRIPSet, + Action: packetfilter.RuleActionMss, + ClampType: packetfilter.ToValue, + MssValue: strconv.Itoa(tcpMssValue), + }, + ) + + if err := h.pFilter.UpdateChainRules(packetfilter.TableTypeRoute, constants.SmPostRoutingChain, rules); err != nil { + return errors.Wrapf(err, "error updating chain %s table type Route", constants.SmPostRoutingChain) } return nil diff --git a/pkg/routeagent_driver/handlers/mtu/mtuhandler_test.go b/pkg/routeagent_driver/handlers/mtu/mtuhandler_test.go index 7568c189a..91c0abdc1 100644 --- a/pkg/routeagent_driver/handlers/mtu/mtuhandler_test.go +++ b/pkg/routeagent_driver/handlers/mtu/mtuhandler_test.go @@ -18,30 +18,29 @@ limitations under the License. package mtu_test import ( + "fmt" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" submV1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1" "github.com/submariner-io/submariner/pkg/event" "github.com/submariner-io/submariner/pkg/ipset" fakeSet "github.com/submariner-io/submariner/pkg/ipset/fake" - "github.com/submariner-io/submariner/pkg/iptables" - fakeIPT "github.com/submariner-io/submariner/pkg/iptables/fake" + "github.com/submariner-io/submariner/pkg/packetfilter" + fakePF "github.com/submariner-io/submariner/pkg/packetfilter/fake" "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" "github.com/submariner-io/submariner/pkg/routeagent_driver/handlers/mtu" ) var _ = Describe("MTUHandler", func() { var ( - ipt *fakeIPT.IPTables + pFilter *fakePF.PacketFilter ipSet *fakeSet.IPSet handler event.Handler ) BeforeEach(func() { - ipt = fakeIPT.New() - iptables.NewFunc = func() (iptables.Interface, error) { - return ipt, nil - } + pFilter = fakePF.New() ipSet = fakeSet.New() ipset.NewFunc = func() ipset.Interface { return ipSet @@ -49,17 +48,17 @@ var _ = Describe("MTUHandler", func() { handler = mtu.NewMTUHandler([]string{"10.1.0.0/24"}, false, 0) }) - AfterEach(func() { - iptables.NewFunc = nil - }) - When("endpoint is added and removed", func() { It("should add and remove iptable rules", func() { Expect(handler.Init()).To(Succeed()) - ipt.AwaitRule(constants.MangleTable, constants.SmPostRoutingChain, ContainSubstring(constants.RemoteCIDRIPSet+" src")) - ipt.AwaitRule(constants.MangleTable, constants.SmPostRoutingChain, ContainSubstring(constants.RemoteCIDRIPSet+" dst")) - ipt.AwaitRule(constants.MangleTable, constants.SmPostRoutingChain, ContainSubstring(constants.LocalCIDRIPSet+" src")) - ipt.AwaitRule(constants.MangleTable, constants.SmPostRoutingChain, ContainSubstring(constants.LocalCIDRIPSet+" dst")) + pFilter.AwaitRule(packetfilter.TableTypeRoute, + constants.SmPostRoutingChain, ContainSubstring(fmt.Sprintf("\"SrcSetName\":%q", constants.RemoteCIDRIPSet))) + pFilter.AwaitRule(packetfilter.TableTypeRoute, + constants.SmPostRoutingChain, ContainSubstring(fmt.Sprintf("\"DestSetName\":%q", constants.RemoteCIDRIPSet))) + pFilter.AwaitRule(packetfilter.TableTypeRoute, + constants.SmPostRoutingChain, ContainSubstring(fmt.Sprintf("\"SrcSetName\":%q", constants.LocalCIDRIPSet))) + pFilter.AwaitRule(packetfilter.TableTypeRoute, + constants.SmPostRoutingChain, ContainSubstring(fmt.Sprintf("\"DestSetName\":%q", constants.LocalCIDRIPSet))) localEndpoint := newSubmEndpoint([]string{"10.1.0.0/24", "172.1.0.0/24"}) Expect(handler.LocalEndpointCreated(localEndpoint)).To(Succeed()) diff --git a/pkg/routeagent_driver/handlers/ovn/gateway_dataplane.go b/pkg/routeagent_driver/handlers/ovn/gateway_dataplane.go index 132dd922d..500076ede 100644 --- a/pkg/routeagent_driver/handlers/ovn/gateway_dataplane.go +++ b/pkg/routeagent_driver/handlers/ovn/gateway_dataplane.go @@ -23,8 +23,9 @@ import ( "strconv" "github.com/pkg/errors" + "github.com/submariner-io/admiral/pkg/log" + "github.com/submariner-io/submariner/pkg/packetfilter" "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" - iptcommon "github.com/submariner-io/submariner/pkg/routeagent_driver/iptables" "github.com/vishvananda/netlink" ) @@ -99,9 +100,9 @@ const ( MSSFor1500MTU = 1500 - IPTCPOverHead - ExpectedIPSECOverhead ) -func (ovn *Handler) getForwardingRuleSpecs() ([][]string, error) { +func (ovn *Handler) getForwardingRuleSpecs() ([]*packetfilter.Rule, error) { if ovn.cableRoutingInterface == nil { - return nil, errors.New("error setting up forwarding iptables, the cable interface isn't discovered yet, " + + return nil, errors.New("error setting up forwarding packetfilter config, the cable interface isn't discovered yet, " + "this will be retried") } @@ -110,22 +111,28 @@ func (ovn *Handler) getForwardingRuleSpecs() ([][]string, error) { // To reroute incoming traffic over the ovn-k8s-mp0 interface, we employ routes in table 149. Before the traffic // hits ovn-k8s-mp0, firewall rules would be processed. Therefore, we include these firewall rules in the FORWARDing // chain to allow such traffic. Similar thing happens for outbound traffic as well, and we use routes in table 150. - rules := [][]string{} + rules := []*packetfilter.Rule{} for _, remoteCIDR := range ovn.getRemoteSubnets().UnsortedList() { - rules = append(rules, - []string{ - "-d", remoteCIDR, "-i", OVNK8sMgmntIntfName, "-o", ovn.cableRoutingInterface.Name, "-j", "ACCEPT", + rules = append(rules, &packetfilter.Rule{ + DestCIDR: remoteCIDR, + Action: packetfilter.RuleActionAccept, + OutInterface: ovn.cableRoutingInterface.Name, + InInterface: OVNK8sMgmntIntfName, + }, + &packetfilter.Rule{ + SrcCIDR: remoteCIDR, + Action: packetfilter.RuleActionAccept, + OutInterface: OVNK8sMgmntIntfName, + InInterface: ovn.cableRoutingInterface.Name, }, - []string{ - "-s", remoteCIDR, "-i", ovn.cableRoutingInterface.Name, "-o", OVNK8sMgmntIntfName, "-j", "ACCEPT", - }) + ) } return rules, nil } -func (ovn *Handler) getMSSClampingRuleSpecs() ([][]string, error) { - rules := [][]string{} +func (ovn *Handler) getMSSClampingRuleSpecs() ([]*packetfilter.Rule, error) { + rules := []*packetfilter.Rule{} // NOTE: This is a workaround for submariner issues: // * https://github.com/submariner-io/submariner/issues/1278 @@ -133,21 +140,24 @@ func (ovn *Handler) getMSSClampingRuleSpecs() ([][]string, error) { // TODO: get the kernel to steer the ICMPs back to ovn-k8s-sub0 interface properly, or write a packet // reflector in the route agent for that type of packets for _, remoteCIDR := range ovn.getRemoteSubnets().UnsortedList() { - rules = append(rules, - []string{ - "-d", remoteCIDR, "-p", "tcp", "-m", "tcp", - "--tcp-flags", "SYN,RST", "SYN", "-j", "TCPMSS", "--set-mss", strconv.Itoa(MSSFor1500MTU), - }, - []string{ - "-s", remoteCIDR, "-p", "tcp", "-m", "tcp", - "--tcp-flags", "SYN,RST", "SYN", "-j", "TCPMSS", "--set-mss", strconv.Itoa(MSSFor1500MTU), - }) + rules = append(rules, &packetfilter.Rule{ + DestCIDR: remoteCIDR, + Action: packetfilter.RuleActionMss, + ClampType: packetfilter.ToValue, + MssValue: strconv.Itoa(MSSFor1500MTU), + }, &packetfilter.Rule{ + SrcCIDR: remoteCIDR, + Action: packetfilter.RuleActionMss, + ClampType: packetfilter.ToValue, + MssValue: strconv.Itoa(MSSFor1500MTU), + }, + ) } return rules, nil } -type forwardRuleSpecGenerator func() ([][]string, error) +type forwardRuleSpecGenerator func() ([]*packetfilter.Rule, error) const ( ForwardingSubmarinerMSSClampChain = "SUBMARINER-FWD-MSSCLAMP" @@ -155,46 +165,68 @@ const ( ) func (ovn *Handler) setupForwardingIptables() error { - if err := ovn.updateIPtableChains(constants.FilterTable, ForwardingSubmarinerMSSClampChain, ovn.getMSSClampingRuleSpecs); err != nil { + if err := ovn.updateIPtableChains(packetfilter.TableTypeFilter, ForwardingSubmarinerMSSClampChain, + ovn.getMSSClampingRuleSpecs); err != nil { return err } - return ovn.updateIPtableChains(constants.FilterTable, ForwardingSubmarinerFWDChain, ovn.getForwardingRuleSpecs) + return ovn.updateIPtableChains(packetfilter.TableTypeFilter, ForwardingSubmarinerFWDChain, ovn.getForwardingRuleSpecs) } func (ovn *Handler) addNoMasqueradeIPTables(subnet string) error { - err := errors.Wrapf(ovn.ipt.AppendUnique(constants.NATTable, constants.SmPostRoutingChain, - []string{"-d", subnet, "-j", "ACCEPT"}...), "error updating %q rules for subnet %q", + err := errors.Wrapf(ovn.pFilter.AppendUnique(packetfilter.TableTypeNAT, constants.SmPostRoutingChain, + &packetfilter.Rule{ + DestCIDR: subnet, + Action: packetfilter.RuleActionAccept, + }), "error updating %q rules for subnet %q", constants.SmPostRoutingChain, subnet) if err != nil { return err } - return errors.Wrapf(ovn.ipt.AppendUnique(constants.NATTable, constants.SmPostRoutingChain, - []string{"-s", subnet, "-j", "ACCEPT"}...), "error updating %q rules for subnet %q", + return errors.Wrapf(ovn.pFilter.AppendUnique(packetfilter.TableTypeNAT, constants.SmPostRoutingChain, + &packetfilter.Rule{ + SrcCIDR: subnet, + Action: packetfilter.RuleActionAccept, + }), "error updating %q rules for subnet %q", constants.SmPostRoutingChain, subnet) } func (ovn *Handler) removeNoMasqueradeIPTables(subnet string) error { - err := errors.Wrapf(ovn.ipt.Delete(constants.NATTable, constants.SmPostRoutingChain, - []string{"-d", subnet, "-j", "ACCEPT"}...), "error updating %q rules for subnet %q", + err := errors.Wrapf(ovn.pFilter.Delete(packetfilter.TableTypeNAT, constants.SmPostRoutingChain, + &packetfilter.Rule{ + DestCIDR: subnet, + Action: packetfilter.RuleActionAccept, + }), "error updating %q rules for subnet %q", constants.SmPostRoutingChain, subnet) if err != nil { return err } - return errors.Wrapf(ovn.ipt.Delete(constants.NATTable, constants.SmPostRoutingChain, - []string{"-s", subnet, "-j", "ACCEPT"}...), "error updating %q rules for subnet %q", + return errors.Wrapf(ovn.pFilter.Delete(packetfilter.TableTypeNAT, constants.SmPostRoutingChain, + &packetfilter.Rule{ + SrcCIDR: subnet, + Action: packetfilter.RuleActionAccept, + }), "error updating %q rules for subnet %q", constants.SmPostRoutingChain, subnet) } func (ovn *Handler) cleanupForwardingIptables() error { - if err := ovn.ipt.ClearChain(constants.FilterTable, ForwardingSubmarinerMSSClampChain); err != nil { + if err := ovn.pFilter.DeleteIPHookChain(&packetfilter.ChainIPHook{ + Name: ForwardingSubmarinerMSSClampChain, + Type: packetfilter.ChainTypeFilter, + Hook: packetfilter.ChainHookForward, + Priority: packetfilter.ChainPriorityFirst, + }); err != nil { return errors.Wrapf(err, "error clearing chain %q", ForwardingSubmarinerMSSClampChain) } - return errors.Wrapf(ovn.ipt.ClearChain(constants.FilterTable, ForwardingSubmarinerFWDChain), - "error clearing chain %q", ForwardingSubmarinerFWDChain) + return errors.Wrapf(ovn.pFilter.DeleteIPHookChain(&packetfilter.ChainIPHook{ + Name: ForwardingSubmarinerFWDChain, + Type: packetfilter.ChainTypeFilter, + Hook: packetfilter.ChainHookForward, + Priority: packetfilter.ChainPriorityFirst, + }), "error clearing chain %q", ForwardingSubmarinerFWDChain) } func (ovn *Handler) getRouteToOVNDataPlane() (*netlink.Route, error) { @@ -210,8 +242,15 @@ func (ovn *Handler) getRouteToOVNDataPlane() (*netlink.Route, error) { } func (ovn *Handler) initIPtablesChains() error { - if err := iptcommon.InitSubmarinerPostRoutingChain(ovn.ipt); err != nil { - return errors.Wrap(err, "error initializing POST routing chain") + logger.V(log.DEBUG).Infof("Install/ensure %q/%s IPHook chain exists", constants.SmPostRoutingChain, "NAT") + + if err := ovn.pFilter.CreateIPHookChainIfNotExists(&packetfilter.ChainIPHook{ + Name: constants.SmPostRoutingChain, + Type: packetfilter.ChainTypeNAT, + Hook: packetfilter.ChainHookPostrouting, + Priority: packetfilter.ChainPriorityFirst, + }); err != nil { + return errors.Wrap(err, "error installing IPHook chain") } if err := ovn.ensureForwardChains(); err != nil { @@ -222,28 +261,32 @@ func (ovn *Handler) initIPtablesChains() error { } func (ovn *Handler) ensureForwardChains() error { - if err := ovn.ipt.CreateChainIfNotExists(constants.FilterTable, ForwardingSubmarinerMSSClampChain); err != nil { - return errors.Wrapf(err, "error creating chain %q", ForwardingSubmarinerMSSClampChain) - } - - if err := ovn.ipt.InsertUnique(constants.FilterTable, "FORWARD", 1, - []string{"-j", ForwardingSubmarinerMSSClampChain}); err != nil { - return errors.Wrapf(err, "error inserting rule for chain %q", ForwardingSubmarinerMSSClampChain) + if err := ovn.pFilter.CreateIPHookChainIfNotExists(&packetfilter.ChainIPHook{ + Name: ForwardingSubmarinerFWDChain, + Type: packetfilter.ChainTypeFilter, + Hook: packetfilter.ChainHookForward, + Priority: packetfilter.ChainPriorityFirst, + }); err != nil { + return errors.Wrap(err, "error installing IPHook chain") } - if err := ovn.ipt.CreateChainIfNotExists(constants.FilterTable, ForwardingSubmarinerFWDChain); err != nil { - return errors.Wrapf(err, "error creating chain %q", ForwardingSubmarinerFWDChain) + if err := ovn.pFilter.CreateIPHookChainIfNotExists(&packetfilter.ChainIPHook{ + Name: ForwardingSubmarinerMSSClampChain, + Type: packetfilter.ChainTypeFilter, + Hook: packetfilter.ChainHookForward, + Priority: packetfilter.ChainPriorityFirst, + }); err != nil { + return errors.Wrap(err, "error installing IPHook chain") } - return errors.Wrapf(ovn.ipt.InsertUnique(constants.FilterTable, "FORWARD", 2, []string{"-j", ForwardingSubmarinerFWDChain}), - "error inserting rule for chain %q", ForwardingSubmarinerFWDChain) + return nil } -func (ovn *Handler) updateIPtableChains(table, chain string, ruleGen forwardRuleSpecGenerator) error { +func (ovn *Handler) updateIPtableChains(table packetfilter.TableType, chain string, ruleGen forwardRuleSpecGenerator) error { ruleSpecs, err := ruleGen() if err != nil { return err } - return errors.Wrap(ovn.ipt.UpdateChainRules(table, chain, ruleSpecs), "error updating chain rules") + return errors.Wrap(ovn.pFilter.UpdateChainRules(table, chain, ruleSpecs), "error updating chain rules") } diff --git a/pkg/routeagent_driver/handlers/ovn/handler.go b/pkg/routeagent_driver/handlers/ovn/handler.go index 45c13a456..451a5a8cf 100644 --- a/pkg/routeagent_driver/handlers/ovn/handler.go +++ b/pkg/routeagent_driver/handlers/ovn/handler.go @@ -33,8 +33,8 @@ import ( clientset "github.com/submariner-io/submariner/pkg/client/clientset/versioned" "github.com/submariner-io/submariner/pkg/cni" "github.com/submariner-io/submariner/pkg/event" - "github.com/submariner-io/submariner/pkg/iptables" "github.com/submariner-io/submariner/pkg/netlink" + "github.com/submariner-io/submariner/pkg/packetfilter" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -59,7 +59,7 @@ type Handler struct { mutex sync.Mutex cableRoutingInterface *net.Interface netLink netlink.Interface - ipt iptables.Interface + pFilter packetfilter.Interface gatewayRouteController *GatewayRouteController nonGatewayRouteController *NonGatewayRouteController stopCh chan struct{} @@ -69,15 +69,15 @@ var logger = log.Logger{Logger: logf.Log.WithName("OVN")} func NewHandler(config *HandlerConfig) *Handler { // We'll panic if env is nil, this is intentional - ipt, err := iptables.New() + pFilter, err := packetfilter.New() if err != nil { - logger.Fatalf("Error initializing iptables in OVN routeagent handler: %s", err) + logger.Fatalf("Error initializing packetfilter in OVN routeagent handler: %s", err) } h := &Handler{ HandlerConfig: *config, netLink: netlink.New(), - ipt: ipt, + pFilter: pFilter, stopCh: make(chan struct{}), } diff --git a/pkg/routeagent_driver/handlers/ovn/handler_test.go b/pkg/routeagent_driver/handlers/ovn/handler_test.go index ee097d00a..d62cb429e 100644 --- a/pkg/routeagent_driver/handlers/ovn/handler_test.go +++ b/pkg/routeagent_driver/handlers/ovn/handler_test.go @@ -32,6 +32,7 @@ import ( "github.com/submariner-io/admiral/pkg/watcher" submarinerv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1" "github.com/submariner-io/submariner/pkg/event/testing" + "github.com/submariner-io/submariner/pkg/packetfilter" "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" "github.com/submariner-io/submariner/pkg/routeagent_driver/handlers/ovn" fakeovn "github.com/submariner-io/submariner/pkg/routeagent_driver/handlers/ovn/fake" @@ -313,8 +314,8 @@ var _ = Describe("Handler", func() { Context("on Uninstall", func() { It("should delete the table rules", func() { - Expect(t.ipTables.ChainExists(constants.FilterTable, ovn.ForwardingSubmarinerFWDChain)).To(BeTrue()) - Expect(t.ipTables.ChainExists(constants.FilterTable, ovn.ForwardingSubmarinerMSSClampChain)).To(BeTrue()) + Expect(t.pFilter.ChainExists(packetfilter.TableTypeFilter, ovn.ForwardingSubmarinerFWDChain)).To(BeTrue()) + Expect(t.pFilter.ChainExists(packetfilter.TableTypeFilter, ovn.ForwardingSubmarinerMSSClampChain)).To(BeTrue()) _ = t.netLink.RuleAdd(&netlink.Rule{ Table: constants.RouteAgentHostNetworkTableID, @@ -331,7 +332,7 @@ var _ = Describe("Handler", func() { t.netLink.AwaitNoRule(constants.RouteAgentHostNetworkTableID, "", "") t.netLink.AwaitNoRule(constants.RouteAgentInterClusterNetworkTableID, "", "") - Expect(t.ipTables.ChainExists(constants.FilterTable, ovn.ForwardingSubmarinerFWDChain)).To(BeFalse()) + Expect(t.pFilter.ChainExists(packetfilter.TableTypeFilter, ovn.ForwardingSubmarinerFWDChain)).To(BeFalse()) }) }) }) diff --git a/pkg/routeagent_driver/handlers/ovn/ovn_suite_test.go b/pkg/routeagent_driver/handlers/ovn/ovn_suite_test.go index ec9992f7e..a51e128b3 100644 --- a/pkg/routeagent_driver/handlers/ovn/ovn_suite_test.go +++ b/pkg/routeagent_driver/handlers/ovn/ovn_suite_test.go @@ -32,10 +32,9 @@ import ( fakesubm "github.com/submariner-io/submariner/pkg/client/clientset/versioned/fake" "github.com/submariner-io/submariner/pkg/event" eventtesting "github.com/submariner-io/submariner/pkg/event/testing" - "github.com/submariner-io/submariner/pkg/iptables" - fakeIPT "github.com/submariner-io/submariner/pkg/iptables/fake" netlinkAPI "github.com/submariner-io/submariner/pkg/netlink" fakenetlink "github.com/submariner-io/submariner/pkg/netlink/fake" + fakePF "github.com/submariner-io/submariner/pkg/packetfilter/fake" "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" "github.com/submariner-io/submariner/pkg/routeagent_driver/handlers/ovn" "github.com/vishvananda/netlink" @@ -73,7 +72,7 @@ type testDriver struct { k8sClient *fakek8s.Clientset dynClient *fakedynamic.FakeDynamicClient netLink *fakenetlink.NetLink - ipTables *fakeIPT.IPTables + pFilter *fakePF.PacketFilter handler event.Handler transitSwitchIP string mgmntIntfIP string @@ -94,11 +93,7 @@ func newTestDriver() *testDriver { netlinkAPI.NewFunc = func() netlinkAPI.Interface { return t.netLink } - - t.ipTables = fakeIPT.New() - iptables.NewFunc = func() (iptables.Interface, error) { - return t.ipTables, nil - } + t.pFilter = fakePF.New() link := &netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ diff --git a/pkg/routeagent_driver/handlers/ovn/route_config_syncer.go b/pkg/routeagent_driver/handlers/ovn/route_config_syncer.go index 36dfc46cf..fb0f6b5ca 100644 --- a/pkg/routeagent_driver/handlers/ovn/route_config_syncer.go +++ b/pkg/routeagent_driver/handlers/ovn/route_config_syncer.go @@ -23,8 +23,9 @@ import ( "time" "github.com/pkg/errors" + "github.com/submariner-io/admiral/pkg/log" + "github.com/submariner-io/submariner/pkg/packetfilter" "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" - iptcommon "github.com/submariner-io/submariner/pkg/routeagent_driver/iptables" "github.com/vishvananda/netlink" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/retry" @@ -143,10 +144,16 @@ func (ovn *Handler) handleInterfaceAddressChange() error { return errors.Wrap(err, "error syncing host network routes") } - if err := iptcommon.InitSubmarinerPostRoutingChain(ovn.ipt); err != nil { - return errors.Wrapf(err, "error syncing IPtable %q chain", constants.PostRoutingChain) - } + logger.V(log.DEBUG).Infof("Install/ensure %q/%s IPHook chain exists", constants.SmPostRoutingChain, "NAT") + if err := ovn.pFilter.CreateIPHookChainIfNotExists(&packetfilter.ChainIPHook{ + Name: constants.SmPostRoutingChain, + Type: packetfilter.ChainTypeNAT, + Hook: packetfilter.ChainHookPostrouting, + Priority: packetfilter.ChainPriorityFirst, + }); err != nil { + return errors.Wrap(err, "error installing IPHook chain") + } return nil }) diff --git a/pkg/routeagent_driver/handlers/ovn/uninstall.go b/pkg/routeagent_driver/handlers/ovn/uninstall.go index 8c80fddda..a893a6ed7 100644 --- a/pkg/routeagent_driver/handlers/ovn/uninstall.go +++ b/pkg/routeagent_driver/handlers/ovn/uninstall.go @@ -20,6 +20,7 @@ package ovn import ( "github.com/pkg/errors" + "github.com/submariner-io/submariner/pkg/packetfilter" "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" "github.com/submariner-io/submariner/pkg/routeagent_driver/handlers/ovn/vsctl" "github.com/vishvananda/netlink" @@ -53,8 +54,33 @@ func (ovn *Handler) Uninstall() error { constants.RouteAgentHostNetworkTableID) } - ovn.flushAndDeleteIPTableChains(constants.FilterTable, constants.ForwardChain, ForwardingSubmarinerFWDChain) - ovn.flushAndDeleteIPTableChains(constants.NATTable, constants.PostRoutingChain, constants.SmPostRoutingChain) + if err := ovn.pFilter.ClearChain(packetfilter.TableTypeFilter, ForwardingSubmarinerFWDChain); err != nil { + logger.Errorf(err, "Error flushing packetfilter chain %q of %q table", ForwardingSubmarinerFWDChain, "Filter") + } + + if err := ovn.pFilter.DeleteIPHookChain(&packetfilter.ChainIPHook{ + Name: ForwardingSubmarinerFWDChain, + Type: packetfilter.ChainTypeFilter, + Hook: packetfilter.ChainHookForward, + Priority: packetfilter.ChainPriorityFirst, + }); err != nil { + logger.Errorf(err, "DeleteIPHookChain %s returned error", + ForwardingSubmarinerFWDChain) + } + + if err := ovn.pFilter.ClearChain(packetfilter.TableTypeNAT, constants.SmPostRoutingChain); err != nil { + logger.Errorf(err, "Error flushing packetfilter chain %q of %q table", constants.SmPostRoutingChain, "NAT") + } + + if err := ovn.pFilter.DeleteIPHookChain(&packetfilter.ChainIPHook{ + Name: constants.SmPostRoutingChain, + Type: packetfilter.ChainTypeNAT, + Hook: packetfilter.ChainHookPostrouting, + Priority: packetfilter.ChainPriorityFirst, + }); err != nil { + logger.Errorf(err, "DeleteIPHookChain %s returned error", + constants.SmPostRoutingChain) + } return nil } @@ -77,27 +103,6 @@ func (ovn *Handler) cleanupRoutes() error { return nil } -func (ovn *Handler) flushAndDeleteIPTableChains(table, tableChain, submarinerChain string) { - logger.Infof("Flushing iptable entries in %q chain of %q table", submarinerChain, table) - - if err := ovn.ipt.ClearChain(table, submarinerChain); err != nil { - logger.Errorf(err, "Error flushing iptables chain %q of %q table", submarinerChain, table) - } - - logger.Infof("Deleting iptable entry in %q chain of %q table", tableChain, table) - - ruleSpec := []string{"-j", submarinerChain} - if err := ovn.ipt.Delete(table, tableChain, ruleSpec...); err != nil { - logger.Errorf(err, "Error deleting iptables rule from %q chain", tableChain) - } - - logger.Infof("Deleting iptable %q chain of %q table", submarinerChain, table) - - if err := ovn.ipt.DeleteChain(table, submarinerChain); err != nil { - logger.Errorf(err, "Error deleting iptable chain %q of table %q", submarinerChain, table) - } -} - // TODO need to be removed when the clusters are fully upgraded to new implementation. func (ovn *Handler) LegacyCleanup() { err := vsctl.DelInternalPort(ovnK8sSubmarinerBridge, ovnK8sSubmarinerInterface) diff --git a/pkg/routeagent_driver/iptables/iptables.go b/pkg/routeagent_driver/iptables/iptables.go deleted file mode 100644 index 597cddbd9..000000000 --- a/pkg/routeagent_driver/iptables/iptables.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -SPDX-License-Identifier: Apache-2.0 - -Copyright Contributors to the Submariner project. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package iptables - -import ( - "github.com/pkg/errors" - "github.com/submariner-io/admiral/pkg/log" - "github.com/submariner-io/submariner/pkg/iptables" - "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" - logf "sigs.k8s.io/controller-runtime/pkg/log" -) - -var logger = log.Logger{Logger: logf.Log.WithName("IPTables")} - -func InitSubmarinerPostRoutingChain(ipt iptables.Interface) error { - logger.V(log.DEBUG).Infof("Install/ensure %s chain exists", constants.SmPostRoutingChain) - - if err := ipt.CreateChainIfNotExists("nat", constants.SmPostRoutingChain); err != nil { - return errors.Wrapf(err, "unable to create %q chain in iptables", constants.SmPostRoutingChain) - } - - logger.V(log.DEBUG).Infof("Insert %s rule that has rules for inter-cluster traffic", constants.SmPostRoutingChain) - - forwardToSubPostroutingRuleSpec := []string{"-j", constants.SmPostRoutingChain} - - if err := ipt.PrependUnique("nat", "POSTROUTING", forwardToSubPostroutingRuleSpec); err != nil { - return errors.Wrapf(err, "unable to insert iptable rule in NAT table, POSTROUTING chain") - } - - return nil -} diff --git a/pkg/routeagent_driver/main.go b/pkg/routeagent_driver/main.go index 1c8af1a67..7639803e6 100644 --- a/pkg/routeagent_driver/main.go +++ b/pkg/routeagent_driver/main.go @@ -40,6 +40,8 @@ import ( "github.com/submariner-io/submariner/pkg/event/controller" eventlogger "github.com/submariner-io/submariner/pkg/event/logger" "github.com/submariner-io/submariner/pkg/node" + packetfilter "github.com/submariner-io/submariner/pkg/packetfilter" + iptables "github.com/submariner-io/submariner/pkg/packetfilter/iptables" "github.com/submariner-io/submariner/pkg/routeagent_driver/cabledriver" cniapi "github.com/submariner-io/submariner/pkg/routeagent_driver/cni" "github.com/submariner-io/submariner/pkg/routeagent_driver/constants" @@ -113,6 +115,10 @@ func main() { return } + // set packetfilter driver to iptables + // TODO: check which driver is supported on platform + packetfilter.SetNewDriverFn(iptables.New) + np := os.Getenv("SUBMARINER_NETWORKPLUGIN") if np == "" {