From 7b54b9846c89c88acb6bd6c159ab712605bda4ae Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Mon, 11 Mar 2024 13:15:56 +0200 Subject: [PATCH 01/27] Sub multiplexing through Filter API --- waku/v2/api/filter.go | 134 +++++++++++++++++++++ waku/v2/node/wakunode2.go | 2 +- waku/v2/peermanager/topic_event_handler.go | 3 +- waku/v2/peerstore/waku_peer_store.go | 50 +++----- waku/v2/protocol/content_filter.go | 9 ++ 5 files changed, 166 insertions(+), 32 deletions(-) create mode 100644 waku/v2/api/filter.go diff --git a/waku/v2/api/filter.go b/waku/v2/api/filter.go new file mode 100644 index 000000000..893a2be12 --- /dev/null +++ b/waku/v2/api/filter.go @@ -0,0 +1,134 @@ +package filter + +import ( + "context" + "sync" + "time" + + "github.com/waku-org/go-waku/waku/v2/protocol" + "github.com/waku-org/go-waku/waku/v2/protocol/filter" + "github.com/waku-org/go-waku/waku/v2/protocol/subscription" +) + +type FilterConfig struct { + MaxPeers uint +} + +type Sub struct { + sync.RWMutex + ContentFilter protocol.ContentFilter + DataCh chan *protocol.Envelope + Config FilterConfig + subs subscription.SubscriptionSet + wf *filter.WakuFilterLightNode + ctx context.Context + cancel context.CancelFunc +} + +func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilter protocol.ContentFilter, config FilterConfig) (*Sub, error) { + sub := new(Sub) + sub.wf = wf + sub.ctx, sub.cancel = context.WithCancel(ctx) + sub.subs = make(subscription.SubscriptionSet) + sub.DataCh = make(chan *protocol.Envelope) + sub.ContentFilter = contentFilter + sub.Config = config + + err := sub.subscribe(contentFilter, sub.Config.MaxPeers) + + if err == nil { + sub.healthCheckLoop() + return sub, nil + } else { + return nil, err + } +} + +func Unsubscribe(apiSub *Sub) error { + apiSub.RLock() + defer apiSub.RUnlock() + for _, s := range apiSub.subs { + apiSub.wf.UnsubscribeWithSubscription(apiSub.ctx, s) + } + apiSub.cancel() + return nil +} + +func (apiSub *Sub) healthCheckLoop() { + // Health checks + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-apiSub.ctx.Done(): + return + case <-ticker.C: + // Returns a map of pubsub topics to peer counts + m := apiSub.checkAliveness() + for t, cnt := range m { + if cnt < apiSub.Config.MaxPeers { + cFilter := protocol.ContentFilter{t, apiSub.ContentFilter.ContentTopics} + apiSub.subscribe(cFilter, apiSub.Config.MaxPeers-cnt) + } + } + } + } +} + +func (apiSub *Sub) checkAliveness() map[string]uint { + apiSub.RLock() + defer apiSub.RUnlock() + ch := make(chan string, len(apiSub.subs)) + wg := &sync.WaitGroup{} + wg.Add(len(apiSub.subs)) + for _, subDetails := range apiSub.subs { + go func(subDetails *subscription.SubscriptionDetails) { + defer wg.Done() + err := apiSub.wf.IsSubscriptionAlive(apiSub.ctx, subDetails) + + if err != nil { + subDetails.Close() + apiSub.Lock() + defer apiSub.Unlock() + delete(apiSub.subs, subDetails.ID) + } else { + ch <- subDetails.ContentFilter.PubsubTopic + } + }(subDetails) + + } + wg.Wait() + close(ch) + // Collect healthy topics + m := make(map[string]uint) + for topic := range ch { + m[topic]++ + } + + return m + +} +func (apiSub *Sub) subscribe(contentFilter protocol.ContentFilter, peerCount uint) error { + // Low-level subscribe, returns a set of SubscriptionDetails + subs, err := apiSub.wf.Subscribe(apiSub.ctx, contentFilter, filter.WithMaxPeersPerContentFilter(int(peerCount))) + if err != nil { + // TODO what if fails? + return err + } + apiSub.Lock() + defer apiSub.Unlock() + for _, s := range subs { + apiSub.subs[s.ID] = s + } + // Multiplex onto single channel + // Goroutines will exit once sub channels are closed + for _, subDetails := range subs { + go func(subDetails *subscription.SubscriptionDetails) { + for env := range subDetails.C { + apiSub.DataCh <- env + } + }(subDetails) + } + return nil + +} diff --git a/waku/v2/node/wakunode2.go b/waku/v2/node/wakunode2.go index 7f2a7cbef..33d3e891e 100644 --- a/waku/v2/node/wakunode2.go +++ b/waku/v2/node/wakunode2.go @@ -832,7 +832,7 @@ func (w *WakuNode) Peers() ([]*Peer, error) { Protocols: protocols, Connected: connected, Addrs: addrs, - PubsubTopics: topics, + PubsubTopics: maps.Keys(topics), }) } return peers, nil diff --git a/waku/v2/peermanager/topic_event_handler.go b/waku/v2/peermanager/topic_event_handler.go index 60e1f759b..3060bf7b9 100644 --- a/waku/v2/peermanager/topic_event_handler.go +++ b/waku/v2/peermanager/topic_event_handler.go @@ -13,6 +13,7 @@ import ( waku_proto "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/protocol/relay" "go.uber.org/zap" + "golang.org/x/exp/maps" ) func (pm *PeerManager) SubscribeToRelayEvtBus(bus event.Bus) error { @@ -103,7 +104,7 @@ func (pm *PeerManager) handleNewRelayTopicUnSubscription(pubsubTopic string) { logging.HostID("peerID", peer)) continue } - if len(peerTopics) == 1 && peerTopics[0] == pubsubTopic { + if len(peerTopics) == 1 && maps.Keys(peerTopics)[0] == pubsubTopic { err := pm.host.Network().ClosePeer(peer) if err != nil { pm.logger.Warn("Failed to disconnect connection towards peer", diff --git a/waku/v2/peerstore/waku_peer_store.go b/waku/v2/peerstore/waku_peer_store.go index 402d265ad..203b23103 100644 --- a/waku/v2/peerstore/waku_peer_store.go +++ b/waku/v2/peerstore/waku_peer_store.go @@ -8,6 +8,8 @@ import ( "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/peerstore" + "github.com/waku-org/go-waku/waku/v2/protocol" + "golang.org/x/exp/maps" ) // Origin is used to determine how the peer is identified, @@ -58,7 +60,7 @@ type WakuPeerstore interface { AddPubSubTopic(p peer.ID, topic string) error RemovePubSubTopic(p peer.ID, topic string) error - PubSubTopics(p peer.ID) ([]string, error) + PubSubTopics(p peer.ID) (protocol.TopicSet, error) SetPubSubTopics(p peer.ID, topics []string) error PeersByPubSubTopics(pubSubTopics []string, specificPeers ...peer.ID) peer.IDSlice PeersByPubSubTopic(pubSubTopic string, specificPeers ...peer.ID) peer.IDSlice @@ -175,13 +177,12 @@ func (ps *WakuPeerstoreImpl) AddPubSubTopic(p peer.ID, topic string) error { if err != nil { return err } - for _, t := range existingTopics { - if t == topic { - return nil - } + + if _, found := existingTopics[topic]; found { + return nil } - existingTopics = append(existingTopics, topic) - return ps.peerStore.Put(p, peerPubSubTopics, existingTopics) + existingTopics[topic] = struct{}{} + return ps.peerStore.Put(p, peerPubSubTopics, maps.Keys(existingTopics)) } // RemovePubSubTopic removes a pubSubTopic from the peer @@ -195,14 +196,9 @@ func (ps *WakuPeerstoreImpl) RemovePubSubTopic(p peer.ID, topic string) error { return nil } - for i := range existingTopics { - if existingTopics[i] == topic { - existingTopics = append(existingTopics[:i], existingTopics[i+1:]...) - break - } - } + delete(existingTopics, topic) - err = ps.SetPubSubTopics(p, existingTopics) + err = ps.SetPubSubTopics(p, maps.Keys(existingTopics)) if err != nil { return err } @@ -215,16 +211,16 @@ func (ps *WakuPeerstoreImpl) SetPubSubTopics(p peer.ID, topics []string) error { } // PubSubTopics fetches list of pubSubTopics for a peer -func (ps *WakuPeerstoreImpl) PubSubTopics(p peer.ID) ([]string, error) { +func (ps *WakuPeerstoreImpl) PubSubTopics(p peer.ID) (protocol.TopicSet, error) { result, err := ps.peerStore.Get(p, peerPubSubTopics) if err != nil { if errors.Is(err, peerstore.ErrNotFound) { - return nil, nil + return protocol.NewTopicSet(), nil } else { return nil, err } } - return result.([]string), nil + return protocol.NewTopicSet((result.([]string))...), nil } // PeersByPubSubTopic Returns list of peers that support list of pubSubTopics @@ -235,22 +231,16 @@ func (ps *WakuPeerstoreImpl) PeersByPubSubTopics(pubSubTopics []string, specific } var result peer.IDSlice for _, p := range specificPeers { - topics, err := ps.PubSubTopics(p) + peerMatch := true + peerTopics, err := ps.PubSubTopics(p) if err == nil { - //Convoluted and crazy logic to find subset of topics - // Could not find a better way to do it? - peerTopicMap := make(map[string]struct{}) - match := true - for _, topic := range topics { - peerTopicMap[topic] = struct{}{} - } - for _, topic := range pubSubTopics { - if _, ok := peerTopicMap[topic]; !ok { - match = false + for _, t := range pubSubTopics { + if _, ok := peerTopics[t]; !ok { + peerMatch = false break } } - if match { + if peerMatch { result = append(result, p) } } //Note: skipping a peer in case of an error as there would be others available. @@ -268,7 +258,7 @@ func (ps *WakuPeerstoreImpl) PeersByPubSubTopic(pubSubTopic string, specificPeer for _, p := range specificPeers { topics, err := ps.PubSubTopics(p) if err == nil { - for _, topic := range topics { + for topic := range topics { if topic == pubSubTopic { result = append(result, p) } diff --git a/waku/v2/protocol/content_filter.go b/waku/v2/protocol/content_filter.go index f09cf52b3..0cff1cbc5 100644 --- a/waku/v2/protocol/content_filter.go +++ b/waku/v2/protocol/content_filter.go @@ -6,6 +6,15 @@ type PubsubTopicStr = string type ContentTopicStr = string type ContentTopicSet map[string]struct{} +type TopicSet map[string]struct{} + +func NewTopicSet(topics ...string) TopicSet { + s := make(TopicSet, len(topics)) + for _, t := range topics { + s[t] = struct{}{} + } + return s +} func NewContentTopicSet(contentTopics ...string) ContentTopicSet { s := make(ContentTopicSet, len(contentTopics)) From 03044dea6f06499b5c1dec0ec4700c19fcf4035c Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Sat, 16 Mar 2024 13:05:58 +0200 Subject: [PATCH 02/27] Update api/filter --- waku/v2/api/filter.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/waku/v2/api/filter.go b/waku/v2/api/filter.go index 893a2be12..26d8d4c34 100644 --- a/waku/v2/api/filter.go +++ b/waku/v2/api/filter.go @@ -8,6 +8,7 @@ import ( "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/protocol/filter" "github.com/waku-org/go-waku/waku/v2/protocol/subscription" + "golang.org/x/exp/maps" ) type FilterConfig struct { @@ -78,7 +79,10 @@ func (apiSub *Sub) healthCheckLoop() { func (apiSub *Sub) checkAliveness() map[string]uint { apiSub.RLock() defer apiSub.RUnlock() - ch := make(chan string, len(apiSub.subs)) + + // Only healthy topics will be pushed here + ch := make(chan string) + wg := &sync.WaitGroup{} wg.Add(len(apiSub.subs)) for _, subDetails := range apiSub.subs { @@ -101,8 +105,11 @@ func (apiSub *Sub) checkAliveness() map[string]uint { close(ch) // Collect healthy topics m := make(map[string]uint) - for topic := range ch { - m[topic]++ + for t := range maps.Keys(protocol.ContentFilterToPubSubTopicMap(apiSub.ContentFilter)) { + m[t] = 0 + } + for t := range ch { + m[t]++ } return m From 295b509fbf7bdad61414f8407c8ba87fd3a3c41c Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Sat, 16 Mar 2024 13:06:34 +0200 Subject: [PATCH 03/27] Add api/filter_test --- waku/v2/api/filter.go | 5 +- waku/v2/api/filter_test.go | 47 +++ .../filter/filter_proto_ident_test.go | 44 +-- waku/v2/protocol/filter/filter_push_test.go | 7 +- .../protocol/filter/filter_subscribe_test.go | 52 +-- waku/v2/protocol/filter/filter_test.go | 353 +----------------- .../filter/filter_unsubscribe_test.go | 61 ++- waku/v2/protocol/filter/test_utils.go | 341 +++++++++++++++++ 8 files changed, 463 insertions(+), 447 deletions(-) create mode 100644 waku/v2/api/filter_test.go create mode 100644 waku/v2/protocol/filter/test_utils.go diff --git a/waku/v2/api/filter.go b/waku/v2/api/filter.go index 26d8d4c34..746f91e15 100644 --- a/waku/v2/api/filter.go +++ b/waku/v2/api/filter.go @@ -1,4 +1,4 @@ -package filter +package api import ( "context" @@ -105,7 +105,8 @@ func (apiSub *Sub) checkAliveness() map[string]uint { close(ch) // Collect healthy topics m := make(map[string]uint) - for t := range maps.Keys(protocol.ContentFilterToPubSubTopicMap(apiSub.ContentFilter)) { + topicMap, _ := protocol.ContentFilterToPubSubTopicMap(apiSub.ContentFilter) + for _, t := range maps.Keys(topicMap) { m[t] = 0 } for t := range ch { diff --git a/waku/v2/api/filter_test.go b/waku/v2/api/filter_test.go new file mode 100644 index 000000000..0f9560d40 --- /dev/null +++ b/waku/v2/api/filter_test.go @@ -0,0 +1,47 @@ +package api + +import ( + "context" + "sync" + "testing" + + "github.com/libp2p/go-libp2p/core/host" + "github.com/stretchr/testify/suite" + "github.com/waku-org/go-waku/waku/v2/protocol" + "github.com/waku-org/go-waku/waku/v2/protocol/filter" + "github.com/waku-org/go-waku/waku/v2/protocol/relay" + "github.com/waku-org/go-waku/waku/v2/protocol/subscription" + "go.uber.org/zap" +) + +func TestFilterApiSuite(t *testing.T) { + suite.Run(t, new(FilterApiTestSuite)) +} + +const defaultTestPubSubTopic = "/waku/2/go/filter/test" +const defaultTestContentTopic = "/test/10/my-app" + +type FilterApiTestSuite struct { + suite.Suite + + testTopic string + testContentTopic string + ctx context.Context + ctxCancel context.CancelFunc + lightNode *filter.WakuFilterLightNode + lightNodeHost host.Host + relayNode *relay.WakuRelay + relaySub *relay.Subscription + fullNode *filter.WakuFilterFullNode + fullNodeHost host.Host + wg *sync.WaitGroup + contentFilter protocol.ContentFilter + subDetails []*subscription.SubscriptionDetails + log *zap.Logger +} + +type WakuMsg struct { + pubSubTopic string + contentTopic string + payload string +} diff --git a/waku/v2/protocol/filter/filter_proto_ident_test.go b/waku/v2/protocol/filter/filter_proto_ident_test.go index 4fae951fb..94274e8a9 100644 --- a/waku/v2/protocol/filter/filter_proto_ident_test.go +++ b/waku/v2/protocol/filter/filter_proto_ident_test.go @@ -31,11 +31,7 @@ import ( func (s *FilterTestSuite) TestCreateSubscription() { // Initial subscribe s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(func() { - _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(s.testContentTopic, utils.GetUnixEpoch()), relay.WithPubSubTopic(s.testTopic)) - s.Require().NoError(err) - - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}, s.subDetails[0].C) } func (s *FilterTestSuite) TestModifySubscription() { @@ -43,21 +39,13 @@ func (s *FilterTestSuite) TestModifySubscription() { // Initial subscribe s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(func() { - _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(s.testContentTopic, utils.GetUnixEpoch()), relay.WithPubSubTopic(s.testTopic)) - s.Require().NoError(err) - - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}, s.subDetails[0].C) // Subscribe to another content_topic newContentTopic := "Topic_modified" s.subDetails = s.subscribe(s.testTopic, newContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(func() { - _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(newContentTopic, utils.GetUnixEpoch()), relay.WithPubSubTopic(s.testTopic)) - s.Require().NoError(err) - - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, newContentTopic, ""}, s.subDetails[0].C) } func (s *FilterTestSuite) TestMultipleMessages() { @@ -65,17 +53,9 @@ func (s *FilterTestSuite) TestMultipleMessages() { // Initial subscribe s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(func() { - _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(s.testContentTopic, utils.GetUnixEpoch(), "first"), relay.WithPubSubTopic(s.testTopic)) - s.Require().NoError(err) - - }, s.subDetails[0].C) - - s.waitForMsg(func() { - _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(s.testContentTopic, utils.GetUnixEpoch(), "second"), relay.WithPubSubTopic(s.testTopic)) - s.Require().NoError(err) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "first"}, s.subDetails[0].C) - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "second"}, s.subDetails[0].C) } func (wf *WakuFilterLightNode) incorrectSubscribeRequest(ctx context.Context, params *FilterSubscribeParameters, @@ -222,7 +202,7 @@ func (wf *WakuFilterLightNode) IncorrectSubscribe(ctx context.Context, contentFi func (s *FilterTestSuite) TestIncorrectSubscribeIdentifier() { log := utils.Logger() - s.log = log + s.Log = log s.wg = &sync.WaitGroup{} // Create test context @@ -231,9 +211,9 @@ func (s *FilterTestSuite) TestIncorrectSubscribeIdentifier() { s.testTopic = defaultTestPubSubTopic s.testContentTopic = defaultTestContentTopic - s.lightNode = s.makeWakuFilterLightNode(true, true) + s.lightNode = s.StartNode(s.MakeWakuFilterLightNode()) - s.relayNode, s.fullNode = s.makeWakuFilterFullNode(s.testTopic, false, true) + s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, false) //Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) @@ -259,7 +239,7 @@ func (wf *WakuFilterLightNode) startWithIncorrectPushProto() error { func (s *FilterTestSuite) TestIncorrectPushIdentifier() { log := utils.Logger() - s.log = log + s.Log = log s.wg = &sync.WaitGroup{} // Create test context @@ -270,9 +250,9 @@ func (s *FilterTestSuite) TestIncorrectPushIdentifier() { s.testTopic = defaultTestPubSubTopic s.testContentTopic = defaultTestContentTopic - s.lightNode = s.makeWakuFilterLightNode(false, true) + s.lightNode = s.MakeWakuFilterLightNode() - s.relayNode, s.fullNode = s.makeWakuFilterFullNode(s.testTopic, false, true) + s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, false) // Re-start light node with unsupported prefix for match func s.lightNode.Stop() @@ -298,7 +278,7 @@ func (s *FilterTestSuite) TestIncorrectPushIdentifier() { // Message should never arrive -> exit after timeout select { case msg := <-s.subDetails[0].C: - s.log.Info("Light node received a msg") + s.Log.Info("Light node received a msg") s.Require().Nil(msg) case <-time.After(1 * time.Second): s.Require().True(true) diff --git a/waku/v2/protocol/filter/filter_push_test.go b/waku/v2/protocol/filter/filter_push_test.go index e5bc530f3..4a852db6e 100644 --- a/waku/v2/protocol/filter/filter_push_test.go +++ b/waku/v2/protocol/filter/filter_push_test.go @@ -2,12 +2,13 @@ package filter import ( "context" + "strconv" + "time" + "github.com/waku-org/go-waku/tests" "github.com/waku-org/go-waku/waku/v2/protocol/relay" "github.com/waku-org/go-waku/waku/v2/utils" "go.uber.org/zap" - "strconv" - "time" ) func (s *FilterTestSuite) TestValidPayloadsASCII() { @@ -131,7 +132,7 @@ func (s *FilterTestSuite) TestLargePayloadsUTF8() { // Generate large string for i := range messages { messages[i].payload, _ = tests.GenerateRandomUTF8String(153600) - s.log.Info("Generated payload with ", zap.String("length", strconv.Itoa(len(messages[i].payload)))) + s.Log.Info("Generated payload with ", zap.String("length", strconv.Itoa(len(messages[i].payload)))) } // All messages should be received diff --git a/waku/v2/protocol/filter/filter_subscribe_test.go b/waku/v2/protocol/filter/filter_subscribe_test.go index b7579f8c5..4736b2a50 100644 --- a/waku/v2/protocol/filter/filter_subscribe_test.go +++ b/waku/v2/protocol/filter/filter_subscribe_test.go @@ -21,22 +21,16 @@ func (s *FilterTestSuite) TestWakuFilter() { s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Should be received - s.waitForMsg(func() { - s.publishMsg(s.testTopic, s.testContentTopic, "first") - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "first"}, s.subDetails[0].C) // Wrong content topic - s.waitForTimeout(func() { - s.publishMsg(s.testTopic, "TopicB", "second") - }, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{s.testTopic, "TopicB", "second"}, s.subDetails[0].C) _, err := s.lightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) // Should not receive after unsubscribe - s.waitForTimeout(func() { - s.publishMsg(s.testTopic, s.testContentTopic, "third") - }, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{s.testTopic, s.testContentTopic, "third"}, s.subDetails[0].C) // Two new subscriptions with same [peer, contentFilter] s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) @@ -46,13 +40,9 @@ func (s *FilterTestSuite) TestWakuFilter() { s.Require().Equal(len(s.lightNode.Subscriptions()), 2) // Should be received on both subscriptions - s.waitForMsg(func() { - s.publishMsg(s.testTopic, s.testContentTopic, "fourth") - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "fourth"}, s.subDetails[0].C) - s.waitForMsg(func() { - s.publishMsg(s.testTopic, s.testContentTopic, "fifth") - }, secondSub[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "fifth"}, secondSub[0].C) s.waitForMsg(nil, s.subDetails[0].C) s.waitForMsg(nil, secondSub[0].C) @@ -62,9 +52,7 @@ func (s *FilterTestSuite) TestWakuFilter() { s.Require().NoError(err) // Should still receive - s.waitForMsg(func() { - s.publishMsg(s.testTopic, s.testContentTopic, "sixth") - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "sixth"}, s.subDetails[0].C) // Unsubscribe from first sub only _, err = s.lightNode.UnsubscribeWithSubscription(s.ctx, s.subDetails[0]) @@ -73,9 +61,7 @@ func (s *FilterTestSuite) TestWakuFilter() { s.Require().Equal(len(s.lightNode.Subscriptions()), 0) // Should not receive after unsubscribe - s.waitForTimeout(func() { - s.publishMsg(s.testTopic, s.testContentTopic, "seventh") - }, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{s.testTopic, s.testContentTopic, "seventh"}, s.subDetails[0].C) } func (s *FilterTestSuite) TestPubSubSingleContentTopic() { @@ -83,9 +69,7 @@ func (s *FilterTestSuite) TestPubSubSingleContentTopic() { s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Message should be received - s.waitForMsg(func() { - s.publishMsg(s.testTopic, s.testContentTopic, "test_msg") - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "test_msg"}, s.subDetails[0].C) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -119,9 +103,9 @@ func (s *FilterTestSuite) TestMultiPubSubMultiContentTopic() { // Create test context s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 20*time.Second) // Test can't exceed 20 seconds - s.lightNode = s.makeWakuFilterLightNode(true, true) + s.lightNode = s.StartNode(s.MakeWakuFilterLightNode()) - s.relayNode, s.fullNode = s.makeWakuFilterFullNode(s.testTopic, true, true) + s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, true) // Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) @@ -133,14 +117,14 @@ func (s *FilterTestSuite) TestMultiPubSubMultiContentTopic() { // Subscribe for _, m := range messages { s.subDetails = append(s.subDetails, s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID())...) - s.log.Info("Subscribing ", zap.String("PubSubTopic", m.pubSubTopic)) + s.Log.Info("Subscribing ", zap.String("PubSubTopic", m.pubSubTopic)) _, err := s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.pubSubTopic)) s.Require().NoError(err) } // Debug to see subscriptions in light node for _, sub := range s.subDetails { - s.log.Info("Light Node subscription ", zap.String("PubSubTopic", sub.ContentFilter.PubsubTopic), zap.String("ContentTopic", sub.ContentFilter.ContentTopicsList()[0])) + s.Log.Info("Light Node subscription ", zap.String("PubSubTopic", sub.ContentFilter.PubsubTopic), zap.String("ContentTopic", sub.ContentFilter.ContentTopicsList()[0])) } // All messages should be received @@ -270,7 +254,7 @@ func (s *FilterTestSuite) TestSubscribeErrorHandling() { func (s *FilterTestSuite) TestMultipleFullNodeSubscriptions() { log := utils.Logger() - s.log = log + s.Log = log s.wg = &sync.WaitGroup{} // Create test context @@ -279,10 +263,10 @@ func (s *FilterTestSuite) TestMultipleFullNodeSubscriptions() { fullNodeIDHex := make([]byte, hex.EncodedLen(len([]byte(s.fullNodeHost.ID())))) _ = hex.Encode(fullNodeIDHex, []byte(s.fullNodeHost.ID())) - s.log.Info("Already subscribed to", zap.String("fullNode", string(fullNodeIDHex))) + s.Log.Info("Already subscribed to", zap.String("fullNode", string(fullNodeIDHex))) // This will overwrite values with the second node info - s.relayNode, s.fullNode = s.makeWakuFilterFullNode(s.testTopic, false, true) + s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, false) // Connect to second full and relay node s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) @@ -292,7 +276,7 @@ func (s *FilterTestSuite) TestMultipleFullNodeSubscriptions() { fullNodeIDHex = make([]byte, hex.EncodedLen(len([]byte(s.fullNodeHost.ID())))) _ = hex.Encode(fullNodeIDHex, []byte(s.fullNodeHost.ID())) - s.log.Info("Subscribing to second", zap.String("fullNode", string(fullNodeIDHex))) + s.Log.Info("Subscribing to second", zap.String("fullNode", string(fullNodeIDHex))) // Subscribe to the second full node s.contentFilter = protocol.ContentFilter{PubsubTopic: s.testTopic, ContentTopics: protocol.NewContentTopicSet(s.testContentTopic)} @@ -308,7 +292,7 @@ func (s *FilterTestSuite) TestSubscribeMultipleLightNodes() { // Create test context s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 10*time.Second) // Test can't exceed 10 seconds - lightNode2 := s.makeWakuFilterLightNode(true, true) + lightNode2 := s.StartNode(s.MakeWakuFilterLightNode()) // Connect node2 lightNode2.h.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) @@ -344,7 +328,7 @@ func (s *FilterTestSuite) TestSubscribeFullNode2FullNode() { // Create test context s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 10*time.Second) - _, fullNode2 := s.makeWakuFilterFullNode(testTopic, false, false) + _, fullNode2 := s.MakeWakuFilterFullNodeNoSharing(testTopic, false) // Connect nodes fullNode2.h.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) diff --git a/waku/v2/protocol/filter/filter_test.go b/waku/v2/protocol/filter/filter_test.go index e5dac7182..5102d9e11 100644 --- a/waku/v2/protocol/filter/filter_test.go +++ b/waku/v2/protocol/filter/filter_test.go @@ -2,25 +2,16 @@ package filter import ( "context" - "crypto/rand" "errors" - "fmt" - "strconv" "sync" "testing" "time" - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/peerstore" - "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/suite" "github.com/waku-org/go-waku/tests" "github.com/waku-org/go-waku/waku/v2/protocol" - "github.com/waku-org/go-waku/waku/v2/protocol/relay" - "github.com/waku-org/go-waku/waku/v2/protocol/subscription" "github.com/waku-org/go-waku/waku/v2/service" - "github.com/waku-org/go-waku/waku/v2/timesource" "github.com/waku-org/go-waku/waku/v2/utils" "go.uber.org/zap" ) @@ -32,309 +23,9 @@ func TestFilterSuite(t *testing.T) { const defaultTestPubSubTopic = "/waku/2/go/filter/test" const defaultTestContentTopic = "/test/10/my-app" -type FilterTestSuite struct { - suite.Suite - - testTopic string - testContentTopic string - ctx context.Context - ctxCancel context.CancelFunc - lightNode *WakuFilterLightNode - lightNodeHost host.Host - relayNode *relay.WakuRelay - relaySub *relay.Subscription - fullNode *WakuFilterFullNode - fullNodeHost host.Host - wg *sync.WaitGroup - contentFilter protocol.ContentFilter - subDetails []*subscription.SubscriptionDetails - log *zap.Logger -} - -type WakuMsg struct { - pubSubTopic string - contentTopic string - payload string -} - -func (s *FilterTestSuite) makeWakuRelay(topic string, shared bool) (*relay.WakuRelay, *relay.Subscription, host.Host, relay.Broadcaster) { - - broadcaster := relay.NewBroadcaster(10) - s.Require().NoError(broadcaster.Start(context.Background())) - - port, err := tests.FindFreePort(s.T(), "", 5) - s.Require().NoError(err) - - host, err := tests.MakeHost(context.Background(), port, rand.Reader) - s.Require().NoError(err) - - relay := relay.NewWakuRelay(broadcaster, 0, timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.log) - relay.SetHost(host) - - if shared { - s.fullNodeHost = host - } - - err = relay.Start(context.Background()) - s.Require().NoError(err) - - sub, err := relay.Subscribe(context.Background(), protocol.NewContentFilter(topic)) - s.Require().NoError(err) - - return relay, sub[0], host, broadcaster -} - -func (s *FilterTestSuite) makeWakuFilterLightNode(start bool, withBroadcaster bool) *WakuFilterLightNode { - port, err := tests.FindFreePort(s.T(), "", 5) - s.Require().NoError(err) - - host, err := tests.MakeHost(context.Background(), port, rand.Reader) - s.Require().NoError(err) - var b relay.Broadcaster - if withBroadcaster { - b = relay.NewBroadcaster(10) - s.Require().NoError(b.Start(context.Background())) - } - filterPush := NewWakuFilterLightNode(b, nil, timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.log) - filterPush.SetHost(host) - s.lightNodeHost = host - if start { - err = filterPush.Start(context.Background()) - s.Require().NoError(err) - } - - return filterPush -} - -func (s *FilterTestSuite) makeWakuFilterFullNode(topic string, withRegisterAll bool, shared bool) (*relay.WakuRelay, *WakuFilterFullNode) { - var sub *relay.Subscription - node, relaySub, host, broadcaster := s.makeWakuRelay(topic, shared) - - if shared { - s.relaySub = relaySub - } - - node2Filter := NewWakuFilterFullNode(timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.log) - node2Filter.SetHost(host) - - if withRegisterAll { - sub = broadcaster.RegisterForAll() - } else { - sub = broadcaster.Register(protocol.NewContentFilter(topic)) - } - - err := node2Filter.Start(s.ctx, sub) - s.Require().NoError(err) - - return node, node2Filter -} - -func (s *FilterTestSuite) waitForMsg(fn func(), ch chan *protocol.Envelope) { - s.wg.Add(1) - var msgFound = false - go func() { - defer s.wg.Done() - select { - case env := <-ch: - for _, topic := range s.contentFilter.ContentTopicsList() { - if topic == env.Message().GetContentTopic() { - msgFound = true - } - } - s.Require().True(msgFound) - case <-time.After(1 * time.Second): - s.Require().Fail("Message timeout") - case <-s.ctx.Done(): - s.Require().Fail("test exceeded allocated time") - } - }() - - if fn != nil { - fn() - } - - s.wg.Wait() -} - -func matchOneOfManyMsg(one WakuMsg, many []WakuMsg) bool { - for _, m := range many { - if m.pubSubTopic == one.pubSubTopic && - m.contentTopic == one.contentTopic && - m.payload == one.payload { - return true - } - } - - return false -} - -func (s *FilterTestSuite) waitForMessages(fn func(), subs []*subscription.SubscriptionDetails, expected []WakuMsg) { - s.wg.Add(1) - msgCount := len(expected) - found := 0 - s.log.Info("Expected messages ", zap.String("count", strconv.Itoa(msgCount))) - s.log.Info("Existing subscriptions ", zap.String("count", strconv.Itoa(len(subs)))) - - go func() { - defer s.wg.Done() - for _, sub := range subs { - s.log.Info("Looking at ", zap.String("pubSubTopic", sub.ContentFilter.PubsubTopic)) - for i := 0; i < msgCount; i++ { - select { - case env, ok := <-sub.C: - if !ok { - continue - } - received := WakuMsg{ - pubSubTopic: env.PubsubTopic(), - contentTopic: env.Message().GetContentTopic(), - payload: string(env.Message().GetPayload()), - } - s.log.Debug("received message ", zap.String("pubSubTopic", received.pubSubTopic), zap.String("contentTopic", received.contentTopic), zap.String("payload", received.payload)) - if matchOneOfManyMsg(received, expected) { - found++ - } - case <-time.After(3 * time.Second): - - case <-s.ctx.Done(): - s.Require().Fail("test exceeded allocated time") - } - } - } - }() - - if fn != nil { - fn() - } - - s.wg.Wait() - s.Require().True(msgCount == found) -} - -func (s *FilterTestSuite) waitForTimeout(fn func(), ch chan *protocol.Envelope) { - s.wg.Add(1) - go func() { - defer s.wg.Done() - select { - case env, ok := <-ch: - if ok { - s.Require().Fail("should not receive another message", zap.String("payload", string(env.Message().Payload))) - } - case <-time.After(1 * time.Second): - // Timeout elapsed, all good - case <-s.ctx.Done(): - s.Require().Fail("waitForTimeout test exceeded allocated time") - } - }() - - fn() - - s.wg.Wait() -} - -func (s *FilterTestSuite) subscribe(pubsubTopic string, contentTopic string, peer peer.ID) []*subscription.SubscriptionDetails { - - for _, sub := range s.subDetails { - if sub.ContentFilter.PubsubTopic == pubsubTopic { - sub.Add(contentTopic) - s.contentFilter = sub.ContentFilter - subDetails, err := s.lightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(peer)) - s.Require().NoError(err) - return subDetails - } - } - - s.contentFilter = protocol.ContentFilter{PubsubTopic: pubsubTopic, ContentTopics: protocol.NewContentTopicSet(contentTopic)} - - subDetails, err := s.lightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(peer)) - s.Require().NoError(err) - - // Sleep to make sure the filter is subscribed - time.Sleep(1 * time.Second) - - return subDetails -} - -func (s *FilterTestSuite) unsubscribe(pubsubTopic string, contentTopic string, peer peer.ID) []*subscription.SubscriptionDetails { - - for _, sub := range s.subDetails { - if sub.ContentFilter.PubsubTopic == pubsubTopic { - topicsCount := len(sub.ContentFilter.ContentTopicsList()) - if topicsCount == 1 { - _, err := s.lightNode.Unsubscribe(s.ctx, sub.ContentFilter, WithPeer(peer)) - s.Require().NoError(err) - } else { - sub.Remove(contentTopic) - } - s.contentFilter = sub.ContentFilter - } - } - - return s.lightNode.Subscriptions() -} - -func (s *FilterTestSuite) publishMsg(topic, contentTopic string, optionalPayload ...string) { - var payload string - if len(optionalPayload) > 0 { - payload = optionalPayload[0] - } else { - payload = "123" - } - - _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(contentTopic, utils.GetUnixEpoch(), payload), relay.WithPubSubTopic(topic)) - s.Require().NoError(err) -} - -func (s *FilterTestSuite) publishMessages(msgs []WakuMsg) { - for _, m := range msgs { - _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(m.contentTopic, utils.GetUnixEpoch(), m.payload), relay.WithPubSubTopic(m.pubSubTopic)) - s.Require().NoError(err) - } -} - -func prepareData(quantity int, topics, contentTopics, payloads bool, sg tests.StringGenerator) []WakuMsg { - var ( - pubsubTopic = defaultTestPubSubTopic // Has to be the same with initial s.testTopic - contentTopic = defaultTestContentTopic // Has to be the same with initial s.testContentTopic - payload = "test_msg" - messages []WakuMsg - strMaxLenght = 4097 - generatedString = "" - ) - - for i := 0; i < quantity; i++ { - msg := WakuMsg{ - pubSubTopic: pubsubTopic, - contentTopic: contentTopic, - payload: payload, - } - - if sg != nil { - generatedString, _ = sg(strMaxLenght) - - } - - if topics { - msg.pubSubTopic = fmt.Sprintf("%s%02d%s", pubsubTopic, i, generatedString) - } - - if contentTopics { - msg.contentTopic = fmt.Sprintf("%s%02d%s", contentTopic, i, generatedString) - } - - if payloads { - msg.payload = fmt.Sprintf("%s%02d%s", payload, i, generatedString) - } - - messages = append(messages, msg) - } - - return messages -} - func (s *FilterTestSuite) SetupTest() { log := utils.Logger() //.Named("filterv2-test") - s.log = log + s.Log = log // Use a pointer to WaitGroup so that to avoid copying // https://pkg.go.dev/sync#WaitGroup s.wg = &sync.WaitGroup{} @@ -347,11 +38,11 @@ func (s *FilterTestSuite) SetupTest() { s.testTopic = defaultTestPubSubTopic s.testContentTopic = defaultTestContentTopic - s.lightNode = s.makeWakuFilterLightNode(true, true) + s.lightNode = s.StartNode(s.MakeWakuFilterLightNode()) //TODO: Add tests to verify broadcaster. - s.relayNode, s.fullNode = s.makeWakuFilterFullNode(s.testTopic, false, true) + s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, false) // Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) @@ -363,7 +54,7 @@ func (s *FilterTestSuite) SetupTest() { func (s *FilterTestSuite) TearDownTest() { s.fullNode.Stop() s.relayNode.Stop() - s.relaySub.Unsubscribe() + s.RelaySub.Unsubscribe() s.lightNode.Stop() s.ctxCancel() } @@ -410,7 +101,7 @@ func (s *FilterTestSuite) TestStartStop() { var wg sync.WaitGroup wg.Add(2) - s.lightNode = s.makeWakuFilterLightNode(false, false) + s.lightNode = s.MakeWakuFilterLightNode() stopNode := func() { for i := 0; i < 100000; i++ { @@ -454,25 +145,19 @@ func (s *FilterTestSuite) TestAutoShard() { s.testContentTopic = cTopic1Str s.testTopic = pubSubTopic.String() - s.lightNode = s.makeWakuFilterLightNode(true, false) - s.relayNode, s.fullNode = s.makeWakuFilterFullNode(pubSubTopic.String(), false, true) + s.lightNode = s.StartNode(s.MakeWakuFilterLightNode()) + s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(pubSubTopic.String(), false) s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) err = s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) - s.log.Info("Testing Autoshard:CreateSubscription") + s.Log.Info("Testing Autoshard:CreateSubscription") s.subDetails = s.subscribe("", s.testContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(func() { - _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(s.testContentTopic, utils.GetUnixEpoch()), relay.WithPubSubTopic(s.testTopic)) - s.Require().NoError(err) - - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}, s.subDetails[0].C) // Wrong content topic - s.waitForTimeout(func() { - s.publishMsg(s.testTopic, "TopicB", "second") - }, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{s.testTopic, "TopicB", "second"}, s.subDetails[0].C) _, err = s.lightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) @@ -480,27 +165,21 @@ func (s *FilterTestSuite) TestAutoShard() { time.Sleep(1 * time.Second) // Should not receive after unsubscribe - s.waitForTimeout(func() { - s.publishMsg(s.testTopic, s.testContentTopic, "third") - }, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{s.testTopic, s.testContentTopic, "third"}, s.subDetails[0].C) s.subDetails = s.subscribe("", s.testContentTopic, s.fullNodeHost.ID()) - s.log.Info("Testing Autoshard:SubscriptionPing") + s.Log.Info("Testing Autoshard:SubscriptionPing") err = s.lightNode.Ping(context.Background(), s.fullNodeHost.ID()) s.Require().NoError(err) // Test ModifySubscription Subscribe to another content_topic - s.log.Info("Testing Autoshard:ModifySubscription") + s.Log.Info("Testing Autoshard:ModifySubscription") newContentTopic := "0/test/1/testTopic1/proto" s.subDetails = s.subscribe("", newContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(func() { - _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(newContentTopic, utils.GetUnixEpoch()), relay.WithPubSubTopic(s.testTopic)) - s.Require().NoError(err) - - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, newContentTopic, ""}, s.subDetails[0].C) _, err = s.lightNode.Unsubscribe(s.ctx, protocol.ContentFilter{ PubsubTopic: s.testTopic, @@ -537,9 +216,9 @@ func (s *FilterTestSuite) TestLightNodeIsListening() { } func (s *FilterTestSuite) BeforeTest(suiteName, testName string) { - s.log.Info("Executing ", zap.String("testName", testName)) + s.Log.Info("Executing ", zap.String("testName", testName)) } func (s *FilterTestSuite) AfterTest(suiteName, testName string) { - s.log.Info("Finished executing ", zap.String("testName", testName)) + s.Log.Info("Finished executing ", zap.String("testName", testName)) } diff --git a/waku/v2/protocol/filter/filter_unsubscribe_test.go b/waku/v2/protocol/filter/filter_unsubscribe_test.go index a498de0ca..d6bc3010d 100644 --- a/waku/v2/protocol/filter/filter_unsubscribe_test.go +++ b/waku/v2/protocol/filter/filter_unsubscribe_test.go @@ -3,13 +3,14 @@ package filter import ( "context" "crypto/rand" + "strconv" + "time" + "github.com/libp2p/go-libp2p/core/peerstore" "github.com/waku-org/go-waku/logging" "github.com/waku-org/go-waku/tests" "github.com/waku-org/go-waku/waku/v2/protocol" "go.uber.org/zap" - "strconv" - "time" ) func (s *FilterTestSuite) TestUnsubscribeSingleContentTopic() { @@ -21,26 +22,18 @@ func (s *FilterTestSuite) TestUnsubscribeSingleContentTopic() { s.subDetails = s.subscribe(s.testTopic, newContentTopic, s.fullNodeHost.ID()) // Message is possible to receive for original contentTopic - s.waitForMsg(func() { - s.publishMsg(s.testTopic, s.testContentTopic, "test_msg") - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "test_msg"}, s.subDetails[0].C) // Message is possible to receive for new contentTopic - s.waitForMsg(func() { - s.publishMsg(s.testTopic, newContentTopic, "test_msg") - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, newContentTopic, "test_msg"}, s.subDetails[0].C) _ = s.unsubscribe(s.testTopic, newContentTopic, s.fullNodeHost.ID()) // Message should not be received for new contentTopic as it was unsubscribed - s.waitForTimeout(func() { - s.publishMsg(s.testTopic, newContentTopic, "test_msg") - }, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{s.testTopic, newContentTopic, "test_msg"}, s.subDetails[0].C) // Message is still possible to receive for original contentTopic - s.waitForMsg(func() { - s.publishMsg(s.testTopic, s.testContentTopic, "test_msg2") - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "test_msg2"}, s.subDetails[0].C) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -68,15 +61,11 @@ func (s *FilterTestSuite) TestUnsubscribeMultiContentTopic() { // Messages should not be received for the last two contentTopics as it was unsubscribed for _, m := range messages[1:] { - s.waitForTimeout(func() { - s.publishMsg(m.pubSubTopic, m.contentTopic, m.payload) - }, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}, s.subDetails[0].C) } // Message is still possible to receive for the first contentTopic - s.waitForMsg(func() { - s.publishMsg(messages[0].pubSubTopic, messages[0].contentTopic, messages[0].payload) - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{messages[0].pubSubTopic, messages[0].contentTopic, messages[0].payload}, s.subDetails[0].C) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -87,9 +76,9 @@ func (s *FilterTestSuite) TestUnsubscribeMultiPubSubMultiContentTopic() { s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 20*time.Second) - s.lightNode = s.makeWakuFilterLightNode(true, true) + s.lightNode = s.StartNode(s.MakeWakuFilterLightNode()) - s.relayNode, s.fullNode = s.makeWakuFilterFullNode(s.testTopic, true, true) + s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, true) // Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNode.h), peerstore.PermanentAddrTTL) @@ -117,9 +106,7 @@ func (s *FilterTestSuite) TestUnsubscribeMultiPubSubMultiContentTopic() { // No messages can be received with previous subscriptions for i, m := range messages { - s.waitForTimeout(func() { - s.publishMsg(m.pubSubTopic, m.contentTopic, m.payload) - }, s.subDetails[i].C) + s.waitForTimeout(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}, s.subDetails[i].C) } } @@ -127,9 +114,9 @@ func (s *FilterTestSuite) TestUnsubscribeErrorHandling() { s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 20*time.Second) - s.lightNode = s.makeWakuFilterLightNode(true, true) + s.lightNode = s.StartNode(s.MakeWakuFilterLightNode()) - s.relayNode, s.fullNode = s.makeWakuFilterFullNode(s.testTopic, true, true) + s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, true) // Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) @@ -218,9 +205,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllWithoutContentTopics() { // Messages should not be received for any contentTopics for _, m := range messages { - s.waitForTimeout(func() { - s.publishMsg(m.pubSubTopic, m.contentTopic, m.payload) - }, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}, s.subDetails[0].C) } } @@ -228,9 +213,9 @@ func (s *FilterTestSuite) TestUnsubscribeAllDiffPubSubContentTopics() { s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 20*time.Second) - s.lightNode = s.makeWakuFilterLightNode(true, true) + s.lightNode = s.StartNode(s.MakeWakuFilterLightNode()) - s.relayNode, s.fullNode = s.makeWakuFilterFullNode(s.testTopic, true, true) + s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, true) // Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNode.h), peerstore.PermanentAddrTTL) @@ -257,9 +242,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllDiffPubSubContentTopics() { // No messages can be received with previous subscriptions for i, m := range messages { - s.waitForTimeout(func() { - s.publishMsg(m.pubSubTopic, m.contentTopic, m.payload) - }, s.subDetails[i].C) + s.waitForTimeout(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}, s.subDetails[i].C) } } @@ -282,15 +265,15 @@ func (s *FilterTestSuite) TestUnsubscribeAllUnrelatedPeer() { host, err := tests.MakeHost(context.Background(), 12345, rand.Reader) s.Require().NoError(err) - s.log.Info("Host ID", logging.HostID("FullNode", s.fullNodeHost.ID())) - s.log.Info("Host ID", logging.HostID("LightNode", s.lightNodeHost.ID())) - s.log.Info("Host ID", logging.HostID("Unrelated", host.ID())) + s.Log.Info("Host ID", logging.HostID("FullNode", s.fullNodeHost.ID())) + s.Log.Info("Host ID", logging.HostID("LightNode", s.lightNodeHost.ID())) + s.Log.Info("Host ID", logging.HostID("Unrelated", host.ID())) // Unsubscribe all with unrelated peer specification pushResult, err := s.lightNode.UnsubscribeAll(s.ctx, WithPeer(host.ID())) for e := range pushResult.errs { - s.log.Info("Push Result ", zap.String("error", strconv.Itoa(e))) + s.Log.Info("Push Result ", zap.String("error", strconv.Itoa(e))) } // All messages should be received because peer ID used was not related to any subscription diff --git a/waku/v2/protocol/filter/test_utils.go b/waku/v2/protocol/filter/test_utils.go new file mode 100644 index 000000000..043fa1f09 --- /dev/null +++ b/waku/v2/protocol/filter/test_utils.go @@ -0,0 +1,341 @@ +package filter + +import ( + "context" + "crypto/rand" + "fmt" + "strconv" + "sync" + "time" + + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/suite" + "github.com/waku-org/go-waku/tests" + "github.com/waku-org/go-waku/waku/v2/protocol" + "github.com/waku-org/go-waku/waku/v2/protocol/relay" + "github.com/waku-org/go-waku/waku/v2/protocol/subscription" + "github.com/waku-org/go-waku/waku/v2/timesource" + "github.com/waku-org/go-waku/waku/v2/utils" + "go.uber.org/zap" +) + +type FilterTestSuite struct { + suite.Suite + + testTopic string + testContentTopic string + ctx context.Context + ctxCancel context.CancelFunc + lightNode *WakuFilterLightNode + lightNodeHost host.Host + relayNode *relay.WakuRelay + fullNode *WakuFilterFullNode + fullNodeHost host.Host + wg *sync.WaitGroup + contentFilter protocol.ContentFilter + subDetails []*subscription.SubscriptionDetails + + Log *zap.Logger + RelaySub *relay.Subscription +} + +type WakuMsg struct { + pubSubTopic string + contentTopic string + payload string +} + +type RelayData struct { + WakuRelay *relay.WakuRelay + Subscription *relay.Subscription + Host host.Host + Broadcaster relay.Broadcaster +} + +func (s *FilterTestSuite) MakeWakuRelay(topic string) RelayData { + + broadcaster := relay.NewBroadcaster(10) + s.Require().NoError(broadcaster.Start(context.Background())) + + port, err := tests.FindFreePort(s.T(), "", 5) + s.Require().NoError(err) + + host, err := tests.MakeHost(context.Background(), port, rand.Reader) + s.Require().NoError(err) + + relay := relay.NewWakuRelay(broadcaster, 0, timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.Log) + relay.SetHost(host) + + err = relay.Start(context.Background()) + s.Require().NoError(err) + + sub, err := relay.Subscribe(context.Background(), protocol.NewContentFilter(topic)) + s.Require().NoError(err) + + return RelayData{relay, sub[0], host, broadcaster} +} + +func (s *FilterTestSuite) MakeWakuFilterFullNodeNoSharing(topic string, withRegisterAll bool) (*relay.WakuRelay, *WakuFilterFullNode) { + + relayData := s.MakeWakuRelay(topic) + + node2Filter := NewWakuFilterFullNode(timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.Log) + node2Filter.SetHost(relayData.Host) + + var sub *relay.Subscription + if withRegisterAll { + sub = relayData.Broadcaster.RegisterForAll() + } else { + sub = relayData.Broadcaster.Register(protocol.NewContentFilter(topic)) + } + + err := node2Filter.Start(s.ctx, sub) + s.Require().NoError(err) + + return relayData.WakuRelay, node2Filter +} + +func (s *FilterTestSuite) MakeWakuFilterFullNode(topic string, withRegisterAll bool) (*relay.WakuRelay, *WakuFilterFullNode) { + relayData := s.MakeWakuRelay(topic) + + s.fullNodeHost = relayData.Host + s.RelaySub = relayData.Subscription + + node2Filter := NewWakuFilterFullNode(timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.Log) + node2Filter.SetHost(relayData.Host) + + var sub *relay.Subscription + if withRegisterAll { + sub = relayData.Broadcaster.RegisterForAll() + } else { + sub = relayData.Broadcaster.Register(protocol.NewContentFilter(topic)) + } + + err := node2Filter.Start(s.ctx, sub) + s.Require().NoError(err) + + return relayData.WakuRelay, node2Filter +} + +func (s *FilterTestSuite) MakeWakuFilterLightNode() *WakuFilterLightNode { + port, err := tests.FindFreePort(s.T(), "", 5) + s.Require().NoError(err) + + host, err := tests.MakeHost(context.Background(), port, rand.Reader) + s.Require().NoError(err) + b := relay.NewBroadcaster(10) + s.Require().NoError(b.Start(context.Background())) + filterPush := NewWakuFilterLightNode(b, nil, timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.Log) + filterPush.SetHost(host) + s.lightNodeHost = host + + return filterPush +} + +func (s *FilterTestSuite) StartNode(node *WakuFilterLightNode) *WakuFilterLightNode { + err := node.Start(context.Background()) + s.Require().NoError(err) + return node +} + +func (s *FilterTestSuite) waitForMsg(msg *WakuMsg, ch chan *protocol.Envelope) { + s.wg.Add(1) + var msgFound = false + go func() { + defer s.wg.Done() + select { + case env := <-ch: + for _, topic := range s.contentFilter.ContentTopicsList() { + if topic == env.Message().GetContentTopic() { + msgFound = true + } + } + s.Require().True(msgFound) + case <-time.After(1 * time.Second): + s.Require().Fail("Message timeout") + case <-s.ctx.Done(): + s.Require().Fail("test exceeded allocated time") + } + }() + + if msg != nil { + s.publishMsg(msg) + } + + s.wg.Wait() +} + +func matchOneOfManyMsg(one WakuMsg, many []WakuMsg) bool { + for _, m := range many { + if m.pubSubTopic == one.pubSubTopic && + m.contentTopic == one.contentTopic && + m.payload == one.payload { + return true + } + } + + return false +} + +func (s *FilterTestSuite) waitForMessages(fn func(), subs []*subscription.SubscriptionDetails, expected []WakuMsg) { + s.wg.Add(1) + msgCount := len(expected) + found := 0 + s.Log.Info("Expected messages ", zap.String("count", strconv.Itoa(msgCount))) + s.Log.Info("Existing subscriptions ", zap.String("count", strconv.Itoa(len(subs)))) + + go func() { + defer s.wg.Done() + for _, sub := range subs { + s.Log.Info("Looking at ", zap.String("pubSubTopic", sub.ContentFilter.PubsubTopic)) + for i := 0; i < msgCount; i++ { + select { + case env, ok := <-sub.C: + if !ok { + continue + } + received := WakuMsg{ + pubSubTopic: env.PubsubTopic(), + contentTopic: env.Message().GetContentTopic(), + payload: string(env.Message().GetPayload()), + } + s.Log.Debug("received message ", zap.String("pubSubTopic", received.pubSubTopic), zap.String("contentTopic", received.contentTopic), zap.String("payload", received.payload)) + if matchOneOfManyMsg(received, expected) { + found++ + } + case <-time.After(3 * time.Second): + + case <-s.ctx.Done(): + s.Require().Fail("test exceeded allocated time") + } + } + } + }() + + if fn != nil { + fn() + } + + s.wg.Wait() + s.Require().True(msgCount == found) +} + +func (s *FilterTestSuite) waitForTimeout(msg *WakuMsg, ch chan *protocol.Envelope) { + s.wg.Add(1) + go func() { + defer s.wg.Done() + select { + case env, ok := <-ch: + if ok { + s.Require().Fail("should not receive another message", zap.String("payload", string(env.Message().Payload))) + } + case <-time.After(1 * time.Second): + // Timeout elapsed, all good + case <-s.ctx.Done(): + s.Require().Fail("waitForTimeout test exceeded allocated time") + } + }() + + s.publishMsg(msg) + + s.wg.Wait() +} + +func (s *FilterTestSuite) subscribe(pubsubTopic string, contentTopic string, peer peer.ID) []*subscription.SubscriptionDetails { + + for _, sub := range s.subDetails { + if sub.ContentFilter.PubsubTopic == pubsubTopic { + sub.Add(contentTopic) + s.contentFilter = sub.ContentFilter + subDetails, err := s.lightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(peer)) + s.Require().NoError(err) + return subDetails + } + } + + s.contentFilter = protocol.ContentFilter{PubsubTopic: pubsubTopic, ContentTopics: protocol.NewContentTopicSet(contentTopic)} + + subDetails, err := s.lightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(peer)) + s.Require().NoError(err) + + // Sleep to make sure the filter is subscribed + time.Sleep(1 * time.Second) + + return subDetails +} + +func (s *FilterTestSuite) unsubscribe(pubsubTopic string, contentTopic string, peer peer.ID) []*subscription.SubscriptionDetails { + + for _, sub := range s.subDetails { + if sub.ContentFilter.PubsubTopic == pubsubTopic { + topicsCount := len(sub.ContentFilter.ContentTopicsList()) + if topicsCount == 1 { + _, err := s.lightNode.Unsubscribe(s.ctx, sub.ContentFilter, WithPeer(peer)) + s.Require().NoError(err) + } else { + sub.Remove(contentTopic) + } + s.contentFilter = sub.ContentFilter + } + } + + return s.lightNode.Subscriptions() +} + +func (s *FilterTestSuite) publishMsg(msg *WakuMsg) { + if len(msg.payload) == 0 { + msg.payload = "123" + } + + _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(msg.contentTopic, utils.GetUnixEpoch(), msg.payload), relay.WithPubSubTopic(msg.pubSubTopic)) + s.Require().NoError(err) +} + +func (s *FilterTestSuite) publishMessages(msgs []WakuMsg) { + for _, m := range msgs { + _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(m.contentTopic, utils.GetUnixEpoch(), m.payload), relay.WithPubSubTopic(m.pubSubTopic)) + s.Require().NoError(err) + } +} + +func prepareData(quantity int, topics, contentTopics, payloads bool, sg tests.StringGenerator) []WakuMsg { + var ( + pubsubTopic = defaultTestPubSubTopic // Has to be the same with initial s.testTopic + contentTopic = defaultTestContentTopic // Has to be the same with initial s.testContentTopic + payload = "test_msg" + messages []WakuMsg + strMaxLenght = 4097 + generatedString = "" + ) + + for i := 0; i < quantity; i++ { + msg := WakuMsg{ + pubSubTopic: pubsubTopic, + contentTopic: contentTopic, + payload: payload, + } + + if sg != nil { + generatedString, _ = sg(strMaxLenght) + + } + + if topics { + msg.pubSubTopic = fmt.Sprintf("%s%02d%s", pubsubTopic, i, generatedString) + } + + if contentTopics { + msg.contentTopic = fmt.Sprintf("%s%02d%s", contentTopic, i, generatedString) + } + + if payloads { + msg.payload = fmt.Sprintf("%s%02d%s", payload, i, generatedString) + } + + messages = append(messages, msg) + } + + return messages +} From 92064425bebafe87981928d5ed8521cda2f3a98d Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Thu, 28 Mar 2024 16:41:42 +0200 Subject: [PATCH 04/27] Upd --- .../filter/filter_proto_ident_test.go | 10 +++++----- waku/v2/protocol/filter/filter_push_test.go | 2 +- .../protocol/filter/filter_subscribe_test.go | 20 +++++++++---------- waku/v2/protocol/filter/filter_test.go | 8 ++++---- .../filter/filter_unsubscribe_test.go | 18 ++++++++--------- waku/v2/protocol/filter/test_utils.go | 12 +++++++++-- 6 files changed, 39 insertions(+), 31 deletions(-) diff --git a/waku/v2/protocol/filter/filter_proto_ident_test.go b/waku/v2/protocol/filter/filter_proto_ident_test.go index 94274e8a9..af0f79266 100644 --- a/waku/v2/protocol/filter/filter_proto_ident_test.go +++ b/waku/v2/protocol/filter/filter_proto_ident_test.go @@ -31,7 +31,7 @@ import ( func (s *FilterTestSuite) TestCreateSubscription() { // Initial subscribe s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}) } func (s *FilterTestSuite) TestModifySubscription() { @@ -39,13 +39,13 @@ func (s *FilterTestSuite) TestModifySubscription() { // Initial subscribe s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}) // Subscribe to another content_topic newContentTopic := "Topic_modified" s.subDetails = s.subscribe(s.testTopic, newContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(&WakuMsg{s.testTopic, newContentTopic, ""}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, newContentTopic, ""}) } func (s *FilterTestSuite) TestMultipleMessages() { @@ -53,9 +53,9 @@ func (s *FilterTestSuite) TestMultipleMessages() { // Initial subscribe s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "first"}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "first"}) - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "second"}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "second"}) } func (wf *WakuFilterLightNode) incorrectSubscribeRequest(ctx context.Context, params *FilterSubscribeParameters, diff --git a/waku/v2/protocol/filter/filter_push_test.go b/waku/v2/protocol/filter/filter_push_test.go index 4a852db6e..b938bc78d 100644 --- a/waku/v2/protocol/filter/filter_push_test.go +++ b/waku/v2/protocol/filter/filter_push_test.go @@ -159,7 +159,7 @@ func (s *FilterTestSuite) TestFuturePayloadEncryptionVersion() { s.Require().NoError(err) // Should be received - s.waitForMsg(nil, s.subDetails[0].C) + s.waitForMsg(nil) _, err = s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) diff --git a/waku/v2/protocol/filter/filter_subscribe_test.go b/waku/v2/protocol/filter/filter_subscribe_test.go index 4736b2a50..463488137 100644 --- a/waku/v2/protocol/filter/filter_subscribe_test.go +++ b/waku/v2/protocol/filter/filter_subscribe_test.go @@ -21,16 +21,16 @@ func (s *FilterTestSuite) TestWakuFilter() { s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Should be received - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "first"}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "first"}) // Wrong content topic - s.waitForTimeout(&WakuMsg{s.testTopic, "TopicB", "second"}, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{s.testTopic, "TopicB", "second"}) _, err := s.lightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) // Should not receive after unsubscribe - s.waitForTimeout(&WakuMsg{s.testTopic, s.testContentTopic, "third"}, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{s.testTopic, s.testContentTopic, "third"}) // Two new subscriptions with same [peer, contentFilter] s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) @@ -40,19 +40,19 @@ func (s *FilterTestSuite) TestWakuFilter() { s.Require().Equal(len(s.lightNode.Subscriptions()), 2) // Should be received on both subscriptions - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "fourth"}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "fourth"}) - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "fifth"}, secondSub[0].C) + s.waitForMsgFromChan(&WakuMsg{s.testTopic, s.testContentTopic, "fifth"}, secondSub[0].C) - s.waitForMsg(nil, s.subDetails[0].C) - s.waitForMsg(nil, secondSub[0].C) + s.waitForMsg(nil) + s.waitForMsgFromChan(nil, secondSub[0].C) // Unsubscribe from second sub only _, err = s.lightNode.UnsubscribeWithSubscription(s.ctx, secondSub[0]) s.Require().NoError(err) // Should still receive - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "sixth"}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "sixth"}) // Unsubscribe from first sub only _, err = s.lightNode.UnsubscribeWithSubscription(s.ctx, s.subDetails[0]) @@ -61,7 +61,7 @@ func (s *FilterTestSuite) TestWakuFilter() { s.Require().Equal(len(s.lightNode.Subscriptions()), 0) // Should not receive after unsubscribe - s.waitForTimeout(&WakuMsg{s.testTopic, s.testContentTopic, "seventh"}, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{s.testTopic, s.testContentTopic, "seventh"}) } func (s *FilterTestSuite) TestPubSubSingleContentTopic() { @@ -69,7 +69,7 @@ func (s *FilterTestSuite) TestPubSubSingleContentTopic() { s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Message should be received - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "test_msg"}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "test_msg"}) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) diff --git a/waku/v2/protocol/filter/filter_test.go b/waku/v2/protocol/filter/filter_test.go index 5102d9e11..c22816864 100644 --- a/waku/v2/protocol/filter/filter_test.go +++ b/waku/v2/protocol/filter/filter_test.go @@ -154,10 +154,10 @@ func (s *FilterTestSuite) TestAutoShard() { s.Log.Info("Testing Autoshard:CreateSubscription") s.subDetails = s.subscribe("", s.testContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}) // Wrong content topic - s.waitForTimeout(&WakuMsg{s.testTopic, "TopicB", "second"}, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{s.testTopic, "TopicB", "second"}) _, err = s.lightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) @@ -165,7 +165,7 @@ func (s *FilterTestSuite) TestAutoShard() { time.Sleep(1 * time.Second) // Should not receive after unsubscribe - s.waitForTimeout(&WakuMsg{s.testTopic, s.testContentTopic, "third"}, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{s.testTopic, s.testContentTopic, "third"}) s.subDetails = s.subscribe("", s.testContentTopic, s.fullNodeHost.ID()) @@ -179,7 +179,7 @@ func (s *FilterTestSuite) TestAutoShard() { newContentTopic := "0/test/1/testTopic1/proto" s.subDetails = s.subscribe("", newContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(&WakuMsg{s.testTopic, newContentTopic, ""}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, newContentTopic, ""}) _, err = s.lightNode.Unsubscribe(s.ctx, protocol.ContentFilter{ PubsubTopic: s.testTopic, diff --git a/waku/v2/protocol/filter/filter_unsubscribe_test.go b/waku/v2/protocol/filter/filter_unsubscribe_test.go index d6bc3010d..1a3183e5c 100644 --- a/waku/v2/protocol/filter/filter_unsubscribe_test.go +++ b/waku/v2/protocol/filter/filter_unsubscribe_test.go @@ -22,18 +22,18 @@ func (s *FilterTestSuite) TestUnsubscribeSingleContentTopic() { s.subDetails = s.subscribe(s.testTopic, newContentTopic, s.fullNodeHost.ID()) // Message is possible to receive for original contentTopic - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "test_msg"}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "test_msg"}) // Message is possible to receive for new contentTopic - s.waitForMsg(&WakuMsg{s.testTopic, newContentTopic, "test_msg"}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, newContentTopic, "test_msg"}) _ = s.unsubscribe(s.testTopic, newContentTopic, s.fullNodeHost.ID()) // Message should not be received for new contentTopic as it was unsubscribed - s.waitForTimeout(&WakuMsg{s.testTopic, newContentTopic, "test_msg"}, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{s.testTopic, newContentTopic, "test_msg"}) // Message is still possible to receive for original contentTopic - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "test_msg2"}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "test_msg2"}) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -61,11 +61,11 @@ func (s *FilterTestSuite) TestUnsubscribeMultiContentTopic() { // Messages should not be received for the last two contentTopics as it was unsubscribed for _, m := range messages[1:] { - s.waitForTimeout(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}) } // Message is still possible to receive for the first contentTopic - s.waitForMsg(&WakuMsg{messages[0].pubSubTopic, messages[0].contentTopic, messages[0].payload}, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{messages[0].pubSubTopic, messages[0].contentTopic, messages[0].payload}) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -106,7 +106,7 @@ func (s *FilterTestSuite) TestUnsubscribeMultiPubSubMultiContentTopic() { // No messages can be received with previous subscriptions for i, m := range messages { - s.waitForTimeout(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}, s.subDetails[i].C) + s.waitForTimeoutFromChan(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}, s.subDetails[i].C) } } @@ -205,7 +205,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllWithoutContentTopics() { // Messages should not be received for any contentTopics for _, m := range messages { - s.waitForTimeout(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}) } } @@ -242,7 +242,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllDiffPubSubContentTopics() { // No messages can be received with previous subscriptions for i, m := range messages { - s.waitForTimeout(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}, s.subDetails[i].C) + s.waitForTimeoutFromChan(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}, s.subDetails[i].C) } } diff --git a/waku/v2/protocol/filter/test_utils.go b/waku/v2/protocol/filter/test_utils.go index 043fa1f09..b416ca601 100644 --- a/waku/v2/protocol/filter/test_utils.go +++ b/waku/v2/protocol/filter/test_utils.go @@ -140,7 +140,11 @@ func (s *FilterTestSuite) StartNode(node *WakuFilterLightNode) *WakuFilterLightN return node } -func (s *FilterTestSuite) waitForMsg(msg *WakuMsg, ch chan *protocol.Envelope) { +func (s *FilterTestSuite) waitForMsg(msg *WakuMsg) { + s.waitForMsgFromChan(msg, s.subDetails[0].C) +} + +func (s *FilterTestSuite) waitForMsgFromChan(msg *WakuMsg, ch chan *protocol.Envelope) { s.wg.Add(1) var msgFound = false go func() { @@ -222,7 +226,11 @@ func (s *FilterTestSuite) waitForMessages(fn func(), subs []*subscription.Subscr s.Require().True(msgCount == found) } -func (s *FilterTestSuite) waitForTimeout(msg *WakuMsg, ch chan *protocol.Envelope) { +func (s *FilterTestSuite) waitForTimeout(msg *WakuMsg) { + s.waitForTimeoutFromChan(msg, s.subDetails[0].C) +} + +func (s *FilterTestSuite) waitForTimeoutFromChan(msg *WakuMsg, ch chan *protocol.Envelope) { s.wg.Add(1) go func() { defer s.wg.Done() From bf9c1bc850fb762e6326cbfa179636d25e015e97 Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Fri, 29 Mar 2024 15:48:08 +0200 Subject: [PATCH 05/27] Get/Make changes --- .../filter/filter_proto_ident_test.go | 9 +- .../protocol/filter/filter_subscribe_test.go | 16 ++-- waku/v2/protocol/filter/filter_test.go | 12 +-- .../filter/filter_unsubscribe_test.go | 15 ++-- waku/v2/protocol/filter/test_utils.go | 83 +++++++++---------- 5 files changed, 69 insertions(+), 66 deletions(-) diff --git a/waku/v2/protocol/filter/filter_proto_ident_test.go b/waku/v2/protocol/filter/filter_proto_ident_test.go index af0f79266..6a3625aed 100644 --- a/waku/v2/protocol/filter/filter_proto_ident_test.go +++ b/waku/v2/protocol/filter/filter_proto_ident_test.go @@ -211,9 +211,10 @@ func (s *FilterTestSuite) TestIncorrectSubscribeIdentifier() { s.testTopic = defaultTestPubSubTopic s.testContentTopic = defaultTestContentTopic - s.lightNode = s.StartNode(s.MakeWakuFilterLightNode()) + s.MakeWakuFilterLightNode() + s.StartLightNode() - s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, false) + s.MakeWakuFilterFullNode(s.testTopic, false) //Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) @@ -250,9 +251,9 @@ func (s *FilterTestSuite) TestIncorrectPushIdentifier() { s.testTopic = defaultTestPubSubTopic s.testContentTopic = defaultTestContentTopic - s.lightNode = s.MakeWakuFilterLightNode() + s.MakeWakuFilterLightNode() - s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, false) + s.MakeWakuFilterFullNode(s.testTopic, false) // Re-start light node with unsupported prefix for match func s.lightNode.Stop() diff --git a/waku/v2/protocol/filter/filter_subscribe_test.go b/waku/v2/protocol/filter/filter_subscribe_test.go index 463488137..348814934 100644 --- a/waku/v2/protocol/filter/filter_subscribe_test.go +++ b/waku/v2/protocol/filter/filter_subscribe_test.go @@ -103,9 +103,10 @@ func (s *FilterTestSuite) TestMultiPubSubMultiContentTopic() { // Create test context s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 20*time.Second) // Test can't exceed 20 seconds - s.lightNode = s.StartNode(s.MakeWakuFilterLightNode()) + s.MakeWakuFilterLightNode() + s.StartLightNode() - s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, true) + s.MakeWakuFilterFullNode(s.testTopic, true) // Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) @@ -266,7 +267,7 @@ func (s *FilterTestSuite) TestMultipleFullNodeSubscriptions() { s.Log.Info("Already subscribed to", zap.String("fullNode", string(fullNodeIDHex))) // This will overwrite values with the second node info - s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, false) + s.MakeWakuFilterFullNode(s.testTopic, false) // Connect to second full and relay node s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) @@ -292,7 +293,9 @@ func (s *FilterTestSuite) TestSubscribeMultipleLightNodes() { // Create test context s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 10*time.Second) // Test can't exceed 10 seconds - lightNode2 := s.StartNode(s.MakeWakuFilterLightNode()) + lightNode2, _ := s.GetWakuFilterLightNode() + err := lightNode2.Start(context.Background()) + s.Require().NoError(err) // Connect node2 lightNode2.h.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) @@ -301,7 +304,7 @@ func (s *FilterTestSuite) TestSubscribeMultipleLightNodes() { // Subscribe separately: light node 1 -> full node contentFilter := protocol.ContentFilter{PubsubTopic: messages[0].pubSubTopic, ContentTopics: protocol.NewContentTopicSet(messages[0].contentTopic)} - _, err := s.lightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.lightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) // Subscribe separately: light node 2 -> full node @@ -328,7 +331,8 @@ func (s *FilterTestSuite) TestSubscribeFullNode2FullNode() { // Create test context s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 10*time.Second) - _, fullNode2 := s.MakeWakuFilterFullNodeNoSharing(testTopic, false) + nodeData := s.GetWakuFilterFullNode(testTopic, false) + fullNode2 := nodeData.fullNode // Connect nodes fullNode2.h.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) diff --git a/waku/v2/protocol/filter/filter_test.go b/waku/v2/protocol/filter/filter_test.go index c22816864..041d1674a 100644 --- a/waku/v2/protocol/filter/filter_test.go +++ b/waku/v2/protocol/filter/filter_test.go @@ -38,11 +38,12 @@ func (s *FilterTestSuite) SetupTest() { s.testTopic = defaultTestPubSubTopic s.testContentTopic = defaultTestContentTopic - s.lightNode = s.StartNode(s.MakeWakuFilterLightNode()) + s.MakeWakuFilterLightNode() + s.StartLightNode() //TODO: Add tests to verify broadcaster. - s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, false) + s.MakeWakuFilterFullNode(s.testTopic, false) // Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) @@ -101,7 +102,7 @@ func (s *FilterTestSuite) TestStartStop() { var wg sync.WaitGroup wg.Add(2) - s.lightNode = s.MakeWakuFilterLightNode() + s.MakeWakuFilterLightNode() stopNode := func() { for i := 0; i < 100000; i++ { @@ -145,8 +146,9 @@ func (s *FilterTestSuite) TestAutoShard() { s.testContentTopic = cTopic1Str s.testTopic = pubSubTopic.String() - s.lightNode = s.StartNode(s.MakeWakuFilterLightNode()) - s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(pubSubTopic.String(), false) + s.MakeWakuFilterLightNode() + s.StartLightNode() + s.MakeWakuFilterFullNode(pubSubTopic.String(), false) s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) err = s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) diff --git a/waku/v2/protocol/filter/filter_unsubscribe_test.go b/waku/v2/protocol/filter/filter_unsubscribe_test.go index 1a3183e5c..fb148befc 100644 --- a/waku/v2/protocol/filter/filter_unsubscribe_test.go +++ b/waku/v2/protocol/filter/filter_unsubscribe_test.go @@ -76,9 +76,10 @@ func (s *FilterTestSuite) TestUnsubscribeMultiPubSubMultiContentTopic() { s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 20*time.Second) - s.lightNode = s.StartNode(s.MakeWakuFilterLightNode()) + s.MakeWakuFilterLightNode() + s.StartLightNode() - s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, true) + s.MakeWakuFilterFullNode(s.testTopic, true) // Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNode.h), peerstore.PermanentAddrTTL) @@ -114,9 +115,10 @@ func (s *FilterTestSuite) TestUnsubscribeErrorHandling() { s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 20*time.Second) - s.lightNode = s.StartNode(s.MakeWakuFilterLightNode()) + s.MakeWakuFilterLightNode() + s.StartLightNode() - s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, true) + s.MakeWakuFilterFullNode(s.testTopic, true) // Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) @@ -213,9 +215,10 @@ func (s *FilterTestSuite) TestUnsubscribeAllDiffPubSubContentTopics() { s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 20*time.Second) - s.lightNode = s.StartNode(s.MakeWakuFilterLightNode()) + s.MakeWakuFilterLightNode() + s.StartLightNode() - s.relayNode, s.fullNode = s.MakeWakuFilterFullNode(s.testTopic, true) + s.MakeWakuFilterFullNode(s.testTopic, true) // Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNode.h), peerstore.PermanentAddrTTL) diff --git a/waku/v2/protocol/filter/test_utils.go b/waku/v2/protocol/filter/test_utils.go index b416ca601..205bd693e 100644 --- a/waku/v2/protocol/filter/test_utils.go +++ b/waku/v2/protocol/filter/test_utils.go @@ -21,24 +21,33 @@ import ( "go.uber.org/zap" ) +type LightNodeData struct { + lightNode *WakuFilterLightNode + lightNodeHost host.Host +} + +type FullNodeData struct { + relayNode *relay.WakuRelay + RelaySub *relay.Subscription + fullNodeHost host.Host + Broadcaster relay.Broadcaster + fullNode *WakuFilterFullNode +} + type FilterTestSuite struct { suite.Suite + LightNodeData + FullNodeData testTopic string testContentTopic string ctx context.Context ctxCancel context.CancelFunc - lightNode *WakuFilterLightNode - lightNodeHost host.Host - relayNode *relay.WakuRelay - fullNode *WakuFilterFullNode - fullNodeHost host.Host wg *sync.WaitGroup contentFilter protocol.ContentFilter subDetails []*subscription.SubscriptionDetails - Log *zap.Logger - RelaySub *relay.Subscription + Log *zap.Logger } type WakuMsg struct { @@ -47,14 +56,7 @@ type WakuMsg struct { payload string } -type RelayData struct { - WakuRelay *relay.WakuRelay - Subscription *relay.Subscription - Host host.Host - Broadcaster relay.Broadcaster -} - -func (s *FilterTestSuite) MakeWakuRelay(topic string) RelayData { +func (s *FilterTestSuite) GetWakuRelay(topic string) FullNodeData { broadcaster := relay.NewBroadcaster(10) s.Require().NoError(broadcaster.Start(context.Background())) @@ -74,15 +76,15 @@ func (s *FilterTestSuite) MakeWakuRelay(topic string) RelayData { sub, err := relay.Subscribe(context.Background(), protocol.NewContentFilter(topic)) s.Require().NoError(err) - return RelayData{relay, sub[0], host, broadcaster} + return FullNodeData{relay, sub[0], host, broadcaster, nil} } -func (s *FilterTestSuite) MakeWakuFilterFullNodeNoSharing(topic string, withRegisterAll bool) (*relay.WakuRelay, *WakuFilterFullNode) { +func (s *FilterTestSuite) GetWakuFilterFullNode(topic string, withRegisterAll bool) FullNodeData { - relayData := s.MakeWakuRelay(topic) + relayData := s.GetWakuRelay(topic) node2Filter := NewWakuFilterFullNode(timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.Log) - node2Filter.SetHost(relayData.Host) + node2Filter.SetHost(relayData.fullNodeHost) var sub *relay.Subscription if withRegisterAll { @@ -94,32 +96,21 @@ func (s *FilterTestSuite) MakeWakuFilterFullNodeNoSharing(topic string, withRegi err := node2Filter.Start(s.ctx, sub) s.Require().NoError(err) - return relayData.WakuRelay, node2Filter -} - -func (s *FilterTestSuite) MakeWakuFilterFullNode(topic string, withRegisterAll bool) (*relay.WakuRelay, *WakuFilterFullNode) { - relayData := s.MakeWakuRelay(topic) - - s.fullNodeHost = relayData.Host - s.RelaySub = relayData.Subscription - - node2Filter := NewWakuFilterFullNode(timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.Log) - node2Filter.SetHost(relayData.Host) + relayData.fullNode = node2Filter - var sub *relay.Subscription - if withRegisterAll { - sub = relayData.Broadcaster.RegisterForAll() - } else { - sub = relayData.Broadcaster.Register(protocol.NewContentFilter(topic)) - } + return relayData +} - err := node2Filter.Start(s.ctx, sub) - s.Require().NoError(err) +func (s *FilterTestSuite) MakeWakuFilterFullNode(topic string, withRegisterAll bool) { + relayData := s.GetWakuFilterFullNode(topic, withRegisterAll) - return relayData.WakuRelay, node2Filter + s.fullNodeHost = relayData.fullNodeHost + s.RelaySub = relayData.RelaySub + s.relayNode = relayData.relayNode + s.fullNode = relayData.fullNode } -func (s *FilterTestSuite) MakeWakuFilterLightNode() *WakuFilterLightNode { +func (s *FilterTestSuite) GetWakuFilterLightNode() (*WakuFilterLightNode, host.Host) { port, err := tests.FindFreePort(s.T(), "", 5) s.Require().NoError(err) @@ -129,15 +120,17 @@ func (s *FilterTestSuite) MakeWakuFilterLightNode() *WakuFilterLightNode { s.Require().NoError(b.Start(context.Background())) filterPush := NewWakuFilterLightNode(b, nil, timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.Log) filterPush.SetHost(host) - s.lightNodeHost = host - return filterPush + return filterPush, host +} + +func (s *FilterTestSuite) MakeWakuFilterLightNode() { + s.lightNode, s.lightNodeHost = s.GetWakuFilterLightNode() } -func (s *FilterTestSuite) StartNode(node *WakuFilterLightNode) *WakuFilterLightNode { - err := node.Start(context.Background()) +func (s *FilterTestSuite) StartLightNode() { + err := s.lightNode.Start(context.Background()) s.Require().NoError(err) - return node } func (s *FilterTestSuite) waitForMsg(msg *WakuMsg) { From b6d8e95277737c0a9ae5233d1c65fbb372b5bd6e Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Fri, 29 Mar 2024 15:51:23 +0200 Subject: [PATCH 06/27] Embedded struct assignments --- waku/v2/protocol/filter/filter_subscribe_test.go | 3 ++- waku/v2/protocol/filter/test_utils.go | 11 ++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/waku/v2/protocol/filter/filter_subscribe_test.go b/waku/v2/protocol/filter/filter_subscribe_test.go index 348814934..3ad0ed73f 100644 --- a/waku/v2/protocol/filter/filter_subscribe_test.go +++ b/waku/v2/protocol/filter/filter_subscribe_test.go @@ -293,7 +293,8 @@ func (s *FilterTestSuite) TestSubscribeMultipleLightNodes() { // Create test context s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 10*time.Second) // Test can't exceed 10 seconds - lightNode2, _ := s.GetWakuFilterLightNode() + lightNodeData := s.GetWakuFilterLightNode() + lightNode2 := lightNodeData.lightNode err := lightNode2.Start(context.Background()) s.Require().NoError(err) diff --git a/waku/v2/protocol/filter/test_utils.go b/waku/v2/protocol/filter/test_utils.go index 205bd693e..c52f5dbfe 100644 --- a/waku/v2/protocol/filter/test_utils.go +++ b/waku/v2/protocol/filter/test_utils.go @@ -104,13 +104,10 @@ func (s *FilterTestSuite) GetWakuFilterFullNode(topic string, withRegisterAll bo func (s *FilterTestSuite) MakeWakuFilterFullNode(topic string, withRegisterAll bool) { relayData := s.GetWakuFilterFullNode(topic, withRegisterAll) - s.fullNodeHost = relayData.fullNodeHost - s.RelaySub = relayData.RelaySub - s.relayNode = relayData.relayNode - s.fullNode = relayData.fullNode + s.FullNodeData = relayData } -func (s *FilterTestSuite) GetWakuFilterLightNode() (*WakuFilterLightNode, host.Host) { +func (s *FilterTestSuite) GetWakuFilterLightNode() LightNodeData { port, err := tests.FindFreePort(s.T(), "", 5) s.Require().NoError(err) @@ -121,11 +118,11 @@ func (s *FilterTestSuite) GetWakuFilterLightNode() (*WakuFilterLightNode, host.H filterPush := NewWakuFilterLightNode(b, nil, timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.Log) filterPush.SetHost(host) - return filterPush, host + return LightNodeData{filterPush, host} } func (s *FilterTestSuite) MakeWakuFilterLightNode() { - s.lightNode, s.lightNodeHost = s.GetWakuFilterLightNode() + s.LightNodeData = s.GetWakuFilterLightNode() } func (s *FilterTestSuite) StartLightNode() { From e88ea3bf26eea65aa58e95fdfe573ef113a66982 Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Fri, 29 Mar 2024 16:16:13 +0200 Subject: [PATCH 07/27] subscribe filter tests --- waku/v2/protocol/filter/filter_ping_test.go | 4 +- .../filter/filter_proto_ident_test.go | 8 ++-- waku/v2/protocol/filter/filter_push_test.go | 16 +++---- .../protocol/filter/filter_subscribe_test.go | 27 ++++++------ waku/v2/protocol/filter/filter_test.go | 8 ++-- .../filter/filter_unsubscribe_test.go | 16 +++---- waku/v2/protocol/filter/test_utils.go | 42 ++++++++++--------- 7 files changed, 63 insertions(+), 58 deletions(-) diff --git a/waku/v2/protocol/filter/filter_ping_test.go b/waku/v2/protocol/filter/filter_ping_test.go index 07fb8d83f..964c087bb 100644 --- a/waku/v2/protocol/filter/filter_ping_test.go +++ b/waku/v2/protocol/filter/filter_ping_test.go @@ -13,7 +13,7 @@ func (s *FilterTestSuite) TestSubscriptionPing() { s.Require().Equal(filterErr.Code, http.StatusNotFound) contentTopic := "abc" - s.subDetails = s.subscribe(s.testTopic, contentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, contentTopic, s.fullNodeHost.ID()) err = s.lightNode.Ping(context.Background(), s.fullNodeHost.ID()) s.Require().NoError(err) @@ -21,7 +21,7 @@ func (s *FilterTestSuite) TestSubscriptionPing() { func (s *FilterTestSuite) TestUnSubscriptionPing() { - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) err := s.lightNode.Ping(context.Background(), s.fullNodeHost.ID()) s.Require().NoError(err) diff --git a/waku/v2/protocol/filter/filter_proto_ident_test.go b/waku/v2/protocol/filter/filter_proto_ident_test.go index 6a3625aed..4721ef3ba 100644 --- a/waku/v2/protocol/filter/filter_proto_ident_test.go +++ b/waku/v2/protocol/filter/filter_proto_ident_test.go @@ -30,20 +30,20 @@ import ( func (s *FilterTestSuite) TestCreateSubscription() { // Initial subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}) } func (s *FilterTestSuite) TestModifySubscription() { // Initial subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}) // Subscribe to another content_topic newContentTopic := "Topic_modified" - s.subDetails = s.subscribe(s.testTopic, newContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, newContentTopic, s.fullNodeHost.ID()) s.waitForMsg(&WakuMsg{s.testTopic, newContentTopic, ""}) } @@ -51,7 +51,7 @@ func (s *FilterTestSuite) TestModifySubscription() { func (s *FilterTestSuite) TestMultipleMessages() { // Initial subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "first"}) diff --git a/waku/v2/protocol/filter/filter_push_test.go b/waku/v2/protocol/filter/filter_push_test.go index b938bc78d..859c9ffd1 100644 --- a/waku/v2/protocol/filter/filter_push_test.go +++ b/waku/v2/protocol/filter/filter_push_test.go @@ -14,7 +14,7 @@ import ( func (s *FilterTestSuite) TestValidPayloadsASCII() { // Subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Prepare data messages := prepareData(100, false, false, true, tests.GenerateRandomASCIIString) @@ -32,7 +32,7 @@ func (s *FilterTestSuite) TestValidPayloadsASCII() { func (s *FilterTestSuite) TestValidPayloadsUTF8() { // Subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Prepare data messages := prepareData(100, false, false, true, tests.GenerateRandomUTF8String) @@ -50,7 +50,7 @@ func (s *FilterTestSuite) TestValidPayloadsUTF8() { func (s *FilterTestSuite) TestValidPayloadsBase64() { // Subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Prepare data messages := prepareData(100, false, false, true, tests.GenerateRandomBase64String) @@ -68,7 +68,7 @@ func (s *FilterTestSuite) TestValidPayloadsBase64() { func (s *FilterTestSuite) TestValidPayloadsJSON() { // Subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Prepare data messages := prepareData(100, false, false, true, tests.GenerateRandomJSONString) @@ -86,7 +86,7 @@ func (s *FilterTestSuite) TestValidPayloadsJSON() { func (s *FilterTestSuite) TestValidPayloadsURLEncoded() { // Subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Prepare data messages := prepareData(100, false, false, true, tests.GenerateRandomURLEncodedString) @@ -104,7 +104,7 @@ func (s *FilterTestSuite) TestValidPayloadsURLEncoded() { func (s *FilterTestSuite) TestValidPayloadsSQL() { // Subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Prepare data messages := prepareData(100, false, false, true, tests.GenerateRandomSQLInsert) @@ -124,7 +124,7 @@ func (s *FilterTestSuite) TestLargePayloadsUTF8() { s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 40*time.Second) // Subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Prepare basic data messages := prepareData(10, false, false, false, nil) @@ -148,7 +148,7 @@ func (s *FilterTestSuite) TestLargePayloadsUTF8() { func (s *FilterTestSuite) TestFuturePayloadEncryptionVersion() { // Subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) message := tests.CreateWakuMessage(s.testContentTopic, utils.GetUnixEpoch(), "test_payload") futureVersion := uint32(100) diff --git a/waku/v2/protocol/filter/filter_subscribe_test.go b/waku/v2/protocol/filter/filter_subscribe_test.go index 3ad0ed73f..99af42f05 100644 --- a/waku/v2/protocol/filter/filter_subscribe_test.go +++ b/waku/v2/protocol/filter/filter_subscribe_test.go @@ -18,7 +18,7 @@ import ( func (s *FilterTestSuite) TestWakuFilter() { // Initial subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Should be received s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "first"}) @@ -33,8 +33,9 @@ func (s *FilterTestSuite) TestWakuFilter() { s.waitForTimeout(&WakuMsg{s.testTopic, s.testContentTopic, "third"}) // Two new subscriptions with same [peer, contentFilter] - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) - secondSub := s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + + secondSub := s.getSub(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Assert that we have 2 subscriptions now s.Require().Equal(len(s.lightNode.Subscriptions()), 2) @@ -66,7 +67,7 @@ func (s *FilterTestSuite) TestWakuFilter() { func (s *FilterTestSuite) TestPubSubSingleContentTopic() { // Initial subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Message should be received s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "test_msg"}) @@ -85,7 +86,7 @@ func (s *FilterTestSuite) TestPubSubMultiContentTopic() { // Subscribe for _, m := range messages { - s.subDetails = s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) } // All messages should be received @@ -117,7 +118,7 @@ func (s *FilterTestSuite) TestMultiPubSubMultiContentTopic() { // Subscribe for _, m := range messages { - s.subDetails = append(s.subDetails, s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID())...) + s.subDetails = append(s.subDetails, s.getSub(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID())...) s.Log.Info("Subscribing ", zap.String("PubSubTopic", m.pubSubTopic)) _, err := s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.pubSubTopic)) s.Require().NoError(err) @@ -147,7 +148,7 @@ func (s *FilterTestSuite) TestPubSubMultiOverlapContentTopic() { // Subscribe for _, m := range messages { - s.subDetails = s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) } // All messages should be received @@ -165,10 +166,10 @@ func (s *FilterTestSuite) TestSubscriptionRefresh() { messages := prepareData(2, false, false, true, nil) // Initial subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Repeat the same subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Both messages should be received s.waitForMessages(func() { @@ -198,7 +199,7 @@ func (s *FilterTestSuite) TestContentTopicsLimit() { // Subscribe for _, m := range messages[:len(messages)-1] { - s.subDetails = s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) } // All messages within limit should get received @@ -370,7 +371,7 @@ func (s *FilterTestSuite) TestIsSubscriptionAlive() { messages := prepareData(2, false, true, false, nil) // Subscribe with the first message only - s.subDetails = s.subscribe(messages[0].pubSubTopic, messages[0].contentTopic, s.fullNodeHost.ID()) + s.subscribe(messages[0].pubSubTopic, messages[0].contentTopic, s.fullNodeHost.ID()) // IsSubscriptionAlive returns no error for the first message err := s.lightNode.IsSubscriptionAlive(s.ctx, s.subDetails[0]) @@ -393,7 +394,7 @@ func (s *FilterTestSuite) TestFilterSubscription() { contentFilter := protocol.ContentFilter{PubsubTopic: s.testTopic, ContentTopics: protocol.NewContentTopicSet(s.testContentTopic)} // Subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Returns no error and SubscriptionDetails for existing subscription _, err := s.lightNode.FilterSubscription(s.fullNodeHost.ID(), contentFilter) @@ -421,7 +422,7 @@ func (s *FilterTestSuite) TestHandleFilterSubscribeOptions() { contentFilter := protocol.ContentFilter{PubsubTopic: s.testTopic, ContentTopics: protocol.NewContentTopicSet(s.testContentTopic)} // Subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // With valid peer opts := []FilterSubscribeOption{WithPeer(s.fullNodeHost.ID())} diff --git a/waku/v2/protocol/filter/filter_test.go b/waku/v2/protocol/filter/filter_test.go index 041d1674a..f8d88b77e 100644 --- a/waku/v2/protocol/filter/filter_test.go +++ b/waku/v2/protocol/filter/filter_test.go @@ -155,7 +155,7 @@ func (s *FilterTestSuite) TestAutoShard() { s.Require().NoError(err) s.Log.Info("Testing Autoshard:CreateSubscription") - s.subDetails = s.subscribe("", s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe("", s.testContentTopic, s.fullNodeHost.ID()) s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}) // Wrong content topic @@ -169,7 +169,7 @@ func (s *FilterTestSuite) TestAutoShard() { // Should not receive after unsubscribe s.waitForTimeout(&WakuMsg{s.testTopic, s.testContentTopic, "third"}) - s.subDetails = s.subscribe("", s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe("", s.testContentTopic, s.fullNodeHost.ID()) s.Log.Info("Testing Autoshard:SubscriptionPing") err = s.lightNode.Ping(context.Background(), s.fullNodeHost.ID()) @@ -179,7 +179,7 @@ func (s *FilterTestSuite) TestAutoShard() { s.Log.Info("Testing Autoshard:ModifySubscription") newContentTopic := "0/test/1/testTopic1/proto" - s.subDetails = s.subscribe("", newContentTopic, s.fullNodeHost.ID()) + s.subscribe("", newContentTopic, s.fullNodeHost.ID()) s.waitForMsg(&WakuMsg{s.testTopic, newContentTopic, ""}) @@ -199,7 +199,7 @@ func (s *FilterTestSuite) TestLightNodeIsListening() { messages := prepareData(2, true, true, false, nil) // Subscribe with the first message only - s.subDetails = s.subscribe(messages[0].pubSubTopic, messages[0].contentTopic, s.fullNodeHost.ID()) + s.subscribe(messages[0].pubSubTopic, messages[0].contentTopic, s.fullNodeHost.ID()) // IsListening returns true for the first message listenStatus := s.lightNode.IsListening(messages[0].pubSubTopic, messages[0].contentTopic) diff --git a/waku/v2/protocol/filter/filter_unsubscribe_test.go b/waku/v2/protocol/filter/filter_unsubscribe_test.go index fb148befc..34dcf1bb0 100644 --- a/waku/v2/protocol/filter/filter_unsubscribe_test.go +++ b/waku/v2/protocol/filter/filter_unsubscribe_test.go @@ -18,8 +18,8 @@ func (s *FilterTestSuite) TestUnsubscribeSingleContentTopic() { var newContentTopic = "TopicB" // Initial subscribe - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) - s.subDetails = s.subscribe(s.testTopic, newContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.testTopic, newContentTopic, s.fullNodeHost.ID()) // Message is possible to receive for original contentTopic s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "test_msg"}) @@ -46,7 +46,7 @@ func (s *FilterTestSuite) TestUnsubscribeMultiContentTopic() { // Subscribe with 3 content topics for _, m := range messages { - s.subDetails = s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) } // All messages should be received @@ -90,7 +90,7 @@ func (s *FilterTestSuite) TestUnsubscribeMultiPubSubMultiContentTopic() { // Subscribe for _, m := range messages { - s.subDetails = append(s.subDetails, s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID())...) + s.subDetails = append(s.subDetails, s.getSub(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID())...) _, err = s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.pubSubTopic)) s.Require().NoError(err) } @@ -149,7 +149,7 @@ func (s *FilterTestSuite) TestUnsubscribeErrorHandling() { // Subscribe with valid topics for _, m := range messages { - s.subDetails = s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) _, err = s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.pubSubTopic)) s.Require().NoError(err) } @@ -193,7 +193,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllWithoutContentTopics() { // Subscribe with 2 content topics for _, m := range messages { - s.subDetails = s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) } // All messages should be received @@ -229,7 +229,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllDiffPubSubContentTopics() { // Subscribe for _, m := range messages { - s.subDetails = append(s.subDetails, s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID())...) + s.subDetails = append(s.subDetails, s.getSub(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID())...) _, err = s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.pubSubTopic)) s.Require().NoError(err) } @@ -256,7 +256,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllUnrelatedPeer() { // Subscribe with 2 content topics for _, m := range messages { - s.subDetails = s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) } // All messages should be received diff --git a/waku/v2/protocol/filter/test_utils.go b/waku/v2/protocol/filter/test_utils.go index c52f5dbfe..2524e4fa8 100644 --- a/waku/v2/protocol/filter/test_utils.go +++ b/waku/v2/protocol/filter/test_utils.go @@ -81,30 +81,30 @@ func (s *FilterTestSuite) GetWakuRelay(topic string) FullNodeData { func (s *FilterTestSuite) GetWakuFilterFullNode(topic string, withRegisterAll bool) FullNodeData { - relayData := s.GetWakuRelay(topic) + nodeData := s.GetWakuRelay(topic) node2Filter := NewWakuFilterFullNode(timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.Log) - node2Filter.SetHost(relayData.fullNodeHost) + node2Filter.SetHost(nodeData.fullNodeHost) var sub *relay.Subscription if withRegisterAll { - sub = relayData.Broadcaster.RegisterForAll() + sub = nodeData.Broadcaster.RegisterForAll() } else { - sub = relayData.Broadcaster.Register(protocol.NewContentFilter(topic)) + sub = nodeData.Broadcaster.Register(protocol.NewContentFilter(topic)) } err := node2Filter.Start(s.ctx, sub) s.Require().NoError(err) - relayData.fullNode = node2Filter + nodeData.fullNode = node2Filter - return relayData + return nodeData } func (s *FilterTestSuite) MakeWakuFilterFullNode(topic string, withRegisterAll bool) { - relayData := s.GetWakuFilterFullNode(topic, withRegisterAll) + nodeData := s.GetWakuFilterFullNode(topic, withRegisterAll) - s.FullNodeData = relayData + s.FullNodeData = nodeData } func (s *FilterTestSuite) GetWakuFilterLightNode() LightNodeData { @@ -241,27 +241,31 @@ func (s *FilterTestSuite) waitForTimeoutFromChan(msg *WakuMsg, ch chan *protocol s.wg.Wait() } -func (s *FilterTestSuite) subscribe(pubsubTopic string, contentTopic string, peer peer.ID) []*subscription.SubscriptionDetails { +func (s *FilterTestSuite) getSub(pubsubTopic string, contentTopic string, peer peer.ID) []*subscription.SubscriptionDetails { + contentFilter := protocol.ContentFilter{PubsubTopic: pubsubTopic, ContentTopics: protocol.NewContentTopicSet(contentTopic)} + + subDetails, err := s.lightNode.Subscribe(s.ctx, contentFilter, WithPeer(peer)) + s.Require().NoError(err) + + time.Sleep(1 * time.Second) + + return subDetails +} +func (s *FilterTestSuite) subscribe(pubsubTopic string, contentTopic string, peer peer.ID) { for _, sub := range s.subDetails { if sub.ContentFilter.PubsubTopic == pubsubTopic { sub.Add(contentTopic) s.contentFilter = sub.ContentFilter subDetails, err := s.lightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(peer)) + s.subDetails = subDetails s.Require().NoError(err) - return subDetails + return } } - s.contentFilter = protocol.ContentFilter{PubsubTopic: pubsubTopic, ContentTopics: protocol.NewContentTopicSet(contentTopic)} - - subDetails, err := s.lightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(peer)) - s.Require().NoError(err) - - // Sleep to make sure the filter is subscribed - time.Sleep(1 * time.Second) - - return subDetails + s.subDetails = s.getSub(pubsubTopic, contentTopic, peer) + s.contentFilter = s.subDetails[0].ContentFilter } func (s *FilterTestSuite) unsubscribe(pubsubTopic string, contentTopic string, peer peer.ID) []*subscription.SubscriptionDetails { From 6b7f1a02ed4a71b109761f7a5bbde8c3675fdb68 Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Mon, 8 Apr 2024 17:28:04 +0300 Subject: [PATCH 08/27] Ref waitForMessages() --- waku/v2/protocol/filter/filter_push_test.go | 28 ++++------------ .../protocol/filter/filter_subscribe_test.go | 20 +++--------- .../filter/filter_unsubscribe_test.go | 32 +++++-------------- waku/v2/protocol/filter/test_utils.go | 11 ++++--- 4 files changed, 26 insertions(+), 65 deletions(-) diff --git a/waku/v2/protocol/filter/filter_push_test.go b/waku/v2/protocol/filter/filter_push_test.go index 859c9ffd1..f76df39e4 100644 --- a/waku/v2/protocol/filter/filter_push_test.go +++ b/waku/v2/protocol/filter/filter_push_test.go @@ -20,9 +20,7 @@ func (s *FilterTestSuite) TestValidPayloadsASCII() { messages := prepareData(100, false, false, true, tests.GenerateRandomASCIIString) // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -38,9 +36,7 @@ func (s *FilterTestSuite) TestValidPayloadsUTF8() { messages := prepareData(100, false, false, true, tests.GenerateRandomUTF8String) // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -56,9 +52,7 @@ func (s *FilterTestSuite) TestValidPayloadsBase64() { messages := prepareData(100, false, false, true, tests.GenerateRandomBase64String) // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -74,9 +68,7 @@ func (s *FilterTestSuite) TestValidPayloadsJSON() { messages := prepareData(100, false, false, true, tests.GenerateRandomJSONString) // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -92,9 +84,7 @@ func (s *FilterTestSuite) TestValidPayloadsURLEncoded() { messages := prepareData(100, false, false, true, tests.GenerateRandomURLEncodedString) // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -110,9 +100,7 @@ func (s *FilterTestSuite) TestValidPayloadsSQL() { messages := prepareData(100, false, false, true, tests.GenerateRandomSQLInsert) // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -136,9 +124,7 @@ func (s *FilterTestSuite) TestLargePayloadsUTF8() { } // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) diff --git a/waku/v2/protocol/filter/filter_subscribe_test.go b/waku/v2/protocol/filter/filter_subscribe_test.go index 99af42f05..a41bc139e 100644 --- a/waku/v2/protocol/filter/filter_subscribe_test.go +++ b/waku/v2/protocol/filter/filter_subscribe_test.go @@ -90,9 +90,7 @@ func (s *FilterTestSuite) TestPubSubMultiContentTopic() { } // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -130,9 +128,7 @@ func (s *FilterTestSuite) TestMultiPubSubMultiContentTopic() { } // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) _, err = s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -152,9 +148,7 @@ func (s *FilterTestSuite) TestPubSubMultiOverlapContentTopic() { } // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -172,9 +166,7 @@ func (s *FilterTestSuite) TestSubscriptionRefresh() { s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) // Both messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) _, err := s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -203,9 +195,7 @@ func (s *FilterTestSuite) TestContentTopicsLimit() { } // All messages within limit should get received - s.waitForMessages(func() { - s.publishMessages(messages[:len(messages)-1]) - }, s.subDetails, messages[:len(messages)-1]) + s.waitForMessages(messages[:len(messages)-1]) // Adding over the limit contentTopic should fail for _, sub := range s.subDetails { diff --git a/waku/v2/protocol/filter/filter_unsubscribe_test.go b/waku/v2/protocol/filter/filter_unsubscribe_test.go index 34dcf1bb0..326b8f643 100644 --- a/waku/v2/protocol/filter/filter_unsubscribe_test.go +++ b/waku/v2/protocol/filter/filter_unsubscribe_test.go @@ -50,9 +50,7 @@ func (s *FilterTestSuite) TestUnsubscribeMultiContentTopic() { } // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) // Unsubscribe with the last 2 content topics for _, m := range messages[1:] { @@ -96,9 +94,7 @@ func (s *FilterTestSuite) TestUnsubscribeMultiPubSubMultiContentTopic() { } // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) // Unsubscribe for _, m := range messages { @@ -155,9 +151,7 @@ func (s *FilterTestSuite) TestUnsubscribeErrorHandling() { } // All messages should be possible to receive for subscribed topics - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) // Unsubscribe with empty pubsub contentFilter := protocol.ContentFilter{PubsubTopic: invalidMessages[0].pubSubTopic, @@ -178,9 +172,7 @@ func (s *FilterTestSuite) TestUnsubscribeErrorHandling() { s.Require().NoError(err) // All messages should be still possible to receive for subscribed topics - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) _, err = s.lightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) @@ -197,9 +189,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllWithoutContentTopics() { } // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) // Unsubscribe all with peer specification _, err := s.lightNode.UnsubscribeAll(s.ctx, WithPeer(s.fullNodeHost.ID())) @@ -235,9 +225,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllDiffPubSubContentTopics() { } // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) // Unsubscribe all without any specification _, err = s.lightNode.UnsubscribeAll(s.ctx) @@ -260,9 +248,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllUnrelatedPeer() { } // All messages should be received - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) // Create new host - not related to any node host, err := tests.MakeHost(context.Background(), 12345, rand.Reader) @@ -280,9 +266,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllUnrelatedPeer() { } // All messages should be received because peer ID used was not related to any subscription - s.waitForMessages(func() { - s.publishMessages(messages) - }, s.subDetails, messages) + s.waitForMessages(messages) // Expect error for unsubscribe from non existing peer s.Require().Error(err) diff --git a/waku/v2/protocol/filter/test_utils.go b/waku/v2/protocol/filter/test_utils.go index 2524e4fa8..f94b86201 100644 --- a/waku/v2/protocol/filter/test_utils.go +++ b/waku/v2/protocol/filter/test_utils.go @@ -173,10 +173,11 @@ func matchOneOfManyMsg(one WakuMsg, many []WakuMsg) bool { return false } -func (s *FilterTestSuite) waitForMessages(fn func(), subs []*subscription.SubscriptionDetails, expected []WakuMsg) { +func (s *FilterTestSuite) waitForMessages(msgs []WakuMsg) { s.wg.Add(1) - msgCount := len(expected) + msgCount := len(msgs) found := 0 + subs := s.subDetails s.Log.Info("Expected messages ", zap.String("count", strconv.Itoa(msgCount))) s.Log.Info("Existing subscriptions ", zap.String("count", strconv.Itoa(len(subs)))) @@ -196,7 +197,7 @@ func (s *FilterTestSuite) waitForMessages(fn func(), subs []*subscription.Subscr payload: string(env.Message().GetPayload()), } s.Log.Debug("received message ", zap.String("pubSubTopic", received.pubSubTopic), zap.String("contentTopic", received.contentTopic), zap.String("payload", received.payload)) - if matchOneOfManyMsg(received, expected) { + if matchOneOfManyMsg(received, msgs) { found++ } case <-time.After(3 * time.Second): @@ -208,8 +209,8 @@ func (s *FilterTestSuite) waitForMessages(fn func(), subs []*subscription.Subscr } }() - if fn != nil { - fn() + if msgs != nil { + s.publishMessages(msgs) } s.wg.Wait() From b932c9688aadfa2b666055ef4bb477eb8aa74b80 Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Mon, 8 Apr 2024 18:22:05 +0300 Subject: [PATCH 09/27] Renames --- waku/v2/api/filter_test.go | 45 ++---- waku/v2/protocol/filter/filter_ping_test.go | 14 +- .../filter/filter_proto_ident_test.go | 48 +++--- waku/v2/protocol/filter/filter_push_test.go | 54 +++---- .../protocol/filter/filter_subscribe_test.go | 148 +++++++++--------- waku/v2/protocol/filter/filter_test.go | 101 ++++-------- .../filter/filter_unsubscribe_test.go | 112 ++++++------- waku/v2/protocol/filter/test_utils.go | 105 +++++++++---- 8 files changed, 309 insertions(+), 318 deletions(-) diff --git a/waku/v2/api/filter_test.go b/waku/v2/api/filter_test.go index 0f9560d40..59fc20cb2 100644 --- a/waku/v2/api/filter_test.go +++ b/waku/v2/api/filter_test.go @@ -2,46 +2,35 @@ package api import ( "context" - "sync" "testing" - "github.com/libp2p/go-libp2p/core/host" "github.com/stretchr/testify/suite" "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/protocol/filter" - "github.com/waku-org/go-waku/waku/v2/protocol/relay" - "github.com/waku-org/go-waku/waku/v2/protocol/subscription" - "go.uber.org/zap" ) func TestFilterApiSuite(t *testing.T) { suite.Run(t, new(FilterApiTestSuite)) } -const defaultTestPubSubTopic = "/waku/2/go/filter/test" -const defaultTestContentTopic = "/test/10/my-app" - type FilterApiTestSuite struct { - suite.Suite - - testTopic string - testContentTopic string - ctx context.Context - ctxCancel context.CancelFunc - lightNode *filter.WakuFilterLightNode - lightNodeHost host.Host - relayNode *relay.WakuRelay - relaySub *relay.Subscription - fullNode *filter.WakuFilterFullNode - fullNodeHost host.Host - wg *sync.WaitGroup - contentFilter protocol.ContentFilter - subDetails []*subscription.SubscriptionDetails - log *zap.Logger + filter.FilterTestSuite +} + +func (s *FilterApiTestSuite) SetupTest() { + s.FilterTestSuite.SetupTest() + s.Log.Info("SetupTest()") } -type WakuMsg struct { - pubSubTopic string - contentTopic string - payload string +func (s *FilterApiTestSuite) TearDownTest() { + s.FilterTestSuite.TearDownTest() +} + +func (s *FilterApiTestSuite) TestSubscribe() { + contentFilter := protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} + apiConfig := FilterConfig{MaxPeers: 2} + apiSub, err := Subscribe(context.Background(), s.LightNode, contentFilter, apiConfig) + s.Require().NoError(err) + + s.Require().NotNil(apiSub.wf) } diff --git a/waku/v2/protocol/filter/filter_ping_test.go b/waku/v2/protocol/filter/filter_ping_test.go index 964c087bb..54607d0ad 100644 --- a/waku/v2/protocol/filter/filter_ping_test.go +++ b/waku/v2/protocol/filter/filter_ping_test.go @@ -6,29 +6,29 @@ import ( ) func (s *FilterTestSuite) TestSubscriptionPing() { - err := s.lightNode.Ping(context.Background(), s.fullNodeHost.ID()) + err := s.LightNode.Ping(context.Background(), s.fullNodeHost.ID()) s.Require().Error(err) filterErr, ok := err.(*FilterError) s.Require().True(ok) s.Require().Equal(filterErr.Code, http.StatusNotFound) contentTopic := "abc" - s.subscribe(s.testTopic, contentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, contentTopic, s.fullNodeHost.ID()) - err = s.lightNode.Ping(context.Background(), s.fullNodeHost.ID()) + err = s.LightNode.Ping(context.Background(), s.fullNodeHost.ID()) s.Require().NoError(err) } func (s *FilterTestSuite) TestUnSubscriptionPing() { - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) - err := s.lightNode.Ping(context.Background(), s.fullNodeHost.ID()) + err := s.LightNode.Ping(context.Background(), s.fullNodeHost.ID()) s.Require().NoError(err) - _, err = s.lightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.LightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) - err = s.lightNode.Ping(context.Background(), s.fullNodeHost.ID()) + err = s.LightNode.Ping(context.Background(), s.fullNodeHost.ID()) s.Require().Error(err) } diff --git a/waku/v2/protocol/filter/filter_proto_ident_test.go b/waku/v2/protocol/filter/filter_proto_ident_test.go index 4721ef3ba..a48a86467 100644 --- a/waku/v2/protocol/filter/filter_proto_ident_test.go +++ b/waku/v2/protocol/filter/filter_proto_ident_test.go @@ -30,32 +30,32 @@ import ( func (s *FilterTestSuite) TestCreateSubscription() { // Initial subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, ""}) } func (s *FilterTestSuite) TestModifySubscription() { // Initial subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}) + s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, ""}) // Subscribe to another content_topic newContentTopic := "Topic_modified" - s.subscribe(s.testTopic, newContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, newContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(&WakuMsg{s.testTopic, newContentTopic, ""}) + s.waitForMsg(&WakuMsg{s.TestTopic, newContentTopic, ""}) } func (s *FilterTestSuite) TestMultipleMessages() { // Initial subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "first"}) + s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, "first"}) - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "second"}) + s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, "second"}) } func (wf *WakuFilterLightNode) incorrectSubscribeRequest(ctx context.Context, params *FilterSubscribeParameters, @@ -208,23 +208,23 @@ func (s *FilterTestSuite) TestIncorrectSubscribeIdentifier() { // Create test context s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 10*time.Second) // Test can't exceed 10 seconds - s.testTopic = defaultTestPubSubTopic - s.testContentTopic = defaultTestContentTopic + s.TestTopic = DefaultTestPubSubTopic + s.TestContentTopic = DefaultTestContentTopic s.MakeWakuFilterLightNode() s.StartLightNode() - s.MakeWakuFilterFullNode(s.testTopic, false) + s.MakeWakuFilterFullNode(s.TestTopic, false) //Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) // Subscribe with incorrect SubscribeID - s.contentFilter = protocol.ContentFilter{PubsubTopic: s.testTopic, ContentTopics: protocol.NewContentTopicSet(s.testContentTopic)} - _, err := s.lightNode.IncorrectSubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + s.contentFilter = protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} + _, err := s.LightNode.IncorrectSubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().Error(err) - _, err = s.lightNode.UnsubscribeAll(s.ctx) + _, err = s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -248,16 +248,16 @@ func (s *FilterTestSuite) TestIncorrectPushIdentifier() { s.ctx = ctx s.ctxCancel = cancel - s.testTopic = defaultTestPubSubTopic - s.testContentTopic = defaultTestContentTopic + s.TestTopic = DefaultTestPubSubTopic + s.TestContentTopic = DefaultTestContentTopic s.MakeWakuFilterLightNode() - s.MakeWakuFilterFullNode(s.testTopic, false) + s.MakeWakuFilterFullNode(s.TestTopic, false) // Re-start light node with unsupported prefix for match func - s.lightNode.Stop() - err := s.lightNode.CommonService.Start(s.ctx, s.lightNode.startWithIncorrectPushProto) + s.LightNode.Stop() + err := s.LightNode.CommonService.Start(s.ctx, s.LightNode.startWithIncorrectPushProto) s.Require().NoError(err) // Connect nodes @@ -266,14 +266,14 @@ func (s *FilterTestSuite) TestIncorrectPushIdentifier() { s.Require().NoError(err) // Subscribe - s.contentFilter = protocol.ContentFilter{PubsubTopic: s.testTopic, ContentTopics: protocol.NewContentTopicSet(s.testContentTopic)} - s.subDetails, err = s.lightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + s.contentFilter = protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} + s.subDetails, err = s.LightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) time.Sleep(1 * time.Second) // Send message - _, err = s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(s.testContentTopic, utils.GetUnixEpoch(), "second"), relay.WithPubSubTopic(s.testTopic)) + _, err = s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(s.TestContentTopic, utils.GetUnixEpoch(), "second"), relay.WithPubSubTopic(s.TestTopic)) s.Require().NoError(err) // Message should never arrive -> exit after timeout @@ -285,6 +285,6 @@ func (s *FilterTestSuite) TestIncorrectPushIdentifier() { s.Require().True(true) } - _, err = s.lightNode.UnsubscribeAll(s.ctx) + _, err = s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } diff --git a/waku/v2/protocol/filter/filter_push_test.go b/waku/v2/protocol/filter/filter_push_test.go index f76df39e4..d8310a22d 100644 --- a/waku/v2/protocol/filter/filter_push_test.go +++ b/waku/v2/protocol/filter/filter_push_test.go @@ -14,15 +14,15 @@ import ( func (s *FilterTestSuite) TestValidPayloadsASCII() { // Subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) // Prepare data - messages := prepareData(100, false, false, true, tests.GenerateRandomASCIIString) + messages := s.prepareData(100, false, false, true, tests.GenerateRandomASCIIString) // All messages should be received s.waitForMessages(messages) - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -30,15 +30,15 @@ func (s *FilterTestSuite) TestValidPayloadsASCII() { func (s *FilterTestSuite) TestValidPayloadsUTF8() { // Subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) // Prepare data - messages := prepareData(100, false, false, true, tests.GenerateRandomUTF8String) + messages := s.prepareData(100, false, false, true, tests.GenerateRandomUTF8String) // All messages should be received s.waitForMessages(messages) - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -46,15 +46,15 @@ func (s *FilterTestSuite) TestValidPayloadsUTF8() { func (s *FilterTestSuite) TestValidPayloadsBase64() { // Subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) // Prepare data - messages := prepareData(100, false, false, true, tests.GenerateRandomBase64String) + messages := s.prepareData(100, false, false, true, tests.GenerateRandomBase64String) // All messages should be received s.waitForMessages(messages) - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -62,15 +62,15 @@ func (s *FilterTestSuite) TestValidPayloadsBase64() { func (s *FilterTestSuite) TestValidPayloadsJSON() { // Subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) // Prepare data - messages := prepareData(100, false, false, true, tests.GenerateRandomJSONString) + messages := s.prepareData(100, false, false, true, tests.GenerateRandomJSONString) // All messages should be received s.waitForMessages(messages) - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -78,15 +78,15 @@ func (s *FilterTestSuite) TestValidPayloadsJSON() { func (s *FilterTestSuite) TestValidPayloadsURLEncoded() { // Subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) // Prepare data - messages := prepareData(100, false, false, true, tests.GenerateRandomURLEncodedString) + messages := s.prepareData(100, false, false, true, tests.GenerateRandomURLEncodedString) // All messages should be received s.waitForMessages(messages) - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -94,15 +94,15 @@ func (s *FilterTestSuite) TestValidPayloadsURLEncoded() { func (s *FilterTestSuite) TestValidPayloadsSQL() { // Subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) // Prepare data - messages := prepareData(100, false, false, true, tests.GenerateRandomSQLInsert) + messages := s.prepareData(100, false, false, true, tests.GenerateRandomSQLInsert) // All messages should be received s.waitForMessages(messages) - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -112,21 +112,21 @@ func (s *FilterTestSuite) TestLargePayloadsUTF8() { s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 40*time.Second) // Subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) // Prepare basic data - messages := prepareData(10, false, false, false, nil) + messages := s.prepareData(10, false, false, false, nil) // Generate large string for i := range messages { - messages[i].payload, _ = tests.GenerateRandomUTF8String(153600) - s.Log.Info("Generated payload with ", zap.String("length", strconv.Itoa(len(messages[i].payload)))) + messages[i].Payload, _ = tests.GenerateRandomUTF8String(153600) + s.Log.Info("Generated payload with ", zap.String("length", strconv.Itoa(len(messages[i].Payload)))) } // All messages should be received s.waitForMessages(messages) - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -134,19 +134,19 @@ func (s *FilterTestSuite) TestLargePayloadsUTF8() { func (s *FilterTestSuite) TestFuturePayloadEncryptionVersion() { // Subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) - message := tests.CreateWakuMessage(s.testContentTopic, utils.GetUnixEpoch(), "test_payload") + message := tests.CreateWakuMessage(s.TestContentTopic, utils.GetUnixEpoch(), "test_payload") futureVersion := uint32(100) message.Version = &futureVersion // Should get accepted - _, err := s.relayNode.Publish(s.ctx, message, relay.WithPubSubTopic(s.testTopic)) + _, err := s.relayNode.Publish(s.ctx, message, relay.WithPubSubTopic(s.TestTopic)) s.Require().NoError(err) // Should be received s.waitForMsg(nil) - _, err = s.lightNode.UnsubscribeAll(s.ctx) + _, err = s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } diff --git a/waku/v2/protocol/filter/filter_subscribe_test.go b/waku/v2/protocol/filter/filter_subscribe_test.go index a41bc139e..ce1415a1e 100644 --- a/waku/v2/protocol/filter/filter_subscribe_test.go +++ b/waku/v2/protocol/filter/filter_subscribe_test.go @@ -18,61 +18,61 @@ import ( func (s *FilterTestSuite) TestWakuFilter() { // Initial subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) // Should be received - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "first"}) + s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, "first"}) // Wrong content topic - s.waitForTimeout(&WakuMsg{s.testTopic, "TopicB", "second"}) + s.waitForTimeout(&WakuMsg{s.TestTopic, "TopicB", "second"}) - _, err := s.lightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err := s.LightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) // Should not receive after unsubscribe - s.waitForTimeout(&WakuMsg{s.testTopic, s.testContentTopic, "third"}) + s.waitForTimeout(&WakuMsg{s.TestTopic, s.TestContentTopic, "third"}) // Two new subscriptions with same [peer, contentFilter] - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) - secondSub := s.getSub(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + secondSub := s.getSub(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) // Assert that we have 2 subscriptions now - s.Require().Equal(len(s.lightNode.Subscriptions()), 2) + s.Require().Equal(len(s.LightNode.Subscriptions()), 2) // Should be received on both subscriptions - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "fourth"}) + s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, "fourth"}) - s.waitForMsgFromChan(&WakuMsg{s.testTopic, s.testContentTopic, "fifth"}, secondSub[0].C) + s.waitForMsgFromChan(&WakuMsg{s.TestTopic, s.TestContentTopic, "fifth"}, secondSub[0].C) s.waitForMsg(nil) s.waitForMsgFromChan(nil, secondSub[0].C) // Unsubscribe from second sub only - _, err = s.lightNode.UnsubscribeWithSubscription(s.ctx, secondSub[0]) + _, err = s.LightNode.UnsubscribeWithSubscription(s.ctx, secondSub[0]) s.Require().NoError(err) // Should still receive - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "sixth"}) + s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, "sixth"}) // Unsubscribe from first sub only - _, err = s.lightNode.UnsubscribeWithSubscription(s.ctx, s.subDetails[0]) + _, err = s.LightNode.UnsubscribeWithSubscription(s.ctx, s.subDetails[0]) s.Require().NoError(err) - s.Require().Equal(len(s.lightNode.Subscriptions()), 0) + s.Require().Equal(len(s.LightNode.Subscriptions()), 0) // Should not receive after unsubscribe - s.waitForTimeout(&WakuMsg{s.testTopic, s.testContentTopic, "seventh"}) + s.waitForTimeout(&WakuMsg{s.TestTopic, s.TestContentTopic, "seventh"}) } func (s *FilterTestSuite) TestPubSubSingleContentTopic() { // Initial subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) // Message should be received - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "test_msg"}) + s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, "test_msg"}) - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -82,17 +82,17 @@ func (s *FilterTestSuite) TestPubSubMultiContentTopic() { // Create test context s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 20*time.Second) // Test can't exceed 10 seconds - messages := prepareData(3, false, true, false, nil) + messages := s.prepareData(3, false, true, false, nil) // Subscribe for _, m := range messages { - s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + s.subscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) } // All messages should be received s.waitForMessages(messages) - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -105,20 +105,20 @@ func (s *FilterTestSuite) TestMultiPubSubMultiContentTopic() { s.MakeWakuFilterLightNode() s.StartLightNode() - s.MakeWakuFilterFullNode(s.testTopic, true) + s.MakeWakuFilterFullNode(s.TestTopic, true) // Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) - messages := prepareData(2, true, true, false, nil) + messages := s.prepareData(2, true, true, false, nil) // Subscribe for _, m := range messages { - s.subDetails = append(s.subDetails, s.getSub(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID())...) - s.Log.Info("Subscribing ", zap.String("PubSubTopic", m.pubSubTopic)) - _, err := s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.pubSubTopic)) + s.subDetails = append(s.subDetails, s.getSub(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID())...) + s.Log.Info("Subscribing ", zap.String("PubSubTopic", m.PubSubTopic)) + _, err := s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.PubSubTopic)) s.Require().NoError(err) } @@ -130,7 +130,7 @@ func (s *FilterTestSuite) TestMultiPubSubMultiContentTopic() { // All messages should be received s.waitForMessages(messages) - _, err = s.lightNode.UnsubscribeAll(s.ctx) + _, err = s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -140,35 +140,35 @@ func (s *FilterTestSuite) TestPubSubMultiOverlapContentTopic() { // Create test context s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 20*time.Second) // Test can't exceed 20 seconds - messages := prepareData(10, false, true, true, nil) + messages := s.prepareData(10, false, true, true, nil) // Subscribe for _, m := range messages { - s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + s.subscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) } // All messages should be received s.waitForMessages(messages) - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } func (s *FilterTestSuite) TestSubscriptionRefresh() { - messages := prepareData(2, false, false, true, nil) + messages := s.prepareData(2, false, false, true, nil) // Initial subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) // Repeat the same subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) // Both messages should be received s.waitForMessages(messages) - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -187,11 +187,11 @@ func (s *FilterTestSuite) TestContentTopicsLimit() { } } - messages := prepareData(maxContentTopics+1, false, true, true, nil) + messages := s.prepareData(maxContentTopics+1, false, true, true, nil) // Subscribe for _, m := range messages[:len(messages)-1] { - s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + s.subscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) } // All messages within limit should get received @@ -199,19 +199,19 @@ func (s *FilterTestSuite) TestContentTopicsLimit() { // Adding over the limit contentTopic should fail for _, sub := range s.subDetails { - if sub.ContentFilter.PubsubTopic == messages[len(messages)-1].pubSubTopic { - sub.Add(messages[len(messages)-1].contentTopic) - _, err := s.lightNode.Subscribe(s.ctx, sub.ContentFilter, WithPeer(s.fullNodeHost.ID())) + if sub.ContentFilter.PubsubTopic == messages[len(messages)-1].PubSubTopic { + sub.Add(messages[len(messages)-1].ContentTopic) + _, err := s.LightNode.Subscribe(s.ctx, sub.ContentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().Error(err) } } // Unsubscribe for cleanup for _, m := range messages { - _ = s.unsubscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + _ = s.unsubscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) } - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -221,25 +221,25 @@ func (s *FilterTestSuite) TestSubscribeErrorHandling() { // Prepare data messages = append(messages, WakuMsg{ - pubSubTopic: "", - contentTopic: s.testContentTopic, - payload: "N/A", + PubSubTopic: "", + ContentTopic: s.TestContentTopic, + Payload: "N/A", }) messages = append(messages, WakuMsg{ - pubSubTopic: s.testTopic, - contentTopic: "", - payload: "N/A", + PubSubTopic: s.TestTopic, + ContentTopic: "", + Payload: "N/A", }) // Subscribe with empty pubsub - s.contentFilter = protocol.ContentFilter{PubsubTopic: messages[0].pubSubTopic, ContentTopics: protocol.NewContentTopicSet(messages[0].contentTopic)} - _, err := s.lightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + s.contentFilter = protocol.ContentFilter{PubsubTopic: messages[0].PubSubTopic, ContentTopics: protocol.NewContentTopicSet(messages[0].ContentTopic)} + _, err := s.LightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().Error(err) // Subscribe with empty content topic - s.contentFilter = protocol.ContentFilter{PubsubTopic: messages[1].pubSubTopic, ContentTopics: protocol.NewContentTopicSet(messages[1].contentTopic)} - _, err = s.lightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + s.contentFilter = protocol.ContentFilter{PubsubTopic: messages[1].PubSubTopic, ContentTopics: protocol.NewContentTopicSet(messages[1].ContentTopic)} + _, err = s.LightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().Error(err) } @@ -258,7 +258,7 @@ func (s *FilterTestSuite) TestMultipleFullNodeSubscriptions() { s.Log.Info("Already subscribed to", zap.String("fullNode", string(fullNodeIDHex))) // This will overwrite values with the second node info - s.MakeWakuFilterFullNode(s.testTopic, false) + s.MakeWakuFilterFullNode(s.TestTopic, false) // Connect to second full and relay node s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) @@ -271,11 +271,11 @@ func (s *FilterTestSuite) TestMultipleFullNodeSubscriptions() { s.Log.Info("Subscribing to second", zap.String("fullNode", string(fullNodeIDHex))) // Subscribe to the second full node - s.contentFilter = protocol.ContentFilter{PubsubTopic: s.testTopic, ContentTopics: protocol.NewContentTopicSet(s.testContentTopic)} - _, err = s.lightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + s.contentFilter = protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} + _, err = s.LightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) - _, err = s.lightNode.UnsubscribeAll(s.ctx) + _, err = s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -285,27 +285,27 @@ func (s *FilterTestSuite) TestSubscribeMultipleLightNodes() { s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 10*time.Second) // Test can't exceed 10 seconds lightNodeData := s.GetWakuFilterLightNode() - lightNode2 := lightNodeData.lightNode + lightNode2 := lightNodeData.LightNode err := lightNode2.Start(context.Background()) s.Require().NoError(err) // Connect node2 lightNode2.h.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) - messages := prepareData(2, true, true, true, nil) + messages := s.prepareData(2, true, true, true, nil) // Subscribe separately: light node 1 -> full node - contentFilter := protocol.ContentFilter{PubsubTopic: messages[0].pubSubTopic, ContentTopics: protocol.NewContentTopicSet(messages[0].contentTopic)} - _, err = s.lightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + contentFilter := protocol.ContentFilter{PubsubTopic: messages[0].PubSubTopic, ContentTopics: protocol.NewContentTopicSet(messages[0].ContentTopic)} + _, err = s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) // Subscribe separately: light node 2 -> full node - contentFilter2 := protocol.ContentFilter{PubsubTopic: messages[1].pubSubTopic, ContentTopics: protocol.NewContentTopicSet(messages[1].contentTopic)} + contentFilter2 := protocol.ContentFilter{PubsubTopic: messages[1].PubSubTopic, ContentTopics: protocol.NewContentTopicSet(messages[1].ContentTopic)} _, err = lightNode2.Subscribe(s.ctx, contentFilter2, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) // Unsubscribe - _, err = s.lightNode.UnsubscribeAll(s.ctx) + _, err = s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) _, err = lightNode2.UnsubscribeAll(s.ctx) @@ -358,13 +358,13 @@ func (s *FilterTestSuite) TestSubscribeFullNode2FullNode() { } func (s *FilterTestSuite) TestIsSubscriptionAlive() { - messages := prepareData(2, false, true, false, nil) + messages := s.prepareData(2, false, true, false, nil) // Subscribe with the first message only - s.subscribe(messages[0].pubSubTopic, messages[0].contentTopic, s.fullNodeHost.ID()) + s.subscribe(messages[0].PubSubTopic, messages[0].ContentTopic, s.fullNodeHost.ID()) // IsSubscriptionAlive returns no error for the first message - err := s.lightNode.IsSubscriptionAlive(s.ctx, s.subDetails[0]) + err := s.LightNode.IsSubscriptionAlive(s.ctx, s.subDetails[0]) s.Require().NoError(err) // Create new host/peer - not related to any node @@ -375,25 +375,25 @@ func (s *FilterTestSuite) TestIsSubscriptionAlive() { s.subDetails[0].PeerID = host.ID() // IsSubscriptionAlive returns error for the second message, peer ID doesn't match - err = s.lightNode.IsSubscriptionAlive(s.ctx, s.subDetails[0]) + err = s.LightNode.IsSubscriptionAlive(s.ctx, s.subDetails[0]) s.Require().Error(err) } func (s *FilterTestSuite) TestFilterSubscription() { - contentFilter := protocol.ContentFilter{PubsubTopic: s.testTopic, ContentTopics: protocol.NewContentTopicSet(s.testContentTopic)} + contentFilter := protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} // Subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) // Returns no error and SubscriptionDetails for existing subscription - _, err := s.lightNode.FilterSubscription(s.fullNodeHost.ID(), contentFilter) + _, err := s.LightNode.FilterSubscription(s.fullNodeHost.ID(), contentFilter) s.Require().NoError(err) otherFilter := protocol.ContentFilter{PubsubTopic: "34583495", ContentTopics: protocol.NewContentTopicSet("sjfa402")} // Returns error and nil SubscriptionDetails for non existent subscription - nonSubscription, err := s.lightNode.FilterSubscription(s.fullNodeHost.ID(), otherFilter) + nonSubscription, err := s.LightNode.FilterSubscription(s.fullNodeHost.ID(), otherFilter) s.Require().Error(err) s.Require().Nil(nonSubscription) @@ -402,23 +402,23 @@ func (s *FilterTestSuite) TestFilterSubscription() { s.Require().NoError(err) // Returns error and nil SubscriptionDetails for unrelated host/peer - nonSubscription, err = s.lightNode.FilterSubscription(host.ID(), contentFilter) + nonSubscription, err = s.LightNode.FilterSubscription(host.ID(), contentFilter) s.Require().Error(err) s.Require().Nil(nonSubscription) } func (s *FilterTestSuite) TestHandleFilterSubscribeOptions() { - contentFilter := protocol.ContentFilter{PubsubTopic: s.testTopic, ContentTopics: protocol.NewContentTopicSet(s.testContentTopic)} + contentFilter := protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} // Subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) // With valid peer opts := []FilterSubscribeOption{WithPeer(s.fullNodeHost.ID())} // Positive case - _, _, err := s.lightNode.handleFilterSubscribeOptions(s.ctx, contentFilter, opts) + _, _, err := s.LightNode.handleFilterSubscribeOptions(s.ctx, contentFilter, opts) s.Require().NoError(err) addr := s.fullNodeHost.Addrs()[0] @@ -427,7 +427,7 @@ func (s *FilterTestSuite) TestHandleFilterSubscribeOptions() { opts = []FilterSubscribeOption{WithPeer(s.fullNodeHost.ID()), WithPeerAddr(addr)} // Should fail on wrong option combination - _, _, err = s.lightNode.handleFilterSubscribeOptions(s.ctx, contentFilter, opts) + _, _, err = s.LightNode.handleFilterSubscribeOptions(s.ctx, contentFilter, opts) s.Require().Error(err) } diff --git a/waku/v2/protocol/filter/filter_test.go b/waku/v2/protocol/filter/filter_test.go index f8d88b77e..4a93609ce 100644 --- a/waku/v2/protocol/filter/filter_test.go +++ b/waku/v2/protocol/filter/filter_test.go @@ -12,7 +12,6 @@ import ( "github.com/waku-org/go-waku/tests" "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/service" - "github.com/waku-org/go-waku/waku/v2/utils" "go.uber.org/zap" ) @@ -20,60 +19,20 @@ func TestFilterSuite(t *testing.T) { suite.Run(t, new(FilterTestSuite)) } -const defaultTestPubSubTopic = "/waku/2/go/filter/test" -const defaultTestContentTopic = "/test/10/my-app" - -func (s *FilterTestSuite) SetupTest() { - log := utils.Logger() //.Named("filterv2-test") - s.Log = log - // Use a pointer to WaitGroup so that to avoid copying - // https://pkg.go.dev/sync#WaitGroup - s.wg = &sync.WaitGroup{} - - // Create test context - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) // Test can't exceed 10 seconds - s.ctx = ctx - s.ctxCancel = cancel - - s.testTopic = defaultTestPubSubTopic - s.testContentTopic = defaultTestContentTopic - - s.MakeWakuFilterLightNode() - s.StartLightNode() - - //TODO: Add tests to verify broadcaster. - - s.MakeWakuFilterFullNode(s.testTopic, false) - - // Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) - err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) - s.Require().NoError(err) - -} - -func (s *FilterTestSuite) TearDownTest() { - s.fullNode.Stop() - s.relayNode.Stop() - s.RelaySub.Unsubscribe() - s.lightNode.Stop() - s.ctxCancel() -} - func (s *FilterTestSuite) TestRunningGuard() { - s.lightNode.Stop() + s.LightNode.Stop() contentFilter := protocol.ContentFilter{PubsubTopic: "test", ContentTopics: protocol.NewContentTopicSet("test")} - _, err := s.lightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err := s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().ErrorIs(err, service.ErrNotStarted) - err = s.lightNode.Start(s.ctx) + err = s.LightNode.Start(s.ctx) s.Require().NoError(err) - _, err = s.lightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) } @@ -82,18 +41,18 @@ func (s *FilterTestSuite) TestFireAndForgetAndCustomWg() { contentFilter := protocol.ContentFilter{PubsubTopic: "test", ContentTopics: protocol.NewContentTopicSet("test")} - _, err := s.lightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err := s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) - result, err := s.lightNode.Unsubscribe(s.ctx, contentFilter, DontWait()) + result, err := s.LightNode.Unsubscribe(s.ctx, contentFilter, DontWait()) s.Require().NoError(err) s.Require().Equal(0, len(result.Errors())) - _, err = s.lightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) wg := sync.WaitGroup{} - _, err = s.lightNode.Unsubscribe(s.ctx, contentFilter, WithWaitGroup(&wg)) + _, err = s.LightNode.Unsubscribe(s.ctx, contentFilter, WithWaitGroup(&wg)) wg.Wait() s.Require().NoError(err) } @@ -106,14 +65,14 @@ func (s *FilterTestSuite) TestStartStop() { stopNode := func() { for i := 0; i < 100000; i++ { - s.lightNode.Stop() + s.LightNode.Stop() } wg.Done() } startNode := func() { for i := 0; i < 100; i++ { - err := s.lightNode.Start(context.Background()) + err := s.LightNode.Start(context.Background()) if errors.Is(err, service.ErrAlreadyStarted) { continue } @@ -133,7 +92,7 @@ func (s *FilterTestSuite) TestAutoShard() { //Workaround as could not find a way to reuse setup test with params // Stop what is run in setup s.fullNode.Stop() - s.lightNode.Stop() + s.LightNode.Stop() ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) // Test can't exceed 10 seconds s.ctx = ctx s.ctxCancel = cancel @@ -143,8 +102,8 @@ func (s *FilterTestSuite) TestAutoShard() { s.Require().NoError(err) //Computing pubSubTopic only for filterFullNode. pubSubTopic := protocol.GetShardFromContentTopic(cTopic1, protocol.GenerationZeroShardsCount) - s.testContentTopic = cTopic1Str - s.testTopic = pubSubTopic.String() + s.TestContentTopic = cTopic1Str + s.TestTopic = pubSubTopic.String() s.MakeWakuFilterLightNode() s.StartLightNode() @@ -155,24 +114,24 @@ func (s *FilterTestSuite) TestAutoShard() { s.Require().NoError(err) s.Log.Info("Testing Autoshard:CreateSubscription") - s.subscribe("", s.testContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, ""}) + s.subscribe("", s.TestContentTopic, s.fullNodeHost.ID()) + s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, ""}) // Wrong content topic - s.waitForTimeout(&WakuMsg{s.testTopic, "TopicB", "second"}) + s.waitForTimeout(&WakuMsg{s.TestTopic, "TopicB", "second"}) - _, err = s.lightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.LightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) time.Sleep(1 * time.Second) // Should not receive after unsubscribe - s.waitForTimeout(&WakuMsg{s.testTopic, s.testContentTopic, "third"}) + s.waitForTimeout(&WakuMsg{s.TestTopic, s.TestContentTopic, "third"}) - s.subscribe("", s.testContentTopic, s.fullNodeHost.ID()) + s.subscribe("", s.TestContentTopic, s.fullNodeHost.ID()) s.Log.Info("Testing Autoshard:SubscriptionPing") - err = s.lightNode.Ping(context.Background(), s.fullNodeHost.ID()) + err = s.LightNode.Ping(context.Background(), s.fullNodeHost.ID()) s.Require().NoError(err) // Test ModifySubscription Subscribe to another content_topic @@ -181,39 +140,39 @@ func (s *FilterTestSuite) TestAutoShard() { newContentTopic := "0/test/1/testTopic1/proto" s.subscribe("", newContentTopic, s.fullNodeHost.ID()) - s.waitForMsg(&WakuMsg{s.testTopic, newContentTopic, ""}) + s.waitForMsg(&WakuMsg{s.TestTopic, newContentTopic, ""}) - _, err = s.lightNode.Unsubscribe(s.ctx, protocol.ContentFilter{ - PubsubTopic: s.testTopic, + _, err = s.LightNode.Unsubscribe(s.ctx, protocol.ContentFilter{ + PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(newContentTopic), }) s.Require().NoError(err) - _, err = s.lightNode.UnsubscribeAll(s.ctx) + _, err = s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } func (s *FilterTestSuite) TestLightNodeIsListening() { - messages := prepareData(2, true, true, false, nil) + messages := s.prepareData(2, true, true, false, nil) // Subscribe with the first message only - s.subscribe(messages[0].pubSubTopic, messages[0].contentTopic, s.fullNodeHost.ID()) + s.subscribe(messages[0].PubSubTopic, messages[0].ContentTopic, s.fullNodeHost.ID()) // IsListening returns true for the first message - listenStatus := s.lightNode.IsListening(messages[0].pubSubTopic, messages[0].contentTopic) + listenStatus := s.LightNode.IsListening(messages[0].PubSubTopic, messages[0].ContentTopic) s.Require().True(listenStatus) // IsListening returns false for the second message - listenStatus = s.lightNode.IsListening(messages[1].pubSubTopic, messages[1].contentTopic) + listenStatus = s.LightNode.IsListening(messages[1].PubSubTopic, messages[1].ContentTopic) s.Require().False(listenStatus) // IsListening returns false for combination as well - listenStatus = s.lightNode.IsListening(messages[0].pubSubTopic, messages[1].contentTopic) + listenStatus = s.LightNode.IsListening(messages[0].PubSubTopic, messages[1].ContentTopic) s.Require().False(listenStatus) - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } diff --git a/waku/v2/protocol/filter/filter_unsubscribe_test.go b/waku/v2/protocol/filter/filter_unsubscribe_test.go index 326b8f643..c32e0f25f 100644 --- a/waku/v2/protocol/filter/filter_unsubscribe_test.go +++ b/waku/v2/protocol/filter/filter_unsubscribe_test.go @@ -18,35 +18,35 @@ func (s *FilterTestSuite) TestUnsubscribeSingleContentTopic() { var newContentTopic = "TopicB" // Initial subscribe - s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) - s.subscribe(s.testTopic, newContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, newContentTopic, s.fullNodeHost.ID()) // Message is possible to receive for original contentTopic - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "test_msg"}) + s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, "test_msg"}) // Message is possible to receive for new contentTopic - s.waitForMsg(&WakuMsg{s.testTopic, newContentTopic, "test_msg"}) + s.waitForMsg(&WakuMsg{s.TestTopic, newContentTopic, "test_msg"}) - _ = s.unsubscribe(s.testTopic, newContentTopic, s.fullNodeHost.ID()) + _ = s.unsubscribe(s.TestTopic, newContentTopic, s.fullNodeHost.ID()) // Message should not be received for new contentTopic as it was unsubscribed - s.waitForTimeout(&WakuMsg{s.testTopic, newContentTopic, "test_msg"}) + s.waitForTimeout(&WakuMsg{s.TestTopic, newContentTopic, "test_msg"}) // Message is still possible to receive for original contentTopic - s.waitForMsg(&WakuMsg{s.testTopic, s.testContentTopic, "test_msg2"}) + s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, "test_msg2"}) - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } func (s *FilterTestSuite) TestUnsubscribeMultiContentTopic() { - var messages = prepareData(3, false, true, true, nil) + var messages = s.prepareData(3, false, true, true, nil) // Subscribe with 3 content topics for _, m := range messages { - s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + s.subscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) } // All messages should be received @@ -54,18 +54,18 @@ func (s *FilterTestSuite) TestUnsubscribeMultiContentTopic() { // Unsubscribe with the last 2 content topics for _, m := range messages[1:] { - _ = s.unsubscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + _ = s.unsubscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) } // Messages should not be received for the last two contentTopics as it was unsubscribed for _, m := range messages[1:] { - s.waitForTimeout(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}) + s.waitForTimeout(&WakuMsg{m.PubSubTopic, m.ContentTopic, m.Payload}) } // Message is still possible to receive for the first contentTopic - s.waitForMsg(&WakuMsg{messages[0].pubSubTopic, messages[0].contentTopic, messages[0].payload}) + s.waitForMsg(&WakuMsg{messages[0].PubSubTopic, messages[0].ContentTopic, messages[0].Payload}) - _, err := s.lightNode.UnsubscribeAll(s.ctx) + _, err := s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } @@ -77,19 +77,19 @@ func (s *FilterTestSuite) TestUnsubscribeMultiPubSubMultiContentTopic() { s.MakeWakuFilterLightNode() s.StartLightNode() - s.MakeWakuFilterFullNode(s.testTopic, true) + s.MakeWakuFilterFullNode(s.TestTopic, true) // Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNode.h), peerstore.PermanentAddrTTL) err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) - messages := prepareData(2, true, true, true, nil) + messages := s.prepareData(2, true, true, true, nil) // Subscribe for _, m := range messages { - s.subDetails = append(s.subDetails, s.getSub(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID())...) - _, err = s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.pubSubTopic)) + s.subDetails = append(s.subDetails, s.getSub(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID())...) + _, err = s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.PubSubTopic)) s.Require().NoError(err) } @@ -98,12 +98,12 @@ func (s *FilterTestSuite) TestUnsubscribeMultiPubSubMultiContentTopic() { // Unsubscribe for _, m := range messages { - _ = s.unsubscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + _ = s.unsubscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) } // No messages can be received with previous subscriptions for i, m := range messages { - s.waitForTimeoutFromChan(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}, s.subDetails[i].C) + s.waitForTimeoutFromChan(&WakuMsg{m.PubSubTopic, m.ContentTopic, m.Payload}, s.subDetails[i].C) } } @@ -114,7 +114,7 @@ func (s *FilterTestSuite) TestUnsubscribeErrorHandling() { s.MakeWakuFilterLightNode() s.StartLightNode() - s.MakeWakuFilterFullNode(s.testTopic, true) + s.MakeWakuFilterFullNode(s.TestTopic, true) // Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) @@ -123,30 +123,30 @@ func (s *FilterTestSuite) TestUnsubscribeErrorHandling() { var messages, invalidMessages []WakuMsg - messages = prepareData(2, false, true, true, nil) + messages = s.prepareData(2, false, true, true, nil) // Prepare "invalid" data for unsubscribe invalidMessages = append(invalidMessages, WakuMsg{ - pubSubTopic: "", - contentTopic: messages[0].contentTopic, - payload: "N/A", + PubSubTopic: "", + ContentTopic: messages[0].ContentTopic, + Payload: "N/A", }, WakuMsg{ - pubSubTopic: messages[0].pubSubTopic, - contentTopic: "", - payload: "N/A", + PubSubTopic: messages[0].PubSubTopic, + ContentTopic: "", + Payload: "N/A", }, WakuMsg{ - pubSubTopic: "/waku/2/go/filter/not_subscribed", - contentTopic: "not_subscribed_topic", - payload: "N/A", + PubSubTopic: "/waku/2/go/filter/not_subscribed", + ContentTopic: "not_subscribed_topic", + Payload: "N/A", }) // Subscribe with valid topics for _, m := range messages { - s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) - _, err = s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.pubSubTopic)) + s.subscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) + _, err = s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.PubSubTopic)) s.Require().NoError(err) } @@ -154,50 +154,50 @@ func (s *FilterTestSuite) TestUnsubscribeErrorHandling() { s.waitForMessages(messages) // Unsubscribe with empty pubsub - contentFilter := protocol.ContentFilter{PubsubTopic: invalidMessages[0].pubSubTopic, - ContentTopics: protocol.NewContentTopicSet(invalidMessages[0].contentTopic)} - _, err = s.lightNode.Unsubscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + contentFilter := protocol.ContentFilter{PubsubTopic: invalidMessages[0].PubSubTopic, + ContentTopics: protocol.NewContentTopicSet(invalidMessages[0].ContentTopic)} + _, err = s.LightNode.Unsubscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().Error(err) // Unsubscribe with empty content topic - contentFilter = protocol.ContentFilter{PubsubTopic: invalidMessages[1].pubSubTopic, - ContentTopics: protocol.NewContentTopicSet(invalidMessages[1].contentTopic)} - _, err = s.lightNode.Unsubscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + contentFilter = protocol.ContentFilter{PubsubTopic: invalidMessages[1].PubSubTopic, + ContentTopics: protocol.NewContentTopicSet(invalidMessages[1].ContentTopic)} + _, err = s.LightNode.Unsubscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().Error(err) // Unsubscribe with non-existent topics, expect no error to prevent attacker from topic guessing - contentFilter = protocol.ContentFilter{PubsubTopic: invalidMessages[2].pubSubTopic, - ContentTopics: protocol.NewContentTopicSet(invalidMessages[2].contentTopic)} - _, err = s.lightNode.Unsubscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + contentFilter = protocol.ContentFilter{PubsubTopic: invalidMessages[2].PubSubTopic, + ContentTopics: protocol.NewContentTopicSet(invalidMessages[2].ContentTopic)} + _, err = s.LightNode.Unsubscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) // All messages should be still possible to receive for subscribed topics s.waitForMessages(messages) - _, err = s.lightNode.UnsubscribeAll(s.ctx) + _, err = s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } func (s *FilterTestSuite) TestUnsubscribeAllWithoutContentTopics() { - var messages = prepareData(2, false, true, true, nil) + var messages = s.prepareData(2, false, true, true, nil) // Subscribe with 2 content topics for _, m := range messages { - s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + s.subscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) } // All messages should be received s.waitForMessages(messages) // Unsubscribe all with peer specification - _, err := s.lightNode.UnsubscribeAll(s.ctx, WithPeer(s.fullNodeHost.ID())) + _, err := s.LightNode.UnsubscribeAll(s.ctx, WithPeer(s.fullNodeHost.ID())) s.Require().NoError(err) // Messages should not be received for any contentTopics for _, m := range messages { - s.waitForTimeout(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}) + s.waitForTimeout(&WakuMsg{m.PubSubTopic, m.ContentTopic, m.Payload}) } } @@ -208,19 +208,19 @@ func (s *FilterTestSuite) TestUnsubscribeAllDiffPubSubContentTopics() { s.MakeWakuFilterLightNode() s.StartLightNode() - s.MakeWakuFilterFullNode(s.testTopic, true) + s.MakeWakuFilterFullNode(s.TestTopic, true) // Connect nodes s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNode.h), peerstore.PermanentAddrTTL) err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) - messages := prepareData(2, true, true, true, nil) + messages := s.prepareData(2, true, true, true, nil) // Subscribe for _, m := range messages { - s.subDetails = append(s.subDetails, s.getSub(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID())...) - _, err = s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.pubSubTopic)) + s.subDetails = append(s.subDetails, s.getSub(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID())...) + _, err = s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.PubSubTopic)) s.Require().NoError(err) } @@ -228,23 +228,23 @@ func (s *FilterTestSuite) TestUnsubscribeAllDiffPubSubContentTopics() { s.waitForMessages(messages) // Unsubscribe all without any specification - _, err = s.lightNode.UnsubscribeAll(s.ctx) + _, err = s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) // No messages can be received with previous subscriptions for i, m := range messages { - s.waitForTimeoutFromChan(&WakuMsg{m.pubSubTopic, m.contentTopic, m.payload}, s.subDetails[i].C) + s.waitForTimeoutFromChan(&WakuMsg{m.PubSubTopic, m.ContentTopic, m.Payload}, s.subDetails[i].C) } } func (s *FilterTestSuite) TestUnsubscribeAllUnrelatedPeer() { - var messages = prepareData(2, false, true, false, nil) + var messages = s.prepareData(2, false, true, false, nil) // Subscribe with 2 content topics for _, m := range messages { - s.subscribe(m.pubSubTopic, m.contentTopic, s.fullNodeHost.ID()) + s.subscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) } // All messages should be received @@ -259,7 +259,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllUnrelatedPeer() { s.Log.Info("Host ID", logging.HostID("Unrelated", host.ID())) // Unsubscribe all with unrelated peer specification - pushResult, err := s.lightNode.UnsubscribeAll(s.ctx, WithPeer(host.ID())) + pushResult, err := s.LightNode.UnsubscribeAll(s.ctx, WithPeer(host.ID())) for e := range pushResult.errs { s.Log.Info("Push Result ", zap.String("error", strconv.Itoa(e))) diff --git a/waku/v2/protocol/filter/test_utils.go b/waku/v2/protocol/filter/test_utils.go index f94b86201..3d7f67a4d 100644 --- a/waku/v2/protocol/filter/test_utils.go +++ b/waku/v2/protocol/filter/test_utils.go @@ -10,6 +10,7 @@ import ( "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/peerstore" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/suite" "github.com/waku-org/go-waku/tests" @@ -22,7 +23,7 @@ import ( ) type LightNodeData struct { - lightNode *WakuFilterLightNode + LightNode *WakuFilterLightNode lightNodeHost host.Host } @@ -39,8 +40,8 @@ type FilterTestSuite struct { LightNodeData FullNodeData - testTopic string - testContentTopic string + TestTopic string + TestContentTopic string ctx context.Context ctxCancel context.CancelFunc wg *sync.WaitGroup @@ -50,10 +51,52 @@ type FilterTestSuite struct { Log *zap.Logger } +const DefaultTestPubSubTopic = "/waku/2/go/filter/test" +const DefaultTestContentTopic = "/test/10/my-app" + type WakuMsg struct { - pubSubTopic string - contentTopic string - payload string + PubSubTopic string + ContentTopic string + Payload string +} + +func (s *FilterTestSuite) SetupTest() { + log := utils.Logger() //.Named("filterv2-test") + s.Log = log + + s.Log.Info("SetupTest()") + // Use a pointer to WaitGroup so that to avoid copying + // https://pkg.go.dev/sync#WaitGroup + s.wg = &sync.WaitGroup{} + + // Create test context + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) // Test can't exceed 10 seconds + s.ctx = ctx + s.ctxCancel = cancel + + s.TestTopic = DefaultTestPubSubTopic + s.TestContentTopic = DefaultTestContentTopic + + s.MakeWakuFilterLightNode() + s.StartLightNode() + + //TODO: Add tests to verify broadcaster. + + s.MakeWakuFilterFullNode(s.TestTopic, false) + + // Connect nodes + s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) + err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.Require().NoError(err) + +} + +func (s *FilterTestSuite) TearDownTest() { + s.fullNode.Stop() + s.LightNode.Stop() + s.RelaySub.Unsubscribe() + s.LightNode.Stop() + s.ctxCancel() } func (s *FilterTestSuite) GetWakuRelay(topic string) FullNodeData { @@ -126,7 +169,7 @@ func (s *FilterTestSuite) MakeWakuFilterLightNode() { } func (s *FilterTestSuite) StartLightNode() { - err := s.lightNode.Start(context.Background()) + err := s.LightNode.Start(context.Background()) s.Require().NoError(err) } @@ -163,9 +206,9 @@ func (s *FilterTestSuite) waitForMsgFromChan(msg *WakuMsg, ch chan *protocol.Env func matchOneOfManyMsg(one WakuMsg, many []WakuMsg) bool { for _, m := range many { - if m.pubSubTopic == one.pubSubTopic && - m.contentTopic == one.contentTopic && - m.payload == one.payload { + if m.PubSubTopic == one.PubSubTopic && + m.ContentTopic == one.ContentTopic && + m.Payload == one.Payload { return true } } @@ -192,11 +235,11 @@ func (s *FilterTestSuite) waitForMessages(msgs []WakuMsg) { continue } received := WakuMsg{ - pubSubTopic: env.PubsubTopic(), - contentTopic: env.Message().GetContentTopic(), - payload: string(env.Message().GetPayload()), + PubSubTopic: env.PubsubTopic(), + ContentTopic: env.Message().GetContentTopic(), + Payload: string(env.Message().GetPayload()), } - s.Log.Debug("received message ", zap.String("pubSubTopic", received.pubSubTopic), zap.String("contentTopic", received.contentTopic), zap.String("payload", received.payload)) + s.Log.Debug("received message ", zap.String("pubSubTopic", received.PubSubTopic), zap.String("contentTopic", received.ContentTopic), zap.String("payload", received.Payload)) if matchOneOfManyMsg(received, msgs) { found++ } @@ -245,7 +288,7 @@ func (s *FilterTestSuite) waitForTimeoutFromChan(msg *WakuMsg, ch chan *protocol func (s *FilterTestSuite) getSub(pubsubTopic string, contentTopic string, peer peer.ID) []*subscription.SubscriptionDetails { contentFilter := protocol.ContentFilter{PubsubTopic: pubsubTopic, ContentTopics: protocol.NewContentTopicSet(contentTopic)} - subDetails, err := s.lightNode.Subscribe(s.ctx, contentFilter, WithPeer(peer)) + subDetails, err := s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(peer)) s.Require().NoError(err) time.Sleep(1 * time.Second) @@ -258,7 +301,7 @@ func (s *FilterTestSuite) subscribe(pubsubTopic string, contentTopic string, pee if sub.ContentFilter.PubsubTopic == pubsubTopic { sub.Add(contentTopic) s.contentFilter = sub.ContentFilter - subDetails, err := s.lightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(peer)) + subDetails, err := s.LightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(peer)) s.subDetails = subDetails s.Require().NoError(err) return @@ -275,7 +318,7 @@ func (s *FilterTestSuite) unsubscribe(pubsubTopic string, contentTopic string, p if sub.ContentFilter.PubsubTopic == pubsubTopic { topicsCount := len(sub.ContentFilter.ContentTopicsList()) if topicsCount == 1 { - _, err := s.lightNode.Unsubscribe(s.ctx, sub.ContentFilter, WithPeer(peer)) + _, err := s.LightNode.Unsubscribe(s.ctx, sub.ContentFilter, WithPeer(peer)) s.Require().NoError(err) } else { sub.Remove(contentTopic) @@ -284,29 +327,29 @@ func (s *FilterTestSuite) unsubscribe(pubsubTopic string, contentTopic string, p } } - return s.lightNode.Subscriptions() + return s.LightNode.Subscriptions() } func (s *FilterTestSuite) publishMsg(msg *WakuMsg) { - if len(msg.payload) == 0 { - msg.payload = "123" + if len(msg.Payload) == 0 { + msg.Payload = "123" } - _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(msg.contentTopic, utils.GetUnixEpoch(), msg.payload), relay.WithPubSubTopic(msg.pubSubTopic)) + _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(msg.ContentTopic, utils.GetUnixEpoch(), msg.Payload), relay.WithPubSubTopic(msg.PubSubTopic)) s.Require().NoError(err) } func (s *FilterTestSuite) publishMessages(msgs []WakuMsg) { for _, m := range msgs { - _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(m.contentTopic, utils.GetUnixEpoch(), m.payload), relay.WithPubSubTopic(m.pubSubTopic)) + _, err := s.relayNode.Publish(s.ctx, tests.CreateWakuMessage(m.ContentTopic, utils.GetUnixEpoch(), m.Payload), relay.WithPubSubTopic(m.PubSubTopic)) s.Require().NoError(err) } } -func prepareData(quantity int, topics, contentTopics, payloads bool, sg tests.StringGenerator) []WakuMsg { +func (s *FilterTestSuite) prepareData(quantity int, topics, contentTopics, payloads bool, sg tests.StringGenerator) []WakuMsg { var ( - pubsubTopic = defaultTestPubSubTopic // Has to be the same with initial s.testTopic - contentTopic = defaultTestContentTopic // Has to be the same with initial s.testContentTopic + pubsubTopic = s.TestTopic // Has to be the same with initial s.testTopic + contentTopic = s.TestContentTopic // Has to be the same with initial s.testContentTopic payload = "test_msg" messages []WakuMsg strMaxLenght = 4097 @@ -315,9 +358,9 @@ func prepareData(quantity int, topics, contentTopics, payloads bool, sg tests.St for i := 0; i < quantity; i++ { msg := WakuMsg{ - pubSubTopic: pubsubTopic, - contentTopic: contentTopic, - payload: payload, + PubSubTopic: pubsubTopic, + ContentTopic: contentTopic, + Payload: payload, } if sg != nil { @@ -326,15 +369,15 @@ func prepareData(quantity int, topics, contentTopics, payloads bool, sg tests.St } if topics { - msg.pubSubTopic = fmt.Sprintf("%s%02d%s", pubsubTopic, i, generatedString) + msg.PubSubTopic = fmt.Sprintf("%s%02d%s", pubsubTopic, i, generatedString) } if contentTopics { - msg.contentTopic = fmt.Sprintf("%s%02d%s", contentTopic, i, generatedString) + msg.ContentTopic = fmt.Sprintf("%s%02d%s", contentTopic, i, generatedString) } if payloads { - msg.payload = fmt.Sprintf("%s%02d%s", payload, i, generatedString) + msg.Payload = fmt.Sprintf("%s%02d%s", payload, i, generatedString) } messages = append(messages, msg) From 70f2d34e82195f51f7918c8efa68e83b6ccecfd3 Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Mon, 8 Apr 2024 19:22:14 +0300 Subject: [PATCH 10/27] FullNodeHost --- waku/v2/api/filter.go | 20 +++-- waku/v2/api/filter_test.go | 12 ++- waku/v2/protocol/filter/filter_ping_test.go | 14 ++-- .../filter/filter_proto_ident_test.go | 18 ++--- waku/v2/protocol/filter/filter_push_test.go | 16 ++-- .../protocol/filter/filter_subscribe_test.go | 76 +++++++++---------- waku/v2/protocol/filter/filter_test.go | 24 +++--- .../filter/filter_unsubscribe_test.go | 44 +++++------ waku/v2/protocol/filter/test_utils.go | 8 +- 9 files changed, 123 insertions(+), 109 deletions(-) diff --git a/waku/v2/api/filter.go b/waku/v2/api/filter.go index 746f91e15..56fce7e75 100644 --- a/waku/v2/api/filter.go +++ b/waku/v2/api/filter.go @@ -5,6 +5,7 @@ import ( "sync" "time" + "github.com/libp2p/go-libp2p/core/peer" "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/protocol/filter" "github.com/waku-org/go-waku/waku/v2/protocol/subscription" @@ -12,7 +13,8 @@ import ( ) type FilterConfig struct { - MaxPeers uint + MaxPeers int + Peers []peer.ID } type Sub struct { @@ -38,7 +40,7 @@ func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilte err := sub.subscribe(contentFilter, sub.Config.MaxPeers) if err == nil { - sub.healthCheckLoop() + go sub.healthCheckLoop() return sub, nil } else { return nil, err @@ -76,7 +78,7 @@ func (apiSub *Sub) healthCheckLoop() { } } -func (apiSub *Sub) checkAliveness() map[string]uint { +func (apiSub *Sub) checkAliveness() map[string]int { apiSub.RLock() defer apiSub.RUnlock() @@ -104,7 +106,7 @@ func (apiSub *Sub) checkAliveness() map[string]uint { wg.Wait() close(ch) // Collect healthy topics - m := make(map[string]uint) + m := make(map[string]int) topicMap, _ := protocol.ContentFilterToPubSubTopicMap(apiSub.ContentFilter) for _, t := range maps.Keys(topicMap) { m[t] = 0 @@ -116,9 +118,15 @@ func (apiSub *Sub) checkAliveness() map[string]uint { return m } -func (apiSub *Sub) subscribe(contentFilter protocol.ContentFilter, peerCount uint) error { +func (apiSub *Sub) subscribe(contentFilter protocol.ContentFilter, peerCount int) error { // Low-level subscribe, returns a set of SubscriptionDetails - subs, err := apiSub.wf.Subscribe(apiSub.ctx, contentFilter, filter.WithMaxPeersPerContentFilter(int(peerCount))) + options := make([]filter.FilterSubscribeOption, 0) + options = append(options, filter.WithMaxPeersPerContentFilter(int(peerCount))) + for _, p := range apiSub.Config.Peers { + options = append(options, filter.WithPeer(p)) + } + subs, err := apiSub.wf.Subscribe(apiSub.ctx, contentFilter, options...) + if err != nil { // TODO what if fails? return err diff --git a/waku/v2/api/filter_test.go b/waku/v2/api/filter_test.go index 59fc20cb2..966f4ca01 100644 --- a/waku/v2/api/filter_test.go +++ b/waku/v2/api/filter_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/suite" "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/protocol/filter" @@ -28,9 +29,14 @@ func (s *FilterApiTestSuite) TearDownTest() { func (s *FilterApiTestSuite) TestSubscribe() { contentFilter := protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} - apiConfig := FilterConfig{MaxPeers: 2} + apiConfig := FilterConfig{MaxPeers: 1, Peers: []peer.ID{s.FullNodeHost.ID()}} + + s.Require().Equal(apiConfig.MaxPeers, 1) + s.Require().Equal(contentFilter.PubsubTopic, s.TestTopic) + + s.Log.Info("About to perform API Subscribe()") apiSub, err := Subscribe(context.Background(), s.LightNode, contentFilter, apiConfig) s.Require().NoError(err) - - s.Require().NotNil(apiSub.wf) + s.Require().Equal(apiSub.ContentFilter, contentFilter) + s.Log.Info("Subscribed") } diff --git a/waku/v2/protocol/filter/filter_ping_test.go b/waku/v2/protocol/filter/filter_ping_test.go index 54607d0ad..cc6dfb2ad 100644 --- a/waku/v2/protocol/filter/filter_ping_test.go +++ b/waku/v2/protocol/filter/filter_ping_test.go @@ -6,29 +6,29 @@ import ( ) func (s *FilterTestSuite) TestSubscriptionPing() { - err := s.LightNode.Ping(context.Background(), s.fullNodeHost.ID()) + err := s.LightNode.Ping(context.Background(), s.FullNodeHost.ID()) s.Require().Error(err) filterErr, ok := err.(*FilterError) s.Require().True(ok) s.Require().Equal(filterErr.Code, http.StatusNotFound) contentTopic := "abc" - s.subscribe(s.TestTopic, contentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, contentTopic, s.FullNodeHost.ID()) - err = s.LightNode.Ping(context.Background(), s.fullNodeHost.ID()) + err = s.LightNode.Ping(context.Background(), s.FullNodeHost.ID()) s.Require().NoError(err) } func (s *FilterTestSuite) TestUnSubscriptionPing() { - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) - err := s.LightNode.Ping(context.Background(), s.fullNodeHost.ID()) + err := s.LightNode.Ping(context.Background(), s.FullNodeHost.ID()) s.Require().NoError(err) - _, err = s.LightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.LightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().NoError(err) - err = s.LightNode.Ping(context.Background(), s.fullNodeHost.ID()) + err = s.LightNode.Ping(context.Background(), s.FullNodeHost.ID()) s.Require().Error(err) } diff --git a/waku/v2/protocol/filter/filter_proto_ident_test.go b/waku/v2/protocol/filter/filter_proto_ident_test.go index a48a86467..dba82e87b 100644 --- a/waku/v2/protocol/filter/filter_proto_ident_test.go +++ b/waku/v2/protocol/filter/filter_proto_ident_test.go @@ -30,20 +30,20 @@ import ( func (s *FilterTestSuite) TestCreateSubscription() { // Initial subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, ""}) } func (s *FilterTestSuite) TestModifySubscription() { // Initial subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, ""}) // Subscribe to another content_topic newContentTopic := "Topic_modified" - s.subscribe(s.TestTopic, newContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, newContentTopic, s.FullNodeHost.ID()) s.waitForMsg(&WakuMsg{s.TestTopic, newContentTopic, ""}) } @@ -51,7 +51,7 @@ func (s *FilterTestSuite) TestModifySubscription() { func (s *FilterTestSuite) TestMultipleMessages() { // Initial subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, "first"}) @@ -217,11 +217,11 @@ func (s *FilterTestSuite) TestIncorrectSubscribeIdentifier() { s.MakeWakuFilterFullNode(s.TestTopic, false) //Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) + s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) // Subscribe with incorrect SubscribeID s.contentFilter = protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} - _, err := s.LightNode.IncorrectSubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err := s.LightNode.IncorrectSubscribe(s.ctx, s.contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().Error(err) _, err = s.LightNode.UnsubscribeAll(s.ctx) @@ -261,13 +261,13 @@ func (s *FilterTestSuite) TestIncorrectPushIdentifier() { s.Require().NoError(err) // Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) - err = s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) + err = s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) // Subscribe s.contentFilter = protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} - s.subDetails, err = s.LightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + s.subDetails, err = s.LightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().NoError(err) time.Sleep(1 * time.Second) diff --git a/waku/v2/protocol/filter/filter_push_test.go b/waku/v2/protocol/filter/filter_push_test.go index d8310a22d..e3aa106f0 100644 --- a/waku/v2/protocol/filter/filter_push_test.go +++ b/waku/v2/protocol/filter/filter_push_test.go @@ -14,7 +14,7 @@ import ( func (s *FilterTestSuite) TestValidPayloadsASCII() { // Subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // Prepare data messages := s.prepareData(100, false, false, true, tests.GenerateRandomASCIIString) @@ -30,7 +30,7 @@ func (s *FilterTestSuite) TestValidPayloadsASCII() { func (s *FilterTestSuite) TestValidPayloadsUTF8() { // Subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // Prepare data messages := s.prepareData(100, false, false, true, tests.GenerateRandomUTF8String) @@ -46,7 +46,7 @@ func (s *FilterTestSuite) TestValidPayloadsUTF8() { func (s *FilterTestSuite) TestValidPayloadsBase64() { // Subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // Prepare data messages := s.prepareData(100, false, false, true, tests.GenerateRandomBase64String) @@ -62,7 +62,7 @@ func (s *FilterTestSuite) TestValidPayloadsBase64() { func (s *FilterTestSuite) TestValidPayloadsJSON() { // Subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // Prepare data messages := s.prepareData(100, false, false, true, tests.GenerateRandomJSONString) @@ -78,7 +78,7 @@ func (s *FilterTestSuite) TestValidPayloadsJSON() { func (s *FilterTestSuite) TestValidPayloadsURLEncoded() { // Subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // Prepare data messages := s.prepareData(100, false, false, true, tests.GenerateRandomURLEncodedString) @@ -94,7 +94,7 @@ func (s *FilterTestSuite) TestValidPayloadsURLEncoded() { func (s *FilterTestSuite) TestValidPayloadsSQL() { // Subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // Prepare data messages := s.prepareData(100, false, false, true, tests.GenerateRandomSQLInsert) @@ -112,7 +112,7 @@ func (s *FilterTestSuite) TestLargePayloadsUTF8() { s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 40*time.Second) // Subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // Prepare basic data messages := s.prepareData(10, false, false, false, nil) @@ -134,7 +134,7 @@ func (s *FilterTestSuite) TestLargePayloadsUTF8() { func (s *FilterTestSuite) TestFuturePayloadEncryptionVersion() { // Subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) message := tests.CreateWakuMessage(s.TestContentTopic, utils.GetUnixEpoch(), "test_payload") futureVersion := uint32(100) diff --git a/waku/v2/protocol/filter/filter_subscribe_test.go b/waku/v2/protocol/filter/filter_subscribe_test.go index ce1415a1e..9e1964bce 100644 --- a/waku/v2/protocol/filter/filter_subscribe_test.go +++ b/waku/v2/protocol/filter/filter_subscribe_test.go @@ -18,7 +18,7 @@ import ( func (s *FilterTestSuite) TestWakuFilter() { // Initial subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // Should be received s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, "first"}) @@ -26,16 +26,16 @@ func (s *FilterTestSuite) TestWakuFilter() { // Wrong content topic s.waitForTimeout(&WakuMsg{s.TestTopic, "TopicB", "second"}) - _, err := s.LightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err := s.LightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().NoError(err) // Should not receive after unsubscribe s.waitForTimeout(&WakuMsg{s.TestTopic, s.TestContentTopic, "third"}) // Two new subscriptions with same [peer, contentFilter] - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) - secondSub := s.getSub(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + secondSub := s.getSub(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // Assert that we have 2 subscriptions now s.Require().Equal(len(s.LightNode.Subscriptions()), 2) @@ -67,7 +67,7 @@ func (s *FilterTestSuite) TestWakuFilter() { func (s *FilterTestSuite) TestPubSubSingleContentTopic() { // Initial subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // Message should be received s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, "test_msg"}) @@ -86,7 +86,7 @@ func (s *FilterTestSuite) TestPubSubMultiContentTopic() { // Subscribe for _, m := range messages { - s.subscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) + s.subscribe(m.PubSubTopic, m.ContentTopic, s.FullNodeHost.ID()) } // All messages should be received @@ -108,15 +108,15 @@ func (s *FilterTestSuite) TestMultiPubSubMultiContentTopic() { s.MakeWakuFilterFullNode(s.TestTopic, true) // Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) - err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) + err := s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) messages := s.prepareData(2, true, true, false, nil) // Subscribe for _, m := range messages { - s.subDetails = append(s.subDetails, s.getSub(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID())...) + s.subDetails = append(s.subDetails, s.getSub(m.PubSubTopic, m.ContentTopic, s.FullNodeHost.ID())...) s.Log.Info("Subscribing ", zap.String("PubSubTopic", m.PubSubTopic)) _, err := s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.PubSubTopic)) s.Require().NoError(err) @@ -144,7 +144,7 @@ func (s *FilterTestSuite) TestPubSubMultiOverlapContentTopic() { // Subscribe for _, m := range messages { - s.subscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) + s.subscribe(m.PubSubTopic, m.ContentTopic, s.FullNodeHost.ID()) } // All messages should be received @@ -160,10 +160,10 @@ func (s *FilterTestSuite) TestSubscriptionRefresh() { messages := s.prepareData(2, false, false, true, nil) // Initial subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // Repeat the same subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // Both messages should be received s.waitForMessages(messages) @@ -191,7 +191,7 @@ func (s *FilterTestSuite) TestContentTopicsLimit() { // Subscribe for _, m := range messages[:len(messages)-1] { - s.subscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) + s.subscribe(m.PubSubTopic, m.ContentTopic, s.FullNodeHost.ID()) } // All messages within limit should get received @@ -201,14 +201,14 @@ func (s *FilterTestSuite) TestContentTopicsLimit() { for _, sub := range s.subDetails { if sub.ContentFilter.PubsubTopic == messages[len(messages)-1].PubSubTopic { sub.Add(messages[len(messages)-1].ContentTopic) - _, err := s.LightNode.Subscribe(s.ctx, sub.ContentFilter, WithPeer(s.fullNodeHost.ID())) + _, err := s.LightNode.Subscribe(s.ctx, sub.ContentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().Error(err) } } // Unsubscribe for cleanup for _, m := range messages { - _ = s.unsubscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) + _ = s.unsubscribe(m.PubSubTopic, m.ContentTopic, s.FullNodeHost.ID()) } _, err := s.LightNode.UnsubscribeAll(s.ctx) @@ -234,12 +234,12 @@ func (s *FilterTestSuite) TestSubscribeErrorHandling() { // Subscribe with empty pubsub s.contentFilter = protocol.ContentFilter{PubsubTopic: messages[0].PubSubTopic, ContentTopics: protocol.NewContentTopicSet(messages[0].ContentTopic)} - _, err := s.LightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err := s.LightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().Error(err) // Subscribe with empty content topic s.contentFilter = protocol.ContentFilter{PubsubTopic: messages[1].PubSubTopic, ContentTopics: protocol.NewContentTopicSet(messages[1].ContentTopic)} - _, err = s.LightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.LightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().Error(err) } @@ -252,8 +252,8 @@ func (s *FilterTestSuite) TestMultipleFullNodeSubscriptions() { // Create test context s.ctx, s.ctxCancel = context.WithTimeout(context.Background(), 10*time.Second) // Test can't exceed 10 seconds - fullNodeIDHex := make([]byte, hex.EncodedLen(len([]byte(s.fullNodeHost.ID())))) - _ = hex.Encode(fullNodeIDHex, []byte(s.fullNodeHost.ID())) + fullNodeIDHex := make([]byte, hex.EncodedLen(len([]byte(s.FullNodeHost.ID())))) + _ = hex.Encode(fullNodeIDHex, []byte(s.FullNodeHost.ID())) s.Log.Info("Already subscribed to", zap.String("fullNode", string(fullNodeIDHex))) @@ -261,18 +261,18 @@ func (s *FilterTestSuite) TestMultipleFullNodeSubscriptions() { s.MakeWakuFilterFullNode(s.TestTopic, false) // Connect to second full and relay node - s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) - err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) + err := s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) - fullNodeIDHex = make([]byte, hex.EncodedLen(len([]byte(s.fullNodeHost.ID())))) - _ = hex.Encode(fullNodeIDHex, []byte(s.fullNodeHost.ID())) + fullNodeIDHex = make([]byte, hex.EncodedLen(len([]byte(s.FullNodeHost.ID())))) + _ = hex.Encode(fullNodeIDHex, []byte(s.FullNodeHost.ID())) s.Log.Info("Subscribing to second", zap.String("fullNode", string(fullNodeIDHex))) // Subscribe to the second full node s.contentFilter = protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} - _, err = s.LightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.LightNode.Subscribe(s.ctx, s.contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().NoError(err) _, err = s.LightNode.UnsubscribeAll(s.ctx) @@ -290,18 +290,18 @@ func (s *FilterTestSuite) TestSubscribeMultipleLightNodes() { s.Require().NoError(err) // Connect node2 - lightNode2.h.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) + lightNode2.h.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) messages := s.prepareData(2, true, true, true, nil) // Subscribe separately: light node 1 -> full node contentFilter := protocol.ContentFilter{PubsubTopic: messages[0].PubSubTopic, ContentTopics: protocol.NewContentTopicSet(messages[0].ContentTopic)} - _, err = s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().NoError(err) // Subscribe separately: light node 2 -> full node contentFilter2 := protocol.ContentFilter{PubsubTopic: messages[1].PubSubTopic, ContentTopics: protocol.NewContentTopicSet(messages[1].ContentTopic)} - _, err = lightNode2.Subscribe(s.ctx, contentFilter2, WithPeer(s.fullNodeHost.ID())) + _, err = lightNode2.Subscribe(s.ctx, contentFilter2, WithPeer(s.FullNodeHost.ID())) s.Require().NoError(err) // Unsubscribe @@ -327,10 +327,10 @@ func (s *FilterTestSuite) TestSubscribeFullNode2FullNode() { fullNode2 := nodeData.fullNode // Connect nodes - fullNode2.h.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) + fullNode2.h.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) // Get stream - stream, err := fullNode2.h.NewStream(s.ctx, s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) + stream, err := fullNode2.h.NewStream(s.ctx, s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) // Prepare subscribe request @@ -344,7 +344,7 @@ func (s *FilterTestSuite) TestSubscribeFullNode2FullNode() { fullNode2.subscribe(s.ctx, stream, subscribeRequest) // Check the pubsub topic related to the first node is stored within the second node - pubsubTopics, hasTopics := fullNode2.subscriptions.Get(s.fullNodeHost.ID()) + pubsubTopics, hasTopics := fullNode2.subscriptions.Get(s.FullNodeHost.ID()) s.Require().True(hasTopics) // Check the pubsub topic is what we have set @@ -361,7 +361,7 @@ func (s *FilterTestSuite) TestIsSubscriptionAlive() { messages := s.prepareData(2, false, true, false, nil) // Subscribe with the first message only - s.subscribe(messages[0].PubSubTopic, messages[0].ContentTopic, s.fullNodeHost.ID()) + s.subscribe(messages[0].PubSubTopic, messages[0].ContentTopic, s.FullNodeHost.ID()) // IsSubscriptionAlive returns no error for the first message err := s.LightNode.IsSubscriptionAlive(s.ctx, s.subDetails[0]) @@ -384,16 +384,16 @@ func (s *FilterTestSuite) TestFilterSubscription() { contentFilter := protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} // Subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // Returns no error and SubscriptionDetails for existing subscription - _, err := s.LightNode.FilterSubscription(s.fullNodeHost.ID(), contentFilter) + _, err := s.LightNode.FilterSubscription(s.FullNodeHost.ID(), contentFilter) s.Require().NoError(err) otherFilter := protocol.ContentFilter{PubsubTopic: "34583495", ContentTopics: protocol.NewContentTopicSet("sjfa402")} // Returns error and nil SubscriptionDetails for non existent subscription - nonSubscription, err := s.LightNode.FilterSubscription(s.fullNodeHost.ID(), otherFilter) + nonSubscription, err := s.LightNode.FilterSubscription(s.FullNodeHost.ID(), otherFilter) s.Require().Error(err) s.Require().Nil(nonSubscription) @@ -412,19 +412,19 @@ func (s *FilterTestSuite) TestHandleFilterSubscribeOptions() { contentFilter := protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} // Subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // With valid peer - opts := []FilterSubscribeOption{WithPeer(s.fullNodeHost.ID())} + opts := []FilterSubscribeOption{WithPeer(s.FullNodeHost.ID())} // Positive case _, _, err := s.LightNode.handleFilterSubscribeOptions(s.ctx, contentFilter, opts) s.Require().NoError(err) - addr := s.fullNodeHost.Addrs()[0] + addr := s.FullNodeHost.Addrs()[0] // Combine mutually exclusive options - opts = []FilterSubscribeOption{WithPeer(s.fullNodeHost.ID()), WithPeerAddr(addr)} + opts = []FilterSubscribeOption{WithPeer(s.FullNodeHost.ID()), WithPeerAddr(addr)} // Should fail on wrong option combination _, _, err = s.LightNode.handleFilterSubscribeOptions(s.ctx, contentFilter, opts) diff --git a/waku/v2/protocol/filter/filter_test.go b/waku/v2/protocol/filter/filter_test.go index 4a93609ce..11bf31eae 100644 --- a/waku/v2/protocol/filter/filter_test.go +++ b/waku/v2/protocol/filter/filter_test.go @@ -25,14 +25,14 @@ func (s *FilterTestSuite) TestRunningGuard() { contentFilter := protocol.ContentFilter{PubsubTopic: "test", ContentTopics: protocol.NewContentTopicSet("test")} - _, err := s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err := s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().ErrorIs(err, service.ErrNotStarted) err = s.LightNode.Start(s.ctx) s.Require().NoError(err) - _, err = s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().NoError(err) } @@ -41,14 +41,14 @@ func (s *FilterTestSuite) TestFireAndForgetAndCustomWg() { contentFilter := protocol.ContentFilter{PubsubTopic: "test", ContentTopics: protocol.NewContentTopicSet("test")} - _, err := s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err := s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().NoError(err) result, err := s.LightNode.Unsubscribe(s.ctx, contentFilter, DontWait()) s.Require().NoError(err) s.Require().Equal(0, len(result.Errors())) - _, err = s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.LightNode.Subscribe(s.ctx, contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().NoError(err) wg := sync.WaitGroup{} @@ -109,18 +109,18 @@ func (s *FilterTestSuite) TestAutoShard() { s.StartLightNode() s.MakeWakuFilterFullNode(pubSubTopic.String(), false) - s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) - err = s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) + err = s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) s.Log.Info("Testing Autoshard:CreateSubscription") - s.subscribe("", s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe("", s.TestContentTopic, s.FullNodeHost.ID()) s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, ""}) // Wrong content topic s.waitForTimeout(&WakuMsg{s.TestTopic, "TopicB", "second"}) - _, err = s.LightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.LightNode.Unsubscribe(s.ctx, s.contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().NoError(err) time.Sleep(1 * time.Second) @@ -128,17 +128,17 @@ func (s *FilterTestSuite) TestAutoShard() { // Should not receive after unsubscribe s.waitForTimeout(&WakuMsg{s.TestTopic, s.TestContentTopic, "third"}) - s.subscribe("", s.TestContentTopic, s.fullNodeHost.ID()) + s.subscribe("", s.TestContentTopic, s.FullNodeHost.ID()) s.Log.Info("Testing Autoshard:SubscriptionPing") - err = s.LightNode.Ping(context.Background(), s.fullNodeHost.ID()) + err = s.LightNode.Ping(context.Background(), s.FullNodeHost.ID()) s.Require().NoError(err) // Test ModifySubscription Subscribe to another content_topic s.Log.Info("Testing Autoshard:ModifySubscription") newContentTopic := "0/test/1/testTopic1/proto" - s.subscribe("", newContentTopic, s.fullNodeHost.ID()) + s.subscribe("", newContentTopic, s.FullNodeHost.ID()) s.waitForMsg(&WakuMsg{s.TestTopic, newContentTopic, ""}) @@ -158,7 +158,7 @@ func (s *FilterTestSuite) TestLightNodeIsListening() { messages := s.prepareData(2, true, true, false, nil) // Subscribe with the first message only - s.subscribe(messages[0].PubSubTopic, messages[0].ContentTopic, s.fullNodeHost.ID()) + s.subscribe(messages[0].PubSubTopic, messages[0].ContentTopic, s.FullNodeHost.ID()) // IsListening returns true for the first message listenStatus := s.LightNode.IsListening(messages[0].PubSubTopic, messages[0].ContentTopic) diff --git a/waku/v2/protocol/filter/filter_unsubscribe_test.go b/waku/v2/protocol/filter/filter_unsubscribe_test.go index c32e0f25f..48427ccfe 100644 --- a/waku/v2/protocol/filter/filter_unsubscribe_test.go +++ b/waku/v2/protocol/filter/filter_unsubscribe_test.go @@ -18,8 +18,8 @@ func (s *FilterTestSuite) TestUnsubscribeSingleContentTopic() { var newContentTopic = "TopicB" // Initial subscribe - s.subscribe(s.TestTopic, s.TestContentTopic, s.fullNodeHost.ID()) - s.subscribe(s.TestTopic, newContentTopic, s.fullNodeHost.ID()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) + s.subscribe(s.TestTopic, newContentTopic, s.FullNodeHost.ID()) // Message is possible to receive for original contentTopic s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, "test_msg"}) @@ -27,7 +27,7 @@ func (s *FilterTestSuite) TestUnsubscribeSingleContentTopic() { // Message is possible to receive for new contentTopic s.waitForMsg(&WakuMsg{s.TestTopic, newContentTopic, "test_msg"}) - _ = s.unsubscribe(s.TestTopic, newContentTopic, s.fullNodeHost.ID()) + _ = s.unsubscribe(s.TestTopic, newContentTopic, s.FullNodeHost.ID()) // Message should not be received for new contentTopic as it was unsubscribed s.waitForTimeout(&WakuMsg{s.TestTopic, newContentTopic, "test_msg"}) @@ -46,7 +46,7 @@ func (s *FilterTestSuite) TestUnsubscribeMultiContentTopic() { // Subscribe with 3 content topics for _, m := range messages { - s.subscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) + s.subscribe(m.PubSubTopic, m.ContentTopic, s.FullNodeHost.ID()) } // All messages should be received @@ -54,7 +54,7 @@ func (s *FilterTestSuite) TestUnsubscribeMultiContentTopic() { // Unsubscribe with the last 2 content topics for _, m := range messages[1:] { - _ = s.unsubscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) + _ = s.unsubscribe(m.PubSubTopic, m.ContentTopic, s.FullNodeHost.ID()) } // Messages should not be received for the last two contentTopics as it was unsubscribed @@ -80,15 +80,15 @@ func (s *FilterTestSuite) TestUnsubscribeMultiPubSubMultiContentTopic() { s.MakeWakuFilterFullNode(s.TestTopic, true) // Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNode.h), peerstore.PermanentAddrTTL) - err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.fullNode.h), peerstore.PermanentAddrTTL) + err := s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) messages := s.prepareData(2, true, true, true, nil) // Subscribe for _, m := range messages { - s.subDetails = append(s.subDetails, s.getSub(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID())...) + s.subDetails = append(s.subDetails, s.getSub(m.PubSubTopic, m.ContentTopic, s.FullNodeHost.ID())...) _, err = s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.PubSubTopic)) s.Require().NoError(err) } @@ -98,7 +98,7 @@ func (s *FilterTestSuite) TestUnsubscribeMultiPubSubMultiContentTopic() { // Unsubscribe for _, m := range messages { - _ = s.unsubscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) + _ = s.unsubscribe(m.PubSubTopic, m.ContentTopic, s.FullNodeHost.ID()) } // No messages can be received with previous subscriptions @@ -117,8 +117,8 @@ func (s *FilterTestSuite) TestUnsubscribeErrorHandling() { s.MakeWakuFilterFullNode(s.TestTopic, true) // Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) - err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) + err := s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) var messages, invalidMessages []WakuMsg @@ -145,7 +145,7 @@ func (s *FilterTestSuite) TestUnsubscribeErrorHandling() { // Subscribe with valid topics for _, m := range messages { - s.subscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) + s.subscribe(m.PubSubTopic, m.ContentTopic, s.FullNodeHost.ID()) _, err = s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.PubSubTopic)) s.Require().NoError(err) } @@ -156,19 +156,19 @@ func (s *FilterTestSuite) TestUnsubscribeErrorHandling() { // Unsubscribe with empty pubsub contentFilter := protocol.ContentFilter{PubsubTopic: invalidMessages[0].PubSubTopic, ContentTopics: protocol.NewContentTopicSet(invalidMessages[0].ContentTopic)} - _, err = s.LightNode.Unsubscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.LightNode.Unsubscribe(s.ctx, contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().Error(err) // Unsubscribe with empty content topic contentFilter = protocol.ContentFilter{PubsubTopic: invalidMessages[1].PubSubTopic, ContentTopics: protocol.NewContentTopicSet(invalidMessages[1].ContentTopic)} - _, err = s.LightNode.Unsubscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.LightNode.Unsubscribe(s.ctx, contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().Error(err) // Unsubscribe with non-existent topics, expect no error to prevent attacker from topic guessing contentFilter = protocol.ContentFilter{PubsubTopic: invalidMessages[2].PubSubTopic, ContentTopics: protocol.NewContentTopicSet(invalidMessages[2].ContentTopic)} - _, err = s.LightNode.Unsubscribe(s.ctx, contentFilter, WithPeer(s.fullNodeHost.ID())) + _, err = s.LightNode.Unsubscribe(s.ctx, contentFilter, WithPeer(s.FullNodeHost.ID())) s.Require().NoError(err) // All messages should be still possible to receive for subscribed topics @@ -185,14 +185,14 @@ func (s *FilterTestSuite) TestUnsubscribeAllWithoutContentTopics() { // Subscribe with 2 content topics for _, m := range messages { - s.subscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) + s.subscribe(m.PubSubTopic, m.ContentTopic, s.FullNodeHost.ID()) } // All messages should be received s.waitForMessages(messages) // Unsubscribe all with peer specification - _, err := s.LightNode.UnsubscribeAll(s.ctx, WithPeer(s.fullNodeHost.ID())) + _, err := s.LightNode.UnsubscribeAll(s.ctx, WithPeer(s.FullNodeHost.ID())) s.Require().NoError(err) // Messages should not be received for any contentTopics @@ -211,15 +211,15 @@ func (s *FilterTestSuite) TestUnsubscribeAllDiffPubSubContentTopics() { s.MakeWakuFilterFullNode(s.TestTopic, true) // Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNode.h), peerstore.PermanentAddrTTL) - err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.fullNode.h), peerstore.PermanentAddrTTL) + err := s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) messages := s.prepareData(2, true, true, true, nil) // Subscribe for _, m := range messages { - s.subDetails = append(s.subDetails, s.getSub(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID())...) + s.subDetails = append(s.subDetails, s.getSub(m.PubSubTopic, m.ContentTopic, s.FullNodeHost.ID())...) _, err = s.relayNode.Subscribe(context.Background(), protocol.NewContentFilter(m.PubSubTopic)) s.Require().NoError(err) } @@ -244,7 +244,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllUnrelatedPeer() { // Subscribe with 2 content topics for _, m := range messages { - s.subscribe(m.PubSubTopic, m.ContentTopic, s.fullNodeHost.ID()) + s.subscribe(m.PubSubTopic, m.ContentTopic, s.FullNodeHost.ID()) } // All messages should be received @@ -254,7 +254,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllUnrelatedPeer() { host, err := tests.MakeHost(context.Background(), 12345, rand.Reader) s.Require().NoError(err) - s.Log.Info("Host ID", logging.HostID("FullNode", s.fullNodeHost.ID())) + s.Log.Info("Host ID", logging.HostID("FullNode", s.FullNodeHost.ID())) s.Log.Info("Host ID", logging.HostID("LightNode", s.lightNodeHost.ID())) s.Log.Info("Host ID", logging.HostID("Unrelated", host.ID())) diff --git a/waku/v2/protocol/filter/test_utils.go b/waku/v2/protocol/filter/test_utils.go index 3d7f67a4d..22594161f 100644 --- a/waku/v2/protocol/filter/test_utils.go +++ b/waku/v2/protocol/filter/test_utils.go @@ -30,7 +30,7 @@ type LightNodeData struct { type FullNodeData struct { relayNode *relay.WakuRelay RelaySub *relay.Subscription - fullNodeHost host.Host + FullNodeHost host.Host Broadcaster relay.Broadcaster fullNode *WakuFilterFullNode } @@ -85,8 +85,8 @@ func (s *FilterTestSuite) SetupTest() { s.MakeWakuFilterFullNode(s.TestTopic, false) // Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) - err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) + err := s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) } @@ -127,7 +127,7 @@ func (s *FilterTestSuite) GetWakuFilterFullNode(topic string, withRegisterAll bo nodeData := s.GetWakuRelay(topic) node2Filter := NewWakuFilterFullNode(timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.Log) - node2Filter.SetHost(nodeData.fullNodeHost) + node2Filter.SetHost(nodeData.FullNodeHost) var sub *relay.Subscription if withRegisterAll { From d5be82ddebdc29ff0c4037d9ed4a765d3876fa0f Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Mon, 8 Apr 2024 19:27:32 +0300 Subject: [PATCH 11/27] Add ConnectHosts() --- waku/v2/protocol/filter/test_utils.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/waku/v2/protocol/filter/test_utils.go b/waku/v2/protocol/filter/test_utils.go index 22594161f..ee88c3584 100644 --- a/waku/v2/protocol/filter/test_utils.go +++ b/waku/v2/protocol/filter/test_utils.go @@ -84,10 +84,7 @@ func (s *FilterTestSuite) SetupTest() { s.MakeWakuFilterFullNode(s.TestTopic, false) - // Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) - err := s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) - s.Require().NoError(err) + s.ConnectHosts(s.lightNodeHost, s.FullNodeHost) } @@ -99,6 +96,12 @@ func (s *FilterTestSuite) TearDownTest() { s.ctxCancel() } +func (s *FilterTestSuite) ConnectHosts(h1, h2 host.Host) { + h1.Peerstore().AddAddr(h2.ID(), tests.GetHostAddress(h2), peerstore.PermanentAddrTTL) + err := h1.Peerstore().AddProtocols(h2.ID(), FilterSubscribeID_v20beta1) + s.Require().NoError(err) +} + func (s *FilterTestSuite) GetWakuRelay(topic string) FullNodeData { broadcaster := relay.NewBroadcaster(10) From b4cd9cf5e0ec9d1f51ce6234593f7ca3cccc1937 Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Mon, 8 Apr 2024 19:33:52 +0300 Subject: [PATCH 12/27] LightNodeHost --- waku/v2/api/filter_test.go | 10 ++++++++-- waku/v2/protocol/filter/filter_proto_ident_test.go | 6 +++--- waku/v2/protocol/filter/filter_subscribe_test.go | 8 ++++---- waku/v2/protocol/filter/filter_test.go | 4 ++-- waku/v2/protocol/filter/filter_unsubscribe_test.go | 14 +++++++------- waku/v2/protocol/filter/test_utils.go | 4 ++-- 6 files changed, 26 insertions(+), 20 deletions(-) diff --git a/waku/v2/api/filter_test.go b/waku/v2/api/filter_test.go index 966f4ca01..844d292b6 100644 --- a/waku/v2/api/filter_test.go +++ b/waku/v2/api/filter_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/protocol/filter" + "go.uber.org/zap" ) func TestFilterApiSuite(t *testing.T) { @@ -29,9 +30,14 @@ func (s *FilterApiTestSuite) TearDownTest() { func (s *FilterApiTestSuite) TestSubscribe() { contentFilter := protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} - apiConfig := FilterConfig{MaxPeers: 1, Peers: []peer.ID{s.FullNodeHost.ID()}} - s.Require().Equal(apiConfig.MaxPeers, 1) + fullNodeData2 := s.GetWakuFilterFullNode(s.TestTopic, true) + s.ConnectHosts(s.LightNodeHost, fullNodeData2.FullNodeHost) + peers := []peer.ID{s.FullNodeHost.ID(), fullNodeData2.FullNodeHost.ID()} + s.Log.Info("FullNodeHost IDs:", zap.Any(peers)) + apiConfig := FilterConfig{MaxPeers: 2, Peers: peers} + + s.Require().Equal(apiConfig.MaxPeers, 2) s.Require().Equal(contentFilter.PubsubTopic, s.TestTopic) s.Log.Info("About to perform API Subscribe()") diff --git a/waku/v2/protocol/filter/filter_proto_ident_test.go b/waku/v2/protocol/filter/filter_proto_ident_test.go index dba82e87b..549071a13 100644 --- a/waku/v2/protocol/filter/filter_proto_ident_test.go +++ b/waku/v2/protocol/filter/filter_proto_ident_test.go @@ -217,7 +217,7 @@ func (s *FilterTestSuite) TestIncorrectSubscribeIdentifier() { s.MakeWakuFilterFullNode(s.TestTopic, false) //Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) + s.LightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) // Subscribe with incorrect SubscribeID s.contentFilter = protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} @@ -261,8 +261,8 @@ func (s *FilterTestSuite) TestIncorrectPushIdentifier() { s.Require().NoError(err) // Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) - err = s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.LightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) + err = s.LightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) // Subscribe diff --git a/waku/v2/protocol/filter/filter_subscribe_test.go b/waku/v2/protocol/filter/filter_subscribe_test.go index 9e1964bce..17e23a276 100644 --- a/waku/v2/protocol/filter/filter_subscribe_test.go +++ b/waku/v2/protocol/filter/filter_subscribe_test.go @@ -108,8 +108,8 @@ func (s *FilterTestSuite) TestMultiPubSubMultiContentTopic() { s.MakeWakuFilterFullNode(s.TestTopic, true) // Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) - err := s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.LightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) + err := s.LightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) messages := s.prepareData(2, true, true, false, nil) @@ -261,8 +261,8 @@ func (s *FilterTestSuite) TestMultipleFullNodeSubscriptions() { s.MakeWakuFilterFullNode(s.TestTopic, false) // Connect to second full and relay node - s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) - err := s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.LightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) + err := s.LightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) fullNodeIDHex = make([]byte, hex.EncodedLen(len([]byte(s.FullNodeHost.ID())))) diff --git a/waku/v2/protocol/filter/filter_test.go b/waku/v2/protocol/filter/filter_test.go index 11bf31eae..c5adb77f8 100644 --- a/waku/v2/protocol/filter/filter_test.go +++ b/waku/v2/protocol/filter/filter_test.go @@ -109,8 +109,8 @@ func (s *FilterTestSuite) TestAutoShard() { s.StartLightNode() s.MakeWakuFilterFullNode(pubSubTopic.String(), false) - s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) - err = s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.LightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) + err = s.LightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) s.Log.Info("Testing Autoshard:CreateSubscription") diff --git a/waku/v2/protocol/filter/filter_unsubscribe_test.go b/waku/v2/protocol/filter/filter_unsubscribe_test.go index 48427ccfe..cac86615c 100644 --- a/waku/v2/protocol/filter/filter_unsubscribe_test.go +++ b/waku/v2/protocol/filter/filter_unsubscribe_test.go @@ -80,8 +80,8 @@ func (s *FilterTestSuite) TestUnsubscribeMultiPubSubMultiContentTopic() { s.MakeWakuFilterFullNode(s.TestTopic, true) // Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.fullNode.h), peerstore.PermanentAddrTTL) - err := s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.LightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.fullNode.h), peerstore.PermanentAddrTTL) + err := s.LightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) messages := s.prepareData(2, true, true, true, nil) @@ -117,8 +117,8 @@ func (s *FilterTestSuite) TestUnsubscribeErrorHandling() { s.MakeWakuFilterFullNode(s.TestTopic, true) // Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) - err := s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.LightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.FullNodeHost), peerstore.PermanentAddrTTL) + err := s.LightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) var messages, invalidMessages []WakuMsg @@ -211,8 +211,8 @@ func (s *FilterTestSuite) TestUnsubscribeAllDiffPubSubContentTopics() { s.MakeWakuFilterFullNode(s.TestTopic, true) // Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.fullNode.h), peerstore.PermanentAddrTTL) - err := s.lightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) + s.LightNodeHost.Peerstore().AddAddr(s.FullNodeHost.ID(), tests.GetHostAddress(s.fullNode.h), peerstore.PermanentAddrTTL) + err := s.LightNodeHost.Peerstore().AddProtocols(s.FullNodeHost.ID(), FilterSubscribeID_v20beta1) s.Require().NoError(err) messages := s.prepareData(2, true, true, true, nil) @@ -255,7 +255,7 @@ func (s *FilterTestSuite) TestUnsubscribeAllUnrelatedPeer() { s.Require().NoError(err) s.Log.Info("Host ID", logging.HostID("FullNode", s.FullNodeHost.ID())) - s.Log.Info("Host ID", logging.HostID("LightNode", s.lightNodeHost.ID())) + s.Log.Info("Host ID", logging.HostID("LightNode", s.LightNodeHost.ID())) s.Log.Info("Host ID", logging.HostID("Unrelated", host.ID())) // Unsubscribe all with unrelated peer specification diff --git a/waku/v2/protocol/filter/test_utils.go b/waku/v2/protocol/filter/test_utils.go index ee88c3584..819d8e329 100644 --- a/waku/v2/protocol/filter/test_utils.go +++ b/waku/v2/protocol/filter/test_utils.go @@ -24,7 +24,7 @@ import ( type LightNodeData struct { LightNode *WakuFilterLightNode - lightNodeHost host.Host + LightNodeHost host.Host } type FullNodeData struct { @@ -84,7 +84,7 @@ func (s *FilterTestSuite) SetupTest() { s.MakeWakuFilterFullNode(s.TestTopic, false) - s.ConnectHosts(s.lightNodeHost, s.FullNodeHost) + s.ConnectHosts(s.LightNodeHost, s.FullNodeHost) } From 5d9fb917479e2c21e7055aa721e32925a08f4c4a Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Fri, 19 Apr 2024 12:35:34 +0300 Subject: [PATCH 13/27] Add Makefile targets for filter tests --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 554461bd5..993faca83 100644 --- a/Makefile +++ b/Makefile @@ -220,3 +220,8 @@ test-postgres: test-postgres-with-race: ${GOBIN} test -race -p 1 -v -count 1 -tags="${PG_BUILD_TAGS}" github.com/waku-org/go-waku/waku/persistence/... +test-filter: + ${GOBIN} test -v github.com/waku-org/go-waku/waku/v2/protocol/filter -run TestFilterSuite -count=1 + +test-filter-api: + ${GOBIN} test -v github.com/waku-org/go-waku/waku/v2/api -run TestFilterApiSuite From 7a908f5950b1cb2769153dbcbb6d883f81581b7a Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Fri, 19 Apr 2024 12:35:57 +0300 Subject: [PATCH 14/27] Test updates --- waku/v2/api/filter.go | 7 ++++++- waku/v2/api/filter_test.go | 23 ++++++++++++++++++++++- waku/v2/protocol/filter/server.go | 2 +- waku/v2/protocol/filter/test_utils.go | 6 +++--- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/waku/v2/api/filter.go b/waku/v2/api/filter.go index 56fce7e75..334a5fa41 100644 --- a/waku/v2/api/filter.go +++ b/waku/v2/api/filter.go @@ -9,6 +9,7 @@ import ( "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/protocol/filter" "github.com/waku-org/go-waku/waku/v2/protocol/subscription" + "go.uber.org/zap" "golang.org/x/exp/maps" ) @@ -26,6 +27,7 @@ type Sub struct { wf *filter.WakuFilterLightNode ctx context.Context cancel context.CancelFunc + log *zap.Logger } func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilter protocol.ContentFilter, config FilterConfig) (*Sub, error) { @@ -36,6 +38,7 @@ func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilte sub.DataCh = make(chan *protocol.Envelope) sub.ContentFilter = contentFilter sub.Config = config + sub.log = func() *zap.Logger { log, _ := zap.NewDevelopment(); return log }().Named("filterv2-api") err := sub.subscribe(contentFilter, sub.Config.MaxPeers) @@ -47,13 +50,14 @@ func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilte } } -func Unsubscribe(apiSub *Sub) error { +func (apiSub *Sub) Unsubscribe() error { apiSub.RLock() defer apiSub.RUnlock() for _, s := range apiSub.subs { apiSub.wf.UnsubscribeWithSubscription(apiSub.ctx, s) } apiSub.cancel() + close(apiSub.DataCh) return nil } @@ -140,6 +144,7 @@ func (apiSub *Sub) subscribe(contentFilter protocol.ContentFilter, peerCount int // Goroutines will exit once sub channels are closed for _, subDetails := range subs { go func(subDetails *subscription.SubscriptionDetails) { + apiSub.log.Info("New multiplex", zap.String("sub ID", subDetails.ID)) for env := range subDetails.C { apiSub.DataCh <- env } diff --git a/waku/v2/api/filter_test.go b/waku/v2/api/filter_test.go index 844d292b6..0a0df1970 100644 --- a/waku/v2/api/filter_test.go +++ b/waku/v2/api/filter_test.go @@ -9,6 +9,7 @@ import ( "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/protocol/filter" "go.uber.org/zap" + "golang.org/x/exp/maps" ) func TestFilterApiSuite(t *testing.T) { @@ -31,10 +32,14 @@ func (s *FilterApiTestSuite) TearDownTest() { func (s *FilterApiTestSuite) TestSubscribe() { contentFilter := protocol.ContentFilter{PubsubTopic: s.TestTopic, ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic)} + // We have one full node already created in SetupTest(), + // create another one fullNodeData2 := s.GetWakuFilterFullNode(s.TestTopic, true) s.ConnectHosts(s.LightNodeHost, fullNodeData2.FullNodeHost) peers := []peer.ID{s.FullNodeHost.ID(), fullNodeData2.FullNodeHost.ID()} - s.Log.Info("FullNodeHost IDs:", zap.Any(peers)) + s.Log.Info("FullNodeHost IDs:", zap.Any("peers", peers)) + // Make sure IDs are different + s.Require().True(peers[0] != peers[1]) apiConfig := FilterConfig{MaxPeers: 2, Peers: peers} s.Require().Equal(apiConfig.MaxPeers, 2) @@ -45,4 +50,20 @@ func (s *FilterApiTestSuite) TestSubscribe() { s.Require().NoError(err) s.Require().Equal(apiSub.ContentFilter, contentFilter) s.Log.Info("Subscribed") + + s.Require().Len(apiSub.subs, 2) + for sub := range apiSub.subs { + s.Log.Info("SubDetails:", zap.String("id", sub)) + } + s.Require().True(maps.Keys(apiSub.subs)[0] != maps.Keys(apiSub.subs)[1]) + // Publish msg and confirm it's received twice because of multiplexing + s.PublishMsg(&filter.WakuMsg{s.TestTopic, s.TestContentTopic, "Test msg"}) + cnt := 0 + for msg := range apiSub.DataCh { + s.Log.Info("Received msg:", zap.Int("cnt", cnt), zap.String("payload", string(msg.Message().Payload))) + cnt++ + } + s.Require().Equal(cnt, 2) + + apiSub.Unsubscribe() } diff --git a/waku/v2/protocol/filter/server.go b/waku/v2/protocol/filter/server.go index 29f38e12d..23dc03ae0 100644 --- a/waku/v2/protocol/filter/server.go +++ b/waku/v2/protocol/filter/server.go @@ -128,7 +128,7 @@ func (wf *WakuFilterFullNode) onRequest(ctx context.Context) func(network.Stream wf.metrics.RecordRequest(subscribeRequest.FilterSubscribeType.String(), time.Since(start)) - logger.Info("received request", zap.String("requestType", subscribeRequest.FilterSubscribeType.String())) + logger.Info("received request", zap.String("serverID", wf.h.ID().String()), zap.String("requestType", subscribeRequest.FilterSubscribeType.String())) } } diff --git a/waku/v2/protocol/filter/test_utils.go b/waku/v2/protocol/filter/test_utils.go index 819d8e329..fbb1c9dbf 100644 --- a/waku/v2/protocol/filter/test_utils.go +++ b/waku/v2/protocol/filter/test_utils.go @@ -201,7 +201,7 @@ func (s *FilterTestSuite) waitForMsgFromChan(msg *WakuMsg, ch chan *protocol.Env }() if msg != nil { - s.publishMsg(msg) + s.PublishMsg(msg) } s.wg.Wait() @@ -283,7 +283,7 @@ func (s *FilterTestSuite) waitForTimeoutFromChan(msg *WakuMsg, ch chan *protocol } }() - s.publishMsg(msg) + s.PublishMsg(msg) s.wg.Wait() } @@ -333,7 +333,7 @@ func (s *FilterTestSuite) unsubscribe(pubsubTopic string, contentTopic string, p return s.LightNode.Subscriptions() } -func (s *FilterTestSuite) publishMsg(msg *WakuMsg) { +func (s *FilterTestSuite) PublishMsg(msg *WakuMsg) { if len(msg.Payload) == 0 { msg.Payload = "123" } From d57c9a3cf72e568844107841efd7fa25e02ab304 Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Fri, 19 Apr 2024 15:40:45 +0300 Subject: [PATCH 15/27] More logs --- waku/v2/api/filter.go | 71 +++++++++++++++++++++++--------------- waku/v2/api/filter_test.go | 9 ++++- 2 files changed, 51 insertions(+), 29 deletions(-) diff --git a/waku/v2/api/filter.go b/waku/v2/api/filter.go index 334a5fa41..5f629a0c0 100644 --- a/waku/v2/api/filter.go +++ b/waku/v2/api/filter.go @@ -43,6 +43,7 @@ func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilte err := sub.subscribe(contentFilter, sub.Config.MaxPeers) if err == nil { + sub.log.Info("go sub.healthCheckLoop()") go sub.healthCheckLoop() return sub, nil } else { @@ -51,8 +52,8 @@ func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilte } func (apiSub *Sub) Unsubscribe() error { - apiSub.RLock() - defer apiSub.RUnlock() + apiSub.Lock() + defer apiSub.Unlock() for _, s := range apiSub.subs { apiSub.wf.UnsubscribeWithSubscription(apiSub.ctx, s) } @@ -68,42 +69,38 @@ func (apiSub *Sub) healthCheckLoop() { for { select { case <-apiSub.ctx.Done(): + apiSub.log.Info("healthCheckLoop: Done()") return case <-ticker.C: - // Returns a map of pubsub topics to peer counts - m := apiSub.checkAliveness() - for t, cnt := range m { - if cnt < apiSub.Config.MaxPeers { - cFilter := protocol.ContentFilter{t, apiSub.ContentFilter.ContentTopics} - apiSub.subscribe(cFilter, apiSub.Config.MaxPeers-cnt) - } - } + apiSub.log.Info("healthCheckLoop: checkAliveness()") + apiSub.checkAliveness() } } } -func (apiSub *Sub) checkAliveness() map[string]int { +func (apiSub *Sub) checkAliveness() { + apiSub.log.Info("ENTER checkAliveness()") apiSub.RLock() defer apiSub.RUnlock() - // Only healthy topics will be pushed here - ch := make(chan string) + // Buffered chan for sub aliveness results + type CheckResult struct { + sub *subscription.SubscriptionDetails + alive bool + } + ch := make(chan CheckResult, len(apiSub.subs)) wg := &sync.WaitGroup{} + + // Run pings asynchronously wg.Add(len(apiSub.subs)) for _, subDetails := range apiSub.subs { go func(subDetails *subscription.SubscriptionDetails) { defer wg.Done() - err := apiSub.wf.IsSubscriptionAlive(apiSub.ctx, subDetails) - - if err != nil { - subDetails.Close() - apiSub.Lock() - defer apiSub.Unlock() - delete(apiSub.subs, subDetails.ID) - } else { - ch <- subDetails.ContentFilter.PubsubTopic - } + ctx, _ := context.WithTimeout(apiSub.ctx, 5*time.Second) + err := apiSub.wf.IsSubscriptionAlive(ctx, subDetails) + + ch <- CheckResult{subDetails, err == nil} }(subDetails) } @@ -115,13 +112,31 @@ func (apiSub *Sub) checkAliveness() map[string]int { for _, t := range maps.Keys(topicMap) { m[t] = 0 } - for t := range ch { - m[t]++ + // Close inactive subs + for s := range ch { + if !s.alive { + s.sub.Close() + delete(apiSub.subs, s.sub.ID) + } else { + m[s.sub.ContentFilter.PubsubTopic]++ + } } + // Re-subscribe asynchronously + for t, cnt := range m { + if cnt < apiSub.Config.MaxPeers { + wg.Add(1) + cFilter := protocol.ContentFilter{t, apiSub.ContentFilter.ContentTopics} + go func() { + defer wg.Done() + apiSub.subscribe(cFilter, apiSub.Config.MaxPeers-cnt) + }() + } + } + wg.Wait() - return m - + apiSub.log.Info("EXIT checkAliveness()") } + func (apiSub *Sub) subscribe(contentFilter protocol.ContentFilter, peerCount int) error { // Low-level subscribe, returns a set of SubscriptionDetails options := make([]filter.FilterSubscribeOption, 0) @@ -144,7 +159,7 @@ func (apiSub *Sub) subscribe(contentFilter protocol.ContentFilter, peerCount int // Goroutines will exit once sub channels are closed for _, subDetails := range subs { go func(subDetails *subscription.SubscriptionDetails) { - apiSub.log.Info("New multiplex", zap.String("sub ID", subDetails.ID)) + apiSub.log.Info("New multiplex", zap.String("subID", subDetails.ID)) for env := range subDetails.C { apiSub.DataCh <- env } diff --git a/waku/v2/api/filter_test.go b/waku/v2/api/filter_test.go index 0a0df1970..0ce1585e0 100644 --- a/waku/v2/api/filter_test.go +++ b/waku/v2/api/filter_test.go @@ -3,6 +3,7 @@ package api import ( "context" "testing" + "time" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/suite" @@ -62,8 +63,14 @@ func (s *FilterApiTestSuite) TestSubscribe() { for msg := range apiSub.DataCh { s.Log.Info("Received msg:", zap.Int("cnt", cnt), zap.String("payload", string(msg.Message().Payload))) cnt++ + break } - s.Require().Equal(cnt, 2) + s.Require().Equal(cnt, 1) + time.Sleep(10 * time.Second) apiSub.Unsubscribe() + for _ = range apiSub.DataCh { + } + s.Log.Info("DataCh is closed") + } From 194bd7fb24b32ca7244be579f2f5afd3d7c5a155 Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Fri, 26 Apr 2024 12:31:04 +0300 Subject: [PATCH 16/27] Remove locks/waitgroups --- waku/v2/api/filter.go | 94 +++++++++++++++++++++----------------- waku/v2/api/filter_test.go | 2 - 2 files changed, 53 insertions(+), 43 deletions(-) diff --git a/waku/v2/api/filter.go b/waku/v2/api/filter.go index 5f629a0c0..c51b4f630 100644 --- a/waku/v2/api/filter.go +++ b/waku/v2/api/filter.go @@ -2,7 +2,6 @@ package api import ( "context" - "sync" "time" "github.com/libp2p/go-libp2p/core/peer" @@ -19,7 +18,6 @@ type FilterConfig struct { } type Sub struct { - sync.RWMutex ContentFilter protocol.ContentFilter DataCh chan *protocol.Envelope Config FilterConfig @@ -40,9 +38,10 @@ func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilte sub.Config = config sub.log = func() *zap.Logger { log, _ := zap.NewDevelopment(); return log }().Named("filterv2-api") - err := sub.subscribe(contentFilter, sub.Config.MaxPeers) + subs, err := sub.subscribe(contentFilter, sub.Config.MaxPeers) if err == nil { + sub.multiplex(subs) sub.log.Info("go sub.healthCheckLoop()") go sub.healthCheckLoop() return sub, nil @@ -51,15 +50,13 @@ func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilte } } -func (apiSub *Sub) Unsubscribe() error { - apiSub.Lock() - defer apiSub.Unlock() +func (apiSub *Sub) Unsubscribe() { + apiSub.cancel() for _, s := range apiSub.subs { apiSub.wf.UnsubscribeWithSubscription(apiSub.ctx, s) } - apiSub.cancel() close(apiSub.DataCh) - return nil + } func (apiSub *Sub) healthCheckLoop() { @@ -76,12 +73,11 @@ func (apiSub *Sub) healthCheckLoop() { apiSub.checkAliveness() } } + } func (apiSub *Sub) checkAliveness() { apiSub.log.Info("ENTER checkAliveness()") - apiSub.RLock() - defer apiSub.RUnlock() // Buffered chan for sub aliveness results type CheckResult struct { @@ -90,54 +86,69 @@ func (apiSub *Sub) checkAliveness() { } ch := make(chan CheckResult, len(apiSub.subs)) - wg := &sync.WaitGroup{} - // Run pings asynchronously - wg.Add(len(apiSub.subs)) - for _, subDetails := range apiSub.subs { - go func(subDetails *subscription.SubscriptionDetails) { - defer wg.Done() + for _, s := range apiSub.subs { + go func() { ctx, _ := context.WithTimeout(apiSub.ctx, 5*time.Second) - err := apiSub.wf.IsSubscriptionAlive(ctx, subDetails) - - ch <- CheckResult{subDetails, err == nil} - }(subDetails) + err := apiSub.wf.IsSubscriptionAlive(ctx, s) + ch <- CheckResult{s, err == nil} + }() } - wg.Wait() - close(ch) - // Collect healthy topics - m := make(map[string]int) + + // Collect healthy topic counts + topicCounts := make(map[string]int) + topicMap, _ := protocol.ContentFilterToPubSubTopicMap(apiSub.ContentFilter) for _, t := range maps.Keys(topicMap) { - m[t] = 0 + topicCounts[t] = 0 } // Close inactive subs + cnt := 0 for s := range ch { + cnt++ if !s.alive { s.sub.Close() delete(apiSub.subs, s.sub.ID) } else { - m[s.sub.ContentFilter.PubsubTopic]++ + topicCounts[s.sub.ContentFilter.PubsubTopic]++ + } + + if cnt == len(apiSub.subs) { + // All values received + break + } + } + close(ch) + for t, cnt := range topicCounts { + if cnt == apiSub.Config.MaxPeers { + delete(topicCounts, t) } } // Re-subscribe asynchronously - for t, cnt := range m { - if cnt < apiSub.Config.MaxPeers { - wg.Add(1) - cFilter := protocol.ContentFilter{t, apiSub.ContentFilter.ContentTopics} - go func() { - defer wg.Done() - apiSub.subscribe(cFilter, apiSub.Config.MaxPeers-cnt) - }() + newSubs := make(chan []*subscription.SubscriptionDetails) + for t, cnt := range topicCounts { + cFilter := protocol.ContentFilter{t, apiSub.ContentFilter.ContentTopics} + go func() { + subs, err := apiSub.subscribe(cFilter, apiSub.Config.MaxPeers-cnt) + if err != nil { + newSubs <- subs + } + }() + } + + for subs := range newSubs { + apiSub.multiplex(subs) + if cnt == len(topicCounts) { + break } } - wg.Wait() + close(newSubs) apiSub.log.Info("EXIT checkAliveness()") } -func (apiSub *Sub) subscribe(contentFilter protocol.ContentFilter, peerCount int) error { +func (apiSub *Sub) subscribe(contentFilter protocol.ContentFilter, peerCount int) ([]*subscription.SubscriptionDetails, error) { // Low-level subscribe, returns a set of SubscriptionDetails options := make([]filter.FilterSubscribeOption, 0) options = append(options, filter.WithMaxPeersPerContentFilter(int(peerCount))) @@ -148,10 +159,13 @@ func (apiSub *Sub) subscribe(contentFilter protocol.ContentFilter, peerCount int if err != nil { // TODO what if fails? - return err + return nil, err } - apiSub.Lock() - defer apiSub.Unlock() + + return subs, nil +} + +func (apiSub *Sub) multiplex(subs []*subscription.SubscriptionDetails) { for _, s := range subs { apiSub.subs[s.ID] = s } @@ -165,6 +179,4 @@ func (apiSub *Sub) subscribe(contentFilter protocol.ContentFilter, peerCount int } }(subDetails) } - return nil - } diff --git a/waku/v2/api/filter_test.go b/waku/v2/api/filter_test.go index 0ce1585e0..ee1d5e719 100644 --- a/waku/v2/api/filter_test.go +++ b/waku/v2/api/filter_test.go @@ -3,7 +3,6 @@ package api import ( "context" "testing" - "time" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/suite" @@ -67,7 +66,6 @@ func (s *FilterApiTestSuite) TestSubscribe() { } s.Require().Equal(cnt, 1) - time.Sleep(10 * time.Second) apiSub.Unsubscribe() for _ = range apiSub.DataCh { } From 8f8c1ded7f709eb1659084951da6b9b4e8ead5b6 Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Mon, 6 May 2024 13:56:19 +0300 Subject: [PATCH 17/27] Refactor --- waku/v2/api/filter.go | 85 ++++++++++++++++++++++++++++---------- waku/v2/api/filter_test.go | 2 + 2 files changed, 66 insertions(+), 21 deletions(-) diff --git a/waku/v2/api/filter.go b/waku/v2/api/filter.go index c51b4f630..0c916a8c3 100644 --- a/waku/v2/api/filter.go +++ b/waku/v2/api/filter.go @@ -28,6 +28,7 @@ type Sub struct { log *zap.Logger } +// Subscribe func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilter protocol.ContentFilter, config FilterConfig) (*Sub, error) { sub := new(Sub) sub.wf = wf @@ -36,7 +37,10 @@ func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilte sub.DataCh = make(chan *protocol.Envelope) sub.ContentFilter = contentFilter sub.Config = config - sub.log = func() *zap.Logger { log, _ := zap.NewDevelopment(); return log }().Named("filterv2-api") + sub.log = func() *zap.Logger { + log, _ := zap.NewDevelopment() + return log + }().Named("filterv2-api") subs, err := sub.subscribe(contentFilter, sub.Config.MaxPeers) @@ -52,10 +56,6 @@ func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilte func (apiSub *Sub) Unsubscribe() { apiSub.cancel() - for _, s := range apiSub.subs { - apiSub.wf.UnsubscribeWithSubscription(apiSub.ctx, s) - } - close(apiSub.DataCh) } @@ -67,24 +67,43 @@ func (apiSub *Sub) healthCheckLoop() { select { case <-apiSub.ctx.Done(): apiSub.log.Info("healthCheckLoop: Done()") + apiSub.cleanup() return case <-ticker.C: apiSub.log.Info("healthCheckLoop: checkAliveness()") - apiSub.checkAliveness() + topicCounts := apiSub.getTopicCounts() + apiSub.resubscribe(topicCounts) } } } -func (apiSub *Sub) checkAliveness() { - apiSub.log.Info("ENTER checkAliveness()") +func (apiSub *Sub) cleanup() { + apiSub.log.Info("ENTER cleanup()") + defer func() { + apiSub.log.Info("EXIT cleanup()") + }() + + for _, s := range apiSub.subs { + apiSub.wf.UnsubscribeWithSubscription(apiSub.ctx, s) + } + close(apiSub.DataCh) + +} + +// Returns active sub counts for each pubsub topic +func (apiSub *Sub) getTopicCounts() map[string]int { + apiSub.log.Info("ENTER getTopicCounts()") + defer func() { + apiSub.log.Info("EXIT getTopicCounts()") + }() // Buffered chan for sub aliveness results type CheckResult struct { sub *subscription.SubscriptionDetails alive bool } - ch := make(chan CheckResult, len(apiSub.subs)) + checkResults := make(chan CheckResult, len(apiSub.subs)) // Run pings asynchronously for _, s := range apiSub.subs { @@ -92,7 +111,8 @@ func (apiSub *Sub) checkAliveness() { ctx, _ := context.WithTimeout(apiSub.ctx, 5*time.Second) err := apiSub.wf.IsSubscriptionAlive(ctx, s) - ch <- CheckResult{s, err == nil} + apiSub.log.Info("Check result:", zap.Any("subID", s.ID), zap.Bool("result", err == nil)) + checkResults <- CheckResult{s, err == nil} }() } @@ -103,49 +123,72 @@ func (apiSub *Sub) checkAliveness() { for _, t := range maps.Keys(topicMap) { topicCounts[t] = 0 } - // Close inactive subs cnt := 0 - for s := range ch { + subLen := len(apiSub.subs) + for s := range checkResults { cnt++ if !s.alive { + // Close inactive subs s.sub.Close() delete(apiSub.subs, s.sub.ID) } else { topicCounts[s.sub.ContentFilter.PubsubTopic]++ } - if cnt == len(apiSub.subs) { + if cnt == subLen { // All values received break } } - close(ch) + + close(checkResults) + return topicCounts +} + +// Attempts to resubscribe on topics that lack subscriptions +func (apiSub *Sub) resubscribe(topicCounts map[string]int) { + apiSub.log.Info("ENTER resubscribe()") + defer func() { + apiSub.log.Info("EXIT resubscribe()") + }() + + // Delete healthy topics for t, cnt := range topicCounts { if cnt == apiSub.Config.MaxPeers { delete(topicCounts, t) } } + + if len(topicCounts) == 0 { + // All topics healthy, return + return + } + // Re-subscribe asynchronously newSubs := make(chan []*subscription.SubscriptionDetails) + for t, cnt := range topicCounts { cFilter := protocol.ContentFilter{t, apiSub.ContentFilter.ContentTopics} go func() { - subs, err := apiSub.subscribe(cFilter, apiSub.Config.MaxPeers-cnt) - if err != nil { - newSubs <- subs - } + subs, _ := apiSub.subscribe(cFilter, apiSub.Config.MaxPeers-cnt) + newSubs <- subs }() } + cnt := 0 + apiSub.log.Info("resubscribe(): before range newSubs") for subs := range newSubs { - apiSub.multiplex(subs) + cnt++ + if subs != nil { + apiSub.multiplex(subs) + } if cnt == len(topicCounts) { + // Received all subscription results break } } + apiSub.log.Info("checkAliveness(): close(newSubs)") close(newSubs) - - apiSub.log.Info("EXIT checkAliveness()") } func (apiSub *Sub) subscribe(contentFilter protocol.ContentFilter, peerCount int) ([]*subscription.SubscriptionDetails, error) { diff --git a/waku/v2/api/filter_test.go b/waku/v2/api/filter_test.go index ee1d5e719..072a30001 100644 --- a/waku/v2/api/filter_test.go +++ b/waku/v2/api/filter_test.go @@ -3,6 +3,7 @@ package api import ( "context" "testing" + "time" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/suite" @@ -66,6 +67,7 @@ func (s *FilterApiTestSuite) TestSubscribe() { } s.Require().Equal(cnt, 1) + time.Sleep(20 * time.Second) apiSub.Unsubscribe() for _ = range apiSub.DataCh { } From 89b0989609258832c10c569f67609b2b3d94d87d Mon Sep 17 00:00:00 2001 From: Vitaly Vlasov Date: Mon, 6 May 2024 14:16:45 +0300 Subject: [PATCH 18/27] Fix TestStaticSharding --- waku/v2/protocol/filter/filter_test.go | 47 +++++++++----------------- 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/waku/v2/protocol/filter/filter_test.go b/waku/v2/protocol/filter/filter_test.go index 54a0ee431..d31171186 100644 --- a/waku/v2/protocol/filter/filter_test.go +++ b/waku/v2/protocol/filter/filter_test.go @@ -12,6 +12,7 @@ import ( "github.com/waku-org/go-waku/tests" "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/service" + "github.com/waku-org/go-waku/waku/v2/utils" "go.uber.org/zap" ) @@ -186,7 +187,7 @@ func (s *FilterTestSuite) AfterTest(suiteName, testName string) { func (s *FilterTestSuite) TestStaticSharding() { log := utils.Logger() - s.log = log + s.Log = log s.wg = &sync.WaitGroup{} ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) // Test can't exceed 10 seconds @@ -194,56 +195,40 @@ func (s *FilterTestSuite) TestStaticSharding() { s.ctxCancel = cancel // Gen pubsub topic "/waku/2/rs/100/100" - s.testTopic = protocol.NewStaticShardingPubsubTopic(uint16(100), uint16(100)).String() + s.TestTopic = protocol.NewStaticShardingPubsubTopic(uint16(100), uint16(100)).String() // Pubsub topics for neg. test cases testTopics := []string{ "/waku/2/rs/100/1024", "/waku/2/rs/100/101", } - s.testContentTopic = "/test/10/my-filter-app/proto" + s.TestContentTopic = "/test/10/my-filter-app/proto" // Prepare new nodes - s.lightNode = s.makeWakuFilterLightNode(true, true) - s.relayNode, s.fullNode = s.makeWakuFilterFullNode(s.testTopic, false, true) + s.MakeWakuFilterLightNode() + s.StartLightNode() + s.MakeWakuFilterFullNode(s.TestTopic, false) // Connect nodes - s.lightNodeHost.Peerstore().AddAddr(s.fullNodeHost.ID(), tests.GetHostAddress(s.fullNodeHost), peerstore.PermanentAddrTTL) - err := s.lightNodeHost.Peerstore().AddProtocols(s.fullNodeHost.ID(), FilterSubscribeID_v20beta1) - s.Require().NoError(err) + s.ConnectHosts(s.LightNodeHost, s.FullNodeHost) - s.subDetails = s.subscribe(s.testTopic, s.testContentTopic, s.fullNodeHost.ID()) - - msg := tests.CreateWakuMessage(s.testContentTopic, utils.GetUnixEpoch()) - msg2 := tests.CreateWakuMessage(s.testContentTopic, utils.GetUnixEpoch()) + s.subscribe(s.TestTopic, s.TestContentTopic, s.FullNodeHost.ID()) // Test positive case for static shard pubsub topic - message gets received - s.waitForMsg(func() { - _, err := s.relayNode.Publish(s.ctx, msg, relay.WithPubSubTopic(s.testTopic)) - s.Require().NoError(err) - - }, s.subDetails[0].C) + s.waitForMsg(&WakuMsg{s.TestTopic, s.TestContentTopic, ""}) // Test two negative cases for static shard pubsub topic - message times out - s.waitForTimeout(func() { - _, err := s.relayNode.Publish(s.ctx, msg2, relay.WithPubSubTopic(testTopics[0])) - s.Require().NoError(err) - - }, s.subDetails[0].C) - - s.waitForTimeout(func() { - _, err := s.relayNode.Publish(s.ctx, msg2, relay.WithPubSubTopic(testTopics[1])) - s.Require().NoError(err) + s.waitForTimeout(&WakuMsg{testTopics[0], s.TestContentTopic, ""}) - }, s.subDetails[0].C) + s.waitForTimeout(&WakuMsg{testTopics[1], s.TestContentTopic, ""}) // Cleanup - _, err = s.lightNode.Unsubscribe(s.ctx, protocol.ContentFilter{ - PubsubTopic: s.testTopic, - ContentTopics: protocol.NewContentTopicSet(s.testContentTopic), + _, err := s.LightNode.Unsubscribe(s.ctx, protocol.ContentFilter{ + PubsubTopic: s.TestTopic, + ContentTopics: protocol.NewContentTopicSet(s.TestContentTopic), }) s.Require().NoError(err) - _, err = s.lightNode.UnsubscribeAll(s.ctx) + _, err = s.LightNode.UnsubscribeAll(s.ctx) s.Require().NoError(err) } From 632f5b07a63106c94f2203f168f33ab9aec3c18d Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Fri, 10 May 2024 15:09:09 +0530 Subject: [PATCH 19/27] fix: remove test causing import loop --- tests/connection_test.go | 78 ---------------------------------------- 1 file changed, 78 deletions(-) delete mode 100644 tests/connection_test.go diff --git a/tests/connection_test.go b/tests/connection_test.go deleted file mode 100644 index 2efa1e50e..000000000 --- a/tests/connection_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package tests - -import ( - "context" - "net" - "testing" - - "github.com/ethereum/go-ethereum/crypto" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" - - "github.com/waku-org/go-waku/waku/v2/node" - "github.com/waku-org/go-waku/waku/v2/payload" - "github.com/waku-org/go-waku/waku/v2/protocol" - "github.com/waku-org/go-waku/waku/v2/protocol/pb" - "github.com/waku-org/go-waku/waku/v2/protocol/relay" - "github.com/waku-org/go-waku/waku/v2/utils" -) - -func TestBasicSendingReceiving(t *testing.T) { - hostAddr, err := net.ResolveTCPAddr("tcp", "0.0.0.0:0") - require.NoError(t, err) - - key, err := RandomHex(32) - require.NoError(t, err) - - prvKey, err := crypto.HexToECDSA(key) - require.NoError(t, err) - - ctx := context.Background() - - wakuNode, err := node.New( - node.WithPrivateKey(prvKey), - node.WithHostAddress(hostAddr), - node.WithWakuRelay(), - ) - require.NoError(t, err) - require.NotNil(t, wakuNode) - - err = wakuNode.Start(ctx) - require.NoError(t, err) - - require.NoError(t, write(ctx, wakuNode, "test")) - - sub, err := wakuNode.Relay().Subscribe(ctx, protocol.NewContentFilter(relay.DefaultWakuTopic)) - require.NoError(t, err) - - value := <-sub[0].Ch - payload, err := payload.DecodePayload(value.Message(), &payload.KeyInfo{Kind: payload.None}) - require.NoError(t, err) - - require.Contains(t, string(payload.Data), "test") -} - -func write(ctx context.Context, wakuNode *node.WakuNode, msgContent string) error { - var contentTopic string = "test" - version := uint32(0) - timestamp := utils.GetUnixEpoch() - - p := new(payload.Payload) - p.Data = []byte(wakuNode.ID() + ": " + msgContent) - p.Key = &payload.KeyInfo{Kind: payload.None} - - payload, err := p.Encode(version) - if err != nil { - return err - } - - msg := &pb.WakuMessage{ - Payload: payload, - Version: proto.Uint32(version), - ContentTopic: contentTopic, - Timestamp: timestamp, - } - - _, err = wakuNode.Relay().Publish(ctx, msg, relay.WithDefaultPubsubTopic()) - return err -} From f0b7b01ca3ec54f54e2fca01a2c2042fe08357d2 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Fri, 10 May 2024 15:25:57 +0530 Subject: [PATCH 20/27] chore: fix lint errors --- waku/v2/api/filter.go | 29 +++++++++++++++++------------ waku/v2/api/filter_test.go | 4 ++-- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/waku/v2/api/filter.go b/waku/v2/api/filter.go index 0c916a8c3..3cf1f9c38 100644 --- a/waku/v2/api/filter.go +++ b/waku/v2/api/filter.go @@ -85,7 +85,11 @@ func (apiSub *Sub) cleanup() { }() for _, s := range apiSub.subs { - apiSub.wf.UnsubscribeWithSubscription(apiSub.ctx, s) + _, err := apiSub.wf.UnsubscribeWithSubscription(apiSub.ctx, s) + if err != nil { + //Logging with info as this is part of cleanup + apiSub.log.Info("failed to unsubscribe filter", zap.Error(err)) + } } close(apiSub.DataCh) @@ -107,13 +111,14 @@ func (apiSub *Sub) getTopicCounts() map[string]int { // Run pings asynchronously for _, s := range apiSub.subs { - go func() { - ctx, _ := context.WithTimeout(apiSub.ctx, 5*time.Second) - err := apiSub.wf.IsSubscriptionAlive(ctx, s) - - apiSub.log.Info("Check result:", zap.Any("subID", s.ID), zap.Bool("result", err == nil)) - checkResults <- CheckResult{s, err == nil} - }() + go func(sub *subscription.SubscriptionDetails) { + ctx, cancelFunc := context.WithTimeout(apiSub.ctx, 5*time.Second) + defer cancelFunc() + err := apiSub.wf.IsSubscriptionAlive(ctx, sub) + + apiSub.log.Info("Check result:", zap.Any("subID", sub.ID), zap.Bool("result", err == nil)) + checkResults <- CheckResult{sub, err == nil} + }(s) } // Collect healthy topic counts @@ -168,11 +173,11 @@ func (apiSub *Sub) resubscribe(topicCounts map[string]int) { newSubs := make(chan []*subscription.SubscriptionDetails) for t, cnt := range topicCounts { - cFilter := protocol.ContentFilter{t, apiSub.ContentFilter.ContentTopics} - go func() { - subs, _ := apiSub.subscribe(cFilter, apiSub.Config.MaxPeers-cnt) + cFilter := protocol.ContentFilter{PubsubTopic: t, ContentTopics: apiSub.ContentFilter.ContentTopics} + go func(count int) { + subs, _ := apiSub.subscribe(cFilter, apiSub.Config.MaxPeers-count) newSubs <- subs - }() + }(cnt) } cnt := 0 diff --git a/waku/v2/api/filter_test.go b/waku/v2/api/filter_test.go index 072a30001..271a169b3 100644 --- a/waku/v2/api/filter_test.go +++ b/waku/v2/api/filter_test.go @@ -58,7 +58,7 @@ func (s *FilterApiTestSuite) TestSubscribe() { } s.Require().True(maps.Keys(apiSub.subs)[0] != maps.Keys(apiSub.subs)[1]) // Publish msg and confirm it's received twice because of multiplexing - s.PublishMsg(&filter.WakuMsg{s.TestTopic, s.TestContentTopic, "Test msg"}) + s.PublishMsg(&filter.WakuMsg{PubSubTopic: s.TestTopic, ContentTopic: s.TestContentTopic, Payload: "Test msg"}) cnt := 0 for msg := range apiSub.DataCh { s.Log.Info("Received msg:", zap.Int("cnt", cnt), zap.String("payload", string(msg.Message().Payload))) @@ -69,7 +69,7 @@ func (s *FilterApiTestSuite) TestSubscribe() { time.Sleep(20 * time.Second) apiSub.Unsubscribe() - for _ = range apiSub.DataCh { + for range apiSub.DataCh { } s.Log.Info("DataCh is closed") From 07fcb77e29d314b0ced5f6bd08e7c27bfab066c3 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Fri, 10 May 2024 15:44:15 +0530 Subject: [PATCH 21/27] fix: build examples --- examples/basic-relay/go.mod | 3 +++ examples/basic-relay/go.sum | 1 + examples/chat2/go.mod | 3 +++ examples/chat2/go.sum | 1 + examples/filter2/go.mod | 4 ++++ examples/filter2/go.sum | 1 + examples/noise/go.mod | 4 ++++ examples/noise/go.sum | 1 + examples/rln/go.mod | 4 ++++ examples/rln/go.sum | 1 + go.sum | 3 --- 11 files changed, 23 insertions(+), 3 deletions(-) diff --git a/examples/basic-relay/go.mod b/examples/basic-relay/go.mod index 1504a8b58..7930f9c12 100644 --- a/examples/basic-relay/go.mod +++ b/examples/basic-relay/go.mod @@ -30,6 +30,7 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/cruxic/go-hmac-drbg v0.0.0-20170206035330-84c46983886d // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -94,6 +95,7 @@ require ( github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect @@ -106,6 +108,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect diff --git a/examples/basic-relay/go.sum b/examples/basic-relay/go.sum index 9d2621f89..ce2241090 100644 --- a/examples/basic-relay/go.sum +++ b/examples/basic-relay/go.sum @@ -628,6 +628,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI= github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= diff --git a/examples/chat2/go.mod b/examples/chat2/go.mod index 2b8a28349..06b20908d 100644 --- a/examples/chat2/go.mod +++ b/examples/chat2/go.mod @@ -38,6 +38,7 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/cruxic/go-hmac-drbg v0.0.0-20170206035330-84c46983886d // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -105,6 +106,7 @@ require ( github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect @@ -118,6 +120,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect diff --git a/examples/chat2/go.sum b/examples/chat2/go.sum index d30229595..cf0be7192 100644 --- a/examples/chat2/go.sum +++ b/examples/chat2/go.sum @@ -660,6 +660,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI= github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= diff --git a/examples/filter2/go.mod b/examples/filter2/go.mod index ad17d25b9..a299123c5 100644 --- a/examples/filter2/go.mod +++ b/examples/filter2/go.mod @@ -26,6 +26,7 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cruxic/go-hmac-drbg v0.0.0-20170206035330-84c46983886d // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -90,6 +91,7 @@ require ( github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect @@ -101,6 +103,7 @@ require ( github.com/raulk/go-watchdog v1.3.0 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect @@ -126,5 +129,6 @@ require ( golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect golang.org/x/tools v0.14.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/examples/filter2/go.sum b/examples/filter2/go.sum index 6bc4d876e..b11572a7b 100644 --- a/examples/filter2/go.sum +++ b/examples/filter2/go.sum @@ -625,6 +625,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI= github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= diff --git a/examples/noise/go.mod b/examples/noise/go.mod index 623b09c33..f838867b1 100644 --- a/examples/noise/go.mod +++ b/examples/noise/go.mod @@ -28,6 +28,7 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cruxic/go-hmac-drbg v0.0.0-20170206035330-84c46983886d // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -92,6 +93,7 @@ require ( github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect @@ -103,6 +105,7 @@ require ( github.com/raulk/go-watchdog v1.3.0 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect @@ -127,5 +130,6 @@ require ( golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect golang.org/x/tools v0.14.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/examples/noise/go.sum b/examples/noise/go.sum index 62cc4a23a..82d7eed79 100644 --- a/examples/noise/go.sum +++ b/examples/noise/go.sum @@ -625,6 +625,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI= github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= diff --git a/examples/rln/go.mod b/examples/rln/go.mod index 25e7bd680..60b5b2356 100644 --- a/examples/rln/go.mod +++ b/examples/rln/go.mod @@ -26,6 +26,7 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cruxic/go-hmac-drbg v0.0.0-20170206035330-84c46983886d // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -91,6 +92,7 @@ require ( github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect @@ -102,6 +104,7 @@ require ( github.com/raulk/go-watchdog v1.3.0 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect @@ -126,5 +129,6 @@ require ( golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect golang.org/x/tools v0.14.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/examples/rln/go.sum b/examples/rln/go.sum index 6bc4d876e..b11572a7b 100644 --- a/examples/rln/go.sum +++ b/examples/rln/go.sum @@ -625,6 +625,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI= github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= diff --git a/go.sum b/go.sum index b7ee60f96..23b463389 100644 --- a/go.sum +++ b/go.sum @@ -564,7 +564,6 @@ github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2C github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -1532,8 +1531,6 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmF github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/waku-org/go-discover v0.0.0-20240129014929-85f2c00b96a3 h1:Kk0KYXZE/uNnARF2TbCQyvyZ/w4SgF8VhquNdOVVsNU= -github.com/waku-org/go-discover v0.0.0-20240129014929-85f2c00b96a3/go.mod h1:eBHgM6T4EG0RZzxpxKy+rGz/6Dw2Nd8DWxS0lm9ESDw= github.com/waku-org/go-discover v0.0.0-20240506173252-4912704efdc5 h1:4K3IS97JryAEV8pRXB//qPcg+8bPXl/O+AOLt3FeCKc= github.com/waku-org/go-discover v0.0.0-20240506173252-4912704efdc5/go.mod h1:eBHgM6T4EG0RZzxpxKy+rGz/6Dw2Nd8DWxS0lm9ESDw= github.com/waku-org/go-libp2p-rendezvous v0.0.0-20240110193335-a67d1cc760a0 h1:R4YYx2QamhBRl/moIxkDCNW+OP7AHbyWLBygDc/xIMo= From d0f3a10d08e02c06103ff03e4d92f808a084ef66 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Fri, 10 May 2024 16:17:34 +0530 Subject: [PATCH 22/27] fix: possible test issue --- waku/v2/protocol/filter/test_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waku/v2/protocol/filter/test_utils.go b/waku/v2/protocol/filter/test_utils.go index fbb1c9dbf..a8d2af48e 100644 --- a/waku/v2/protocol/filter/test_utils.go +++ b/waku/v2/protocol/filter/test_utils.go @@ -113,7 +113,7 @@ func (s *FilterTestSuite) GetWakuRelay(topic string) FullNodeData { host, err := tests.MakeHost(context.Background(), port, rand.Reader) s.Require().NoError(err) - relay := relay.NewWakuRelay(broadcaster, 0, timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.Log) + relay := relay.NewWakuRelay(broadcaster, 0, timesource.NewDefaultClock(), prometheus.DefaultRegisterer, s.Log, relay.WithMaxMsgSize(1024*1024)) relay.SetHost(host) err = relay.Start(context.Background()) From f74d74f3346c9c5ef63dab9fb367b4c544035882 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Fri, 10 May 2024 16:51:05 +0530 Subject: [PATCH 23/27] chore: fix for test fail in CI --- waku/v2/api/filter_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/waku/v2/api/filter_test.go b/waku/v2/api/filter_test.go index 271a169b3..c9f0829ad 100644 --- a/waku/v2/api/filter_test.go +++ b/waku/v2/api/filter_test.go @@ -56,7 +56,8 @@ func (s *FilterApiTestSuite) TestSubscribe() { for sub := range apiSub.subs { s.Log.Info("SubDetails:", zap.String("id", sub)) } - s.Require().True(maps.Keys(apiSub.subs)[0] != maps.Keys(apiSub.subs)[1]) + subsArray := maps.Keys(apiSub.subs) + s.Require().True(subsArray[0] != subsArray[1]) // Publish msg and confirm it's received twice because of multiplexing s.PublishMsg(&filter.WakuMsg{PubSubTopic: s.TestTopic, ContentTopic: s.TestContentTopic, Payload: "Test msg"}) cnt := 0 @@ -67,7 +68,7 @@ func (s *FilterApiTestSuite) TestSubscribe() { } s.Require().Equal(cnt, 1) - time.Sleep(20 * time.Second) + time.Sleep(2 * time.Second) apiSub.Unsubscribe() for range apiSub.DataCh { } From e581e42e305f8c4d2ce89ba36c0a02c19f0816d1 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Fri, 10 May 2024 17:14:55 +0530 Subject: [PATCH 24/27] chore: change in test validation --- waku/v2/protocol/filter/test_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waku/v2/protocol/filter/test_utils.go b/waku/v2/protocol/filter/test_utils.go index a8d2af48e..2784c16b7 100644 --- a/waku/v2/protocol/filter/test_utils.go +++ b/waku/v2/protocol/filter/test_utils.go @@ -260,7 +260,7 @@ func (s *FilterTestSuite) waitForMessages(msgs []WakuMsg) { } s.wg.Wait() - s.Require().True(msgCount == found) + s.Require().Equal(msgCount, found) } func (s *FilterTestSuite) waitForTimeout(msg *WakuMsg) { From e6da6d8cf275ea0b2d7f52a4f16b803b575ed3d4 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Mon, 13 May 2024 14:55:12 +0530 Subject: [PATCH 25/27] chore: address review comments --- waku/v2/api/filter.go | 46 +++++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/waku/v2/api/filter.go b/waku/v2/api/filter.go index 3cf1f9c38..cd1daad14 100644 --- a/waku/v2/api/filter.go +++ b/waku/v2/api/filter.go @@ -2,6 +2,7 @@ package api import ( "context" + "sync" "time" "github.com/libp2p/go-libp2p/core/peer" @@ -12,6 +13,8 @@ import ( "golang.org/x/exp/maps" ) +const FilterPingTimeout = 5 + type FilterConfig struct { MaxPeers int Peers []peer.ID @@ -44,14 +47,13 @@ func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilte subs, err := sub.subscribe(contentFilter, sub.Config.MaxPeers) - if err == nil { - sub.multiplex(subs) - sub.log.Info("go sub.healthCheckLoop()") - go sub.healthCheckLoop() - return sub, nil - } else { + if err != nil { return nil, err } + sub.multiplex(subs) + sub.log.Info("go sub.healthCheckLoop()") + go sub.healthCheckLoop() + return sub, nil } func (apiSub *Sub) Unsubscribe() { @@ -61,7 +63,7 @@ func (apiSub *Sub) Unsubscribe() { func (apiSub *Sub) healthCheckLoop() { // Health checks - ticker := time.NewTicker(5 * time.Second) + ticker := time.NewTicker(FilterPingTimeout * time.Second) defer ticker.Stop() for { select { @@ -97,22 +99,20 @@ func (apiSub *Sub) cleanup() { // Returns active sub counts for each pubsub topic func (apiSub *Sub) getTopicCounts() map[string]int { - apiSub.log.Info("ENTER getTopicCounts()") - defer func() { - apiSub.log.Info("EXIT getTopicCounts()") - }() - // Buffered chan for sub aliveness results type CheckResult struct { sub *subscription.SubscriptionDetails alive bool } checkResults := make(chan CheckResult, len(apiSub.subs)) + var wg sync.WaitGroup // Run pings asynchronously for _, s := range apiSub.subs { + wg.Add(1) go func(sub *subscription.SubscriptionDetails) { - ctx, cancelFunc := context.WithTimeout(apiSub.ctx, 5*time.Second) + defer wg.Done() + ctx, cancelFunc := context.WithTimeout(apiSub.ctx, FilterPingTimeout*time.Second) defer cancelFunc() err := apiSub.wf.IsSubscriptionAlive(ctx, sub) @@ -129,7 +129,8 @@ func (apiSub *Sub) getTopicCounts() map[string]int { topicCounts[t] = 0 } cnt := 0 - subLen := len(apiSub.subs) + wg.Wait() + close(checkResults) for s := range checkResults { cnt++ if !s.alive { @@ -139,23 +140,13 @@ func (apiSub *Sub) getTopicCounts() map[string]int { } else { topicCounts[s.sub.ContentFilter.PubsubTopic]++ } - - if cnt == subLen { - // All values received - break - } } - close(checkResults) return topicCounts } // Attempts to resubscribe on topics that lack subscriptions func (apiSub *Sub) resubscribe(topicCounts map[string]int) { - apiSub.log.Info("ENTER resubscribe()") - defer func() { - apiSub.log.Info("EXIT resubscribe()") - }() // Delete healthy topics for t, cnt := range topicCounts { @@ -181,7 +172,7 @@ func (apiSub *Sub) resubscribe(topicCounts map[string]int) { } cnt := 0 - apiSub.log.Info("resubscribe(): before range newSubs") + apiSub.log.Debug("resubscribe(): before range newSubs") for subs := range newSubs { cnt++ if subs != nil { @@ -214,12 +205,11 @@ func (apiSub *Sub) subscribe(contentFilter protocol.ContentFilter, peerCount int } func (apiSub *Sub) multiplex(subs []*subscription.SubscriptionDetails) { - for _, s := range subs { - apiSub.subs[s.ID] = s - } + // Multiplex onto single channel // Goroutines will exit once sub channels are closed for _, subDetails := range subs { + apiSub.subs[subDetails.ID] = subDetails go func(subDetails *subscription.SubscriptionDetails) { apiSub.log.Info("New multiplex", zap.String("subID", subDetails.ID)) for env := range subDetails.C { From 6d89d0d56250e7e8d13861fcf5f146088a8ab72b Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Tue, 14 May 2024 12:12:10 +0530 Subject: [PATCH 26/27] chore:address review comments --- waku/v2/api/filter.go | 19 +++++++++++-------- waku/v2/api/filter_test.go | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/waku/v2/api/filter.go b/waku/v2/api/filter.go index cd1daad14..6a44afac6 100644 --- a/waku/v2/api/filter.go +++ b/waku/v2/api/filter.go @@ -32,7 +32,7 @@ type Sub struct { } // Subscribe -func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilter protocol.ContentFilter, config FilterConfig) (*Sub, error) { +func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilter protocol.ContentFilter, config FilterConfig, log *zap.Logger) (*Sub, error) { sub := new(Sub) sub.wf = wf sub.ctx, sub.cancel = context.WithCancel(ctx) @@ -40,10 +40,7 @@ func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilte sub.DataCh = make(chan *protocol.Envelope) sub.ContentFilter = contentFilter sub.Config = config - sub.log = func() *zap.Logger { - log, _ := zap.NewDevelopment() - return log - }().Named("filterv2-api") + sub.log = log.Named("filter-api") subs, err := sub.subscribe(contentFilter, sub.Config.MaxPeers) @@ -128,11 +125,9 @@ func (apiSub *Sub) getTopicCounts() map[string]int { for _, t := range maps.Keys(topicMap) { topicCounts[t] = 0 } - cnt := 0 wg.Wait() close(checkResults) for s := range checkResults { - cnt++ if !s.alive { // Close inactive subs s.sub.Close() @@ -197,7 +192,15 @@ func (apiSub *Sub) subscribe(contentFilter protocol.ContentFilter, peerCount int subs, err := apiSub.wf.Subscribe(apiSub.ctx, contentFilter, options...) if err != nil { - // TODO what if fails? + if len(subs) > 0 { + // Partial Failure, for now proceed as we don't expect this to happen wrt specific topics. + // Rather it can happen in case subscription with one of the peer fails. + // This can further get automatically handled at resubscribe, + apiSub.log.Error("partial failure in Filter subscribe", zap.Error(err)) + return subs, nil + } + // In case of complete subscription failure, application or user needs to handle and probably retry based on error + // TODO: Once filter error handling indicates specific error, this can be addressed based on the error at this layer. return nil, err } diff --git a/waku/v2/api/filter_test.go b/waku/v2/api/filter_test.go index c9f0829ad..22432e7e2 100644 --- a/waku/v2/api/filter_test.go +++ b/waku/v2/api/filter_test.go @@ -47,7 +47,7 @@ func (s *FilterApiTestSuite) TestSubscribe() { s.Require().Equal(contentFilter.PubsubTopic, s.TestTopic) s.Log.Info("About to perform API Subscribe()") - apiSub, err := Subscribe(context.Background(), s.LightNode, contentFilter, apiConfig) + apiSub, err := Subscribe(context.Background(), s.LightNode, contentFilter, apiConfig, s.Log) s.Require().NoError(err) s.Require().Equal(apiSub.ContentFilter, contentFilter) s.Log.Info("Subscribed") From f6c9810f0748a12f82053dcd1a213dbb193c03e4 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Wed, 15 May 2024 12:16:53 +0530 Subject: [PATCH 27/27] chore: address review comments --- waku/v2/api/filter.go | 58 +++++++++++++++++------------- waku/v2/protocol/content_filter.go | 14 +++++++- waku/v2/protocol/filter/server.go | 2 +- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/waku/v2/api/filter.go b/waku/v2/api/filter.go index 6a44afac6..f6f83f140 100644 --- a/waku/v2/api/filter.go +++ b/waku/v2/api/filter.go @@ -2,6 +2,7 @@ package api import ( "context" + "encoding/json" "sync" "time" @@ -13,11 +14,20 @@ import ( "golang.org/x/exp/maps" ) -const FilterPingTimeout = 5 +const FilterPingTimeout = 5 * time.Second +const MultiplexChannelBuffer = 100 type FilterConfig struct { - MaxPeers int - Peers []peer.ID + MaxPeers int `json:"maxPeers"` + Peers []peer.ID `json:"peers"` +} + +func (fc FilterConfig) String() string { + jsonStr, err := json.Marshal(fc) + if err != nil { + return "" + } + return string(jsonStr) } type Sub struct { @@ -37,18 +47,17 @@ func Subscribe(ctx context.Context, wf *filter.WakuFilterLightNode, contentFilte sub.wf = wf sub.ctx, sub.cancel = context.WithCancel(ctx) sub.subs = make(subscription.SubscriptionSet) - sub.DataCh = make(chan *protocol.Envelope) + sub.DataCh = make(chan *protocol.Envelope, MultiplexChannelBuffer) sub.ContentFilter = contentFilter sub.Config = config sub.log = log.Named("filter-api") - + sub.log.Debug("filter subscribe params", zap.Int("maxPeers", config.MaxPeers), zap.Stringer("contentFilter", contentFilter)) subs, err := sub.subscribe(contentFilter, sub.Config.MaxPeers) if err != nil { return nil, err } sub.multiplex(subs) - sub.log.Info("go sub.healthCheckLoop()") go sub.healthCheckLoop() return sub, nil } @@ -60,16 +69,16 @@ func (apiSub *Sub) Unsubscribe() { func (apiSub *Sub) healthCheckLoop() { // Health checks - ticker := time.NewTicker(FilterPingTimeout * time.Second) + ticker := time.NewTicker(FilterPingTimeout) defer ticker.Stop() for { select { case <-apiSub.ctx.Done(): - apiSub.log.Info("healthCheckLoop: Done()") + apiSub.log.Debug("healthCheckLoop: Done()") apiSub.cleanup() return case <-ticker.C: - apiSub.log.Info("healthCheckLoop: checkAliveness()") + apiSub.log.Debug("healthCheckLoop: checkAliveness()") topicCounts := apiSub.getTopicCounts() apiSub.resubscribe(topicCounts) } @@ -78,9 +87,9 @@ func (apiSub *Sub) healthCheckLoop() { } func (apiSub *Sub) cleanup() { - apiSub.log.Info("ENTER cleanup()") + apiSub.log.Debug("ENTER cleanup()") defer func() { - apiSub.log.Info("EXIT cleanup()") + apiSub.log.Debug("EXIT cleanup()") }() for _, s := range apiSub.subs { @@ -109,11 +118,11 @@ func (apiSub *Sub) getTopicCounts() map[string]int { wg.Add(1) go func(sub *subscription.SubscriptionDetails) { defer wg.Done() - ctx, cancelFunc := context.WithTimeout(apiSub.ctx, FilterPingTimeout*time.Second) + ctx, cancelFunc := context.WithTimeout(apiSub.ctx, FilterPingTimeout) defer cancelFunc() err := apiSub.wf.IsSubscriptionAlive(ctx, sub) - apiSub.log.Info("Check result:", zap.Any("subID", sub.ID), zap.Bool("result", err == nil)) + apiSub.log.Debug("Check result:", zap.Any("subID", sub.ID), zap.Bool("result", err == nil)) checkResults <- CheckResult{sub, err == nil} }(s) } @@ -154,32 +163,33 @@ func (apiSub *Sub) resubscribe(topicCounts map[string]int) { // All topics healthy, return return } + var wg sync.WaitGroup // Re-subscribe asynchronously newSubs := make(chan []*subscription.SubscriptionDetails) for t, cnt := range topicCounts { cFilter := protocol.ContentFilter{PubsubTopic: t, ContentTopics: apiSub.ContentFilter.ContentTopics} + wg.Add(1) go func(count int) { - subs, _ := apiSub.subscribe(cFilter, apiSub.Config.MaxPeers-count) + defer wg.Done() + subs, err := apiSub.subscribe(cFilter, apiSub.Config.MaxPeers-count) + if err != nil { + return + } //Not handling scenario where all requested subs are not received as that will get handled in next cycle. newSubs <- subs }(cnt) } - - cnt := 0 + wg.Wait() + close(newSubs) apiSub.log.Debug("resubscribe(): before range newSubs") for subs := range newSubs { - cnt++ if subs != nil { apiSub.multiplex(subs) } - if cnt == len(topicCounts) { - // Received all subscription results - break - } } - apiSub.log.Info("checkAliveness(): close(newSubs)") - close(newSubs) + apiSub.log.Debug("checkAliveness(): close(newSubs)") + //close(newSubs) } func (apiSub *Sub) subscribe(contentFilter protocol.ContentFilter, peerCount int) ([]*subscription.SubscriptionDetails, error) { @@ -214,7 +224,7 @@ func (apiSub *Sub) multiplex(subs []*subscription.SubscriptionDetails) { for _, subDetails := range subs { apiSub.subs[subDetails.ID] = subDetails go func(subDetails *subscription.SubscriptionDetails) { - apiSub.log.Info("New multiplex", zap.String("subID", subDetails.ID)) + apiSub.log.Debug("New multiplex", zap.String("subID", subDetails.ID)) for env := range subDetails.C { apiSub.DataCh <- env } diff --git a/waku/v2/protocol/content_filter.go b/waku/v2/protocol/content_filter.go index 0cff1cbc5..0a859e0d2 100644 --- a/waku/v2/protocol/content_filter.go +++ b/waku/v2/protocol/content_filter.go @@ -1,6 +1,8 @@ package protocol -import "golang.org/x/exp/maps" +import ( + "golang.org/x/exp/maps" +) type PubsubTopicStr = string type ContentTopicStr = string @@ -37,6 +39,16 @@ type ContentFilter struct { ContentTopics ContentTopicSet `json:"contentTopics"` } +func (cf ContentFilter) String() string { + var ret string + ret += "{ pubsubTopic: " + cf.PubsubTopic + ", contentTopics: [ " + for ct := range cf.ContentTopics { + ret += ct + } + ret += " ] }" + return ret +} + func (cf ContentFilter) ContentTopicsList() []string { return cf.ContentTopics.ToList() } diff --git a/waku/v2/protocol/filter/server.go b/waku/v2/protocol/filter/server.go index 2416a62ba..2bf63bb50 100644 --- a/waku/v2/protocol/filter/server.go +++ b/waku/v2/protocol/filter/server.go @@ -128,7 +128,7 @@ func (wf *WakuFilterFullNode) onRequest(ctx context.Context) func(network.Stream wf.metrics.RecordRequest(subscribeRequest.FilterSubscribeType.String(), time.Since(start)) - logger.Info("received request", zap.String("serverID", wf.h.ID().String()), zap.String("requestType", subscribeRequest.FilterSubscribeType.String())) + logger.Info("received request", zap.Stringer("serverID", wf.h.ID()), zap.Stringer("requestType", subscribeRequest.FilterSubscribeType)) } }