improve feed/group request by allowing more users per request.

j-fbriere · Feb 14, 2024 · 0d7d9e2 · 0d7d9e2
1 parent aeaa3c5
commit 0d7d9e2
Show file tree

Hide file tree

Showing 8 changed files with 73 additions and 76 deletions.
diff --git a/android/fastlane/metadata/android/en-GB/changelogs/37.txt b/android/fastlane/metadata/android/en-GB/changelogs/37.txt
diff --git a/fastlane/metadata/android/en-US/changelogs/37.txt b/fastlane/metadata/android/en-US/changelogs/37.txt
diff --git a/lib/group/_feed.dart b/lib/group/_feed.dart
@@ -13,28 +13,28 @@ import 'package:squawker/constants.dart';
 import 'package:squawker/database/entities.dart';
 import 'package:squawker/database/repository.dart';
 import 'package:squawker/generated/l10n.dart';
-import 'package:squawker/group/group_screen.dart';
 import 'package:squawker/profile/profile.dart';
 import 'package:squawker/tweet/_video.dart';
 import 'package:squawker/tweet/conversation.dart';
 import 'package:squawker/tweet/tweet.dart';
 import 'package:squawker/ui/errors.dart';
+import 'package:squawker/utils/crypto_util.dart';
 import 'package:squawker/utils/iterables.dart';
 import 'package:pref/pref.dart';
 import 'package:provider/provider.dart';
 import 'package:synchronized/synchronized.dart';
 
 class SubscriptionGroupFeed extends StatefulWidget {
   final SubscriptionGroupGet group;
-  final List<SubscriptionGroupFeedChunk> chunks;
+  final List<String> searchQueries;
   final bool includeReplies;
   final bool includeRetweets;
   final ItemScrollController? scrollController;
 
   const SubscriptionGroupFeed(
       {Key? key,
       required this.group,
-      required this.chunks,
+      required this.searchQueries,
       required this.includeReplies,
       required this.includeRetweets,
       required this.scrollController})
@@ -148,47 +148,6 @@ class SubscriptionGroupFeedState extends State<SubscriptionGroupFeed> with Widge
     }
   }
 
-  String _buildSearchQuery(List<Subscription> users) {
-    var query = '';
-    if (!widget.includeReplies) {
-      query += '-filter:replies AND ';
-    }
-
-    if (!widget.includeRetweets) {
-      query += '-filter:retweets AND ';
-    } else {
-      query += 'include:nativeretweets AND ';
-    }
-
-    var remainingLength = 512 - query.length;
-
-    int cnt = 0;
-    for (var user in users) {
-      var queryToAdd = '';
-      if (user is UserSubscription) {
-        queryToAdd = 'from:${user.screenName}';
-      } else if (user is SearchSubscription) {
-        queryToAdd = '"${user.id}"';
-      }
-
-      // If we can add this user to the query and still be less than ~512 characters, do so
-      if (query.length + queryToAdd.length < remainingLength) {
-        if (cnt > 0) {
-          query += ' OR ';
-        }
-
-        query += queryToAdd;
-      } else {
-        // Otherwise, add the search future and start a new one
-        assert(false, 'should never reach here');
-        query = queryToAdd;
-      }
-      cnt++;
-    }
-
-    return query;
-  }
-
   /// Search for our next "page" of tweets.
   ///
   /// Here, each page is actually a set of mappings, where the ID of each set is the hash of all the user IDs in that
@@ -218,10 +177,10 @@ class SubscriptionGroupFeedState extends State<SubscriptionGroupFeed> with Widge
       }
 
       _errorResponse = null;
-      RateFetchContext fetchContext = RateFetchContext(prefs.get(optionEnhancedFeeds) ? Twitter.graphqlSearchTimelineUriPath : Twitter.searchTweetsUriPath, widget.chunks.length);
+      RateFetchContext fetchContext = RateFetchContext(prefs.get(optionEnhancedFeeds) ? Twitter.graphqlSearchTimelineUriPath : Twitter.searchTweetsUriPath, widget.searchQueries.length);
       await fetchContext.init();
-      for (var chunk in widget.chunks) {
-        var hash = chunk.hash;
+      for (var searchQuery in widget.searchQueries) {
+        String hash = await sha1Hash(searchQuery);
 
         futures.add(Future(() async {
           var tweets = <TweetChain>[];
@@ -263,17 +222,16 @@ class SubscriptionGroupFeedState extends State<SubscriptionGroupFeed> with Widge
 
           if (requestToDo) {
             // Perform our search for the next page of results for this chunk, and add those tweets to our collection
-            var query = _buildSearchQuery(chunk.users);
             TweetStatus result;
             try {
               if (prefs.get(optionEnhancedFeeds)) {
-                result = await Twitter.searchTweetsGraphql(query, widget.includeReplies, limit: 100,
+                result = await Twitter.searchTweetsGraphql(searchQuery, widget.includeReplies, limit: 100,
                     cursor: searchCursor,
                     leanerFeeds: prefs.get(optionLeanerFeeds),
                     fetchContext: fetchContext);
               }
               else {
-                result = await Twitter.searchTweets(query, widget.includeReplies, limit: 100,
+                result = await Twitter.searchTweets(searchQuery, widget.includeReplies, limit: 100,
                     cursor: searchCursor,
                     cursorType: cursorType,
                     leanerFeeds: prefs.get(optionLeanerFeeds),
@@ -442,7 +400,7 @@ class SubscriptionGroupFeedState extends State<SubscriptionGroupFeed> with Widge
       );
     }
 
-    if (widget.chunks.isEmpty) {
+    if (widget.searchQueries.isEmpty) {
       return Scaffold(
         body: Center(
           child: Text(L10n.of(context).this_group_contains_no_subscriptions),

diff --git a/lib/group/group_screen.dart b/lib/group/group_screen.dart
@@ -1,7 +1,7 @@
-import 'package:crypto/crypto.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_triple/flutter_triple.dart';
 import 'package:provider/provider.dart';
+import 'package:quiver/iterables.dart';
 import 'package:scrollable_positioned_list/scrollable_positioned_list.dart';
 import 'package:squawker/database/entities.dart';
 import 'package:squawker/generated/l10n.dart';
@@ -11,7 +11,6 @@ import 'package:squawker/group/_settings.dart';
 import 'package:squawker/ui/errors.dart';
 import 'package:squawker/utils/data_service.dart';
 import 'package:squawker/utils/iterables.dart';
-import 'package:quiver/iterables.dart';
 
 class GroupScreenArguments {
   final String id;
@@ -43,6 +42,47 @@ class SubscriptionGroupScreenContent extends StatelessWidget {
 
   SubscriptionGroupScreenContent({Key? key, required this.id}) : super(key: key);
 
+  String _buildSearchQuery(List<Subscription> users, bool includeReplies, bool includeRetweets) {
+    StringBuffer query = StringBuffer();
+    bool firstDone = false;
+
+    if (includeReplies) {
+      query.write('-filter:replies AND ');
+    }
+    if (!includeRetweets) {
+      query.write('-filter:retweets AND ');
+    } else {
+      query.write('include:nativeretweets AND ');
+    }
+    while (users.isNotEmpty) {
+      Subscription user = users[0];
+      String queryToAdd;
+
+      if (user is UserSubscription) {
+        queryToAdd = firstDone ? ' OR from:${user.screenName}' : 'from:${user.screenName}';
+      } else { // user is SearchSubscription
+        queryToAdd = firstDone ? ' OR ${user.id}' : user.id;
+      }
+      if (query.length + queryToAdd.length > 512) {
+        break;
+      }
+      query.write(queryToAdd);
+      users.removeAt(0);
+      firstDone = true;
+    }
+
+    return query.toString();
+  }
+
+  List<String> _buildSearchQueries(List<Subscription> users, bool includeReplies, bool includeRetweets) {
+    List<String> searchQueryLst = [];
+    while (users.isNotEmpty) {
+      String searchQuery = _buildSearchQuery(users, includeReplies, includeRetweets);
+      searchQueryLst.add(searchQuery);
+    }
+    return searchQueryLst;
+  }
+
   @override
   Widget build(BuildContext context) {
     return ScopedBuilder<GroupModel, SubscriptionGroupGet>.transition(
@@ -60,9 +100,7 @@ class SubscriptionGroupScreenContent extends StatelessWidget {
         var filteredUsers = group.id == '-1' ? group.subscriptions.where((elm) => elm.inFeed) : group.subscriptions;
         var users = filteredUsers.sorted((a, b) => a.createdAt.compareTo(b.createdAt)).toList();
 
-        var chunks = partition(users, 16)
-            .map((e) => SubscriptionGroupFeedChunk(e, group.includeReplies, group.includeRetweets))
-            .toList();
+        List<String> searchQueries = _buildSearchQueries(users, group.includeReplies, group.includeRetweets);
 
         GlobalKey<SubscriptionGroupFeedState>? sgfKey = DataService().map['feed_key_${group.id.replaceAll('-', '_')}'];
         if (sgfKey == null) {
@@ -73,7 +111,7 @@ class SubscriptionGroupScreenContent extends StatelessWidget {
         return SubscriptionGroupFeed(
           key: sgfKey,
           group: group,
-          chunks: chunks,
+          searchQueries: searchQueries,
           includeReplies: group.includeReplies,
           includeRetweets: group.includeRetweets,
           scrollController: scrollController,
@@ -83,20 +121,6 @@ class SubscriptionGroupScreenContent extends StatelessWidget {
   }
 }
 
-class SubscriptionGroupFeedChunk {
-  final List<Subscription> users;
-  final bool includeReplies;
-  final bool includeRetweets;
-
-  SubscriptionGroupFeedChunk(this.users, this.includeReplies, this.includeRetweets);
-
-  String get hash {
-    var toHash = '${users.map((e) => e.id).join(', ')}$includeReplies$includeRetweets';
-
-    return sha1.convert(toHash.codeUnits).toString();
-  }
-}
-
 class SubscriptionGroupScreen extends StatelessWidget {
   final ScrollController scrollController;
   final String id;

diff --git a/lib/settings/_account.dart b/lib/settings/_account.dart
@@ -94,13 +94,13 @@ class _SettingsAccountFragmentState extends State<SettingsAccountFragment> {
               shrinkWrap: true,
               itemBuilder: (BuildContext context, int index) {
                 List<String> infoLst = [];
-                if (_regularAccountsTokens[index].profile!.name != null) {
+                if (_regularAccountsTokens[index].profile!.name?.isNotEmpty ?? false) {
                   infoLst.add(_regularAccountsTokens[index].profile!.name!);
                 }
-                if (_regularAccountsTokens[index].profile!.email != null) {
+                if (_regularAccountsTokens[index].profile!.email?.isNotEmpty ?? false) {
                   infoLst.add(_regularAccountsTokens[index].profile!.email!);
                 }
-                if (_regularAccountsTokens[index].profile!.phone != null) {
+                if (_regularAccountsTokens[index].profile!.phone?.isNotEmpty ?? false) {
                   infoLst.add(_regularAccountsTokens[index].profile!.phone!);
                 }
                 return SwipeActionCell(
@@ -119,7 +119,7 @@ class _SettingsAccountFragmentState extends State<SettingsAccountFragment> {
                   ],
                   child: Card(
                     child: ListTile(
-                      leading:  const Icon(Icons.person),
+                      leading: const Icon(Icons.person),
                       title: Text(_regularAccountsTokens[index].screenName),
                       subtitle: infoLst.isEmpty ? null : Text(infoLst.join(', ')),
                     )

diff --git a/lib/utils/crypto_util.dart b/lib/utils/crypto_util.dart
@@ -1,6 +1,7 @@
 import 'dart:convert';
 import 'dart:math';
 import 'package:cryptography/cryptography.dart';
+import 'package:hex/hex.dart';
 
 const String oauthConsumerKey = '3nVuSoBZnx6U4vzUxf5w';
 const String oauthConsumerSecret = 'Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys';
@@ -50,3 +51,8 @@ Future<String> aesGcm256Decrypt(String key, String encryptedText) async {
   return utf8.decode(decryptedText);
 }
 
+Future<String> sha1Hash(String text) async {
+  final algorithm = Sha1();
+  final hash = await algorithm.hash(text.codeUnits);
+  return HEX.encode(hash.bytes);
+}
diff --git a/pubspec.lock b/pubspec.lock
@@ -509,6 +509,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "2.1.2"
+  hex:
+    dependency: "direct main"
+    description:
+      name: hex
+      sha256: "4e7cd54e4b59ba026432a6be2dd9d96e4c5205725194997193bf871703b82c4a"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.2.0"
   html:
     dependency: transitive
     description:

diff --git a/pubspec.yaml b/pubspec.yaml
@@ -16,7 +16,7 @@ publish_to: 'none' # Remove this line if you wish to publish to pub.dev
 # Read more about iOS versioning at
 # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html
 
-version: 3.7.2+36
+version: 3.7.3+37
 
 environment:
   sdk: ">=2.18.0 <3.0.0"
@@ -53,6 +53,7 @@ dependencies:
   flutter_swipe_action_cell: ^3.1.3
   flutter_triple: ^2.2.0
   flutter_windowmanager: ^0.2.0
+  hex: ^0.2.0
   html_unescape: ^2.0.0
   http: ^1.1.2
   infinite_scroll_pagination: ^4.0.0