Merge pull request #160 from CSCfi/indexer-improvements

Reduce indexing batch size. Specify retry count in NEST client
CSCfi · Nov 7, 2024 · c0c589d · c0c589d
2 parents 4e2a499 + 8ee8f84
commit c0c589d
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 3 deletions.
diff --git a/aspnetcore/src/ElasticService/ElasticSearchExtensions.cs b/aspnetcore/src/ElasticService/ElasticSearchExtensions.cs
@@ -36,8 +36,10 @@ private static ConnectionSettings GetConnectionSettings(IConfiguration configura
     {
         var elasticSearchClusterUrl = configuration["ElasticSearch:Url"] ?? throw new InvalidOperationException("ElasticSearch url missing.");
 
+        // https://www.elastic.co/guide/en/elasticsearch/client/net-api/7.17/configuration-options.html
         var connectionSettings = new ConnectionSettings(new Uri(elasticSearchClusterUrl))
-            .DefaultFieldNameInferrer(i => i); // This forces elastic to store .Net objects using type names, instead of camel casing. This enables using nameof when referring to fields.
+            .DefaultFieldNameInferrer(i => i) // This forces elastic to store .Net objects using type names, instead of camel casing. This enables using nameof when referring to fields.
+            .MaximumRetries(5);
 
         if (Debugger.IsAttached)
         {

diff --git a/aspnetcore/src/ElasticService/ElasticSearchIndexService.cs b/aspnetcore/src/ElasticService/ElasticSearchIndexService.cs
@@ -8,7 +8,7 @@ public class ElasticSearchIndexService : IElasticSearchIndexService
     private readonly IElasticClient _elasticClient;
     private readonly ILogger<ElasticSearchIndexService> _logger;
 
-    private const int BatchSize = 2000;
+    private const int BatchSize = 1500;
 
     public ElasticSearchIndexService(IElasticClient elasticClient, ILogger<ElasticSearchIndexService> logger)
     {

diff --git a/aspnetcore/src/Indexer/Indexer.cs b/aspnetcore/src/Indexer/Indexer.cs
@@ -148,7 +148,7 @@ private async Task IndexEntities(string indexName, IIndexRepository repository,
                 */
 
                 int skipAmount = 0;
-                int takeAmount = 50000;
+                int takeAmount = 30000;
                 int numOfResults = 0;
                 int processedCount = 0;