From 8ee8f841bf8195c54ddb5f3f114cca18046853ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20Sa=CC=88rkikoski?= Date: Thu, 7 Nov 2024 14:13:09 +0200 Subject: [PATCH] Reduce indexing batch size. Specify retry count in Elasticsearch NEST client. --- aspnetcore/src/ElasticService/ElasticSearchExtensions.cs | 4 +++- aspnetcore/src/ElasticService/ElasticSearchIndexService.cs | 2 +- aspnetcore/src/Indexer/Indexer.cs | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/aspnetcore/src/ElasticService/ElasticSearchExtensions.cs b/aspnetcore/src/ElasticService/ElasticSearchExtensions.cs index 72780da..34d954b 100644 --- a/aspnetcore/src/ElasticService/ElasticSearchExtensions.cs +++ b/aspnetcore/src/ElasticService/ElasticSearchExtensions.cs @@ -36,8 +36,10 @@ private static ConnectionSettings GetConnectionSettings(IConfiguration configura { var elasticSearchClusterUrl = configuration["ElasticSearch:Url"] ?? throw new InvalidOperationException("ElasticSearch url missing."); + // https://www.elastic.co/guide/en/elasticsearch/client/net-api/7.17/configuration-options.html var connectionSettings = new ConnectionSettings(new Uri(elasticSearchClusterUrl)) - .DefaultFieldNameInferrer(i => i); // This forces elastic to store .Net objects using type names, instead of camel casing. This enables using nameof when referring to fields. + .DefaultFieldNameInferrer(i => i) // This forces elastic to store .Net objects using type names, instead of camel casing. This enables using nameof when referring to fields. + .MaximumRetries(5); if (Debugger.IsAttached) { diff --git a/aspnetcore/src/ElasticService/ElasticSearchIndexService.cs b/aspnetcore/src/ElasticService/ElasticSearchIndexService.cs index 59a5e9e..d1d0d01 100644 --- a/aspnetcore/src/ElasticService/ElasticSearchIndexService.cs +++ b/aspnetcore/src/ElasticService/ElasticSearchIndexService.cs @@ -8,7 +8,7 @@ public class ElasticSearchIndexService : IElasticSearchIndexService private readonly IElasticClient _elasticClient; private readonly ILogger _logger; - private const int BatchSize = 2000; + private const int BatchSize = 1500; public ElasticSearchIndexService(IElasticClient elasticClient, ILogger logger) { diff --git a/aspnetcore/src/Indexer/Indexer.cs b/aspnetcore/src/Indexer/Indexer.cs index 26f7e46..5c8ed9c 100644 --- a/aspnetcore/src/Indexer/Indexer.cs +++ b/aspnetcore/src/Indexer/Indexer.cs @@ -148,7 +148,7 @@ private async Task IndexEntities(string indexName, IIndexRepository repository, */ int skipAmount = 0; - int takeAmount = 50000; + int takeAmount = 30000; int numOfResults = 0; int processedCount = 0;