From e15965d2250516a9c537933b6429ee4a4254ecde Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Mon, 19 Feb 2024 15:32:30 +1300 Subject: [PATCH 1/4] Earlier exit when failing to fetch blocks with workers --- packages/node-core/src/indexer/worker/worker.service.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/node-core/src/indexer/worker/worker.service.ts b/packages/node-core/src/indexer/worker/worker.service.ts index e315c5df1d..6c2aff4c43 100644 --- a/packages/node-core/src/indexer/worker/worker.service.ts +++ b/packages/node-core/src/indexer/worker/worker.service.ts @@ -69,6 +69,7 @@ export abstract class BaseWorkerService< return; } logger.error(e, `Failed to fetch block ${height}`); + throw e; } } From a5cc51b17f735f802cc6361504937b1c07d9f6a3 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Mon, 19 Feb 2024 15:34:56 +1300 Subject: [PATCH 2/4] Update changelog --- packages/node-core/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md index 24667e710d..f9009b2dae 100644 --- a/packages/node-core/CHANGELOG.md +++ b/packages/node-core/CHANGELOG.md @@ -7,10 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - Schema Migration support for Enums, Relations, Subscription (#2251) + ### Fixed - Fixed non-atomic schema migration execution (#2244) - Testing Suites should run with unfinalizedBlocks `false` (#2258) +### Changed +- Throw error earlier when failing to fetch blocks when workers are enabled (#2256) + ## [7.2.1] - 2024-02-07 ### Added - Update `ParentProject` to use `untilBlock` as and alias for `block` (#2235) From 78ab10e4e2ffe60fce793f2b4ba038b6f2eab6b1 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Tue, 20 Feb 2024 09:45:43 +1300 Subject: [PATCH 3/4] Improve code reuse and error handling --- packages/node-core/src/api.service.ts | 22 ++++++++++++++---- .../blockDispatcher/block-dispatcher.ts | 2 +- packages/node/src/indexer/api.service.ts | 23 +++++-------------- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/packages/node-core/src/api.service.ts b/packages/node-core/src/api.service.ts index 208acbbd5d..5d938ccce6 100644 --- a/packages/node-core/src/api.service.ts +++ b/packages/node-core/src/api.service.ts @@ -43,18 +43,30 @@ export abstract class ApiService = any[] private timeouts: Record = {}; async fetchBlocks(heights: number[], numAttempts = MAX_RECONNECT_ATTEMPTS): Promise { + return this.retryFetch(async () => { + // Get the latest fetch function from the provider + const apiInstance = this.connectionPoolService.api; + return apiInstance.fetchBlocks(heights); + }, numAttempts); + } + + protected async retryFetch(fn: () => Promise, numAttempts = MAX_RECONNECT_ATTEMPTS): Promise { let reconnectAttempts = 0; + let lastError: Error | null = null; while (reconnectAttempts < numAttempts) { try { - // Get the latest fetch function from the provider - const apiInstance = this.connectionPoolService.api; - return await apiInstance.fetchBlocks(heights); + return await fn(); } catch (e: any) { - logger.error(e, `Failed to fetch blocks ${heights[0]}...${heights[heights.length - 1]}`); - + lastError = e; reconnectAttempts++; } } + if (lastError !== null) { + logger.error( + `Maximum number of retries (${numAttempts}) reached. See the following error for the underlying reason.` + ); + throw lastError; + } throw new Error(`Maximum number of retries (${numAttempts}) reached.`); } diff --git a/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts index a2d870d43c..8ebe3dc331 100644 --- a/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts @@ -206,7 +206,7 @@ export abstract class BlockDispatcher // Do nothing, fetching the block was flushed, this could be caused by forked blocks or dynamic datasources return; } - logger.warn(e, 'Failed to enqueue fetched block to process'); + logger.error(e, 'Failed to enqueue fetched block to process'); process.exit(1); }); diff --git a/packages/node/src/indexer/api.service.ts b/packages/node/src/indexer/api.service.ts index 524e38e0d8..16b647a59d 100644 --- a/packages/node/src/indexer/api.service.ts +++ b/packages/node/src/indexer/api.service.ts @@ -263,27 +263,16 @@ export class ApiService return `api.rpc.${ext?.section ?? '*'}.${ext?.method ?? '*'}`; } + // Overrides the super function because of the specVer async fetchBlocks( heights: number[], overallSpecVer?: number, numAttempts = MAX_RECONNECT_ATTEMPTS, ): Promise { - let reconnectAttempts = 0; - while (reconnectAttempts < numAttempts) { - try { - const apiInstance = this.connectionPoolService.api; - return await apiInstance.fetchBlocks(heights, overallSpecVer); - } catch (e: any) { - logger.error( - e, - `Failed to fetch blocks ${heights[0]}...${ - heights[heights.length - 1] - }`, - ); - - reconnectAttempts++; - } - } - throw new Error(`Maximum number of retries (${numAttempts}) reached.`); + return this.retryFetch(async () => { + // Get the latest fetch function from the provider + const apiInstance = this.connectionPoolService.api; + return apiInstance.fetchBlocks(heights, overallSpecVer); + }, numAttempts); } } From 2e37b667cacadce45caf4b43eaa42f233070c34d Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Tue, 20 Feb 2024 09:48:02 +1300 Subject: [PATCH 4/4] Update changelog --- packages/node-core/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md index f9009b2dae..753e733386 100644 --- a/packages/node-core/CHANGELOG.md +++ b/packages/node-core/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Testing Suites should run with unfinalizedBlocks `false` (#2258) ### Changed -- Throw error earlier when failing to fetch blocks when workers are enabled (#2256) +- Improve error handling when fetching blocks (#2256) ## [7.2.1] - 2024-02-07 ### Added