diff --git a/opencti-platform/opencti-graphql/src/database/engine.js b/opencti-platform/opencti-graphql/src/database/engine.js index e1d73bac5366..fbca2cb9ecdb 100644 --- a/opencti-platform/opencti-graphql/src/database/engine.js +++ b/opencti-platform/opencti-graphql/src/database/engine.js @@ -3864,25 +3864,47 @@ const createDeleteOperationElement = async (context, user, mainElement, deletedE }; // TODO: get rid of this function and let elastic fail queries, so we can fix all of them by using the right type of data -export const prepareElementForIndexing = (element) => { +export const prepareElementForIndexing = async (element) => { const thing = {}; - Object.keys(element).forEach((key) => { + const keyItems = Object.keys(element); + let startProcessingTime = new Date().getTime(); + for (let index = 0; index < keyItems.length; index += 1) { + const key = keyItems[index]; const value = element[key]; if (Array.isArray(value)) { // Array of Date, objects, string or number - const filteredArray = value.filter((i) => i); - thing[key] = filteredArray.length > 0 ? filteredArray.map((f) => { - if (isDateAttribute(key)) { // Date is an object but natively supported - return f; - } - if (R.is(String, f)) { // For string, trim by default - return f.trim(); + const preparedArray = []; + let innerProcessingTime = new Date().getTime(); + let extendLoopSplit = 0; + for (let valueIndex = 0; valueIndex < value.length; valueIndex += 1) { + const valueElement = value[valueIndex]; + if (valueElement) { + if (isDateAttribute(key)) { // Date is an object but natively supported + preparedArray.push(valueElement); + } else if (R.is(String, valueElement)) { // For string, trim by default + preparedArray.push(valueElement.trim()); + } else if (R.is(Object, valueElement) && Object.keys(value).length > 0) { // For complex object, prepare inner elements + const complexPrepared = await prepareElementForIndexing(valueElement); + preparedArray.push(complexPrepared); + } else { + // For all other types, no transform (list of boolean is not supported) + preparedArray.push(valueElement); + } } - if (R.is(Object, f) && Object.keys(value).length > 0) { // For complex object, prepare inner elements - return prepareElementForIndexing(f); + // Prevent event loop locking more than MAX_EVENT_LOOP_PROCESSING_TIME + if (new Date().getTime() - innerProcessingTime > MAX_EVENT_LOOP_PROCESSING_TIME) { + // If we extends the preparation 5 times, log an error + // It will help to understand what kind of key have so much elements + if (extendLoopSplit === 5) { + logApp.error('[ENGINE] Element preparation too many values', { id: element.id, key, size: value.length }); + } + extendLoopSplit += 1; + innerProcessingTime = new Date().getTime(); + await new Promise((resolve) => { + setImmediate(resolve); + }); } - // For all other types, no transform (list of boolean is not supported) - return f; - }) : []; + } + thing[key] = preparedArray; } else if (isDateAttribute(key)) { // Date is an object but natively supported thing[key] = value; } else if (isBooleanAttribute(key)) { // Patch field is string generic so need to be cast to boolean @@ -3890,13 +3912,20 @@ export const prepareElementForIndexing = (element) => { } else if (isNumericAttribute(key)) { thing[key] = isNotEmptyField(value) ? Number(value) : undefined; } else if (R.is(Object, value) && Object.keys(value).length > 0) { // For complex object, prepare inner elements - thing[key] = prepareElementForIndexing(value); + thing[key] = await prepareElementForIndexing(value); } else if (R.is(String, value)) { // For string, trim by default thing[key] = value.trim(); } else { // For all other types (numeric, ...), no transform thing[key] = value; } - }); + // Prevent event loop locking more than MAX_EVENT_LOOP_PROCESSING_TIME + if (new Date().getTime() - startProcessingTime > MAX_EVENT_LOOP_PROCESSING_TIME) { + startProcessingTime = new Date().getTime(); + await new Promise((resolve) => { + setImmediate(resolve); + }); + } + } return thing; }; const prepareRelation = (thing) => { @@ -4228,7 +4257,7 @@ const getInstanceToUpdate = async (context, user, instance) => { }; export const elUpdateElement = async (context, user, instance) => { const instanceToUse = await getInstanceToUpdate(context, user, instance); - const esData = prepareElementForIndexing(instanceToUse); + const esData = await prepareElementForIndexing(instanceToUse); validateDataBeforeIndexing(esData); const dataToReplace = R.pipe(R.dissoc('representative'), R.dissoc('_id'))(esData); const replacePromise = elReplace(instanceToUse._index, instanceToUse._id ?? instanceToUse.internal_id, { doc: dataToReplace }); diff --git a/opencti-platform/opencti-graphql/src/migrations/1672652100502-entity-settings-mapping.js b/opencti-platform/opencti-graphql/src/migrations/1672652100502-entity-settings-mapping.js index fc956f6e4933..c8c5753b5dd7 100644 --- a/opencti-platform/opencti-graphql/src/migrations/1672652100502-entity-settings-mapping.js +++ b/opencti-platform/opencti-graphql/src/migrations/1672652100502-entity-settings-mapping.js @@ -50,7 +50,7 @@ export const up = async (next) => { settingsFromEl.platform_entities_files_ref = null; settingsFromEl.platform_hidden_types = null; - const esData = prepareElementForIndexing(settingsFromEl); + const esData = await prepareElementForIndexing(settingsFromEl); await elReplace(settingsFromEl._index, settingsFromEl.internal_id, { doc: esData }); next(); diff --git a/opencti-platform/opencti-graphql/tests/01-unit/database/engine-test.js b/opencti-platform/opencti-graphql/tests/01-unit/database/engine-test.js index 466a8b132722..cb1df7214a22 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/database/engine-test.js +++ b/opencti-platform/opencti-graphql/tests/01-unit/database/engine-test.js @@ -2,26 +2,26 @@ import { describe, expect, it } from 'vitest'; import { prepareElementForIndexing } from '../../../src/database/engine'; describe('prepareElementForIndexing testing', () => { - it('should base trim applied', () => { - const element = prepareElementForIndexing({ name: ' test' }); + it('should base trim applied', async () => { + const element = await prepareElementForIndexing({ name: ' test' }); expect(element.name).toBe('test'); }); - it('should inner trim applied', () => { - const element = prepareElementForIndexing({ num: 10, data: { test: ' spacing ' } }); + it('should inner trim applied', async () => { + const element = await prepareElementForIndexing({ num: 10, data: { test: ' spacing ' } }); expect(element.num).toBe(10); expect(element.data.test).toBe('spacing'); }); - it('should array trim applied', () => { - const element = prepareElementForIndexing({ test: [20, ' trim01 ', ' trim 02 '] }); + it('should array trim applied', async () => { + const element = await prepareElementForIndexing({ test: [20, ' trim01 ', ' trim 02 '] }); expect(element.test).toEqual([20, 'trim01', 'trim 02']); }); - it('should inner array trim applied', () => { - const element = prepareElementForIndexing({ test: { values: [20, ' trim01 ', ' trim 02 '] } }); + it('should inner array trim applied', async () => { + const element = await prepareElementForIndexing({ test: { values: [20, ' trim01 ', ' trim 02 '] } }); expect(element.test.values).toEqual([20, 'trim01', 'trim 02']); }); - it('should do nothing with date value', () => { + it('should do nothing with date value', async () => { const now = new Date(); - const element = prepareElementForIndexing({ date: now }); + const element = await prepareElementForIndexing({ date: now }); expect(element.date).toEqual(now); }); });