[backend] Prevent prepareElementForIndexing to block event loop (#9748)

opencti-platform/opencti-graphql/src/database/engine.js

@@ -3864,39 +3864,68 @@ const createDeleteOperationElement = async (context, user, mainElement, deletedE
 };
 
 // TODO: get rid of this function and let elastic fail queries, so we can fix all of them by using the right type of data
-export const prepareElementForIndexing = (element) => {
+export const prepareElementForIndexing = async (element) => {
   const thing = {};
-  Object.keys(element).forEach((key) => {
+  const keyItems = Object.keys(element);
+  let startProcessingTime = new Date().getTime();
+  for (let index = 0; index < keyItems.length; index += 1) {
+    const key = keyItems[index];
     const value = element[key];
     if (Array.isArray(value)) { // Array of Date, objects, string or number
-      const filteredArray = value.filter((i) => i);
-      thing[key] = filteredArray.length > 0 ? filteredArray.map((f) => {
-        if (isDateAttribute(key)) { // Date is an object but natively supported
-          return f;
-        }
-        if (R.is(String, f)) { // For string, trim by default
-          return f.trim();
+      const preparedArray = [];
+      let innerProcessingTime = new Date().getTime();
+      let extendLoopSplit = 0;
+      for (let valueIndex = 0; valueIndex < value.length; valueIndex += 1) {
+        const valueElement = value[valueIndex];
+        if (valueElement) {
+          if (isDateAttribute(key)) { // Date is an object but natively supported
+            preparedArray.push(valueElement);
+          } else if (R.is(String, valueElement)) { // For string, trim by default
+            preparedArray.push(valueElement.trim());
+          } else if (R.is(Object, valueElement) && Object.keys(value).length > 0) { // For complex object, prepare inner elements
+            const complexPrepared = await prepareElementForIndexing(valueElement);
+            preparedArray.push(complexPrepared);
+          } else {
+            // For all other types, no transform (list of boolean is not supported)
+            preparedArray.push(valueElement);
+          }
         }
-        if (R.is(Object, f) && Object.keys(value).length > 0) { // For complex object, prepare inner elements
-          return prepareElementForIndexing(f);
+        // Prevent event loop locking more than MAX_EVENT_LOOP_PROCESSING_TIME
+        if (new Date().getTime() - innerProcessingTime > MAX_EVENT_LOOP_PROCESSING_TIME) {
+          // If we extends the preparation 5 times, log an error
+          // It will help to understand what kind of key have so much elements
+          if (extendLoopSplit === 5) {
+            logApp.error('[ENGINE] Element preparation too many values', { id: element.id, key, size: value.length });
+          }
+          extendLoopSplit += 1;
+          innerProcessingTime = new Date().getTime();
+          await new Promise((resolve) => {
+            setImmediate(resolve);
+          });
         }
-        // For all other types, no transform (list of boolean is not supported)
-        return f;
-      }) : [];
+      }
+      thing[key] = preparedArray;
     } else if (isDateAttribute(key)) { // Date is an object but natively supported
       thing[key] = value;
     } else if (isBooleanAttribute(key)) { // Patch field is string generic so need to be cast to boolean
       thing[key] = typeof value === 'boolean' ? value : value?.toLowerCase() === 'true';
     } else if (isNumericAttribute(key)) {
       thing[key] = isNotEmptyField(value) ? Number(value) : undefined;
     } else if (R.is(Object, value) && Object.keys(value).length > 0) { // For complex object, prepare inner elements
-      thing[key] = prepareElementForIndexing(value);
+      thing[key] = await prepareElementForIndexing(value);
     } else if (R.is(String, value)) { // For string, trim by default
       thing[key] = value.trim();
     } else { // For all other types (numeric, ...), no transform
       thing[key] = value;
     }
-  });
+    // Prevent event loop locking more than MAX_EVENT_LOOP_PROCESSING_TIME
+    if (new Date().getTime() - startProcessingTime > MAX_EVENT_LOOP_PROCESSING_TIME) {
+      startProcessingTime = new Date().getTime();
+      await new Promise((resolve) => {
+        setImmediate(resolve);
+      });
+    }
+  }
   return thing;
 };
 const prepareRelation = (thing) => {
@@ -4228,7 +4257,7 @@ const getInstanceToUpdate = async (context, user, instance) => {
 };
 export const elUpdateElement = async (context, user, instance) => {
   const instanceToUse = await getInstanceToUpdate(context, user, instance);
-  const esData = prepareElementForIndexing(instanceToUse);
+  const esData = await prepareElementForIndexing(instanceToUse);
   validateDataBeforeIndexing(esData);
   const dataToReplace = R.pipe(R.dissoc('representative'), R.dissoc('_id'))(esData);
   const replacePromise = elReplace(instanceToUse._index, instanceToUse._id ?? instanceToUse.internal_id, { doc: dataToReplace });

opencti-platform/opencti-graphql/src/migrations/1672652100502-entity-settings-mapping.js

@@ -50,7 +50,7 @@ export const up = async (next) => {
   settingsFromEl.platform_entities_files_ref = null;
   settingsFromEl.platform_hidden_types = null;
 
-  const esData = prepareElementForIndexing(settingsFromEl);
+  const esData = await prepareElementForIndexing(settingsFromEl);
   await elReplace(settingsFromEl._index, settingsFromEl.internal_id, { doc: esData });
 
   next();

opencti-platform/opencti-graphql/tests/01-unit/database/engine-test.js

@@ -2,26 +2,26 @@ import { describe, expect, it } from 'vitest';
 import { prepareElementForIndexing } from '../../../src/database/engine';
 
 describe('prepareElementForIndexing testing', () => {
-  it('should base trim applied', () => {
-    const element = prepareElementForIndexing({ name: '  test' });
+  it('should base trim applied', async () => {
+    const element = await prepareElementForIndexing({ name: '  test' });
     expect(element.name).toBe('test');
   });
-  it('should inner trim applied', () => {
-    const element = prepareElementForIndexing({ num: 10, data: { test: '  spacing   ' } });
+  it('should inner trim applied', async () => {
+    const element = await prepareElementForIndexing({ num: 10, data: { test: '  spacing   ' } });
     expect(element.num).toBe(10);
     expect(element.data.test).toBe('spacing');
   });
-  it('should array trim applied', () => {
-    const element = prepareElementForIndexing({ test: [20, '  trim01  ', '  trim 02    '] });
+  it('should array trim applied', async () => {
+    const element = await prepareElementForIndexing({ test: [20, '  trim01  ', '  trim 02    '] });
     expect(element.test).toEqual([20, 'trim01', 'trim 02']);
   });
-  it('should inner array trim applied', () => {
-    const element = prepareElementForIndexing({ test: { values: [20, '  trim01  ', '  trim 02    '] } });
+  it('should inner array trim applied', async () => {
+    const element = await prepareElementForIndexing({ test: { values: [20, '  trim01  ', '  trim 02    '] } });
     expect(element.test.values).toEqual([20, 'trim01', 'trim 02']);
   });
-  it('should do nothing with date value', () => {
+  it('should do nothing with date value', async () => {
     const now = new Date();
-    const element = prepareElementForIndexing({ date: now });
+    const element = await prepareElementForIndexing({ date: now });
     expect(element.date).toEqual(now);
   });
 });