csvLoader module

.codeclimate.yml

@@ -21,7 +21,7 @@ checks:
       threshold: 4
   similar-code:
     config:
-      threshold: 55
+      threshold: 70 
 plugins:
   eslint:
     enabled: true

.eslintrc

@@ -32,6 +32,7 @@
       "no-confusing-arrow": "off",
       "newline-per-chained-call": "off",
       "no-prototype-builtins": "off",
+      "implicit-arrow-linebreak": "off",
 
       "object-curly-spacing": ["warn", "always"],
       "max-len": ["error", 150],
@@ -64,7 +65,6 @@
       "no-empty": ["warn"],
       "no-cond-assign": ["warn"],
       "no-multiple-empty-lines": ["warn"],
-      "implicit-arrow-linebreak": ["warn"],
       "lines-between-class-members": ["warn"],
       "max-lines": ["warn", 250],
       "max-params": ["warn", 4],

app/api/attachments/specs/attachments.spec.js

@@ -79,7 +79,14 @@ describe('attachments', () => {
 
     it('should not delte the local file if other siblings are using it', async () => {
       expect(await fs.exists(`${paths.attachmentsPath}attachment.txt`)).toBe(true);
-      const sibling = { sharedId: toDeleteId.toString(), attachments: [{ filename: 'attachment.txt', originalname: 'common name 1.not' }] };
+      const sibling = {
+        title: 'title',
+        sharedId: toDeleteId.toString(),
+        attachments: [{
+          filename: 'attachment.txt',
+          originalname: 'common name 1.not'
+        }]
+      };
       await entities.saveMultiple([sibling]);
       const response = await attachments.delete(attachmentToDelete);
       const dbEntity = await entities.getById(toDeleteId);

app/api/attachments/specs/fixtures.js

@@ -12,6 +12,7 @@ const attachmentToEdit = db.id();
 export default {
   entities: [
     {
+      title: 'title',
       sharedId: toDeleteId.toString(),
       _id: toDeleteId,
       file: { originalname: 'source doc', filename: 'mainFile.txt' },
@@ -25,6 +26,7 @@ export default {
       ],
     },
     {
+      title: 'title',
       sharedId,
       _id: entityId,
       file: { originalname: 'source doc', filename: 'filename' },
@@ -38,6 +40,7 @@ export default {
       ],
     },
     {
+      title: 'title',
       sharedId,
       _id: entityIdEn,
       file: { originalname: 'source doc', filename: 'filenameEn' },
@@ -46,6 +49,7 @@ export default {
       ],
     },
     {
+      title: 'title',
       sharedId,
       _id: entityIdPt,
       file: { originalname: 'source doc', filename: 'filenamePt' },

app/api/csv/csvLoader.js

@@ -0,0 +1,73 @@
+import csv from 'csvtojson';
+
+import EventEmitter from 'events';
+import entities from 'api/entities';
+import fs from 'fs';
+import templates, { templateUtils } from 'api/templates';
+
+import typeParsers from './typeParsers';
+
+const toSafeName = rawEntity =>
+  Object.keys(rawEntity).reduce(
+    (translatedObject, key) => ({
+      ...translatedObject,
+      [templateUtils.safeName(key)]: rawEntity[key]
+    }),
+    {}
+  );
+
+const toMetadata = async (template, entityToImport) =>
+  template.properties
+  .filter(prop => entityToImport[prop.name])
+  .reduce(
+    async (meta, prop) => ({
+      ...(await meta),
+      [prop.name]: typeParsers[prop.type] ?
+        await typeParsers[prop.type](entityToImport, prop) :
+        await typeParsers.default(entityToImport, prop)
+    }),
+    Promise.resolve({})
+  );
+
+const importEntity = async (template, rawEntity, { user = {}, language }) =>
+  entities.save(
+    {
+      title: rawEntity.title,
+      template: template._id,
+      metadata: await toMetadata(template, rawEntity)
+    },
+    { user, language }
+  );
+
+export default class CSVLoader extends EventEmitter {
+  constructor() {
+    super();
+    this._errors = {};
+  }
+
+  errors() {
+    return this._errors;
+  }
+
+  async load(csvPath, templateId, options = { language: 'en' }) {
+    const template = await templates.getById(templateId);
+
+    await csv({
+      delimiter: [',', ';']
+    })
+    .fromStream(fs.createReadStream(csvPath))
+    .subscribe(async (rawEntity, index) => {
+      try {
+        const entity = await importEntity(template, toSafeName(rawEntity), options);
+        this.emit('entityLoaded', entity);
+      } catch (e) {
+        this._errors[index] = e;
+        this.emit('loadError', e, toSafeName(rawEntity));
+      }
+    });
+
+    if (Object.keys(this._errors).length) {
+      throw new Error('errors ocurred !');
+    }
+  }
+}

app/api/csv/index.js

@@ -0,0 +1,3 @@
+import csvLoader from './csvLoader.js';
+
+export default csvLoader;

app/api/csv/specs/csvLoader.spec.js

@@ -0,0 +1,172 @@
+import db from 'api/utils/testing_db';
+import entities from 'api/entities';
+import path from 'path';
+import translations from 'api/i18n';
+
+import CSVLoader from '../csvLoader';
+import fixtures, { template1Id } from './fixtures';
+import typeParsers from '../typeParsers';
+
+describe('csvLoader', () => {
+  afterAll(async () => db.disconnect());
+  const csvFile = path.join(__dirname, '/test.csv');
+  const loader = new CSVLoader();
+
+  typeParsers.select = () => Promise.resolve('thesauri');
+  typeParsers.text = () => Promise.resolve('text');
+  typeParsers.default = () => Promise.resolve('default');
+
+  beforeAll(async () => {
+    await db.clearAllAndLoad(fixtures);
+    spyOn(entities, 'indexEntities').and.returnValue(Promise.resolve());
+    spyOn(translations, 'updateContext').and.returnValue(Promise.resolve());
+  });
+
+  it('should use the passed language', async () => {
+    spyOn(entities, 'save').and.returnValue(Promise.resolve({}));
+    try {
+      await loader.load(csvFile, template1Id, { language: 'es' });
+    } catch (e) {
+      throw loader.errors()[Object.keys(loader.errors())[0]];
+    }
+    expect(entities.save.calls.argsFor(0)[1].language).toBe('es');
+  });
+
+  it('should use the passed user', async () => {
+    spyOn(entities, 'save').and.returnValue(Promise.resolve({}));
+    try {
+      await loader.load(csvFile, template1Id, { user: { username: 'user' } });
+    } catch (e) {
+      throw loader.errors()[Object.keys(loader.errors())[0]];
+    }
+    expect(entities.save.calls.argsFor(0)[1].user).toEqual({ username: 'user' });
+  });
+
+  describe('load', () => {
+    let imported;
+    const events = [];
+
+    beforeAll(async () => {
+      loader.on('entityLoaded', (entity) => {
+        events.push(entity.title);
+      });
+
+      try {
+        await loader.load(csvFile, template1Id);
+      } catch (e) {
+        throw loader.errors()[Object.keys(loader.errors())[0]];
+      }
+
+      imported = await entities.get();
+    });
+
+
+    it('should load title', () => {
+      const textValues = imported.map(i => i.title);
+      expect(textValues).toEqual(['title1', 'title2', 'title3']);
+    });
+
+    it('should emit event after each entity has been imported', () => {
+      expect(events).toEqual(['title1', 'title2', 'title3']);
+    });
+
+    it('should only import valid metadata', () => {
+      const metadataImported = Object.keys(imported[0].metadata);
+      expect(metadataImported).toEqual([
+        'text_label',
+        'select_label',
+        'not_defined_type'
+      ]);
+    });
+
+    it('should ignore properties not configured in the template', () => {
+      const textValues = imported
+      .map(i => i.metadata.non_configured)
+      .filter(i => i);
+
+      expect(textValues.length).toEqual(0);
+    });
+
+    describe('metadata parsing', () => {
+      it('should parse metadata properties by type using typeParsers', () => {
+        const textValues = imported.map(i => i.metadata.text_label);
+        expect(textValues).toEqual(['text', 'text', 'text']);
+
+        const thesauriValues = imported.map(i => i.metadata.select_label);
+        expect(thesauriValues).toEqual(['thesauri', 'thesauri', 'thesauri']);
+      });
+
+      describe('when parser not defined', () => {
+        it('should use default parser', () => {
+          const noTypeValues = imported.map(i => i.metadata.not_defined_type);
+          expect(noTypeValues).toEqual(['default', 'default', 'default']);
+        });
+      });
+    });
+  });
+
+  describe('when errors happen', () => {
+    beforeAll(async () => {
+      spyOn(entities, 'save').and.callFake((entity) => {
+        if (entity.title === 'title1' || entity.title === 'title3') {
+          return Promise.reject(new Error(`error-${entity.title}`));
+        }
+        return Promise.resolve();
+      });
+      await db.clearAllAndLoad(fixtures);
+    });
+
+    it('should emit an error', async () => {
+      const loader = new CSVLoader();
+
+      const eventErrors = {};
+      loader.on('errorLoading', (error, entity) => {
+        eventErrors[entity.title] = error;
+      });
+
+      try {
+        await loader.load(csvFile, template1Id);
+      } catch (e) {
+        expect(eventErrors).toEqual({
+          title1: new Error('error-title1'),
+          title3: new Error('error-title3'),
+        });
+      }
+    });
+
+    it('should save errors and index them by csv line, should throw an error on finish', async () => {
+      const loader = new CSVLoader();
+
+      try {
+        await loader.load(csvFile, template1Id);
+        fail('should fail');
+      } catch (e) {
+        expect(loader.errors()).toEqual({
+          0: new Error('error-title1'),
+          2: new Error('error-title3'),
+        });
+      }
+    });
+
+    it('should fail when parsing throws an error', async () => {
+      entities.save.and.callFake(() => Promise.resolve());
+      spyOn(typeParsers, 'text').and.callFake((entity) => {
+        if (entity.title === 'title2') {
+          return Promise.reject(new Error(`error-${entity.title}`));
+        }
+        return Promise.resolve();
+      });
+
+      const loader = new CSVLoader();
+
+      try {
+        await loader.load(csvFile, template1Id);
+        fail('should fail');
+      } catch (e) {
+        expect(loader.errors()).toEqual({
+          1: new Error('error-title2'),
+        });
+      }
+    });
+  });
+});

app/api/csv/specs/fixtures.js

@@ -0,0 +1,68 @@
+import db from 'api/utils/testing_db';
+import { templateTypes } from 'shared/templateTypes';
+import { templateUtils } from 'api/templates';
+
+const template1Id = db.id();
+const thesauri1Id = db.id();
+const templateToRelateId = db.id();
+
+export default {
+  templates: [
+    {
+      _id: templateToRelateId,
+      name: 'template to relate',
+      properties: []
+    },
+    {
+      _id: template1Id,
+      name: 'base template',
+      properties: [
+        {
+          type: templateTypes.text,
+          label: 'text label',
+          name: templateUtils.safeName('text label'),
+        },
+        {
+          type: templateTypes.select,
+          label: 'select label',
+          name: templateUtils.safeName('select label'),
+          content: thesauri1Id,
+        },
+        {
+          type: 'non_defined_type',
+          label: 'not defined type',
+          name: templateUtils.safeName('not defined type'),
+        },
+        {
+          type: templateTypes.text,
+          label: 'not configured on csv',
+          name: templateUtils.safeName('not configured on csv'),
+        },
+      ]
+    },
+  ],
+
+  dictionaries: [
+    {
+      _id: thesauri1Id,
+      name: 'thesauri1',
+      values: [],
+    }
+  ],
+
+  settings: [
+    {
+      _id: db.id(),
+      site_name: 'Uwazi',
+      languages: [
+        { key: 'en', label: 'English', default: true },
+      ]
+    }
+  ]
+};
+
+export {
+  template1Id,
+  thesauri1Id,
+  templateToRelateId
+};

app/api/csv/specs/test.csv

@@ -0,0 +1,5 @@
+Title , text label  , non configured, select label, not defined type,
+
+title1, text value 1, ______________, thesauri1   , notType1        ,
+title2, text value 2, ______________, thesauri2   , notType2        ,
+title3, text value 3, ______________, thesauri2   , notType3        ,