Update sailor & fixes (#67)

* update * update * update * merge origin update-object * merge origin update-object * merge origin update-object * merge origin update-object * merge origin update-object * merge origin update-object * merge origin update-object * merge origin update-object * update * update * update * update * Update component.json Co-authored-by: Oleg Oshurkov <[email protected]>
elasticio · May 12, 2022 · f12fd33 · f12fd33
1 parent d559561
commit f12fd33
Show file tree

Hide file tree

Showing 10 changed files with 1,549 additions and 339 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,24 +1,125 @@
-version: 2
+version: 2.1
+parameters:
+  node-version:
+    type: string
+    default: "16.13.2"
+orbs:
+  node: circleci/[email protected]
+  slack: circleci/[email protected]
+commands:
+  notify_on_failure:
+    steps:
+      - slack/notify:
+          event: fail
+          custom: |
+            {
+            	"blocks": [
+            		{
+            			"type": "section",
+            			"fields": [
+            				{
+            					"type": "mrkdwn",
+            					"text": ":red_circle: *$CIRCLE_PROJECT_REPONAME*:*$CIRCLE_TAG* build failed"
+            				}
+            			]
+            		},
+            		{
+            			"type": "actions",
+            			"elements": [
+            				{
+            					"type": "button",
+            					"text": {
+            						"type": "plain_text",
+            						"text": "View Job"
+            					},
+            					"url": "${CIRCLE_BUILD_URL}"
+            				}
+            			]
+            		}
+            	]
+            }
+  notify_on_pass:
+    steps:
+      - slack/notify:
+          event: pass
+          custom: |
+            {
+            	"blocks": [
+            		{
+            			"type": "section",
+            			"fields": [
+            				{
+            					"type": "mrkdwn",
+            					"text": ":tada: *$CIRCLE_PROJECT_REPONAME*:*$CIRCLE_TAG* was successfully built and published"
+            				}
+            			]
+            		},
+            		{
+            			"type": "actions",
+            			"elements": [
+            				{
+            					"type": "button",
+            					"text": {
+            						"type": "plain_text",
+            						"text": "View Job"
+            					},
+            					"url": "${CIRCLE_BUILD_URL}"
+            				}
+            			]
+            		}
+            	]
+            }
 jobs:
   test:
     docker:
       - image: circleci/node:14-stretch
     steps:
       - checkout
-      - restore_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
+      - node/install:
+          node-version: << pipeline.parameters.node-version >>
+      # - run:
+      #     name: Audit Dependencies
+      #     command: npm audit --audit-level=high
+      - node/install-packages:
+          cache-path: ./node_modules
+          override-ci-command: npm install
       - run:
-          name: Installing Dependencies
-          command: npm install
-      - save_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
-          paths:
-            - node_modules
+          name: Running Mocha Tests
+          command: npm test
+  build:
+    docker:
+      - image: circleci/node:14-stretch
+        user: root
+    steps:
+      - checkout
+      - node/install:
+          node-version: << pipeline.parameters.node-version >>
+      - setup_remote_docker:
+          version: 19.03.13
+          docker_layer_caching: true
+      # build and push Docker image
+      - run:
+          name: Install component-build-helper lib
+          command: npm install -g @elastic.io/component-build-helper
       - run:
-          name: Running Mocha Unit&Integration Tests
-          command: npm test && npm run integration-test
+          name: Build and publish docker image
+          command: build_component_docker
+      - notify_on_failure
+      - notify_on_pass
 workflows:
-  version: 2
-  build_and_test:
+  test:
+    jobs:
+      - test:
+          name: "Running tests"
+          filters:
+            tags:
+              ignore: /.*/
+  publish_release:
     jobs:
-      - test
+      - build:
+          name: "Build and publish docker image"
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/
diff --git a/.env.example b/.env.example
@@ -0,0 +1,3 @@
+ACCESS_KEY_ID=DKIAJJ5GKdfgdfgFEn
+ACCESS_KEY_SECRET=gergji37fhkds4f34cf34c3fds4
+REGION=ca-central-1
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,11 @@
+# 1.4.3 (April 22, 2022)
+
+* Update `component-commons-library` to 2.0.2
+* Update `oih-standard-library` to 2.0.2
+* Update `elasticio-sailor-nodejs` to 2.6.27
+* Added component-pusher to circleci
+* Fix dependencies
+
 # 1.4.2 (November 26, 2021)
 
 * Upgrade sailor version to 2.6.26

diff --git a/component.json b/component.json
@@ -1,8 +1,7 @@
 {
   "title": "AWS S3",
-  "version": "1.4.2",
+  "version": "1.4.3",
   "description": "Integration component that can read and write to AWS S3",
-  "buildType": "docker",
   "docsUrl": "https://github.com/elasticio/amazon-s3-component",
   "credentials": {
     "fields": {
@@ -134,7 +133,7 @@
             },
             "bucketName": {
               "type": "string",
-              "title":"Bucket Name and folder",
+              "title": "Bucket Name and folder",
               "required": false
             }
           }
@@ -187,7 +186,7 @@
               "required": true
             },
             "bucketName": {
-              "title":"Bucket Name and folder",
+              "title": "Bucket Name and folder",
               "type": "string",
               "required": false
             }
@@ -225,7 +224,7 @@
           "type": "object",
           "properties": {
             "bucketName": {
-              "title":"Bucket Name and folder",
+              "title": "Bucket Name and folder",
               "type": "string",
               "required": false
             }
@@ -267,7 +266,7 @@
               "required": true
             },
             "bucketName": {
-              "title":"Bucket Name and Folder",
+              "title": "Bucket Name and Folder",
               "type": "string",
               "required": false
             }
@@ -296,7 +295,7 @@
           "type": "object",
           "properties": {
             "bucketName": {
-              "title":"Bucket Name",
+              "title": "Bucket Name",
               "type": "string",
               "required": true
             },

diff --git a/lib/actions/readFile.js b/lib/actions/readFile.js
@@ -1,12 +1,11 @@
-/* eslint-disable no-use-before-define,consistent-return,func-names */
+/* eslint-disable no-use-before-define,consistent-return,func-names,no-param-reassign */
+const { AttachmentProcessor } = require('@elastic.io/component-commons-library');
 const { messages } = require('elasticio-node');
 const convert = require('xml-js');
 const mime = require('mime-types');
 const iconv = require('iconv-lite');
 const { Client } = require('../client');
-const params = require('../parameters');
-
-const attachmentProcessor = require('../utils/attachmentProcessor');
+const { REQUEST_MAX_BODY_LENGTH } = require('../parameters');
 
 exports.process = async function (msg, cfg) {
   const client = new Client(this.logger, cfg);
@@ -15,11 +14,11 @@ exports.process = async function (msg, cfg) {
 
   const result = await client.getObject(bucketName, filename);
 
-  if (result.ContentLength > params.ATTACHMENT_MAX_SIZE) {
+  if (result.ContentLength > REQUEST_MAX_BODY_LENGTH) {
     this.logger.error('File %s with size %d bytes is too big for attachment usage. '
-      + 'Current attachment max size is %d bytes', filename, result.ContentLength, params.ATTACHMENT_MAX_SIZE);
+      + 'Current attachment max size is %d bytes', filename, result.ContentLength, REQUEST_MAX_BODY_LENGTH);
     throw new Error(`File ${filename} with size ${result.ContentLength} bytes is too big for attachment usage. `
-      + `Current attachment max size is ${params.ATTACHMENT_MAX_SIZE} bytes`);
+      + `Current attachment max size is ${REQUEST_MAX_BODY_LENGTH} bytes`);
   }
 
   const fileContent = iconv.decode(result.Body, 'iso-8859-15');
@@ -32,10 +31,18 @@ exports.process = async function (msg, cfg) {
     const xmlDoc = JSON.parse(convert.xml2json(fileContent));
     await this.emit('data', messages.newMessageWithBody(xmlDoc));
   } else {
-    const response = await attachmentProcessor.addAttachment.call(this, msg, filename,
-      fileContent, contentType);
-    const output = messages.newMessageWithBody(response);
-    output.attachments = response.attachments;
+    const attachmentProcessor = new AttachmentProcessor();
+    const response = await attachmentProcessor.uploadAttachment(result.Body, contentType);
+    const attachmentUrl = `${response.config.url}${response.data.objectId}?storage_type=maester`;
+    msg.attachments = {
+      [filename]: {
+        url: attachmentUrl,
+        size: fileContent.length,
+        'content-type': contentType,
+      },
+    };
+    const output = messages.newMessageWithBody(msg);
+    output.attachments = msg.attachments;
     return output;
   }
 };
diff --git a/lib/parameters.js b/lib/parameters.js
@@ -1 +1 @@
-exports.ATTACHMENT_MAX_SIZE = process.env.ATTACHMENT_MAX_SIZE ? parseInt(process.env.ATTACHMENT_MAX_SIZE, 10) : 10000000;
+exports.REQUEST_MAX_BODY_LENGTH = process.env.REQUEST_MAX_BODY_LENGTH ? parseInt(process.env.REQUEST_MAX_BODY_LENGTH, 10) : 104857600; // 100MB
diff --git a/lib/utils/attachmentProcessor.js b/lib/utils/attachmentProcessor.js
diff --git a/lib/utils/pollingUtil.js b/lib/utils/pollingUtil.js
@@ -72,13 +72,14 @@ class AwsS3Polling extends PollingTrigger {
   async s3FileToAttachment(msg, filename, size) {
     this.logger.info('Adding file to attachment...');
     const s3Stream = this.client.getObjectReadStream(this.cfg.bucketName, filename);
-    const uploadResult = await this.attachmentProcessor.uploadAttachment(s3Stream);
+    const uploadResult = await this.attachmentProcessor.uploadAttachment(s3Stream, 'stream');
+    const attachmentUrl = `${uploadResult.config.url}${uploadResult.data.objectId}?storage_type=maester`;
     this.logger.info('File successfully uploaded to attachment storage');
 
     /* eslint-disable-next-line no-param-reassign */
     msg.attachments = {
       [filename]: {
-        url: uploadResult.config.url,
+        url: attachmentUrl,
         size,
       },
     };
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		exports.ATTACHMENT_MAX_SIZE = process.env.ATTACHMENT_MAX_SIZE ? parseInt(process.env.ATTACHMENT_MAX_SIZE, 10) : 10000000;
		exports.REQUEST_MAX_BODY_LENGTH = process.env.REQUEST_MAX_BODY_LENGTH ? parseInt(process.env.REQUEST_MAX_BODY_LENGTH, 10) : 104857600; // 100MB