From 2a14de69686b5970adfd9a5aa974754a0aaf98f4 Mon Sep 17 00:00:00 2001 From: Ming Xia Shi Date: Mon, 15 Jan 2024 14:34:56 +0800 Subject: [PATCH 1/5] add cacheable-lookup for dns cache --- commands/http.js | 3 +++ package.json | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/commands/http.js b/commands/http.js index 69fe787..512873f 100644 --- a/commands/http.js +++ b/commands/http.js @@ -4,6 +4,7 @@ const URL = require("url"); const fs = require("fs"); const HTTPRetryRequest = require("../libs/HTTPRetryRequest"); const { checkIfEmpty, HEADERS, HEADERVALUES } = require("../libs/utilities"); +const CacheableLookup = require("cacheable-lookup"); module.exports = { /** @@ -191,6 +192,8 @@ module.exports = { opts.headers = { ...headerObject }; + const cacheable = new CacheableLookup(); + opts.lookup = cacheable.lookup; log.sys("Commencing to execute HTTP call with", reqURL, JSON.stringify(opts)); diff --git a/package.json b/package.json index 07a1b20..f4654ea 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,8 @@ "system-sleep": "^1.3.6", "twilio": "^3.73.0", "url": "^0.11.0", - "wait-on": "^4.0.2" + "wait-on": "^4.0.2", + "cacheable-lookup": "^7.0.0" }, "scripts": { "test": "cross-env NODE_ENV=test mocha tests/**/*.spec.js", From e59bbdf5ff8acd6609c8e7daf28295656a1507b3 Mon Sep 17 00:00:00 2001 From: Ming Xia Shi Date: Mon, 15 Jan 2024 14:45:23 +0800 Subject: [PATCH 2/5] update package-lock.json for cacheable-lookup --- package-lock.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/package-lock.json b/package-lock.json index 77895a2..97bafc4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "@slack/web-api": "^5.8.0", "@slack/webhook": "^5.0.3", "axios": "^0.25.0", + "cacheable-lookup": "^7.0.0", "cloudevents": "^5.3.0", "googleapis": "^89.0.0", "https": "^1.0.0", @@ -831,6 +832,14 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, + "node_modules/cacheable-lookup": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz", + "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==", + "engines": { + "node": ">=14.16" + } + }, "node_modules/cacheable-request": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", @@ -5869,6 +5878,11 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, + "cacheable-lookup": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz", + "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==" + }, "cacheable-request": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", From 4b158540c4377b8d3e17a75bd2f6b0f39b7b2fe9 Mon Sep 17 00:00:00 2001 From: Ming Xia Shi Date: Mon, 15 Jan 2024 16:02:26 +0800 Subject: [PATCH 3/5] fix require issue --- commands/http.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/http.js b/commands/http.js index 512873f..7e7b5c7 100644 --- a/commands/http.js +++ b/commands/http.js @@ -4,7 +4,7 @@ const URL = require("url"); const fs = require("fs"); const HTTPRetryRequest = require("../libs/HTTPRetryRequest"); const { checkIfEmpty, HEADERS, HEADERVALUES } = require("../libs/utilities"); -const CacheableLookup = require("cacheable-lookup"); +import CacheableLookup from "cacheable-lookup"; module.exports = { /** From e32350ddce19f854a1b409bd604ae8acb71681a0 Mon Sep 17 00:00:00 2001 From: Ming Xia Shi Date: Mon, 15 Jan 2024 17:10:50 +0800 Subject: [PATCH 4/5] fix import cacheable-lookup --- commands/http.js | 482 ++++++++++++++++++++++++----------------------- 1 file changed, 242 insertions(+), 240 deletions(-) diff --git a/commands/http.js b/commands/http.js index 7e7b5c7..a74218a 100644 --- a/commands/http.js +++ b/commands/http.js @@ -4,280 +4,282 @@ const URL = require("url"); const fs = require("fs"); const HTTPRetryRequest = require("../libs/HTTPRetryRequest"); const { checkIfEmpty, HEADERS, HEADERVALUES } = require("../libs/utilities"); -import CacheableLookup from "cacheable-lookup"; -module.exports = { - /** - * @todo implement a fetch that takes in; - * URL [String] - * method [Select - Options: get, post, put, patch, delete, options] - * headers [Text Area - new line delimitered list?] - * content type [Select - Options: any, text, xml, json, html] - * body [Text Area - optional depending on method] - * @param {string} successcodes Represents a list of HTTP Status Codes, used for success checks - * @param {string} retrycodes Represents a list of HTTP Status Codes, used for retry checks - * @param {string} errorcodes Represents a list of HTTP Status Codes, used for error checks - * @param {int} retrynumber Represents the number of retries that will be perfomed until success is obtained or the number of retries is achived - * @param {int} retrydelay Represents the number of miliseconds that will delay the next retry - * @param {int} systemretrynumber @readonly Represents the number of retries that will be perfomed until success is obtained or the number of retries is achived, in case of special case of exceptions, defauls to 3 - * @param {int} systemretrydelay @readonly Represents the number of miliseconds that will delay the next retry, in case of special case of exceptions, defaults to 5 seconds - * Allow untrusted SSL certs [Boolean Toggle] - */ - execute() { - log.debug("Started HTTP Call Plugin"); +/** + * @todo implement a fetch that takes in; + * URL [String] + * method [Select - Options: get, post, put, patch, delete, options] + * headers [Text Area - new line delimitered list?] + * content type [Select - Options: any, text, xml, json, html] + * body [Text Area - optional depending on method] + * @param {string} successcodes Represents a list of HTTP Status Codes, used for success checks + * @param {string} retrycodes Represents a list of HTTP Status Codes, used for retry checks + * @param {string} errorcodes Represents a list of HTTP Status Codes, used for error checks + * @param {int} retrynumber Represents the number of retries that will be perfomed until success is obtained or the number of retries is achived + * @param {int} retrydelay Represents the number of miliseconds that will delay the next retry + * @param {int} systemretrynumber @readonly Represents the number of retries that will be perfomed until success is obtained or the number of retries is achived, in case of special case of exceptions, defauls to 3 + * @param {int} systemretrydelay @readonly Represents the number of miliseconds that will delay the next retry, in case of special case of exceptions, defaults to 5 seconds + * Allow untrusted SSL certs [Boolean Toggle] + */ +async function execute() { + log.debug("Started HTTP Call Plugin"); - //Destructure and get properties ready. - const taskProps = utils.resolveInputParameters(); + //Destructure and get properties ready. + const taskProps = utils.resolveInputParameters(); - const { url, method, header, contentType, body, allowUntrustedCerts, outputFilePath, successcodes = "1xx,2xx", retrycodes = "502,503", errorcodes = "", retrynumber = 0, retrydelay = 200, systemretrynumber = 3, systemretrydelay = 5000 } = taskProps; + const { url, method, header, contentType, body, allowUntrustedCerts, outputFilePath, successcodes = "1xx,2xx", retrycodes = "502,503", errorcodes = "", retrynumber = 0, retrydelay = 200, systemretrynumber = 3, systemretrydelay = 5000 } = taskProps; - // Input force defaults - let newretrynumber = 0; - let newretrydelay = 200; - let newsystemretrynumber = 3; - let newsystemretrydelay = 5000; - let newbody = ""; - // Input not "empty", set value - if (!checkIfEmpty(successcodes)) { - newsuccesscodes = successcodes - .toString() - .trim() - .toLowerCase(); // Input normalization + // Input force defaults + let newretrynumber = 0; + let newretrydelay = 200; + let newsystemretrynumber = 3; + let newsystemretrydelay = 5000; + let newbody = ""; + // Input not "empty", set value + if (!checkIfEmpty(successcodes)) { + newsuccesscodes = successcodes + .toString() + .trim() + .toLowerCase(); // Input normalization + } + if (!checkIfEmpty(retrycodes)) { + newretrycodes = retrycodes + .toString() + .trim() + .toLowerCase(); // Input normalization + } + if (!checkIfEmpty(errorcodes)) { + newerrorcodes = errorcodes + .toString() + .trim() + .toLowerCase(); // Input normalization + } + if (!checkIfEmpty(retrynumber)) { + newretrynumber = parseInt(retrynumber, 10); + if (isNaN(newretrynumber)) { + log.err("Invalid input for: retrynumber"); + process.exit(1); } - if (!checkIfEmpty(retrycodes)) { - newretrycodes = retrycodes - .toString() - .trim() - .toLowerCase(); // Input normalization + if (newretrynumber < 1 || newretrynumber > 9) { + log.err("Invalid input for: retrynumber [1,9]"); + process.exit(1); } - if (!checkIfEmpty(errorcodes)) { - newerrorcodes = errorcodes - .toString() - .trim() - .toLowerCase(); // Input normalization + } + if (!checkIfEmpty(retrydelay)) { + newretrydelay = parseInt(retrydelay, 10); + // parse test + if (isNaN(newretrydelay)) { + log.err("Invalid input for: retrydelay"); + process.exit(1); } - if (!checkIfEmpty(retrynumber)) { - newretrynumber = parseInt(retrynumber, 10); - if (isNaN(newretrynumber)) { - log.err("Invalid input for: retrynumber"); - process.exit(1); - } - if (newretrynumber < 1 || newretrynumber > 9) { - log.err("Invalid input for: retrynumber [1,9]"); - process.exit(1); - } + // bounds exceeded + if (newretrydelay < 100 || newretrydelay > 300000) { + log.err("Invalid input for: retrydelay [100,300000]"); + process.exit(1); } - if (!checkIfEmpty(retrydelay)) { - newretrydelay = parseInt(retrydelay, 10); - // parse test - if (isNaN(newretrydelay)) { - log.err("Invalid input for: retrydelay"); - process.exit(1); - } - // bounds exceeded - if (newretrydelay < 100 || newretrydelay > 300000) { - log.err("Invalid input for: retrydelay [100,300000]"); - process.exit(1); - } + } + if (!checkIfEmpty(systemretrynumber)) { + newsystemretrynumber = parseInt(systemretrynumber, 10); + if (isNaN(newsystemretrynumber)) { + newsystemretrynumber = 3; } - if (!checkIfEmpty(systemretrynumber)) { - newsystemretrynumber = parseInt(systemretrynumber, 10); - if (isNaN(newsystemretrynumber)) { - newsystemretrynumber = 3; - } - if (newsystemretrynumber < 1 || newsystemretrynumber > 9) { - newsystemretrynumber = 3; - } + if (newsystemretrynumber < 1 || newsystemretrynumber > 9) { + newsystemretrynumber = 3; } - if (!checkIfEmpty(systemretrydelay)) { - newsystemretrydelay = parseInt(systemretrydelay, 10); - // parse test - if (isNaN(newsystemretrydelay)) { - newsystemretrydelay = 5000; - } - // bounds exceeded - if (newsystemretrydelay < 100 || newsystemretrydelay > 300000) { - newsystemretrydelay = 5000; - } + } + if (!checkIfEmpty(systemretrydelay)) { + newsystemretrydelay = parseInt(systemretrydelay, 10); + // parse test + if (isNaN(newsystemretrydelay)) { + newsystemretrydelay = 5000; } - if (!checkIfEmpty(body)) { - newbody = body.replace(/(\n|\r|\t)/gm, ""); + // bounds exceeded + if (newsystemretrydelay < 100 || newsystemretrydelay > 300000) { + newsystemretrydelay = 5000; } + } + if (!checkIfEmpty(body)) { + newbody = body.replace(/(\n|\r|\t)/gm, ""); + } - /** - * turn header into object based upon new line delimeters - */ + /** + * turn header into object based upon new line delimeters + */ - const headerObject = {}; - if (!checkIfEmpty(header)) { - let headerSplitArr = header.split("\n"); - log.debug(headerSplitArr); - headerSplitArr.forEach(line => { - let arrHearder = line.split(":"); - if (arrHearder && arrHearder.length) { - let key = arrHearder - .shift() - .trim() - .replace(/("|')/g, ""); //take first string, the header - let value = arrHearder - .join(":") - .trim() - .replace(/("|')/g, ""); //rejoin all strings - if (key) { - // as per RFC 2616 Sec4.2 - value is optional https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 - headerObject[key] = value; - } + const headerObject = {}; + if (!checkIfEmpty(header)) { + let headerSplitArr = header.split("\n"); + log.debug(headerSplitArr); + headerSplitArr.forEach(line => { + let arrHearder = line.split(":"); + if (arrHearder && arrHearder.length) { + let key = arrHearder + .shift() + .trim() + .replace(/("|')/g, ""); //take first string, the header + let value = arrHearder + .join(":") + .trim() + .replace(/("|')/g, ""); //rejoin all strings + if (key) { + // as per RFC 2616 Sec4.2 - value is optional https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 + headerObject[key] = value; } - }); - } - - if (contentType && contentType !== '""' && contentType !== '" "') { - headerObject[HEADERS.CONTENTTYPE] = contentType; - } - if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.APPLICATIONJSON) { - // force "body" to JSON parse and convert back to string, to avoid special characters. - try { - newbody = JSON.stringify(JSON.parse(newbody)); - } catch (e) { - log.err(`Invalid input for: body, JSON parse failed for content type: ${contentType} \n ${e}`); - process.exit(1); } - } + }); + } - if (!checkIfEmpty(newbody)) { - headerObject[HEADERS.CONTENTLENGTH] = ~-encodeURI(newbody).split(/%..|./).length; + if (contentType && contentType !== '""' && contentType !== '" "') { + headerObject[HEADERS.CONTENTTYPE] = contentType; + } + if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.APPLICATIONJSON) { + // force "body" to JSON parse and convert back to string, to avoid special characters. + try { + newbody = JSON.stringify(JSON.parse(newbody)); + } catch (e) { + log.err(`Invalid input for: body, JSON parse failed for content type: ${contentType} \n ${e}`); + process.exit(1); } + } - log.debug(headerObject); + if (!checkIfEmpty(newbody)) { + headerObject[HEADERS.CONTENTLENGTH] = ~-encodeURI(newbody).split(/%..|./).length; + } - var agent = null; - if (process.env.HTTP_PROXY) { - if (!process.env.NO_PROXY) { + log.debug(headerObject); + + var agent = null; + if (process.env.HTTP_PROXY) { + if (!process.env.NO_PROXY) { + log.debug("Using Proxy", process.env.HTTP_PROXY); + log.debug("NO_PROXY list not provided by env"); + agent = new HttpsProxyAgent(process.env.HTTP_PROXY); + } else { + log.debug("NO_PROXY list detected", process.env.NO_PROXY); + const noProxyList = process.env.NO_PROXY.split(","); + let urltoUrl = new URL(url); + let urlHost = urltoUrl.host.split(":")[0]; + log.debug("urlHost:", urlHost); + const skipProxy = noProxyList.some(domain => { + log.debug("domain:", domain); + log.debug(urlHost.endsWith(domain)); + return urlHost.endsWith(domain); + }); + log.debug("skipProxy", skipProxy); + if (!skipProxy) { log.debug("Using Proxy", process.env.HTTP_PROXY); - log.debug("NO_PROXY list not provided by env"); agent = new HttpsProxyAgent(process.env.HTTP_PROXY); - } else { - log.debug("NO_PROXY list detected", process.env.NO_PROXY); - const noProxyList = process.env.NO_PROXY.split(","); - let urltoUrl = new URL(url); - let urlHost = urltoUrl.host.split(":")[0]; - log.debug("urlHost:", urlHost); - const skipProxy = noProxyList.some(domain => { - log.debug("domain:", domain); - log.debug(urlHost.endsWith(domain)); - return urlHost.endsWith(domain); - }); - log.debug("skipProxy", skipProxy); - if (!skipProxy) { - log.debug("Using Proxy", process.env.HTTP_PROXY); - agent = new HttpsProxyAgent(process.env.HTTP_PROXY); - } else if (skipProxy) { - log.debug("Not specifying proxy. Domain was found in no_proxy list"); - } + } else if (skipProxy) { + log.debug("Not specifying proxy. Domain was found in no_proxy list"); } } + } - let allowUntrustedFlag = false; + let allowUntrustedFlag = false; - if ((typeof allowUntrustedCerts === "string" && allowUntrustedCerts === "true") || (typeof allowUntrustedCerts === "boolean" && allowUntrustedCerts)) { - log.sys(`Attempting HTTP request allowing untrusted certs`); - allowUntrustedFlag = true; - } + if ((typeof allowUntrustedCerts === "string" && allowUntrustedCerts === "true") || (typeof allowUntrustedCerts === "boolean" && allowUntrustedCerts)) { + log.sys(`Attempting HTTP request allowing untrusted certs`); + allowUntrustedFlag = true; + } - const reqURL = new URL.URL(url); - let opts = {}; - opts.rejectUnauthorized = !allowUntrustedFlag; - opts.agent = agent; - opts.method = method; - opts.headers = { - ...headerObject - }; - const cacheable = new CacheableLookup(); - opts.lookup = cacheable.lookup; + const reqURL = new URL.URL(url); + let opts = {}; + opts.rejectUnauthorized = !allowUntrustedFlag; + opts.agent = agent; + opts.method = method; + opts.headers = { + ...headerObject + }; + const CacheableLookup = await import("cacheable-lookup"); + const cacheable = new CacheableLookup(); + opts.lookup = cacheable.lookup; - log.sys("Commencing to execute HTTP call with", reqURL, JSON.stringify(opts)); + log.sys("Commencing to execute HTTP call with", reqURL, JSON.stringify(opts)); - let config = { - SUCCESS_CODES: successcodes.toString(), // Force string - RETRY_CODES: retrycodes.toString(), // Force string - ERROR_CODES: errorcodes.toString(), // Force string - MAX_RETRIES: newretrynumber, - DELAY: newretrydelay, - SYSTEM_MAX_RETRIES: newsystemretrynumber, - SYSTEM_DELAY: newsystemretrydelay - }; - if (!checkIfEmpty(newbody)) { - log.debug("writing request body: \n ", newbody); - config.body = newbody; - } + let config = { + SUCCESS_CODES: successcodes.toString(), // Force string + RETRY_CODES: retrycodes.toString(), // Force string + ERROR_CODES: errorcodes.toString(), // Force string + MAX_RETRIES: newretrynumber, + DELAY: newretrydelay, + SYSTEM_MAX_RETRIES: newsystemretrynumber, + SYSTEM_DELAY: newsystemretrydelay + }; + if (!checkIfEmpty(newbody)) { + log.debug("writing request body: \n ", newbody); + config.body = newbody; + } - new HTTPRetryRequest(config, reqURL, opts) - .then(res => { - log.debug(`statusCode: ${res.statusCode}`); - if (headerObject[HEADERS.CONTENTTYPE]) { - if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.APPLICATIONJSON) { - try { - log.debug(`output: ${res.body.toString()}`); - //make sure non-empty output is a valid JSON, - //if not throw exception - if (!(res.body === null || res.body.toString().match(/^ *$/) !== null)) { - JSON.parse(res.body.toString()); - } - } catch (e) { - log.err(e); - process.exit(1); + new HTTPRetryRequest(config, reqURL, opts) + .then(res => { + log.debug(`statusCode: ${res.statusCode}`); + if (headerObject[HEADERS.CONTENTTYPE]) { + if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.APPLICATIONJSON) { + try { + log.debug(`output: ${res.body.toString()}`); + //make sure non-empty output is a valid JSON, + //if not throw exception + if (!(res.body === null || res.body.toString().match(/^ *$/) !== null)) { + JSON.parse(res.body.toString()); } + } catch (e) { + log.err(e); + process.exit(1); } - if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.APPLICATIONXML) { - // TODO: implement JSDOM or DOMParser(i.e. not yet in nodejs) - } - if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.TEXTHTML) { - // TODO: implement JSDOM or DOMParser(i.e. not yet in nodejs) - } - } - - try { - resultstatusCode = parseInt(res.statusCode, 10); - } catch (e) { - resultstatusCode = res.statusCode; } - utils.setOutputParameter("statusCode", resultstatusCode); - if (!(res.body === null || res.body.toString().match(/^ *$/) !== null)) { - log.sys("Response Received:", res.body.toString()); + if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.APPLICATIONXML) { + // TODO: implement JSDOM or DOMParser(i.e. not yet in nodejs) } - if (outputFilePath && outputFilePath.length && outputFilePath !== '""' && outputFilePath !== '" "') { - //https://nodejs.org/docs/latest-v14.x/api/fs.html#fs_fs_writefilesync_file_data_options - fs.writeFileSync(outputFilePath, res.body, { - encoding: "utf8", - mode: 666, - flag: "w" - }); - log.debug("The task output parameter successfully saved to provided file path."); - } else { - utils.setOutputParameter("response", res.body.toString()); - log.debug("The task output parameter successfully saved to standard response file."); + if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.TEXTHTML) { + // TODO: implement JSDOM or DOMParser(i.e. not yet in nodejs) } - log.good("Response successfully received!"); - }) - .catch(err => { - // log.err("HTTP Promise error:", err, "Cause: ", err.cause); - if (err.statusCode || err.statusMessage || err.body) { - log.err(`HTTP Promise error:`); - log.err(` \n Status Code: ${err.statusCode}`); - log.err(` \n Status Message: ${err.statusMessage}`); - log.err(` \n Response: ${err.body?.toString()}`); - } else { - log.err(`HTTP Promise error:`, err); - } - (async () => { - await (async function(msg) { - utils.setOutputParameter("statusCode", msg?.statusCode ?? ""); - utils.setOutputParameter("response", msg?.body?.toString() ?? msg.message); - })(err); - process.exit(1); - })(); - }); + } - log.debug("Finished HTTP Call File Plugin"); - } + try { + resultstatusCode = parseInt(res.statusCode, 10); + } catch (e) { + resultstatusCode = res.statusCode; + } + utils.setOutputParameter("statusCode", resultstatusCode); + if (!(res.body === null || res.body.toString().match(/^ *$/) !== null)) { + log.sys("Response Received:", res.body.toString()); + } + if (outputFilePath && outputFilePath.length && outputFilePath !== '""' && outputFilePath !== '" "') { + //https://nodejs.org/docs/latest-v14.x/api/fs.html#fs_fs_writefilesync_file_data_options + fs.writeFileSync(outputFilePath, res.body, { + encoding: "utf8", + mode: 666, + flag: "w" + }); + log.debug("The task output parameter successfully saved to provided file path."); + } else { + utils.setOutputParameter("response", res.body.toString()); + log.debug("The task output parameter successfully saved to standard response file."); + } + log.good("Response successfully received!"); + }) + .catch(err => { + // log.err("HTTP Promise error:", err, "Cause: ", err.cause); + if (err.statusCode || err.statusMessage || err.body) { + log.err(`HTTP Promise error:`); + log.err(` \n Status Code: ${err.statusCode}`); + log.err(` \n Status Message: ${err.statusMessage}`); + log.err(` \n Response: ${err.body?.toString()}`); + } else { + log.err(`HTTP Promise error:`, err); + } + (async () => { + await (async function(msg) { + utils.setOutputParameter("statusCode", msg?.statusCode ?? ""); + utils.setOutputParameter("response", msg?.body?.toString() ?? msg.message); + })(err); + process.exit(1); + })(); + }); + + log.debug("Finished HTTP Call File Plugin"); +} + +module.exports = { + execute }; From af5d212b22a9397690442cfed206a2e4f342a93f Mon Sep 17 00:00:00 2001 From: Ming Xia Shi Date: Mon, 15 Jan 2024 18:28:28 +0800 Subject: [PATCH 5/5] downgrade cacheable-lookup --- commands/http.js | 482 +++++++++++++++++++++++----------------------- package-lock.json | 16 +- package.json | 2 +- 3 files changed, 249 insertions(+), 251 deletions(-) diff --git a/commands/http.js b/commands/http.js index a74218a..512873f 100644 --- a/commands/http.js +++ b/commands/http.js @@ -4,282 +4,280 @@ const URL = require("url"); const fs = require("fs"); const HTTPRetryRequest = require("../libs/HTTPRetryRequest"); const { checkIfEmpty, HEADERS, HEADERVALUES } = require("../libs/utilities"); +const CacheableLookup = require("cacheable-lookup"); -/** - * @todo implement a fetch that takes in; - * URL [String] - * method [Select - Options: get, post, put, patch, delete, options] - * headers [Text Area - new line delimitered list?] - * content type [Select - Options: any, text, xml, json, html] - * body [Text Area - optional depending on method] - * @param {string} successcodes Represents a list of HTTP Status Codes, used for success checks - * @param {string} retrycodes Represents a list of HTTP Status Codes, used for retry checks - * @param {string} errorcodes Represents a list of HTTP Status Codes, used for error checks - * @param {int} retrynumber Represents the number of retries that will be perfomed until success is obtained or the number of retries is achived - * @param {int} retrydelay Represents the number of miliseconds that will delay the next retry - * @param {int} systemretrynumber @readonly Represents the number of retries that will be perfomed until success is obtained or the number of retries is achived, in case of special case of exceptions, defauls to 3 - * @param {int} systemretrydelay @readonly Represents the number of miliseconds that will delay the next retry, in case of special case of exceptions, defaults to 5 seconds - * Allow untrusted SSL certs [Boolean Toggle] - */ -async function execute() { - log.debug("Started HTTP Call Plugin"); +module.exports = { + /** + * @todo implement a fetch that takes in; + * URL [String] + * method [Select - Options: get, post, put, patch, delete, options] + * headers [Text Area - new line delimitered list?] + * content type [Select - Options: any, text, xml, json, html] + * body [Text Area - optional depending on method] + * @param {string} successcodes Represents a list of HTTP Status Codes, used for success checks + * @param {string} retrycodes Represents a list of HTTP Status Codes, used for retry checks + * @param {string} errorcodes Represents a list of HTTP Status Codes, used for error checks + * @param {int} retrynumber Represents the number of retries that will be perfomed until success is obtained or the number of retries is achived + * @param {int} retrydelay Represents the number of miliseconds that will delay the next retry + * @param {int} systemretrynumber @readonly Represents the number of retries that will be perfomed until success is obtained or the number of retries is achived, in case of special case of exceptions, defauls to 3 + * @param {int} systemretrydelay @readonly Represents the number of miliseconds that will delay the next retry, in case of special case of exceptions, defaults to 5 seconds + * Allow untrusted SSL certs [Boolean Toggle] + */ + execute() { + log.debug("Started HTTP Call Plugin"); - //Destructure and get properties ready. - const taskProps = utils.resolveInputParameters(); + //Destructure and get properties ready. + const taskProps = utils.resolveInputParameters(); - const { url, method, header, contentType, body, allowUntrustedCerts, outputFilePath, successcodes = "1xx,2xx", retrycodes = "502,503", errorcodes = "", retrynumber = 0, retrydelay = 200, systemretrynumber = 3, systemretrydelay = 5000 } = taskProps; + const { url, method, header, contentType, body, allowUntrustedCerts, outputFilePath, successcodes = "1xx,2xx", retrycodes = "502,503", errorcodes = "", retrynumber = 0, retrydelay = 200, systemretrynumber = 3, systemretrydelay = 5000 } = taskProps; - // Input force defaults - let newretrynumber = 0; - let newretrydelay = 200; - let newsystemretrynumber = 3; - let newsystemretrydelay = 5000; - let newbody = ""; - // Input not "empty", set value - if (!checkIfEmpty(successcodes)) { - newsuccesscodes = successcodes - .toString() - .trim() - .toLowerCase(); // Input normalization - } - if (!checkIfEmpty(retrycodes)) { - newretrycodes = retrycodes - .toString() - .trim() - .toLowerCase(); // Input normalization - } - if (!checkIfEmpty(errorcodes)) { - newerrorcodes = errorcodes - .toString() - .trim() - .toLowerCase(); // Input normalization - } - if (!checkIfEmpty(retrynumber)) { - newretrynumber = parseInt(retrynumber, 10); - if (isNaN(newretrynumber)) { - log.err("Invalid input for: retrynumber"); - process.exit(1); + // Input force defaults + let newretrynumber = 0; + let newretrydelay = 200; + let newsystemretrynumber = 3; + let newsystemretrydelay = 5000; + let newbody = ""; + // Input not "empty", set value + if (!checkIfEmpty(successcodes)) { + newsuccesscodes = successcodes + .toString() + .trim() + .toLowerCase(); // Input normalization } - if (newretrynumber < 1 || newretrynumber > 9) { - log.err("Invalid input for: retrynumber [1,9]"); - process.exit(1); + if (!checkIfEmpty(retrycodes)) { + newretrycodes = retrycodes + .toString() + .trim() + .toLowerCase(); // Input normalization } - } - if (!checkIfEmpty(retrydelay)) { - newretrydelay = parseInt(retrydelay, 10); - // parse test - if (isNaN(newretrydelay)) { - log.err("Invalid input for: retrydelay"); - process.exit(1); + if (!checkIfEmpty(errorcodes)) { + newerrorcodes = errorcodes + .toString() + .trim() + .toLowerCase(); // Input normalization } - // bounds exceeded - if (newretrydelay < 100 || newretrydelay > 300000) { - log.err("Invalid input for: retrydelay [100,300000]"); - process.exit(1); + if (!checkIfEmpty(retrynumber)) { + newretrynumber = parseInt(retrynumber, 10); + if (isNaN(newretrynumber)) { + log.err("Invalid input for: retrynumber"); + process.exit(1); + } + if (newretrynumber < 1 || newretrynumber > 9) { + log.err("Invalid input for: retrynumber [1,9]"); + process.exit(1); + } } - } - if (!checkIfEmpty(systemretrynumber)) { - newsystemretrynumber = parseInt(systemretrynumber, 10); - if (isNaN(newsystemretrynumber)) { - newsystemretrynumber = 3; + if (!checkIfEmpty(retrydelay)) { + newretrydelay = parseInt(retrydelay, 10); + // parse test + if (isNaN(newretrydelay)) { + log.err("Invalid input for: retrydelay"); + process.exit(1); + } + // bounds exceeded + if (newretrydelay < 100 || newretrydelay > 300000) { + log.err("Invalid input for: retrydelay [100,300000]"); + process.exit(1); + } } - if (newsystemretrynumber < 1 || newsystemretrynumber > 9) { - newsystemretrynumber = 3; + if (!checkIfEmpty(systemretrynumber)) { + newsystemretrynumber = parseInt(systemretrynumber, 10); + if (isNaN(newsystemretrynumber)) { + newsystemretrynumber = 3; + } + if (newsystemretrynumber < 1 || newsystemretrynumber > 9) { + newsystemretrynumber = 3; + } } - } - if (!checkIfEmpty(systemretrydelay)) { - newsystemretrydelay = parseInt(systemretrydelay, 10); - // parse test - if (isNaN(newsystemretrydelay)) { - newsystemretrydelay = 5000; + if (!checkIfEmpty(systemretrydelay)) { + newsystemretrydelay = parseInt(systemretrydelay, 10); + // parse test + if (isNaN(newsystemretrydelay)) { + newsystemretrydelay = 5000; + } + // bounds exceeded + if (newsystemretrydelay < 100 || newsystemretrydelay > 300000) { + newsystemretrydelay = 5000; + } } - // bounds exceeded - if (newsystemretrydelay < 100 || newsystemretrydelay > 300000) { - newsystemretrydelay = 5000; + if (!checkIfEmpty(body)) { + newbody = body.replace(/(\n|\r|\t)/gm, ""); } - } - if (!checkIfEmpty(body)) { - newbody = body.replace(/(\n|\r|\t)/gm, ""); - } - /** - * turn header into object based upon new line delimeters - */ + /** + * turn header into object based upon new line delimeters + */ - const headerObject = {}; - if (!checkIfEmpty(header)) { - let headerSplitArr = header.split("\n"); - log.debug(headerSplitArr); - headerSplitArr.forEach(line => { - let arrHearder = line.split(":"); - if (arrHearder && arrHearder.length) { - let key = arrHearder - .shift() - .trim() - .replace(/("|')/g, ""); //take first string, the header - let value = arrHearder - .join(":") - .trim() - .replace(/("|')/g, ""); //rejoin all strings - if (key) { - // as per RFC 2616 Sec4.2 - value is optional https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 - headerObject[key] = value; + const headerObject = {}; + if (!checkIfEmpty(header)) { + let headerSplitArr = header.split("\n"); + log.debug(headerSplitArr); + headerSplitArr.forEach(line => { + let arrHearder = line.split(":"); + if (arrHearder && arrHearder.length) { + let key = arrHearder + .shift() + .trim() + .replace(/("|')/g, ""); //take first string, the header + let value = arrHearder + .join(":") + .trim() + .replace(/("|')/g, ""); //rejoin all strings + if (key) { + // as per RFC 2616 Sec4.2 - value is optional https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 + headerObject[key] = value; + } } - } - }); - } + }); + } - if (contentType && contentType !== '""' && contentType !== '" "') { - headerObject[HEADERS.CONTENTTYPE] = contentType; - } - if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.APPLICATIONJSON) { - // force "body" to JSON parse and convert back to string, to avoid special characters. - try { - newbody = JSON.stringify(JSON.parse(newbody)); - } catch (e) { - log.err(`Invalid input for: body, JSON parse failed for content type: ${contentType} \n ${e}`); - process.exit(1); + if (contentType && contentType !== '""' && contentType !== '" "') { + headerObject[HEADERS.CONTENTTYPE] = contentType; + } + if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.APPLICATIONJSON) { + // force "body" to JSON parse and convert back to string, to avoid special characters. + try { + newbody = JSON.stringify(JSON.parse(newbody)); + } catch (e) { + log.err(`Invalid input for: body, JSON parse failed for content type: ${contentType} \n ${e}`); + process.exit(1); + } } - } - if (!checkIfEmpty(newbody)) { - headerObject[HEADERS.CONTENTLENGTH] = ~-encodeURI(newbody).split(/%..|./).length; - } + if (!checkIfEmpty(newbody)) { + headerObject[HEADERS.CONTENTLENGTH] = ~-encodeURI(newbody).split(/%..|./).length; + } - log.debug(headerObject); + log.debug(headerObject); - var agent = null; - if (process.env.HTTP_PROXY) { - if (!process.env.NO_PROXY) { - log.debug("Using Proxy", process.env.HTTP_PROXY); - log.debug("NO_PROXY list not provided by env"); - agent = new HttpsProxyAgent(process.env.HTTP_PROXY); - } else { - log.debug("NO_PROXY list detected", process.env.NO_PROXY); - const noProxyList = process.env.NO_PROXY.split(","); - let urltoUrl = new URL(url); - let urlHost = urltoUrl.host.split(":")[0]; - log.debug("urlHost:", urlHost); - const skipProxy = noProxyList.some(domain => { - log.debug("domain:", domain); - log.debug(urlHost.endsWith(domain)); - return urlHost.endsWith(domain); - }); - log.debug("skipProxy", skipProxy); - if (!skipProxy) { + var agent = null; + if (process.env.HTTP_PROXY) { + if (!process.env.NO_PROXY) { log.debug("Using Proxy", process.env.HTTP_PROXY); + log.debug("NO_PROXY list not provided by env"); agent = new HttpsProxyAgent(process.env.HTTP_PROXY); - } else if (skipProxy) { - log.debug("Not specifying proxy. Domain was found in no_proxy list"); + } else { + log.debug("NO_PROXY list detected", process.env.NO_PROXY); + const noProxyList = process.env.NO_PROXY.split(","); + let urltoUrl = new URL(url); + let urlHost = urltoUrl.host.split(":")[0]; + log.debug("urlHost:", urlHost); + const skipProxy = noProxyList.some(domain => { + log.debug("domain:", domain); + log.debug(urlHost.endsWith(domain)); + return urlHost.endsWith(domain); + }); + log.debug("skipProxy", skipProxy); + if (!skipProxy) { + log.debug("Using Proxy", process.env.HTTP_PROXY); + agent = new HttpsProxyAgent(process.env.HTTP_PROXY); + } else if (skipProxy) { + log.debug("Not specifying proxy. Domain was found in no_proxy list"); + } } } - } - let allowUntrustedFlag = false; + let allowUntrustedFlag = false; - if ((typeof allowUntrustedCerts === "string" && allowUntrustedCerts === "true") || (typeof allowUntrustedCerts === "boolean" && allowUntrustedCerts)) { - log.sys(`Attempting HTTP request allowing untrusted certs`); - allowUntrustedFlag = true; - } + if ((typeof allowUntrustedCerts === "string" && allowUntrustedCerts === "true") || (typeof allowUntrustedCerts === "boolean" && allowUntrustedCerts)) { + log.sys(`Attempting HTTP request allowing untrusted certs`); + allowUntrustedFlag = true; + } - const reqURL = new URL.URL(url); - let opts = {}; - opts.rejectUnauthorized = !allowUntrustedFlag; - opts.agent = agent; - opts.method = method; - opts.headers = { - ...headerObject - }; - const CacheableLookup = await import("cacheable-lookup"); - const cacheable = new CacheableLookup(); - opts.lookup = cacheable.lookup; + const reqURL = new URL.URL(url); + let opts = {}; + opts.rejectUnauthorized = !allowUntrustedFlag; + opts.agent = agent; + opts.method = method; + opts.headers = { + ...headerObject + }; + const cacheable = new CacheableLookup(); + opts.lookup = cacheable.lookup; - log.sys("Commencing to execute HTTP call with", reqURL, JSON.stringify(opts)); + log.sys("Commencing to execute HTTP call with", reqURL, JSON.stringify(opts)); - let config = { - SUCCESS_CODES: successcodes.toString(), // Force string - RETRY_CODES: retrycodes.toString(), // Force string - ERROR_CODES: errorcodes.toString(), // Force string - MAX_RETRIES: newretrynumber, - DELAY: newretrydelay, - SYSTEM_MAX_RETRIES: newsystemretrynumber, - SYSTEM_DELAY: newsystemretrydelay - }; - if (!checkIfEmpty(newbody)) { - log.debug("writing request body: \n ", newbody); - config.body = newbody; - } + let config = { + SUCCESS_CODES: successcodes.toString(), // Force string + RETRY_CODES: retrycodes.toString(), // Force string + ERROR_CODES: errorcodes.toString(), // Force string + MAX_RETRIES: newretrynumber, + DELAY: newretrydelay, + SYSTEM_MAX_RETRIES: newsystemretrynumber, + SYSTEM_DELAY: newsystemretrydelay + }; + if (!checkIfEmpty(newbody)) { + log.debug("writing request body: \n ", newbody); + config.body = newbody; + } - new HTTPRetryRequest(config, reqURL, opts) - .then(res => { - log.debug(`statusCode: ${res.statusCode}`); - if (headerObject[HEADERS.CONTENTTYPE]) { - if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.APPLICATIONJSON) { - try { - log.debug(`output: ${res.body.toString()}`); - //make sure non-empty output is a valid JSON, - //if not throw exception - if (!(res.body === null || res.body.toString().match(/^ *$/) !== null)) { - JSON.parse(res.body.toString()); + new HTTPRetryRequest(config, reqURL, opts) + .then(res => { + log.debug(`statusCode: ${res.statusCode}`); + if (headerObject[HEADERS.CONTENTTYPE]) { + if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.APPLICATIONJSON) { + try { + log.debug(`output: ${res.body.toString()}`); + //make sure non-empty output is a valid JSON, + //if not throw exception + if (!(res.body === null || res.body.toString().match(/^ *$/) !== null)) { + JSON.parse(res.body.toString()); + } + } catch (e) { + log.err(e); + process.exit(1); } - } catch (e) { - log.err(e); - process.exit(1); + } + if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.APPLICATIONXML) { + // TODO: implement JSDOM or DOMParser(i.e. not yet in nodejs) + } + if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.TEXTHTML) { + // TODO: implement JSDOM or DOMParser(i.e. not yet in nodejs) } } - if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.APPLICATIONXML) { - // TODO: implement JSDOM or DOMParser(i.e. not yet in nodejs) + + try { + resultstatusCode = parseInt(res.statusCode, 10); + } catch (e) { + resultstatusCode = res.statusCode; } - if (headerObject[HEADERS.CONTENTTYPE] === HEADERVALUES.TEXTHTML) { - // TODO: implement JSDOM or DOMParser(i.e. not yet in nodejs) + utils.setOutputParameter("statusCode", resultstatusCode); + if (!(res.body === null || res.body.toString().match(/^ *$/) !== null)) { + log.sys("Response Received:", res.body.toString()); } - } - - try { - resultstatusCode = parseInt(res.statusCode, 10); - } catch (e) { - resultstatusCode = res.statusCode; - } - utils.setOutputParameter("statusCode", resultstatusCode); - if (!(res.body === null || res.body.toString().match(/^ *$/) !== null)) { - log.sys("Response Received:", res.body.toString()); - } - if (outputFilePath && outputFilePath.length && outputFilePath !== '""' && outputFilePath !== '" "') { - //https://nodejs.org/docs/latest-v14.x/api/fs.html#fs_fs_writefilesync_file_data_options - fs.writeFileSync(outputFilePath, res.body, { - encoding: "utf8", - mode: 666, - flag: "w" - }); - log.debug("The task output parameter successfully saved to provided file path."); - } else { - utils.setOutputParameter("response", res.body.toString()); - log.debug("The task output parameter successfully saved to standard response file."); - } - log.good("Response successfully received!"); - }) - .catch(err => { - // log.err("HTTP Promise error:", err, "Cause: ", err.cause); - if (err.statusCode || err.statusMessage || err.body) { - log.err(`HTTP Promise error:`); - log.err(` \n Status Code: ${err.statusCode}`); - log.err(` \n Status Message: ${err.statusMessage}`); - log.err(` \n Response: ${err.body?.toString()}`); - } else { - log.err(`HTTP Promise error:`, err); - } - (async () => { - await (async function(msg) { - utils.setOutputParameter("statusCode", msg?.statusCode ?? ""); - utils.setOutputParameter("response", msg?.body?.toString() ?? msg.message); - })(err); - process.exit(1); - })(); - }); - - log.debug("Finished HTTP Call File Plugin"); -} + if (outputFilePath && outputFilePath.length && outputFilePath !== '""' && outputFilePath !== '" "') { + //https://nodejs.org/docs/latest-v14.x/api/fs.html#fs_fs_writefilesync_file_data_options + fs.writeFileSync(outputFilePath, res.body, { + encoding: "utf8", + mode: 666, + flag: "w" + }); + log.debug("The task output parameter successfully saved to provided file path."); + } else { + utils.setOutputParameter("response", res.body.toString()); + log.debug("The task output parameter successfully saved to standard response file."); + } + log.good("Response successfully received!"); + }) + .catch(err => { + // log.err("HTTP Promise error:", err, "Cause: ", err.cause); + if (err.statusCode || err.statusMessage || err.body) { + log.err(`HTTP Promise error:`); + log.err(` \n Status Code: ${err.statusCode}`); + log.err(` \n Status Message: ${err.statusMessage}`); + log.err(` \n Response: ${err.body?.toString()}`); + } else { + log.err(`HTTP Promise error:`, err); + } + (async () => { + await (async function(msg) { + utils.setOutputParameter("statusCode", msg?.statusCode ?? ""); + utils.setOutputParameter("response", msg?.body?.toString() ?? msg.message); + })(err); + process.exit(1); + })(); + }); -module.exports = { - execute + log.debug("Finished HTTP Call File Plugin"); + } }; diff --git a/package-lock.json b/package-lock.json index 97bafc4..7281422 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@slack/web-api": "^5.8.0", "@slack/webhook": "^5.0.3", "axios": "^0.25.0", - "cacheable-lookup": "^7.0.0", + "cacheable-lookup": "^6.1.0", "cloudevents": "^5.3.0", "googleapis": "^89.0.0", "https": "^1.0.0", @@ -833,11 +833,11 @@ "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, "node_modules/cacheable-lookup": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz", - "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-6.1.0.tgz", + "integrity": "sha512-KJ/Dmo1lDDhmW2XDPMo+9oiy/CeqosPguPCrgcVzKyZrL6pM1gU2GmPY/xo6OQPTUaA/c0kwHuywB4E6nmT9ww==", "engines": { - "node": ">=14.16" + "node": ">=10.6.0" } }, "node_modules/cacheable-request": { @@ -5879,9 +5879,9 @@ "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, "cacheable-lookup": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz", - "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==" + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-6.1.0.tgz", + "integrity": "sha512-KJ/Dmo1lDDhmW2XDPMo+9oiy/CeqosPguPCrgcVzKyZrL6pM1gU2GmPY/xo6OQPTUaA/c0kwHuywB4E6nmT9ww==" }, "cacheable-request": { "version": "6.1.0", diff --git a/package.json b/package.json index f4654ea..0c130d7 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "twilio": "^3.73.0", "url": "^0.11.0", "wait-on": "^4.0.2", - "cacheable-lookup": "^7.0.0" + "cacheable-lookup": "^6.1.0" }, "scripts": { "test": "cross-env NODE_ENV=test mocha tests/**/*.spec.js",