Adding basic remote fetching for external resources

SphinxKnight · Feb 28, 2018 · 43c11b1 · 43c11b1
1 parent 9ccb4c2
commit 43c11b1
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 16 deletions.
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -6,6 +6,7 @@
     },
     "extends": "eslint:recommended",
     "parserOptions": {
+        "ecmaVersion": 8,
         "sourceType": "module"
     },
     "rules": {

diff --git a/bin/css/css-extracter.js b/bin/css/css-extracter.js
@@ -2,8 +2,9 @@ const fs = require("fs");
 const htmlParser = require("htmlparser2");
 const readline = require("readline");
 const path = require("path");
+const fetchURL = require("../../lib/fetchURL");
 
-exports.analyzeFile = function analyzeFile (fileName,callback){
+exports.analyzeFile = function (fileName,callback){
     const rl = readline.createInterface({
         input: fs.createReadStream(fileName),
         crlfDelay: Infinity
@@ -22,11 +23,16 @@ exports.analyzeFile = function analyzeFile (fileName,callback){
                 inStyle = true;
             }
             if(name === "link" && attribs.rel === "stylesheet"){
-                const fragment = {
-                    "fileName": path.relative(path.dirname(fileName),path.resolve(path.dirname(fileName),attribs.href)),
-                    "lineShift": 0,
-                    "content": fs.readFileSync(attribs.href,"utf-8")
-                };
+                const fragment = {};
+                fragment.lineShift = 0;
+                if(attribs.href.startsWith("http")){
+                    fragment.fileName = attribs.href;
+                    const content = fetchURL.fetchURL(attribs.href);
+                    fragment.content = content;
+                } else {
+                    fragment.fileName = path.relative(path.dirname(fileName),path.resolve(path.dirname(fileName),attribs.href));
+                    fragment.content = fs.readFileSync(attribs.href,"utf-8");
+                }
                 acc.push(fragment);
             }
         },

diff --git a/bin/js/js-extracter.js b/bin/js/js-extracter.js
@@ -2,6 +2,7 @@ const fs = require("fs");
 const htmlParser = require("htmlparser2");
 const readline = require("readline");
 const path = require("path");
+const fetchURL = require("../../lib/fetchURL");
 
 exports.analyzeFile = function analyzeFile (fileName, callback){
     const rl = readline.createInterface({
@@ -21,11 +22,16 @@ exports.analyzeFile = function analyzeFile (fileName, callback){
                 inScript = true;
                 numLineBlock = numLine;
                 if("src" in attribs){
-                    const fragment = {
-                        "fileName":path.relative(path.dirname(fileName),path.resolve(path.dirname(fileName),attribs.src)),
-                        "lineShift": 0,
-                        "content":fs.readFileSync(attribs.src,"utf-8")
-                    };
+                    const fragment = {};
+                    fragment.lineShift = 0;
+                    if(attribs.src.startsWith("http")){
+                        fragment.fileName = attribs.src;
+                        const content = fetchURL.fetchURL(attribs.src);
+                        fragment.content = content;
+                    } else {
+                        fragment.fileName = path.relative(path.dirname(fileName),path.resolve(path.dirname(fileName),attribs.src));
+                        fragment.content = fs.readFileSync(attribs.src,"utf-8");
+                    }
                     acc.push(fragment);
                 }
             }

diff --git a/index.js b/index.js
@@ -75,8 +75,8 @@ if(mode === "normal"){
 
     // Let's get the CSS inside the site
     cssExtracter.analyzeFile(fileName, (e, acc) => {
-        acc.map((block) => {
-            cssAnalyzer.analyzeString(block.content, scope, block.lineShift, block.fileName, (e, d) => {
+        acc.map(async (block) => {
+            cssAnalyzer.analyzeString(await block.content, scope, block.lineShift, block.fileName, (e, d) => {
                 if (e) {
                     console.error(e);
                     return false;
@@ -92,8 +92,8 @@ if(mode === "normal"){
 
     // Let's get the JavaScript inside the site
     jsExtracter.analyzeFile(fileName, (e, acc) => {
-        acc.map((block) => {
-            jsAnalyzer.analyzeString(block.content, scope, block.lineShift, block.fileName, (e, d) => {
+        acc.map(async (block) => {
+            jsAnalyzer.analyzeString(await block.content, scope, block.lineShift, block.fileName, (e, d) => {
                 if (e) {
                     console.error(e);
                     return false;
@@ -163,7 +163,7 @@ function sortReport (reportLineA, reportLineB){
 }
 
 function printReportLine (reportLine) {
-    console.log("\t\t" + reportLine.browser + " incompatible - @" + reportLine.fileName + "#L" + reportLine.line + " - " + reportLine.featureName + (reportLine.featureVersion ? (" - minVer: " + reportLine.featureVersion) : " not implemented"));
+    console.log(`\t\t${reportLine.browser} incompatible - ${reportLine.fileName}#L${reportLine.line} - ${reportLine.featureName} ${(reportLine.featureVersion ? (" - minVer: " + reportLine.featureVersion) : " not implemented")}`);
 }
 
 exports.htmlStringAnalyzer = function htmlStringAnalyzer (string, browserScope){

diff --git a/lib/fetchURL.js b/lib/fetchURL.js
@@ -0,0 +1,22 @@
+const http = require("http");
+const https = require("https");
+
+exports.fetchURL = (url) => {
+    return new Promise((resolve, reject) => {
+        let client = http;
+        if (url.toString().startsWith("https")) {
+            client = https;
+        }
+        client.get(url, (resp) => {
+            let data = "";
+            resp.on("data", (chunk) => {
+                data += chunk;
+            });
+            resp.on("end", () => {
+                resolve(data);
+            });
+        }).on("error", (err) => {
+            reject(err);
+        });
+    });
+};