step

.gitignore

@@ -1,4 +1,5 @@
 node_modules
 .vscode
 archive
+pnpm-lock.yaml
 dist

src/crawler.ts

@@ -4,11 +4,12 @@ import { isFunction, setDefaults, flattenDeep } from "./lib/utils.js";
 import { getValidOptions, alignOptions } from "./options.js";
 import type { crawlerOptions, requestOptions } from "./types/crawler.js";
 import { promisify } from "util";
+import { load } from "cheerio";
 import got from "got";
 import seenreq from "seenreq";
 import iconv from "iconv-lite";
-import cheerio from "cheerio";
 
+//@todo change log method
 process.env.NODE_ENV = process.env.NODE_ENV ?? process.argv[2] ?? "debug";
 
 if (process.env.NODE_ENV !== "debug") {
@@ -89,11 +90,11 @@ class Crawler extends EventEmitter {
         return charset;
     };
 
-    // private _getContentType = (headers: Record<string, string>): string[] => {
-    //     let contentType = headers["content-type"];
-    //     if (!contentType) return [];
-    //     return contentType.split(";").map((type: string) => type.trim());
-    // }
+    private _checkHtml = (headers: Record<string, string>): boolean => {
+        const contentType = headers["content-type"];
+        if (/xml|html/i.test(contentType)) return true;
+        return false;
+    };
 
     private _schedule = async (options: crawlerOptions): Promise<void> => {
         this.emit("schedule", options);
@@ -198,12 +199,17 @@ class Crawler extends EventEmitter {
             resError = error;
         }
 
-        // @todo: jQuery injection
-
-        // const injectableTypes = ["html", "xhtml", "text/xml", "application/xml", "+xml"];
-        // if (this._getContentType(response.headers).some(type => injectableTypes.includes(type))) {
-        //     console.warn("response body is not HTML, skip injecting. Set jQuery to false to suppress this message");
-        // }
+        if (options.jQuery === true) {
+            if (response.body === "" || !this._checkHtml(response.headers)) {
+                console.warn("response body is not HTML, skip injecting. Set jQuery to false to suppress this message");
+            } else {
+                try {
+                    response.$ = load(response.body);
+                } catch (err) {
+                    console.error(err);
+                }
+            }
+        }
 
         if (options.callback && typeof options.callback === "function") {
             return options.callback(resError, response, options.release);

src/options.ts

@@ -64,7 +64,5 @@ export const alignOptions = (options: any): any => {
         }
     });
     cleanObject(gotOptions);
-    // console.log(gotOptions);
-    // process.exit(0);
     return gotOptions;
 };

test.js

@@ -1,38 +1,16 @@
-// const crawler = require('./dist/index.js');
-// const result = crawler.add({
-//     url: "http://www.google.com",
-//     method: "GET",
-//     incomingEncoding: "utf8",
-//     callback: (err, res, done) => {
-//         console.log(response.body)
-//     }
-// });
-// console.log(result)
-// import got from "got";
-// import fs from "fs";
-// const result = await got({
-//     url: "http://www.google.com"
-// });
-// fs.writeFileSync("result", result.body);
-// let a = 5
-// console.log(a++ % 6)
-const a = {
-    a: {
-        q1: 1,
-        q2: 2,
-        q3: 3
+import { crawler } from './dist/index.js';
+crawler.add({
+    url: 'https://www.google.com',
+    method: 'GET',
+    headers: {
+        'Content-Type': 'application/json'
     },
-    b: {
-        q1: 1,
-        q2: 2,
-        q3: 3
+    callback: (err, res, done) => {
+        if (err) {
+            console.log(err);
+        } else {
+            console.log(res.body);
+        }
+        done();
     }
-}
-// for(const value of Object.values(a)){
-//     value.q1++;
-//     value.q2 = 0;
-//     delete value.q3;
-// }
-// console.log(a)
-const { q1, q4 } = a.a;
-console.log(q1, q4)
+});