From 24bdc6b99194f863e53c81fb2d928406414f1ea6 Mon Sep 17 00:00:00 2001 From: MiniAst Date: Thu, 1 Aug 2024 12:41:34 +0800 Subject: [PATCH] update dependencies & types --- README.md | 2 +- eslint.config.js | 1 - package.json | 14 +- pnpm-lock.yaml | 297 +++++++++++++++++++++---------------------- src/lib/utils.ts | 8 +- src/options.ts | 2 +- src/types/crawler.ts | 28 +++- 7 files changed, 182 insertions(+), 170 deletions(-) diff --git a/README.md b/README.md index 27e5d5e..50449d6 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ ------ -[![npm package](https://nodei.co/npm/crawler.png?downloads=true&downloadRank=true&stars=true)](https://www.npmjs.com/package/crawler/v/2.0.0) +[![npm package](https://nodei.co/npm/crawler.png?downloads=true&downloadRank=true&stars=true)](https://www.npmjs.com/package/crawler/v/2.0.2) [![CircleCI](https://circleci.com/gh/bda-research/node-crawler/tree/master.svg?style=svg)](https://circleci.com/gh/bda-research/node-crawler/tree/master) [![NPM download][download-image]][download-url] diff --git a/eslint.config.js b/eslint.config.js index 20295b3..00e5bcd 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -21,7 +21,6 @@ const options = [ files: ["src/**/*.ts"], rules: { "@typescript-eslint/no-explicit-any": "warn", - "@typescript-eslint/ban-types": "warn", "@typescript-eslint/no-unused-vars": [ "error", { diff --git a/package.json b/package.json index a7cf947..0eb2c60 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "license": "MIT", "dependencies": { "cheerio": "1.0.0-rc.12", - "got": "^14.4.1", + "got": "^14.4.2", "hpagent": "^1.2.0", "http2-wrapper": "^2.2.1", "iconv-lite": "^0.6.3", @@ -44,19 +44,19 @@ "tslog": "^4.9.3" }, "devDependencies": { - "@eslint/js": "^9.7.0", + "@eslint/js": "^9.8.0", "@types/got": "^9.6.12", - "@types/node": "^20.14.10", + "@types/node": "^20.14.13", "ava": "^6.1.3", "c8": "^10.1.2", - "eslint": "^9.7.0", + "eslint": "^9.8.0", "globals": "^15.8.0", "nock": "^13.5.4", "sinon": "^18.0.0", "tough-cookie": "^4.1.4", - "tsx": "^4.16.2", - "typescript": "5.4.5", - "typescript-eslint": "8.0.0-alpha.30" + "tsx": "^4.16.3", + "typescript": "^5.5.4", + "typescript-eslint": "^8.0.0" }, "ava": { "files": [ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 052b4ef..1962f40 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: 1.0.0-rc.12 version: 1.0.0-rc.12 got: - specifier: ^14.4.1 - version: 14.4.1 + specifier: ^14.4.2 + version: 14.4.2 hpagent: specifier: ^1.2.0 version: 1.2.0 @@ -31,14 +31,14 @@ importers: version: 4.9.3 devDependencies: '@eslint/js': - specifier: ^9.7.0 - version: 9.7.0 + specifier: ^9.8.0 + version: 9.8.0 '@types/got': specifier: ^9.6.12 version: 9.6.12 '@types/node': - specifier: ^20.14.10 - version: 20.14.10 + specifier: ^20.14.13 + version: 20.14.13 ava: specifier: ^6.1.3 version: 6.1.3 @@ -46,8 +46,8 @@ importers: specifier: ^10.1.2 version: 10.1.2 eslint: - specifier: ^9.7.0 - version: 9.7.0 + specifier: ^9.8.0 + version: 9.8.0 globals: specifier: ^15.8.0 version: 15.8.0 @@ -61,14 +61,14 @@ importers: specifier: ^4.1.4 version: 4.1.4 tsx: - specifier: ^4.16.2 - version: 4.16.2 + specifier: ^4.16.3 + version: 4.16.3 typescript: - specifier: 5.4.5 - version: 5.4.5 + specifier: ^5.5.4 + version: 5.5.4 typescript-eslint: - specifier: 8.0.0-alpha.30 - version: 8.0.0-alpha.30(eslint@9.7.0)(typescript@5.4.5) + specifier: ^8.0.0 + version: 8.0.0(eslint@9.8.0)(typescript@5.5.4) packages: @@ -223,16 +223,16 @@ packages: resolution: {integrity: sha512-G/M/tIiMrTAxEWRfLfQJMmGNX28IxBg4PBz8XqQhqUHLFI6TL2htpIB1iQCj144V5ee/JaKyT9/WZ0MGZWfA7A==} engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} - '@eslint/config-array@0.17.0': - resolution: {integrity: sha512-A68TBu6/1mHHuc5YJL0U0VVeGNiklLAL6rRmhTCP2B5XjWLMnrX+HkO+IAXyHvks5cyyY1jjK5ITPQ1HGS2EVA==} + '@eslint/config-array@0.17.1': + resolution: {integrity: sha512-BlYOpej8AQ8Ev9xVqroV7a02JK3SkBAaN9GfMMH9W6Ch8FlQlkjGw4Ir7+FgYwfirivAf4t+GtzuAxqfukmISA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} '@eslint/eslintrc@3.1.0': resolution: {integrity: sha512-4Bfj15dVJdoy3RfZmmo86RK1Fwzn6SstsvK9JS+BaVKqC6QQQQyXekNaC+g+LKNgkQ+2VhGAzm6hO40AhMR3zQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@eslint/js@9.7.0': - resolution: {integrity: sha512-ChuWDQenef8OSFnvuxv0TCVxEwmu3+hPNKvM9B34qpM0rDRbjL8t5QkQeHHeAfsKQjuH9wS82WeCi1J/owatng==} + '@eslint/js@9.8.0': + resolution: {integrity: sha512-MfluB7EUfxXtv3i/++oh89uzAr4PDI4nn201hsp+qaXqsjAWzinlZEHEfPgAX4doIlKvPG/i0A9dpKxOLII8yA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} '@eslint/object-schema@2.1.4': @@ -292,9 +292,9 @@ packages: '@sec-ant/readable-stream@0.4.1': resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} - '@sindresorhus/is@6.3.1': - resolution: {integrity: sha512-FX4MfcifwJyFOI2lPoX7PQxCqx8BG1HCho7WdiXwpEQx1Ycij0JxkfYtGK7yqNScrZGSlt6RE6sw8QYoH7eKnQ==} - engines: {node: '>=16'} + '@sindresorhus/is@7.0.0': + resolution: {integrity: sha512-WDTlVTyvFivSOuyvMeedzg2hdoBLZ3f1uNVuEida2Rl9BrfjrIRjWA/VZIrMRLvSwJYCAlCRA3usDt1THytxWQ==} + engines: {node: '>=18'} '@sindresorhus/merge-streams@2.3.0': resolution: {integrity: sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==} @@ -328,14 +328,14 @@ packages: '@types/istanbul-lib-coverage@2.0.6': resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==} - '@types/node@20.14.10': - resolution: {integrity: sha512-MdiXf+nDuMvY0gJKxyfZ7/6UFsETO7mGKF54MVD/ekJS6HdFtpZFBgrh6Pseu64XTb2MLyFPlbW6hj8HYRQNOQ==} + '@types/node@20.14.13': + resolution: {integrity: sha512-+bHoGiZb8UiQ0+WEtmph2IWQCjIqg8MDZMAV+ppRRhUZnquF5mQkP/9vpSwJClEiSM/C7fZZExPzfU0vJTyp8w==} '@types/tough-cookie@4.0.5': resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==} - '@typescript-eslint/eslint-plugin@8.0.0-alpha.30': - resolution: {integrity: sha512-2CBUupdkfbE3eATph4QeZejvT+M+1bVur+zXlVx09WN31phap51ps/qemeclnCbGEz6kTgBDmScrr9XmmF8/Pg==} + '@typescript-eslint/eslint-plugin@8.0.0': + resolution: {integrity: sha512-STIZdwEQRXAHvNUS6ILDf5z3u95Gc8jzywunxSNqX00OooIemaaNIA0vEgynJlycL5AjabYLLrIyHd4iazyvtg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: '@typescript-eslint/parser': ^8.0.0 || ^8.0.0-alpha.0 @@ -345,8 +345,8 @@ packages: typescript: optional: true - '@typescript-eslint/parser@8.0.0-alpha.30': - resolution: {integrity: sha512-tAYgFmgXU1MlCK3nbblUvJlDSibBvxtAQXGrF3IG0KmnRza9FXILZifHWL0rrwacDn40K53K607Fk2QkMjiGgw==} + '@typescript-eslint/parser@8.0.0': + resolution: {integrity: sha512-pS1hdZ+vnrpDIxuFXYQpLTILglTjSYJ9MbetZctrUawogUsPdz31DIIRZ9+rab0LhYNTsk88w4fIzVheiTbWOQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: eslint: ^8.57.0 || ^9.0.0 @@ -355,12 +355,12 @@ packages: typescript: optional: true - '@typescript-eslint/scope-manager@8.0.0-alpha.30': - resolution: {integrity: sha512-FGW/iPWGyPFamAVZ60oCAthMqQrqafUGebF8UKuq/ha+e9SVG6YhJoRzurlQXOVf8dHfOhJ0ADMXyFnMc53clg==} + '@typescript-eslint/scope-manager@8.0.0': + resolution: {integrity: sha512-V0aa9Csx/ZWWv2IPgTfY7T4agYwJyILESu/PVqFtTFz9RIS823mAze+NbnBI8xiwdX3iqeQbcTYlvB04G9wyQw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@typescript-eslint/type-utils@8.0.0-alpha.30': - resolution: {integrity: sha512-FrnhlCKEKZKRbpDviHkIU9tayIUGTOfa+SjvrRv6p/AJIUv6QT8oRboRjLH/cCuwUEbM0k5UtRWYug4albHUqQ==} + '@typescript-eslint/type-utils@8.0.0': + resolution: {integrity: sha512-mJAFP2mZLTBwAn5WI4PMakpywfWFH5nQZezUQdSKV23Pqo6o9iShQg1hP2+0hJJXP2LnZkWPphdIq4juYYwCeg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '*' @@ -368,12 +368,12 @@ packages: typescript: optional: true - '@typescript-eslint/types@8.0.0-alpha.30': - resolution: {integrity: sha512-4WzLlw27SO9pK9UFj/Hu7WGo8WveT0SEiIpFVsV2WwtQmLps6kouwtVCB8GJPZKJyurhZhcqCoQVQFmpv441Vg==} + '@typescript-eslint/types@8.0.0': + resolution: {integrity: sha512-wgdSGs9BTMWQ7ooeHtu5quddKKs5Z5dS+fHLbrQI+ID0XWJLODGMHRfhwImiHoeO2S5Wir2yXuadJN6/l4JRxw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@typescript-eslint/typescript-estree@8.0.0-alpha.30': - resolution: {integrity: sha512-WSXbc9ZcXI+7yC+6q95u77i8FXz6HOLsw3ST+vMUlFy1lFbXyFL/3e6HDKQCm2Clt0krnoCPiTGvIn+GkYPn4Q==} + '@typescript-eslint/typescript-estree@8.0.0': + resolution: {integrity: sha512-5b97WpKMX+Y43YKi4zVcCVLtK5F98dFls3Oxui8LbnmRsseKenbbDinmvxrWegKDMmlkIq/XHuyy0UGLtpCDKg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '*' @@ -381,14 +381,14 @@ packages: typescript: optional: true - '@typescript-eslint/utils@8.0.0-alpha.30': - resolution: {integrity: sha512-rfhqfLqFyXhHNDwMnHiVGxl/Z2q/3guQ1jLlGQ0hi9Rb7inmwz42crM+NnLPR+2vEnwyw1P/g7fnQgQ3qvFx4g==} + '@typescript-eslint/utils@8.0.0': + resolution: {integrity: sha512-k/oS/A/3QeGLRvOWCg6/9rATJL5rec7/5s1YmdS0ZU6LHveJyGFwBvLhSRBv6i9xaj7etmosp+l+ViN1I9Aj/Q==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: eslint: ^8.57.0 || ^9.0.0 - '@typescript-eslint/visitor-keys@8.0.0-alpha.30': - resolution: {integrity: sha512-XZuNurZxBqmr6ZIRIwWFq7j5RZd6ZlkId/HZEWyfciK+CWoyOxSF9Pv2VXH9Rlu2ZG2PfbhLz2Veszl4Pfn7yA==} + '@typescript-eslint/visitor-keys@8.0.0': + resolution: {integrity: sha512-oN0K4nkHuOyF3PVMyETbpP5zp6wfyOvm7tWhTMfoqxSSsPmJIh6JNASuZDlODE8eE+0EB9uar+6+vxr9DBTYOA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} '@vercel/nft@0.26.5': @@ -634,8 +634,8 @@ packages: resolution: {integrity: sha512-uqCUKXE5q1PNBXjPqvwhwJf9SwMoAHBgWJ6DcrnS5o+W2JOiIILl0JEdVD8SGujrNS02GGxgwAg2PN2zONgtjg==} engines: {node: '>=6'} - debug@4.3.5: - resolution: {integrity: sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==} + debug@4.3.6: + resolution: {integrity: sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==} engines: {node: '>=6.0'} peerDependencies: supports-color: '*' @@ -739,8 +739,8 @@ packages: resolution: {integrity: sha512-OtIRv/2GyiF6o/d8K7MYKKbXrOUBIK6SfkIRM4Z0dY3w+LiQ0vy3F57m0Z71bjbyeiWFiHJ8brqnmE6H6/jEuw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - eslint@9.7.0: - resolution: {integrity: sha512-FzJ9D/0nGiCGBf8UXO/IGLTgLVzIxze1zpfA8Ton2mjLovXdAPlYDv+MQDcqj3TmrhAGYfOpz9RfR+ent0AgAw==} + eslint@9.8.0: + resolution: {integrity: sha512-K8qnZ/QJzT2dLKdZJVX6W4XOwBzutMYmt0lqUS+JdXgd+HTYFlonFgkJ8s44d/zMPPCnOOk0kMWCApCPhiOy9A==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} hasBin: true @@ -858,16 +858,12 @@ packages: resolution: {integrity: sha512-2nk+7SIVb14QrgXFHcm84tD4bKQz0RxPuMT8Ag5KPOq7J5fEmAg0UbXdTOSHqNuHSU28k55qnceesxXRZGzKWA==} engines: {node: '>=18'} - get-stream@8.0.1: - resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} - engines: {node: '>=16'} - get-stream@9.0.1: resolution: {integrity: sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==} engines: {node: '>=18'} - get-tsconfig@4.7.5: - resolution: {integrity: sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw==} + get-tsconfig@4.7.6: + resolution: {integrity: sha512-ZAqrLlu18NbDdRaHq+AKXzAmqIUPswPWKUchfytdAjiRFnCe5ojG2bstg6mRiZabkKfCoL/e98pbBELIV/YCeA==} glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} @@ -901,8 +897,8 @@ packages: resolution: {integrity: sha512-s3Fq41ZVh7vbbe2PN3nrW7yC7U7MFVc5c98/iTl9c2GawNMKx/J648KQRW6WKkuU8GIbbh2IXfIRQjOZnXcTnw==} engines: {node: '>=18'} - got@14.4.1: - resolution: {integrity: sha512-IvDJbJBUeexX74xNQuMIVgCRRuNOm5wuK+OC3Dc2pnSoh1AOmgc7JVj7WC+cJ4u0aPcO9KZ2frTXcqK4W/5qTQ==} + got@14.4.2: + resolution: {integrity: sha512-+Te/qEZ6hr7i+f0FNgXx/6WQteSM/QqueGvxeYQQFm0GDfoxLVJ/oiwUKYMTeioColWUTdewZ06hmrBjw6F7tw==} engines: {node: '>=20'} graceful-fs@4.2.11: @@ -1309,8 +1305,8 @@ packages: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} - pretty-ms@9.0.0: - resolution: {integrity: sha512-E9e9HJ9R9NasGOgPaPE8VMeiPKAyWR5jcFpNnwIejslIhWqdqOrb2wShBsncMPUb+BcCd2OPYfh7p2W6oemTng==} + pretty-ms@9.1.0: + resolution: {integrity: sha512-o1piW0n3tgKIKCwk2vpM/vOV13zjJzvP37Ioze54YlTHE06m4tjEbzg9WsKkvTuyYln2DHjo5pY4qrZGI0otpw==} engines: {node: '>=18'} propagate@2.0.1: @@ -1392,8 +1388,8 @@ packages: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true - semver@7.6.2: - resolution: {integrity: sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==} + semver@7.6.3: + resolution: {integrity: sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==} engines: {node: '>=10'} hasBin: true @@ -1516,8 +1512,8 @@ packages: resolution: {integrity: sha512-oDWuGVONxhVEBtschLf2cs/Jy8i7h1T+CpdkTNWQgdAF7DhRo2G8vMCgILKe7ojdEkLhICWgI1LYSSKaJsRgcw==} engines: {node: '>=16'} - tsx@4.16.2: - resolution: {integrity: sha512-C1uWweJDgdtX2x600HjaFaucXTilT7tgUZHbOE4+ypskZ1OP8CRCSDkCxG6Vya9EwaFIVagWwpaVAn5wzypaqQ==} + tsx@4.16.3: + resolution: {integrity: sha512-MP8AEUxVnboD2rCC6kDLxnpDBNWN9k3BSVU/0/nNxgm70bPBnfn+yCKcnOsIVPQwdkbKYoFOlKjjWZWJ2XCXUg==} engines: {node: '>=18.0.0'} hasBin: true @@ -1529,16 +1525,20 @@ packages: resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==} engines: {node: '>=4'} + type-detect@4.1.0: + resolution: {integrity: sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==} + engines: {node: '>=4'} + type-fest@0.13.1: resolution: {integrity: sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==} engines: {node: '>=10'} - type-fest@4.21.0: - resolution: {integrity: sha512-ADn2w7hVPcK6w1I0uWnM//y1rLXZhzB9mr0a3OirzclKF1Wp6VzevUmzz/NRAWunOT6E8HrnpGY7xOfc6K57fA==} + type-fest@4.23.0: + resolution: {integrity: sha512-ZiBujro2ohr5+Z/hZWHESLz3g08BBdrdLMieYFULJO+tWc437sn8kQsWLJoZErY8alNhxre9K4p3GURAG11n+w==} engines: {node: '>=16'} - typescript-eslint@8.0.0-alpha.30: - resolution: {integrity: sha512-/vGhBMsK1TpadQh1eQ02c5pyiPGmKR9cVzX5C9plZ+LC0HPLpWoJbbTVfQN7BkIK7tUxDt2BFr3pFL5hDDrx7g==} + typescript-eslint@8.0.0: + resolution: {integrity: sha512-yQWBJutWL1PmpmDddIOl9/Mi6vZjqNCjqSGBMQ4vsc2Aiodk0SnbQQWPXbSy0HNuKCuGkw1+u4aQ2mO40TdhDQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '*' @@ -1546,8 +1546,8 @@ packages: typescript: optional: true - typescript@5.4.5: - resolution: {integrity: sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==} + typescript@5.5.4: + resolution: {integrity: sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==} engines: {node: '>=14.17'} hasBin: true @@ -1704,17 +1704,17 @@ snapshots: '@esbuild/win32-x64@0.21.5': optional: true - '@eslint-community/eslint-utils@4.4.0(eslint@9.7.0)': + '@eslint-community/eslint-utils@4.4.0(eslint@9.8.0)': dependencies: - eslint: 9.7.0 + eslint: 9.8.0 eslint-visitor-keys: 3.4.3 '@eslint-community/regexpp@4.11.0': {} - '@eslint/config-array@0.17.0': + '@eslint/config-array@0.17.1': dependencies: '@eslint/object-schema': 2.1.4 - debug: 4.3.5 + debug: 4.3.6 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -1722,7 +1722,7 @@ snapshots: '@eslint/eslintrc@3.1.0': dependencies: ajv: 6.12.6 - debug: 4.3.5 + debug: 4.3.6 espree: 10.1.0 globals: 14.0.0 ignore: 5.3.1 @@ -1733,7 +1733,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@eslint/js@9.7.0': {} + '@eslint/js@9.8.0': {} '@eslint/object-schema@2.1.4': {} @@ -1770,7 +1770,7 @@ snapshots: nopt: 5.0.0 npmlog: 5.0.1 rimraf: 3.0.2 - semver: 7.6.2 + semver: 7.6.3 tar: 6.2.1 transitivePeerDependencies: - encoding @@ -1798,7 +1798,7 @@ snapshots: '@sec-ant/readable-stream@0.4.1': {} - '@sindresorhus/is@6.3.1': {} + '@sindresorhus/is@7.0.0': {} '@sindresorhus/merge-streams@2.3.0': {} @@ -1818,7 +1818,7 @@ snapshots: dependencies: '@sinonjs/commons': 2.0.0 lodash.get: 4.4.2 - type-detect: 4.0.8 + type-detect: 4.1.0 '@sinonjs/text-encoding@0.7.2': {} @@ -1828,7 +1828,7 @@ snapshots: '@types/got@9.6.12': dependencies: - '@types/node': 20.14.10 + '@types/node': 20.14.13 '@types/tough-cookie': 4.0.5 form-data: 2.5.1 @@ -1836,91 +1836,91 @@ snapshots: '@types/istanbul-lib-coverage@2.0.6': {} - '@types/node@20.14.10': + '@types/node@20.14.13': dependencies: undici-types: 5.26.5 '@types/tough-cookie@4.0.5': {} - '@typescript-eslint/eslint-plugin@8.0.0-alpha.30(@typescript-eslint/parser@8.0.0-alpha.30(eslint@9.7.0)(typescript@5.4.5))(eslint@9.7.0)(typescript@5.4.5)': + '@typescript-eslint/eslint-plugin@8.0.0(@typescript-eslint/parser@8.0.0(eslint@9.8.0)(typescript@5.5.4))(eslint@9.8.0)(typescript@5.5.4)': dependencies: '@eslint-community/regexpp': 4.11.0 - '@typescript-eslint/parser': 8.0.0-alpha.30(eslint@9.7.0)(typescript@5.4.5) - '@typescript-eslint/scope-manager': 8.0.0-alpha.30 - '@typescript-eslint/type-utils': 8.0.0-alpha.30(eslint@9.7.0)(typescript@5.4.5) - '@typescript-eslint/utils': 8.0.0-alpha.30(eslint@9.7.0)(typescript@5.4.5) - '@typescript-eslint/visitor-keys': 8.0.0-alpha.30 - eslint: 9.7.0 + '@typescript-eslint/parser': 8.0.0(eslint@9.8.0)(typescript@5.5.4) + '@typescript-eslint/scope-manager': 8.0.0 + '@typescript-eslint/type-utils': 8.0.0(eslint@9.8.0)(typescript@5.5.4) + '@typescript-eslint/utils': 8.0.0(eslint@9.8.0)(typescript@5.5.4) + '@typescript-eslint/visitor-keys': 8.0.0 + eslint: 9.8.0 graphemer: 1.4.0 ignore: 5.3.1 natural-compare: 1.4.0 - ts-api-utils: 1.3.0(typescript@5.4.5) + ts-api-utils: 1.3.0(typescript@5.5.4) optionalDependencies: - typescript: 5.4.5 + typescript: 5.5.4 transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.0.0-alpha.30(eslint@9.7.0)(typescript@5.4.5)': + '@typescript-eslint/parser@8.0.0(eslint@9.8.0)(typescript@5.5.4)': dependencies: - '@typescript-eslint/scope-manager': 8.0.0-alpha.30 - '@typescript-eslint/types': 8.0.0-alpha.30 - '@typescript-eslint/typescript-estree': 8.0.0-alpha.30(typescript@5.4.5) - '@typescript-eslint/visitor-keys': 8.0.0-alpha.30 - debug: 4.3.5 - eslint: 9.7.0 + '@typescript-eslint/scope-manager': 8.0.0 + '@typescript-eslint/types': 8.0.0 + '@typescript-eslint/typescript-estree': 8.0.0(typescript@5.5.4) + '@typescript-eslint/visitor-keys': 8.0.0 + debug: 4.3.6 + eslint: 9.8.0 optionalDependencies: - typescript: 5.4.5 + typescript: 5.5.4 transitivePeerDependencies: - supports-color - '@typescript-eslint/scope-manager@8.0.0-alpha.30': + '@typescript-eslint/scope-manager@8.0.0': dependencies: - '@typescript-eslint/types': 8.0.0-alpha.30 - '@typescript-eslint/visitor-keys': 8.0.0-alpha.30 + '@typescript-eslint/types': 8.0.0 + '@typescript-eslint/visitor-keys': 8.0.0 - '@typescript-eslint/type-utils@8.0.0-alpha.30(eslint@9.7.0)(typescript@5.4.5)': + '@typescript-eslint/type-utils@8.0.0(eslint@9.8.0)(typescript@5.5.4)': dependencies: - '@typescript-eslint/typescript-estree': 8.0.0-alpha.30(typescript@5.4.5) - '@typescript-eslint/utils': 8.0.0-alpha.30(eslint@9.7.0)(typescript@5.4.5) - debug: 4.3.5 - ts-api-utils: 1.3.0(typescript@5.4.5) + '@typescript-eslint/typescript-estree': 8.0.0(typescript@5.5.4) + '@typescript-eslint/utils': 8.0.0(eslint@9.8.0)(typescript@5.5.4) + debug: 4.3.6 + ts-api-utils: 1.3.0(typescript@5.5.4) optionalDependencies: - typescript: 5.4.5 + typescript: 5.5.4 transitivePeerDependencies: - eslint - supports-color - '@typescript-eslint/types@8.0.0-alpha.30': {} + '@typescript-eslint/types@8.0.0': {} - '@typescript-eslint/typescript-estree@8.0.0-alpha.30(typescript@5.4.5)': + '@typescript-eslint/typescript-estree@8.0.0(typescript@5.5.4)': dependencies: - '@typescript-eslint/types': 8.0.0-alpha.30 - '@typescript-eslint/visitor-keys': 8.0.0-alpha.30 - debug: 4.3.5 + '@typescript-eslint/types': 8.0.0 + '@typescript-eslint/visitor-keys': 8.0.0 + debug: 4.3.6 globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.5 - semver: 7.6.2 - ts-api-utils: 1.3.0(typescript@5.4.5) + semver: 7.6.3 + ts-api-utils: 1.3.0(typescript@5.5.4) optionalDependencies: - typescript: 5.4.5 + typescript: 5.5.4 transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.0.0-alpha.30(eslint@9.7.0)(typescript@5.4.5)': + '@typescript-eslint/utils@8.0.0(eslint@9.8.0)(typescript@5.5.4)': dependencies: - '@eslint-community/eslint-utils': 4.4.0(eslint@9.7.0) - '@typescript-eslint/scope-manager': 8.0.0-alpha.30 - '@typescript-eslint/types': 8.0.0-alpha.30 - '@typescript-eslint/typescript-estree': 8.0.0-alpha.30(typescript@5.4.5) - eslint: 9.7.0 + '@eslint-community/eslint-utils': 4.4.0(eslint@9.8.0) + '@typescript-eslint/scope-manager': 8.0.0 + '@typescript-eslint/types': 8.0.0 + '@typescript-eslint/typescript-estree': 8.0.0(typescript@5.5.4) + eslint: 9.8.0 transitivePeerDependencies: - supports-color - typescript - '@typescript-eslint/visitor-keys@8.0.0-alpha.30': + '@typescript-eslint/visitor-keys@8.0.0': dependencies: - '@typescript-eslint/types': 8.0.0-alpha.30 + '@typescript-eslint/types': 8.0.0 eslint-visitor-keys: 3.4.3 '@vercel/nft@0.26.5': @@ -1959,7 +1959,7 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.3.5 + debug: 4.3.6 transitivePeerDependencies: - supports-color @@ -2024,7 +2024,7 @@ snapshots: common-path-prefix: 3.0.0 concordance: 5.0.4 currently-unhandled: 0.4.1 - debug: 4.3.5 + debug: 4.3.6 emittery: 1.0.3 figures: 6.1.0 globby: 14.0.2 @@ -2039,7 +2039,7 @@ snapshots: package-config: 5.0.0 picomatch: 3.0.1 plur: 5.1.0 - pretty-ms: 9.0.0 + pretty-ms: 9.1.0 resolve-cwd: 3.0.0 stack-utils: 2.0.6 strip-ansi: 7.1.0 @@ -2181,7 +2181,7 @@ snapshots: js-string-escape: 1.0.1 lodash: 4.17.21 md5-hex: 3.0.1 - semver: 7.6.2 + semver: 7.6.3 well-known-symbols: 2.0.0 console-control-strings@1.1.0: {} @@ -2214,7 +2214,7 @@ snapshots: dependencies: time-zone: 1.0.0 - debug@4.3.5: + debug@4.3.6: dependencies: ms: 2.1.2 @@ -2311,20 +2311,20 @@ snapshots: eslint-visitor-keys@4.0.0: {} - eslint@9.7.0: + eslint@9.8.0: dependencies: - '@eslint-community/eslint-utils': 4.4.0(eslint@9.7.0) + '@eslint-community/eslint-utils': 4.4.0(eslint@9.8.0) '@eslint-community/regexpp': 4.11.0 - '@eslint/config-array': 0.17.0 + '@eslint/config-array': 0.17.1 '@eslint/eslintrc': 3.1.0 - '@eslint/js': 9.7.0 + '@eslint/js': 9.8.0 '@humanwhocodes/module-importer': 1.0.1 '@humanwhocodes/retry': 0.3.0 '@nodelib/fs.walk': 1.2.8 ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.5 + debug: 4.3.6 escape-string-regexp: 4.0.0 eslint-scope: 8.0.2 eslint-visitor-keys: 4.0.0 @@ -2458,14 +2458,12 @@ snapshots: get-east-asian-width@1.2.0: {} - get-stream@8.0.1: {} - get-stream@9.0.1: dependencies: '@sec-ant/readable-stream': 0.4.1 is-stream: 4.0.1 - get-tsconfig@4.7.5: + get-tsconfig@4.7.6: dependencies: resolve-pkg-maps: 1.0.0 @@ -2517,20 +2515,19 @@ snapshots: slash: 5.1.0 unicorn-magic: 0.1.0 - got@14.4.1: + got@14.4.2: dependencies: - '@sindresorhus/is': 6.3.1 + '@sindresorhus/is': 7.0.0 '@szmarczak/http-timer': 5.0.1 cacheable-lookup: 7.0.0 cacheable-request: 12.0.1 decompress-response: 6.0.0 form-data-encoder: 4.0.2 - get-stream: 8.0.1 http2-wrapper: 2.2.1 lowercase-keys: 3.0.0 p-cancelable: 4.0.1 responselike: 3.0.0 - type-fest: 4.21.0 + type-fest: 4.23.0 graceful-fs@4.2.11: {} @@ -2561,7 +2558,7 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.3.5 + debug: 4.3.6 transitivePeerDependencies: - supports-color @@ -2686,7 +2683,7 @@ snapshots: make-dir@4.0.0: dependencies: - semver: 7.6.2 + semver: 7.6.3 matcher@5.0.0: dependencies: @@ -2758,7 +2755,7 @@ snapshots: nock@13.5.4: dependencies: - debug: 4.3.5 + debug: 4.3.6 json-stringify-safe: 5.0.1 propagate: 2.0.1 transitivePeerDependencies: @@ -2867,7 +2864,7 @@ snapshots: prelude-ls@1.2.1: {} - pretty-ms@9.0.0: + pretty-ms@9.1.0: dependencies: parse-ms: 4.0.0 @@ -2929,7 +2926,7 @@ snapshots: semver@6.3.1: {} - semver@7.6.2: {} + semver@7.6.3: {} serialize-error@7.0.1: dependencies: @@ -3048,16 +3045,16 @@ snapshots: tr46@0.0.3: {} - ts-api-utils@1.3.0(typescript@5.4.5): + ts-api-utils@1.3.0(typescript@5.5.4): dependencies: - typescript: 5.4.5 + typescript: 5.5.4 tslog@4.9.3: {} - tsx@4.16.2: + tsx@4.16.3: dependencies: esbuild: 0.21.5 - get-tsconfig: 4.7.5 + get-tsconfig: 4.7.6 optionalDependencies: fsevents: 2.3.3 @@ -3067,22 +3064,24 @@ snapshots: type-detect@4.0.8: {} + type-detect@4.1.0: {} + type-fest@0.13.1: {} - type-fest@4.21.0: {} + type-fest@4.23.0: {} - typescript-eslint@8.0.0-alpha.30(eslint@9.7.0)(typescript@5.4.5): + typescript-eslint@8.0.0(eslint@9.8.0)(typescript@5.5.4): dependencies: - '@typescript-eslint/eslint-plugin': 8.0.0-alpha.30(@typescript-eslint/parser@8.0.0-alpha.30(eslint@9.7.0)(typescript@5.4.5))(eslint@9.7.0)(typescript@5.4.5) - '@typescript-eslint/parser': 8.0.0-alpha.30(eslint@9.7.0)(typescript@5.4.5) - '@typescript-eslint/utils': 8.0.0-alpha.30(eslint@9.7.0)(typescript@5.4.5) + '@typescript-eslint/eslint-plugin': 8.0.0(@typescript-eslint/parser@8.0.0(eslint@9.8.0)(typescript@5.5.4))(eslint@9.8.0)(typescript@5.5.4) + '@typescript-eslint/parser': 8.0.0(eslint@9.8.0)(typescript@5.5.4) + '@typescript-eslint/utils': 8.0.0(eslint@9.8.0)(typescript@5.5.4) optionalDependencies: - typescript: 5.4.5 + typescript: 5.5.4 transitivePeerDependencies: - eslint - supports-color - typescript@5.4.5: {} + typescript@5.5.4: {} undici-types@5.26.5: {} diff --git a/src/lib/utils.ts b/src/lib/utils.ts index ef81180..5bd322b 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -44,8 +44,8 @@ export const isValidUrl = (url: string): boolean => { * Flattens an array of arrays recursively. * */ -export function flattenDeep(array: any[]): any[] { - const result: any[] = []; +export function flattenDeep(array: (T | T[])[]): T[] { + const result: T[] = []; array.forEach(element => { if (Array.isArray(element)) { result.push(...flattenDeep(element)); @@ -56,8 +56,8 @@ export function flattenDeep(array: any[]): any[] { return result; } -export function pick(target: T, keys: (keyof T)[]): Pick { - const result: any = {}; +export function pick(target: T, keys: K[]): Pick { + const result = {} as Pick; keys.forEach(key => { if (target[key] !== undefined) { result[key] = target[key]; diff --git a/src/options.ts b/src/options.ts index dea0189..dca7ace 100644 --- a/src/options.ts +++ b/src/options.ts @@ -5,8 +5,8 @@ import { RequestConfig, RequestOptions } from "./types/crawler.js"; export const globalOnlyOptions = [ "maxConnections", - "priorityLevels", "rateLimit", + "priorityLevels", "skipDuplicates", "homogeneous", "userAgents", diff --git a/src/types/crawler.ts b/src/types/crawler.ts index bbafb47..ef9afca 100644 --- a/src/types/crawler.ts +++ b/src/types/crawler.ts @@ -72,8 +72,20 @@ export type RequestOptions = { * @description rateLimiter ID */ rateLimiterId?: number; + /** + * @default 2 + * @description The retry count of the request. + */ retries?: number; + /** + * @default 3000 + * @description The interval between retries in milliseconds. + */ retryInterval?: number; + /** + * @default 20000 + * @description The global timeout of the request in milliseconds. + */ timeout?: number; priority?: number; seenreq?: any; @@ -91,8 +103,8 @@ export type RequestOptions = { /** * @deprecated Please use "url" instead. */ - uri?: string | Function; - url?: string | Function; + uri?: string | ((urlFn: (url: string) => void) => void); + url?: string | ((urlFn: (url: string) => void) => void); /** * @deprecated Please use "searchParams" instead. @@ -124,8 +136,10 @@ export type RequestOptions = { cookieJar?: object; /** - * @description If true, the crawler will parse the response body as JSON. * @default false + * + * If true, the crawler will parse the response body as JSON. + * This will set 'jQuery' to false. */ isJson?: boolean; @@ -134,14 +148,14 @@ export type RequestOptions = { /** * @deprecated Please use "parseJson" instead. */ - jsonReviver?: Function; - parseJson?: Function; + jsonReviver?: (text: string) => unknown; + parseJson?: (text: string) => unknown; /** * @deprecated Please use "stringifyJson" instead. */ - jsonReplacer?: Function; - stringifyJson?: Function; + jsonReplacer?: (object: unknown) => string; + stringifyJson?: (object: unknown) => string; preRequest?: (options: RequestOptions, done?: (error?: Error | null) => void) => void; release?: () => void;