axelburks · pull · Dec 3, 2023 · Dec 2, 2023 · Dec 2, 2023 · Dec 2, 2023
diff --git a/lib/config.js b/lib/config.js
@@ -292,8 +292,8 @@ const calculateValue = () => {
             cookie: envs.TOPHUB_COOKIE,
         },
         twitter: {
-            oauthTokens: envs.TWITTER_OAUTH_TOKEN && envs.TWITTER_OAUTH_TOKEN.split(','),
-            oauthTokenSecrets: envs.TWITTER_OAUTH_TOKEN_SECRET && envs.TWITTER_OAUTH_TOKEN_SECRET.split(','),
+            oauthTokens: envs.TWITTER_OAUTH_TOKEN?.split(','),
+            oauthTokenSecrets: envs.TWITTER_OAUTH_TOKEN_SECRET?.split(','),
         },
         weibo: {
             app_key: envs.WEIBO_APP_KEY,

diff --git a/lib/v2/brave/latest.js b/lib/v2/brave/latest.js
@@ -20,13 +20,15 @@ module.exports = async (ctx) => {
 
             const title = item.text();
             const device = item.parent().find('h2').text();
+            const matchVersion = title.match(/(v[\d.]+)/);
+            const matchDate = title.match(/\((.*?)\)/);
 
             return {
                 title: `[${device}] ${title}`,
                 link: currentUrl,
-                guid: `${currentUrl}#${device}-${title.match(/(v[\d.]+)/)[1]}`,
+                guid: `${currentUrl}#${device}-${matchVersion?.[1] ?? title}`,
                 description: item.next().html(),
-                pubDate: parseDate(title.match(/\((.*?)\)/)[1].replace(/(st|nd|rd|th)?,/, ''), ['MMMM D YYYY', 'MMM D YYYY']),
+                pubDate: parseDate(matchDate?.[1].replace(/(st|nd|rd|th)?,/, ''), ['MMMM D YYYY', 'MMM D YYYY']),
             };
         });
 

diff --git a/lib/v2/cna/maintainer.js b/lib/v2/cna/maintainer.js
@@ -1,3 +1,4 @@
 module.exports = {
     '/:id?': ['nczitzk'],
+    '/web/:id?': ['dzx-dzx'],
 };
diff --git a/lib/v2/cna/radar.js b/lib/v2/cna/radar.js
@@ -8,6 +8,12 @@ module.exports = {
                 source: ['/list/:id', '/topic/newstopic/:id'],
                 target: (params) => `/cna/${params.id.replace('.aspx', '')}`,
             },
+            {
+                title: '分类(网页爬虫方法)',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#zhong-yang-tong-xun-she',
+                source: ['/list/:id'],
+                target: (params) => `/cna/web/${params.id.replace('.aspx', '')}`,
+            },
         ],
     },
 };
diff --git a/lib/v2/cna/router.js b/lib/v2/cna/router.js
@@ -1,3 +1,4 @@
 module.exports = (router) => {
     router.get('/:id?', require('./'));
+    router.get('/web/:id?', require('./web/'));
 };
diff --git a/lib/v2/cna/web/index.js b/lib/v2/cna/web/index.js
@@ -0,0 +1,62 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id || 'aall';
+
+    let rootUrl;
+
+    if (/^\d+$/.test(id)) {
+        rootUrl = `https://www.cna.com.tw/topic/newstopic/${id}.aspx`;
+    } else {
+        rootUrl = `https://www.cna.com.tw/list/${id}.aspx`;
+    }
+    const response = await got({
+        method: 'get',
+        url: rootUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+    const list = $('*:is(.pcBox .caItem, .mainList li a div) h2')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 10)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                link: item.parents('a').attr('href'),
+                pubDate: timezone(parseDate(item.next().text()), +8),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+                const content = cheerio.load(detailResponse.data);
+                const topImage = content('.fullPic').html();
+
+                item.description = (topImage === null ? '' : topImage) + content('.paragraph').eq(0).html();
+                item.category = [
+                    ...content("meta[property='article:tag']")
+                        .get()
+                        .map((e) => e.attribs.content),
+                    content('.active > a').text(),
+                ];
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: rootUrl,
+        item: items,
+    };
+};
diff --git a/lib/v2/twitter/web-api/twitter-api.js b/lib/v2/twitter/web-api/twitter-api.js
@@ -9,7 +9,7 @@ const queryString = require('query-string');
 let tokenIndex = 0;
 
 const twitterGot = async (url, params) => {
-    if (!config.twitter.oauthTokens.length || !config.twitter.oauthTokenSecrets.length || config.twitter.oauthTokens.length !== config.twitter.oauthTokenSecrets.length) {
+    if (!config.twitter.oauthTokens?.length || !config.twitter.oauthTokenSecrets?.length || config.twitter.oauthTokens.length !== config.twitter.oauthTokenSecrets.length) {
         throw Error('Invalid twitter oauth tokens');
     }
 

diff --git a/website/docs/routes/bbs.mdx b/website/docs/routes/bbs.mdx
diff --git a/website/docs/routes/blog.mdx b/website/docs/routes/blog.mdx
@@ -76,11 +76,11 @@ Freebuf 的文章页面带有反爬虫机制，所以目前无法获取文章的
 
 ### Articles {#google-sites-articles}
 
-<Route author="hoilc" example="/google/sites/outlierseconomics" path="/google/sites/:id" paramsDesc={['Site ID, can be found in URL']} />
+<Route author="hoilc" example="/google/sites/outlierseconomics" path="/google/sites/:id" paramsDesc={['Site ID, can be found in URL']} notOperational="1" />
 
 ### Recent Changes {#google-sites-recent-changes}
 
-<Route author="nczitzk" example="/google/sites/recentChanges/outlierseconomics" path="/google/sites/recentChanges/:id" paramsDesc={['Site ID, can be found in URL']}/>
+<Route author="nczitzk" example="/google/sites/recentChanges/outlierseconomics" path="/google/sites/recentChanges/:id" paramsDesc={['Site ID, can be found in URL']} notOperational="1"/>
 
 ## Gwern Branwen {#gwern-branwen}
 
@@ -132,11 +132,11 @@ username 为博主用户名，而非`xxx.hashnode.dev`中`xxx`所代表的 blog
 
 ### DIYgod 的动森日记 {#hi-diygod-diygod-de-dong-sen-ri-ji}
 
-<Route author="DIYgod" example="/blogs/diygod/animal-crossing" path="/blogs/diygod/animal-crossing"/>
+<Route author="DIYgod" example="/blogs/diygod/animal-crossing" path="/blogs/diygod/animal-crossing" notOperational="1"/>
 
 ### DIYgod 的可爱的手办们 {#hi-diygod-diygod-de-ke-ai-de-shou-ban-men}
 
-<Route author="DIYgod" example="/blogs/diygod/gk" path="/blogs/diygod/gk"/>
+<Route author="DIYgod" example="/blogs/diygod/gk" path="/blogs/diygod/gk" notOperational="1"/>
 
 ## Ian Spriggss {#ian-spriggss}
 
@@ -154,7 +154,7 @@ username 为博主用户名，而非`xxx.hashnode.dev`中`xxx`所代表的 blog
 
 ### JustRun {#justrun-justrun}
 
-<Route author="nczitzk" example="/justrun" path="/justrun"/>
+<Route author="nczitzk" example="/justrun" path="/justrun" notOperational="1"/>
 
 ## Kun Cheng {#kun-cheng}
 
@@ -178,7 +178,7 @@ username 为博主用户名，而非`xxx.hashnode.dev`中`xxx`所代表的 blog
 
 ### Ash Maurya's blog {#love-the-problem-ash-maurya-s-blog}
 
-<Route author="james-tindal" example="/ash-maurya" path="/ash-maurya"/>
+<Route author="james-tindal" example="/ash-maurya" path="/ash-maurya" notOperational="1"/>
 
 ## MacMenuBar {#macmenubar}
 
@@ -244,7 +244,7 @@ Personalized recommendations require the cookie value after logging in, so only
 
 ### Blog {#miris-whispers-blog}
 
-<Route author="chazeon" example="/miris/blog" path="/miris/blog" />
+<Route author="chazeon" example="/miris/blog" path="/miris/blog" notOperational="1" />
 
 ## Paul Graham {#paul-graham}
 
@@ -262,13 +262,13 @@ Personalized recommendations require the cookie value after logging in, so only
 
 ### Blog {#polkadot-blog}
 
-<Route author="iceqing" example="/polkadot/blog" path="/polkadot/blog"/>
+<Route author="iceqing" example="/polkadot/blog" path="/polkadot/blog" notOperational="1"/>
 
 ## PolkaWorld {#polkaworld}
 
 ### Newest Articles {#polkaworld-newest-articles}
 
-<Route author="iceqing" example="/polkaworld/newest" path="/polkaworld/newest"/>
+<Route author="iceqing" example="/polkaworld/newest" path="/polkaworld/newest" notOperational="1"/>
 
 :::tip
 
@@ -322,11 +322,11 @@ Limit the number of entries to be retrieved by adding `?limit=x` to the end of t
 
 ### 最新文章 {#whoscall-zui-xin-wen-zhang}
 
-<Route author="nczitzk" example="/whoscall" path="/whoscall"/>
+<Route author="nczitzk" example="/whoscall" path="/whoscall" notOperational="1"/>
 
 ### 分類 {#whoscall-fen-lei}
 
-<Route author="nczitzk" example="/whoscall/categories/5-Whoscall 百科" path="/whoscall/categories/:category?" paramsDesc={['分类，见下表，可在对应分類页 URL 中找到，默认为最新文章']}>
+<Route author="nczitzk" example="/whoscall/categories/5-Whoscall 百科" path="/whoscall/categories/:category?" paramsDesc={['分类，见下表，可在对应分類页 URL 中找到，默认为最新文章']} notOperational="1">
 
 | News   | Whoscall 百科   | 防詐小學堂     | Whoscall 日常   |
 | ------ | --------------- | -------------- | --------------- |
@@ -336,7 +336,7 @@ Limit the number of entries to be retrieved by adding `?limit=x` to the end of t
 
 ### 標籤 {#whoscall-biao-qian}
 
-<Route author="nczitzk" example="/whoscall/tags/whoscall小百科" path="/whoscall/tags/:tag?" paramsDesc={['標籤，见下表，可在对应標籤页 URL 中找到，默认为最新文章']}>
+<Route author="nczitzk" example="/whoscall/tags" path="/whoscall/tags/:tag?" paramsDesc={['標籤，见下表，可在对应標籤页 URL 中找到，默认为最新文章']} notOperational="1">
 
 | 防疫也防詐 | 防詐專家 | 來電辨識 | whoscall 日常 |
 | ---------- | -------- | -------- | ------------- |
@@ -359,7 +359,7 @@ Limit the number of entries to be retrieved by adding `?limit=x` to the end of t
 
 ### 首页 {#a-li-yun-xi-tong-zu-ji-shu-bo-ke-shou-ye}
 
-<Route author="attenuation" example="/aliyun-kernel/index" path="/aliyun-kernel/index"/>
+<Route author="attenuation" example="/aliyun-kernel/index" path="/aliyun-kernel/index" notOperational="1"/>
 
 ## 博客园 {#bo-ke-yuan}
 
@@ -425,19 +425,19 @@ Limit the number of entries to be retrieved by adding `?limit=x` to the end of t
 
 ### 博客 {#hua-kang-zi-xing-gu-shi-bo-ke}
 
-<Route author="tpnonthealps" example="/fontstory" path="/fontstory" />
+<Route author="tpnonthealps" example="/fontstory" path="/fontstory" notOperational="1" />
 
 ## 黄健宏博客 {#huang-jian-hong-bo-ke}
 
 ### 文章 {#huang-jian-hong-bo-ke-wen-zhang}
 
-<Route author="stormbuf" example="/huangz" path="/huangz" radar="1"/>
+<Route author="stormbuf" example="/huangz" path="/huangz" radar="1" notOperational="1"/>
 
 ## 建宁闲谈 {#jian-ning-xian-tan}
 
 ### 文章 {#jian-ning-xian-tan-wen-zhang}
 
-<Route author="changlan" example="/blogs/jianning" path="/blogs/jianning" radar="1"/>
+<Route author="changlan" example="/blogs/jianning" path="/blogs/jianning" radar="1" notOperational="1"/>
 
 ## 劍心．回憶 {#jian-xin-hui-yi}
 
@@ -491,7 +491,7 @@ Limit the number of entries to be retrieved by adding `?limit=x` to the end of t
 
 ### 最新文章 {#jiao-liu-dao-zi-yuan-wang-zui-xin-wen-zhang}
 
-<Route author="TonyRL" example="/jiaoliudao" path="/jiaoliudao" radar="1" />
+<Route author="TonyRL" example="/jiaoliudao" path="/jiaoliudao" radar="1" notOperational="1" />
 
 ## 敬维博客 {#jing-wei-bo-ke}
 
@@ -503,7 +503,7 @@ Limit the number of entries to be retrieved by adding `?limit=x` to the end of t
 
 ### 推送 {#mei-ri-an-quan-tui-song}
 
-<Route author="LogicJake" example="/security/pulses" path="/security/pulses"/>
+<Route author="LogicJake" example="/security/pulses" path="/security/pulses" anticrawler="1"/>
 
 ## 美团技术团队 {#mei-tuan-ji-shu-tuan-dui}
 
@@ -545,7 +545,7 @@ Limit the number of entries to be retrieved by adding `?limit=x` to the end of t
 
 ### 首页 {#yu-cong-bo-ke-shou-ye}
 
-<Route author="XinRoom" example="/ddosi" path="/ddosi"/>
+<Route author="XinRoom" example="/ddosi" path="/ddosi" anticrawler="1"/>
 
 ### 分类 {#yu-cong-bo-ke-fen-lei}
 

diff --git a/website/docs/routes/design.mdx b/website/docs/routes/design.mdx
@@ -50,7 +50,7 @@ Behance user's profile URL, like [https://www.behance.net/mishapetrick](https://
 
 ### Works & News {#digic-picture-works-news}
 
-<Route author="MisteryMonster" example="/digic-pictures/works/real-time-engine" path="/digic-pictures/:menu/:tag?" paramsDesc={['`news`, `works`', 'Under WORK types: `/game-cinematics`, `/feature`, `/making-of`, `/commercials-vfx`, `/real-time-engine`']} />
+<Route author="MisteryMonster" example="/digic-pictures/works/real-time-engine" path="/digic-pictures/:menu/:tag?" paramsDesc={['`news`, `works`', 'Under WORK types: `/game-cinematics`, `/feature`, `/making-of`, `/commercials-vfx`, `/real-time-engine`']} notOperational="1" />
 
 ## Dribbble {#dribbble}
 
@@ -185,7 +185,7 @@ Default is under 'https://www.methodstudios.com/en/features'.
 
 ### 文章列表 {#shopping-design-wen-zhang-lie-biao}
 
-<Route author="miles170" example="/shoppingdesign/posts" path="/shoppingdesign/posts"/>
+<Route author="miles170" example="/shoppingdesign/posts" path="/shoppingdesign/posts" notOperational="1"/>
 
 ## UI 中国 {#ui-zhong-guo}
 
@@ -213,27 +213,27 @@ Default is under 'https://www.methodstudios.com/en/features'.
 
 ### 设计专题 {#you-she-wang-she-ji-zhuan-ti}
 
-<Route author="nczitzk" example="/uisdc/zt/design-history" path="/uisdc/zt/:title?" paramsDesc={['专题名称，可在标签页的 URL 中找到，如 `design-history`；也可填入 `hot` 展示最热门专题，默认展示最新鲜专题']}>
+<Route author="nczitzk" example="/uisdc/zt/design-history" path="/uisdc/zt/:title?" paramsDesc={['专题名称，可在标签页的 URL 中找到，如 `design-history`；也可填入 `hot` 展示最热门专题，默认展示最新鲜专题']} notOperational="1">
 
 更多设计专题请参见 [优设专题](https://www.uisdc.com/zt)
 
 </Route>
 
 ### 细节猎人 {#you-she-wang-xi-jie-lie-ren}
 
-<Route author="nczitzk" example="/uisdc/topic/all" path="/uisdc/topic/:title?/:sort?" paramsDesc={['标签名称，可在标签页的 URL 中找到', '排序方式，`hot` 指最热门，默认为最新鲜']}>
+<Route author="nczitzk" example="/uisdc/topic/all" path="/uisdc/topic/:title?/:sort?" paramsDesc={['标签名称，可在标签页的 URL 中找到', '排序方式，`hot` 指最热门，默认为最新鲜']} notOperational="1">
 
 更多细节标签请参见 [全部标签](https://www.uisdc.com/alltopics)
 
 </Route>
 
 ### 设计话题 {#you-she-wang-she-ji-hua-ti}
 
-<Route author="nczitzk" example="/uisdc/talk" path="/uisdc/talk/:sort?" paramsDesc={['排序方式，`hot` 指最热门，默认为最新鲜']}/>
+<Route author="nczitzk" example="/uisdc/talk" path="/uisdc/talk/:sort?" paramsDesc={['排序方式，`hot` 指最热门，默认为最新鲜']} notOperational="1"/>
 
 ### 行业新闻 {#you-she-wang-hang-ye-xin-wen}
 
-<Route author="nczitzk" example="/uisdc/hangye" path="/uisdc/hangye/:caty?" paramsDesc={['分类，见下表，默认为全部新闻']}>
+<Route author="nczitzk" example="/uisdc/hangye" path="/uisdc/hangye/:caty?" paramsDesc={['分类，见下表，默认为全部新闻']} anticrawler="1">
 
 | 全部新闻 | 活动赛事        | 品牌资讯   | 新品推荐     |
 | -------- | --------------- | ---------- | ------------ |
@@ -243,13 +243,13 @@ Default is under 'https://www.methodstudios.com/en/features'.
 
 ### 优设读报 {#you-she-wang-you-she-du-bao}
 
-<Route author="nczitzk" example="/uisdc/news" path="/uisdc/news"/>
+<Route author="nczitzk" example="/uisdc/news" path="/uisdc/news" notOperational="1"/>
 
 ## 站酷 {#zhan-ku}
 
 ### 发现（+ 推荐预设） {#zhan-ku-fa-xian-tui-jian-yu-she}
 
-<Route author="junbaor nczitzk" example="/zcool/discover/all" path="/zcool/discover/:type?" paramsDesc={['预设参数，见下表']} radar="1">
+<Route author="junbaor nczitzk" example="/zcool/discover/all" path="/zcool/discover/:type?" paramsDesc={['预设参数，见下表']} radar="1" notOperational="1">
 
 推荐类型
 
@@ -261,7 +261,7 @@ Default is under 'https://www.methodstudios.com/en/features'.
 
 ### 发现（+ 查询参数） {#zhan-ku-fa-xian-cha-xun-can-shu}
 
-<Route author="nczitzk" example="/zcool/discover/cate=0&subCate=0" path="/zcool/discover/:query?" paramsDesc={['查询参数']} radar="1">
+<Route author="nczitzk" example="/zcool/discover/cate=0&subCate=0" path="/zcool/discover/:query?" paramsDesc={['查询参数']} radar="1" notOperational="1">
 
 在 [站酷发现页](https://www.zcool.com.cn/discover) 中选择好所有可选的查询参数后会跳转到对应搜索结果页面。此时地址栏 `https://www.zcool.com.cn/discover/` 后的字段即为查询参数。
 
@@ -275,7 +275,7 @@ Default is under 'https://www.methodstudios.com/en/features'.
 
 ### 发现 {#zhan-ku-fa-xian}
 
-<Route author="nczitzk" example="/zcool/discover" path="/zcool/discover/:query?/:subCate?/:hasVideo?/:city?/:college?/:recommendLevel?/:sort?" paramsDesc={['查询参数或分类，若填写分类见下表，默认为空 或 `0` 即精选', '子分类，见下表，默认为 `0` 即该父分类下全部', '是否含视频，默认为 `0` 即全部，亦可选 `1` 即含视频', '地区代码，填入发现页中 `选择城市` 中的各级地名，如 `亚洲`、`中国`、`北京`、`纽约`、`巴黎`等', '学校，默认为 `0` 即全部', '推荐等级，见下表，默认为 `2` 即编辑精选', '排序方式，可选 `0` 即最新发布 或 `9` 即默认排序，默认为 `9`']} radar="1">
+<Route author="nczitzk" example="/zcool/discover" path="/zcool/discover/:query?/:subCate?/:hasVideo?/:city?/:college?/:recommendLevel?/:sort?" paramsDesc={['查询参数或分类，若填写分类见下表，默认为空 或 `0` 即精选', '子分类，见下表，默认为 `0` 即该父分类下全部', '是否含视频，默认为 `0` 即全部，亦可选 `1` 即含视频', '地区代码，填入发现页中 `选择城市` 中的各级地名，如 `亚洲`、`中国`、`北京`、`纽约`、`巴黎`等', '学校，默认为 `0` 即全部', '推荐等级，见下表，默认为 `2` 即编辑精选', '排序方式，可选 `0` 即最新发布 或 `9` 即默认排序，默认为 `9`']} radar="1" notOperational="1">
 
 查看 **精选** 分类下的全部内容，其他参数选择默认，可直接使用路由 [`/zcool/discover/0`](https://rsshub.app/zcool/discover/0)
 
@@ -399,7 +399,7 @@ UI 17
 
 ### 作品总榜单 {#zhan-ku-zuo-pin-zong-bang-dan}
 
-<Route author="yuuow" example="/zcool/top/design" path="/zcool/top/:type" paramsDesc={['推荐类型,详见下面的表格']} radar="1">
+<Route author="yuuow" example="/zcool/top/design" path="/zcool/top/:type" paramsDesc={['推荐类型,详见下面的表格']} radar="1" notOperational="1">
 
 榜单类型
 
@@ -411,7 +411,7 @@ UI 17
 
 ### 用户作品 {#zhan-ku-yong-hu-zuo-pin}
 
-<Route author="junbaor" example="/zcool/user/baiyong" path="/zcool/user/:uid" paramsDesc={['个性域名前缀或者用户ID']} radar="1">
+<Route author="junbaor" example="/zcool/user/baiyong" path="/zcool/user/:uid" paramsDesc={['个性域名前缀或者用户ID']} radar="1" notOperational="1">
 
 例如: