refactor(route): wsj news (DIYgod#12087)

* feat(route): wsj news * chore: docs update * chore: docs update * chore: use map to rewrite the function * chore: update utils * Update docs/en/traditional-media.md ---------
benzking · Mar 13, 2023 · 633aec3 · 633aec3
1 parent 6f04aff
commit 633aec3
Show file tree

Hide file tree

Showing 7 changed files with 177 additions and 146 deletions.
diff --git a/docs/en/traditional-media.md b/docs/en/traditional-media.md
@@ -678,7 +678,17 @@ Provides all of the articles by the specified New York Times author.
 
 ### News
 
-<RouteEn author="oppilate" example="/wsj/en-us/opinion" path="/wsj/:lang/:category?" :paramsDesc="['Language, `en-us`, `zh-cn`, `zh-tw` are supported', 'Category, only supported in `en-us`. Supports `opinion`, `world_news`, `us_bussiness`, `market_news`, `technology`, `lifestyle`.']">
+<RouteEn author="oppilate NavePnow" example="/wsj/en-us/opinion" path="/wsj/:lang/:category?" :paramsDesc="['Language, `en-us`, `zh-cn`, `zh-tw`', 'Category. See below']">
+
+en_us
+| World | U.S. | Politics | Economy | Business | Tech | Markets | Opinion | Books & Arts | Real Estate | Life & Work | Sytle | Sports |
+| ------ | ------- | -------- | -------- | ----- | --------- | --------- | --------- | --------- | --------- |--------- | --------- | --------- |
+| world | us | politics | economy | business | technology | markets | opinion | books-arts | realestate | life-work | style-entertainment | sports |
+
+zh-cn / zh-tw
+| 国际 | 中国 | 金融市场 | 经济 | 商业 | 科技 | 派 | 专栏与观点 |
+| ------ | ------- | -------- | -------- | ----- | --------- | --------- | --------- |
+| world | china | markets | economy | business | technology | life-arts | opinion |
 
 Provide full article RSS for WSJ topics.
 

diff --git a/docs/traditional-media.md b/docs/traditional-media.md
@@ -1182,7 +1182,19 @@ IT・科学 tech_science
 
 ### 新闻
 
-<Route author="oppilate" example="/wsj/en-us/opinion" path="/wsj/:lang/:category?" :paramsDesc="['语言，支持 `en-us`、`zh-cn`、`zh-tw`', '分类，仅 `en-us` 支持分类订阅。支持 `opinion`, `world_news`, `us_bussiness`, `market_news`, `technology`, `lifestyle`。']">
+<Route author="oppilate NavePnow" example="/wsj/en-us/opinion" path="/wsj/:lang/:category?" :paramsDesc="['语言，支持 `en-us`、`zh-cn`、`zh-tw`', '分类，见下表']">
+
+en_us
+
+| World | U.S. | Politics | Economy | Business | Tech       | Markets | Opinion | Books & Arts | Real Estate | Life & Work | Sytle               | Sports |
+| ----- | ---- | -------- | ------- | -------- | ---------- | ------- | ------- | ------------ | ----------- | ----------- | ------------------- | ------ |
+| world | us   | politics | economy | business | technology | markets | opinion | books-arts   | realestate  | life-work   | style-entertainment | sports |
+
+zh-cn / zh-tw
+
+| 国际    | 中国    | 金融市场    | 经济      | 商业       | 科技         | 派         | 专栏与观点   |
+| ----- | ----- | ------- | ------- | -------- | ---------- | --------- | ------- |
+| world | china | markets | economy | business | technology | life-arts | opinion |
 
 通过提取文章全文，以提供比官方源更佳的阅读体验。
 

diff --git a/lib/v2/wsj/index.js b/lib/v2/wsj/index.js
diff --git a/lib/v2/wsj/news.js b/lib/v2/wsj/news.js
@@ -0,0 +1,51 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { asyncPoolAll, parseArticle } = require('./utils');
+const hostMap = {
+    'en-us': 'https://www.wsj.com',
+    'zh-cn': 'https://cn.wsj.com/zh-hans',
+    'zh-tw': 'https://cn.wsj.com/zh-hant',
+};
+module.exports = async (ctx) => {
+    const lang = ctx.params.lang;
+    const category = ctx.params.category || '';
+    const host = hostMap[lang];
+    let subTitle = ` - ${lang.toUpperCase()}`;
+    let url = host;
+    if (category.length > 0) {
+        url = `${host}/news/${category}`;
+        subTitle = `${subTitle} - ${category}`;
+    }
+    const response = await got({
+        method: 'get',
+        url,
+    });
+
+    const $ = cheerio.load(response.data);
+    const contents = $('script:contains("window.__STATE__")').text();
+    const data = JSON.parse(contents.match(/\{.*\}/)[0]).data;
+    const filteredKeys = Object.entries(data)
+        .filter(([key, value]) => {
+            if (!key.startsWith('article')) {
+                return false;
+            }
+            const link = value.data.data.url;
+            return link.includes('wsj.com/articles/');
+        })
+        .map(([key]) => key);
+    const list = filteredKeys.map((key) => {
+        const item = {};
+        item.title = data[key].data.data.headline;
+        item.link = data[key].data.data.url;
+        item.test = key;
+        return item;
+    });
+    const items = await asyncPoolAll(1, list, (item) => parseArticle(item, ctx));
+
+    ctx.state.data = {
+        title: `WSJ${subTitle}`,
+        link: url,
+        description: `WSJ${subTitle}`,
+        item: items,
+    };
+};
diff --git a/lib/v2/wsj/radar.js b/lib/v2/wsj/radar.js
@@ -5,15 +5,15 @@ module.exports = {
             {
                 title: '新闻',
                 docs: 'https://docs.rsshub.app/traditional-media.html#hua-er-jie-ri-bao-the-wall-street-journal-wsj',
-                souce: '/',
+                source: '/',
                 target: '/wsj/zh-cn',
             },
         ],
         www: [
             {
                 title: '新闻',
                 docs: 'https://docs.rsshub.app/traditional-media.html#hua-er-jie-ri-bao-the-wall-street-journal-wsj',
-                souce: '/',
+                source: '/',
                 target: '/wsj/en-us',
             },
         ],

diff --git a/lib/v2/wsj/router.js b/lib/v2/wsj/router.js
@@ -1,3 +1,3 @@
 module.exports = (router) => {
-    router.get('/:lang/:category?', require('./index'));
+    router.get('/:lang/:category?', require('./news'));
 };
diff --git a/lib/v2/wsj/utils.js b/lib/v2/wsj/utils.js
@@ -0,0 +1,99 @@
+const asyncPool = require('tiny-async-pool');
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const UA = require('@/utils/rand-user-agent')({ browser: 'chrome', os: 'android', device: 'mobile' });
+
+// const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36';
+const parseArticle = (item, ctx) =>
+    ctx.cache.tryGet(item.link, async () => {
+        // Fetch the AMP version
+        const url = item.link.replace(/(?<=^https:\/\/\w+\.wsj\.com)/, '/amp');
+        const response = await got({
+            url,
+            method: 'get',
+            headers: {
+                'User-Agent': UA,
+            },
+        });
+        const html = response.data;
+        const $ = cheerio.load(html);
+        const content = $('.articleBody > section');
+
+        // Cover
+        const cover = $('.articleLead > div.is-lead-inset > div.header > .img-header > div.image-container > amp-img > img');
+
+        if (cover.length > 0) {
+            $(`<img src=${cover[0].attribs.content}>`).insertBefore(content[0].childNodes[0]);
+            $(cover).remove();
+        }
+
+        // Summary
+        const summary = $('head > meta[name="description"]').attr('content');
+
+        // Metadata (categories & updatedAt)
+        const updatedAt = $('meta[itemprop="dateModified"]').attr('content');
+        const publishedAt = $('meta[itemprop="datePublished"]').attr('content');
+        const author = $('.author > a[rel="author"]').text();
+
+        const categories = $('meta[name="keywords"]')
+            .attr('content')
+            .split(',')
+            .map((c) => c.trim());
+
+        // Images
+        content.find('amp-img').each((i, e) => {
+            const img = $(`<img width="${e.attribs.width}" height="${e.attribs.height}" src="${e.attribs.src}" alt="${e.attribs.alt}">`);
+
+            // Caption follows, no need to handle caption
+            $(img).insertBefore(e);
+            $(e).remove();
+        });
+
+        // iframes (youtube videos and interactive elements)
+        content.find('amp-iframe').each((i, e) => {
+            const iframe = $(`<iframe width="${e.attribs.width}" height="${e.attribs.height}" src="${e.attribs.src}">`);
+            $(iframe).insertBefore(e);
+            $(e).remove();
+        });
+
+        // Remove unwanted DOMs
+        const unwanted_element_selectors = ['amp-ad', '.wsj-ad'];
+        unwanted_element_selectors.forEach((selector) => {
+            content.find(selector).each((i, e) => {
+                $(e).remove();
+            });
+        });
+
+        // Paywall
+        content.find('.paywall').each((i, e) => {
+            // Caption follows, no need to handle caption
+            $(e.childNodes).insertBefore(e);
+            $(e).remove();
+        });
+
+        return {
+            title: item.title,
+            pubDate: parseDate(publishedAt),
+            updated: parseDate(updatedAt),
+            author,
+            link: item.link,
+            summary,
+            description: content.html(),
+            category: categories,
+            icon: 'https://s.wsj.net/media/wsj_launcher-icon-4x.png',
+            logo: 'https://vir.wsj.net/fp/assets/webpack4/img/wsj-logo-big-black.165e51cc.svg',
+        };
+    });
+
+const asyncPoolAll = async (...args) => {
+    const results = [];
+    for await (const result of asyncPool(...args)) {
+        results.push(result);
+    }
+    return results;
+};
+module.exports = {
+    asyncPoolAll,
+    parseArticle,
+};