Skip to content

Commit

Permalink
refactor: avoid promise overhead (DIYgod#8028)
Browse files Browse the repository at this point in the history
  • Loading branch information
SukkaW authored Aug 16, 2021
1 parent 8053b33 commit 6e3b58e
Show file tree
Hide file tree
Showing 543 changed files with 5,546 additions and 5,945 deletions.
4 changes: 3 additions & 1 deletion .eslintrc
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@
"no-await-in-loop": 2,
"require-atomic-updates": 0,
"no-prototype-builtins": 0,
"no-new-func": 2
"no-new-func": 2,
"require-await": 2,
"no-return-await": 2
}
}
8 changes: 4 additions & 4 deletions lib/middleware/access-control.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,24 +29,24 @@ module.exports = async (ctx, next) => {
await next();
} else {
if (!isControlled || allowLocalhost) {
return await grant();
return grant();
}

if (config.accessKey) {
if (config.accessKey === accessKey || accessCode === md5(requestPath + config.accessKey)) {
return await grant();
return grant();
}
}

if (config.whitelist) {
if (config.whitelist.find((white) => ip.includes(white) || requestPath.includes(white) || requestUA.includes(white))) {
return await grant();
return grant();
}
}

if (config.blacklist) {
if (!config.blacklist.find((black) => ip.includes(black) || requestPath.includes(black) || requestUA.includes(black))) {
return await grant();
return grant();
}
}

Expand Down
2 changes: 1 addition & 1 deletion lib/middleware/cache.js
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ module.exports = function (app) {
const key = 'koa-redis-cache:' + md5(ctx.request.path);

if (!available) {
return await next();
return next();
}

try {
Expand Down
2 changes: 1 addition & 1 deletion lib/middleware/parameter.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ module.exports = async (ctx, next) => {
// sort items
ctx.state.data.item = ctx.state.data.item.sort((a, b) => +new Date(b.pubDate || 0) - +new Date(a.pubDate || 0));

const handleItem = async (item) => {
const handleItem = (item) => {
item.title && (item.title = entities.decodeXML(item.title + ''));

// handle pubDate
Expand Down
2 changes: 1 addition & 1 deletion lib/router.js
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ const lazyloadRouteHandler = (routeHandlerPath) => (ctx) => {
// index
router.get('/', lazyloadRouteHandler('./routes/index'));

router.get('/robots.txt', async (ctx) => {
router.get('/robots.txt', (ctx) => {
if (config.disallowRobot) {
ctx.set('Content-Type', 'text/plain');
ctx.body = 'User-agent: *\nDisallow: /';
Expand Down
32 changes: 15 additions & 17 deletions lib/routes/199it/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,23 +8,21 @@ module.exports = async (ctx) => {
const $ = cheerio.load(res.data);
const list = $('.entry-content').get();

const out = await Promise.all(
list.slice(0, 9).map(async (item) => {
const $ = cheerio.load(item);
const time = $('.post-time time').attr('datetime');
const title = $('.entry-title a').attr('title');
const partial = $('.entry-title a').attr('href');
const address = partial;
const single = {
title,
pubDate: new Date(time).toUTCString(),
link: address,
guid: address,
};
ctx.cache.set(address, JSON.stringify(single));
return Promise.resolve(single);
})
);
const out = list.slice(0, 9).map((item) => {
const $ = cheerio.load(item);
const time = $('.post-time time').attr('datetime');
const title = $('.entry-title a').attr('title');
const partial = $('.entry-title a').attr('href');
const address = partial;
const single = {
title,
pubDate: new Date(time).toUTCString(),
link: address,
guid: address,
};
ctx.cache.set(address, JSON.stringify(single));
return single;
});
ctx.state.data = {
title: '199it',
link: url,
Expand Down
21 changes: 10 additions & 11 deletions lib/routes/199it/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,18 @@ module.exports = async (ctx, keyword, currentUrl) => {
.get();

const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);

item.description = content('div.entry-content').html();
item.description = content('div.entry-content').html();

return item;
})
return item;
})
)
);

Expand Down
31 changes: 15 additions & 16 deletions lib/routes/1point3acres/blog.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,21 @@ module.exports = async (ctx) => {
.get();

const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);

content('h2, .meta_tags, .pagination, .subinfopost, #comments').remove();

item.description = content('.post').html();
item.pubDate = new Date(detailResponse.data.match(/"datePublished":"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00)","dateModified":/)[1]).toUTCString();

return item;
})
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);

content('h2, .meta_tags, .pagination, .subinfopost, #comments').remove();

item.description = content('.post').html();
item.pubDate = new Date(detailResponse.data.match(/"datePublished":"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00)","dateModified":/)[1]).toUTCString();

return item;
})
)
);

Expand Down
17 changes: 8 additions & 9 deletions lib/routes/36kr/motif.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,14 @@ module.exports = async (ctx) => {
title: `36氪专题 - ${motifInfo.categoryTitle}`,
link: motifUrl,
item: await Promise.all(
articleList.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const contentResponse = await got({ method: 'get', url: item.link });
const result = contentResponse.data.match(/"articleDetailData":(.*?),"articleRecommendData":/) || contentResponse.data.match(/"videoDetail":(.*?),"authorVideos":/);
const contentData = JSON.parse(result[1]);
item.description = contentData.data.widgetContent;
return item;
})
articleList.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const contentResponse = await got({ method: 'get', url: item.link });
const result = contentResponse.data.match(/"articleDetailData":(.*?),"articleRecommendData":/) || contentResponse.data.match(/"videoDetail":(.*?),"authorVideos":/);
const contentData = JSON.parse(result[1]);
item.description = contentData.data.widgetContent;
return item;
})
)
),
description: `${motifInfo.categoryTitle}`,
Expand Down
15 changes: 7 additions & 8 deletions lib/routes/36kr/news.js
Original file line number Diff line number Diff line change
Expand Up @@ -81,14 +81,13 @@ module.exports = async (ctx) => {
title: `36氪资讯 - ${cfg.title}`,
link: newsUrl,
item: await Promise.all(
informationList.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const contentResponse = await got({ method: 'get', url: item.link });
const content = cheerio.load(contentResponse.data);
item.description = content('div.common-width.content.articleDetailContent.kr-rich-text-wrapper').html();
return item;
})
informationList.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const contentResponse = await got({ method: 'get', url: item.link });
const content = cheerio.load(contentResponse.data);
item.description = content('div.common-width.content.articleDetailContent.kr-rich-text-wrapper').html();
return item;
})
)
),
};
Expand Down
2 changes: 1 addition & 1 deletion lib/routes/36kr/search/article.js
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ module.exports = async (ctx) => {
pubDate: new Date(item.publishTime).toUTCString(),
};

const other = await ctx.cache.tryGet(link, async () => await load(link));
const other = await ctx.cache.tryGet(link, () => load(link));
return Promise.resolve(Object.assign({}, single, other));
})
);
Expand Down
17 changes: 8 additions & 9 deletions lib/routes/36kr/user.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,14 @@ module.exports = async (ctx) => {
title: `36氪用户 - ${authorInfo.userNick}`,
link: userUrl,
item: await Promise.all(
articleList.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const contentResponse = await got({ method: 'get', url: item.link });
const result = contentResponse.data.match(/"articleDetailData":(.*?),"articleRecommendData":/) || contentResponse.data.match(/"videoDetail":(.*?),"authorVideos":/);
const contentData = JSON.parse(result[1]);
item.description = contentData.data.widgetContent;
return item;
})
articleList.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const contentResponse = await got({ method: 'get', url: item.link });
const result = contentResponse.data.match(/"articleDetailData":(.*?),"articleRecommendData":/) || contentResponse.data.match(/"videoDetail":(.*?),"authorVideos":/);
const contentData = JSON.parse(result[1]);
item.description = contentData.data.widgetContent;
return item;
})
)
),
description: `${authorInfo.label} | ${authorInfo.summary}`,
Expand Down
33 changes: 16 additions & 17 deletions lib/routes/3k8/latest.js
Original file line number Diff line number Diff line change
Expand Up @@ -23,23 +23,22 @@ module.exports = async (ctx) => {
.get();

const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
responseType: 'buffer',
});

const content = cheerio.load(iconv.decode(detailResponse.data, 'gb2312'));

item.title = content('h1.title').text();
item.pubDate = new Date(content('div.news-mes span').eq(0).text().replace('发布时间:', '').trim() + ' GMT+8').toUTCString();
item.description = content('div.news-content').html();

return item;
})
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
responseType: 'buffer',
});

const content = cheerio.load(iconv.decode(detailResponse.data, 'gb2312'));

item.title = content('h1.title').text();
item.pubDate = new Date(content('div.news-mes span').eq(0).text().replace('发布时间:', '').trim() + ' GMT+8').toUTCString();
item.description = content('div.news-content').html();

return item;
})
)
);

Expand Down
4 changes: 2 additions & 2 deletions lib/routes/3ycy/home.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@ module.exports = async (ctx) => {
}

const items = await Promise.all(
count.map(async (i) => {
count.map((i) => {
const item = $(list[i]);
const title = item.find('h3 a').first();
const link = title.attr('href');

return await ctx.cache.tryGet('3ycy' + link, async () => {
return ctx.cache.tryGet('3ycy' + link, async () => {
const date = item.find('div.postSubtitle').text().trim();
const match = /(\d+\/\d+\/\d{4}\s+\d+:\d+:\d+\s*(?:AM|PM))/i.exec(date);
const pubDate = match && new Date(match[1] + ' GMT+8').toUTCString();
Expand Down
35 changes: 17 additions & 18 deletions lib/routes/6park/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,24 +29,23 @@ module.exports = async (ctx) => {
.get();

const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);

item.title = content('title').text().replace(' -6park.com', '');
item.author = detailResponse.data.match(/: .*>(.*)<.*\[/)[1];
item.pubDate = new Date(detailResponse.data.match(/ (.*) /)[1]).toUTCString();
item.description = content('pre')
.html()
.replace(/<font color="#E6E6DD">6park.com<\/font>/g, '');

return item;
})
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);

item.title = content('title').text().replace(' -6park.com', '');
item.author = detailResponse.data.match(/: .*>(.*)<.*\[/)[1];
item.pubDate = new Date(detailResponse.data.match(/ (.*) /)[1]).toUTCString();
item.description = content('pre')
.html()
.replace(/<font color="#E6E6DD">6park.com<\/font>/g, '');

return item;
})
)
);

Expand Down
2 changes: 1 addition & 1 deletion lib/routes/8kcos/cat.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ module.exports = async (ctx) => {
Cheerio.load(resp.body)('#wrap > div > div > div > ul > li.item')
.map((_, e) => {
const { href } = Cheerio.load(e)('h2 > a')[0].attribs;
return ctx.cache.tryGet(href, async () => loadArticle(href));
return ctx.cache.tryGet(href, () => loadArticle(href));
})
.toArray()
),
Expand Down
2 changes: 1 addition & 1 deletion lib/routes/8kcos/latest.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ module.exports = async (ctx) => {
Cheerio.load(response.body)('#wrap > div > div > div > ul > li.item')
.map((_, e) => {
const { href } = Cheerio.load(e)('div > h2 > a')[0].attribs;
return ctx.cache.tryGet(href, async () => loadArticle(href));
return ctx.cache.tryGet(href, () => loadArticle(href));
})
.toArray()
)
Expand Down
21 changes: 10 additions & 11 deletions lib/routes/91ddcc/stage.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,18 @@ module.exports = async (ctx) => {
.get();

const items = await Promise.all(
list.map(
async (item) =>
await ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);

item.description = content('div.article-con').html();
item.description = content('div.article-con').html();

return item;
})
return item;
})
)
);

Expand Down
Loading

0 comments on commit 6e3b58e

Please sign in to comment.