diff --git a/.circleci/config.yml b/.circleci/config.yml index 26f434b..b59ba08 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -61,9 +61,22 @@ jobs: name: Add publish token command: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc - run: npm publish - periodic: + pages: docker: - image: circleci/node:latest + steps: + - checkout + - run: npm i + - run: npm run build + - run: + name: Copy circleci config + command: cp -r .circleci docs/ + - run: + name: Post to gh-pages + command: curl ci-cd.net/v1/gh/pages | bash -s docs/. + periodic: + docker: + - image: circleci/node:latest-browsers steps: - checkout - run: npm i @@ -83,11 +96,31 @@ workflows: version: 2.1 test: jobs: - - test_node_10 - - test_node_12 - - test_node_latest - - test_browsers - - lint_sort_and_types + - test_node_10: + filters: + branches: + ignore: + - gh-pages + - test_node_12: + filters: + branches: + ignore: + - gh-pages + - test_node_latest: + filters: + branches: + ignore: + - gh-pages + - test_browsers: + filters: + branches: + ignore: + - gh-pages + - lint_sort_and_types: + filters: + branches: + ignore: + - gh-pages - publish: requires: - test_node_10 @@ -99,6 +132,9 @@ workflows: branches: only: - master + - pages: + requires: + - publish scheduled: jobs: - periodic diff --git a/.gitignore b/.gitignore index 0c6ebda..15cc13f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ +.cache node_modules tests/fixtures/user-agents.net.bot-crawler.txt tests/fixtures/crawler-user-agents-monperrus.json tests/.browser-tests.js +docs diff --git a/.npmignore b/.npmignore index ee08500..1397af2 100644 --- a/.npmignore +++ b/.npmignore @@ -1,5 +1,6 @@ .* tests/* scripts/* +src/* tsconfig.json CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 0499389..14a233b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +# next + +## Crawlers list update +Add patterns for: +- Google WebLight Proxy +- HighWinds Content Delivery System +- [Hydra by addthis](https://github.com/addthis/hydra) +- [RebelMouse](https://www.rebelmouse.com/rebelmouse-public-api) +- Scanners: Jorgee Vulnerability, ClamAV Website, Burp Collaborator +- Monitoring services: Xymon, AlertSite, Hobbit, updown.io, Monit, Dotcom + +## Testing +Add some legit browser user-agent strings +Fix periodic tests environment +Add [a tester page](https://gorangajic.github.io/isbot) to check user agents easily + # 3.0.0: Maintainability and performance through automation The API and code **has not changed** diff --git a/README.md b/README.md index f02dd6b..774343d 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ Detect bots/crawlers/spiders using the user agent string. [![](https://img.shields.io/circleci/build/github/gorangajic/isbot?style=flat-square)](https://circleci.com/gh/gorangajic/isbot) [![](https://img.shields.io/npm/dt/isbot?style=flat-square)](https://www.npmjs.com/package/isbot) [![](https://img.shields.io/github/last-commit/gorangajic/isbot?style=flat-square)](https://github.com/gorangajic/isbot/graphs/commit-activity) [![](https://img.shields.io/librariesio/sourcerank/npm/isbot?style=flat-square)](https://libraries.io/npm/isbot) -### install +## Check it out +[Visit the online checker](https://gorangajic.github.io/isbot) ## Usage @@ -60,6 +61,25 @@ Return the respective match for bot user agent rule isbot.find('Googlebot/2.1 (+http://www.google.com/bot.html)') // 'bot' ``` +## Definitions +- **Bot.** Autonomous program imitating or replacing some aspect of a human behaviour, performing repetitive tasks much faster than human users could. +- **Good bot.** Automated programs who visit websites in order to collect useful information. Web crawlers, site scrapers, stress testers, preview builders and other programs are welcomed on most websites because they serve purposes of mutual benefits. +- **Bad bot.** Programs which are designed to perform malicious actions, ultimately hurting businesses. Testing credential databases, DDoS attacks, spam bots. + +## Clarifications +### What does "isbot" do? +This package aims to identify "Good bots". Those who voluntarily identify themselves by setting a unique, preferably descriptive, user agent, usually by setting a dedicated request header. + +### What doesn't "isbot" do? +It does not try to recognise malicious bots or programs disguising themselves as real users. + +### Why would I want to identify good bots? +Recognising good bots such as web crawlers is useful for multiple purposes. Although it is not recommended to serve different content to web crawlers like Googlebot, you can still elect to +- Flag bot pageviews to consider in business analysis +- Prefer to serve cached content and relieve service load +- Omit third party solutions' code (tags, pixels) +> It is not recommended to whitelist requests for any reason based on user agent header only. Instead other methods of identification can be added such as reverse dns lookup. + ## Data sources ### Crawlers user agents: diff --git a/list.json b/list.json index 1df2a12..91ad51f 100644 --- a/list.json +++ b/list.json @@ -37,6 +37,7 @@ "^CakePHP", "^Camo Asset Proxy", "^CATExplorador/\\d", + "^ClamAV[\\s/]", "^Client", "^Custom$", "^DAP ", @@ -83,13 +84,16 @@ "^Gregarius", "^GuzzleHttp", "^Hatena", + "^Hobbit", "^http", "^HubSpot", + "^HWCDN/", "^IBrowse", "^ICE Browser", "^iframely/", "^infoX-WISG", "^java", + "^JetBrains", "^Jetty/", "^Jigsaw", "^libtorrent", @@ -114,6 +118,7 @@ "^Microsoft-WebDAV-MiniRedir", "^Mixmax-LinkPreview", "^MixnodeCache/", + "^Monit", "^MovableType", "^Mozilla/4\\.0 \\(compatible;\\)$", "^Mozilla/5\\.0 \\(compatible(; Optimizer)?\\)", @@ -148,6 +153,7 @@ "^python", "^RamblerMail", "^raynette_httprequest", + "^RebelMouse", "^Riddler", "^RssBandit", "^RSSOwl", @@ -188,6 +194,7 @@ "^WhatWeb", "^Windows-RSS-Platform", "^Xenu Link Sleuth", + "^Xymon", "^Yahoo", "^Yandex", "^Zabbix", @@ -210,6 +217,7 @@ "bot", "BrandVerity/", "BrowseX", + "burpcollaborator", "capture", "Catchpoint", "check", @@ -219,11 +227,14 @@ "collect", "Commons-HttpClient", "crawl", + "daemon", "DareBoost", "Datanyze", "dataprovider", + "DejaClick", "detector", "developers\\.google\\.com\\/\\+\\/web\\/snippet\\/", + "DMBrowser", "Embedly", "feed", "fetch", @@ -244,19 +255,21 @@ "Google-Site-Verification", "Google-Structured-Data-Testing-Tool", "Google-Youtube", + "googleweblight", "GroupHigh/", "GTmetrix", "HeadlessChrome/", "heritrix", "httrack", "HubSpot Marketing Grader", + "Hydra", "ichiro", - "IDA", "images", "index", "ips-agent", "java/", "JavaFX", + "Jorgee", "kulturarw3/", "library", "link preview", @@ -266,8 +279,6 @@ "Mediapartners-Google", "Miniflux/", "monitor", - "MSIE or Firefox mutant", - "Netcraft Web Server Survey", "NetcraftSurveyAgent/", "NetNewsWire", "NewsFox", @@ -296,11 +307,13 @@ "ScoutJet", "scrape", "search", + "server", "SeznamEmailProxy", "shrinktheweb\\.com", "SkypeUriPreview", "slurp", "Snacktory", + "Sogou", "SpeedMode; Proxy;", "spider", "spy", @@ -317,13 +330,13 @@ "Twingly Recon", "url", "Vagabondo", + "validator", "vkshare", "Wappalyzer", "webcorp/", "WebDataStats/", "Webglance", "webkit2png", - "Widget Server", "WinHTTP", "wmtips.com/\\d", "woorankreview", diff --git a/package.json b/package.json index c5f322e..f2e403f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "isbot", - "version": "3.0.0", + "version": "3.0.1", "description": "🤖 detect bots/crawlers/spiders via the user agent.", "keywords": [ "bot", @@ -12,7 +12,7 @@ ], "author": "Goran Gajic", "license": "MIT", - "homepage": "https://github.com/gorangajic/isbot", + "homepage": "https://gorangajic.github.io/isbot", "repository": { "type": "git", "url": "https://github.com/gorangajic/isbot" @@ -23,6 +23,10 @@ "main": "index.js", "types": "index.d.ts", "scripts": { + "prebuild": "which parcel || npm i parcel-bundler --no-save", + "build": "parcel build src/index.html --out-dir docs --public-url .", + "prestart": "which parcel || npm i parcel-bundler --no-save", + "start": "parcel src/index.html --out-dir docs", "prepare": "./scripts/download-fixtures.sh", "sort": "./scripts/sort.js", "unit": "node --experimental-worker ./node_modules/.bin/mocha tests/*.js --exclude 'tests/.*'", diff --git a/src/favicon.ico b/src/favicon.ico new file mode 100644 index 0000000..4f277cf Binary files /dev/null and b/src/favicon.ico differ diff --git a/src/index.html b/src/index.html new file mode 100644 index 0000000..5868eb0 --- /dev/null +++ b/src/index.html @@ -0,0 +1,23 @@ + + + + + + + isbot 🤖/👨‍🦰 + + + + + + + +
+

Is this a bot user agent string?

+ + +

Powered by Javascript package isbot

+
+ + + diff --git a/src/opensearch.xml b/src/opensearch.xml new file mode 100644 index 0000000..4dfaa63 --- /dev/null +++ b/src/opensearch.xml @@ -0,0 +1,9 @@ + + isbot + isbot CHECK + isbot CHECK + UTF-8 + UTF-8 + false + + diff --git a/src/script.js b/src/script.js new file mode 100644 index 0000000..11c4464 --- /dev/null +++ b/src/script.js @@ -0,0 +1,44 @@ +import isbot from '..'; + +(function () { + const textarea = document.querySelector('textarea') + const output = document.querySelector('output') + let timer + + const query = window.location.search.replace(/\?ua=(.*)$/, '$1') + + textarea.innerHTML = query + ? decodeURIComponent(query) + : navigator.userAgent + textarea.addEventListener('keyup', change) + textarea.addEventListener('paste', change) + textarea.addEventListener('focus', () => textarea.select()) + check() + + function change ({ target: { value } }) { + clearTimeout(timer) + timer = setTimeout(check, 200, value) + } + + function check (value = textarea.innerHTML) { + value = value.trim() + if (value === '') { + output.innerHTML = 'Insert user agent string in the text box' + return + } + + const result = isbot(value) + + output.innerHTML = [ + result + ? 'I think so, yes' + : 'I don\'t think so, no', + result + ? `The pattern that I recognise is ${isbot.find(value)}` + : 'I could not find a pattern I recognise' + ].join('\n') + + output.className = '' + setTimeout(() => { output.className = 'highlight' }, 100) + } +})() diff --git a/src/styles.css b/src/styles.css new file mode 100644 index 0000000..9eee652 --- /dev/null +++ b/src/styles.css @@ -0,0 +1,45 @@ +* { + margin: 0; + padding: 0; +} +html { + font: 100 100%/1.4 -apple-system, "BlinkMacSystemFont", "Helvetica Neue", "Helvetica", "Lucida Grande", "Arial", sans-serif; +} +html, body { + height: 100%; + display: flex; + justify-content: center; + align-items: center; +} +form { + width: 90vw; + padding: 1em 0; + text-align: center; +} +h1, textarea, output { + margin-bottom: 1em; +} +h1 { + font-weight: 700; + font-size: 3em; +} +textarea { + font-size: 1em; + width: 100%; + height: 5em; + padding: .25em; + resize: vertical; + box-sizing: border-box; + font-size: 2em; +} +output { + display: block; + white-space: pre-line; + font-size: 2em; +} +kbd { + display: inline-block; + height: 1em; +} +kbd:before { content: "‟"; } +kbd:after { content: "”"; } diff --git a/tests/efficiency.js b/tests/efficiency.js index 881c3a8..24045b4 100644 --- a/tests/efficiency.js +++ b/tests/efficiency.js @@ -39,7 +39,7 @@ describe('efficiency', () => { done() }) - it(`${length} rules are is necessary, and have examples`, function (done) { + it(`${length} rules are necessary, and have examples`, function (done) { const errors = [] update(`${length - list.length}/${length}. Error count: ${errors.length}`) diff --git a/tests/fixtures/manual-crawlers-list.yml b/tests/fixtures/manual-crawlers-list.yml index 45e6ba3..7edae4d 100644 --- a/tests/fixtures/manual-crawlers-list.yml +++ b/tests/fixtures/manual-crawlers-list.yml @@ -1,12 +1,52 @@ +AlertSite Monitoring: + - Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0 DejaClick/2.9.7.2 +Burp Collaborator Scanner: + - Godzilla/17.0 (Unknown Operator; Nexus 5X Build/MMB29P) NoWebKit/5.36 (DOM, like Rhino; n7vbji2k0wt3rsbev55blgx17sdk8lwek97zvo.burpcollaborator.net) TOR/540.5.35487 Torrent 2654.76 + - Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36 root@8f6gtjctw9glvnasfrq438tprig56wty20ghwkf9.burpcollaborator.net + - Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'+(select load_file('x97bjuqgzjb9cjiodeygpdyxkoqledf164uzhp5e.burpcollaborator.netszc'))+' + - http://5iojs2zo8rkhlrrwmm7oyl75twztnlo9ex9kz8o.burpcollaborator.net/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36 +ClamAV Website Scanner: + - ClamAV 0.95.3 + - "ClamAV/0.101.5 (OS: linux-gnu, ARCH: x86_64, CPU: x86_64)" coccoc: - Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine) +Dotcom Monitor: + - Mozilla/5.0 (Linux; U; Android 2.2; en-us; SCH-I800 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1 DMBrowser-BV + - Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36 DMBrowser/2.1 (BV; LV) + - Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0) DMBrowser/2.1 (UV) Flipboard: - Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.6; +http://flipboard.com/browserproxy) +Google WebLight Proxy: + - Dalvik/2.1.0 (Linux; U; Android 5.1.1_r1; Samsung Galaxy S4 Build/SGH-I337M)(googleweblight) + - Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko; googleweblight) Chrome/38.0.1025.166 Mobile Safari/535.19 + - Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko; googleweblight) Chrome/38.0.1025.166 Mobile Safari/535.19,gzip(gfe) + - Mozilla/5.0 (Windows; Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko; googleweblight) Chrome/38.0.1025.166 Mobile Safari/535.19 + - Nokia5800d-1/5.0 (SymbianOS/9.4; Series60/5.3 Nokia5800d-1/111.050.1511; Mozilla/5.0; Profile/MIDP-2.1 Configuration/CLDC-1.1; googleweblight) AppleWebKit/533.4 (KHTML, like Gecko, googleweblight) NokiaBrowser/8.3.7.4 Mobile Safari/533.4 3gpp-gba +HighWinds Content Delivery System: + - HWCDN/GFS v1.80.995-4.38.2369.el7 CDS/AT2 + - HWCDN/GFS v1.80.995-4.38.2369.el7 CDS/DA2 + - HWCDN/GFS v1.89.1003-1.3.2692.el7 CDS/AT2 +Hobbit: + - Hobbit bbtest-net/4.3.0-0.beta2 +Hydra by addthis: + - Mozilla/4.0 (Hydra) + - Mozilla/5.0 (Hydra Proxy) +Jorgee Vulnerability Scanner: + - Mozilla/5.0 Jorgee +Medusa by pymedusa: + - python/unicode Medusa/0.2.14 (Windows; 10; 3174120f-3388-11e9-b8bd-1c1b0d9d2a41) + - python/unicode Medusa/0.2.14 (Windows; 10; d26d37e1-3389-11e9-8d14-1c1b0d9d2a41) +Monit: + - Monit/5.23.0 proximic: - Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php) +RebelMouse: + - RebelMouse/0.1 Mozilla/5.0 (compatible; http://rebelmouse.com) Gecko/20100101 Firefox/7.0.1 RuxitSynthetic: - Chrome/80.0.3987.87 Safari/537.36 RuxitSynthetic/1.0 v6191106029376332690 t7889551165227354132 -TinEye: - - Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0 TinEye/1.0 (via http://www.tineye.com/) -Yahoo Ad monitoring: - - Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) yahoo.adquality.lwd.desktop/1523838870-60 +Sogou Search Dog: + - Mozilla/5.0 (Linux; Android 9; NX569J Build/PQ3A.190505.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/68.0.3440.106 Mobile Safari/537.36 AWP/2.0 SogouMSE,SogouMobileBrowser/5.21.8 +updown.io monitoring: + - updown.io daemon 2.2 +Xymon Site Monitor: + - Xymon xymonnet/4.3.17 diff --git a/tests/fixtures/manual-legit-browsers.yml b/tests/fixtures/manual-legit-browsers.yml index 6c383f5..86ca3fe 100644 --- a/tests/fixtures/manual-legit-browsers.yml +++ b/tests/fixtures/manual-legit-browsers.yml @@ -1,3 +1,5 @@ +Amazon 4K Fire TV: + - Mozilla/5.0 (Linux; Android 5.1; AFTS Build/LMY47O) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/41.99900.2250.0242 Safari/537.36 Android: - Dalvik/2.1.0 (Linux; U; Android 5.1.1; AFTT Build/LVY48F) CTV - Dalvik/2.1.0 (Linux; U; Android 9; SM-G965U Build/PPR1.180610.011) @@ -6,6 +8,9 @@ AOL: - Mozilla/4.0 (compatible; MSIE 6.0; AOL 7.0; Windows NT 5.1) - Mozilla/4.0 (compatible; MSIE 6.0; AOL 9.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322) - Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 ADG/11.0.2566 AOLBUILD/11.0.2566 Safari/537.36 +Apple TV: + - AppleTV5,3/9.1.1 + - AppleTV6,2/11.1 Arora: - Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/533.3 (KHTML, like Gecko) Arora/0.11.0 Safari/533.3 - Mozilla/5.0 (Windows; U; ; en-NZ) AppleWebKit/527 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0 @@ -47,6 +52,8 @@ Chrome: - Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.83 Safari/537.1 - Mozilla/5.0 (X11; CrOS x86_64 11895.118.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.159 Safari/537.36 - Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36 +Chromecast: + - Mozilla/5.0 (CrKey armv7l 1.5.16041) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.0 Safari/537.36 Chromium: - Mozilla/5.0 (Linux; Ubuntu 14.04 like Android 4.4) AppleWebKit/537.36 Chromium/35.0.1870.2 Mobile Safari/537.36 - Mozilla/5.0 (SMART-TV; X11; Linux armv7l) AppleWebKit/537.42 (KHTML, like Gecko) Chromium/25.0.1349.2 Chrome/25.0.1349.2 Safari/537.42 @@ -78,10 +85,12 @@ Edge: - Mozilla/5.0 (Windows IoT 10.0; Android 6.0.1; WebView/3.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Mobile Safari/537.36 Edge/17.17134 - Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134 - Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36 Edg/81.0.416.62 - - Mozilla/5.0 (Windows NT 10.0; Win64; x64; Xbox; Xbox One) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363 - Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; Lumia 650) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254 - Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; NOKIA; Lumia 930) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Mobile Safari/537.36 Edge/14.14393 - Mozilla/5.0 (iPhone; CPU iPhone OS 13_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 EdgiOS/44.10.19 Mobile/15E148 Safari/605.1.15 +Edge Xbox One: + - Mozilla/5.0 (Windows NT 10.0; Win64; x64; XBOX_ONE_ED) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393 + - Mozilla/5.0 (Windows NT 10.0; Win64; x64; Xbox; Xbox One) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363 Electron Application: - Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) PlutoTV/0.2.0 Chrome/49.0.2623.75 Electron/0.37.2 Safari/537.36 - Mozilla/5.0 (Windows NT 10.0.18362; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Slack/4.4.2 Chrome/78.0.3904.130 Electron/7.2.1 Safari/537.36 WindowsStore/10.0.18362 Sonic Slack_SSB/4.4.2 @@ -124,6 +133,8 @@ Flock: - Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.8b5) Gecko/20051102 Flock/0.4 Firefox/1.0+ - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.12; photobucket) Gecko/20070531 Firefox/1.5.0.12 Flock/0.7.14 - Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.0.19) Gecko/2010062819 Firefox/3.0.19 Flock/2.6.1 +Google Nexus Player: + - Dalvik/2.1.0 (Linux; U; Android 6.0.1; Nexus Player Build/MMB29T) Google Search App: - Mozilla/5.0 (iPad; CPU OS 9_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) GSA/10.0.63022 Mobile/13B143 Safari/600.1.4 - Mozilla/5.0 (iPhone; CPU iPhone OS 13_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/102.0.304944559 Mobile/15E148 Safari/604.1 @@ -162,6 +173,8 @@ Internet TV: Kindle: - Dalvik/2.1.0 (Linux; U; Android 5.1.1; KFSUWI Build/LVY48F) - Mozilla/5.0 (Linux; U; Android 2.3.4; en-us; Kindle Fire Build/GINGERBREAD) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533 + - Mozilla/5.0 (Linux; U; en-US) AppleWebKit/528.5+ (KHTML, like Gecko, Safari/528.5+) Version/4.0 Kindle/3.0 (screen 600x800; rotate) + - Mozilla/5.0 (X11; U; Linux armv7l like Android; en-us) AppleWebKit/531.2+ (KHTML, like Gecko) Version/5.0 Safari/533.2+ Kindle/3.0+ Konqueror: - Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.21 (KHTML, like Gecko) konqueror/4.14.16 Safari/537.21 - Mozilla/5.0 (compatible; Konqueror/3.2; Linux) (KHTML, like Gecko) @@ -180,6 +193,8 @@ Maxthon: Mercury: - Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; IE8Mercury; rv:11.0) like Gecko - Mozilla/5.0 (iPad; CPU OS 7_0_4 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Mercury/8.9.4 Mobile/11B554a Safari/9537.53 +Minix NEO X5: + - Mozilla/5.0 (Linux; U; Android 4.2.2; he-il; NEO-X5-116A Build/JDQ39) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30 Miui: - Mozilla/5.0 (Linux; U; Android 7.0; en-us; Redmi Note 4 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/53.0.2785.146 Mobile Safari/537.36 XiaoMi/MiuiBrowser/9.2.8 - Mozilla/5.0 (Linux; U; Android 9; pl-pl; Redmi Note 7 Build/PKQ1.180904.001) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.141 Mobile Safari/537.36 XiaoMi/MiuiBrowser/11.4.3-g @@ -193,6 +208,10 @@ Netscape Navigator: - Mozilla/5.0 (Windows; U; Win 9x 4.90; SG; rv:1.9.2.4) Gecko/20101104 Netscape/9.1.0285 - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.2) Gecko/20040804 Netscape/7.2 (ax) - Mozilla/5.0 (X11; U; Linux i686; en-US; rv:0.9.3) Gecko/20010801 +Nintendo 3DS: + - Mozilla/5.0 (Nintendo 3DS; U; ; en) Version/1.7412.EU +Nintendo Wii U: + - Mozilla/5.0 (Nintendo WiiU) AppleWebKit/536.30 (KHTML, like Gecko) NX/3.0.4.2.12 NintendoBrowser/4.3.1.11264.US Nokia: - Mozilla/5.0 (Linux; Android 4.1.2; Nokia_X Build/JZO54K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.82 Mobile Safari/537.36 NokiaBrowser/1.2.0.12 - Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537 @@ -255,6 +274,8 @@ PlayStation 4: - Mozilla/5.0 (PlayStation 4 5.55) AppleWebKit/601.2 (KHTML, like Gecko) - Mozilla/5.0 (PlayStation 4/SmartTV) AppleWebKit/605.1.15 (KHTML, like Gecko) THEWORLD - PS4Application libhttp/1.000 (PS4) CoreMedia libhttp/6.72 (PlayStation 4) +Playstation Vita: + - Mozilla/5.0 (PlayStation Vita 3.61) AppleWebKit/537.73 (KHTML, like Gecko) Silk/3.2 PowerPoint: - Microsoft Office PowerPoint 2013 - Microsoft Office/16.0 (Windows NT 10.0; Microsoft PowerPoint 16.0.10325; Pro) @@ -267,6 +288,8 @@ QQ: - Mozilla/5.0 (Linux; U; Android 6.0; zh-cn; FRD-AL00 Build/HUAWEIFRD-AL00) AppleWebKit/537.36 (KHTML, like Gecko)Version/4.0 Chrome/37.0.0.0 MQQBrowser/6.0 Mobile Safari/537.36 - Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36 QQBrowser/3.8.3858.400 - Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6788.400 QQBrowser/10.3.2727.400 +Roku Ultra: + - Roku4640X/DVP-7.70 (297.70E04154A) Safari: - Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15 - Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9 diff --git a/tests/helpers/index.js b/tests/helpers/index.js index e70801e..cade73a 100644 --- a/tests/helpers/index.js +++ b/tests/helpers/index.js @@ -45,6 +45,7 @@ module.exports.crawlers = [ const BOTS = new RegExp([ 'adbeat.com', 'chrome-lighthouse', + 'googleweblight', 'JavaFX', 'phantomjs', 'swurl', diff --git a/tsconfig.json b/tsconfig.json index 96b2e0c..5d45787 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,6 +12,6 @@ "esModuleInterop": true, "resolveJsonModule": true }, - "include": ["."], - "exclude": ["tests", "scripts", "node_modules"] + "include": ["index.js", "list.json", "index.d.ts"], + "exclude": [] }