more bots

biggora · Oct 29, 2016 · c50ad80 · c50ad80
1 parent 113618a
commit c50ad80
Showing 1 changed file with 25 additions and 10 deletions.
diff --git a/lib/express-useragent.js b/lib/express-useragent.js
@@ -8,23 +8,38 @@
 
     var BOTS = [
         '\\+https:\\/\\/developers.google.com\\/\\+\\/web\\/snippet\\/',
-        'googlebot',
+        'ad\\smonitoring', // Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html)
+        'adsbot', // Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)
         'baiduspider',
-        'gurujibot',
-        'yandexbot',
-        'slurp',
-        'msnbot',
         'bingbot',
+        'cloudflare', // Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34
+        'crawler', // TinEye/1.1 (http://tineye.com/crawler.html), Hubspot webcrawler, aicrawler, etc
+        'curl',
+        'embedly',
+        'exabot', //Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)
         'facebookexternalhit',
+        'googlebot',
+        'gurujibot',
+        'heritrix',
         'linkedinbot',
-        'twitterbot',
+        'msnbot',
+        'phantomjs',
+        'phantom\\.js',
+        'pingdom',
+        'python',
+        'rtlnieuws',
         'slackbot',
+        'slurp',
+        'spbot',
         'telegrambot',
-        'applebot',
-        'pingdom',
+        'test\\scertificate',
+        'testing',
+        'tiabot',
         'tumblr ',
-        'Embedly',
-        'spbot'
+        'twitterbot',
+        'web\\sscraper',
+        'wget',
+        'yandexbot',
     ];
     var IS_BOT_REGEXP = new RegExp('^.*(' + BOTS.join('|') + ').*$');