Adds detection for various bots

Tests/Parser/Client/fixtures/library.yml

@@ -659,3 +659,15 @@
     type: library
     name: Boto3
     version: 1.17.18
+-
+  user_agent: Custom-AsyncHttpClient
+  client:
+    type: library
+    name: Custom Async HTTP Client
+    version: ""
+-
+  user_agent: Custom-HttpClient
+  client:
+    type: library
+    name: Custom HTTP Client
+    version: ""

Tests/fixtures/bots.yml

@@ -7573,3 +7573,112 @@
     producer:
       name: Elasticsearch B.V.
       url: https://www.elastic.co/
+-
+  user_agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0 Time/1710630666672
+  bot:
+    name: Generic Bot
+-
+  user_agent: Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN;; rv:1.9.0.9) Gecko/2009042410 Firefox/3.0.9 Wyzo/3.0.3 Time/1715201908281
+  bot:
+    name: Generic Bot
+-
+  user_agent: WDG_Validator/1.6.2
+  bot:
+    name: WDG HTML Validator
+    category: Validator
+    url: http://www.htmlhelp.com/tools/validator/
+-
+  user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; BuiltWith/1.0; +http://builtwith.com/biup) Chrome/74.0.3729.131 Safari/537.36
+  bot:
+    name: BuiltWith
+    category: Crawler
+    url: https://builtwith.com/biup
+    producer:
+      name: BuiltWith Pty Ltd
+      url: https://builtwith.com/
+-
+  user_agent: Google Chrome Mozilla/5.0 (Windows NT 10.0; Win64; x64; CONTACT; scan@aegis.network) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36
+  bot:
+    name: Aegis
+    category: Crawler
+    url: https://web.archive.org/web/20180910002802/http://www.aegis.network/
+-
+  user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 (compatible; CrawlyProjectCrawler/0.1.3; crawlyproject@digitaldragon.dev +https://crawlyproject.digitaldragon.dev/)
+  bot:
+    name: Crawly Project
+    category: Crawler
+    url: https://web.archive.org/web/20240326141952/https://crawlyproject.digitaldragon.dev/
+-
+  user_agent: BDFetch
+  bot:
+    name: BDFetch
+    category: Crawler
+    url: https://web.archive.org/web/20130821043949/http://www.branddimensions.com/
+-
+  user_agent: r00ts3c-owned-you
+  bot:
+    name: Generic Bot
+-
+  user_agent: nvdorz
+  bot:
+    name: Generic Bot
+-
+  user_agent: Root Slut
+  bot:
+    name: Generic Bot
+-
+  user_agent: Abcd
+  bot:
+    name: Generic Bot
+-
+  user_agent: NiggaBalls
+  bot:
+    name: Generic Bot
+-
+  user_agent: Dark
+  bot:
+    name: Generic Bot
+-
+  user_agent: BotPoke
+  bot:
+    name: Generic Bot
+-
+  user_agent: url
+  bot:
+    name: Generic Bot
+-
+  user_agent: facebookscraper/1.0( http://www.facebook.com/sharescraper_help.php)
+  bot:
+    name: Facebook Crawler
+    category: Social Media Agent
+    url: https://developers.facebook.com/docs/sharing/webmasters/crawler/
+    producer:
+      name: Meta Platforms, Inc.
+      url: https://www.meta.com/
+-
+  user_agent: PunkMap (https://github.com/openeasm/punkmap)
+  bot:
+    name: Punk Map
+    category: Security Checker
+    url: https://github.com/openeasm/punkmap
+-
+  user_agent: "'Mozilla/5.0 (compatible; GenomeCrawlerd/1.0; +https://www.nokia.com/networks/ip-networks/deepfield/genome/)'"
+  bot:
+    name: Deepfield Genome
+    category: Crawler
+    url: https://www.nokia.com/networks/ip-networks/deepfield/genome/
+    producer:
+      name: Nokia Corporation
+      url: https://www.nokia.com/
+-
+  user_agent: Gaisbot/3.0 (robot@gais.cs.ccu.edu.tw; http://gais.cs.ccu.edu.tw/robot.php)
+  bot:
+    name: Gaisbot
+    category: Crawler
+    url: https://web.archive.org/web/20090604121511/https://gais.cs.ccu.edu.tw/robot.php
+-
+  user_agent: FAST-WebCrawler/3.8 (crawler at trd dot overture dot com; http://www.alltheweb.com/help/webmaster/crawler)
+  bot:
+    name: AlltheWeb
+    category: Crawler
+    url: https://web.archive.org/web/20041020050801/http://www.alltheweb.com/help/webmaster/crawler

regexes/bots.yml

@@ -551,7 +551,7 @@
     name: 'SEOmoz, Inc.'
     url: 'http://moz.com/'
 
-- regex: 'facebookexternalhit|facebookplatform|facebookexternalua|facebookcatalog'
+- regex: 'facebook(?:catalog|externalhit|externalua|platform|scraper)'
   name: 'Facebook Crawler'
   category: 'Social Media Agent'
   url: 'https://developers.facebook.com/docs/sharing/webmasters/crawler/'
@@ -2455,7 +2455,7 @@
     name: 'PPC Labs LLC'
     url: 'https://www.adbeat.com/'
 
-- regex: 'BW/[\d.]+'
+- regex: '(?:BuiltWith|BW)/[\d.]+'
   name: 'BuiltWith'
   category: 'Crawler'
   url: 'https://builtwith.com/biup'
@@ -4440,8 +4440,51 @@
     name: 'Elasticsearch B.V.'
     url: 'https://www.elastic.co/'
 
+- regex: 'WDG_Validator/[\d.]+'
+  name: 'WDG HTML Validator'
+  category: 'Validator'
+  url: 'http://www.htmlhelp.com/tools/validator/'
+
+- regex: 'scan@aegis.network'
+  name: 'Aegis'
+  category: 'Crawler'
+  url: 'https://web.archive.org/web/20180910002802/http://www.aegis.network/'
+
+- regex: 'CrawlyProjectCrawler/[\d.]+'
+  name: 'Crawly Project'
+  category: 'Crawler'
+  url: 'https://web.archive.org/web/20240326141952/https://crawlyproject.digitaldragon.dev/'
+
+- regex: 'BDFetch'
+  name: 'BDFetch'
+  category: 'Crawler'
+  url: 'https://web.archive.org/web/20130821043949/http://www.branddimensions.com/'
+
+- regex: 'PunkMap'
+  name: 'Punk Map'
+  category: 'Security Checker'
+  url: 'https://github.com/openeasm/punkmap'
+
+- regex: 'GenomeCrawlerd/[\d.]+'
+  name: 'Deepfield Genome'
+  category: 'Crawler'
+  url: 'https://www.nokia.com/networks/ip-networks/deepfield/genome/'
+  producer:
+    name: 'Nokia Corporation'
+    url: 'https://www.nokia.com/'
+
+- regex: 'Gaisbot/[\d.]+'
+  name: 'Gaisbot'
+  category: 'Crawler'
+  url: 'https://web.archive.org/web/20090604121511/https://gais.cs.ccu.edu.tw/robot.php'
+
+- regex: 'FAST-WebCrawler/[\d.]+'
+  name: 'AlltheWeb'
+  category: 'Crawler'
+  url: 'https://web.archive.org/web/20041020050801/http://www.alltheweb.com/help/webmaster/crawler'
+
 # Generic bots
-- regex: 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|vortex(?!(?: Build|Plus| CM62| HD65))|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\.o\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \(cow\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherweb|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|^xenu|^(?:chrome|firefox|KvshClient|Zeus|ZmEu)$'
+- regex: 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|vortex(?!(?: Build|Plus| CM62| HD65))|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\.o\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \(cow\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherweb|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|^xenu|^(?:chrome|firefox|Abcd|Dark|KvshClient|url|Zeus|ZmEu)$'
   name: 'Generic Bot'
 
 # Generic detections

regexes/client/libraries.yml

@@ -611,3 +611,13 @@
   name: 'Boto3'
   version: '$1'
   url: 'https://github.com/boto/boto3'
+
+- regex: 'Custom-HttpClient'
+  name: 'Custom HTTP Client'
+  version: ''
+  url: ''
+
+- regex: 'Custom-AsyncHttpClient'
+  name: 'Custom Async HTTP Client'
+  version: ''
+  url: ''