From afec4faf0f5a78e80f9563f129f7d09d5c0dee0e Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 31 Oct 2023 09:55:10 +0100 Subject: [PATCH 01/24] Adds detection for HTML Parser --- Tests/Parser/Client/fixtures/library.yml | 7 ++++++- regexes/client/libraries.yml | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Tests/Parser/Client/fixtures/library.yml b/Tests/Parser/Client/fixtures/library.yml index 0366dc1ff5..9b7f934bdd 100644 --- a/Tests/Parser/Client/fixtures/library.yml +++ b/Tests/Parser/Client/fixtures/library.yml @@ -431,7 +431,6 @@ type: library name: cri-o version: 1.16.1 - - user_agent: go-containerregistry/v0.11.0 client: @@ -528,3 +527,9 @@ type: library name: Axios version: "1.2.0" +- + user_agent: HTMLParser/1.6 + client: + type: library + name: HTML Parser + version: "1.6" diff --git a/regexes/client/libraries.yml b/regexes/client/libraries.yml index ad22d6787b..f52d5be197 100644 --- a/regexes/client/libraries.yml +++ b/regexes/client/libraries.yml @@ -5,6 +5,11 @@ # @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or later ############### +- regex: 'HTMLParser(?:/(\d+[\.\d]+))?' + name: 'HTML Parser' + version: '$1' + url: 'https://htmlparser.sourceforge.net/' + # got (https://github.com/sindresorhus/got) - regex: '^got ' name: 'got' From a94bd66f0a4fae323542527fb680ca108676bc1d Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 31 Oct 2023 09:56:41 +0100 Subject: [PATCH 02/24] Improves detection for Python urllib --- Tests/Parser/Client/fixtures/library.yml | 6 ++++++ regexes/client/libraries.yml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Tests/Parser/Client/fixtures/library.yml b/Tests/Parser/Client/fixtures/library.yml index 9b7f934bdd..caef3cd480 100644 --- a/Tests/Parser/Client/fixtures/library.yml +++ b/Tests/Parser/Client/fixtures/library.yml @@ -533,3 +533,9 @@ type: library name: HTML Parser version: "1.6" +- + user_agent: python-urllib3/1.26.9 + client: + type: library + name: Python urllib + version: 1.26.9 diff --git a/regexes/client/libraries.yml b/regexes/client/libraries.yml index f52d5be197..cbcf41283d 100644 --- a/regexes/client/libraries.yml +++ b/regexes/client/libraries.yml @@ -101,7 +101,7 @@ version: '$1' url: 'https://pypi.org/project/httplib2/' -- regex: 'Python-urllib(?:/?(\d+[\.\d]+))?' +- regex: 'Python-urllib3?(?:/?(\d+[\.\d]+))?' name: 'Python urllib' version: '$1' From 72522d84d4b4996469f7df8451b5bd7d9885d287 Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 31 Oct 2023 09:59:43 +0100 Subject: [PATCH 03/24] Adds detection for msray --- Tests/Parser/Client/fixtures/library.yml | 6 ++++++ regexes/client/libraries.yml | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/Tests/Parser/Client/fixtures/library.yml b/Tests/Parser/Client/fixtures/library.yml index caef3cd480..4f8f0371fa 100644 --- a/Tests/Parser/Client/fixtures/library.yml +++ b/Tests/Parser/Client/fixtures/library.yml @@ -539,3 +539,9 @@ type: library name: Python urllib version: 1.26.9 +- + user_agent: msray-plus + client: + type: library + name: msray + version: "" diff --git a/regexes/client/libraries.yml b/regexes/client/libraries.yml index cbcf41283d..4ff7494d33 100644 --- a/regexes/client/libraries.yml +++ b/regexes/client/libraries.yml @@ -5,6 +5,11 @@ # @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or later ############### +- regex: 'msray-plus' + name: 'msray' + version: '' + url: 'https://github.com/super-l/msray' + - regex: 'HTMLParser(?:/(\d+[\.\d]+))?' name: 'HTML Parser' version: '$1' From c152c3ec5447270685ce2eb7766df64d69d7b107 Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 31 Oct 2023 10:01:55 +0100 Subject: [PATCH 04/24] Adds detection for Slim --- Tests/Parser/Client/fixtures/library.yml | 6 ++++++ regexes/client/libraries.yml | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/Tests/Parser/Client/fixtures/library.yml b/Tests/Parser/Client/fixtures/library.yml index 4f8f0371fa..ce7a795b24 100644 --- a/Tests/Parser/Client/fixtures/library.yml +++ b/Tests/Parser/Client/fixtures/library.yml @@ -545,3 +545,9 @@ type: library name: msray version: "" +- + user_agent: Slim Framework + client: + type: library + name: Slim + version: "" diff --git a/regexes/client/libraries.yml b/regexes/client/libraries.yml index 4ff7494d33..e62c31db81 100644 --- a/regexes/client/libraries.yml +++ b/regexes/client/libraries.yml @@ -5,6 +5,11 @@ # @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or later ############### +- regex: 'Slim Framework' + name: 'Slim' + version: '' + url: 'https://www.slimframework.com/' + - regex: 'msray-plus' name: 'msray' version: '' From 6f91cdd85a12b10960ca3ad9ea33e1baed91aa64 Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 31 Oct 2023 10:04:35 +0100 Subject: [PATCH 05/24] Adds detection for Fuzz Faster U Fool --- Tests/Parser/Client/fixtures/library.yml | 6 ++++++ regexes/client/libraries.yml | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/Tests/Parser/Client/fixtures/library.yml b/Tests/Parser/Client/fixtures/library.yml index ce7a795b24..b94e31041e 100644 --- a/Tests/Parser/Client/fixtures/library.yml +++ b/Tests/Parser/Client/fixtures/library.yml @@ -551,3 +551,9 @@ type: library name: Slim version: "" +- + user_agent: Fuzz Faster U Fool v1.5.0-dev + client: + type: library + name: Fuzz Faster U Fool + version: 1.5.0 diff --git a/regexes/client/libraries.yml b/regexes/client/libraries.yml index e62c31db81..f8fc8ee5bb 100644 --- a/regexes/client/libraries.yml +++ b/regexes/client/libraries.yml @@ -5,6 +5,11 @@ # @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or later ############### +- regex: 'Fuzz Faster U Fool v(\d+[\.\d]+)' + name: 'Fuzz Faster U Fool' + version: '$1' + url: 'https://github.com/ffuf/ffuf' + - regex: 'Slim Framework' name: 'Slim' version: '' From 835c83bcc35c49f266de91f5257c7dde1a562b88 Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 31 Oct 2023 10:12:26 +0100 Subject: [PATCH 06/24] Adds detection for Matomo --- Tests/fixtures/bots.yml | 9 +++++++++ regexes/bots.yml | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/Tests/fixtures/bots.yml b/Tests/fixtures/bots.yml index a94571c59a..33f7f47506 100644 --- a/Tests/fixtures/bots.yml +++ b/Tests/fixtures/bots.yml @@ -5828,3 +5828,12 @@ name: phpMyAdmin category: Service Agent url: https://www.phpmyadmin.net/ +- + user_agent: Matomo/4.15.1 + bot: + name: Matomo + category: Service Agent + url: https://github.com/matomo-org/matomo + producer: + name: InnoCraft Ltd + url: https://matomo.org/ diff --git a/regexes/bots.yml b/regexes/bots.yml index 92f808036a..9761ea47c4 100644 --- a/regexes/bots.yml +++ b/regexes/bots.yml @@ -3472,6 +3472,14 @@ category: 'Service Agent' url: 'https://www.phpmyadmin.net/' +- regex: 'Matomo/([\d+.]+)' + name: 'Matomo' + category: 'Service Agent' + url: 'https://github.com/matomo-org/matomo' + producer: + name: 'InnoCraft Ltd' + url: 'https://matomo.org/' + # Generic detections - regex: '[a-z0-9\-_]*((? Date: Tue, 31 Oct 2023 10:15:30 +0100 Subject: [PATCH 07/24] Improves detection for generic bots --- Tests/fixtures/bots.yml | 4 ++++ regexes/bots.yml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Tests/fixtures/bots.yml b/Tests/fixtures/bots.yml index 33f7f47506..ff91ff71e4 100644 --- a/Tests/fixtures/bots.yml +++ b/Tests/fixtures/bots.yml @@ -5837,3 +5837,7 @@ producer: name: InnoCraft Ltd url: https://matomo.org/ +- + user_agent: CustomUserAgent/1.0 + bot: + name: Generic Bot diff --git a/regexes/bots.yml b/regexes/bots.yml index 9761ea47c4..a73b1d6b10 100644 --- a/regexes/bots.yml +++ b/regexes/bots.yml @@ -2035,7 +2035,7 @@ - regex: 'RSSRadio \(Push Notification Scanner;support@dorada\.co\.uk\)' name: 'RSSRadio Bot' -- regex: '(A6-Indexer|nuhk|TsolCrawler|Yammybot|Openbot|Gulper Web Bot|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr.com|tweetedtimes.com|TrendsmapResolver|teoma|blitzbot|oegp|furlbot|http%20client|polybot|htdig|mogimogi|larbin|scrubby|searchsight|seekbot|semanticdiscovery|snappy|vortex(?!(?: Build|Plus))|zeal(?!ot)|fast-webcrawler|converacrawler|dataparksearch|findlinks|BrowserMob|HttpMonitor|ThumbShotsBot|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|RackspaceBot|robots|SeopultContentAnalyzer|7Siters|centuryb.o.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|My User Agent|cortex|CF-UC User Agent|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|daumoa,damoa,daum,daumos,duamoa,duam,duamos|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \(cow\)|custom_user_agent|Test Certificate Info|iplabel|Magellan)' +- regex: '(A6-Indexer|nuhk|TsolCrawler|Yammybot|Openbot|Gulper Web Bot|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr.com|tweetedtimes.com|TrendsmapResolver|teoma|blitzbot|oegp|furlbot|http%20client|polybot|htdig|mogimogi|larbin|scrubby|searchsight|seekbot|semanticdiscovery|snappy|vortex(?!(?: Build|Plus))|zeal(?!ot)|fast-webcrawler|converacrawler|dataparksearch|findlinks|BrowserMob|HttpMonitor|ThumbShotsBot|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|RackspaceBot|robots|SeopultContentAnalyzer|7Siters|centuryb.o.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|My User Agent|cortex|CF-UC User Agent|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|daumoa,damoa,daum,daumos,duamoa,duam,duamos|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \(cow\)|custom_user_agent|Test Certificate Info|iplabel|Magellan|CustomUserAgent)' name: 'Generic Bot' - regex: '^sentry' From 6fd5560d9009b0a1074402d0f9b65d3783d7f60f Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 31 Oct 2023 10:19:05 +0100 Subject: [PATCH 08/24] Adds detection for Prometheus --- Tests/fixtures/bots.yml | 9 +++++++++ regexes/bots.yml | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/Tests/fixtures/bots.yml b/Tests/fixtures/bots.yml index ff91ff71e4..bac8e3fbdb 100644 --- a/Tests/fixtures/bots.yml +++ b/Tests/fixtures/bots.yml @@ -5841,3 +5841,12 @@ user_agent: CustomUserAgent/1.0 bot: name: Generic Bot +- + user_agent: Prometheus/2.40.5 + bot: + name: Prometheus + category: Service Agent + url: https://github.com/prometheus/prometheus + producer: + name: The Linux Foundation + url: https://www.cncf.io/ diff --git a/regexes/bots.yml b/regexes/bots.yml index a73b1d6b10..242f71151f 100644 --- a/regexes/bots.yml +++ b/regexes/bots.yml @@ -3480,6 +3480,14 @@ name: 'InnoCraft Ltd' url: 'https://matomo.org/' +- regex: 'Prometheus/([\d+.]+)' + name: 'Prometheus' + category: 'Service Agent' + url: 'https://github.com/prometheus/prometheus' + producer: + name: 'The Linux Foundation' + url: 'https://www.cncf.io/' + # Generic detections - regex: '[a-z0-9\-_]*((? Date: Tue, 31 Oct 2023 10:35:27 +0100 Subject: [PATCH 09/24] Improves detection for generic bots --- Tests/fixtures/bots.yml | 8 ++++++++ regexes/bots.yml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Tests/fixtures/bots.yml b/Tests/fixtures/bots.yml index bac8e3fbdb..99124ef0b6 100644 --- a/Tests/fixtures/bots.yml +++ b/Tests/fixtures/bots.yml @@ -5850,3 +5850,11 @@ producer: name: The Linux Foundation url: https://www.cncf.io/ +- + user_agent: firefox + bot: + name: Generic Bot +- + user_agent: Chrome + bot: + name: Generic Bot diff --git a/regexes/bots.yml b/regexes/bots.yml index 242f71151f..0d10cf816d 100644 --- a/regexes/bots.yml +++ b/regexes/bots.yml @@ -3489,5 +3489,5 @@ url: 'https://www.cncf.io/' # Generic detections -- regex: '[a-z0-9\-_]*((? Date: Tue, 31 Oct 2023 10:49:05 +0100 Subject: [PATCH 10/24] Adds detection for ArchiveBot --- Tests/fixtures/bots.yml | 9 +++++++++ regexes/bots.yml | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/Tests/fixtures/bots.yml b/Tests/fixtures/bots.yml index 99124ef0b6..5d3a7e0faf 100644 --- a/Tests/fixtures/bots.yml +++ b/Tests/fixtures/bots.yml @@ -5858,3 +5858,12 @@ user_agent: Chrome bot: name: Generic Bot +- + user_agent: ArchiveTeam ArchiveBot/20220523.4a672db (wpull 2.0.3) and not Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36 + bot: + name: ArchiveBot + category: Crawler + url: https://wiki.archiveteam.org/index.php?title=ArchiveBot + producer: + name: ArchiveTeam + url: https://wiki.archiveteam.org/ diff --git a/regexes/bots.yml b/regexes/bots.yml index 0d10cf816d..a76af6d51c 100644 --- a/regexes/bots.yml +++ b/regexes/bots.yml @@ -3488,6 +3488,14 @@ name: 'The Linux Foundation' url: 'https://www.cncf.io/' +- regex: 'ArchiveTeam ArchiveBot' + name: 'ArchiveBot' + category: 'Crawler' + url: 'https://wiki.archiveteam.org/index.php?title=ArchiveBot' + producer: + name: 'ArchiveTeam' + url: 'https://wiki.archiveteam.org/' + # Generic detections - regex: '[a-z0-9\-_]*((? Date: Tue, 31 Oct 2023 10:51:06 +0100 Subject: [PATCH 11/24] Adds detection for MADBbot --- Tests/fixtures/bots.yml | 6 ++++++ regexes/bots.yml | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/Tests/fixtures/bots.yml b/Tests/fixtures/bots.yml index 5d3a7e0faf..a0b7d67121 100644 --- a/Tests/fixtures/bots.yml +++ b/Tests/fixtures/bots.yml @@ -5867,3 +5867,9 @@ producer: name: ArchiveTeam url: https://wiki.archiveteam.org/ +- + user_agent: MADBbot/0.1 (Gathering webpages for data analytics; https://madb.zapto.org/bot.html; ma-db-crawl@googlegroups.com) + bot: + name: MADBbot + category: Crawler + url: https://madb.zapto.org/bot.html diff --git a/regexes/bots.yml b/regexes/bots.yml index a76af6d51c..668e42c5a9 100644 --- a/regexes/bots.yml +++ b/regexes/bots.yml @@ -3496,6 +3496,11 @@ name: 'ArchiveTeam' url: 'https://wiki.archiveteam.org/' +- regex: 'MADBbot/([\d+.]+)' + name: 'MADBbot' + category: 'Crawler' + url: 'https://madb.zapto.org/bot.html' + # Generic detections - regex: '[a-z0-9\-_]*((? Date: Tue, 31 Oct 2023 12:51:16 +0100 Subject: [PATCH 12/24] Adds detection for Kali --- Parser/OperatingSystem.php | 3 ++- Tests/Parser/fixtures/oss.yml | 8 ++++++++ regexes/oss.yml | 7 +++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Parser/OperatingSystem.php b/Parser/OperatingSystem.php index 4e3668f19d..9d8fcec44d 100644 --- a/Parser/OperatingSystem.php +++ b/Parser/OperatingSystem.php @@ -87,6 +87,7 @@ class OperatingSystem extends AbstractParser 'INF' => 'Inferno', 'JME' => 'Java ME', 'KOS' => 'KaiOS', + 'KAL' => 'Kali', 'KAN' => 'Kanotix', 'KNO' => 'Knoppix', 'KTV' => 'KreaTV', @@ -203,7 +204,7 @@ class OperatingSystem extends AbstractParser 'ORD', 'TOS', 'RSO', 'DEE', 'FRE', 'MAG', 'FEN', 'CAI', 'PCL', 'HAS', 'LOS', 'DVK', 'ROK', 'OWR', 'OTV', 'KTV', 'PUR', 'PLA', 'FUC', 'PAR', 'FOR', 'MON', 'KAN', 'ZEN', 'LND', 'LNS', 'CHN', 'AMZ', 'TEN', 'CST', - 'NOV', 'ROU', 'ZOR', 'RED', + 'NOV', 'ROU', 'ZOR', 'RED', 'KAL', ], 'Mac' => ['MAC'], 'Mobile Gaming Console' => ['PSP', 'NDS', 'XBX'], diff --git a/Tests/Parser/fixtures/oss.yml b/Tests/Parser/fixtures/oss.yml index d4025ccc78..a3cd2d3080 100644 --- a/Tests/Parser/fixtures/oss.yml +++ b/Tests/Parser/fixtures/oss.yml @@ -3934,3 +3934,11 @@ version: "14.1" platform: family: Mac +- + user_agent: python-requests/2.7.0 CPython/2.7.15 Linux/4.16.0-kali2-amd64 + os: + name: Kali + short_name: KAL + version: "2" + platform: + family: GNU/Linux diff --git a/regexes/oss.yml b/regexes/oss.yml index 088f04b037..9e87a9865f 100644 --- a/regexes/oss.yml +++ b/regexes/oss.yml @@ -5,6 +5,13 @@ # @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or later ############### +########## +# Kali (https://www.kali.org/) +########## +- regex: 'Linux/.+kali(\d)' + name: 'Kali' + version: '$1' + ########## # PICO OS (https://www.picoxr.com/global/software/pico-os) ########## From bacc7a804ffa2a2f96e492d8b37596c38ac7414f Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 31 Oct 2023 12:57:16 +0100 Subject: [PATCH 13/24] Adds detection for Oracle Linux --- Parser/OperatingSystem.php | 3 ++- Tests/Parser/fixtures/oss.yml | 8 ++++++++ regexes/oss.yml | 7 +++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Parser/OperatingSystem.php b/Parser/OperatingSystem.php index 9d8fcec44d..a0d3e0bf51 100644 --- a/Parser/OperatingSystem.php +++ b/Parser/OperatingSystem.php @@ -120,6 +120,7 @@ class OperatingSystem extends AbstractParser 'OBS' => 'OpenBSD', 'OWR' => 'OpenWrt', 'OTV' => 'Opera TV', + 'ORA' => 'Oracle Linux', 'ORD' => 'Ordissimo', 'PAR' => 'Pardus', 'PCL' => 'PCLinuxOS', @@ -204,7 +205,7 @@ class OperatingSystem extends AbstractParser 'ORD', 'TOS', 'RSO', 'DEE', 'FRE', 'MAG', 'FEN', 'CAI', 'PCL', 'HAS', 'LOS', 'DVK', 'ROK', 'OWR', 'OTV', 'KTV', 'PUR', 'PLA', 'FUC', 'PAR', 'FOR', 'MON', 'KAN', 'ZEN', 'LND', 'LNS', 'CHN', 'AMZ', 'TEN', 'CST', - 'NOV', 'ROU', 'ZOR', 'RED', 'KAL', + 'NOV', 'ROU', 'ZOR', 'RED', 'KAL', 'ORA', ], 'Mac' => ['MAC'], 'Mobile Gaming Console' => ['PSP', 'NDS', 'XBX'], diff --git a/Tests/Parser/fixtures/oss.yml b/Tests/Parser/fixtures/oss.yml index a3cd2d3080..b0b2b7dbe3 100644 --- a/Tests/Parser/fixtures/oss.yml +++ b/Tests/Parser/fixtures/oss.yml @@ -3942,3 +3942,11 @@ version: "2" platform: family: GNU/Linux +- + user_agent: python-requests/2.6.0 CPython/2.7.5 Linux/4.1.12-124.15.4.el7uek.x86_64 + os: + name: Oracle Linux + short_name: ORA + version: "7" + platform: x64 + family: GNU/Linux diff --git a/regexes/oss.yml b/regexes/oss.yml index 9e87a9865f..da7c817bb1 100644 --- a/regexes/oss.yml +++ b/regexes/oss.yml @@ -5,6 +5,13 @@ # @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or later ############### +########## +# Oracle Linux (https://www.oracle.com/linux/) +########## +- regex: '.+.el(\d+(?:[_\.]\d+)*)uek' + name: 'Oracle Linux' + version: '$1' + ########## # Kali (https://www.kali.org/) ########## From c0e2d82bcfa1dbf73bd5c442e2dc8a4dd605f0bb Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 31 Oct 2023 12:59:19 +0100 Subject: [PATCH 14/24] Improves version detection for TencentOS --- Tests/Parser/fixtures/oss.yml | 2 +- regexes/oss.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/Parser/fixtures/oss.yml b/Tests/Parser/fixtures/oss.yml index b0b2b7dbe3..bf8ad65b5a 100644 --- a/Tests/Parser/fixtures/oss.yml +++ b/Tests/Parser/fixtures/oss.yml @@ -3364,7 +3364,7 @@ os: name: TencentOS short_name: TEN - version: 4.14.105 + version: "3" platform: family: GNU/Linux - diff --git a/regexes/oss.yml b/regexes/oss.yml index da7c817bb1..5032186918 100644 --- a/regexes/oss.yml +++ b/regexes/oss.yml @@ -15,7 +15,7 @@ ########## # Kali (https://www.kali.org/) ########## -- regex: 'Linux/.+kali(\d)' +- regex: '.+kali(\d)' name: 'Kali' version: '$1' @@ -64,7 +64,7 @@ ########## # TencentOS (https://github.com/Tencent/TencentOS-kernel) ########## -- regex: 'Linux/(\d+[\.\d]*).+tlinux' +- regex: '.+tlinux(\d)' name: 'TencentOS' version: '$1' From 0cc4ed61244c2b516e22b6d0f33090180e456099 Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 31 Oct 2023 13:01:30 +0100 Subject: [PATCH 15/24] Improves version detection for CentOS --- Tests/Parser/fixtures/oss.yml | 8 ++++++++ regexes/oss.yml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Tests/Parser/fixtures/oss.yml b/Tests/Parser/fixtures/oss.yml index bf8ad65b5a..32c09d80e9 100644 --- a/Tests/Parser/fixtures/oss.yml +++ b/Tests/Parser/fixtures/oss.yml @@ -3950,3 +3950,11 @@ version: "7" platform: x64 family: GNU/Linux +- + user_agent: python-requests/2.7.0 CPython/2.7.3 Linux/2.6.18-308.el5 + os: + name: CentOS + short_name: CES + version: "5" + platform: + family: GNU/Linux diff --git a/regexes/oss.yml b/regexes/oss.yml index 5032186918..c3dd1c365a 100644 --- a/regexes/oss.yml +++ b/regexes/oss.yml @@ -569,7 +569,7 @@ name: 'CentOS Stream' version: '$1' -- regex: '.+.el(\d+(?:[_\.]\d+)*).(?:centos|x86_64)' +- regex: '.+.el(\d+(?:[_\.]\d+)*)' name: 'CentOS' version: '$1' From 101a380fc96472fb677dd20e265f29886b344876 Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 7 Nov 2023 10:52:58 +0100 Subject: [PATCH 16/24] Move links from comment to url and update some links --- regexes/client/libraries.yml | 46 +++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/regexes/client/libraries.yml b/regexes/client/libraries.yml index e27065d466..05342149fa 100644 --- a/regexes/client/libraries.yml +++ b/regexes/client/libraries.yml @@ -29,71 +29,85 @@ - regex: '^got(?:/(\d+\.[.\d]+))? \(' name: 'got' version: '$1' + url: 'https://github.com/sindresorhus/got' -# Typhoeus (https://github.com/typhoeus/typhoeus) +# Typhoeus - regex: 'Typhoeus' name: 'Typhoeus' version: '' + url: 'https://github.com/typhoeus/typhoeus' -# req (https://github.com/imroc/req) +# req - regex: 'req/v([\.\d]+)' name: 'req' version: '$1' + url: 'https://github.com/imroc/req' -# quic-go (https://github.com/lucas-clemente/quic-go) +# quic-go - regex: 'quic-go-HTTP/3' name: 'quic-go' version: '' + url: 'https://github.com/lucas-clemente/quic-go' -# Azure Data Factory (https://azure.microsoft.com/en-us/products/data-factory/) +# Azure Data Factory - regex: 'azure-data-factory(?:/(\d+[\.\d]+))?' name: 'Azure Data Factory' version: '$1' + url: 'https://azure.microsoft.com/en-us/products/data-factory/' -# Dart (https://dart.dev/) +# Dart - regex: 'Dart(?:/(\d+[\.\d]+))?' name: 'Dart' version: '$1' + url: 'https://dart.dev/' -# r-curl (https://github.com/jeroen/curl) +# r-curl - regex: 'r-curl(?:/(\d+[\.\d]+))?' name: 'r-curl' version: '$1' + url: 'https://github.com/jeroen/curl' -# HTTPX (https://www.python-httpx.org/) +# HTTPX - regex: 'python-httpx(?:/(\d+[\.\d]+))?' name: 'HTTPX' version: '$1' + url: 'https://www.python-httpx.org/' -# fasthttp (https://github.com/valyala/fasthttp) +# fasthttp - regex: 'fasthttp(?:/(\d+[\.\d]+))?' name: 'fasthttp' version: '$1' + url: 'https://github.com/valyala/fasthttp' -# GeoIP Update (https://github.com/maxmind/geoipupdate) +# GeoIP Update - regex: 'geoipupdate(?:/(\d+[\.\d]+))?' name: 'GeoIP Update' version: '$1' + url: 'https://github.com/maxmind/geoipupdate' -# PHP cURL Class (https://github.com/php-curl-class/php-curl-class) +# PHP cURL Class - regex: 'PHP-Curl-Class(?:/(\d+[\.\d]+))?' name: 'PHP cURL Class' version: '$1' + url: 'https://github.com/php-curl-class/php-curl-class' -# cPanel HTTP Client (https://www.cpanel.net/) +# cPanel HTTP Client - regex: 'Cpanel-HTTP-Client(?:/(\d+[\.\d]+))?' name: 'cPanel HTTP Client' version: '$1' + url: 'https://www.cpanel.net/' -# AnyEvent HTTP (http://software.schmorp.de/pkg/AnyEvent) +# AnyEvent HTTP - regex: 'AnyEvent-HTTP(?:/(\d+[\.\d]+))?' name: 'AnyEvent HTTP' version: '$1' + url: 'http://software.schmorp.de/pkg/AnyEvent' -# SlimerJS (https://www.slimerjs.org/) +# SlimerJS - regex: 'SlimerJS/(\d+[\.\d]+)' name: 'SlimerJS' version: '$1' + url: 'https://www.slimerjs.org/' - regex: 'Wget(?:/(\d+[\.\d]+))?' name: 'Wget' @@ -162,12 +176,12 @@ - regex: 'HTTP_Request2(?:/(\d+[\.\d]+))?' name: 'HTTP_Request2' version: '$1' - url: 'http://pear.php.net/package/http_request2' + url: 'https://pear.php.net/package/http_request2' - regex: 'Mechanize(?:/(\d+[\.\d]+))?' name: 'Mechanize' version: '$1' - url: 'http://github.com/sparklemotion/mechanize/' + url: 'https://github.com/sparklemotion/mechanize' - regex: 'aiohttp(?:/(\d+[\.\d]+))?' name: 'aiohttp' @@ -208,7 +222,7 @@ - regex: 'RestSharp/(\d+[\.\d]+)' name: 'RestSharp' version: '$1' - url: 'http://restsharp.org/' + url: 'https://github.com/restsharp/RestSharp' - regex: 'scalaj-http/(\d+[\.\d]+)' name: 'ScalaJ HTTP' From 56f7b805d0199b7595b0435d15823feae700823d Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 7 Nov 2023 11:33:24 +0100 Subject: [PATCH 17/24] Fix regex for Oracle Linux --- regexes/oss.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regexes/oss.yml b/regexes/oss.yml index dac0c943f7..38aad8b09d 100644 --- a/regexes/oss.yml +++ b/regexes/oss.yml @@ -8,7 +8,7 @@ ########## # Oracle Linux (https://www.oracle.com/linux/) ########## -- regex: '.+.el(\d+(?:[_\.]\d+)*)uek' +- regex: 'Linux/(?:\d+\.[\d.-]+)\.el(\d+(?:[_.]\d+)*)uek' name: 'Oracle Linux' version: '$1' From 62c436ce4f207a4eae5ad32d13b70935413d57b5 Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 7 Nov 2023 11:34:09 +0100 Subject: [PATCH 18/24] Fix regex for CentOS --- regexes/oss.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regexes/oss.yml b/regexes/oss.yml index 38aad8b09d..65f4773f00 100644 --- a/regexes/oss.yml +++ b/regexes/oss.yml @@ -569,7 +569,7 @@ name: 'CentOS Stream' version: '$1' -- regex: '.+.el(\d+(?:[_\.]\d+)*)' +- regex: 'Linux/(?:\d+\.[\d.-]+)\.el(\d+(?:[_.]\d+)*)' name: 'CentOS' version: '$1' From 977172b76a48d4eb7a633412585febe42a2bee27 Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 7 Nov 2023 11:37:09 +0100 Subject: [PATCH 19/24] Improve detection for generic bots --- regexes/bots.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regexes/bots.yml b/regexes/bots.yml index 668e42c5a9..eb959e256e 100644 --- a/regexes/bots.yml +++ b/regexes/bots.yml @@ -3502,5 +3502,5 @@ url: 'https://madb.zapto.org/bot.html' # Generic detections -- regex: '[a-z0-9\-_]*((? Date: Tue, 7 Nov 2023 11:42:58 +0100 Subject: [PATCH 20/24] Revert "Fix regex for CentOS" This reverts commit 62c436ce4f207a4eae5ad32d13b70935413d57b5. --- regexes/oss.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regexes/oss.yml b/regexes/oss.yml index 65f4773f00..38aad8b09d 100644 --- a/regexes/oss.yml +++ b/regexes/oss.yml @@ -569,7 +569,7 @@ name: 'CentOS Stream' version: '$1' -- regex: 'Linux/(?:\d+\.[\d.-]+)\.el(\d+(?:[_.]\d+)*)' +- regex: '.+.el(\d+(?:[_\.]\d+)*)' name: 'CentOS' version: '$1' From 2da17485e6edb191e7e0a61d0aab723542f9dfa8 Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 7 Nov 2023 11:43:12 +0100 Subject: [PATCH 21/24] Revert "Improves version detection for CentOS" This reverts commit 0cc4ed61244c2b516e22b6d0f33090180e456099. --- Tests/Parser/fixtures/oss.yml | 8 -------- regexes/oss.yml | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/Tests/Parser/fixtures/oss.yml b/Tests/Parser/fixtures/oss.yml index 3a2f225e26..6c50eeb516 100644 --- a/Tests/Parser/fixtures/oss.yml +++ b/Tests/Parser/fixtures/oss.yml @@ -3942,11 +3942,3 @@ version: "7" platform: x64 family: GNU/Linux -- - user_agent: python-requests/2.7.0 CPython/2.7.3 Linux/2.6.18-308.el5 - os: - name: CentOS - short_name: CES - version: "5" - platform: - family: GNU/Linux diff --git a/regexes/oss.yml b/regexes/oss.yml index 38aad8b09d..dae4138e66 100644 --- a/regexes/oss.yml +++ b/regexes/oss.yml @@ -569,7 +569,7 @@ name: 'CentOS Stream' version: '$1' -- regex: '.+.el(\d+(?:[_\.]\d+)*)' +- regex: '.+.el(\d+(?:[_\.]\d+)*).(?:centos|x86_64)' name: 'CentOS' version: '$1' From 6d7128cdd9e1bace02297c5fccc57a845d7da687 Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Tue, 7 Nov 2023 20:11:16 +0100 Subject: [PATCH 22/24] Change names --- Tests/Parser/Client/fixtures/library.yml | 6 +++--- regexes/client/libraries.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Tests/Parser/Client/fixtures/library.yml b/Tests/Parser/Client/fixtures/library.yml index b94e31041e..005b33802b 100644 --- a/Tests/Parser/Client/fixtures/library.yml +++ b/Tests/Parser/Client/fixtures/library.yml @@ -543,17 +543,17 @@ user_agent: msray-plus client: type: library - name: msray + name: Msray-Plus version: "" - user_agent: Slim Framework client: type: library - name: Slim + name: Slim Framework version: "" - user_agent: Fuzz Faster U Fool v1.5.0-dev client: type: library - name: Fuzz Faster U Fool + name: FFUF version: 1.5.0 diff --git a/regexes/client/libraries.yml b/regexes/client/libraries.yml index 05342149fa..a9cea7ae17 100644 --- a/regexes/client/libraries.yml +++ b/regexes/client/libraries.yml @@ -6,17 +6,17 @@ ############### - regex: 'Fuzz Faster U Fool v(\d+[\.\d]+)' - name: 'Fuzz Faster U Fool' + name: 'FFUF' version: '$1' url: 'https://github.com/ffuf/ffuf' - regex: 'Slim Framework' - name: 'Slim' + name: 'Slim Framework' version: '' url: 'https://www.slimframework.com/' - regex: 'msray-plus' - name: 'msray' + name: 'Msray-Plus' version: '' url: 'https://github.com/super-l/msray' From f88b6c2f85731441e2c76c1b8bbe541500d2b844 Mon Sep 17 00:00:00 2001 From: Liviu-Mihail Concioiu Date: Thu, 9 Nov 2023 13:59:46 +0100 Subject: [PATCH 23/24] Remove newline --- regexes/bots.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/regexes/bots.yml b/regexes/bots.yml index 03df17337c..fbfd6dc22a 100644 --- a/regexes/bots.yml +++ b/regexes/bots.yml @@ -3511,4 +3511,3 @@ # Generic detections - regex: '[a-z0-9\-_]*((? Date: Mon, 13 Nov 2023 23:29:01 +0100 Subject: [PATCH 24/24] Improves version detection for iOS and macOS --- Tests/Parser/fixtures/oss.yml | 8 ++++++++ regexes/oss.yml | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/Tests/Parser/fixtures/oss.yml b/Tests/Parser/fixtures/oss.yml index c7323fa76a..1d36ce719a 100644 --- a/Tests/Parser/fixtures/oss.yml +++ b/Tests/Parser/fixtures/oss.yml @@ -3966,3 +3966,11 @@ version: 1.0.0 platform: "" family: GNU/Linux +- + user_agent: Aloha/1 CFNetwork/1490.0.1 Darwin/23.2.0 + os: + name: iOS + short_name: IOS + version: "17.2" + platform: + family: iOS diff --git a/regexes/oss.yml b/regexes/oss.yml index b3836e070a..02df1d04c8 100644 --- a/regexes/oss.yml +++ b/regexes/oss.yml @@ -837,6 +837,8 @@ - regex: '^(?!com.apple.Safari.SearchHelper|Safari).*CFNetwork/.+ Darwin/(\d+[\.\d]+)(?!.*(?:x86_64|i386|PowerMac|Power%20Macintosh))' name: 'iOS' versions: + - regex: 'Darwin/23.2.0' + version: '17.2' - regex: 'Darwin/23.1.0' version: '17.1' - regex: 'Darwin/23.0.0' @@ -1062,6 +1064,8 @@ - regex: '(?:CFNetwork|StudioDisplay)/.+Darwin(?:/|; )(?:[\d\.]+).+(?:x86_64|i386|Power%20Macintosh)|(?:x86_64-apple-)?darwin(?:[\d\.]+)|PowerMac|com.apple.Safari.SearchHelper|^Safari' name: 'Mac' versions: + - regex: '(?:x86_64-apple-)?Darwin(?:/|; )?23.2.0' + version: '14.2' - regex: '(?:x86_64-apple-)?Darwin(?:/|; )?23.1.0' version: '14.1' - regex: '(?:x86_64-apple-)?Darwin(?:/|; )?23.0.0'