From beea7a22c40532058fe036e3a29d050054300a9b Mon Sep 17 00:00:00 2001 From: Moon Patel Date: Fri, 22 Mar 2024 21:35:39 +0530 Subject: [PATCH 1/3] fixes 1886 observable analyzer Tor_Nodes_DanMeUk added analyzer code for Tor_Node_DanMeUk migrations for Tor_Nodes_DanMeUk analyzer added Tor_Nodes_DanMeUk analyzer to FREE_TO_USE_ANALYZERS updated docs checking pre-commit --- .../0071_analyzer_config_tor_nodes_danmeuk.py | 101 ++++++++++++++++++ .../observable_analyzers/tor_nodes_danmeuk.py | 87 +++++++++++++++ ..._tor_nodes_danmeuk_analyzer_free_to_use.py | 34 ++++++ docs/source/Usage.md | 1 + 4 files changed, 223 insertions(+) create mode 100644 api_app/analyzers_manager/migrations/0071_analyzer_config_tor_nodes_danmeuk.py create mode 100644 api_app/analyzers_manager/observable_analyzers/tor_nodes_danmeuk.py create mode 100644 api_app/playbooks_manager/migrations/0029_add_tor_nodes_danmeuk_analyzer_free_to_use.py diff --git a/api_app/analyzers_manager/migrations/0071_analyzer_config_tor_nodes_danmeuk.py b/api_app/analyzers_manager/migrations/0071_analyzer_config_tor_nodes_danmeuk.py new file mode 100644 index 0000000000..a1cf0b6775 --- /dev/null +++ b/api_app/analyzers_manager/migrations/0071_analyzer_config_tor_nodes_danmeuk.py @@ -0,0 +1,101 @@ +from django.db import migrations +from django.db.models.fields.related_descriptors import ( + ForwardManyToOneDescriptor, + ForwardOneToOneDescriptor, + ManyToManyDescriptor, +) + +plugin = {'python_module': {'health_check_schedule': None, 'update_schedule': {'minute': '0', 'hour': '*', 'day_of_week': '*', 'day_of_month': '*', 'month_of_year': '*'}, 'module': 'tor_nodes_danmeuk.TorNodesDanMeUK', 'base_path': 'api_app.analyzers_manager.observable_analyzers'}, 'name': 'Tor_Nodes_DanMeUk', 'description': 'check if an IP is a Tor Node', 'disabled': False, 'soft_time_limit': 30, 'routing_key': 'default', 'health_check_status': True, 'type': 'observable', 'docker_based': False, 'maximum_tlp': 'RED', 'observable_supported': ['ip'], 'supported_filetypes': [], 'run_hash': False, 'run_hash_type': '', 'not_supported_filetypes': [], 'model': 'analyzers_manager.AnalyzerConfig'} + +params = [] + +values = [] + + +def _get_real_obj(Model, field, value): + def _get_obj(Model, other_model, value): + if isinstance(value, dict): + real_vals = {} + for key, real_val in value.items(): + real_vals[key] = _get_real_obj(other_model, key, real_val) + value = other_model.objects.get_or_create(**real_vals)[0] + # it is just the primary key serialized + else: + if isinstance(value, int): + if Model.__name__ == "PluginConfig": + value = other_model.objects.get(name=plugin["name"]) + else: + value = other_model.objects.get(pk=value) + else: + value = other_model.objects.get(name=value) + return value + + if ( + type(getattr(Model, field)) + in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor] + and value + ): + other_model = getattr(Model, field).get_queryset().model + value = _get_obj(Model, other_model, value) + elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value: + other_model = getattr(Model, field).rel.model + value = [_get_obj(Model, other_model, val) for val in value] + return value + +def _create_object(Model, data): + mtm, no_mtm = {}, {} + for field, value in data.items(): + value = _get_real_obj(Model, field, value) + if type(getattr(Model, field)) is ManyToManyDescriptor: + mtm[field] = value + else: + no_mtm[field] = value + try: + o = Model.objects.get(**no_mtm) + except Model.DoesNotExist: + o = Model(**no_mtm) + o.full_clean() + o.save() + for field, value in mtm.items(): + attribute = getattr(o, field) + if value is not None: + attribute.set(value) + return False + return True + +def migrate(apps, schema_editor): + Parameter = apps.get_model("api_app", "Parameter") + PluginConfig = apps.get_model("api_app", "PluginConfig") + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + if not Model.objects.filter(name=plugin["name"]).exists(): + exists = _create_object(Model, plugin) + if not exists: + for param in params: + _create_object(Parameter, param) + for value in values: + _create_object(PluginConfig, value) + + + +def reverse_migrate(apps, schema_editor): + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + Model.objects.get(name=plugin["name"]).delete() + + + +class Migration(migrations.Migration): + atomic = False + dependencies = [ + ('api_app', '0061_job_depth_analysis'), + ('analyzers_manager', '0070_urlhaus_threatfox_disable_param'), + ] + + operations = [ + migrations.RunPython( + migrate, reverse_migrate + ) + ] + + \ No newline at end of file diff --git a/api_app/analyzers_manager/observable_analyzers/tor_nodes_danmeuk.py b/api_app/analyzers_manager/observable_analyzers/tor_nodes_danmeuk.py new file mode 100644 index 0000000000..2c767ac3fd --- /dev/null +++ b/api_app/analyzers_manager/observable_analyzers/tor_nodes_danmeuk.py @@ -0,0 +1,87 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import logging +import os + +import requests +from django.conf import settings + +from api_app.analyzers_manager import classes +from api_app.analyzers_manager.exceptions import AnalyzerRunException +from tests.mock_utils import MockUpResponse, if_mock_connections, patch + +logger = logging.getLogger(__name__) + +db_name = "tor_nodes_addresses.txt" +database_location = f"{settings.MEDIA_ROOT}/{db_name}" + + +class TorNodesDanMeUK(classes.ObservableAnalyzer): + def run(self): + result = {"found": False} + if not os.path.isfile(database_location) and not self.update(): + raise AnalyzerRunException("Failed extraction of tor db") + + if not os.path.exists(database_location): + raise AnalyzerRunException( + f"database location {database_location} does not exist" + ) + + with open(database_location, "r", encoding="utf-8") as f: + db = f.read() + + db_list = db.split("\n") + if self.observable_name in db_list: + result["found"] = True + + return result + + @classmethod + def update(cls): + try: + logger.info("starting download of tor nodes from https://dan.me.uk") + url = "https://www.dan.me.uk/torlist/?full" + r = requests.get(url) + r.raise_for_status() + + data_extracted = r.content.decode() + tor_nodes_list = data_extracted.split("\n") + + with open(database_location, "w", encoding="utf-8") as f: + for ip in tor_nodes_list: + if ip: + f.write(f"{ip}\n") + + if not os.path.exists(database_location): + return False + + logger.info("ended download of tor nodes from https://dan.me.uk") + return True + except Exception as e: + logger.exception(e) + + return False + + @classmethod + def _monkeypatch(cls): + patches = [ + if_mock_connections( + patch( + "requests.get", + return_value=MockUpResponse( + {}, + 200, + content=b"""100.10.37.131 +100.14.156.183 +100.16.153.149 +100.4.55.171 +100.8.8.137 +101.100.141.137 +101.55.125.10 +102.119.243.196""", + ), + ), + ) + ] + return super()._monkeypatch(patches=patches) diff --git a/api_app/playbooks_manager/migrations/0029_add_tor_nodes_danmeuk_analyzer_free_to_use.py b/api_app/playbooks_manager/migrations/0029_add_tor_nodes_danmeuk_analyzer_free_to_use.py new file mode 100644 index 0000000000..e7010c1e11 --- /dev/null +++ b/api_app/playbooks_manager/migrations/0029_add_tor_nodes_danmeuk_analyzer_free_to_use.py @@ -0,0 +1,34 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + + +from django.db import migrations + + +def migrate(apps, schema_editor): + playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig") + AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig") + pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS") + pc.analyzers.add(AnalyzerConfig.objects.get(name="Tor_Nodes_DanMeUk").id) + pc.full_clean() + pc.save() + + +def reverse_migrate(apps, schema_editor): + playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig") + AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig") + pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS") + pc.analyzers.remove(AnalyzerConfig.objects.get(name="Tor_Nodes_DanMeUk").id) + pc.full_clean() + pc.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("playbooks_manager", "0028_add_bgp_ranking_to_free_to_use"), + ("analyzers_manager", "0071_analyzer_config_tor_nodes_danmeuk"), + ] + + operations = [ + migrations.RunPython(migrate, reverse_migrate), + ] diff --git a/docs/source/Usage.md b/docs/source/Usage.md index 13573cb267..766f5fa112 100644 --- a/docs/source/Usage.md +++ b/docs/source/Usage.md @@ -293,6 +293,7 @@ The following is the list of the available analyzers you can run out-of-the-box. * `TalosReputation`: check an IP reputation from [Talos](https://talosintelligence.com/reputation_center/) * `ThreatFox`: search for an IOC in [ThreatFox](https://threatfox.abuse.ch/api/)'s database * `Threatminer`: retrieve data from [Threatminer](https://www.threatminer.org/) API +* `TorNodesDanMeUk`: check if an IP is a Tor Node * `TorProject`: check if an IP is a Tor Exit Node * `Triage_Search`: Search for reports of observables or upload from URL on triage cloud * `Tranco`: Check if a domain is in the latest [Tranco](https://tranco-list.eu/) ranking top sites list From e74873fcd1a608e64f82754ad822cfbeb8c16537 Mon Sep 17 00:00:00 2001 From: Moon Patel Date: Mon, 25 Mar 2024 15:50:52 +0530 Subject: [PATCH 2/3] Updated Usage.md --- docs/source/Usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/Usage.md b/docs/source/Usage.md index 766f5fa112..fc56915826 100644 --- a/docs/source/Usage.md +++ b/docs/source/Usage.md @@ -293,7 +293,7 @@ The following is the list of the available analyzers you can run out-of-the-box. * `TalosReputation`: check an IP reputation from [Talos](https://talosintelligence.com/reputation_center/) * `ThreatFox`: search for an IOC in [ThreatFox](https://threatfox.abuse.ch/api/)'s database * `Threatminer`: retrieve data from [Threatminer](https://www.threatminer.org/) API -* `TorNodesDanMeUk`: check if an IP is a Tor Node +* `TorNodesDanMeUk`: check if an IP is a Tor Node using a list of all Tor nodes provided by [dan.me.uk](https://www.dan.me.uk/tornodes) * `TorProject`: check if an IP is a Tor Exit Node * `Triage_Search`: Search for reports of observables or upload from URL on triage cloud * `Tranco`: Check if a domain is in the latest [Tranco](https://tranco-list.eu/) ranking top sites list From eaa3cef9ef6af55243e523e5e559d481fa0d50b8 Mon Sep 17 00:00:00 2001 From: Matteo Lodi <30625432+mlodic@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:40:09 +0100 Subject: [PATCH 3/3] Update api_app/analyzers_manager/migrations/0071_analyzer_config_tor_nodes_danmeuk.py --- .../migrations/0071_analyzer_config_tor_nodes_danmeuk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/analyzers_manager/migrations/0071_analyzer_config_tor_nodes_danmeuk.py b/api_app/analyzers_manager/migrations/0071_analyzer_config_tor_nodes_danmeuk.py index a1cf0b6775..1a8969e7cc 100644 --- a/api_app/analyzers_manager/migrations/0071_analyzer_config_tor_nodes_danmeuk.py +++ b/api_app/analyzers_manager/migrations/0071_analyzer_config_tor_nodes_danmeuk.py @@ -5,7 +5,7 @@ ManyToManyDescriptor, ) -plugin = {'python_module': {'health_check_schedule': None, 'update_schedule': {'minute': '0', 'hour': '*', 'day_of_week': '*', 'day_of_month': '*', 'month_of_year': '*'}, 'module': 'tor_nodes_danmeuk.TorNodesDanMeUK', 'base_path': 'api_app.analyzers_manager.observable_analyzers'}, 'name': 'Tor_Nodes_DanMeUk', 'description': 'check if an IP is a Tor Node', 'disabled': False, 'soft_time_limit': 30, 'routing_key': 'default', 'health_check_status': True, 'type': 'observable', 'docker_based': False, 'maximum_tlp': 'RED', 'observable_supported': ['ip'], 'supported_filetypes': [], 'run_hash': False, 'run_hash_type': '', 'not_supported_filetypes': [], 'model': 'analyzers_manager.AnalyzerConfig'} +plugin = {'python_module': {'health_check_schedule': None, 'update_schedule': {'minute': '0', 'hour': '*/6', 'day_of_week': '*', 'day_of_month': '*', 'month_of_year': '*'}, 'module': 'tor_nodes_danmeuk.TorNodesDanMeUK', 'base_path': 'api_app.analyzers_manager.observable_analyzers'}, 'name': 'Tor_Nodes_DanMeUk', 'description': 'check if an IP is a Tor Node', 'disabled': False, 'soft_time_limit': 30, 'routing_key': 'default', 'health_check_status': True, 'type': 'observable', 'docker_based': False, 'maximum_tlp': 'RED', 'observable_supported': ['ip'], 'supported_filetypes': [], 'run_hash': False, 'run_hash_type': '', 'not_supported_filetypes': [], 'model': 'analyzers_manager.AnalyzerConfig'} params = []