From e144e9b7fad7d78156db3918435e3655b9981ea6 Mon Sep 17 00:00:00 2001 From: Federico Fantini <122294644+federicofantini@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:37:42 +0200 Subject: [PATCH] Pdf uri extractor and pivoting (#2391) * uri extraction * added download file analyzer and pivot configs * fixed code review doctor * made code review changes added job creation check to avoid graph related issues * added abstract update method * fixed migration order * fixed validated_data dict access * fixed migrations order * fixed migrations order --- .../file_analyzers/pdf_info.py | 17 + ...100_analyzer_config_downloadfilefromuri.py | 295 ++++++++++++++++++ .../download_file_from_uri.py | 59 ++++ ...028_pivot_config_resubmitdownloadedfile.py | 148 +++++++++ .../0029_pivot_config_downloadfilefromuri.py | 148 +++++++++ api_app/pivots_manager/pivots/compare.py | 2 + api_app/pivots_manager/pivots/load_file.py | 15 + .../0048_playbook_config_download_file.py | 119 +++++++ api_app/serializers/job.py | 12 +- 9 files changed, 814 insertions(+), 1 deletion(-) create mode 100644 api_app/analyzers_manager/migrations/0100_analyzer_config_downloadfilefromuri.py create mode 100644 api_app/analyzers_manager/observable_analyzers/download_file_from_uri.py create mode 100644 api_app/pivots_manager/migrations/0028_pivot_config_resubmitdownloadedfile.py create mode 100644 api_app/pivots_manager/migrations/0029_pivot_config_downloadfilefromuri.py create mode 100644 api_app/pivots_manager/pivots/load_file.py create mode 100644 api_app/playbooks_manager/migrations/0048_playbook_config_download_file.py diff --git a/api_app/analyzers_manager/file_analyzers/pdf_info.py b/api_app/analyzers_manager/file_analyzers/pdf_info.py index 20cd29ea4e..c97c63bba2 100644 --- a/api_app/analyzers_manager/file_analyzers/pdf_info.py +++ b/api_app/analyzers_manager/file_analyzers/pdf_info.py @@ -18,6 +18,10 @@ class PDFInfo(FileAnalyzer): def flatten(list_of_lists: List[List[Any]]) -> List[Any]: return [item for sublist in list_of_lists for item in sublist] + @classmethod + def update(cls) -> bool: + pass + def run(self): self.results = {"peepdf": {}, "pdfid": {}} # the analysis fails only when BOTH fails @@ -25,6 +29,19 @@ def run(self): pdfid_success = self.__pdfid_analysis() if not peepdf_success and not pdfid_success: raise AnalyzerRunException("both peepdf and pdfid failed") + + # pivot uris in the pdf only if we have one page + if "reports" in self.results["pdfid"] and isinstance( + self.results["pdfid"]["reports"], list + ): + for elem in self.results["pdfid"]["reports"]: + if "/Page" in elem and elem["/Page"] == 1: + self.results["uris"] = [] + for s in self.results["peepdf"]["stats"]: + self.results["uris"].extend(s["uris"]) + + logger.info(f"extracted urls from file {self.md5}: {self.results['uris']}") + return self.results def __peepdf_analysis(self): diff --git a/api_app/analyzers_manager/migrations/0100_analyzer_config_downloadfilefromuri.py b/api_app/analyzers_manager/migrations/0100_analyzer_config_downloadfilefromuri.py new file mode 100644 index 0000000000..6a23e15925 --- /dev/null +++ b/api_app/analyzers_manager/migrations/0100_analyzer_config_downloadfilefromuri.py @@ -0,0 +1,295 @@ +from django.db import migrations +from django.db.models.fields.related_descriptors import ( + ForwardManyToOneDescriptor, + ForwardOneToOneDescriptor, + ManyToManyDescriptor, +) + +plugin = { + "python_module": { + "health_check_schedule": None, + "update_schedule": None, + "module": "download_file_from_uri.DownloadFileFromUri", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "DownloadFileFromUri", + "description": "performs an http request to an uri and download the file through the http proxy", + "disabled": False, + "soft_time_limit": 60, + "routing_key": "default", + "health_check_status": True, + "type": "observable", + "docker_based": False, + "maximum_tlp": "RED", + "observable_supported": ["url"], + "supported_filetypes": [], + "run_hash": False, + "run_hash_type": "", + "not_supported_filetypes": [], + "model": "analyzers_manager.AnalyzerConfig", +} + +params = [ + { + "python_module": { + "module": "download_file_from_uri.DownloadFileFromUri", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "http_proxy", + "type": "str", + "description": "http proxy url", + "is_secret": True, + "required": True, + }, + { + "python_module": { + "module": "download_file_from_uri.DownloadFileFromUri", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "header_user_agent", + "type": "str", + "description": "http header user-agent field", + "is_secret": False, + "required": True, + }, + { + "python_module": { + "module": "download_file_from_uri.DownloadFileFromUri", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "header_cookies", + "type": "str", + "description": "http header cookies field (e.g. $Version=1; Skin=new;)", + "is_secret": False, + "required": True, + }, + { + "python_module": { + "module": "download_file_from_uri.DownloadFileFromUri", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "header_content_type", + "type": "str", + "description": "http header content-type field", + "is_secret": False, + "required": True, + }, + { + "python_module": { + "module": "download_file_from_uri.DownloadFileFromUri", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "header_accept", + "type": "str", + "description": "http header accept field", + "is_secret": False, + "required": True, + }, + { + "python_module": { + "module": "download_file_from_uri.DownloadFileFromUri", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "timeout", + "type": "int", + "description": "http requests timeout", + "is_secret": False, + "required": True, + }, +] + +values = [ + { + "parameter": { + "python_module": { + "module": "download_file_from_uri.DownloadFileFromUri", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "header_user_agent", + "type": "str", + "description": "http header user-agent field", + "is_secret": False, + "required": True, + }, + "analyzer_config": "DownloadFileFromUri", + "connector_config": None, + "visualizer_config": None, + "ingestor_config": None, + "pivot_config": None, + "for_organization": False, + "value": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/125.0.2535.92", + "updated_at": "2024-06-19T10:23:03.145744Z", + "owner": None, + }, + { + "parameter": { + "python_module": { + "module": "download_file_from_uri.DownloadFileFromUri", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "header_cookies", + "type": "str", + "description": "http header cookies field (e.g. $Version=1; Skin=new;)", + "is_secret": False, + "required": True, + }, + "analyzer_config": "DownloadFileFromUri", + "connector_config": None, + "visualizer_config": None, + "ingestor_config": None, + "pivot_config": None, + "for_organization": False, + "value": "", + "updated_at": "2024-06-19T10:23:03.145744Z", + "owner": None, + }, + { + "parameter": { + "python_module": { + "module": "download_file_from_uri.DownloadFileFromUri", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "header_content_type", + "type": "str", + "description": "http header content-type field", + "is_secret": False, + "required": True, + }, + "analyzer_config": "DownloadFileFromUri", + "connector_config": None, + "visualizer_config": None, + "ingestor_config": None, + "pivot_config": None, + "for_organization": False, + "value": "application/octet-stream", + "updated_at": "2024-06-19T10:23:03.145744Z", + "owner": None, + }, + { + "parameter": { + "python_module": { + "module": "download_file_from_uri.DownloadFileFromUri", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "header_accept", + "type": "str", + "description": "http header accept field", + "is_secret": False, + "required": True, + }, + "analyzer_config": "DownloadFileFromUri", + "connector_config": None, + "visualizer_config": None, + "ingestor_config": None, + "pivot_config": None, + "for_organization": False, + "value": "application/octet-stream", + "updated_at": "2024-06-19T10:23:03.145744Z", + "owner": None, + }, + { + "parameter": { + "python_module": { + "module": "download_file_from_uri.DownloadFileFromUri", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "timeout", + "type": "int", + "description": "http requests timeout", + "is_secret": False, + "required": True, + }, + "analyzer_config": "DownloadFileFromUri", + "connector_config": None, + "visualizer_config": None, + "ingestor_config": None, + "pivot_config": None, + "for_organization": False, + "value": 50, + "updated_at": "2024-06-19T10:23:03.145744Z", + "owner": None, + }, +] + + +def _get_real_obj(Model, field, value): + def _get_obj(Model, other_model, value): + if isinstance(value, dict): + real_vals = {} + for key, real_val in value.items(): + real_vals[key] = _get_real_obj(other_model, key, real_val) + value = other_model.objects.get_or_create(**real_vals)[0] + # it is just the primary key serialized + else: + if isinstance(value, int): + if Model.__name__ == "PluginConfig": + value = other_model.objects.get(name=plugin["name"]) + else: + value = other_model.objects.get(pk=value) + else: + value = other_model.objects.get(name=value) + return value + + if ( + type(getattr(Model, field)) + in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor] + and value + ): + other_model = getattr(Model, field).get_queryset().model + value = _get_obj(Model, other_model, value) + elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value: + other_model = getattr(Model, field).rel.model + value = [_get_obj(Model, other_model, val) for val in value] + return value + + +def _create_object(Model, data): + mtm, no_mtm = {}, {} + for field, value in data.items(): + value = _get_real_obj(Model, field, value) + if type(getattr(Model, field)) is ManyToManyDescriptor: + mtm[field] = value + else: + no_mtm[field] = value + try: + o = Model.objects.get(**no_mtm) + except Model.DoesNotExist: + o = Model(**no_mtm) + o.full_clean() + o.save() + for field, value in mtm.items(): + attribute = getattr(o, field) + if value is not None: + attribute.set(value) + return False + return True + + +def migrate(apps, schema_editor): + Parameter = apps.get_model("api_app", "Parameter") + PluginConfig = apps.get_model("api_app", "PluginConfig") + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + if not Model.objects.filter(name=plugin["name"]).exists(): + exists = _create_object(Model, plugin) + if not exists: + for param in params: + _create_object(Parameter, param) + for value in values: + _create_object(PluginConfig, value) + + +def reverse_migrate(apps, schema_editor): + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + Model.objects.get(name=plugin["name"]).delete() + + +class Migration(migrations.Migration): + atomic = False + dependencies = [ + ("api_app", "0062_alter_parameter_python_module"), + ("analyzers_manager", "0099_analyzer_config_spamhaus_wqs"), + ] + + operations = [migrations.RunPython(migrate, reverse_migrate)] diff --git a/api_app/analyzers_manager/observable_analyzers/download_file_from_uri.py b/api_app/analyzers_manager/observable_analyzers/download_file_from_uri.py new file mode 100644 index 0000000000..a52f4a600e --- /dev/null +++ b/api_app/analyzers_manager/observable_analyzers/download_file_from_uri.py @@ -0,0 +1,59 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. +import base64 +import logging + +import requests + +from api_app.analyzers_manager.classes import ObservableAnalyzer +from api_app.analyzers_manager.exceptions import AnalyzerRunException + +logger = logging.getLogger(__name__) + + +class DownloadFileFromUri(ObservableAnalyzer): + _http_proxy: str + header_user_agent: str + header_cookies: str + header_content_type: str + header_accept: str + timeout: int + + @classmethod + def update(cls) -> bool: + pass + + def run(self): + result = {"stored_base64": ""} + + proxies = {"http": self._http_proxy} if self._http_proxy else {} + headers = { + "User-Agent": self.header_user_agent, + "Cookie": self.header_cookies, + "Content-type": self.header_content_type, + "Accept": self.header_accept, + } + + try: + r = requests.get( + self.observable_name, + headers=headers, + proxies=proxies, + timeout=self.timeout, + ) + except Exception as e: + raise AnalyzerRunException(f"requests exception: {e}") + else: + if r.content: + if "text/html" not in r.headers["Content-Type"]: + result["stored_base64"] = base64.b64encode(r.content).decode( + "ascii" + ) + else: + logger.info( + f"discarded text/html response for {self.observable_name}" + ) + else: + logger.info(f"no response content for {self.observable_name}") + + return result diff --git a/api_app/pivots_manager/migrations/0028_pivot_config_resubmitdownloadedfile.py b/api_app/pivots_manager/migrations/0028_pivot_config_resubmitdownloadedfile.py new file mode 100644 index 0000000000..ed77a9cd29 --- /dev/null +++ b/api_app/pivots_manager/migrations/0028_pivot_config_resubmitdownloadedfile.py @@ -0,0 +1,148 @@ +from django.db import migrations +from django.db.models.fields.related_descriptors import ( + ForwardManyToOneDescriptor, + ForwardOneToOneDescriptor, + ManyToManyDescriptor, +) + +plugin = { + "python_module": { + "health_check_schedule": None, + "update_schedule": None, + "module": "load_file.LoadFile", + "base_path": "api_app.pivots_manager.pivots", + }, + "related_analyzer_configs": ["DownloadFileFromUri"], + "related_connector_configs": [], + "playbook_to_execute": "FREE_TO_USE_ANALYZERS", + "name": "ResubmitDownloadedFile", + "description": "Pivot for plugins DownloadFileFromUri " + "that executes playbook FREE_TO_USE_ANALYZERS", + "disabled": False, + "soft_time_limit": 60, + "routing_key": "stored_base64", + "health_check_status": True, + "model": "pivots_manager.PivotConfig", +} + +params = [ + { + "python_module": { + "module": "load_file.LoadFile", + "base_path": "api_app.pivots_manager.pivots", + }, + "name": "field_to_compare", + "type": "str", + "description": "Dotted path to the field", + "is_secret": False, + "required": True, + } +] + +values = [ + { + "parameter": { + "python_module": { + "module": "load_file.LoadFile", + "base_path": "api_app.pivots_manager.pivots", + }, + "name": "field_to_compare", + "type": "str", + "description": "Dotted path to the field", + "is_secret": False, + "required": True, + }, + "analyzer_config": None, + "connector_config": None, + "visualizer_config": None, + "ingestor_config": None, + "pivot_config": "ResubmitDownloadedFile", + "for_organization": False, + "value": "stored_base64", + "updated_at": "2024-06-19T12:30:03.194133Z", + "owner": None, + } +] + + +def _get_real_obj(Model, field, value): + def _get_obj(Model, other_model, value): + if isinstance(value, dict): + real_vals = {} + for key, real_val in value.items(): + real_vals[key] = _get_real_obj(other_model, key, real_val) + value = other_model.objects.get_or_create(**real_vals)[0] + # it is just the primary key serialized + else: + if isinstance(value, int): + if Model.__name__ == "PluginConfig": + value = other_model.objects.get(name=plugin["name"]) + else: + value = other_model.objects.get(pk=value) + else: + value = other_model.objects.get(name=value) + return value + + if ( + type(getattr(Model, field)) + in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor] + and value + ): + other_model = getattr(Model, field).get_queryset().model + value = _get_obj(Model, other_model, value) + elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value: + other_model = getattr(Model, field).rel.model + value = [_get_obj(Model, other_model, val) for val in value] + return value + + +def _create_object(Model, data): + mtm, no_mtm = {}, {} + for field, value in data.items(): + value = _get_real_obj(Model, field, value) + if type(getattr(Model, field)) is ManyToManyDescriptor: + mtm[field] = value + else: + no_mtm[field] = value + try: + o = Model.objects.get(**no_mtm) + except Model.DoesNotExist: + o = Model(**no_mtm) + o.full_clean() + o.save() + for field, value in mtm.items(): + attribute = getattr(o, field) + if value is not None: + attribute.set(value) + return False + return True + + +def migrate(apps, schema_editor): + Parameter = apps.get_model("api_app", "Parameter") + PluginConfig = apps.get_model("api_app", "PluginConfig") + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + if not Model.objects.filter(name=plugin["name"]).exists(): + exists = _create_object(Model, plugin) + if not exists: + for param in params: + _create_object(Parameter, param) + for value in values: + _create_object(PluginConfig, value) + + +def reverse_migrate(apps, schema_editor): + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + Model.objects.get(name=plugin["name"]).delete() + + +class Migration(migrations.Migration): + atomic = False + dependencies = [ + ("pivots_manager", "0027_pivot_config_takedownrequesttoabuseip"), + ("analyzers_manager", "0100_analyzer_config_downloadfilefromuri"), + ] + + operations = [migrations.RunPython(migrate, reverse_migrate)] diff --git a/api_app/pivots_manager/migrations/0029_pivot_config_downloadfilefromuri.py b/api_app/pivots_manager/migrations/0029_pivot_config_downloadfilefromuri.py new file mode 100644 index 0000000000..db3cf2cb95 --- /dev/null +++ b/api_app/pivots_manager/migrations/0029_pivot_config_downloadfilefromuri.py @@ -0,0 +1,148 @@ +from django.db import migrations +from django.db.models.fields.related_descriptors import ( + ForwardManyToOneDescriptor, + ForwardOneToOneDescriptor, + ManyToManyDescriptor, +) + +plugin = { + "python_module": { + "health_check_schedule": None, + "update_schedule": None, + "module": "compare.Compare", + "base_path": "api_app.pivots_manager.pivots", + }, + "related_analyzer_configs": ["PDF_Info"], + "related_connector_configs": [], + "playbook_to_execute": "Download_File", + "name": "DownloadFileFromUri", + "description": "Pivot for plugins PDF_Info that executes playbook Download_File", + "disabled": False, + "soft_time_limit": 60, + "routing_key": "uris", + "health_check_status": True, + "model": "pivots_manager.PivotConfig", +} + +params = [ + { + "python_module": { + "module": "compare.Compare", + "base_path": "api_app.pivots_manager.pivots", + }, + "name": "field_to_compare", + "type": "str", + "description": "Dotted path to the field", + "is_secret": False, + "required": True, + } +] + +values = [ + { + "parameter": { + "python_module": { + "module": "compare.Compare", + "base_path": "api_app.pivots_manager.pivots", + }, + "name": "field_to_compare", + "type": "str", + "description": "Dotted path to the field", + "is_secret": False, + "required": True, + }, + "analyzer_config": None, + "connector_config": None, + "visualizer_config": None, + "ingestor_config": None, + "pivot_config": "DownloadFileFromUri", + "for_organization": False, + "value": "uris", + "updated_at": "2024-06-19T12:28:27.945297Z", + "owner": None, + } +] + + +def _get_real_obj(Model, field, value): + def _get_obj(Model, other_model, value): + if isinstance(value, dict): + real_vals = {} + for key, real_val in value.items(): + real_vals[key] = _get_real_obj(other_model, key, real_val) + value = other_model.objects.get_or_create(**real_vals)[0] + # it is just the primary key serialized + else: + if isinstance(value, int): + if Model.__name__ == "PluginConfig": + value = other_model.objects.get(name=plugin["name"]) + else: + value = other_model.objects.get(pk=value) + else: + value = other_model.objects.get(name=value) + return value + + if ( + type(getattr(Model, field)) + in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor] + and value + ): + other_model = getattr(Model, field).get_queryset().model + value = _get_obj(Model, other_model, value) + elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value: + other_model = getattr(Model, field).rel.model + value = [_get_obj(Model, other_model, val) for val in value] + return value + + +def _create_object(Model, data): + mtm, no_mtm = {}, {} + for field, value in data.items(): + value = _get_real_obj(Model, field, value) + if type(getattr(Model, field)) is ManyToManyDescriptor: + mtm[field] = value + else: + no_mtm[field] = value + try: + o = Model.objects.get(**no_mtm) + except Model.DoesNotExist: + o = Model(**no_mtm) + o.full_clean() + o.save() + for field, value in mtm.items(): + attribute = getattr(o, field) + if value is not None: + attribute.set(value) + return False + return True + + +def migrate(apps, schema_editor): + Parameter = apps.get_model("api_app", "Parameter") + PluginConfig = apps.get_model("api_app", "PluginConfig") + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + if not Model.objects.filter(name=plugin["name"]).exists(): + exists = _create_object(Model, plugin) + if not exists: + for param in params: + _create_object(Parameter, param) + for value in values: + _create_object(PluginConfig, value) + + +def reverse_migrate(apps, schema_editor): + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + Model.objects.get(name=plugin["name"]).delete() + + +class Migration(migrations.Migration): + atomic = False + dependencies = [ + ("api_app", "0062_alter_parameter_python_module"), + ("playbooks_manager", "0048_playbook_config_download_file"), + ("pivots_manager", "0028_pivot_config_resubmitdownloadedfile"), + ] + + operations = [migrations.RunPython(migrate, reverse_migrate)] diff --git a/api_app/pivots_manager/pivots/compare.py b/api_app/pivots_manager/pivots/compare.py index 5df88133e0..dff23158aa 100644 --- a/api_app/pivots_manager/pivots/compare.py +++ b/api_app/pivots_manager/pivots/compare.py @@ -29,6 +29,8 @@ def _get_value(self, field: str) -> Any: if isinstance(content, (int, dict)): raise ValueError(f"You can't use a {type(content)} as pivot") + if not content: + raise ValueError("Empty value") return content def should_run(self) -> Tuple[bool, Optional[str]]: diff --git a/api_app/pivots_manager/pivots/load_file.py b/api_app/pivots_manager/pivots/load_file.py new file mode 100644 index 0000000000..ad116aade0 --- /dev/null +++ b/api_app/pivots_manager/pivots/load_file.py @@ -0,0 +1,15 @@ +import base64 +from typing import Any + +from api_app.pivots_manager.pivots.compare import Compare + + +class LoadFile(Compare): + field_to_compare: str + + @classmethod + def update(cls) -> bool: + pass + + def get_value_to_pivot_to(self) -> Any: + return base64.b64decode(self._value) diff --git a/api_app/playbooks_manager/migrations/0048_playbook_config_download_file.py b/api_app/playbooks_manager/migrations/0048_playbook_config_download_file.py new file mode 100644 index 0000000000..e64c312b8f --- /dev/null +++ b/api_app/playbooks_manager/migrations/0048_playbook_config_download_file.py @@ -0,0 +1,119 @@ +from django.db import migrations +from django.db.models.fields.related_descriptors import ( + ForwardManyToOneDescriptor, + ForwardOneToOneDescriptor, + ManyToManyDescriptor, +) + +plugin = { + "analyzers": ["DownloadFileFromUri"], + "connectors": [], + "pivots": ["ResubmitDownloadedFile"], + "for_organization": False, + "name": "Download_File", + "description": "A playbook containing the DownloadFileFromUri Analyzer.", + "disabled": False, + "type": ["url"], + "runtime_configuration": { + "pivots": {}, + "analyzers": {}, + "connectors": {}, + "visualizers": {}, + }, + "scan_mode": 1, + "scan_check_time": "1 00:00:00", + "tlp": "RED", + "starting": True, + "owner": None, + "tags": [], + "model": "playbooks_manager.PlaybookConfig", +} + +params = [] + +values = [] + + +def _get_real_obj(Model, field, value): + def _get_obj(Model, other_model, value): + if isinstance(value, dict): + real_vals = {} + for key, real_val in value.items(): + real_vals[key] = _get_real_obj(other_model, key, real_val) + value = other_model.objects.get_or_create(**real_vals)[0] + # it is just the primary key serialized + else: + if isinstance(value, int): + if Model.__name__ == "PluginConfig": + value = other_model.objects.get(name=plugin["name"]) + else: + value = other_model.objects.get(pk=value) + else: + value = other_model.objects.get(name=value) + return value + + if ( + type(getattr(Model, field)) + in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor] + and value + ): + other_model = getattr(Model, field).get_queryset().model + value = _get_obj(Model, other_model, value) + elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value: + other_model = getattr(Model, field).rel.model + value = [_get_obj(Model, other_model, val) for val in value] + return value + + +def _create_object(Model, data): + mtm, no_mtm = {}, {} + for field, value in data.items(): + value = _get_real_obj(Model, field, value) + if type(getattr(Model, field)) is ManyToManyDescriptor: + mtm[field] = value + else: + no_mtm[field] = value + try: + o = Model.objects.get(**no_mtm) + except Model.DoesNotExist: + o = Model(**no_mtm) + o.full_clean() + o.save() + for field, value in mtm.items(): + attribute = getattr(o, field) + if value is not None: + attribute.set(value) + return False + return True + + +def migrate(apps, schema_editor): + Parameter = apps.get_model("api_app", "Parameter") + PluginConfig = apps.get_model("api_app", "PluginConfig") + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + if not Model.objects.filter(name=plugin["name"]).exists(): + exists = _create_object(Model, plugin) + if not exists: + for param in params: + _create_object(Parameter, param) + for value in values: + _create_object(PluginConfig, value) + + +def reverse_migrate(apps, schema_editor): + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + Model.objects.get(name=plugin["name"]).delete() + + +class Migration(migrations.Migration): + atomic = False + dependencies = [ + ("api_app", "0062_alter_parameter_python_module"), + ("playbooks_manager", "0047_add_crt_sh_to_free_to_use"), + ("analyzers_manager", "0100_analyzer_config_downloadfilefromuri"), + ("pivots_manager", "0028_pivot_config_resubmitdownloadedfile"), + ] + + operations = [migrations.RunPython(migrate, reverse_migrate)] diff --git a/api_app/serializers/job.py b/api_app/serializers/job.py index 5972d9c021..71e28987d2 100644 --- a/api_app/serializers/job.py +++ b/api_app/serializers/job.py @@ -344,7 +344,17 @@ def create(self, validated_data: Dict) -> Job: warnings = validated_data.pop("warnings") send_task = validated_data.pop("send_task", False) parent_job = validated_data.pop("parent_job", None) - if validated_data["scan_mode"] == ScanMode.CHECK_PREVIOUS_ANALYSIS.value: + + # if we have a parent job and a new playbook to excute force new analysis + # in order to avoid graph related issues + if validated_data[ + "scan_mode" + ] == ScanMode.CHECK_PREVIOUS_ANALYSIS.value and not ( + "parent" in validated_data + and validated_data["parent"] + and "playbook_to_execute" in validated_data + and validated_data["playbook_to_execute"] + ): try: return self.check_previous_jobs(validated_data) except self.Meta.model.DoesNotExist: