From 19e30b51bf340a401a36f30029c9a28e4f7747d3 Mon Sep 17 00:00:00 2001 From: 0ssigeno Date: Fri, 22 Mar 2024 11:39:12 +0100 Subject: [PATCH 01/10] Improve values retrieval Signed-off-by: 0ssigeno --- api_app/classes.py | 13 ++++++---- api_app/models.py | 59 ++++++++++++--------------------------------- api_app/queryset.py | 52 +++++++++++++++++++++++++++++++++++---- 3 files changed, 71 insertions(+), 53 deletions(-) diff --git a/api_app/classes.py b/api_app/classes.py index c4f576a304..b08e218d1a 100644 --- a/api_app/classes.py +++ b/api_app/classes.py @@ -81,14 +81,17 @@ def __repr__(self): return f"({self.__class__.__name__}, job: #{self.job_id})" def config(self, runtime_configuration: typing.Dict): - for param, value in self._config.read_params( + self.__parameters = self._config.read_configured_params( self._user, runtime_configuration - ).items(): - attribute_name = f"_{param.name}" if param.is_secret else param.name - setattr(self, attribute_name, value) + ) + for parameter in self.__parameters: + attribute_name = ( + f"_{parameter.name}" if parameter.is_secret else parameter.name + ) + setattr(self, attribute_name, parameter.value) logger.debug( f"Adding to {self.__class__.__name__} " - f"param {attribute_name} with value {value} " + f"param {attribute_name} with value {parameter.value} " ) def before_run(self): diff --git a/api_app/models.py b/api_app/models.py index f940e9eab2..cc0e21fd44 100644 --- a/api_app/models.py +++ b/api_app/models.py @@ -652,20 +652,6 @@ def refresh_cache_keys(self): config: PythonConfig config.refresh_cache_keys() - def get_valid_value_for_test(self): - if not settings.STAGE_CI and not settings.MOCK_CONNECTIONS: - raise PluginConfig.DoesNotExist - if "url" in self.name: - return "https://intelowl.com" - elif "pdns_credentials" == self.name: - return "user|pwd" - elif "test" in self.name: - raise PluginConfig.DoesNotExist - elif self.type == ParamTypes.INT.value: - return 10 - else: - return "test" - @cached_property def config_class(self) -> Type["PythonConfig"]: return self.python_module.python_class.config_model @@ -1128,10 +1114,8 @@ def plugin_type(cls) -> str: def _get_params(self, user: User, runtime_configuration: Dict) -> Dict[str, Any]: return { - parameter.name: value - for parameter, value in self.read_params( - user, runtime_configuration - ).items() + parameter.name: parameter.value + for parameter in self.read_configured_params(user, runtime_configuration) if not parameter.is_secret } @@ -1227,33 +1211,22 @@ def _is_configured(self, user: User = None) -> bool: def config_exception(cls): raise NotImplementedError() - def read_params( + def read_configured_params( self, user: User = None, config_runtime: Dict = None - ) -> Dict[Parameter, Any]: - # priority - # 1 - Runtime config - # 2 - Value inside the db - result = {} - for param in self.parameters.annotate_configured( + ) -> ParameterQuerySet: + params = self.parameters.annotate_configured( self, user - ).annotate_value_for_user(self, user): - param: Parameter - if param.name in config_runtime: - result[param] = config_runtime[param.name] - else: - if param.configured: - result[param] = param.value - else: - if settings.STAGE_CI or settings.MOCK_CONNECTIONS: - result[param] = param.get_valid_value_for_test() - continue - if param.required: - raise TypeError( - f"Required param {param.name} " - f"of plugin {param.python_module.module}" - " does not have a valid value" - ) - return result + ).annotate_value_for_user(self, user, config_runtime) + not_configured_params = params.filter(required=True, value__isnull=True) + if not_configured_params.exists(): + param = not_configured_params.first() + raise TypeError( + f"Required param {param.name} " + f"of plugin {param.python_module.module}" + " does not have a valid value" + ) + + return params def generate_health_check_periodic_task(self): from intel_owl.tasks import health_check diff --git a/api_app/queryset.py b/api_app/queryset.py index 3c75a797ab..b216a818ee 100644 --- a/api_app/queryset.py +++ b/api_app/queryset.py @@ -33,7 +33,7 @@ from django.db.models.lookups import Exact from django.utils.timezone import now -from api_app.choices import TLP +from api_app.choices import TLP, ParamTypes from certego_saas.apps.organization.membership import Membership from certego_saas.apps.user.models import User @@ -280,10 +280,11 @@ def _alias_org_value_for_user( parameter__pk=OuterRef("pk"), **{config.snake_case_name: config.pk} ) .visible_for_user_by_org(user) - .values("value")[:1] + .values("value")[:1], + output_field=JSONField(null=True, blank=True), ) if user and user.has_membership() - else Value(None, output_field=JSONField()), + else Value(None, output_field=JSONField(null=True, blank=True)), ) def _alias_default_value(self, config: "PythonConfig") -> "ParameterQuerySet": @@ -295,27 +296,68 @@ def _alias_default_value(self, config: "PythonConfig") -> "ParameterQuerySet": parameter__pk=OuterRef("pk"), **{config.snake_case_name: config.pk} ) .default_values() - .values("value")[:1] + .values("value")[:1], + output_field=JSONField(null=True, blank=True), + ) + ) + + def _alias_runtime_config(self, runtime_config=None): + if not runtime_config: + runtime_config = {} + return self.alias( + runtime_value=Value( + runtime_config.get(F("name"), None), + output_field=JSONField(null=True, blank=True), + ) + ) + + def _alias_for_test(self): + if not settings.STAGE_CI and not settings.MOCK_CONNECTIONS: + return self.alias( + test_value=Value( + None, + output_field=JSONField(null=True, blank=True), + ) + ) + return self.alias( + test_value=Case( + When(name__icontains="url", then=Value("https://intelowl.com")), + When(name="pdns_credentials", then=Value("user|pwd")), + When(name__contains="test", then=Value(None)), + When(type=ParamTypes.INT.value, then=Value(10)), + default=Value("test"), + output_field=JSONField(null=True, blank=True), ) ) def annotate_value_for_user( - self, config: "PythonConfig", user: User = None + self, config: "PythonConfig", user: User = None, runtime_config=None ) -> "ParameterQuerySet": return ( self.prefetch_related("values") ._alias_owner_value_for_user(config, user) ._alias_org_value_for_user(config, user) ._alias_default_value(config) + ._alias_runtime_config(runtime_config) + ._alias_for_test() # importance order .annotate( + # 1. runtime + # 2. owner + # 3. organization + # 4. (if TEST environment) test value + # 5. default value value=Case( + When(runtime_value__isnull=False, then=F("runtime_value")), When(owner_value__isnull=False, then=F("owner_value")), When(org_value__isnull=False, then=F("org_value")), + When(test_value__isnull=False, then=F("test_value")), default=F("default_value"), + output_field=JSONField(null=True, blank=True), ), is_from_org=Case( When( + runtime_value__isnull=True, org_value__isnull=True, owner_value__isnull=False, then=Value(True), From c600de1d77a534972fc41275af47f8f2fd735801 Mon Sep 17 00:00:00 2001 From: 0ssigeno Date: Fri, 22 Mar 2024 11:39:33 +0100 Subject: [PATCH 02/10] disable_for_rate_limit now is disabled if there is no api_key or the api_key was set at org level Signed-off-by: 0ssigeno --- api_app/classes.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api_app/classes.py b/api_app/classes.py index b08e218d1a..1fb84b0341 100644 --- a/api_app/classes.py +++ b/api_app/classes.py @@ -287,7 +287,12 @@ def disable_for_rate_limit(self): self._user.membership.organization ) if org_configuration.rate_limit_timeout is not None: - org_configuration.disable_for_rate_limit() + api_key_parameter = self.__parameters.filter( + name__contains="api_key" + ).first() + # if we do not have api keys OR the api key was org based + if not api_key_parameter or api_key_parameter.is_from_org: + org_configuration.disable_for_rate_limit() else: logger.warning( f"You are trying to disable {self.name}" From b76296b7d16eca647f324d7e6f37819fc5ddbc1d Mon Sep 17 00:00:00 2001 From: 0ssigeno Date: Fri, 22 Mar 2024 15:43:10 +0100 Subject: [PATCH 03/10] Fixes Signed-off-by: 0ssigeno --- api_app/queryset.py | 64 +++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 20 deletions(-) diff --git a/api_app/queryset.py b/api_app/queryset.py index b216a818ee..58a6fb3dc3 100644 --- a/api_app/queryset.py +++ b/api_app/queryset.py @@ -265,7 +265,8 @@ def _alias_owner_value_for_user( parameter__pk=OuterRef("pk"), **{config.snake_case_name: config.pk} ) .visible_for_user_owned(user) - .values("value")[:1] + .values("value")[:1], + output_field=JSONField(), ) ) @@ -281,10 +282,10 @@ def _alias_org_value_for_user( ) .visible_for_user_by_org(user) .values("value")[:1], - output_field=JSONField(null=True, blank=True), + output_field=JSONField(), ) if user and user.has_membership() - else Value(None, output_field=JSONField(null=True, blank=True)), + else Value(None, output_field=JSONField()), ) def _alias_default_value(self, config: "PythonConfig") -> "ParameterQuerySet": @@ -297,7 +298,7 @@ def _alias_default_value(self, config: "PythonConfig") -> "ParameterQuerySet": ) .default_values() .values("value")[:1], - output_field=JSONField(null=True, blank=True), + output_field=JSONField(), ) ) @@ -307,7 +308,7 @@ def _alias_runtime_config(self, runtime_config=None): return self.alias( runtime_value=Value( runtime_config.get(F("name"), None), - output_field=JSONField(null=True, blank=True), + output_field=JSONField(), ) ) @@ -316,24 +317,31 @@ def _alias_for_test(self): return self.alias( test_value=Value( None, - output_field=JSONField(null=True, blank=True), ) ) - return self.alias( + return self.annotate( test_value=Case( - When(name__icontains="url", then=Value("https://intelowl.com")), - When(name="pdns_credentials", then=Value("user|pwd")), - When(name__contains="test", then=Value(None)), - When(type=ParamTypes.INT.value, then=Value(10)), - default=Value("test"), - output_field=JSONField(null=True, blank=True), + When( + name__icontains="url", + then=Value("https://intelowl.com", output_field=JSONField()), + ), + When( + name="pdns_credentials", + then=Value("user|pwd", output_field=JSONField()), + ), + When(name__contains="test", then=Value(None, output_field=JSONField())), + When( + type=ParamTypes.INT.value, then=Value(10, output_field=JSONField()) + ), + default=Value("test", output_field=JSONField()), + output_field=JSONField(), ) ) def annotate_value_for_user( self, config: "PythonConfig", user: User = None, runtime_config=None ) -> "ParameterQuerySet": - return ( + res = ( self.prefetch_related("values") ._alias_owner_value_for_user(config, user) ._alias_org_value_for_user(config, user) @@ -348,12 +356,26 @@ def annotate_value_for_user( # 4. (if TEST environment) test value # 5. default value value=Case( - When(runtime_value__isnull=False, then=F("runtime_value")), - When(owner_value__isnull=False, then=F("owner_value")), - When(org_value__isnull=False, then=F("org_value")), - When(test_value__isnull=False, then=F("test_value")), - default=F("default_value"), - output_field=JSONField(null=True, blank=True), + When( + runtime_value__isnull=False, + then=Cast(F("runtime_value"), output_field=JSONField()), + ), + When( + owner_value__isnull=False, + then=Cast(F("owner_value"), output_field=JSONField()), + ), + When( + org_value__isnull=False, + then=Cast(F("org_value"), output_field=JSONField()), + ), + # Yeah, I can't use isnull=False here + # because _reasons_ of the JsonField in conjunction with the Case clause + When( + ~Q(test_value__exact=None), + then=Cast(F("test_value"), output_field=JSONField()), + ), + default=Cast(F("default_value"), output_field=JSONField()), + output_field=JSONField(), ), is_from_org=Case( When( @@ -366,6 +388,8 @@ def annotate_value_for_user( ), ) ) + print(res.query) + return res class AbstractReportQuerySet(SendToBiQuerySet): From d3e914dd799b8238bc4eb8f35724bdb5b419e463 Mon Sep 17 00:00:00 2001 From: 0ssigeno Date: Fri, 22 Mar 2024 15:43:48 +0100 Subject: [PATCH 04/10] Blake Signed-off-by: 0ssigeno --- api_app/queryset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api_app/queryset.py b/api_app/queryset.py index 58a6fb3dc3..bc4fb1c0de 100644 --- a/api_app/queryset.py +++ b/api_app/queryset.py @@ -369,7 +369,8 @@ def annotate_value_for_user( then=Cast(F("org_value"), output_field=JSONField()), ), # Yeah, I can't use isnull=False here - # because _reasons_ of the JsonField in conjunction with the Case clause + # because _reasons_ of the JsonField in conjunction with + # the Case clause When( ~Q(test_value__exact=None), then=Cast(F("test_value"), output_field=JSONField()), From 0c990fbbf43442264fb42d5529a7336663e0b216 Mon Sep 17 00:00:00 2001 From: 0ssigeno Date: Fri, 22 Mar 2024 15:46:35 +0100 Subject: [PATCH 05/10] Fix return Signed-off-by: 0ssigeno --- api_app/queryset.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api_app/queryset.py b/api_app/queryset.py index bc4fb1c0de..3cb9f989dc 100644 --- a/api_app/queryset.py +++ b/api_app/queryset.py @@ -341,7 +341,7 @@ def _alias_for_test(self): def annotate_value_for_user( self, config: "PythonConfig", user: User = None, runtime_config=None ) -> "ParameterQuerySet": - res = ( + return ( self.prefetch_related("values") ._alias_owner_value_for_user(config, user) ._alias_org_value_for_user(config, user) @@ -389,8 +389,6 @@ def annotate_value_for_user( ), ) ) - print(res.query) - return res class AbstractReportQuerySet(SendToBiQuerySet): From 15b512434a4a07ff19a45d3701149f48dbd7fa74 Mon Sep 17 00:00:00 2001 From: 0ssigeno Date: Fri, 22 Mar 2024 16:27:45 +0100 Subject: [PATCH 06/10] Fix Signed-off-by: 0ssigeno --- api_app/queryset.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/api_app/queryset.py b/api_app/queryset.py index 3cb9f989dc..52e3915315 100644 --- a/api_app/queryset.py +++ b/api_app/queryset.py @@ -266,7 +266,6 @@ def _alias_owner_value_for_user( ) .visible_for_user_owned(user) .values("value")[:1], - output_field=JSONField(), ) ) @@ -282,7 +281,6 @@ def _alias_org_value_for_user( ) .visible_for_user_by_org(user) .values("value")[:1], - output_field=JSONField(), ) if user and user.has_membership() else Value(None, output_field=JSONField()), @@ -298,7 +296,6 @@ def _alias_default_value(self, config: "PythonConfig") -> "ParameterQuerySet": ) .default_values() .values("value")[:1], - output_field=JSONField(), ) ) @@ -319,7 +316,7 @@ def _alias_for_test(self): None, ) ) - return self.annotate( + return self.alias( test_value=Case( When( name__icontains="url", @@ -333,7 +330,7 @@ def _alias_for_test(self): When( type=ParamTypes.INT.value, then=Value(10, output_field=JSONField()) ), - default=Value("test", output_field=JSONField()), + default=Value(F("default_value"), output_field=JSONField()), output_field=JSONField(), ) ) From 7f4bc57ab2b5d56f6313149cb34b7d6b94974786 Mon Sep 17 00:00:00 2001 From: 0ssigeno Date: Fri, 22 Mar 2024 16:56:50 +0100 Subject: [PATCH 07/10] Fix Signed-off-by: 0ssigeno --- api_app/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/queryset.py b/api_app/queryset.py index 52e3915315..58d5a34960 100644 --- a/api_app/queryset.py +++ b/api_app/queryset.py @@ -330,7 +330,7 @@ def _alias_for_test(self): When( type=ParamTypes.INT.value, then=Value(10, output_field=JSONField()) ), - default=Value(F("default_value"), output_field=JSONField()), + default="default_value", output_field=JSONField(), ) ) From 1eb0299c2bdc5efe311e69098b46e2f4842c4e6c Mon Sep 17 00:00:00 2001 From: 0ssigeno Date: Fri, 22 Mar 2024 17:12:26 +0100 Subject: [PATCH 08/10] Fix order Signed-off-by: 0ssigeno --- api_app/queryset.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/api_app/queryset.py b/api_app/queryset.py index 58d5a34960..67a0091cb0 100644 --- a/api_app/queryset.py +++ b/api_app/queryset.py @@ -330,7 +330,7 @@ def _alias_for_test(self): When( type=ParamTypes.INT.value, then=Value(10, output_field=JSONField()) ), - default="default_value", + default=Value("test", output_field=JSONField()), output_field=JSONField(), ) ) @@ -350,8 +350,9 @@ def annotate_value_for_user( # 1. runtime # 2. owner # 3. organization - # 4. (if TEST environment) test value - # 5. default value + # 4. default value + # 5. (if TEST environment) test value + # 5. (if NOT TEST environment) None value=Case( When( runtime_value__isnull=False, @@ -365,14 +366,11 @@ def annotate_value_for_user( org_value__isnull=False, then=Cast(F("org_value"), output_field=JSONField()), ), - # Yeah, I can't use isnull=False here - # because _reasons_ of the JsonField in conjunction with - # the Case clause When( - ~Q(test_value__exact=None), - then=Cast(F("test_value"), output_field=JSONField()), + default_value__isnull=False, + then=Cast(F("default_value"), output_field=JSONField()), ), - default=Cast(F("default_value"), output_field=JSONField()), + default=Cast(F("test_value"), output_field=JSONField()), output_field=JSONField(), ), is_from_org=Case( From 4c4cdbdeef1b3baa32ddfc64e2349be5f1a5623b Mon Sep 17 00:00:00 2001 From: 0ssigeno Date: Fri, 22 Mar 2024 17:20:36 +0100 Subject: [PATCH 09/10] Fix other analyzers Signed-off-by: 0ssigeno --- .../observable_analyzers/mmdb_server.py | 13 ++----- .../observable_analyzers/otx.py | 2 +- .../observable_analyzers/phoneinfoga_scan.py | 38 ++++++++----------- 3 files changed, 19 insertions(+), 34 deletions(-) diff --git a/api_app/analyzers_manager/observable_analyzers/mmdb_server.py b/api_app/analyzers_manager/observable_analyzers/mmdb_server.py index ee528b7284..dd280c51ac 100644 --- a/api_app/analyzers_manager/observable_analyzers/mmdb_server.py +++ b/api_app/analyzers_manager/observable_analyzers/mmdb_server.py @@ -3,9 +3,6 @@ import requests from api_app.analyzers_manager import classes -from api_app.analyzers_manager.exceptions import ( # AnalyzerConfigurationException - AnalyzerRunException, -) from tests.mock_utils import MockUpResponse, if_mock_connections, patch logger = logging.getLogger(__name__) @@ -19,16 +16,12 @@ class MmdbServer(classes.ObservableAnalyzer): def update(self) -> bool: pass - base_url = str + base_url: str observable_name: str def run(self): - try: - response = requests.get(self.base_url + self.observable_name) - response.raise_for_status() - except requests.RequestException as e: - logger.exception("Error while querying mmdb server: {e}") - raise AnalyzerRunException(e) + response = requests.get(self.base_url + self.observable_name) + response.raise_for_status() return response.json() @classmethod diff --git a/api_app/analyzers_manager/observable_analyzers/otx.py b/api_app/analyzers_manager/observable_analyzers/otx.py index 04b604494f..0c7e387888 100644 --- a/api_app/analyzers_manager/observable_analyzers/otx.py +++ b/api_app/analyzers_manager/observable_analyzers/otx.py @@ -154,7 +154,7 @@ def run(self): logger.warning( f"Sections: {not_supported_requested_section_list}" f" are not supported for indicator type: {otx_type}. " - f"We remove them from the search." + "We remove them from the search." ) for not_supported in not_supported_requested_section_list: self.sections.remove(not_supported) diff --git a/api_app/analyzers_manager/observable_analyzers/phoneinfoga_scan.py b/api_app/analyzers_manager/observable_analyzers/phoneinfoga_scan.py index 201a023981..882547c20c 100644 --- a/api_app/analyzers_manager/observable_analyzers/phoneinfoga_scan.py +++ b/api_app/analyzers_manager/observable_analyzers/phoneinfoga_scan.py @@ -3,9 +3,6 @@ import requests from api_app.analyzers_manager import classes -from api_app.analyzers_manager.exceptions import ( # AnalyzerConfigurationException - AnalyzerRunException, -) from tests.mock_utils import MockUpResponse logger = logging.getLogger(__name__) @@ -30,28 +27,23 @@ def update(self) -> bool: _GOOGLE_API_KEY: str = "" def run(self): - response: None url: str = f"http://phoneinfoga:5000/api/v2/scanners/{self.scanner_name}/run" - try: - response = requests.post( - url, - headers={ - "Content-Type": "application/json", - "accept": "application/json", + response = requests.post( + url, + headers={ + "Content-Type": "application/json", + "accept": "application/json", + }, + json={ + "number": self.observable_name, + "options": { + "NUMVERIFY_API_KEY": self._NUMVERIFY_API_KEY, + "GOOGLECSE_CX": self._GOOGLECSE_CX, + "GOOGLE_API_KEY": self._GOOGLE_API_KEY, }, - json={ - "number": self.observable_name, - "options": { - "NUMVERIFY_API_KEY": self._NUMVERIFY_API_KEY, - "GOOGLECSE_CX": self._GOOGLECSE_CX, - "GOOGLE_API_KEY": self._GOOGLE_API_KEY, - }, - }, - ) - response.raise_for_status() - except requests.RequestException as e: - logger.exception("Error while querying phoneinfoga analyzer: {e}") - raise AnalyzerRunException(e) + }, + ) + response.raise_for_status() return response.json() @staticmethod From cdb5af3c23c3da2b72a17b32daba082efa921a00 Mon Sep 17 00:00:00 2001 From: 0ssigeno Date: Fri, 22 Mar 2024 17:41:14 +0100 Subject: [PATCH 10/10] Deepsource fixes Signed-off-by: 0ssigeno --- .../migrations/0001_initial_squashed.py | 1 - .../observable_analyzers/feodo_tracker.py | 7 +- .../observable_analyzers/ip2location.py | 8 +- .../observable_analyzers/phishtank.py | 582 ------------------ .../observable_analyzers/vt/vt3_base.py | 12 +- api_app/classes.py | 13 +- .../migrations/0001_initial_squashed.py | 30 - api_app/migrations/0001_2_initial_squashed.py | 3 - api_app/migrations/0061_job_depth_analysis.py | 1 - .../migrations/0001_initial_squashed.py | 44 -- .../migrations/0001_initial_squashed.py | 44 -- 11 files changed, 21 insertions(+), 724 deletions(-) diff --git a/api_app/analyzers_manager/migrations/0001_initial_squashed.py b/api_app/analyzers_manager/migrations/0001_initial_squashed.py index 2b136b0ee4..809a0f1497 100644 --- a/api_app/analyzers_manager/migrations/0001_initial_squashed.py +++ b/api_app/analyzers_manager/migrations/0001_initial_squashed.py @@ -6,7 +6,6 @@ import django.utils.timezone from django.db import migrations, models -import api_app.defaults import api_app.fields import api_app.validators diff --git a/api_app/analyzers_manager/observable_analyzers/feodo_tracker.py b/api_app/analyzers_manager/observable_analyzers/feodo_tracker.py index 663a162f6d..c7369f2d45 100644 --- a/api_app/analyzers_manager/observable_analyzers/feodo_tracker.py +++ b/api_app/analyzers_manager/observable_analyzers/feodo_tracker.py @@ -42,14 +42,13 @@ def default_locations(cls) -> Tuple[str, str]: def run(self): result = {"found": False} - db_location, url = ( + db_location, _ = ( self.recommend_locations if self.use_recommended_url else self.default_locations ) - if self.update_on_run or not os.path.exists(db_location): - if not self.update(): - raise AnalyzerRunException("Unable to update database") + if self.update_on_run or not os.path.exists(db_location) and not self.update(): + raise AnalyzerRunException("Unable to update database") try: with open(db_location, "r", encoding="utf-8") as f: db = json.load(f) diff --git a/api_app/analyzers_manager/observable_analyzers/ip2location.py b/api_app/analyzers_manager/observable_analyzers/ip2location.py index 5e702e2586..6fe7f1efaa 100644 --- a/api_app/analyzers_manager/observable_analyzers/ip2location.py +++ b/api_app/analyzers_manager/observable_analyzers/ip2location.py @@ -20,10 +20,10 @@ def run(self): try: payload = {"ip": self.observable_name} - """ There are two free versions of the service: - 1. keyless : Requires No API key and has a daily limit of 500 queries - 2. keyed: Requires API key. - """ + # There are two free versions of the service: + # 1. keyless : Requires No API key and has a daily limit of 500 queries + # 2. keyed: Requires API key. + if self.api_version == "keyed": payload["key"] = self._api_key_name diff --git a/api_app/analyzers_manager/observable_analyzers/phishtank.py b/api_app/analyzers_manager/observable_analyzers/phishtank.py index 72b0d78342..abd88ef0c6 100644 --- a/api_app/analyzers_manager/observable_analyzers/phishtank.py +++ b/api_app/analyzers_manager/observable_analyzers/phishtank.py @@ -55,585 +55,3 @@ def _monkeypatch(cls): ) ] return super()._monkeypatch(patches=patches) - - -""" - -IntelOwl logov5.2.3 -Home -Dashboard -Jobs -Plugins -Scan -4 -SB -Organization -Organization Config - Invitations -certego's plugin configuration -Note: Your plugin configuration overrides your organization's configuration. - Parameters - Secrets - -Analyzer - -AbuseIPDB - -api_key_name -********** - -Analyzer - -Auth0 - -api_key_name -********** - -Analyzer - -CIRCLPassiveDNS - -pdns_credentials -********** - -Analyzer - -CIRCLPassiveSSL - -pdns_credentials -********** - -Analyzer - -Censys_Search - -api_id_name -********** - -Analyzer - -Censys_Search - -api_secret_name -********** - -Analyzer - -Cuckoo_Scan - -url_key_name -********** - -Analyzer - -Dehashed_Search - -api_key_name -********** - -Analyzer - -DNSDB - -api_key_name -********** - -Analyzer - -Dragonfly_Emulation_PE - -api_key_name -********** - -Analyzer - -Dragonfly_Emulation_PE - -url_key_name -********** - -Analyzer - -Dragonfly_Emulation_PE - -certificate_path_name -********** - -Analyzer - -GoogleSafebrowsing - -api_key_name -********** - -Analyzer - -HybridAnalysis_Get_File - -api_key_name -********** - -Analyzer - -Intezer_Get - -api_key_name -********** - -Analyzer - -Intezer_Scan - -api_key_name -********** - -Analyzer - -MISPFIRST_Check_Hash - -api_key_name -********** - -Analyzer - -MISPFIRST - -url_key_name -********** - -Analyzer - -MaxMindGeoIP - -api_key_name -********** - -Analyzer - -MWDB_Get - -api_key_name -********** - -Analyzer - -MWDB_Scan - -api_key_name -********** - -Analyzer - -ONYPHE - -api_key_name -********** - -Analyzer - -OTX_Check_Hash - -api_key_name -********** - -Analyzer - -Phishtank - -api_key_name -********** - -Analyzer - -Pulsedive - -api_key_name -********** - -Analyzer - -Quokka_PDNS_Wildcard_Left - -api_key_name -********** - -Analyzer - -Quokka_PDNS_Wildcard_Left - -url_key_name -********** - -Analyzer - -Shodan_Honeyscore - -api_key_name -********** - -Analyzer - -UnpacMe - -api_key_name -********** - -Analyzer - -VirusTotal_v3_Intelligence_Search - -api_key_name -********** - -Analyzer - -VirusTotal_v3_Get_File - -api_key_name -********** - -Analyzer - -XForceExchange - -api_key_name -********** - -Analyzer - -XForceExchange - -api_password_name -********** - -Analyzer - -Triage_Search - -api_key_name -********** - -Analyzer - -Triage_Scan - -api_key_name -********** - -Analyzer - -YARAify_File_Scan - -api_key_identifier -********** - -Analyzer - -Stalkphish - -api_key_name -********** - -Analyzer - -GoogleWebRisk - -service_account_json -********** - -Analyzer - -Yara - -private_repositories -********** - -Analyzer - -Crowdsec - -api_key_name -********** - -Analyzer - -SublimeSecurity - -url -********** - -Analyzer - -SublimeSecurity - -api_key -********** - -Analyzer - -SublimeSecurity - -message_source_id -********** - -Connector - -Quokka - -url -********** - -Connector - -Quokka - -api_key_name -********** - -Connector - -Quokka - -engine_url -********** - -Analyzer - -DNSDB_SIE - -api_key_name -********** - -Analyzer - -DNSDB_SIE_Names - -api_key_name -********** - -Analyzer - -DNSDB_SIE_Wildcard_Left - -api_key_name -********** - -Analyzer - -HybridAnalysis_Get_Observable - -api_key_name -********** - -Analyzer - -VirusTotal_v3_Get_Observable - -api_key_name -08043cd5c1ed59fb025737eeef91caf1e9f38cff0f4392e2b5027a1399f20ce0 - -Analyzer - -Dragonfly_Emulation_ELF - -api_key_name -********** - -Analyzer - -OTXQuery - -api_key_name -********** - -Analyzer - -Quokka_RDNS_Names - -url_key_name -********** - -Analyzer - -Quokka_PDNS - -url_key_name -********** - -Analyzer - -Quokka_RDNS - -url_key_name -********** - -Analyzer - -Quokka_PDNS - -api_key_name -********** - -Analyzer - -Quokka_RDNS - -api_key_name -********** - -Analyzer - -Quokka_RDNS_Names - -api_key_name -********** - -Analyzer - -MISPFIRST_Check_Hash - -url_key_name -********** - -Analyzer - -MISPFIRST - -api_key_name -********** - -Connector - -Quokka - -engine_api_key_name -********** - -Analyzer - -CapeSandbox - -api_key_name -********** - -Analyzer - -CapeSandbox - -url_key_name -********** - -Analyzer - -CapeSandbox - -certificate -********** - -Visualizer - -Quokka_Domain_Url - -quokka_base_url -********** - -Visualizer - -Quokka_Ip - -quokka_base_url -********** - -Visualizer - -Quokka_Hash - -quokka_base_url -********** - -Visualizer - -Quokka_File - -quokka_base_url -********** - -Visualizer - -Quokka_Static_File - -quokka_base_url -********** - -Visualizer - -Sample_Static_Analysis - -quokka_base_url -********** - -Visualizer - -Quokka_File - -capybox_base_url -********** - -Visualizer - -Quokka_Hash - -capybox_base_url -********** - -Visualizer - -Quokka_Static_File - -capybox_base_url -********** - -Visualizer - -Sample_Static_Analysis - -capybox_base_url -********** - -Visualizer - -Quokka_Observable - -quokka_base_url -********** - -Analyzer - -DNS0_rrsets_data - -api_key -********** - -Analyzer - -DNS0_rrsets_name - -api_key -********** - -Analyzer - -DNS0_names - -api_key -********** - -Visualizer - -Quokka_Observable - -capybox_base_url -********** -15 - -""" diff --git a/api_app/analyzers_manager/observable_analyzers/vt/vt3_base.py b/api_app/analyzers_manager/observable_analyzers/vt/vt3_base.py index 359a7c945b..77451e2fa7 100644 --- a/api_app/analyzers_manager/observable_analyzers/vt/vt3_base.py +++ b/api_app/analyzers_manager/observable_analyzers/vt/vt3_base.py @@ -5,7 +5,7 @@ import logging import time from datetime import datetime, timedelta -from typing import Dict +from typing import Dict, Tuple import requests @@ -229,6 +229,12 @@ def _vt_get_report( self._vt_get_relationships( observable_name, relationships_requested, uri, result ) + uri_prefix, uri_postfix = self._get_url_prefix_postfix() + result["link"] = f"https://www.virustotal.com/gui/{uri_prefix}/{uri_postfix}" + + return result + + def _get_url_prefix_postfix(self) -> Tuple[str, str]: if self._job.observable_classification == ObservableClassification.DOMAIN.value: uri_prefix = "domain" uri_postfix = self._job.observable_name @@ -241,9 +247,7 @@ def _vt_get_report( else: # hash uri_prefix = "search" uri_postfix = self._job.observable_name - result["link"] = f"https://www.virustotal.com/gui/{uri_prefix}/{uri_postfix}" - - return result + return uri_prefix, uri_postfix def _vt_scan_file(self, md5: str, rescan_instead: bool = False) -> dict: if rescan_instead: diff --git a/api_app/classes.py b/api_app/classes.py index 1fb84b0341..4c77e890e5 100644 --- a/api_app/classes.py +++ b/api_app/classes.py @@ -135,13 +135,12 @@ def after_run_failed(self, e: Exception): self.report.errors.append(str(e)) self.report.status = self.report.Status.FAILED self.report.save(update_fields=["status", "errors"]) - if isinstance(e, HTTPError): - if ( - e.response - and hasattr(e.response, "status_code") - and e.response.status_code == 429 - ): - self.disable_for_rate_limit() + if isinstance(e, HTTPError) and ( + e.response + and hasattr(e.response, "status_code") + and e.response.status_code == 429 + ): + self.disable_for_rate_limit() if settings.STAGE_CI: raise e diff --git a/api_app/ingestors_manager/migrations/0001_initial_squashed.py b/api_app/ingestors_manager/migrations/0001_initial_squashed.py index 03ea261b4c..07cdbba096 100644 --- a/api_app/ingestors_manager/migrations/0001_initial_squashed.py +++ b/api_app/ingestors_manager/migrations/0001_initial_squashed.py @@ -7,41 +7,11 @@ from django.conf import settings from django.db import migrations, models -import api_app.defaults import api_app.interfaces -# Functions from the following migrations need manual copying. -# Move them and any dependencies into this file, then update the -# RunPython operations to refer to the local versions: -# api_app.ingestors_manager.migrations.0003_ingestor_config -# api_app.ingestors_manager.migrations.0010_ingestorconfig_routing_key_and_more - class Migration(migrations.Migration): initial = True - replaces = [ - # ("ingestors_manager", "0001_initial"), - # ( - # "ingestors_manager", - # "0002_remove_ingestorconfig_disabled_in_organizations_and_more", - # ), - # ("ingestors_manager", "0003_ingestor_config"), - # ("ingestors_manager", "0004_alter_ingestorreport_name"), - # ( - # "ingestors_manager", - # "0005_rename_ingestors_m_python__5c8ce0_idx_ingestors_m_python__b7a859_idx_and_more", - # ), - # ("ingestors_manager", "0006_alter_python_module"), - # ("ingestors_manager", "0007_alter_ingestorconfig_python_module"), - # ("ingestors_manager", "0008_alter_ingestorconfig_python_module"), - # ("ingestors_manager", "0009_ingestorconfig_maximum_jobs"), - # ("ingestors_manager", "0010_ingestorconfig_routing_key_and_more"), - # ("ingestors_manager", "0011_ingestorconfig_health_check_task"), - # ("ingestors_manager", "0012_ingestorconfig_health_check_status"), - # ("ingestors_manager", "0013_ingestorreport_parameters"), - # ("ingestors_manager", "0014_ingestorreport_sent_to_bi"), - # ("ingestors_manager", "0015_ingestorreport_ingestorreportsbisearch"), - ] dependencies = [ ("api_app", "0001_1_initial_squashed"), diff --git a/api_app/migrations/0001_2_initial_squashed.py b/api_app/migrations/0001_2_initial_squashed.py index 6e9b6870d8..8ca29930b5 100644 --- a/api_app/migrations/0001_2_initial_squashed.py +++ b/api_app/migrations/0001_2_initial_squashed.py @@ -1,6 +1,3 @@ -import django.contrib.postgres.fields -import django.contrib.postgres.fields.jsonb -import django.core.validators import django.db.migrations.operations.special import django.db.models.deletion from django.conf import settings diff --git a/api_app/migrations/0061_job_depth_analysis.py b/api_app/migrations/0061_job_depth_analysis.py index 6347114db5..85d8667a4a 100644 --- a/api_app/migrations/0061_job_depth_analysis.py +++ b/api_app/migrations/0061_job_depth_analysis.py @@ -6,7 +6,6 @@ def migrate(apps, schema_editor): # I have to use the import here because i really need the class methods - from api_app.models import Job as JobNonStoric PivotMap = apps.get_model("pivots_manager", "PivotMap") Investigation = apps.get_model("investigations_manager", "Investigation") diff --git a/api_app/playbooks_manager/migrations/0001_initial_squashed.py b/api_app/playbooks_manager/migrations/0001_initial_squashed.py index ddadf0ce2f..0ef67848b1 100644 --- a/api_app/playbooks_manager/migrations/0001_initial_squashed.py +++ b/api_app/playbooks_manager/migrations/0001_initial_squashed.py @@ -11,53 +11,9 @@ import api_app.fields import api_app.validators -# Functions from the following migrations need manual copying. -# Move them and any dependencies into this file, then update the -# RunPython operations to refer to the local versions: -# api_app.playbooks_manager.migrations.0004_datamigration -# api_app.playbooks_manager.migrations.0005_static_analysis -# api_app.playbooks_manager.migrations.0006_rename_static_analysis -# api_app.playbooks_manager.migrations.0007_fix_static_analysis -# api_app.playbooks_manager.migrations.0008_fix_free_to_use_playbook -# api_app.playbooks_manager.migrations.0010_domain_reputation_playbook -# api_app.playbooks_manager.migrations.0011_fix_static_analysis -# api_app.playbooks_manager.migrations.0012_ip_reputation_playbook -# api_app.playbooks_manager.migrations.0014_remove_old_playbook -# api_app.playbooks_manager.migrations.0015_dns_playbook -# api_app.playbooks_manager.migrations.0018_playbookconfig_scan_check_time_and_more -# api_app.playbooks_manager.migrations.0019_update_static_analysis -# api_app.playbooks_manager.migrations.0022_add_dns0_to_free_playbook - class Migration(migrations.Migration): initial = True - replaces = [ - # ("playbooks_manager", "0001_initial"), - # ("playbooks_manager", "0002_alter_cachedplaybook_job"), - # ("playbooks_manager", "0003_playbook"), - # ("playbooks_manager", "0004_datamigration"), - # ("playbooks_manager", "0005_static_analysis"), - # ("playbooks_manager", "0006_rename_static_analysis"), - # ("playbooks_manager", "0007_fix_static_analysis"), - # ("playbooks_manager", "0008_fix_free_to_use_playbook"), - # ("playbooks_manager", "0009_alter_playbookconfig_name"), - # ("playbooks_manager", "0010_domain_reputation_playbook"), - # ("playbooks_manager", "0011_fix_static_analysis"), - # ("playbooks_manager", "0012_ip_reputation_playbook"), - # ("playbooks_manager", "0013_alter_playbookconfig_options"), - # ("playbooks_manager", "0014_remove_old_playbook"), - # ("playbooks_manager", "0015_dns_playbook"), - # ("playbooks_manager", "0016_playbookconfig_disabled_in_organizations"), - # ("playbooks_manager", "0017_playbookconfig_pivots"), - # ("playbooks_manager", "0018_playbookconfig_scan_check_time_and_more"), - # ("playbooks_manager", "0019_update_static_analysis"), - # ( - # "playbooks_manager", - # "0020_playbookconfig_for_organization_playbookconfig_owner_and_more", - # ), - # ("playbooks_manager", "0021_alter_playbookconfig_name_and_more"), - # ("playbooks_manager", "0022_add_dns0_to_free_playbook"), - ] dependencies = [ ("api_app", "0001_1_initial_squashed"), diff --git a/api_app/visualizers_manager/migrations/0001_initial_squashed.py b/api_app/visualizers_manager/migrations/0001_initial_squashed.py index 50f11469e7..3126d3d360 100644 --- a/api_app/visualizers_manager/migrations/0001_initial_squashed.py +++ b/api_app/visualizers_manager/migrations/0001_initial_squashed.py @@ -6,55 +6,11 @@ import django.utils.timezone from django.db import migrations, models -import api_app.defaults -import api_app.validators import api_app.visualizers_manager.validators -# Functions from the following migrations need manual copying. -# Move them and any dependencies into this file, then update the -# RunPython operations to refer to the local versions: -# api_app.visualizers_manager.migrations.00011_dns_visualizer -# api_app.visualizers_manager.migrations.0002_datamigration -# api_app.visualizers_manager.migrations.0005_visualizerconfig_disabled_in_org -# api_app.visualizers_manager.migrations.0007_auto_20230308_1623 -# api_app.visualizers_manager.migrations.0008_parent_playbook_foreign_key -# api_app.visualizers_manager.migrations.0009_remove_parent_playbook -# api_app.visualizers_manager.migrations.0013_params -# api_app.visualizers_manager.migrations.0018_visualizer_config -# api_app.visualizers_manager.migrations.0019_dns_visualizer_change -# api_app.visualizers_manager.migrations.0020_visualizer_config -# api_app.visualizers_manager.migrations.0022_remove_visualizerconfig_analyzers_and_more - class Migration(migrations.Migration): initial = True - replaces = [ - # ("visualizers_manager", "0001_initial"), - # ("visualizers_manager", "0002_datamigration"), - # ("visualizers_manager", "0003_auto_20230301_1415"), - # ("visualizers_manager", "0004_alter_visualizerreport_report"), - # ("visualizers_manager", "0005_visualizerconfig_disabled_in_org"), - # ("visualizers_manager", "0006_alter_visualizerreport_job"), - # ("visualizers_manager", "0007_auto_20230308_1623"), - # ("visualizers_manager", "0008_parent_playbook_foreign_key"), - # ("visualizers_manager", "0009_remove_parent_playbook"), - # ("visualizers_manager", "00010_remove_runtime_configuration"), - # ("visualizers_manager", "00011_dns_visualizer"), - # ( - # "visualizers_manager", - # "0012_alter_visualizerconfig_disabled_in_organizations_and_more", - # ), - # ("visualizers_manager", "0013_params"), - # ("visualizers_manager", "0014_alter_visualizerreport_report"), - # ("visualizers_manager", "0015_alter_visualizerreport_unique_together"), - # ("visualizers_manager", "0016_visualizerreport_name"), - # ("visualizers_manager", "0017_alter_visualizerreport_options"), - # ("visualizers_manager", "0018_visualizer_config"), - # ("visualizers_manager", "0019_dns_visualizer_change"), - # ("visualizers_manager", "0020_visualizer_config"), - # ("visualizers_manager", "0021_alter_visualizerconfig_options"), - # ("visualizers_manager", "0022_remove_visualizerconfig_analyzers_and_more"), - ] dependencies = [ ("api_app", "0001_1_initial_squashed"),