fixed advanced search (#2748)

* fixed advanced search * fixed tests
intelowlproject · Feb 18, 2025 · 3198062 · 3198062
1 parent a1a5dbf
commit 3198062
Show file tree

Hide file tree

Showing 11 changed files with 27 additions and 119 deletions.
diff --git a/api_app/documents.py b/api_app/documents.py
diff --git a/api_app/management/commands/elastic_templates.py b/api_app/management/commands/elastic_templates.py
@@ -4,7 +4,6 @@
 from django.conf import settings
 from django.core.management import BaseCommand
 from elasticsearch import ApiError
-from elasticsearch_dsl import connections
 
 logger = logging.getLogger(__name__)
 
@@ -22,7 +21,7 @@ def handle(self, *args, **options):
                 settings.CONFIG_ROOT / "elastic_search_mappings" / "plugin_report.json"
             ) as file_content:
                 try:
-                    connections.get_connection().indices.put_template(
+                    settings.ELASTICSEARCH_DSL_CLIENT.indices.put_template(
                         name="plugin-report", body=json.load(file_content)
                     )
                     success_msg = (

diff --git a/api_app/views.py b/api_app/views.py
@@ -1622,7 +1622,10 @@ def get(self, request):
 
         # 3 return data
         elastic_response = (
-            Search(index=f"plugin-report-{get_environment()}*")
+            Search(
+                using=settings.ELASTICSEARCH_DSL_CLIENT,
+                index=f"plugin-report-{get_environment()}*",
+            )
             .query(QElastic("bool", filter=filter_list))
             .extra(size=10000)  # max allowed size
             .execute()

diff --git a/docker/env_file_app_ci b/docker/env_file_app_ci
@@ -31,8 +31,6 @@ AWS_SECRET_ACCESS_KEY=
 ELASTICSEARCH_DSL_ENABLED=False
 ELASTICSEARCH_DSL_HOST=
 ELASTICSEARCH_DSL_PASSWORD=changeme
-ELASTICSEARCH_DSL_NO_OF_SHARDS=1
-ELASTICSEARCH_DSL_NO_OF_REPLICAS=0
 
 ELASTICSEARCH_BI_ENABLED=False
 ELASTICSEARCH_BI_HOST=

diff --git a/docker/env_file_app_template b/docker/env_file_app_template
@@ -57,8 +57,6 @@ ELASTICSEARCH_DSL_ENABLED=False
 ELASTICSEARCH_DSL_HOST=
 ELASTICSEARCH_DSL_PASSWORD=
 # consult to: https://django-elasticsearch-dsl.readthedocs.io/en/latest/settings.html
-ELASTICSEARCH_DSL_NO_OF_SHARDS=1
-ELASTICSEARCH_DSL_NO_OF_REPLICAS=0
 
 ELASTICSEARCH_BI_ENABLED=False
 ELASTICSEARCH_BI_HOST=

diff --git a/intel_owl/settings/__init__.py b/intel_owl/settings/__init__.py
@@ -20,7 +20,6 @@
     "prettyjson",
     # celery, elasticsearch
     "django_celery_results",
-    "django_elasticsearch_dsl",
     # rest framework libs
     "rest_framework",
     "rest_framework_filters",

diff --git a/intel_owl/settings/elasticsearch.py b/intel_owl/settings/elasticsearch.py
@@ -5,17 +5,14 @@
 
 from intel_owl import secrets
 
+# business intelligence (bi)
 ELASTICSEARCH_BI_ENABLED = (
     secrets.get_secret("ELASTICSEARCH_BI_ENABLED", False) == "True"
 )
 if ELASTICSEARCH_BI_ENABLED:
     ELASTICSEARCH_BI_HOST = secrets.get_secret("ELASTICSEARCH_BI_HOST").split(",")
     ELASTICSEARCH_BI_INDEX = secrets.get_secret("ELASTICSEARCH_BI_INDEX")
-
-    if not ELASTICSEARCH_BI_HOST or not ELASTICSEARCH_BI_INDEX:
-        print("Elasticsearch not correctly configured")
-
-    else:
+    if ELASTICSEARCH_BI_HOST and ELASTICSEARCH_BI_INDEX:
         ELASTICSEARCH_BI_CLIENT = Elasticsearch(
             ELASTICSEARCH_BI_HOST,
             maxsize=20,
@@ -27,44 +24,40 @@
             print(
                 f"ELASTICSEARCH BI client configuration did not connect correctly: {ELASTICSEARCH_BI_CLIENT.info()}"
             )
+    else:
+        print("Elasticsearch not correctly configured")
 
+
+# advanced search
 ELASTICSEARCH_DSL_ENABLED = (
     secrets.get_secret("ELASTICSEARCH_DSL_ENABLED", False) == "True"
 )
 if ELASTICSEARCH_DSL_ENABLED:
     ELASTICSEARCH_DSL_HOST = secrets.get_secret("ELASTICSEARCH_DSL_HOST")
     if ELASTICSEARCH_DSL_HOST:
-        elastic_client_settings = {"hosts": ELASTICSEARCH_DSL_HOST}
+        elastic_search_conf = {"hosts": ELASTICSEARCH_DSL_HOST}
 
         ELASTICSEARCH_DSL_PASSWORD = secrets.get_secret("ELASTICSEARCH_DSL_PASSWORD")
         if ELASTICSEARCH_DSL_PASSWORD:
-            elastic_client_settings["basic_auth"] = (
+            elastic_search_conf["basic_auth"] = (
                 "elastic",
                 ELASTICSEARCH_DSL_PASSWORD,
             )
-        ca_path = "/opt/deploy/intel_owl/certs/elastic_ca/ca.crt"
-        cert_path = "/opt/deploy/intel_owl/certs/elastic_instance/elasticsearch.crt"
         if "elasticsearch:9200" in ELASTICSEARCH_DSL_HOST:
             # in case we use Elastic as container we need the generated
             # in case we use Elastic as external service it should have a valid cert
-            elastic_client_settings["verify_certs"] = cert_path
-            elastic_client_settings["ca_certs"] = ca_path
-        ELASTICSEARCH_DSL = {"default": elastic_client_settings}
-
-        ELASTICSEARCH_DSL_INDEX_SETTINGS = {
-            "number_of_shards": int(
-                secrets.get_secret("ELASTICSEARCH_DSL_NO_OF_SHARDS")
-            ),
-            "number_of_replicas": int(
-                secrets.get_secret("ELASTICSEARCH_DSL_NO_OF_REPLICAS")
-            ),
-        }
+            elastic_search_conf["verify_certs"] = (
+                "/opt/deploy/intel_owl/certs/elastic_instance/elasticsearch.crt"
+            )
+            elastic_search_conf["ca_certs"] = (
+                "/opt/deploy/intel_owl/certs/elastic_ca/ca.crt"
+            )
+        ELASTICSEARCH_DSL_CLIENT = Elasticsearch(**elastic_search_conf)
+        if not ELASTICSEARCH_DSL_CLIENT.ping():
+            print(
+                f"ELASTICSEARCH DSL client configuration did not connect correctly: {ELASTICSEARCH_DSL_CLIENT.info()}"
+            )
     else:
         print(
             "you have to configure ELASTIC_HOST with the URL of your ElasticSearch instance"
         )
-else:
-    ELASTICSEARCH_DSL_AUTOSYNC = False
-    ELASTICSEARCH_DSL = {
-        "default": {"hosts": ""},
-    }
diff --git a/intel_owl/tasks.py b/intel_owl/tasks.py
@@ -20,7 +20,6 @@
 from django_celery_beat.models import PeriodicTask
 from elasticsearch import ApiError
 from elasticsearch.helpers import bulk
-from elasticsearch_dsl import connections
 
 from api_app.choices import ReportStatus, Status
 from intel_owl import secrets
@@ -496,7 +495,7 @@ def _convert_report_to_elastic_document(
         )
         logger.info(f"documents to add to elastic: {len(all_report_document_list)}")
         try:
-            bulk(connections.get_connection(), all_report_document_list)
+            bulk(settings.ELASTICSEARCH_DSL_CLIENT, all_report_document_list)
         except ApiError as error:
             logger.critical(error)
         else:

diff --git a/requirements/project-requirements.txt b/requirements/project-requirements.txt
@@ -11,7 +11,6 @@ django-ses == 4.4.0
 django-iam-dbauth==0.2.1
 django-prettyjson==0.4.1
 django-silk==5.3.2
-django-elasticsearch-dsl==8.0
 django-treebeard==4.7
 django-solo==2.4.0
 django_extensions==3.2.3

diff --git a/tests/api_app/test_views.py b/tests/api_app/test_views.py
@@ -1368,6 +1368,7 @@ def test_get_403(self):
 
 
 @override_settings(ELASTICSEARCH_DSL_ENABLED=True)
+@override_settings(ELASTICSEARCH_DSL_CLIENT=None)
 class ElasticTestCase(CustomViewSetTestCase):
     uri = reverse("plugin-report-queries")
 

diff --git a/tests/intel_owl/test_tasks.py b/tests/intel_owl/test_tasks.py
@@ -24,7 +24,7 @@
 
 @patch("intel_owl.tasks.get_environment", return_value="unittest")
 @patch("intel_owl.tasks.now", return_value=_now)
-@patch("intel_owl.tasks.connections.get_connection")
+@override_settings(ELASTICSEARCH_DSL_CLIENT=None)
 class SendElasticTestCase(CustomTestCase):
 
     def setUp(self):