diff --git a/02b4dedb-9076-43ac-95e8-5d9d8c30b8e3_output.json b/02b4dedb-9076-43ac-95e8-5d9d8c30b8e3_output.json new file mode 100644 index 0000000..63fd0ab --- /dev/null +++ b/02b4dedb-9076-43ac-95e8-5d9d8c30b8e3_output.json @@ -0,0 +1,4 @@ +{ + "id": "02b4dedb-9076-43ac-95e8-5d9d8c30b8e3", + "malicious": false +} \ No newline at end of file diff --git a/5771d496-b0d5-4fcf-ba45-43903b27747d_output.json b/5771d496-b0d5-4fcf-ba45-43903b27747d_output.json new file mode 100644 index 0000000..f012cfe --- /dev/null +++ b/5771d496-b0d5-4fcf-ba45-43903b27747d_output.json @@ -0,0 +1,4 @@ +{ + "id": "5771d496-b0d5-4fcf-ba45-43903b27747d", + "malicious": false +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index c6dcd7a..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: "3.9" - -services: - redis: - image: redis:latest - restart: always - ports: - - "6379:6379" - - worker: - build: . - restart: always - environment: - CELERY_BROKER_URL: redis://redis:6379 - CELERY_RESULT_BACKEND: redis://redis:6379 - command: poetry run celery --app spamoverflow.tasks.scan worker --loglevel=info \ No newline at end of file diff --git a/endpoints.http b/endpoints.http index 4b56fbd..b4d1291 100644 --- a/endpoints.http +++ b/endpoints.http @@ -3,7 +3,7 @@ GET {{baseUrl}}/api/v1/customers/testing/emails?start=2024-03-22T02:29:03+00:00 ### Get information for a particular email. -GET {{baseUrl}}/api/v1/customers/testing/emails/ac05ffd6-c35e-463e-bb0e-cdd98e73f71e +GET {{baseUrl}}/api/v1/customers/testing/emails/62f600b1-fbff-4bab-bba1-b3d82b270966 ### Post a new email scan request. POST {{baseUrl}}/api/v1/customers/testing/emails diff --git a/instance/db.sqlite b/instance/db.sqlite index 3b913db..bb744a4 100644 Binary files a/instance/db.sqlite and b/instance/db.sqlite differ diff --git a/main.tf b/main.tf index 9df3f13..97ab44f 100644 --- a/main.tf +++ b/main.tf @@ -296,3 +296,9 @@ resource "aws_appautoscaling_policy" "app-cpu" { target_value = 20 } } + +# ----------------- QUEUE ----------------- # + +resource "aws_sqs_queue" "scan_queue" { + name = "scan" +} diff --git a/spamoverflow/__init__.py b/spamoverflow/__init__.py index 073f1a6..68b1aa0 100644 --- a/spamoverflow/__init__.py +++ b/spamoverflow/__init__.py @@ -12,6 +12,7 @@ def create_app(config_overrides=None): # Load the models from spamoverflow.models import db from spamoverflow.models.email import Email + db.init_app(app) # Create the database tables @@ -24,3 +25,6 @@ def create_app(config_overrides=None): app.register_blueprint(api) return app + + + diff --git a/spamoverflow/__pycache__/__init__.cpython-310.pyc b/spamoverflow/__pycache__/__init__.cpython-310.pyc index d327391..b27e310 100644 Binary files a/spamoverflow/__pycache__/__init__.cpython-310.pyc and b/spamoverflow/__pycache__/__init__.cpython-310.pyc differ diff --git a/spamoverflow/tasks/__pycache__/ical.cpython-310.pyc b/spamoverflow/tasks/__pycache__/ical.cpython-310.pyc deleted file mode 100644 index e9c2a24..0000000 Binary files a/spamoverflow/tasks/__pycache__/ical.cpython-310.pyc and /dev/null differ diff --git a/spamoverflow/tasks/__pycache__/scan.cpython-310.pyc b/spamoverflow/tasks/__pycache__/scan.cpython-310.pyc deleted file mode 100644 index 7e2518f..0000000 Binary files a/spamoverflow/tasks/__pycache__/scan.cpython-310.pyc and /dev/null differ diff --git a/spamoverflow/tasks/__pycache__/spamscan.cpython-310.pyc b/spamoverflow/tasks/__pycache__/spamscan.cpython-310.pyc deleted file mode 100644 index 1f86ed8..0000000 Binary files a/spamoverflow/tasks/__pycache__/spamscan.cpython-310.pyc and /dev/null differ diff --git a/spamoverflow/tasks/scan.py b/spamoverflow/tasks/scan.py deleted file mode 100644 index 8f03312..0000000 --- a/spamoverflow/tasks/scan.py +++ /dev/null @@ -1,67 +0,0 @@ -import json -import os -import subprocess - -from celery import Celery -from spamoverflow.models.email import Email -from spamoverflow.models import db -from spamoverflow import create_app - -app = create_app() - -celery = Celery(__name__) -celery.conf.broker_url = os.environ.get("CELERY_BROKER_URL") -celery.conf.result_backend = os.environ.get("CELERY_RESULT_BACKEND") -celery.conf.task_default_queue = os.environ.get("CELERY_DEFAULT_QUEUE", "scan") - -# Get the directory of the spamhammer and set up spamhammer executable -current_directory = os.path.dirname(os.path.abspath(__file__)) -parent_directory = os.path.abspath(os.path.join(current_directory, os.pardir)) -binary_name = 'spamhammer.exe' -binary_path = os.path.join(parent_directory, binary_name) - -@celery.task(name="scan") -def create_scan(tasks): - - data = tasks - email_id = data.get('id') - email_json = data.get('email_json') - - #with app.app_context(): - #Find the email - email = Email.query.filter_by(id=email_id).first() - - # Denote file paths for the input and output file for spamhammer - input_file_path = f"{email_id}_input.json" - output_file_path = f"{email_id}_output" - - try: - # Open input JSON file for writing - with open(input_file_path, 'w') as input_file: - json.dump(email_json, input_file) - - # Run spamhammer with the required arguments - arguments = ["scan", "--input", input_file_path, "--output", output_file_path] - subprocess.run([binary_path] + arguments) - - # Open output JSON file for reading - with open(f"{output_file_path}.json", 'r') as output_file: - output_data = json.load(output_file) - - # Update the scanning status - email.status = "scanned" - email.malicious = output_data.get('malicious') - db.session.commit() - - except Exception as e: - # Update teh scanning status - email.status = "failed" - db.session.commit() - - # Remove input and output JSON files as they are no longer required - if os.path.exists(input_file_path): - os.remove(input_file_path) - if os.path.exists(output_file_path): - os.remove(output_file_path) - - return email_id \ No newline at end of file diff --git a/spamoverflow/views/__pycache__/routes.cpython-310.pyc b/spamoverflow/views/__pycache__/routes.cpython-310.pyc index b44631a..1b38fc8 100644 Binary files a/spamoverflow/views/__pycache__/routes.cpython-310.pyc and b/spamoverflow/views/__pycache__/routes.cpython-310.pyc differ diff --git a/spamoverflow/views/routes.py b/spamoverflow/views/routes.py index bca4908..607d91f 100644 --- a/spamoverflow/views/routes.py +++ b/spamoverflow/views/routes.py @@ -10,16 +10,13 @@ import uuid import re from sqlalchemy import func -from celery.result import AsyncResult -from spamoverflow.tasks import scan -""" # Get the directory of the spamhammer and set up spamhammer executable current_directory = os.path.dirname(os.path.abspath(__file__)) parent_directory = os.path.abspath(os.path.join(current_directory, os.pardir)) binary_name = 'spamhammer.exe' binary_path = os.path.join(parent_directory, binary_name) -""" + api = Blueprint('api', __name__, url_prefix='/api/v1') @@ -182,15 +179,8 @@ def create_email(customer_id): db.session.add(email) db.session.commit() - # Create a task for a worker to pick up - scan_url = f"{request.host_url}api/v1/email/scan" - response = requests.post(scan_url, json={"id": id, "email_json": email_json}) - - #if response.status_code == 202: - # Successfully started the background task - # return jsonify(email.to_dict()), 201 + process_email(id, email_json) - #process_email(id, email_json) return jsonify(email.to_dict()), 201 except Exception as e: @@ -295,33 +285,4 @@ def health(): return jsonify({'status': 'Service is healthy'}), 200 except Exception as e: return jsonify({'error': 'Service is not healthy: {}'.format(str(e))}), 500 - -@api.route('/email/scan', methods=['POST']) -def create_scan(): - data = request.json - task = scan.create_scan.delay(data) - - result = { - 'task_id': task.id, - 'task_url': f'{request.host_url}api/v1/email/scan/{task.id}/status' - } - - return jsonify(result), 202 - -@api.route('/email/scan//status', methods=['GET']) -def get_task(task_id): - task_result = AsyncResult(task_id) - result = { - "task_id": task_id, - "task_status": task_result.status, - "result_url": f'{request.host_url}api/v1/email/scan/{task_id}/result' - } - return jsonify(result), 200 - -@api.route('/email/scan//result', methods=['GET']) -def get_calendar(task_id): - task_result = AsyncResult(task_id) - if task_result.status == 'SUCCESS': - return task_result.result, 200, {'Content-Type': 'text/calendar'} - else: - return jsonify({'error': 'Task not finished'}), 404 \ No newline at end of file + \ No newline at end of file