Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Websocket #2173

Merged
merged 60 commits into from
Mar 12, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
60 commits
Select commit Hold shift + click to select a range
f546fab
setup websocket
drosetti Jan 29, 2024
bb1ed9c
fixed frontend
drosetti Feb 1, 2024
8425ff9
added channel layer and used ws from different part of job workflow
drosetti Feb 6, 2024
08a1b51
added request to retry plugin
drosetti Feb 6, 2024
5640007
first request with http then open ws
drosetti Feb 7, 2024
a0f34bd
close ws when job is reported
drosetti Feb 7, 2024
2cbdb35
fix
drosetti Feb 7, 2024
8784c6a
updated props
drosetti Feb 7, 2024
0de008b
removed polling message
drosetti Feb 9, 2024
4356713
Redis as main broker
0ssigeno Feb 14, 2024
8909d76
Added change for rabbit in start
0ssigeno Feb 14, 2024
f28459d
rework auth form durin (token) to drf cookie
drosetti Feb 14, 2024
5a0657a
close ws server side
drosetti Feb 15, 2024
ecf848c
Fixes for docker mgmt
0ssigeno Feb 15, 2024
f4e32f0
Docs
0ssigeno Feb 15, 2024
0966870
Fixes
0ssigeno Feb 15, 2024
e84145b
added auth layer in ws
drosetti Feb 20, 2024
317f5a8
fix redis
drosetti Feb 20, 2024
96e23b7
fixed csrf token in frontend dev env
drosetti Feb 21, 2024
1e10aca
moved middleware + improved UX job loading
drosetti Feb 21, 2024
3be3403
moved token view from durin to drf
drosetti Feb 22, 2024
50e79f7
fix
drosetti Feb 22, 2024
c9ad009
migrate token from durin to drf
drosetti Feb 22, 2024
c2df6a7
frontend refator
drosetti Feb 22, 2024
b2819bc
frontend test user token page
drosetti Feb 23, 2024
37d306b
fixed auth tests
drosetti Feb 27, 2024
a929d99
test api to manage api token
drosetti Feb 28, 2024
b8f3502
removed comment
drosetti Feb 28, 2024
e7be422
unittest ws
drosetti Mar 5, 2024
e562685
added daphne healthcheck
drosetti Mar 5, 2024
753386d
refactor
drosetti Mar 5, 2024
e2a6e84
send data via ws when the job or a plugin is killed
drosetti Mar 5, 2024
a3031b0
refactor ws + permission to kill job
drosetti Mar 6, 2024
81aef2e
Merge branch 'develop' into websocket
drosetti Mar 6, 2024
e12ad88
linter
drosetti Mar 6, 2024
ba43dd9
suggestion
drosetti Mar 6, 2024
2339b53
prettier
drosetti Mar 6, 2024
16d348f
black formatter
drosetti Mar 6, 2024
c02ef97
isort
drosetti Mar 6, 2024
1c6fcee
fix test
drosetti Mar 6, 2024
5f7c74c
fixes
drosetti Mar 7, 2024
ebe0884
fix
drosetti Mar 7, 2024
1b39ab3
fix
drosetti Mar 7, 2024
036d9ef
renamed package es
drosetti Mar 7, 2024
ad6d9fb
removed custom collection for job channels
drosetti Mar 8, 2024
9e71337
Update intel_owl/settings/celery.py
0ssigeno Mar 8, 2024
33c8ef9
fixes
drosetti Mar 8, 2024
07e7fad
update comment
drosetti Mar 8, 2024
cc18efb
Update docs/source/Advanced-Configuration.md
mlodic Mar 11, 2024
0640541
updated v6 doc
mlodic Mar 11, 2024
9989d9e
Fixes
0ssigeno Mar 11, 2024
a972284
removed redis cache setting
drosetti Mar 11, 2024
902810a
Added daphne file
0ssigeno Mar 11, 2024
de5831c
More logs
0ssigeno Mar 11, 2024
034796d
Fixes
0ssigeno Mar 11, 2024
86b2519
deep source
drosetti Mar 11, 2024
aa3e447
fix
drosetti Mar 11, 2024
27d327b
fix
drosetti Mar 11, 2024
e5a7d25
fix
drosetti Mar 11, 2024
487f65d
Merge branch 'develop' into websocket
drosetti Mar 12, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api_app/serializers/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ class Meta:
read_only=True, slug_field="name", many=True
)

def get_pivots_to_execute(self, obj: Job):
def get_pivots_to_execute(self, obj: Job): # skipcq: PYL-R0201
return obj.pivots_to_execute.all().values_list("name", flat=True)


Expand Down
33 changes: 16 additions & 17 deletions api_app/signals.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,28 +55,27 @@ def post_delete_job(sender, instance: Job, **kwargs):
instance.analysis.delete()


@receiver(models.signals.post_migrate)
@receiver(models.signals.post_migrate, sender=BeatConfig)
def post_migrate_beat(
sender, app_config, verbosity, interactive, stdout, using, plan, apps, **kwargs
):
from django_celery_beat.models import PeriodicTask

if isinstance(sender, BeatConfig):
from intel_owl.tasks import update

for module in PythonModule.objects.filter(health_check_schedule__isnull=False):
for config in module.configs.filter(health_check_task__isnull=True):
config.generate_health_check_periodic_task()
for module in PythonModule.objects.filter(
update_schedule__isnull=False, update_task__isnull=True
):
module.generate_update_periodic_task()

for task in PeriodicTask.objects.filter(
enabled=True, task=f"{update.__module__}.{update.__name__}"
):
task.enabled &= settings.REPO_DOWNLOADER_ENABLED
task.save()
from intel_owl.tasks import update

for module in PythonModule.objects.filter(health_check_schedule__isnull=False):
for config in module.configs.filter(health_check_task__isnull=True):
config.generate_health_check_periodic_task()
for module in PythonModule.objects.filter(
update_schedule__isnull=False, update_task__isnull=True
):
module.generate_update_periodic_task()

for task in PeriodicTask.objects.filter(
enabled=True, task=f"{update.__module__}.{update.__name__}"
):
task.enabled &= settings.REPO_DOWNLOADER_ENABLED
task.save()


@receiver(models.signals.post_save, sender=PluginConfig)
Expand Down
9 changes: 4 additions & 5 deletions authentication/migrations/0002_migrate_from_durin.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,10 @@ def move_token_from_durin(apps, schema_editor):
for durin_token in AuthToken.objects.all():
# export only CLI token (client name PyIntelOwl)
# only in case user didn't have a rest framework token
if durin_token.client.name == "PyIntelOwl":
if not Token.objects.filter(user_id=durin_token.user.id).exists():
Token.objects.create(
key=durin_token.token, user_id=durin_token.user.pk
)
if durin_token.client.name == "PyIntelOwl" and not Token.objects.filter(user_id=durin_token.user.id).exists():
Token.objects.create(
key=durin_token.token, user_id=durin_token.user.pk
)

# delete durin db data
AuthToken.objects.all().delete()
Expand Down
4 changes: 2 additions & 2 deletions authentication/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ def post(request: Request) -> Response:
class LogoutView(APIView):
permission_classes = [IsAuthenticated]

def post(self, request, *args, **kwargs):
def post(self, request, *args, **kwargs): # skipcq: PYL-R0201
user = request.user
logger.info(f"perform_logout received request from '{user.username}''.")
logout(request)
Expand Down Expand Up @@ -261,7 +261,7 @@ def get(self, request, *args, **kwargs):
serializer = TokenSerializer(instance)
return Response(serializer.data)

def post(self, request):
def post(self, request): # skipcq: PYL-R0201
username = request.user.username
logger.info(f"user {username} send a request to create the API token")
serializer = TokenSerializer(data={}, context={"request": request})
Expand Down
3 changes: 0 additions & 3 deletions tests/api_app/websocket/test_websocket.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ def setUp(self) -> None:
status=Job.Status.REPORTED_WITHOUT_FAILS.value,
observable_name="8.8.8.8",
observable_classification=ObservableTypes.IP,
received_request_time=datetime.datetime.now(),
)

async def test_job_unauthorized(self):
Expand Down Expand Up @@ -115,7 +114,6 @@ async def test_job_running(self):
status=Job.Status.PENDING.value,
observable_name="test.com",
observable_classification=ObservableTypes.DOMAIN,
received_request_time=datetime.datetime.now(),
)
class_dns_python_module, _ = await sync_to_async(
PythonModule.objects.get_or_create
Expand Down Expand Up @@ -222,7 +220,6 @@ async def test_job_killed(self):
status=Job.Status.RUNNING.value,
observable_name="test.com",
observable_classification=ObservableTypes.DOMAIN,
received_request_time=datetime.datetime.now(),
)

await sync_to_async(self.client.force_login)(self.user)
Expand Down
Loading