From d761114d5af9922c9260833f27608b072b03b087 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:32:37 +0200 Subject: [PATCH] Refactor the CodebaseRelationListView #858 (#864) Signed-off-by: Thomas Druez --- CHANGELOG.rst | 4 + scanpipe/api/serializers.py | 4 +- scanpipe/filters.py | 95 +++++++-- scanpipe/models.py | 11 ++ scanpipe/templates/scanpipe/base.html | 2 +- .../filter_dropdown_choices_field.html | 2 +- .../includes/pagination_header_relations.html | 16 +- .../templates/scanpipe/relation_list.html | 69 +++---- .../tests/data/d2d/about_files/expected.json | 186 ++++++++++++------ scanpipe/tests/data/flume-ng-node-d2d.json | 108 ++++++---- scanpipe/tests/test_api.py | 6 +- scanpipe/tests/test_views.py | 2 +- scanpipe/views.py | 102 +++------- 13 files changed, 370 insertions(+), 237 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dcdb85e54..8d40dca70 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,10 @@ Changelog v32.5.3 (unreleased) -------------------- +- Improve the performance of the codebase relations list view to support large number + of entries. + https://github.com/nexB/scancode.io/issues/858 + - Improve DiscoveredPackageListView query performances refining the prefetch_related. https://github.com/nexB/scancode.io/issues/856 diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py index be773bb31..6e7cd5cdb 100644 --- a/scanpipe/api/serializers.py +++ b/scanpipe/api/serializers.py @@ -385,9 +385,11 @@ class CodebaseRelationSerializer(serializers.ModelSerializer): class Meta: model = CodebaseRelation fields = [ - "from_resource", "to_resource", + "status", "map_type", + "score", + "from_resource", ] diff --git a/scanpipe/filters.py b/scanpipe/filters.py index 80bb75fcf..dd9c97fbf 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -32,6 +32,7 @@ from django_filters.widgets import LinkWidget from packageurl.contrib.django.filters import PackageURLFilter +from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage @@ -334,20 +335,25 @@ def filter(self, qs, value): return qs +MAP_TYPE_CHOICES = ( + ("about_file", "about file"), + ("java_to_class", "java to class"), + ("jar_to_source", "jar to source"), + ("js_compiled", "js compiled"), + ("js_colocation", "js colocation"), + ("js_path", "js path"), + ("path", "path"), + ("sha1", "sha1"), +) + + class RelationMapTypeFilter(django_filters.ChoiceFilter): def __init__(self, *args, **kwargs): kwargs["choices"] = ( ("none", "No map"), ("any", "Any map"), ("many", "Many map"), - ("about_file", "about file"), - ("java_to_class", "java to class"), - ("jar_to_source", "jar to source"), - ("js_compiled", "js compiled"), - ("js_colocation", "js colocation"), - ("js_path", "js path"), - ("path", "path"), - ("sha1", "sha1"), + *MAP_TYPE_CHOICES, ) super().__init__(*args, **kwargs) @@ -367,6 +373,19 @@ def filter(self, qs, value): return qs.status() return super().filter(qs, value) + @staticmethod + def get_status_choices(qs, include_any=False): + """Return the list of unique status for resources in ``project``.""" + default_choices = [(EMPTY_VAR, "No status")] + if include_any: + default_choices.append(("any", "Any status")) + + status_values = ( + qs.order_by("status").values_list("status", flat=True).distinct() + ) + value_choices = [(status, status) for status in status_values if status] + return default_choices + value_choices + class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): dropdown_widget_fields = [ @@ -452,22 +471,17 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if status_filter := self.filters.get("status"): - status_filter.extra.update({"choices": self.get_status_choices()}) + status_filter.extra.update( + { + "choices": status_filter.get_status_choices( + self.queryset, include_any=True + ) + } + ) license_expression_filer = self.filters["detected_license_expression"] license_expression_filer.extra["widget"] = HasValueDropdownWidget() - def get_status_choices(self): - default_choices = [ - (EMPTY_VAR, "No status"), - ("any", "Any status"), - ] - status_values = ( - self.queryset.order_by("status").values_list("status", flat=True).distinct() - ) - value_choices = [(status, status) for status in status_values if status] - return default_choices + value_choices - @classmethod def filter_for_lookup(cls, field, lookup_type): """Add support for JSONField storing "list" using the JSONListFilter.""" @@ -632,3 +646,44 @@ class Meta: "model", "message", ] + + +class RelationFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): + dropdown_widget_fields = [ + "status", + "map_type", + ] + + search = django_filters.CharFilter( + label="Search", + field_name="to_resource__path", + lookup_expr="icontains", + ) + sort = django_filters.OrderingFilter( + label="Sort", + fields=[ + "from_resource", + "to_resource", + "map_type", + ], + ) + map_type = django_filters.ChoiceFilter(choices=MAP_TYPE_CHOICES) + status = StatusFilter(field_name="to_resource__status") + + class Meta: + model = CodebaseRelation + fields = [ + "search", + "map_type", + "status", + ] + + def __init__(self, *args, **kwargs): + project = kwargs.pop("project") + super().__init__(*args, **kwargs) + if project: + status_filter = self.filters.get("status") + qs = CodebaseResource.objects.filter(project=project) + status_filter.extra.update( + {"choices": status_filter.get_status_choices(qs)} + ) diff --git a/scanpipe/models.py b/scanpipe/models.py index e4ec33cb5..a3cb26fe6 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -2251,6 +2251,17 @@ class Meta: def __str__(self): return f"{self.from_resource.pk} > {self.to_resource.pk} using {self.map_type}" + @property + def status(self): + return self.to_resource.status + + @property + def score(self): + score = self.extra_data.get("path_score", "") + if diff_ratio := self.extra_data.get("diff_ratio", ""): + score += f" diff_ratio: {diff_ratio}" + return score + class VulnerabilityMixin(models.Model): """Add the vulnerability related fields and methods.""" diff --git a/scanpipe/templates/scanpipe/base.html b/scanpipe/templates/scanpipe/base.html index 5e6c923d8..b13784df0 100644 --- a/scanpipe/templates/scanpipe/base.html +++ b/scanpipe/templates/scanpipe/base.html @@ -74,7 +74,7 @@ #project-extra-data pre {background-color: initial; color: initial; padding: initial; white-space: pre-wrap; word-break: break-all;} #codebase-relation-list table {table-layout: fixed;} #codebase-relation-list th#column-status {width: 110px;} - #codebase-relation-list th#column-related_from__map_type {width: 145px;} + #codebase-relation-list th#column-map_type {width: 145px;} {% block extrahead %}{% endblock %} diff --git a/scanpipe/templates/scanpipe/includes/filter_dropdown_choices_field.html b/scanpipe/templates/scanpipe/includes/filter_dropdown_choices_field.html index 34234242e..60f4ed24c 100644 --- a/scanpipe/templates/scanpipe/includes/filter_dropdown_choices_field.html +++ b/scanpipe/templates/scanpipe/includes/filter_dropdown_choices_field.html @@ -1,7 +1,7 @@
- +