Refactor the CodebaseRelationListView #858 (#864)

Signed-off-by: Thomas Druez <tdruez@nexb.com>

CHANGELOG.rst

@@ -4,6 +4,10 @@ Changelog
 v32.5.3 (unreleased)
 --------------------
 
+- Improve the performance of the codebase relations list view to support large number
+  of entries.
+  https://github.com/nexB/scancode.io/issues/858
+
 - Improve DiscoveredPackageListView query performances refining the prefetch_related.
   https://github.com/nexB/scancode.io/issues/856
 

scanpipe/api/serializers.py

@@ -385,9 +385,11 @@ class CodebaseRelationSerializer(serializers.ModelSerializer):
     class Meta:
         model = CodebaseRelation
         fields = [
-            "from_resource",
             "to_resource",
+            "status",
             "map_type",
+            "score",
+            "from_resource",
         ]
 
 

scanpipe/filters.py

@@ -32,6 +32,7 @@
 from django_filters.widgets import LinkWidget
 from packageurl.contrib.django.filters import PackageURLFilter
 
+from scanpipe.models import CodebaseRelation
 from scanpipe.models import CodebaseResource
 from scanpipe.models import DiscoveredDependency
 from scanpipe.models import DiscoveredPackage
@@ -334,20 +335,25 @@ def filter(self, qs, value):
         return qs
 
 
+MAP_TYPE_CHOICES = (
+    ("about_file", "about file"),
+    ("java_to_class", "java to class"),
+    ("jar_to_source", "jar to source"),
+    ("js_compiled", "js compiled"),
+    ("js_colocation", "js colocation"),
+    ("js_path", "js path"),
+    ("path", "path"),
+    ("sha1", "sha1"),
+)
+
+
 class RelationMapTypeFilter(django_filters.ChoiceFilter):
     def __init__(self, *args, **kwargs):
         kwargs["choices"] = (
             ("none", "No map"),
             ("any", "Any map"),
             ("many", "Many map"),
-            ("about_file", "about file"),
-            ("java_to_class", "java to class"),
-            ("jar_to_source", "jar to source"),
-            ("js_compiled", "js compiled"),
-            ("js_colocation", "js colocation"),
-            ("js_path", "js path"),
-            ("path", "path"),
-            ("sha1", "sha1"),
+            *MAP_TYPE_CHOICES,
         )
         super().__init__(*args, **kwargs)
 
@@ -367,6 +373,19 @@ def filter(self, qs, value):
             return qs.status()
         return super().filter(qs, value)
 
+    @staticmethod
+    def get_status_choices(qs, include_any=False):
+        """Return the list of unique status for resources in ``project``."""
+        default_choices = [(EMPTY_VAR, "No status")]
+        if include_any:
+            default_choices.append(("any", "Any status"))
+
+        status_values = (
+            qs.order_by("status").values_list("status", flat=True).distinct()
+        )
+        value_choices = [(status, status) for status in status_values if status]
+        return default_choices + value_choices
+
 
 class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
     dropdown_widget_fields = [
@@ -452,22 +471,17 @@ class Meta:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         if status_filter := self.filters.get("status"):
-            status_filter.extra.update({"choices": self.get_status_choices()})
+            status_filter.extra.update(
+                {
+                    "choices": status_filter.get_status_choices(
+                        self.queryset, include_any=True
+                    )
+                }
+            )
 
         license_expression_filer = self.filters["detected_license_expression"]
         license_expression_filer.extra["widget"] = HasValueDropdownWidget()
 
-    def get_status_choices(self):
-        default_choices = [
-            (EMPTY_VAR, "No status"),
-            ("any", "Any status"),
-        ]
-        status_values = (
-            self.queryset.order_by("status").values_list("status", flat=True).distinct()
-        )
-        value_choices = [(status, status) for status in status_values if status]
-        return default_choices + value_choices
-
     @classmethod
     def filter_for_lookup(cls, field, lookup_type):
         """Add support for JSONField storing "list" using the JSONListFilter."""
@@ -632,3 +646,44 @@ class Meta:
             "model",
             "message",
         ]
+
+
+class RelationFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
+    dropdown_widget_fields = [
+        "status",
+        "map_type",
+    ]
+
+    search = django_filters.CharFilter(
+        label="Search",
+        field_name="to_resource__path",
+        lookup_expr="icontains",
+    )
+    sort = django_filters.OrderingFilter(
+        label="Sort",
+        fields=[
+            "from_resource",
+            "to_resource",
+            "map_type",
+        ],
+    )
+    map_type = django_filters.ChoiceFilter(choices=MAP_TYPE_CHOICES)
+    status = StatusFilter(field_name="to_resource__status")
+
+    class Meta:
+        model = CodebaseRelation
+        fields = [
+            "search",
+            "map_type",
+            "status",
+        ]
+
+    def __init__(self, *args, **kwargs):
+        project = kwargs.pop("project")
+        super().__init__(*args, **kwargs)
+        if project:
+            status_filter = self.filters.get("status")
+            qs = CodebaseResource.objects.filter(project=project)
+            status_filter.extra.update(
+                {"choices": status_filter.get_status_choices(qs)}
+            )

scanpipe/models.py

@@ -2251,6 +2251,17 @@ class Meta:
     def __str__(self):
         return f"{self.from_resource.pk} > {self.to_resource.pk} using {self.map_type}"
 
+    @property
+    def status(self):
+        return self.to_resource.status
+
+    @property
+    def score(self):
+        score = self.extra_data.get("path_score", "")
+        if diff_ratio := self.extra_data.get("diff_ratio", ""):
+            score += f" diff_ratio: {diff_ratio}"
+        return score
+
 
 class VulnerabilityMixin(models.Model):
     """Add the vulnerability related fields and methods."""

scanpipe/templates/scanpipe/base.html

@@ -74,7 +74,7 @@
       #project-extra-data pre {background-color: initial; color: initial; padding: initial; white-space: pre-wrap; word-break: break-all;}
       #codebase-relation-list table {table-layout: fixed;}
       #codebase-relation-list th#column-status {width: 110px;}
-      #codebase-relation-list th#column-related_from__map_type {width: 145px;}
+      #codebase-relation-list th#column-map_type {width: 145px;}
     </style>
     {% block extrahead %}{% endblock %}
   </head>

scanpipe/templates/scanpipe/includes/filter_dropdown_choices_field.html

@@ -1,7 +1,7 @@
 <div class="dropdown is-hoverable {% if is_right %}is-right{% endif %}">
   <div class="dropdown-trigger">
     <a class="{% if filter.data %}has-text-link{% else %}is-grey-link{% endif %}" aria-haspopup="true" aria-controls="{{ filter.id_for_label }}">
-      <span class="icon width-1"><i class="fa-solid fa-filter"></i></span>
+      <span class="icon width-1 height-1"><i class="fa-solid fa-filter"></i></span>
     </a>
   </div>
   <div class="dropdown-menu" id="{{ filter.id_for_label }}" role="menu">

scanpipe/templates/scanpipe/includes/pagination_header_relations.html

@@ -2,5 +2,19 @@
 {% load humanize %}
 
 {% block paginator_count %}
-  {{ paginator.count|intcomma }} to/ resources ({{ relation_count|intcomma }} relations)
+  <span style="vertical-align: middle">
+    {{ paginator.count|intcomma }} relations
+  </span>
+  <a href="{% url 'project_resources' project.slug %}?tag=to&relation_map_type=none&status=_EMPTY_" target="_blank" class="button is-small is-info is-outlined">
+    <span>Un-mapped <strong>to/</strong> resources</span>
+    <span class="icon">
+      <i class="fa-solid fa-external-link-alt"></i>
+    </span>
+  </a>
+  <a href="{% url 'project_resources' project.slug %}?tag=from&relation_map_type=none&status=_EMPTY_" target="_blank" class="button is-small is-info is-outlined">
+    <span>Un-mapped <strong>from/</strong> resources</span>
+    <span class="icon">
+      <i class="fa-solid fa-external-link-alt"></i>
+    </span>
+  </a>
 {% endblock %}

scanpipe/templates/scanpipe/relation_list.html

@@ -20,51 +20,36 @@
     <table class="table is-bordered is-narrow is-hoverable is-fullwidth">
       {% include 'scanpipe/includes/list_view_thead.html' %}
       <tbody>
-        {% for resource in object_list %}
-          {% for relation in resource.related_from.all %}
-            <tr>
-              {% if forloop.first %}
-                <td class="break-all" rowspan="{{ resource.related_from.all|length }}">
-                  {# CAUTION: Avoid relying on get_absolute_url to prevent unnecessary query triggers #}
-                  <a href="{% url 'resource_detail' project.slug resource.path %}#viewer">{{ resource.path }}</a>
-                </td>
+        {% for relation in object_list %}
+          <tr>
+            <td class="break-all">
+              {# CAUTION: Avoid relying on get_absolute_url to prevent unnecessary query triggers #}
+              <a href="{% url 'resource_detail' project.slug relation.to_resource.path %}#viewer">{{ relation.to_resource.path }}</a>
+            </td>
+            <td class="break-all">
+              <a href="?status={{ relation.to_resource.status }}" class="is-black-link">{{ relation.to_resource.status }}</a>
+            </td>
+            <td class="break-all">
+              <a href="?map_type={{ relation.map_type }}" class="is-black-link">{{ relation.map_type }}</a>
+              {% if relation.extra_data.path_score %}
+                {{ relation.extra_data.path_score }}
+              {% endif %}
+              {% if relation.map_type == "path" and relation.to_resource.is_text and relation.from_resource.is_text %}
+                <div>
+                  <a href="{% url 'resource_diff' project.slug %}?from_path={{ relation.to_resource.path }}&to_path={{ relation.from_resource.path }}" target="_blank">diff</a>
+                  {% if relation.extra_data.diff_ratio %}
+                    ratio: {{ relation.extra_data.diff_ratio }}
+                  {% endif %}
+                </div>
               {% endif %}
-              <td>
-                <a href="?status={{ resource.status }}" class="is-black-link">{{ resource.status }}</a>
-              </td>
-              <td>
-                <a href="?relation_map_type={{ relation.map_type }}" class="is-black-link">{{ relation.map_type }}</a>
-                {% if relation.extra_data.path_score %}
-                  {{ relation.extra_data.path_score }}
-                {% endif %}
-                {% if relation.map_type == "path" and resource.is_text and relation.from_resource.is_text %}
-                  <div>
-                    <a href="{% url 'resource_diff' project.slug %}?from_path={{ resource.path }}&to_path={{ relation.from_resource.path }}" target="_blank">diff</a>
-                    {% if relation.extra_data.diff_ratio %}
-                      ratio: {{ relation.extra_data.diff_ratio }}
-                    {% endif %}
-                  </div>
-                {% endif %}
-              </td>
-              <td class="break-all">
-                <a href="{% url 'resource_detail' project.slug resource.path %}#viewer">{{ relation.from_resource.path }}</a>
-              </td>
-            </tr>
-          {% empty %}
-            <tr>
-              <td class="break-all">
-                <a class="has-text-danger" href="{% url 'resource_detail' project.slug resource.path %}#viewer">{{ resource.path }}</a>
-              </td>
-              <td>
-                <a href="?status={{ resource.status }}" class="is-black-link">{{ resource.status }}</a>
-              </td>
-              <td></td>
-              <td></td>
-            </tr>
-          {% endfor %}
+            </td>
+            <td class="break-all">
+              <a href="{% url 'resource_detail' project.slug relation.from_resource.path %}#viewer">{{ relation.from_resource.path }}</a>
+            </td>
+          </tr>
         {% empty %}
           <tr>
-            <td colspan="42" class="has-text-centered p-3">
+            <td colspan="4" class="has-text-centered p-3">
               No Relations found. <a href="?">Clear search and filters</a>
             </td>
           </tr>