Add ability to filter on codebase resource detected values #153 (2)

Signed-off-by: Thomas Druez <tdruez@nexb.com>

scanpipe/filters.py

@@ -20,6 +20,8 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
+from django.db import models
+
 import django_filters
 from packageurl.contrib.django.filters import PackageURLFilter
 
@@ -37,6 +39,18 @@ class Meta:
         fields = ["search"]
 
 
+class JSONContainsFilter(django_filters.CharFilter):
+    """
+    Allow "contains" lookup on a JSONField converted to text.
+    This is useful for datastructures stored as list of dictionaries, where  Django's
+    default lookups are not available.
+    Require the implementation of "json_field_contains" method on the QuerySet.
+    """
+
+    def filter(self, qs, value):
+        return qs.json_field_contains(self.field_name, value)
+
+
 class ResourceFilterSet(django_filters.FilterSet):
     class Meta:
         model = CodebaseResource
@@ -56,8 +70,25 @@ class Meta:
             "mime_type",
             "file_type",
             "compliance_alert",
+            "copyrights",
+            "holders",
+            "authors",
+            "licenses",
+            "license_expressions",
+            "emails",
+            "urls",
         ]
 
+    @classmethod
+    def filter_for_lookup(cls, field, lookup_type):
+        """
+        Add support for JSONField storing "list" using the JSONListFilter.
+        """
+        if isinstance(field, models.JSONField) and field.default == list:
+            return JSONContainsFilter, {}
+
+        return super().filter_for_lookup(field, lookup_type)
+
 
 class PackageFilterSet(django_filters.FilterSet):
     purl = PackageURLFilter()

scanpipe/models.py

@@ -33,6 +33,8 @@
 from django.db import models
 from django.db import transaction
 from django.db.models import Q
+from django.db.models import TextField
+from django.db.models.functions import Cast
 from django.forms import model_to_dict
 from django.urls import reverse
 from django.utils import timezone
@@ -723,6 +725,18 @@ def has_licenses(self):
     def has_no_licenses(self):
         return self.filter(licenses=[])
 
+    def json_field_contains(self, field_name, value):
+        """
+        Filter the QuerySet looking for the `value` string in the `field_name` JSON
+        field converted into text.
+        Empty values are excluded as there's no need to cast those into text.
+        """
+        return (
+            self.filter(~Q(**{field_name: []}))
+            .annotate(**{f"{field_name}_as_text": Cast(field_name, TextField())})
+            .filter(**{f"{field_name}_as_text__contains": value})
+        )
+
 
 class ScanFieldsModelMixin(models.Model):
     """

scanpipe/templates/scanpipe/project_detail.html

@@ -142,18 +142,18 @@ <h3 id="resource-charts" class="title is-4 has-text-centered mb-3">
       </div>
       <div class="columns is-gapless">
         <div class="column">
-          <div id="holderChart"></div>
+          <div id="holderChart" data-url="{% url 'project_resources' project.uuid %}" data-field="holders"></div>
         </div>
         <div class="column">
-          <div id="copyrightChart"></div>
+          <div id="copyrightChart" data-url="{% url 'project_resources' project.uuid %}" data-field="copyrights"></div>
         </div>
       </div>
       <div class="columns is-gapless">
         <div class="column">
-          <div id="licenseKeyChart"></div>
+          <div id="licenseKeyChart" data-url="{% url 'project_resources' project.uuid %}" data-field="licenses"></div>
         </div>
         <div class="column">
-          <div id="licenseCategoryChart"></div>
+          <div id="licenseCategoryChart" data-url="{% url 'project_resources' project.uuid %}" data-field="licenses"></div>
         </div>
       </div>
       {% if file_compliance_alert %}

scanpipe/tests/test_models.py

@@ -649,7 +649,7 @@ def test_scanpipe_scan_fields_model_mixin_methods(self):
         resource.refresh_from_db()
         self.assertEqual(["mit"], resource2.license_expressions)
 
-    def test_scanpipe_codebase_resource_type_methods(self):
+    def test_scanpipe_codebase_resource_queryset_methods(self):
         CodebaseResource.objects.all().delete()
 
         file = CodebaseResource.objects.create(
@@ -727,6 +727,24 @@ def test_scanpipe_codebase_resource_type_methods(self):
         self.assertEqual(1, CodebaseResource.objects.in_package().count())
         self.assertEqual(2, CodebaseResource.objects.not_in_package().count())
 
+    def test_scanpipe_codebase_resource_queryset_json_field_contains(self):
+        resource1 = CodebaseResource.objects.create(project=self.project1, path="1")
+        resource1.holders = [
+            {"value": "H1", "end_line": 51, "start_line": 50},
+            {"value": "H2", "end_line": 61, "start_line": 60},
+        ]
+        resource1.save()
+
+        resource2 = CodebaseResource.objects.create(project=self.project1, path="2")
+        resource2.holders = [{"value": "H3", "end_line": 558, "start_line": 556}]
+        resource2.save()
+
+        qs = CodebaseResource.objects
+        self.assertQuerysetEqual([resource2], qs.json_field_contains("holders", "H3"))
+        self.assertQuerysetEqual([resource1], qs.json_field_contains("holders", "H1"))
+        expected = [resource1, resource2]
+        self.assertQuerysetEqual(expected, qs.json_field_contains("holders", "H"))
+
     def test_scanpipe_codebase_resource_descendants(self):
         path = "codebase/asgiref-3.3.0-py3-none-any.whl-extract/asgiref"
         resource = self.project_asgiref.codebaseresources.get(path=path)
@@ -803,7 +821,7 @@ def test_scanpipe_discovered_package_model_create_from_data(self):
         self.assertEqual(package_count, DiscoveredPackage.objects.count())
 
     def test_scanpipe_discovered_package_model_queryset_methods(self):
-        package1 = DiscoveredPackage.create_from_data(self.project1, package_data1)
+        DiscoveredPackage.create_from_data(self.project1, package_data1)
         inputs = [
             ("pkg:deb/debian/adduser@3.118?arch=all", 1),
             ("pkg:deb/debian/adduser@3.118", 1),