Modify ScannableURI fields #49 #285

    * Create test for ScannableURI API

Signed-off-by: Jono Yang <[email protected]>

minecode/api.py

@@ -114,7 +114,7 @@ def get_next_download_url(self, request, *args, **kwargs):
                 scannable_uri.save()
             else:
                 response = {
-                    'package_uuid': "",
+                    'scannable_uri_uuid': "",
                     'download_url': "",
                 }
             return Response(response)

minecode/migrations/0032_remove_scannableuri_minecode_sc_scan_st_d6a459_idx_and_more.py

@@ -0,0 +1,64 @@
+# Generated by Django 5.0.1 on 2024-02-15 23:16
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("minecode", "0031_importableuri"),
+        ("packagedb", "0083_delete_apiuser"),
+    ]
+
+    operations = [
+        migrations.RemoveIndex(
+            model_name="scannableuri",
+            name="minecode_sc_scan_st_d6a459_idx",
+        ),
+        migrations.AlterUniqueTogether(
+            name="scannableuri",
+            unique_together=set(),
+        ),
+        migrations.AddField(
+            model_name="scannableuri",
+            name="scan_date",
+            field=models.DateTimeField(
+                blank=True,
+                db_index=True,
+                help_text="Timestamp set to the date when a scan was taken by a worker",
+                null=True,
+            ),
+        ),
+        migrations.AddField(
+            model_name="scannableuri",
+            name="scan_project_url",
+            field=models.CharField(
+                blank=True,
+                db_index=True,
+                help_text="URL to scan project for this Package",
+                max_length=2048,
+                null=True,
+            ),
+        ),
+        migrations.AddField(
+            model_name="scannableuri",
+            name="uuid",
+            field=models.UUIDField(default=uuid.uuid4, null=True),
+        ),
+        migrations.AddIndex(
+            model_name="scannableuri",
+            index=models.Index(
+                fields=["scan_status", "scan_date", "last_status_poll_date"],
+                name="minecode_sc_scan_st_5e04d7_idx",
+            ),
+        ),
+        migrations.RemoveField(
+            model_name="scannableuri",
+            name="scan_request_date",
+        ),
+        migrations.RemoveField(
+            model_name="scannableuri",
+            name="scan_uuid",
+        ),
+    ]

minecode/migrations/0033_scannableuri_populate_uuid.py

@@ -0,0 +1,43 @@
+# Generated by Django 5.0.1 on 2024-02-16 00:16
+
+from django.db import migrations
+import uuid
+
+
+def populate_uuid(apps, schema_editor):
+    ScannableURI = apps.get_model("minecode", "ScannableURI")
+    unupdated = []
+    scannable_uris = ScannableURI.objects.all().iterator(chunk_size=5000)
+    for i, scannable_uri in enumerate(scannable_uris):
+        if  i > 0 and not i % 5000:
+            ScannableURI.objects.bulk_update(
+                objs=unupdated,
+                fields=[
+                    "uuid"
+                ]
+            )
+            unupdated = []
+        scannable_uri.uuid = uuid.uuid4()
+        unupdated.append(scannable_uri)
+
+    if unupdated:
+        ScannableURI.objects.bulk_update(
+            objs=unupdated,
+            fields=[
+                "uuid",
+            ]
+        )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        (
+            "minecode",
+            "0032_remove_scannableuri_minecode_sc_scan_st_d6a459_idx_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.RunPython(populate_uuid, reverse_code=migrations.RunPython.noop),
+    ]

minecode/migrations/0034_scannableuri_alter_uuid_field.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.1 on 2024-02-16 00:16
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("minecode", "0033_scannableuri_populate_uuid"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="scannableuri",
+            name="uuid",
+            field=models.UUIDField(default=uuid.uuid4, editable=False, unique=True),
+        ),
+    ]

minecode/models.py

@@ -737,15 +737,22 @@ class ScannableURI(BaseURI):
 
     class Meta:
         verbose_name = 'Scannable URI'
-        unique_together = ['canonical', 'scan_project_url']
 
         indexes = [
             # to get the scannables
             models.Index(
-                fields=['scan_status', 'scan_date', 'last_status_poll_date', ]),
+                fields=[
+                    'scan_status',
+                    'scan_date',
+                    'last_status_poll_date',
+                ]
+            ),
             # ordered by for the main queue query e.g. '-priority'
             models.Index(
-                fields=['-priority'])
+                fields=[
+                    '-priority'
+                ]
+            )
         ]
 
     def save(self, *args, **kwargs):

minecode/tests/test_api.py

@@ -0,0 +1,80 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from uuid import uuid4
+import json
+import os
+
+from django.contrib.postgres.search import SearchVector
+from django.test import TestCase
+from django.urls import reverse
+from django.utils import timezone
+from rest_framework import status
+from rest_framework.test import APIClient
+
+from minecode.models import PriorityResourceURI
+from minecode.models import ScannableURI
+from minecode.utils_test import JsonBasedTesting
+from packagedb.models import Package
+from packagedb.models import PackageContentType
+from packagedb.models import PackageSet
+from packagedb.models import Resource
+from minecode.models import ScannableURI
+
+from unittest import mock
+from univers.versions import MavenVersion
+
+class ScannableURIAPITestCase(JsonBasedTesting, TestCase):
+    test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles')
+
+    def setUp(self):
+        self.package1 = Package.objects.create(
+            download_url='https://test-url.com/package1.tar.gz',
+            type='type1',
+            name='name1',
+            version='1.0',
+        )
+        self.scannable_uri1 = ScannableURI.objects.create(
+            uri='https://test-url.com/package1.tar.gz',
+            package=self.package1
+        )
+
+        self.package2 = Package.objects.create(
+            download_url='https://test-url.com/package2.tar.gz',
+            type='type2',
+            name='name2',
+            version='2.0',
+        )
+        self.scannable_uri2 = ScannableURI.objects.create(
+            uri='https://test-url.com/package2.tar.gz',
+            package=self.package2
+        )
+
+        self.client = APIClient()
+
+    def test_api_scannable_uri_list_endpoint(self):
+        response = self.client.get('/api/scan_queue/')
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(2, response.data.get('count'))
+
+    def test_api_scannable_uri_get_next_download_url(self):
+        response = self.client.get('/api/scan_queue/get_next_download_url/')
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data.get('scannable_uri_uuid'), self.scannable_uri1.uuid)
+        self.assertEqual(response.data.get('download_url'), self.scannable_uri1.uri)
+
+        response = self.client.get('/api/scan_queue/get_next_download_url/')
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data.get('scannable_uri_uuid'), self.scannable_uri2.uuid)
+        self.assertEqual(response.data.get('download_url'), self.scannable_uri2.uri)
+
+        response = self.client.get('/api/scan_queue/get_next_download_url/')
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data.get('scannable_uri_uuid'), '')
+        self.assertEqual(response.data.get('download_url'), '')