Merge pull request #79 from keshav-space/npm-on-demand

Add support for npm priority queue

minecode/tests/test_npm.py

@@ -13,15 +13,18 @@
 import os
 import re
 
+from django.test import TestCase as DjangoTestCase
 from mock import Mock
 from mock import patch
+from packagedcode.npm import NpmPackageJsonHandler
+from packageurl import PackageURL
 
-from minecode.utils_test import mocked_requests_get
-from minecode.utils_test import JsonBasedTesting
-
+import packagedb
 from minecode import mappers
 from minecode import route
 from minecode.models import ResourceURI
+from minecode.utils_test import JsonBasedTesting
+from minecode.utils_test import mocked_requests_get
 from minecode.visitors import npm
 
 
@@ -166,3 +169,40 @@ def test_regex_npm_mapper(self):
         regex = re.compile(r'^https://registry.npmjs.org/[^\/]+$')
         result = re.match(regex, 'https://registry.npmjs.org/react-mobile-navigation-modal')
         self.assertTrue(result)
+
+class NpmPriorityQueueTests(JsonBasedTesting, DjangoTestCase):
+    test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles')
+
+    def setUp(self):
+        super(NpmPriorityQueueTests, self).setUp()
+        expected_json_loc = self.get_test_loc('npm/lodash_package-expected.json')
+        with open(expected_json_loc) as f:
+            self.expected_json_contents = json.load(f)
+
+        self.scan_package = NpmPackageJsonHandler._parse(
+            json_data=self.expected_json_contents,
+        )
+
+    def test_get_package_json(self, regen=False):
+        json_contents = npm.get_package_json(
+            namespace=self.scan_package.namespace,
+            name=self.scan_package.name,
+            version=self.scan_package.version
+        )
+        if regen:
+            with open(self.expected_pom_loc, 'w') as f:
+                f.write(json_contents)
+        self.assertEqual(self.expected_json_contents, json_contents)
+
+    def test_map_npm_package(self):
+        package_count = packagedb.models.Package.objects.all().count()
+        self.assertEqual(0, package_count)
+        package_url = PackageURL.from_string(self.scan_package.purl)
+        npm.map_npm_package(package_url)
+        package_count = packagedb.models.Package.objects.all().count()
+        self.assertEqual(1, package_count)
+        package = packagedb.models.Package.objects.all().first()
+        expected_purl_str = 'pkg:npm/[email protected]'
+        expected_download_url = 'https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz'
+        self.assertEqual(expected_purl_str, package.purl)
+        self.assertEqual(expected_download_url, package.download_url)

minecode/tests/testfiles/npm/lodash_package-expected.json

@@ -0,0 +1,82 @@
+{
+   "name":"lodash",
+   "version":"4.17.21",
+   "description":"Lodash modular utilities.",
+   "keywords":[
+      "modules",
+      "stdlib",
+      "util"
+   ],
+   "homepage":"https://lodash.com/",
+   "repository":{
+      "type":"git",
+      "url":"git+https://github.com/lodash/lodash.git"
+   },
+   "icon":"https://lodash.com/icon.svg",
+   "license":"MIT",
+   "main":"lodash.js",
+   "author":{
+      "name":"John-David Dalton",
+      "email":"[email protected]"
+   },
+   "contributors":[
+      {
+         "name":"John-David Dalton",
+         "email":"[email protected]"
+      },
+      {
+         "name":"Mathias Bynens",
+         "email":"[email protected]"
+      }
+   ],
+   "scripts":{
+      "test":"echo \"See https://travis-ci.org/lodash-archive/lodash-cli for testing details.\""
+   },
+   "gitHead":"c6e281b878b315c7a10d90f9c2af4cdb112d9625",
+   "bugs":{
+      "url":"https://github.com/lodash/lodash/issues"
+   },
+   "_id":"[email protected]",
+   "_nodeVersion":"14.15.5",
+   "_npmVersion":"6.14.11",
+   "dist":{
+      "integrity":"sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
+      "shasum":"679591c564c3bffaae8454cf0b3df370c3d6911c",
+      "tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+      "fileCount":1054,
+      "unpackedSize":1412415,
+      "npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgMS3ZCRA9TVsSAnZWagAA8+4P/jx+SJ6Ue5oAJjz0L7gw\nLDD5YvP8aoliFq4GYkwUXfVQvOwomIPfa+U5Kao/hDfuwFQ/Bq5D5nSsl2bj\nrjJgvlKXna0SId8AgDgY2fB7zSfninuJvalY4iTWMN8DFSpG0XE2QFfoKpd3\njDmuzcNtgr79QV6DgjOVkHiP1IGNDlLTc1QEKiwo/5CdGQi1q/iCj6dViQMJ\nByuuuV2Qzi3f/FI25cG797WZar1MHhhlcnB50HiVBGp54IZOyuqdqWPduZQo\nvhONtonxPGBm3/J+uAkeUSSyL3Ud+FzLvdg8WEI9gDL0yvU4k0FcsnOONEYn\nngLaKEsw2xAnPBYW3Lf73Jnpwx6FAT3k49kgzxiNYSxEo7x4wiuNtBoDMyNw\nEKj6SZ0bUNmaJgiMfDnnDjCKjI3JrO1hho8z6CkwuvxuWLlW9wSsVayggzAI\nEhfeTeISugVHh332oDY2MI/Ysu8MnVN8fGmqeYQBBFj3aWatuA2NvVjACnX/\n54G7FtCU8TxZpm9shFRSopBx8PeI3r+icx1CT8YVFypY416PLnidHyqtME1G\neuRd1nWEz18hvVUAEHmuvHo+EPP3tITmTTUPQcZGMdBcZC+4UBmPMWX466HE\nbHw4aOnUWMa0sWfsERC5xzRZAb4lgMPEoTOnZyN4usMy7x9TzGZKZvU24HUE\nmpae\r\n=NOmG\r\n-----END PGP SIGNATURE-----\r\n",
+      "signatures":[
+         {
+            "keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA",
+            "sig":"MEUCIF3Yithbtmy1aEBNlfNWbLswAfPIyQUuNUGARD3Ex2t4AiEA6TlN2ZKJCUpS/Sf2Z6MduF1BNSvayHIpu5wAcICcKXw="
+         }
+      ]
+   },
+   "_npmUser":{
+      "name":"bnjmnt4n",
+      "email":"[email protected]"
+   },
+   "directories":{
+
+   },
+   "maintainers":[
+      {
+         "name":"mathias",
+         "email":"[email protected]"
+      },
+      {
+         "name":"jdalton",
+         "email":"[email protected]"
+      },
+      {
+         "name":"bnjmnt4n",
+         "email":"[email protected]"
+      }
+   ],
+   "_npmOperationalInternal":{
+      "host":"s3://npm-registry-packages",
+      "tmp":"tmp/lodash_4.17.21_1613835736675_0.01913912595366596"
+   },
+   "_hasShrinkwrap":false
+}

minecode/visitors/npm.py

@@ -10,14 +10,17 @@
 
 import logging
 import json
+import requests
 
 from packageurl import PackageURL
 
 from packagedcode.npm import npm_api_url
 from packagedcode.npm import split_scoped_package_name
+from packagedcode.npm import NpmPackageJsonHandler
 
 from minecode import seed
 from minecode import visit_router
+from minecode import priority_router
 from minecode.visitors import NonPersistentHttpVisitor
 from minecode.visitors import URI
 
@@ -103,3 +106,73 @@ def get_uris(self, content):
                 data=json.dumps(doc, separators=(',', ':'), ensure_ascii=False),
                 # note: visited is True since there nothing more to visit
                 visited=True)
+
+def get_package_json(namespace, name, version):
+    """
+    Return the contents of the package.json file of the package described by the purl
+    field arguments in a string.
+    """
+    # Create URLs using purl fields
+    url = npm_api_url(
+        namespace=namespace,
+        name=name,
+        version=version,
+    )
+
+    try:
+        response = requests.get(url)
+        response.raise_for_status() 
+        return response.json()
+    except requests.exceptions.HTTPError as err:
+        logger.error(f"HTTP error occurred: {err}")
+
+
+def map_npm_package(package_url):
+    """
+    Add a npm `package_url` to the PackageDB.
+
+    Return an error string if any errors are encountered during the process
+    """
+    from minecode.management.commands.priority_queue import add_package_to_scan_queue
+    from minecode.management.commands.run_map import merge_or_create_package
+
+    package_json = get_package_json(
+        namespace = package_url.namespace,
+        name=package_url.name,
+        version=package_url.version,
+        )
+
+    if not package_json:
+        error = f'Package does not exist on npmjs: {package_url}'
+        logger.error(error)
+        return error
+
+    package = NpmPackageJsonHandler._parse(
+        json_data=package_json
+    )
+
+    db_package, _, _, _ = merge_or_create_package(package, visit_level=0)
+    # Submit package for scanning
+    if db_package:
+        add_package_to_scan_queue(db_package)
+
+
+@priority_router.route('pkg:npm/.*')
+def process_request(purl_str):
+    """
+    Process `priority_resource_uri` containing a npm Package URL (PURL) as a
+    URI.
+
+    This involves obtaining Package information for the PURL from npm and
+    using it to create a new PackageDB entry. The package is then added to the
+    scan queue afterwards.
+    """
+    package_url = PackageURL.from_string(purl_str)
+    if not package_url.version:
+        return
+
+    error_msg = map_npm_package(package_url)
+
+    if error_msg:
+        return error_msg
+