Add addtional visitors and mappers

minecode/src/discovery/mappers/maven.py

@@ -0,0 +1,131 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import json
+import logging
+import packageurl
+from packageurl import PackageURL
+
+from commoncode.text import as_unicode
+from packagedcode.models import PackageData
+from packagedcode.maven import _parse
+
+from discovery import map_router
+from discovery.mappers import Mapper
+from discovery.utils import parse_date
+from discovery.visitors.maven import Artifact
+
+
+TRACE = False
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+if TRACE:
+    import sys
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+
+@map_router.route('maven-index://.*')
+class MavenIndexArtifactMapper(Mapper):
+    """
+    Process the minimal artifacts collected for a Maven Jar or POM in an
+    index visit.
+    """
+
+    def get_packages(self, uri, resource_uri):
+        yield get_mini_package(resource_uri.data, uri, resource_uri.package_url)
+
+
+def get_mini_package(data, uri, purl):
+    """
+    Return a MavenPomPackage built from the minimal artifact data available in a
+    nexus index, given a `data` JSON string, a `uri` string and a `purl`
+    PacxkageURL string. Return None if the package cannot be built.
+    """
+    if not data:
+        return
+
+    artdata = json.loads(data)
+
+    # FIXME: this should a slot in Artifact
+    download_url = artdata.pop('download_url')
+    # FIXME: what if this is an ArtifactExtended??
+    artifact = Artifact(**artdata)
+
+    if purl:
+        if isinstance(purl, str):
+            purl = PackageURL.from_string(purl)
+        assert isinstance(purl, PackageURL)
+
+    qualifiers = None
+    if purl and purl.qualifiers:
+        qualifiers = packageurl.normalize_qualifiers(purl.qualifiers, encode=False)
+    if qualifiers:
+        assert isinstance(qualifiers, dict)
+    logger.debug('get_mini_package: qualifiers: {}'.format(qualifiers))
+
+    package = PackageData(
+        type='maven',
+        namespace=artifact.group_id,
+        name=artifact.artifact_id,
+        version=artifact.version,
+        qualifiers=qualifiers,
+        description=artifact.description,
+        download_url=download_url,
+        release_date=parse_date(artifact.last_modified),
+        size=artifact.size,
+        sha1=artifact.sha1 or None,
+    )
+    logger.debug('get_mini_package: package.qualifiers: {}'.format(package.qualifiers))
+    logger.debug('get_mini_package for uri: {}, package: {}'.format(uri, package))
+    return package
+
+
+# FIXME this should be valid for any POM
+@map_router.route('https?://repo1.maven.org/maven2/.*\.pom')
+class MavenPomMapper(Mapper):
+    """
+    Map a proper full POM visited as XML.
+    """
+    def get_packages(self, uri, resource_uri):
+
+        logger.debug('MavenPomMapper.get_packages: uri: {}, resource_uri: {}, purl:'
+                     .format(uri, resource_uri.uri, resource_uri.package_url))
+        package = get_package(resource_uri.data, resource_uri.package_url)
+        if package:
+            logger.debug('MavenPomMapper.get_packages: uri: {}, package: {}'
+                         .format(uri, package))
+            yield package
+
+
+def get_package(text, package_url=None,
+                baseurl='https://repo1.maven.org/maven2'):
+    """
+    Return a ScannedPackage built from a POM XML string `text`.
+    """
+    text = as_unicode(text)
+    package = _parse(
+        datasource_id='maven_pom',
+        package_type='maven',
+        primary_language='Java',
+        text=text
+    )
+    if package:
+        # FIXME: this should be part of the parse call
+        if package_url:
+            purl = PackageURL.from_string(package_url)
+            package.set_purl(purl)
+        # Build proper download_url given a POM: this must be the URL for
+        # the Jar which is the key to the PackageDB record
+        # FIXME the download is hardcoded to Maven Central?
+        # package.download_url = package.repository_download_url(baseurl=baseurl)
+    return package

minecode/src/discovery/mappers/sourceforge.py

@@ -0,0 +1,93 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import json
+
+from packagedcode import models as scan_models
+
+from discovery import map_router
+from discovery.mappers import Mapper
+
+
+@map_router.route('https?://sourceforge.net/api/project/name/[a-z0-9.-]+/json',
+                  'https?://sourceforge.net/rest/p/[a-z0-9.-]+')
+class SourceforgeProjectJsonAPIMapper(Mapper):
+
+    def get_packages(self, uri, resource_uri):
+        """
+        Yield Package built from resource_uri record for a single
+        package version.
+        Yield as many Package as there are download URLs.
+        """
+        metadata = json.loads(resource_uri.data)
+        return build_packages_from_metafile(metadata, resource_uri.package_url)
+
+
+def build_packages_from_metafile(metadata, purl=None):
+    """
+    Yield Package built from package a `metadata` content
+    metadata: json metadata content
+    purl: String value of the package url of the ResourceURI object
+    """
+    short_desc = metadata.get('summary')
+    long_desc = metadata.get('short_description')
+    descriptions = [d for d in (short_desc, long_desc) if d and d.strip()]
+    description = '\n'.join(descriptions)
+    name = metadata.get('shortname')
+    # short name is more reasonable here for name, since it's an abbreviation
+    # for the project and unique
+    if not name:
+        name = metadata.get('name')
+    if name:
+        common_data = dict(
+            type='sourceforge',
+            name=metadata.get('shortname', metadata.get('name')),
+            description=description,
+            homepage_url=metadata.get('external_homepage', metadata.get('url')),
+        )
+
+        devs = metadata.get('developers') or []
+        for dev in devs:
+            parties = common_data.get('parties')
+            if not parties:
+                common_data['parties'] = []
+            if dev.get('name'):
+                common_data['parties'].append(
+                    scan_models.Party(name=dev.get('name'), role='contributor', url=dev.get('url')))
+
+        categories = metadata.get('categories', {})
+        languages = categories.get('language', [])
+        langs = []
+        for lang in languages:
+            lshort = lang.get('shortname')
+            if lshort:
+                langs.append(lshort)
+        langs = ', '.join(langs)
+        common_data['primary_language'] = langs or None
+
+        declared_licenses = []
+        licenses = categories.get('license') or []
+        for l in licenses:
+            license_name = l.get('fullname')
+            # full name is first priority than shortname since shortname is like gpl, it doesn't show detailed gpl version etc.
+            if license_name:
+                declared_licenses.append(l.get('shortname'))
+            if license_name:
+                declared_licenses.append(license_name)
+        if declared_licenses:
+            common_data['declared_license'] = '\n'.join(declared_licenses)
+
+        keywords = []
+        topics = categories.get('topic', [])
+        for topic in topics:
+            keywords.append(topic.get('shortname'))
+        common_data['keywords'] = keywords or None
+        package = scan_models.Package(**common_data)
+        package.set_purl(purl)
+        yield package

minecode/src/discovery/visitors/java_stream.LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Gustav Arngården
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

minecode/src/discovery/visitors/java_stream.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+
+# The MIT License (MIT)
+#
+# Copyright (c) 2014 Gustav Arngården
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+"""
+Reading from Java DataInputStream format.
+"""
+
+import struct
+
+
+class DataInputStream(object):
+    def __init__(self, stream):
+        self.stream = stream
+
+    def read(self, n=1):
+        data = self.stream.read(n)
+        if len(data) != n:
+            # this is a problem but in most cases we have reached EOF
+            raise EOFError
+        return data
+
+    def read_byte(self):
+        return struct.unpack('b', self.read(1))[0]
+
+    def read_long(self):
+        return struct.unpack('>q', self.read(8))[0]
+
+    def read_utf(self):
+        utf_length = struct.unpack('>H', self.read(2))[0]
+        return self.read(utf_length)
+
+    def read_int(self):
+        return struct.unpack('>i', self.read(4))[0]

minecode/src/discovery/visitors/java_stream.py.ABOUT

@@ -0,0 +1,11 @@
+about_resource: java_stream.py
+name: java_stream.py
+version: 7d118ceef9746981e6bc198861125ca2bb6f920f
+homepage_url: https://github.com/arngarden/python_java_datastream
+owner: Gustav Arngården
+copyright: Copyright (c) 2014 Gustav Arngården
+license: mit
+download_url: https://raw.githubusercontent.com/arngarden/python_java_datastream/7d118ceef9746981e6bc198861125ca2bb6f920f/data_input_stream.py
+
+vcs_tool: git
+vcs_repo: https://github.com/arngarden/python_java_datastream