Merge branch 'aboutcode-org#295-requirements.txt' of https://github.c…

…om/Abhishek-Dev09/scancode-toolkit into aboutcode-org#295-requirements.txt

src/packagedcode/__init__.py

@@ -33,6 +33,7 @@
 from packagedcode import conda
 from packagedcode import cargo
 from packagedcode import freebsd
+from packagedcode import golang
 from packagedcode import haxe
 from packagedcode import maven
 from packagedcode import npm
@@ -73,6 +74,7 @@
     models.ChromeExtension,
     models.IOSApp,
     pypi.PythonPackage,
+    golang.GolangPackage,
     models.CabPackage,
     models.MsiInstallerPackage,
     models.InstallShieldPackage,

src/packagedcode/go_mod.py

@@ -0,0 +1,210 @@
+
+# Copyright (c) 2019 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/scancode-toolkit/
+# The ScanCode software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode require an acknowledgment.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with ScanCode or any ScanCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  ScanCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  ScanCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
+
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import io
+import logging
+import re
+
+import attr
+
+
+"""
+https://golang.org/ref/mod#go.mod-files
+
+For example:
+
+module example.com/my/thing
+
+go 1.12
+
+require example.com/other/thing v1.0.2
+require example.com/new/thing v2.3.4
+exclude example.com/old/thing v1.2.3
+require (
+    example.com/new/thing v2.3.4
+    example.com/old/thing v1.2.3
+)
+require (
+    example.com/new/thing v2.3.4
+    example.com/old/thing v1.2.3
+)
+
+"""
+
+"""
+module is in the form
+module github.com/alecthomas/participle
+
+For example:
+>>> ob = GoMod()
+>>> p = ob.parse_module('module github.com/alecthomas/participle')
+>>> assert p.group('module') == ('github.com/alecthomas/participle')
+
+require or exclude can be in the form
+require github.com/davecgh/go-spew v1.1.1
+or
+exclude github.com/davecgh/go-spew v1.1.1
+or
+github.com/davecgh/go-spew v1.1.1
+
+For example:
+>>> ob = GoMod()
+
+>>> p = ob.parse_require('require github.com/davecgh/go-spew v1.1.1')
+>>> assert p.group('namespace') == ('github.com/davecgh')
+>>> assert p.group('name') == ('go-spew')
+>>> assert p.group('version') == ('v1.1.1')
+
+>>> p = ob.parse_exclude('exclude github.com/davecgh/go-spew v1.1.1')
+>>> assert p.group('namespace') == ('github.com/davecgh')
+>>> assert p.group('name') == ('go-spew')
+>>> assert p.group('version') == ('v1.1.1')
+
+>>> p = ob.parse_dep_link('github.com/davecgh/go-spew v1.1.1')
+>>> assert p.group('namespace') == ('github.com/davecgh')
+>>> assert p.group('name') == ('go-spew')
+>>> assert p.group('version') == ('v1.1.1')
+"""
+
+
+TRACE = False
+
+logger = logging.getLogger(__name__)
+
+if TRACE:
+    import sys
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+
+@attr.s()
+class GoMod(object):
+    # Regex expressions to parse different types of dependency
+    parse_module = re.compile(
+        r'^module\s'
+        r'(?P<module>[a-z].*)'
+    ).match
+
+    parse_module_name = re.compile(
+        r'^module(\s)*'
+        r'(?P<namespace>(.*))'
+        r'/'
+        r'(?P<name>[^\s]*)'
+    ).match
+
+    parse_require = re.compile(
+        r'^require(\s)*'
+        r'(?P<namespace>(.*))'
+        r'/'
+        r'(?P<name>[^\s]*)'
+        r'\s'
+        r'(?P<version>(.*))'
+    ).match
+
+    parse_exclude = re.compile(
+        r'^exclude(\s)*'
+        r'(?P<namespace>(.*))'
+        r'/'
+        r'(?P<name>[^\s]*)'
+        r'\s'
+        r'(?P<version>(.*))'
+    ).match
+
+    parse_dep_link = re.compile(
+        r'(?P<namespace>(.*))'
+        r'/'
+        r'(?P<name>[^\s]*)'
+        r'\s'
+        r'(?P<version>(.*))'
+    ).match
+
+    def preprocess(self, line):
+        """
+        Return line string after removing commented portion and excess spaces.
+        """
+        if "//" in line:
+            line = line[:line.index('//')]
+        line = line.strip()
+        return line
+
+    def parse_gomod(self, location):
+        """
+        Return a dictionary containing all the important go.mod file data.
+        """
+        with io.open(location, encoding='utf-8', closefd=True) as data:
+            lines = data.readlines()
+
+        gomod_data = {}
+        require = []
+        exclude = []
+
+        for i, line in enumerate(lines):
+            line = self.preprocess(line)
+            parsed_module = self.parse_module(line)
+            if parsed_module:
+                gomod_data['module'] = parsed_module.group('module')
+
+            parsed_module_name = self.parse_module_name(line)
+            if parsed_module_name:
+                gomod_data['name'] = parsed_module_name.group('name')
+                gomod_data['namespace'] = parsed_module_name.group('namespace')
+
+            parsed_require = self.parse_require(line)
+            if parsed_require:
+                line_req = [parsed_require.group('namespace'), parsed_require.group('name'), parsed_require.group('version')]
+                require.append(line_req)
+
+            parsed_exclude = self.parse_exclude(line)
+            if parsed_exclude:
+                line_exclude = [parsed_exclude.group('namespace'), parsed_exclude.group('name'), parsed_exclude.group('version')]
+                exclude.append(line_exclude)
+
+            if 'require' in line and '(' in line:
+                for req in lines[i+1:]:
+                    req = self.preprocess(req)
+                    if ')' in req:
+                        break
+                    parsed_dep_link = self.parse_dep_link(req)
+                    if parsed_dep_link:
+                        line_req = [parsed_dep_link.group('namespace'), parsed_dep_link.group('name'), parsed_dep_link.group('version')]
+                        require.append(line_req)
+
+            if 'exclude' in line and '(' in line:
+                for exc in lines[i+1:]:
+                    exc = self.preprocess(exc)
+                    if ')' in exc:
+                        break
+                    parsed_dep_link = self.parse_dep_link(exc)
+                    if parsed_dep_link:
+                        line_exclude = [parsed_dep_link.group('namespace'), parsed_dep_link.group('name'), parsed_dep_link.group('version')]
+                        exclude.append(line_exclude)
+
+        gomod_data['require'] = require
+        gomod_data['exclude'] = exclude
+
+        return gomod_data

src/packagedcode/golang.py

@@ -0,0 +1,143 @@
+
+# Copyright (c) 2019 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/scancode-toolkit/
+# The ScanCode software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode require an acknowledgment.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with ScanCode or any ScanCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  ScanCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  ScanCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
+
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from collections import OrderedDict
+import io
+import logging
+import re
+
+import attr
+from packageurl import PackageURL
+
+from commoncode import filetype
+from commoncode import fileutils
+from packagedcode.go_mod import GoMod
+from packagedcode import models
+
+
+"""
+Handle Go packages including go.mod and go.sum files.
+"""
+"""
+Sample go.mod file:
+module github.com/alecthomas/participle
+require (
+	github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/stretchr/testify v1.4.0
+)
+go 1.13
+"""
+
+# TODO:
+# go.mod file does not contain version number.
+# valid download url need version number
+# CHECK: https://forum.golangbridge.org/t/url-to-download-package/19811
+
+TRACE = False
+
+logger = logging.getLogger(__name__)
+
+if TRACE:
+    import sys
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+
+@attr.s()
+class GolangPackage(models.Package):
+    metafiles = ('go.mod',)
+    default_type = 'golang'
+    default_primary_language = 'Go'
+    default_web_baseurl = 'https://pkg.go.dev'
+    default_download_baseurl = None
+    default_api_baseurl = None
+
+    @classmethod
+    def recognize(cls, location):
+        if fileutils.file_name(location).lower() == 'go.mod':
+            gomod_obj = GoMod()
+            gomod_data = gomod_obj.parse_gomod(location)
+            yield build_gomod_package(gomod_data)
+
+    @classmethod
+    def get_package_root(cls, manifest_resource, codebase):
+        return manifest_resource.parent(codebase)
+
+    def repository_homepage_url(self, baseurl=default_web_baseurl):
+        return '{}/{}/{}'.format(baseurl, self.namespace, self.name)
+
+def build_gomod_package(gomod_data):
+    """
+    Return a Package object from a go.mod file or None.
+    """
+    package_dependencies = []
+    require = gomod_data.get('require') or []
+    for namespace, name, version in require:
+        package_dependencies.append(
+            models.DependentPackage(
+                purl=PackageURL(
+                    type='golang',
+                    namespace=namespace,
+                    name=name
+                ).to_string(),
+                requirement=version,
+                scope='require',
+                is_runtime=True,
+                is_optional=False,
+                is_resolved=False,
+            )
+        )
+
+    exclude = gomod_data.get('exclude') or []
+    for namespace, name, version in exclude:
+        package_dependencies.append(
+            models.DependentPackage(
+                purl=PackageURL(
+                    type='golang',
+                    namespace=namespace,
+                    name=name
+                ).to_string(),
+                requirement=version,
+                scope='exclude',
+                is_runtime=True,
+                is_optional=False,
+                is_resolved=False,
+            )
+        )
+
+    name = gomod_data.get('name')
+    namespace = gomod_data.get('namespace')
+    homepage_url = 'https://pkg.go.dev/{}'.format(gomod_data.get('module'))
+
+    return GolangPackage(
+        name=name,
+        namespace=namespace,
+        homepage_url=homepage_url,
+        dependencies=package_dependencies
+    )