Remove remaining references to Python 2 #295

This removes any references to Python 2 and the commoncode.compat module

Signed-off-by: Philippe Ombredanne <[email protected]>

etc/scripts/scanserv.py

@@ -45,10 +45,9 @@ def run_scan(location, **kwargs):
 
 
 if __name__ == '__channelexec__':
-    from commoncode import compat
     for kwargs in channel:  # NOQA
         # a mapping of kwargs or a location string
-        if isinstance(kwargs, (str, compat.unicode)):
+        if isinstance(kwargs, (str, str)):
             channel.send(run_scan(kwargs))  # NOQA
         elif isinstance(kwargs, dict):
             channel.send(run_scan(**kwargs))  # NOQA

etc/scripts/synclic.py

@@ -45,7 +45,6 @@
 
 from commoncode import fetch
 from commoncode import fileutils
-from commoncode import compat
 
 import licensedcode
 from licensedcode.models import load_licenses
@@ -898,7 +897,7 @@ def update_external(_attrib, _sc_val, _ext_val):
 
             continue
 
-        if (isinstance(scancode_value, compat.unicode) and isinstance(external_value, compat.unicode)):
+        if (isinstance(scancode_value, str) and isinstance(external_value, str)):
             # keep the stripped and normalized spaces value
             # normalized spaces
             normalized_scancode_value = ' '.join(scancode_value.split())

src/cluecode/finder.py

@@ -33,8 +33,6 @@
 from six import string_types
 import urlpy
 
-from commoncode import compat
-from commoncode.system import py3
 from commoncode.text import toascii
 from cluecode import finder_data
 from textcode import analysis
@@ -251,7 +249,7 @@ def find_urls(location, unique=True):
         if TRACE_URL:
             logger_debug('find_urls: lineno:', lineno, '_line:', repr(_line),
                          'type(url):', type(url), 'url:', repr(url))
-        yield compat.unicode(url), lineno
+        yield str(url), lineno
 
 
 EMPTY_URLS = set(['https', 'http', 'ftp', 'www', ])
@@ -462,7 +460,7 @@ def get_ip(s):
         return False
 
     try:
-        ip = ipaddress.ip_address(compat.unicode(s))
+        ip = ipaddress.ip_address(str(s))
         return ip
     except ValueError:
         return False

src/formattedcode/output_csv.py

@@ -33,7 +33,6 @@
 from six import string_types
 import unicodecsv
 
-from commoncode import compat
 from formattedcode import FileOptionType
 from plugincode.output import output_impl
 from plugincode.output import OutputPlugin
@@ -320,7 +319,7 @@ def flatten_package(_package, path, prefix='package__'):
                     if isinstance(component_val, list):
                         component_val = '\n'.join(component_val)
 
-                    if not isinstance(component_val, compat.unicode):
+                    if not isinstance(component_val, str):
                         component_val = repr(component_val)
 
                     existing = pack.get(component_new_key) or []
@@ -338,7 +337,7 @@ def flatten_package(_package, path, prefix='package__'):
 
         pack[nk] = ''
 
-        if isinstance(val, compat.unicode):
+        if isinstance(val, str):
             pack[nk] = val
         else:
             # Use repr if not a string

src/formattedcode/output_html.py

@@ -40,16 +40,13 @@
 import click
 import simplejson
 
-from commoncode import compat
 from commoncode.fileutils import PATH_TYPE
 from commoncode.fileutils import as_posixpath
 from commoncode.fileutils import copytree
 from commoncode.fileutils import delete
 from commoncode.fileutils import file_name
 from commoncode.fileutils import file_base_name
-from commoncode.fileutils import fsencode
 from commoncode.fileutils import parent_directory
-from commoncode.system import on_linux
 from formattedcode import FileOptionType
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.cliutils import OUTPUT_GROUP
@@ -133,7 +130,7 @@ def write_templated(output_file, results, version, template_loc):
     template = get_template(template_loc)
 
     for template_chunk in generate_output(results, version, template):
-        assert isinstance(template_chunk, compat.unicode)
+        assert isinstance(template_chunk, str)
         try:
             output_file.write(template_chunk)
         except Exception:

src/formattedcode/output_json.py

@@ -134,8 +134,7 @@ def write_results(codebase, output_file, pretty=False, **kwargs):
 
         # Write files
         codebase_files = OutputPlugin.get_files(codebase, **kwargs)
-        # OutputPlugin.get_files() returns a `map()`, which isn's JSON
-        # serializable in Python 3
+        # OutputPlugin.get_files() returns a generator, not JSON-serializable
         codebase_files = list(codebase_files)
         s.write('files', codebase_files)
 

src/formattedcode/output_spdx.py

@@ -196,8 +196,8 @@ def write_spdx(output_file, files, tool_name, tool_version, notice, input_file,
     """
     Write scan output as SPDX Tag/value or RDF.
     """
+    as_rdf = not as_tagvalue
     _patch_license_list()
-
     absinput = abspath(input_file)
 
     if isdir(absinput):
@@ -341,24 +341,20 @@ def write_spdx(output_file, files, tool_name, tool_version, notice, input_file,
 
         if as_tagvalue:
             from spdx.writers.tagvalue import write_document  # NOQA
-        else:
+        elif as_rdf:
             from spdx.writers.rdf import write_document  # NOQA
 
         if as_tagvalue:
-            # unicode text everywhere
             spdx_output = StringIO()
-        else:
-            # rdf as utf-encoded bytes on Py2
+        elif as_rdf:
+            # rdf is utf-encoded bytes
             spdx_output = BytesIO()
 
         write_document(doc, spdx_output, validate=False)
         result = spdx_output.getvalue()
 
-        if as_tagvalue:
-            # unicode text everywhere
-            pass
-        else:
-            # rdf as utf-encoded bytes on Py2
+        if as_rdf:
+            # rdf is utf-encoded bytes
             result = result.decode('utf-8')
 
         output_file.write(result)

src/licensedcode/index.py

@@ -41,12 +41,6 @@
 import sys
 from time import time
 
-# Python 2 and 3 support
-try:
-    import itertools.izip as zip  # NOQA
-except ImportError:
-    pass
-
 from intbitset import intbitset
 from six import string_types
 

src/licensedcode/spans.py

@@ -37,9 +37,6 @@
 
 from intbitset import intbitset
 
-from commoncode import compat
-
-
 """
 Ranges and intervals of integers using bitmaps.
 Used as a compact and faster data structure for token and position sets.
@@ -110,7 +107,7 @@ def __init__(self, *args):
 
         elif len_args == 1:
             # args0 is a single int or an iterable of ints
-            if isinstance(args[0], compat.integer_types):
+            if isinstance(args[0], int):
                 self._set = intbitset(args)
             else:
                 # some sequence or iterable
@@ -207,7 +204,7 @@ def __contains__(self, other):
         if isinstance(other, Span):
             return self._set.issuperset(other._set)
 
-        if isinstance(other, compat.integer_types):
+        if isinstance(other, int):
             return self._set.__contains__(other)
 
         if isinstance(other, (set, frozenset)):

src/licensedcode/tokenize.py

@@ -28,15 +28,6 @@
 from __future__ import unicode_literals
 
 from itertools import islice
-
-# Python 2 and 3 support
-try:
-    # Python 2
-    import itertools.izip as zip  # NOQA
-except ImportError:
-    # Python 3
-    pass
-
 from binascii import crc32
 import re
 

src/packagedcode/cocoapods.py

@@ -29,15 +29,11 @@
 import re
 
 import attr
-from packageurl import PackageURL
 
-from commoncode.fileutils import py2
 from commoncode import filetype
-from commoncode import fileutils
 from packagedcode import models
 from packagedcode.spec import Spec
 
-
 """
 Handle cocoapods packages manifests for macOS and iOS
 including .podspec, Podfile and Podfile.lock files.
@@ -46,7 +42,6 @@
 
 # TODO: override the license detection to detect declared_license correctly.
 
-
 TRACE = False
 
 logger = logging.getLogger(__name__)
@@ -104,11 +99,16 @@ def build_package(podspec_data):
     name = podspec_data.get('name')
     version = podspec_data.get('version')
     declared_license = podspec_data.get('license')
-    summary = podspec_data.get('summary')
-    description = podspec_data.get('description')
+    summary = podspec_data.get('summary', '')
+    description = podspec_data.get('description', '')
     homepage_url = podspec_data.get('homepage_url')
     source = podspec_data.get('source')
     authors = podspec_data.get('author') or []
+    if summary and not description.startswith(summary):
+        desc = [summary]
+        if description:
+            desc += [description]
+        description = '\n'.join(desc)
 
     author_names = []
     author_email = []
@@ -166,7 +166,7 @@ def party_mapper(author, email):
 def parse_person(person):
     """
     Return name and email from person string.
-    
+
     https://guides.cocoapods.org/syntax/podspec.html#authors
     Author can be in the form:
         s.author = 'Rohit Potter'
@@ -187,4 +187,4 @@ def parse_person(person):
         name = parsed.group('name')
         email = parsed.group('email')
 
-    return name, email 
+    return name, email

src/packagedcode/pypi.py

@@ -52,13 +52,6 @@
 from packagedcode.utils import build_description
 from packagedcode.utils import combine_expressions
 
-try:
-    # Python 2
-    unicode = unicode  # NOQA
-
-except NameError:  # pragma: nocover
-    # Python 3
-    unicode = str  # NOQA
 
 """
 Detect and collect Python packages information.
@@ -774,7 +767,7 @@ def compute_normalized_license(declared_license):
         values = list(declared_license.values())
     elif isinstance(declared_license, list):
         values = list(declared_license)
-    elif isinstance(declared_license, (str, unicode,)):
+    elif isinstance(declared_license, str):
         values = [declared_license]
     else:
         return

src/scancode/__init__.py

@@ -35,7 +35,6 @@
 from click.types import BoolParamType
 from six import string_types
 
-from commoncode import compat
 from commoncode import fileutils
 
 # Tracing flags

src/scancode/cli.py

@@ -43,14 +43,6 @@
 from time import time
 import traceback
 
-# Python 2 and 3 support
-try:
-    # Python 2
-    import itertools.imap as map  # NOQA
-except ImportError:
-    # Python 3
-    pass
-
 # this exception is not available on posix
 try:
     WindowsError  # NOQA
@@ -68,15 +60,13 @@ class WindowsError(Exception):
 from commoncode.cliutils import path_progress_message
 from commoncode.cliutils import progressmanager
 from commoncode.cliutils import PluggableCommandLineOption
-from commoncode import compat
 from commoncode.fileutils import as_posixpath
 from commoncode.fileutils import PATH_TYPE
 from commoncode.fileutils import POSIX_PATH_SEP
 from commoncode.timeutils import time2tstamp
 from commoncode.resource import Codebase
 from commoncode.resource import VirtualCodebase
 from commoncode.system import on_windows
-from commoncode.system import on_linux
 
 # these are important to register plugin managers
 from plugincode import PluginManager

src/scancode/interrupt.py

@@ -51,7 +51,6 @@ class TimeoutError(Exception):  # NOQA
 NO_ERROR = None
 NO_VALUE = None
 
-
 if not on_windows:
     """
     Some code based in part and inspired from the RobotFramework and
@@ -113,21 +112,9 @@ def handler(signum, frame):
     from ctypes import pythonapi
     from multiprocessing import TimeoutError as MpTimeoutError
 
-    try:
-        # python 3
-        from queue import Empty as Queue_Empty  # NOQA
-        from queue import Queue  # NOQA
-    except:
-        # python 2
-        from Queue import Empty as Queue_Empty  # NOQA
-        from Queue import Queue  # NOQA
-
-    try:
-        # python 3
-        from _thread import start_new_thread
-    except ImportError:
-        # python 2
-        from thread import start_new_thread
+    from queue import Empty as Queue_Empty
+    from queue import Queue
+    from _thread import start_new_thread
 
     def interruptible(func, args=None, kwargs=None, timeout=DEFAULT_TIMEOUT):
         """

src/scancode/plugin_ignore.py

@@ -32,7 +32,6 @@
 from plugincode.pre_scan import pre_scan_impl
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.cliutils import PRE_SCAN_GROUP
-from commoncode import compat
 
 
 # Tracing flags
@@ -53,8 +52,7 @@ def logger_debug(*args):
     logger.setLevel(logging.DEBUG)
 
     def logger_debug(*args):
-        return logger.debug(
-            ' '.join(isinstance(a, compat.unicode) and a or repr(a) for a in args))
+        return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
 
 
 @pre_scan_impl