Remove unused imports and other minor cleanups #295

Signed-off-by: Philippe Ombredanne <[email protected]>
aboutcode-org · Dec 4, 2020 · 30ebe90 · 30ebe90
1 parent 11165a4
commit 30ebe90
Show file tree

Hide file tree

Showing 28 changed files with 26 additions and 58 deletions.
diff --git a/etc/scripts/add_spdx_keys_as_rules.py b/etc/scripts/add_spdx_keys_as_rules.py
@@ -28,7 +28,6 @@
 import os.path
 
 import click
-click.disable_unicode_literals_warning = True
 
 import synclic
 

diff --git a/etc/scripts/buildcopytests.py b/etc/scripts/buildcopytests.py
@@ -29,7 +29,6 @@
 import sys
 
 import click
-click.disable_unicode_literals_warning = True
 
 import scancode_config
 

diff --git a/etc/scripts/buildlictests.py b/etc/scripts/buildlictests.py
@@ -29,7 +29,6 @@
 import sys
 
 import click
-click.disable_unicode_literals_warning = True
 from itertools import chain
 
 import scancode_config

diff --git a/etc/scripts/buildrules.py b/etc/scripts/buildrules.py
@@ -29,7 +29,6 @@
 
 import attr
 import click
-click.disable_unicode_literals_warning = True
 from license_expression import Licensing
 import saneyaml
 

diff --git a/etc/scripts/genlicspdx.py b/etc/scripts/genlicspdx.py
@@ -28,7 +28,6 @@
 import os
 
 import click
-click.disable_unicode_literals_warning = True
 
 from licensedcode.models import load_licenses
 from scancode.cli import run_scan

diff --git a/etc/scripts/genurlrules.py b/etc/scripts/genurlrules.py
@@ -27,7 +27,6 @@
 import os
 
 import click
-click.disable_unicode_literals_warning = True
 
 from license_expression import Licensing
 

diff --git a/etc/scripts/synclic.py b/etc/scripts/synclic.py
@@ -36,7 +36,6 @@
 import zipfile
 
 import click
-click.disable_unicode_literals_warning = True
 import requests
 
 from commoncode import fetch

diff --git a/etc/scripts/updateignorables.py b/etc/scripts/updateignorables.py
@@ -26,7 +26,6 @@
 
 
 import click
-click.disable_unicode_literals_warning = True
 
 from licensedcode import cache
 from licensedcode import models

diff --git a/src/cluecode/finder.py b/src/cluecode/finder.py
@@ -128,7 +128,7 @@ def re_filt(matches):
                 continue
             yield key, match, line, lineno
 
-    matcher = re.compile(pattern, re.UNICODE | re.I).match
+    matcher = re.compile(pattern, re.UNICODE | re.IGNORECASE).match
     return re_filt
 
 # A good reference page of email address regex is:
@@ -575,7 +575,7 @@ def find_pattern(location, pattern, unique=False):
     Return all match groups joined as one unicode string.
     Only return unique items if unique is True.
     """
-    pattern = re.compile(pattern, re.UNICODE | re.I)
+    pattern = re.compile(pattern, re.UNICODE | re.IGNORECASE)
     matches = find(location, [(None, pattern,)])
     if unique:
         matches = unique_filter(matches)

diff --git a/src/formattedcode/output_html.py b/src/formattedcode/output_html.py
@@ -22,7 +22,6 @@
 #  ScanCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
 
-
 import io
 from operator import itemgetter
 from os.path import abspath
@@ -35,7 +34,6 @@
 import click
 import simplejson
 
-from commoncode.fileutils import PATH_TYPE
 from commoncode.fileutils import as_posixpath
 from commoncode.fileutils import copytree
 from commoncode.fileutils import delete
@@ -96,8 +94,12 @@ class CustomTemplateOutput(OutputPlugin):
 
         PluggableCommandLineOption(('--custom-template',),
             type=click.Path(
-                exists=True, file_okay=True, dir_okay=False,
-                readable=True, path_type=PATH_TYPE),
+                exists=True,
+                file_okay=True,
+                dir_okay=False,
+                readable=True,
+                path_type=str
+            ),
             required_options=['custom_output'],
             metavar='FILE',
             help='Use this Jinja template FILE as a custom template.',

diff --git a/src/licensedcode/dmp.py b/src/licensedcode/dmp.py
@@ -47,7 +47,7 @@ def logger_debug(*args): pass
     logger = logging.getLogger(__name__)
 
     def logger_debug(*args):
-        return logger.debug(' '.join(isinstance(a, basestring) and a or repr(a) for a in args))
+        return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
 
     logging.basicConfig(stream=sys.stdout)
     logger.setLevel(logging.DEBUG)
@@ -103,7 +103,7 @@ def int2unicode(nums):
     """
     Convert an array of positive integers to a unicode string.
     """
-    return u''.join(unichr(i + 1) for i in nums)
+    return u''.join(chr(i + 1) for i in nums)
 
 
 def trim(diffs):

diff --git a/src/packagedcode/alpine.py b/src/packagedcode/alpine.py
@@ -27,7 +27,6 @@
 from datetime import datetime
 import email
 import io
-import os
 from os import path
 import posixpath
 import re

diff --git a/src/packagedcode/bower.py b/src/packagedcode/bower.py
@@ -23,7 +23,6 @@
 #  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
 
 
-from functools import partial
 import io
 import json
 import logging
@@ -32,7 +31,6 @@
 from packageurl import PackageURL
 
 from commoncode import filetype
-from commoncode import ignore
 from packagedcode import models
 from packagedcode.utils import combine_expressions
 

diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py
@@ -22,20 +22,14 @@
 #  ScanCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
 
-
-import io
 import logging
-import re
 
 import attr
-from packageurl import PackageURL
 
-from commoncode import filetype
 from commoncode import fileutils
 from packagedcode import go_mod
 from packagedcode import models
 
-
 """
 Handle Go packages including go.mod and go.sum files.
 """

diff --git a/src/packagedcode/opam.py b/src/packagedcode/opam.py
@@ -30,8 +30,6 @@
 import attr
 from packageurl import PackageURL
 
-from commoncode import filetype
-from commoncode import fileutils
 from packagedcode import models
 
 

diff --git a/src/packagedcode/phpcomposer.py b/src/packagedcode/phpcomposer.py
@@ -30,7 +30,6 @@
 import sys
 
 import attr
-from packageurl import PackageURL
 
 from commoncode import filetype
 from commoncode import fileutils

diff --git a/src/packagedcode/plugin_package.py b/src/packagedcode/plugin_package.py
@@ -26,7 +26,6 @@
 
 import attr
 import click
-click.disable_unicode_literals_warning = True
 
 from plugincode.scan import ScanPlugin
 from plugincode.scan import scan_impl

diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py
@@ -446,7 +446,7 @@ def find_dunder_version(location):
     SPDX-License-Identifier: BSD-3-Clause
     (C) 2001-2020 Chris Liechti <[email protected]>
     """
-    pattern = re.compile(r"^__version__\s*=\s*['\"]([^'\"]*)['\"]", re.M)
+    pattern = re.compile(r"^__version__\s*=\s*['\"]([^'\"]*)['\"]", re.MULTILINE)
     match = find_pattern(location, pattern)
     if TRACE: logger_debug('find_dunder_version:', 'location:', location, 'match:', match)
     return match
@@ -457,7 +457,7 @@ def find_plain_version(location):
     Return a plain version attribute string or None from searching the module
     file at `location`.
     """
-    pattern = re.compile(r"^version\s*=\s*['\"]([^'\"]*)['\"]", re.M)
+    pattern = re.compile(r"^version\s*=\s*['\"]([^'\"]*)['\"]", re.MULTILINE)
     match = find_pattern(location, pattern)
     if TRACE: logger_debug('find_plain_version:', 'location:', location, 'match:', match)
     return match
@@ -479,7 +479,7 @@ def find_setup_py_dunder_version(location):
     SPDX-License-Identifier: BSD-3-Clause
     (C) 2001-2020 Chris Liechti <[email protected]>
     """
-    pattern = re.compile(r"^\s*version\s*=\s*(.*__version__)", re.M)
+    pattern = re.compile(r"^\s*version\s*=\s*(.*__version__)", re.MULTILINE)
     match = find_pattern(location, pattern)
     if TRACE: logger_debug('find_setup_py_dunder_version:', 'location:', location, 'match:', match)
     return match

diff --git a/src/packagedcode/recognize.py b/src/packagedcode/recognize.py
@@ -30,9 +30,7 @@
 
 from commoncode import filetype
 from commoncode.fileutils import file_name
-from commoncode.fileutils import fsencode
 from commoncode.fileutils import splitext_name
-from commoncode.system import on_linux
 from packagedcode import PACKAGE_TYPES
 from typecode import contenttype
 

diff --git a/src/scancode/cli.py b/src/scancode/cli.py
@@ -46,7 +46,6 @@ class WindowsError(Exception):
         pass
 
 import click  # NOQA
-click.disable_unicode_literals_warning = True
 
 
 from commoncode import cliutils
@@ -55,7 +54,6 @@ class WindowsError(Exception):
 from commoncode.cliutils import progressmanager
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.fileutils import as_posixpath
-from commoncode.fileutils import PATH_TYPE
 from commoncode.fileutils import POSIX_PATH_SEP
 from commoncode.timeutils import time2tstamp
 from commoncode.resource import Codebase
@@ -191,8 +189,7 @@ def validate_depth(ctx, param, value):
 
 @click.argument('input',
     metavar='<OUTPUT FORMAT OPTION(s)> <input>...', nargs=-1,
-    # ensure that the input path is bytes on Linux, unicode elsewhere
-    type=click.Path(exists=True, readable=True, path_type=PATH_TYPE))
+    type=click.Path(exists=True, readable=True, path_type=str))
 
 @click.option('--strip-root',
     is_flag=True,
@@ -526,7 +523,7 @@ def echo_func(*_args, **_kwargs):
         if not common_prefix:
             # we have no common prefix, but all relative. therefore the
             # parent/root is the current ddirectory
-            common_prefix = PATH_TYPE('.')
+            common_prefix = str('.')
 
         elif not os.path.isdir(common_prefix):
             msg = 'Invalid inputs: all input paths must share a common parent directory.'

diff --git a/src/scancode/cli_test_utils.py b/src/scancode/cli_test_utils.py
@@ -28,7 +28,6 @@
 import os
 import time
 
-from commoncode.system import on_linux
 from commoncode.system import on_windows
 from scancode_config import scancode_root_dir
 

diff --git a/src/textcode/strings.py b/src/textcode/strings.py
@@ -26,7 +26,6 @@
 import re
 import string
 
-from commoncode.text import toascii
 
 """
 Extract raw ASCII strings from (possibly) binary strings.
@@ -50,20 +49,18 @@
 MIN_LEN = 4
 MIN_LEN_STR = b'4'
 
-def strings_from_file(location, buff_size=1024 * 1024, ascii=False, clean=True, min_len=MIN_LEN):
+def strings_from_file(location, buff_size=1024 * 1024, clean=True, min_len=MIN_LEN):
     """
-    Yield unicode strings made only of ASCII characters found in file at location.
-    Process the file in chunks (to limit memory usage). If ascii is True, strings
-    are converted to plain ASCII "str or byte" strings instead of unicode.
+    Yield unicode strings made only of printable ASCII characters found in file
+    at `location``. Process the file in chunks of `buff_size` bytes (to limit
+    memory usage).
     """
     with open(location, 'rb') as f:
         while 1:
             buf = f.read(buff_size)
             if not buf:
                 break
             for s in strings_from_string(buf, clean=clean, min_len=min_len):
-                if ascii:
-                    s = toascii(s)
                 s = s.strip()
                 if len(s) >= min_len:
                     yield s
@@ -94,10 +91,10 @@ def strings_from_file(location, buff_size=1024 * 1024, ascii=False, clean=True,
 
 def strings_from_string(binary_string, clean=False, min_len=0):
     """
-    Yield strings extracted from a (possibly binary) string. The strings are ASCII
-    printable characters only. If clean is True, also clean and filter short and
-    repeated strings.
-    Note: we do not keep the offset of where a string was found (e.g. match.start).
+    Yield strings extracted from a (possibly binary) string `binary_string`. The
+    strings are ASCII printable characters only. If `clean` is True, also clean
+    and filter short and repeated strings. Note: we do not keep the offset of
+    where a string was found (e.g. match.start).
     """
     for match in ascii_strings(binary_string):
         s = decode(match.group())

diff --git a/tests/cluecode/test_plugin_email_url.py b/tests/cluecode/test_plugin_email_url.py
@@ -26,7 +26,6 @@
 import os
 
 import click
-click.disable_unicode_literals_warning = True
 
 from commoncode.testcase import FileDrivenTesting
 

diff --git a/tests/licensedcode/test_plugin_license.py b/tests/licensedcode/test_plugin_license.py
@@ -26,7 +26,6 @@
 import os
 
 import click
-click.disable_unicode_literals_warning = True
 import pytest
 
 from commoncode.testcase import FileDrivenTesting

diff --git a/tests/licensedcode/test_plugin_license_text.py b/tests/licensedcode/test_plugin_license_text.py
@@ -26,7 +26,6 @@
 import os
 
 import click
-click.disable_unicode_literals_warning = True
 
 from commoncode.testcase import FileDrivenTesting
 

diff --git a/tests/scancode/test_cli.py b/tests/scancode/test_cli.py
@@ -28,11 +28,9 @@
 import os
 
 import click
-click.disable_unicode_literals_warning = True
 import pytest
 
 from commoncode import fileutils
-from commoncode.fileutils import fsencode
 from commoncode.testcase import FileDrivenTesting
 from commoncode.system import on_linux
 from commoncode.system import on_mac
@@ -442,7 +440,7 @@ def test_scan_does_not_fail_when_scanning_unicode_test_files_from_express():
     test_path = u'unicode_fixtures.tar.gz'
 
     test_dir = test_env.extract_test_tar_raw(test_path)
-    test_dir = fsencode(test_dir)
+    test_dir = os.fsencode(test_dir)
 
     args = ['-n0', '--info', '--license', '--copyright', '--package', '--email',
             '--url', '--strip-root', '--json', '-', test_dir]

diff --git a/tests/scancode/test_scancode_checks.py b/tests/scancode/test_scancode_checks.py
@@ -35,6 +35,8 @@
  - code style
  - ABOUT files
  - release archives creation
+ 
+NOTE: it has imports limited to the standard library by design
 """
 
 

diff --git a/tests/summarycode/test_score.py b/tests/summarycode/test_score.py
@@ -27,7 +27,6 @@
 import os
 
 import click
-click.disable_unicode_literals_warning = True
 import pytest
 
 from commoncode.testcase import FileDrivenTesting