From 30ebe907a021263d20e5a79ca1b3614b8e94cfbb Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 4 Dec 2020 19:03:54 +0100 Subject: [PATCH] Remove unused imports and other minor cleanups #295 Signed-off-by: Philippe Ombredanne --- etc/scripts/add_spdx_keys_as_rules.py | 1 - etc/scripts/buildcopytests.py | 1 - etc/scripts/buildlictests.py | 1 - etc/scripts/buildrules.py | 1 - etc/scripts/genlicspdx.py | 1 - etc/scripts/genurlrules.py | 1 - etc/scripts/synclic.py | 1 - etc/scripts/updateignorables.py | 1 - src/cluecode/finder.py | 4 ++-- src/formattedcode/output_html.py | 10 ++++++---- src/licensedcode/dmp.py | 4 ++-- src/packagedcode/alpine.py | 1 - src/packagedcode/bower.py | 2 -- src/packagedcode/golang.py | 6 ------ src/packagedcode/opam.py | 2 -- src/packagedcode/phpcomposer.py | 1 - src/packagedcode/plugin_package.py | 1 - src/packagedcode/pypi.py | 6 +++--- src/packagedcode/recognize.py | 2 -- src/scancode/cli.py | 7 ++----- src/scancode/cli_test_utils.py | 1 - src/textcode/strings.py | 19 ++++++++----------- tests/cluecode/test_plugin_email_url.py | 1 - tests/licensedcode/test_plugin_license.py | 1 - .../licensedcode/test_plugin_license_text.py | 1 - tests/scancode/test_cli.py | 4 +--- tests/scancode/test_scancode_checks.py | 2 ++ tests/summarycode/test_score.py | 1 - 28 files changed, 26 insertions(+), 58 deletions(-) diff --git a/etc/scripts/add_spdx_keys_as_rules.py b/etc/scripts/add_spdx_keys_as_rules.py index 7d4f2bfcf0c..a0c15fb1f78 100644 --- a/etc/scripts/add_spdx_keys_as_rules.py +++ b/etc/scripts/add_spdx_keys_as_rules.py @@ -28,7 +28,6 @@ import os.path import click -click.disable_unicode_literals_warning = True import synclic diff --git a/etc/scripts/buildcopytests.py b/etc/scripts/buildcopytests.py index 8183859d57a..d7c9e164595 100644 --- a/etc/scripts/buildcopytests.py +++ b/etc/scripts/buildcopytests.py @@ -29,7 +29,6 @@ import sys import click -click.disable_unicode_literals_warning = True import scancode_config diff --git a/etc/scripts/buildlictests.py b/etc/scripts/buildlictests.py index c9fde2a847d..6da0203c3e7 100644 --- a/etc/scripts/buildlictests.py +++ b/etc/scripts/buildlictests.py @@ -29,7 +29,6 @@ import sys import click -click.disable_unicode_literals_warning = True from itertools import chain import scancode_config diff --git a/etc/scripts/buildrules.py b/etc/scripts/buildrules.py index dd7146ddbab..376f55fece1 100644 --- a/etc/scripts/buildrules.py +++ b/etc/scripts/buildrules.py @@ -29,7 +29,6 @@ import attr import click -click.disable_unicode_literals_warning = True from license_expression import Licensing import saneyaml diff --git a/etc/scripts/genlicspdx.py b/etc/scripts/genlicspdx.py index bda03ff0ab6..e877148335b 100644 --- a/etc/scripts/genlicspdx.py +++ b/etc/scripts/genlicspdx.py @@ -28,7 +28,6 @@ import os import click -click.disable_unicode_literals_warning = True from licensedcode.models import load_licenses from scancode.cli import run_scan diff --git a/etc/scripts/genurlrules.py b/etc/scripts/genurlrules.py index 1f55f4ffd24..5f1017fe7d3 100644 --- a/etc/scripts/genurlrules.py +++ b/etc/scripts/genurlrules.py @@ -27,7 +27,6 @@ import os import click -click.disable_unicode_literals_warning = True from license_expression import Licensing diff --git a/etc/scripts/synclic.py b/etc/scripts/synclic.py index 0f4c2ded685..028a4c8cfa0 100644 --- a/etc/scripts/synclic.py +++ b/etc/scripts/synclic.py @@ -36,7 +36,6 @@ import zipfile import click -click.disable_unicode_literals_warning = True import requests from commoncode import fetch diff --git a/etc/scripts/updateignorables.py b/etc/scripts/updateignorables.py index d32f2ae36b1..520d6ba935e 100644 --- a/etc/scripts/updateignorables.py +++ b/etc/scripts/updateignorables.py @@ -26,7 +26,6 @@ import click -click.disable_unicode_literals_warning = True from licensedcode import cache from licensedcode import models diff --git a/src/cluecode/finder.py b/src/cluecode/finder.py index be38aa0162d..f0753162152 100644 --- a/src/cluecode/finder.py +++ b/src/cluecode/finder.py @@ -128,7 +128,7 @@ def re_filt(matches): continue yield key, match, line, lineno - matcher = re.compile(pattern, re.UNICODE | re.I).match + matcher = re.compile(pattern, re.UNICODE | re.IGNORECASE).match return re_filt # A good reference page of email address regex is: @@ -575,7 +575,7 @@ def find_pattern(location, pattern, unique=False): Return all match groups joined as one unicode string. Only return unique items if unique is True. """ - pattern = re.compile(pattern, re.UNICODE | re.I) + pattern = re.compile(pattern, re.UNICODE | re.IGNORECASE) matches = find(location, [(None, pattern,)]) if unique: matches = unique_filter(matches) diff --git a/src/formattedcode/output_html.py b/src/formattedcode/output_html.py index 6087f9724f0..dc2f99f8d32 100644 --- a/src/formattedcode/output_html.py +++ b/src/formattedcode/output_html.py @@ -22,7 +22,6 @@ # ScanCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode-toolkit/ for support and download. - import io from operator import itemgetter from os.path import abspath @@ -35,7 +34,6 @@ import click import simplejson -from commoncode.fileutils import PATH_TYPE from commoncode.fileutils import as_posixpath from commoncode.fileutils import copytree from commoncode.fileutils import delete @@ -96,8 +94,12 @@ class CustomTemplateOutput(OutputPlugin): PluggableCommandLineOption(('--custom-template',), type=click.Path( - exists=True, file_okay=True, dir_okay=False, - readable=True, path_type=PATH_TYPE), + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + path_type=str + ), required_options=['custom_output'], metavar='FILE', help='Use this Jinja template FILE as a custom template.', diff --git a/src/licensedcode/dmp.py b/src/licensedcode/dmp.py index 708d48050ce..1d3fe8caca6 100644 --- a/src/licensedcode/dmp.py +++ b/src/licensedcode/dmp.py @@ -47,7 +47,7 @@ def logger_debug(*args): pass logger = logging.getLogger(__name__) def logger_debug(*args): - return logger.debug(' '.join(isinstance(a, basestring) and a or repr(a) for a in args)) + return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args)) logging.basicConfig(stream=sys.stdout) logger.setLevel(logging.DEBUG) @@ -103,7 +103,7 @@ def int2unicode(nums): """ Convert an array of positive integers to a unicode string. """ - return u''.join(unichr(i + 1) for i in nums) + return u''.join(chr(i + 1) for i in nums) def trim(diffs): diff --git a/src/packagedcode/alpine.py b/src/packagedcode/alpine.py index 7d60ed44e12..fc7ebdfd626 100644 --- a/src/packagedcode/alpine.py +++ b/src/packagedcode/alpine.py @@ -27,7 +27,6 @@ from datetime import datetime import email import io -import os from os import path import posixpath import re diff --git a/src/packagedcode/bower.py b/src/packagedcode/bower.py index 4e084e30521..0d68451f78e 100644 --- a/src/packagedcode/bower.py +++ b/src/packagedcode/bower.py @@ -23,7 +23,6 @@ # Visit https://github.com/nexB/scancode-toolkit/ for support and download. -from functools import partial import io import json import logging @@ -32,7 +31,6 @@ from packageurl import PackageURL from commoncode import filetype -from commoncode import ignore from packagedcode import models from packagedcode.utils import combine_expressions diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index 6519d4bd0fb..02b79710b97 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -22,20 +22,14 @@ # ScanCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -import io import logging -import re import attr -from packageurl import PackageURL -from commoncode import filetype from commoncode import fileutils from packagedcode import go_mod from packagedcode import models - """ Handle Go packages including go.mod and go.sum files. """ diff --git a/src/packagedcode/opam.py b/src/packagedcode/opam.py index 6341be30f11..e1aaa08ee3d 100644 --- a/src/packagedcode/opam.py +++ b/src/packagedcode/opam.py @@ -30,8 +30,6 @@ import attr from packageurl import PackageURL -from commoncode import filetype -from commoncode import fileutils from packagedcode import models diff --git a/src/packagedcode/phpcomposer.py b/src/packagedcode/phpcomposer.py index 870bb335cd9..7de322b5586 100644 --- a/src/packagedcode/phpcomposer.py +++ b/src/packagedcode/phpcomposer.py @@ -30,7 +30,6 @@ import sys import attr -from packageurl import PackageURL from commoncode import filetype from commoncode import fileutils diff --git a/src/packagedcode/plugin_package.py b/src/packagedcode/plugin_package.py index 363eaef92ed..67694aad7a3 100644 --- a/src/packagedcode/plugin_package.py +++ b/src/packagedcode/plugin_package.py @@ -26,7 +26,6 @@ import attr import click -click.disable_unicode_literals_warning = True from plugincode.scan import ScanPlugin from plugincode.scan import scan_impl diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index d968f1f4432..07349e80c57 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -446,7 +446,7 @@ def find_dunder_version(location): SPDX-License-Identifier: BSD-3-Clause (C) 2001-2020 Chris Liechti """ - pattern = re.compile(r"^__version__\s*=\s*['\"]([^'\"]*)['\"]", re.M) + pattern = re.compile(r"^__version__\s*=\s*['\"]([^'\"]*)['\"]", re.MULTILINE) match = find_pattern(location, pattern) if TRACE: logger_debug('find_dunder_version:', 'location:', location, 'match:', match) return match @@ -457,7 +457,7 @@ def find_plain_version(location): Return a plain version attribute string or None from searching the module file at `location`. """ - pattern = re.compile(r"^version\s*=\s*['\"]([^'\"]*)['\"]", re.M) + pattern = re.compile(r"^version\s*=\s*['\"]([^'\"]*)['\"]", re.MULTILINE) match = find_pattern(location, pattern) if TRACE: logger_debug('find_plain_version:', 'location:', location, 'match:', match) return match @@ -479,7 +479,7 @@ def find_setup_py_dunder_version(location): SPDX-License-Identifier: BSD-3-Clause (C) 2001-2020 Chris Liechti """ - pattern = re.compile(r"^\s*version\s*=\s*(.*__version__)", re.M) + pattern = re.compile(r"^\s*version\s*=\s*(.*__version__)", re.MULTILINE) match = find_pattern(location, pattern) if TRACE: logger_debug('find_setup_py_dunder_version:', 'location:', location, 'match:', match) return match diff --git a/src/packagedcode/recognize.py b/src/packagedcode/recognize.py index e56e9855a61..ced5a78f620 100644 --- a/src/packagedcode/recognize.py +++ b/src/packagedcode/recognize.py @@ -30,9 +30,7 @@ from commoncode import filetype from commoncode.fileutils import file_name -from commoncode.fileutils import fsencode from commoncode.fileutils import splitext_name -from commoncode.system import on_linux from packagedcode import PACKAGE_TYPES from typecode import contenttype diff --git a/src/scancode/cli.py b/src/scancode/cli.py index 6b394475bed..9621cf17d68 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -46,7 +46,6 @@ class WindowsError(Exception): pass import click # NOQA -click.disable_unicode_literals_warning = True from commoncode import cliutils @@ -55,7 +54,6 @@ class WindowsError(Exception): from commoncode.cliutils import progressmanager from commoncode.cliutils import PluggableCommandLineOption from commoncode.fileutils import as_posixpath -from commoncode.fileutils import PATH_TYPE from commoncode.fileutils import POSIX_PATH_SEP from commoncode.timeutils import time2tstamp from commoncode.resource import Codebase @@ -191,8 +189,7 @@ def validate_depth(ctx, param, value): @click.argument('input', metavar=' ...', nargs=-1, - # ensure that the input path is bytes on Linux, unicode elsewhere - type=click.Path(exists=True, readable=True, path_type=PATH_TYPE)) + type=click.Path(exists=True, readable=True, path_type=str)) @click.option('--strip-root', is_flag=True, @@ -526,7 +523,7 @@ def echo_func(*_args, **_kwargs): if not common_prefix: # we have no common prefix, but all relative. therefore the # parent/root is the current ddirectory - common_prefix = PATH_TYPE('.') + common_prefix = str('.') elif not os.path.isdir(common_prefix): msg = 'Invalid inputs: all input paths must share a common parent directory.' diff --git a/src/scancode/cli_test_utils.py b/src/scancode/cli_test_utils.py index 5bc9c40a4f4..fd4a650c6de 100644 --- a/src/scancode/cli_test_utils.py +++ b/src/scancode/cli_test_utils.py @@ -28,7 +28,6 @@ import os import time -from commoncode.system import on_linux from commoncode.system import on_windows from scancode_config import scancode_root_dir diff --git a/src/textcode/strings.py b/src/textcode/strings.py index 67975d7b017..eeee3b36e91 100644 --- a/src/textcode/strings.py +++ b/src/textcode/strings.py @@ -26,7 +26,6 @@ import re import string -from commoncode.text import toascii """ Extract raw ASCII strings from (possibly) binary strings. @@ -50,11 +49,11 @@ MIN_LEN = 4 MIN_LEN_STR = b'4' -def strings_from_file(location, buff_size=1024 * 1024, ascii=False, clean=True, min_len=MIN_LEN): +def strings_from_file(location, buff_size=1024 * 1024, clean=True, min_len=MIN_LEN): """ - Yield unicode strings made only of ASCII characters found in file at location. - Process the file in chunks (to limit memory usage). If ascii is True, strings - are converted to plain ASCII "str or byte" strings instead of unicode. + Yield unicode strings made only of printable ASCII characters found in file + at `location``. Process the file in chunks of `buff_size` bytes (to limit + memory usage). """ with open(location, 'rb') as f: while 1: @@ -62,8 +61,6 @@ def strings_from_file(location, buff_size=1024 * 1024, ascii=False, clean=True, if not buf: break for s in strings_from_string(buf, clean=clean, min_len=min_len): - if ascii: - s = toascii(s) s = s.strip() if len(s) >= min_len: yield s @@ -94,10 +91,10 @@ def strings_from_file(location, buff_size=1024 * 1024, ascii=False, clean=True, def strings_from_string(binary_string, clean=False, min_len=0): """ - Yield strings extracted from a (possibly binary) string. The strings are ASCII - printable characters only. If clean is True, also clean and filter short and - repeated strings. - Note: we do not keep the offset of where a string was found (e.g. match.start). + Yield strings extracted from a (possibly binary) string `binary_string`. The + strings are ASCII printable characters only. If `clean` is True, also clean + and filter short and repeated strings. Note: we do not keep the offset of + where a string was found (e.g. match.start). """ for match in ascii_strings(binary_string): s = decode(match.group()) diff --git a/tests/cluecode/test_plugin_email_url.py b/tests/cluecode/test_plugin_email_url.py index 1ebbb441c89..1b8f31ce237 100644 --- a/tests/cluecode/test_plugin_email_url.py +++ b/tests/cluecode/test_plugin_email_url.py @@ -26,7 +26,6 @@ import os import click -click.disable_unicode_literals_warning = True from commoncode.testcase import FileDrivenTesting diff --git a/tests/licensedcode/test_plugin_license.py b/tests/licensedcode/test_plugin_license.py index b053567bc27..3da492359fb 100644 --- a/tests/licensedcode/test_plugin_license.py +++ b/tests/licensedcode/test_plugin_license.py @@ -26,7 +26,6 @@ import os import click -click.disable_unicode_literals_warning = True import pytest from commoncode.testcase import FileDrivenTesting diff --git a/tests/licensedcode/test_plugin_license_text.py b/tests/licensedcode/test_plugin_license_text.py index 6ac3e72d8cb..93566cc7a13 100644 --- a/tests/licensedcode/test_plugin_license_text.py +++ b/tests/licensedcode/test_plugin_license_text.py @@ -26,7 +26,6 @@ import os import click -click.disable_unicode_literals_warning = True from commoncode.testcase import FileDrivenTesting diff --git a/tests/scancode/test_cli.py b/tests/scancode/test_cli.py index a09bc066f3a..d4f9accd859 100644 --- a/tests/scancode/test_cli.py +++ b/tests/scancode/test_cli.py @@ -28,11 +28,9 @@ import os import click -click.disable_unicode_literals_warning = True import pytest from commoncode import fileutils -from commoncode.fileutils import fsencode from commoncode.testcase import FileDrivenTesting from commoncode.system import on_linux from commoncode.system import on_mac @@ -442,7 +440,7 @@ def test_scan_does_not_fail_when_scanning_unicode_test_files_from_express(): test_path = u'unicode_fixtures.tar.gz' test_dir = test_env.extract_test_tar_raw(test_path) - test_dir = fsencode(test_dir) + test_dir = os.fsencode(test_dir) args = ['-n0', '--info', '--license', '--copyright', '--package', '--email', '--url', '--strip-root', '--json', '-', test_dir] diff --git a/tests/scancode/test_scancode_checks.py b/tests/scancode/test_scancode_checks.py index a233c2ff44c..4d37b406541 100644 --- a/tests/scancode/test_scancode_checks.py +++ b/tests/scancode/test_scancode_checks.py @@ -35,6 +35,8 @@ - code style - ABOUT files - release archives creation + +NOTE: it has imports limited to the standard library by design """ diff --git a/tests/summarycode/test_score.py b/tests/summarycode/test_score.py index 7f21a3df26c..5877b6bffdf 100644 --- a/tests/summarycode/test_score.py +++ b/tests/summarycode/test_score.py @@ -27,7 +27,6 @@ import os import click -click.disable_unicode_literals_warning = True import pytest from commoncode.testcase import FileDrivenTesting