diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index b6b79ac4d5..8a8b13af96 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -101,12 +101,12 @@ jobs: # and are only run after the PR gets merged GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}} run: | - # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit; + # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit # tests that require a GitHub token are skipped automatically when no GitHub token is available if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then if [ ! -z $GITHUB_TOKEN ]; then - SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; - python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; + SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())" + python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')" fi echo "GitHub token installed!" else @@ -191,7 +191,17 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test|CryptographyDeprecationWarning: TripleDES has been moved|algorithms.TripleDES" + IGNORE_PATTERNS="no GitHub token available" + IGNORE_PATTERNS+="|skipping SvnRepository test" + IGNORE_PATTERNS+="|requires Lmod as modules tool" + IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device" + IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[56]" + IGNORE_PATTERNS+="|from cryptography.* import " + IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 2" + IGNORE_PATTERNS+="|Blowfish" + IGNORE_PATTERNS+="|GC3Pie not available, skipping test" + IGNORE_PATTERNS+="|CryptographyDeprecationWarning: TripleDES has been moved" + IGNORE_PATTERNS+="|algorithms.TripleDES" # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 2b6aaa751a..7095fedefd 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -46,6 +46,7 @@ import functools import os import re +from collections import OrderedDict from contextlib import contextmanager import easybuild.tools.filetools as filetools @@ -74,7 +75,7 @@ from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name from easybuild.tools.modules import modules_tool, NoModulesTool -from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type +from easybuild.tools.py2vs3 import create_base_metaclass, string_type from easybuild.tools.systemtools import check_os_dependency, pick_dep_version from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 1d5d45adb1..cb92b36b76 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -45,6 +45,7 @@ import re import sys import tempfile +from collections import OrderedDict from easybuild.base import fancylogger from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR @@ -64,7 +65,6 @@ from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data from easybuild.tools.github import fetch_files_from_commit, fetch_files_from_pr from easybuild.tools.multidiff import multidiff -from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.toolchain.toolchain import is_system_toolchain from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import only_if_module_is_available, quote_str diff --git a/easybuild/tools/build_details.py b/easybuild/tools/build_details.py index 487e6372a3..f5ce1ebc01 100644 --- a/easybuild/tools/build_details.py +++ b/easybuild/tools/build_details.py @@ -31,8 +31,8 @@ * Stijn De Weirdt (Ghent University) """ import time +from collections import OrderedDict from easybuild.tools.filetools import det_size -from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.systemtools import get_system_info from easybuild.tools.version import EASYBLOCKS_VERSION, FRAMEWORK_VERSION diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index b4387454ba..5aa0abf4c8 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -40,6 +40,7 @@ import inspect import json import os +from collections import OrderedDict from easybuild.tools import LooseVersion from easybuild.base import fancylogger @@ -60,7 +61,7 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import read_file from easybuild.tools.modules import modules_tool -from easybuild.tools.py2vs3 import OrderedDict, ascii_lowercase +from easybuild.tools.py2vs3 import ascii_lowercase from easybuild.tools.toolchain.toolchain import DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME, is_system_toolchain from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import INDENT_2SPACES, INDENT_4SPACES diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index df10ec859e..de7dc63a7d 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -45,6 +45,7 @@ import sys import tempfile import pwd +from collections import OrderedDict import easybuild.tools.environment as env from easybuild.base import fancylogger # build_log should always stay there, to ensure EasyBuildLog @@ -97,7 +98,7 @@ from easybuild.tools.module_generator import ModuleGeneratorLua, avail_module_generators from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes from easybuild.tools.modules import Lmod -from easybuild.tools.py2vs3 import OrderedDict, string_type +from easybuild.tools.py2vs3 import string_type from easybuild.tools.robot import det_robot_path from easybuild.tools.run import run_cmd from easybuild.tools.package.utilities import avail_package_naming_schemes diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index b6da8a9245..a8f98480f9 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -32,10 +32,10 @@ * Jørgen Nordmoen (University of Oslo) """ import functools +from collections import OrderedDict from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style -from easybuild.tools.py2vs3 import OrderedDict try: from rich.console import Console, Group diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py index f619279060..b52f55a1a2 100644 --- a/easybuild/tools/py2vs3/py2.py +++ b/easybuild/tools/py2vs3/py2.py @@ -52,7 +52,7 @@ # reload function (built-in in Python 2) -reload = reload +reload = reload # noqa: F821 # string type that can be used in 'isinstance' calls string_type = basestring @@ -89,9 +89,12 @@ def subprocess_terminate(proc, timeout): proc.terminate() +# Wrapped in exec to avoid invalid syntax warnings for Python 3 +exec(''' def raise_with_traceback(exception_class, message, traceback): """Raise exception of specified class with given message and traceback.""" raise exception_class, message, traceback # noqa: E999 +''') def extract_method_name(method_func): diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 73eb657510..24e910fb94 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -42,6 +42,7 @@ import sys import termios import warnings +from collections import OrderedDict from ctypes.util import find_library from socket import gethostname from easybuild.tools.py2vs3 import subprocess_popen_text @@ -64,7 +65,7 @@ from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import IGNORE from easybuild.tools.filetools import is_readable, read_file, which -from easybuild.tools.py2vs3 import OrderedDict, string_type +from easybuild.tools.py2vs3 import string_type from easybuild.tools.run import run_cmd @@ -735,7 +736,6 @@ def get_os_name(): # platform.linux_distribution was removed in Python 3.8, # see https://docs.python.org/2/library/platform.html#platform.linux_distribution if hasattr(platform, 'linux_distribution'): - # platform.linux_distribution is more useful, but only available since Python 2.6 # this allows to differentiate between Fedora, CentOS, RHEL and Scientific Linux (Rocks is just CentOS) with warnings.catch_warnings(): warnings.simplefilter("ignore", category=PendingDeprecationWarning) diff --git a/eb b/eb index ad14ba61d0..0a510c70ad 100755 --- a/eb +++ b/eb @@ -40,8 +40,8 @@ keyboard_interrupt() { trap keyboard_interrupt SIGINT -# Python 2.6+ or 3.5+ required -REQ_MIN_PY2VER=6 +# Python 2.7+ or 3.5+ required +REQ_MIN_PY2VER=7 REQ_MIN_PY3VER=5 EASYBUILD_MAIN='easybuild.main' diff --git a/requirements.txt b/requirements.txt index 85a9df78e7..d2ce2ccdc1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,12 @@ # keyring is required to provide GitHub token to EasyBuild; # for recent versions of keyring, keyrings.alt must be installed too -keyring +# 19.0 dropped Python 2 support +keyring<19.0; python_version < '3.0' +keyring; python_version >= '3.0' keyrings.alt # GitPython 3.1.15 deprecates Python 3.5 -GitPython==3.1.14; python_version >= '3.0' and python_version < '3.6' +GitPython<3.1.15; python_version >= '3.0' and python_version < '3.6' GitPython; python_version >= '3.6' or python_version <= '3.0' # autopep8 diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index eb21430f47..569660b2ab 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -38,6 +38,7 @@ import sys import tempfile import textwrap +from collections import OrderedDict from easybuild.tools import LooseVersion from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -72,7 +73,7 @@ from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.options import parse_external_modules_metadata -from easybuild.tools.py2vs3 import OrderedDict, reload +from easybuild.tools.py2vs3 import reload from easybuild.tools.robot import det_robot_path, resolve_dependencies from easybuild.tools.systemtools import AARCH64, KNOWN_ARCH_CONSTANTS, POWER, X86_64 from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext, get_os_name, get_os_version diff --git a/test/framework/repository.py b/test/framework/repository.py index 720e0fa8e8..feb96a13cf 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -41,7 +41,6 @@ from easybuild.tools.filetools import read_file from easybuild.tools.repository.filerepo import FileRepository from easybuild.tools.repository.gitrepo import GitRepository -from easybuild.tools.repository.hgrepo import HgRepository from easybuild.tools.repository.svnrepo import SvnRepository from easybuild.tools.repository.repository import init_repository from easybuild.tools.run import run_cmd @@ -129,27 +128,6 @@ def test_svnrepo(self): self.assertExists(os.path.join(repo.wc, 'trunk', 'README.md')) shutil.rmtree(repo.wc) - # this test is disabled because it fails in Travis as a result of bitbucket disabling TLS 1.0/1.1 - # we can consider re-enabling it when moving to a more recent Ubuntu version in the Travis config - # (which implies dropping support for Python 2.6) - # cfr. https://github.com/easybuilders/easybuild-framework/pull/2678 - def DISABLED_test_hgrepo(self): - """Test using HgRepository.""" - # only run this test if pysvn Python module is available - try: - import hglib # noqa - except ImportError: - print("(skipping HgRepository test)") - return - - # GitHub also supports SVN - test_repo_url = 'https://kehoste@bitbucket.org/kehoste/testrepository' - - repo = HgRepository(test_repo_url) - repo.init() - self.assertExists(os.path.join(repo.wc, 'README')) - shutil.rmtree(repo.wc) - def test_init_repository(self): """Test use of init_repository function.""" repo = init_repository('FileRepository', self.path)