add support for python 3.12

.github/workflows/tests.yml

@@ -34,6 +34,12 @@ jobs:
         - python-version: '3.10'
           os: ubuntu-latest
           tox-env: py
+        - python-version: '3.11'
+          os: ubuntu-latest
+          tox-env: py
+        - python-version: '3.12'
+          os: ubuntu-latest
+          tox-env: py
 
     steps:
     - uses: actions/checkout@v2
@@ -74,6 +80,10 @@ jobs:
           tox-env: py
         - python-version: '3.10'
           tox-env: py
+        - python-version: '3.11'
+          tox-env: py
+        - python-version: '3.12'
+          tox-env: py
 
     steps:
     - uses: actions/checkout@v2
@@ -108,6 +118,10 @@ jobs:
           tox-env: py
         - python-version: '3.10'
           tox-env: py
+        - python-version: '3.11'
+          tox-env: py
+        - python-version: '3.12'
+          tox-env: py
 
     steps:
     - uses: actions/checkout@v2

setup.py

@@ -32,6 +32,8 @@
     install_requires=[
         'click',
         'docker',
+        'importlib-metadata; python_version < "3.8"',
+        'packaging',
         'pip',
         'PyYAML',
         'retrying',
@@ -52,6 +54,8 @@
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
         'Operating System :: OS Independent',
         'Environment :: Console',
         'Topic :: Internet :: WWW/HTTP',

shub/deploy_egg.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 import os
 import tempfile
+from shutil import which
 
 import click
 
@@ -9,7 +10,7 @@
 from shub.exceptions import (BadParameterException, NotFoundException,
                              SubcommandException)
 from shub.utils import (decompress_egg_files, download_from_pypi,
-                        find_executable, run_cmd)
+                        run_cmd)
 
 
 HELP = """
@@ -86,7 +87,7 @@ def _checkout(repo, git_branch=None, target_dir='egg-tmp-clone'):
     ]
     missing_exes = []
     for cmd in vcs_commands:
-        exe = find_executable(cmd[0])
+        exe = which(cmd[0])
         if not exe:
             missing_exes.append(cmd[0])
             continue
@@ -109,7 +110,7 @@ def _checkout(repo, git_branch=None, target_dir='egg-tmp-clone'):
 
     if git_branch:
         try:
-            run_cmd([find_executable('git'), 'checkout', git_branch])
+            run_cmd([which('git'), 'checkout', git_branch])
         except SubcommandException:
             raise BadParameterException("Branch %s is not valid" % git_branch)
         click.echo("%s branch was checked out" % git_branch)

shub/image/run/wrapper.py

@@ -29,7 +29,7 @@
 import logging
 import datetime
 from multiprocessing import Process
-from distutils.spawn import find_executable
+from shutil import which
 
 
 def _consume_from_fifo(fifo_path):
@@ -68,7 +68,7 @@
     # non-daemon to allow it to finish reading from pipe before exit.
     Process(target=_consume_from_fifo, args=[fifo_path]).start()
     # replace current process with original start-crawl
-    os.execv(find_executable('start-crawl'), sys.argv)
+    os.execv(which('start-crawl'), sys.argv)
 
 
 if __name__ == '__main__':

shub/image/utils.py

@@ -9,7 +9,6 @@
 import yaml
 from tqdm import tqdm
 from six import binary_type
-import pkg_resources
 
 from shub import config as shub_config
 from shub import utils as shub_utils
@@ -18,6 +17,11 @@
     ShubDeprecationWarning, print_warning, BadParameterException,
 )
 
+if sys.version_info < (3, 8):
+    import importlib_metadata as metadata
+else:
+    from importlib import metadata
+
 
 DEFAULT_DOCKER_API_VERSION = '1.21'
 STATUS_FILE_LOCATION = '.releases'
@@ -83,8 +87,8 @@ def get_docker_client(validate=True):
         import docker
     except ImportError:
         raise ImportError(DOCKER_PY_UNAVAILABLE_MSG)
-    for dep in pkg_resources.working_set:
-        if dep.project_name == 'docker-py':
+    for dep in metadata.distributions():
+        if dep.name == 'docker-py':
             raise ImportError(DOCKER_PY_UNAVAILABLE_MSG)
 
     docker_host = os.environ.get('DOCKER_HOST')

shub/utils.py

@@ -11,9 +11,9 @@
 import time
 
 from collections import deque
-from six.moves.configparser import SafeConfigParser
-from distutils.spawn import find_executable
-from distutils.version import LooseVersion, StrictVersion
+from configparser import ConfigParser
+from shutil import which
+from packaging.version import Version
 from glob import glob
 from importlib import import_module
 from tempfile import NamedTemporaryFile, TemporaryFile
@@ -143,17 +143,21 @@ def _check_deploy_files_size(files):
 
 def write_and_echo_logs(keep_log, last_logs, rsp, verbose):
     """It will write logs to temporal file and echo if verbose is True."""
-    with NamedTemporaryFile(prefix='shub_deploy_', suffix='.log',
-                            delete=(not keep_log)) as log_file:
-        for line in rsp.iter_lines():
-            if verbose:
-                click.echo(line)
-            last_logs.append(line)
-            log_file.write(line + b'\n')
+    log_contents = b""
+    for line in rsp.iter_lines():
+        if verbose:
+            click.echo(line)
+        last_logs.append(line)
+        log_contents += line + b'\n'
+    deployed = _is_deploy_successful(last_logs)
+    if not deployed:
+        keep_log = True
+    echo_short_log_if_deployed(deployed, last_logs, verbose=verbose)
 
-        deployed = _is_deploy_successful(last_logs)
-        echo_short_log_if_deployed(deployed, last_logs, log_file, verbose)
-        if not log_file.delete:
+    with NamedTemporaryFile(prefix='shub_deploy_', suffix='.log',
+                            delete=not keep_log) as log_file:
+        log_file.write(log_contents)
+        if keep_log:
             click.echo("Deploy log location: %s" % log_file.name)
         if not deployed:
             try:
@@ -163,12 +167,11 @@ def write_and_echo_logs(keep_log, last_logs, rsp, verbose):
             raise RemoteErrorException("Deploy failed: {}".format(last_log))
 
 
-def echo_short_log_if_deployed(deployed, last_logs, log_file, verbose):
+def echo_short_log_if_deployed(deployed, last_logs, log_file=None, verbose=False):
     if deployed:
         if not verbose:
             click.echo(last_logs[-1])
     else:
-        log_file.delete = False
         if not verbose:
             click.echo("Deploy log last %s lines:" % len(last_logs))
             for line in last_logs:
@@ -212,7 +215,7 @@ def patch_sys_executable():
 
 
 def find_exe(exe_name):
-    exe = find_executable(exe_name)
+    exe = which(exe_name)
     if not exe:
         raise NotFoundException("Please install {}".format(exe_name))
     return exe
@@ -275,7 +278,7 @@ def pwd_version():
 
 
 def pwd_git_version():
-    git = find_executable('git')
+    git = which('git')
     if not git:
         return None
     try:
@@ -290,7 +293,7 @@ def pwd_git_version():
 
 
 def pwd_hg_version():
-    hg = find_executable('hg')
+    hg = which('hg')
     if not hg:
         return None
     try:
@@ -302,7 +305,7 @@ def pwd_hg_version():
 
 
 def pwd_bzr_version():
-    bzr = find_executable('bzr')
+    bzr = which('bzr')
     if not bzr:
         return None
     try:
@@ -485,9 +488,9 @@ def inside_project():
 
 
 def get_config(use_closest=True):
-    """Get Scrapy config file as a SafeConfigParser"""
+    """Get Scrapy config file as a ConfigParser"""
     sources = get_sources(use_closest)
-    cfg = SafeConfigParser()
+    cfg = ConfigParser()
     cfg.read(sources)
     return cfg
 
@@ -506,7 +509,7 @@ def get_sources(use_closest=True):
 
 
 def get_scrapycfg_targets(cfgfiles=None):
-    cfg = SafeConfigParser()
+    cfg = ConfigParser()
     cfg.read(cfgfiles or [])
     baset = dict(cfg.items('deploy')) if cfg.has_section('deploy') else {}
     targets = {}
@@ -627,8 +630,8 @@ def update_available(silent_fail=True):
     """
     try:
         release_data = latest_github_release()
-        latest_rls = StrictVersion(release_data['name'].lstrip('v'))
-        used_rls = StrictVersion(shub.__version__)
+        latest_rls = Version(release_data['name'].lstrip('v'))
+        used_rls = Version(shub.__version__)
         if used_rls >= latest_rls:
             return None
         return release_data['html_url']
@@ -643,15 +646,15 @@ def download_from_pypi(dest, pkg=None, reqfile=None, extra_args=None):
     if (not pkg and not reqfile) or (pkg and reqfile):
         raise ValueError('Call with either pkg or reqfile')
     extra_args = extra_args or []
-    pip_version = LooseVersion(getattr(pip, '__version__', '1.0'))
+    pip_version = Version(getattr(pip, '__version__', '1.0'))
     cmd = 'install'
     no_wheel = []
     target = [pkg] if pkg else ['-r', reqfile]
-    if pip_version >= LooseVersion('1.4'):
+    if pip_version >= Version('1.4'):
         no_wheel = ['--no-use-wheel']
-    if pip_version >= LooseVersion('7'):
+    if pip_version >= Version('7'):
         no_wheel = ['--no-binary=:all:']
-    if pip_version >= LooseVersion('8'):
+    if pip_version >= Version('8'):
         cmd = 'download'
     with patch_sys_executable():
         pip_main([cmd, '-d', dest, '--no-deps'] + no_wheel + extra_args +

tests/test_config.py

@@ -126,7 +126,7 @@ def test_load(self):
         }
         self.assertEqual(projects, self.conf.projects)
         endpoints = {'external': 'ext_endpoint'}
-        self.assertDictContainsSubset(endpoints, self.conf.endpoints)
+        self.assertLessEqual(endpoints.items(), self.conf.endpoints.items())
         apikeys = {'default': 'key', 'otheruser': 'otherkey'}
         self.assertEqual(apikeys, self.conf.apikeys)
         stacks = {'dev': 'scrapy:v1.1'}
@@ -145,7 +145,7 @@ def test_load_partial(self):
         """
         conf = self._get_conf_with_yml(yml)
         endpoints = {'external': 'ext_endpoint'}
-        self.assertDictContainsSubset(endpoints, conf.endpoints)
+        self.assertLessEqual(endpoints.items(), conf.endpoints.items())
         self.assertEqual(conf.projects, {})
         self.assertEqual(conf.apikeys, {})
         self.assertEqual(conf.images, {})
@@ -176,9 +176,9 @@ def test_load_shortcut_mixed(self):
                 dev: dev_stack
             stack: prod_stack
         """
-        self.assertDictContainsSubset(
-            self._get_conf_with_yml(yml).stacks,
-            {'default': 'prod_stack', 'dev': 'dev_stack'},
+        self.assertLessEqual(
+            self._get_conf_with_yml(yml).stacks.items(),
+            {'default': 'prod_stack', 'dev': 'dev_stack'}.items()
         )
 
     def test_load_shortcut_conflict(self):
@@ -586,7 +586,7 @@ def test_get_image_ambiguous_global_image_and_global_stack(self):
             image: true
             stack: scrapy:1.3
         """)
-        with self.assertRaisesRegexp(BadConfigException, '(?i)ambiguous'):
+        with self.assertRaisesRegex(BadConfigException, '(?i)ambiguous'):
             self.conf.get_image('default')
 
     def test_get_image_ambiguous_global_image_and_project_stack(self):
@@ -601,9 +601,9 @@ def test_get_image_ambiguous_global_image_and_project_stack(self):
                 stack: scrapy:1.3
             image: true
             """)
-        with self.assertRaisesRegexp(BadConfigException, '(?i)ambiguous'):
+        with self.assertRaisesRegex(BadConfigException, '(?i)ambiguous'):
             self.conf.get_image('bad')
-        with self.assertRaisesRegexp(BadConfigException, '(?i)disabled'):
+        with self.assertRaisesRegex(BadConfigException, '(?i)disabled'):
             self.conf.get_image('good')
 
     def test_get_image_ambiguous_project_image_and_project_stack(self):
@@ -614,7 +614,7 @@ def test_get_image_ambiguous_project_image_and_project_stack(self):
                 image: true
                 stack: scrapy:1.3
             """)
-        with self.assertRaisesRegexp(BadConfigException, '(?i)ambiguous'):
+        with self.assertRaisesRegex(BadConfigException, '(?i)ambiguous'):
             self.conf.get_image('default')
 
     def test_get_target_conf(self):

tests/test_schedule.py

@@ -63,9 +63,9 @@ def test_forwards_args_and_settings(self, mock_client):
             "--argument ARGWITHEQUAL=val2=val2".split(' '),
         )
         job_args = mock_proj.jobs.run.call_args[1]['job_args']
-        self.assertDictContainsSubset(
-            {'ARG': 'val1', 'ARGWITHEQUAL': 'val2=val2'},
-            job_args,
+        self.assertLessEqual(
+            {'ARG': 'val1', 'ARGWITHEQUAL': 'val2=val2'}.items(),
+            job_args.items(),
         )
         job_settings = mock_proj.jobs.run.call_args[1]['job_settings']
         self.assertEqual(
@@ -116,9 +116,9 @@ def test_forwards_environment(self, mock_client):
             "testspider -e VAR1=VAL1 --environment VAR2=VAL2".split(' '),
         )
         call_kwargs = mock_proj.jobs.run.call_args[1]
-        self.assertDictContainsSubset(
-            {'VAR1': 'VAL1', 'VAR2': 'VAL2'},
-            call_kwargs['environment'],
+        self.assertLessEqual(
+            {'VAR1': 'VAL1', 'VAR2': 'VAL2'}.items(),
+            call_kwargs['environment'].items(),
         )