Fix windows build (celery#6104)

* do not load memcache nor couchbase lib during windows build

those libraries depends on native libraries libcouchbase and libmemcached
that are not installed on Appveyor.
As only unit tests runs on Appveyor, it should be fine

* Add python 3.8 workaround for app trap

* skip tests file_descriptor_safety tests on windows

AsyncPool is not supported on Windows so Pool does have _fileno_to_outq
attribute, making the test fail

* Fix crossplatform log and pid files in multi mode

it relates to celery#6017

* Use tox to build and test on windows

* remove tox_install_command

* drop python 2.7 from windows build

appveyor.yml

@@ -12,46 +12,44 @@ environment:
     # a later point release.
     # See: https://www.appveyor.com/docs/installed-software#python
 
-    - PYTHON: "C:\\Python27-x64"
-      PYTHON_VERSION: "2.7.x"
-      PYTHON_ARCH: "64"
-      WINDOWS_SDK_VERSION: "v7.0"
-
     - PYTHON: "C:\\Python35-x64"
       PYTHON_VERSION: "3.5.x"
       PYTHON_ARCH: "64"
       WINDOWS_SDK_VERSION: "v7.1"
+      TOXENV: "3.5-unit"
 
     - PYTHON: "C:\\Python36-x64"
       PYTHON_VERSION: "3.6.x"
       PYTHON_ARCH: "64"
       WINDOWS_SDK_VERSION: "v7.1"
-
+      TOXENV: "3.6-unit"
+
     - PYTHON: "C:\\Python37-x64"
       PYTHON_VERSION: "3.7.x"
       PYTHON_ARCH: "64"
       WINDOWS_SDK_VERSION: "v7.1"
-
+      TOXENV: "3.7-unit"
+
     - PYTHON: "C:\\Python38-x64"
       PYTHON_VERSION: "3.8.x"
       PYTHON_ARCH: "64"
       WINDOWS_SDK_VERSION: "v7.1"
+      TOXENV: "3.8-unit"
 
 
 init:
   - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%"
 
 install:
   - "powershell extra\\appveyor\\install.ps1"
-  - "%PYTHON%/python -m pip install -U pip setuptools"
+  - "%PYTHON%/python -m pip install -U pip setuptools tox"
   - "%PYTHON%/Scripts/pip.exe install -U eventlet"
   - "%PYTHON%/Scripts/pip.exe install -U -r requirements/extras/thread.txt"
-  - "%PYTHON%/Scripts/pip.exe install -U -r requirements/test-ci-default.txt"
 
 build: off
 
 test_script:
-  - "%WITH_COMPILER% %PYTHON%/python setup.py test"
+  - "%WITH_COMPILER% %PYTHON%/Scripts/tox -v -- -v"
 
 after_test:
   - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel"

celery/apps/multi.py

@@ -151,8 +151,8 @@ def _setdefaultopt(self, d, alt, value):
                 return d[opt]
             except KeyError:
                 pass
-        path_split = value.split("/")
-        dir_path = "/".join(path_split[0:-1])
+        value = os.path.normpath(value)
+        dir_path = os.path.dirname(value)
         if not os.path.exists(dir_path):
             os.makedirs(dir_path)
         return d.setdefault(alt[0], value)

celery/contrib/testing/app.py

@@ -32,7 +32,7 @@ class Trap(object):
     def __getattr__(self, name):
         # Workaround to allow unittest.mock to patch this object
         # in Python 3.8 and above.
-        if name == '_is_coroutine':
+        if name == '_is_coroutine' or name == '__func__':
             return None
         print(name)
         raise RuntimeError('Test depends on current_app')

requirements/extras/couchbase.txt

@@ -1,2 +1,2 @@
-couchbase < 3.0.0
+couchbase < 3.0.0; platform_system != "Windows"
 couchbase-cffi < 3.0.0;platform_python_implementation=="PyPy"

requirements/extras/memcache.txt

@@ -1 +1 @@
-pylibmc
+pylibmc; platform_system != "Windows"

t/unit/apps/test_multi.py

@@ -3,6 +3,7 @@
 import errno
 import signal
 import sys
+import os
 
 import pytest
 from case import Mock, call, patch, skip
@@ -113,8 +114,8 @@ def assert_line_in(name, args):
 
         def _args(name, *args):
             return args + (
-                '--pidfile=/var/run/celery/{}.pid'.format(name),
-                '--logfile=/var/log/celery/{}%I.log'.format(name),
+                '--pidfile={}.pid'.format(os.path.join(os.path.normpath('/var/run/celery/'), name)),
+                '--logfile={}%I.log'.format(os.path.join(os.path.normpath('/var/log/celery/'), name)),
                 '--executable={0}'.format(sys.executable),
                 '',
             )
@@ -194,10 +195,10 @@ def test_from_kwargs(self):
             '--executable={0}'.format(n.executable),
             '-O fair',
             '-n [email protected]',
-            '--logfile=/var/log/celery/foo%I.log',
+            '--logfile={}'.format(os.path.normpath('/var/log/celery/foo%I.log')),
             '-Q q1,q2',
             '--max-tasks-per-child=30',
-            '--pidfile=/var/run/celery/foo.pid',
+            '--pidfile={}'.format(os.path.normpath('/var/run/celery/foo.pid')),
             '',
         ])
 
@@ -275,7 +276,7 @@ def test_handle_process_exit__signalled(self):
 
     def test_logfile(self):
         assert self.node.logfile == self.expander.return_value
-        self.expander.assert_called_with('/var/log/celery/%n%I.log')
+        self.expander.assert_called_with(os.path.normpath('/var/log/celery/%n%I.log'))
 
 
 class test_Cluster:
@@ -375,8 +376,8 @@ def test_getpids(self):
         assert sorted(node_0.argv) == sorted([
             '',
             '--executable={0}'.format(node_0.executable),
-            '--logfile=/var/log/celery/foo%I.log',
-            '--pidfile=/var/run/celery/foo.pid',
+            '--logfile={}'.format(os.path.normpath('/var/log/celery/foo%I.log')),
+            '--pidfile={}'.format(os.path.normpath('/var/run/celery/foo.pid')),
             '-m celery worker --detach',
             '-n [email protected]',
         ])
@@ -386,8 +387,8 @@ def test_getpids(self):
         assert sorted(node_1.argv) == sorted([
             '',
             '--executable={0}'.format(node_1.executable),
-            '--logfile=/var/log/celery/bar%I.log',
-            '--pidfile=/var/run/celery/bar.pid',
+            '--logfile={}'.format(os.path.normpath('/var/log/celery/bar%I.log')),
+            '--pidfile={}'.format(os.path.normpath('/var/run/celery/bar.pid')),
             '-m celery worker --detach',
             '-n [email protected]',
         ])
@@ -404,8 +405,8 @@ def __init__(self, path):
 
             def read_pid(self):
                 try:
-                    return {'/var/run/celery/foo.pid': 10,
-                            '/var/run/celery/bar.pid': 11}[self.path]
+                    return {os.path.normpath('/var/run/celery/foo.pid'): 10,
+                            os.path.normpath('/var/run/celery/bar.pid'): 11}[self.path]
                 except KeyError:
                     raise ValueError()
         self.Pidfile.side_effect = pids

t/unit/worker/test_worker.py

@@ -792,6 +792,7 @@ def test_with_autoscaler(self):
         )
         assert worker.autoscaler
 
+    @skip.if_win32()
     @pytest.mark.nothreads_not_lingering
     @mock.sleepdeprived(module=autoscale)
     def test_with_autoscaler_file_descriptor_safety(self):
@@ -841,6 +842,7 @@ def test_with_autoscaler_file_descriptor_safety(self):
         worker.terminate()
         worker.pool.terminate()
 
+    @skip.if_win32()
     @pytest.mark.nothreads_not_lingering
     @mock.sleepdeprived(module=autoscale)
     def test_with_file_descriptor_safety(self):

tox.ini

@@ -68,7 +68,6 @@ basepython =
     flake8,apicheck,linkcheck,configcheck,pydocstyle,bandit: python3.8
     flakeplus: python2.7
 usedevelop = True
-install_command = {toxinidir}/tox_install_command.sh {opts} {packages}
 
 [testenv:apicheck]
 setenv =