Enabled parallel tests

.github/workflows/ci.yml

@@ -80,16 +80,11 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install --upgrade tox tox-gh-actions
 
+      - name: Docker compose up
+        run: docker compose up -d --wait
+
       - name: Tox tests
-        run: |
-          REDIS_PRIMARY=$(tests/start_redis.sh)
-          REDIS_SENTINEL=$(tests/start_redis.sh --sentinel)
-          CONTAINERS="$REDIS_PRIMARY $REDIS_SENTINEL"
-          trap "docker stop $CONTAINERS && docker rm $CONTAINERS" EXIT
-          tests/wait_for_redis.sh $REDIS_PRIMARY 6379
-          tests/wait_for_redis.sh $REDIS_SENTINEL 26379
-
-          tox
+        run: tox
         env:
           DJANGO: ${{ matrix.django-version }}
           REDIS: ${{ matrix.redis-version }}

docker/docker-compose.yml

@@ -0,0 +1,30 @@
+services:
+
+  redis:
+    image: redis:latest
+    container_name: redis-primary
+    command: redis-server --enable-debug-command yes --protected-mode no
+    ports:
+      - 6379:6379
+    healthcheck:
+      test: redis-cli ping
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+  sentinel:
+    image: redis:latest
+    container_name: redis-sentinel
+    depends_on:
+      redis:
+        condition: service_healthy
+    entrypoint: "redis-sentinel /redis.conf --port 26379"
+    ports:
+      - 26379:26379
+    volumes:
+      - "./sentinel.conf:/redis.conf"
+    healthcheck:
+      test: redis-cli -p 26379 ping
+      interval: 5s
+      timeout: 5s
+      retries: 5

docker/sentinel.conf

@@ -0,0 +1,4 @@
+sentinel monitor default_service 127.0.0.1 6379 1
+sentinel down-after-milliseconds default_service 3200
+sentinel failover-timeout default_service 10000
+sentinel parallel-syncs default_service 1

setup.cfg

@@ -85,18 +85,8 @@ REDIS =
 [testenv]
 passenv = CI, GITHUB*
 commands =
-  {envpython} -m pytest --cov-report= --ds=settings.sqlite {posargs}
-  {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_herd {posargs}
-  {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_json {posargs}
-  {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_lz4 {posargs}
-  {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_msgpack {posargs}
-  {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_sentinel {posargs}
-  {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_sentinel_opts {posargs}
-  {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_sharding {posargs}
-  {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_usock {posargs}
-  {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_zlib {posargs}
-  {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_zstd {posargs}
-  {envpython} -m pytest --cov-append --cov-report= --ds=settings.sqlite_gzip {posargs}
+  {envpython} -m pytest --cov-report= -n 4 {posargs}
+commands_post =
   {envpython} -m coverage report
   {envpython} -m coverage xml
 
@@ -109,8 +99,9 @@ deps =
     pytest
     pytest-cov
     pytest-django
-    pytest-pythonpath
     pytest-mock
+    pytest-pythonpath
+    pytest-xdist
     redismaster: https://github.com/redis/redis-py/archive/master.tar.gz
     lz4>=0.15
     pyzstd>=0.15
@@ -119,7 +110,7 @@ deps =
 basepython = python3
 envdir={toxworkdir}/lint
 commands =
-    black: black --target-version py36 {posargs:--check --diff} setup.py django_redis/ tests/
+    black: black --target-version py38 {posargs:--check --diff} setup.py django_redis/ tests/
     ruff: ruff {posargs:check --show-fixes} django_redis/ tests/
     mypy: mypy {posargs:--cobertura-xml-report .} django_redis tests
 deps =
@@ -136,8 +127,6 @@ deps =
 skip_install = true
 
 [tool:pytest]
-DJANGO_SETTINGS_MODULE = settings.sqlite
-
 addopts =
     --doctest-modules
     --cov=django_redis

tests/README.rst

@@ -4,17 +4,5 @@ Running the test suite
 .. code-block:: bash
 
   # start redis and a sentinel (uses docker with image redis:latest)
-  PRIMARY=$(tests/start_redis.sh)
-  SENTINEL=$(tests/start_redis.sh --sentinel)
-
-  # or just wait 5 - 10 seconds and most likely this would be the case
-  tests/wait_for_redis.sh $PRIMARY 6379
-  tests/wait_for_redis.sh $SENTINEL 26379
-
-  # run the tests
-  tox
-
-  # shut down redis
-  for container in $PRIMARY $SENTINEL; do
-    docker stop $container && docker rm $container
-  done
+cd docker
+docker-compose up

tests/conftest.py

@@ -1,12 +1,63 @@
+from os import environ
 from typing import Iterable
 
 import pytest
-from django.core.cache import cache as default_cache
+from pytest import Metafunc
+from xdist.scheduler import LoadScopeScheduling
 
 from django_redis.cache import BaseCache
+from tests.settings_wrapper import SettingsWrapper
+
+
+class FixtureScheduling(LoadScopeScheduling):
+    """Split by [] value. This is very hackish and might blow up any time!"""
+
+    def _split_scope(self, nodeid):
+        if "[sqlite" in nodeid:
+            return nodeid.rsplit("[")[-1].replace("]", "")
+        return None
+
+
+def pytest_xdist_make_scheduler(log, config):
+    return FixtureScheduling(config, log)
+
+
+@pytest.fixture()
+def settings():
+    """A Django settings object which restores changes after the testrun"""
+    wrapper = SettingsWrapper()
+    yield wrapper
+    wrapper.finalize()
 
 
 @pytest.fixture
-def cache() -> Iterable[BaseCache]:
+def cache(cache_settings: str) -> Iterable[BaseCache]:
+    from django import setup
+
+    environ["DJANGO_SETTINGS_MODULE"] = f"settings.{cache_settings}"
+    setup()
+
+    from django.core.cache import cache as default_cache
+
     yield default_cache
     default_cache.clear()
+
+
+def pytest_generate_tests(metafunc: Metafunc):
+    if "cache" in metafunc.fixturenames or "session" in metafunc.fixturenames:
+        # Mark
+        settings = [
+            "sqlite",
+            "sqlite_gzip",
+            "sqlite_herd",
+            "sqlite_json",
+            "sqlite_lz4",
+            "sqlite_msgpack",
+            "sqlite_sentinel",
+            "sqlite_sentinel_opts",
+            "sqlite_sharding",
+            "sqlite_usock",
+            "sqlite_zlib",
+            "sqlite_zstd",
+        ]
+        metafunc.parametrize("cache_settings", settings)

tests/settings/sqlite_gzip.py

@@ -3,31 +3,31 @@
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": ["redis://127.0.0.1:6379?db=1", "redis://127.0.0.1:6379?db=1"],
+        "LOCATION": ["redis://127.0.0.1:6379?db=2", "redis://127.0.0.1:6379?db=2"],
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "COMPRESSOR": "django_redis.compressors.gzip.GzipCompressor",
         },
     },
     "doesnotexist": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:56379?db=1",
+        "LOCATION": "redis://127.0.0.1:56379?db=2",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "COMPRESSOR": "django_redis.compressors.gzip.GzipCompressor",
         },
     },
     "sample": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379?db=1,redis://127.0.0.1:6379?db=1",
+        "LOCATION": "redis://127.0.0.1:6379?db=2,redis://127.0.0.1:6379?db=2",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "COMPRESSOR": "django_redis.compressors.gzip.GzipCompressor",
         },
     },
     "with_prefix": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379?db=1",
+        "LOCATION": "redis://127.0.0.1:6379?db=2",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "COMPRESSOR": "django_redis.compressors.gzip.GzipCompressor",

tests/settings/sqlite_herd.py

@@ -3,22 +3,22 @@
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": ["redis://127.0.0.1:6379?db=5"],
+        "LOCATION": ["redis://127.0.0.1:6379?db=3"],
         "OPTIONS": {"CLIENT_CLASS": "django_redis.client.HerdClient"},
     },
     "doesnotexist": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:56379?db=1",
+        "LOCATION": "redis://127.0.0.1:56379?db=3",
         "OPTIONS": {"CLIENT_CLASS": "django_redis.client.HerdClient"},
     },
     "sample": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379?db=1,redis://127.0.0.1:6379?db=1",
+        "LOCATION": "redis://127.0.0.1:6379?db=3,redis://127.0.0.1:6379?db=3",
         "OPTIONS": {"CLIENT_CLASS": "django_redis.client.HerdClient"},
     },
     "with_prefix": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379?db=1",
+        "LOCATION": "redis://127.0.0.1:6379?db=3",
         "OPTIONS": {"CLIENT_CLASS": "django_redis.client.HerdClient"},
         "KEY_PREFIX": "test-prefix",
     },

tests/settings/sqlite_json.py

@@ -3,31 +3,31 @@
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": ["redis://127.0.0.1:6379?db=1", "redis://127.0.0.1:6379?db=1"],
+        "LOCATION": ["redis://127.0.0.1:6379?db=4", "redis://127.0.0.1:6379?db=4"],
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "SERIALIZER": "django_redis.serializers.json.JSONSerializer",
         },
     },
     "doesnotexist": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:56379?db=1",
+        "LOCATION": "redis://127.0.0.1:56379?db=4",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "SERIALIZER": "django_redis.serializers.json.JSONSerializer",
         },
     },
     "sample": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379?db=1,redis://127.0.0.1:6379?db=1",
+        "LOCATION": "redis://127.0.0.1:6379?db=4,redis://127.0.0.1:6379?db=4",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "SERIALIZER": "django_redis.serializers.json.JSONSerializer",
         },
     },
     "with_prefix": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379?db=1",
+        "LOCATION": "redis://127.0.0.1:6379?db=4",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "SERIALIZER": "django_redis.serializers.json.JSONSerializer",

tests/settings/sqlite_lz4.py

@@ -3,31 +3,31 @@
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": ["redis://127.0.0.1:6379?db=1", "redis://127.0.0.1:6379?db=1"],
+        "LOCATION": ["redis://127.0.0.1:6379?db=5", "redis://127.0.0.1:6379?db=5"],
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "COMPRESSOR": "django_redis.compressors.lz4.Lz4Compressor",
         },
     },
     "doesnotexist": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:56379?db=1",
+        "LOCATION": "redis://127.0.0.1:56379?db=5",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "COMPRESSOR": "django_redis.compressors.lz4.Lz4Compressor",
         },
     },
     "sample": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "127.0.0.1:6379?db=1,127.0.0.1:6379?db=1",
+        "LOCATION": "127.0.0.1:6379?db=5,127.0.0.1:6379?db=5",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "COMPRESSOR": "django_redis.compressors.lz4.Lz4Compressor",
         },
     },
     "with_prefix": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379?db=1",
+        "LOCATION": "redis://127.0.0.1:6379?db=5",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "COMPRESSOR": "django_redis.compressors.lz4.Lz4Compressor",

tests/settings/sqlite_msgpack.py

@@ -3,31 +3,31 @@
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": ["redis://127.0.0.1:6379?db=1", "redis://127.0.0.1:6379?db=1"],
+        "LOCATION": ["redis://127.0.0.1:6379?db=6", "redis://127.0.0.1:6379?db=6"],
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "SERIALIZER": "django_redis.serializers.msgpack.MSGPackSerializer",
         },
     },
     "doesnotexist": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:56379?db=1",
+        "LOCATION": "redis://127.0.0.1:56379?db=6",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "SERIALIZER": "django_redis.serializers.msgpack.MSGPackSerializer",
         },
     },
     "sample": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379?db=1,redis://127.0.0.1:6379?db=1",
+        "LOCATION": "redis://127.0.0.1:6379?db=6,redis://127.0.0.1:6379?db=6",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "SERIALIZER": "django_redis.serializers.msgpack.MSGPackSerializer",
         },
     },
     "with_prefix": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379?db=1",
+        "LOCATION": "redis://127.0.0.1:6379?db=6",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "SERIALIZER": "django_redis.serializers.msgpack.MSGPackSerializer",