chore: enable redis cache

aether-kernel/conf/pip/requirements.txt

@@ -13,11 +13,11 @@
 ################################################################################
 
 aether.python==1.0.17
-aether.sdk==1.2.21
+aether.sdk==1.2.22
 attrs==19.3.0
 autopep8==1.5.2
-boto3==1.12.45
-botocore==1.15.45
+boto3==1.12.48
+botocore==1.15.48
 cachetools==4.1.0
 certifi==2020.4.5.1
 cffi==1.14.0
@@ -38,7 +38,7 @@ django-filter==2.2.0
 django-minio-storage==0.3.7
 django-model-utils==4.0.0
 django-prometheus==2.0.0
-django-redis-sessions==0.6.1
+django-redis==4.11.0
 django-silk==4.0.1
 django-storages==1.9.1
 django-uwsgi==0.2.2
@@ -89,7 +89,7 @@ pyparsing==2.4.7
 pyrsistent==0.16.0
 python-dateutil==2.8.1
 python-json-logger==0.1.11
-pytz==2019.3
+pytz==2020.1
 redis==3.4.1
 requests==2.23.0
 rsa==4.0

aether-kernel/entrypoint.sh

@@ -111,6 +111,9 @@ function setup {
     # migrate data model if needed
     ./manage.py migrate --noinput
 
+    # clean out expired sessions
+    ./manage.py clearsessions
+
     # arguments: -u=admin -p=secretsecret [email protected] -t=01234656789abcdefghij
     ./manage.py setup_admin -u=$ADMIN_USERNAME -p=$ADMIN_PASSWORD -t=$ADMIN_TOKEN
 

aether-odk-module/aether/odk/api/collect/auth_utils.py

@@ -26,7 +26,6 @@
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.core.exceptions import ObjectDoesNotExist
-from django.utils import six
 from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _
 
@@ -246,4 +245,4 @@ def _create_nonce(timestamp, realm):
 
 
 def _hash_fn(data):
-    return md5(six.b(data)).hexdigest()
+    return md5(data.encode('latin-1')).hexdigest()

aether-odk-module/aether/odk/api/collect/tests/test_authentication.py

@@ -19,8 +19,11 @@
 import base64
 import hashlib
 import os
+import sys
 import time
 
+from importlib import reload, import_module
+
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.test import RequestFactory, override_settings
@@ -271,11 +274,25 @@ def test_replay_attack(self):
             self.assertEqual(response.status_code, 200)
 
 
+@override_settings(
+    DEFAULT_REALM='digest',
+    GATEWAY_ENABLED=False,
+)
 class MultitenancyAuthenticationTests(UrlsTestCase):
 
     def setUp(self):
         super(MultitenancyAuthenticationTests, self).setUp()
 
+        # in aether.sdk.multitenancy.utils
+        # def get_current_realm(request, default_realm=settings.DEFAULT_REALM):
+        modules_sdk = [
+            'aether.sdk.multitenancy.utils',
+            'aether.sdk.auth.utils',
+        ]
+        for module_sdk in modules_sdk:
+            reload(sys.modules[module_sdk])
+            import_module(module_sdk)
+
         self.username = 'surveyor'
         self.password = '~t]:vS3Q>e{2k]CE'
         self.url = reverse('xform-list-xml')
@@ -291,6 +308,8 @@ def setUp(self):
         self.assertTrue(user.is_valid(), user.errors)
         user.save()
         self.user = get_user_model().objects.get(pk=user.data['id'])
+        self.assertEqual(self.user.username, f'{settings.DEFAULT_REALM}__{self.username}')
+        self.assertTrue(self.user.groups.filter(name=settings.DEFAULT_REALM).exists())
 
     def test_challenge(self):
         response = self.client.get(self.url)
@@ -310,7 +329,7 @@ def test__parsed_username(self):
         self.assertEqual(response.status_code, 401)
 
         self.assertIn('WWW-Authenticate', response)
-        auth = build_digest_header(self.username,
+        auth = build_digest_header(self.username,  # without realm prefix
                                    self.password,
                                    response['WWW-Authenticate'],
                                    'GET',
@@ -324,7 +343,7 @@ def test__unparsed_username(self):
         self.assertEqual(response.status_code, 401)
 
         self.assertIn('WWW-Authenticate', response)
-        auth = build_digest_header(f'{settings.DEFAULT_REALM}__{self.username}',
+        auth = build_digest_header(self.user.username,  # with realm prefix
                                    self.password,
                                    response['WWW-Authenticate'],
                                    'GET',
@@ -364,6 +383,16 @@ class GatewayAuthenticationTests(UrlsTestCase):
     def setUp(self):
         super(GatewayAuthenticationTests, self).setUp()
 
+        # in aether.sdk.multitenancy.utils
+        # def get_current_realm(request, default_realm=settings.DEFAULT_REALM):
+        modules_sdk = [
+            'aether.sdk.multitenancy.utils',
+            'aether.sdk.auth.utils',
+        ]
+        for module_sdk in modules_sdk:
+            reload(sys.modules[module_sdk])
+            import_module(module_sdk)
+
         self.username = 'surveyor'
         self.password = '~t]:vS3Q>e{2k]CE'
 

aether-odk-module/conf/pip/requirements.txt

@@ -12,10 +12,10 @@
 #
 ################################################################################
 
-aether.sdk==1.2.21
+aether.sdk==1.2.22
 autopep8==1.5.2
-boto3==1.12.45
-botocore==1.15.45
+boto3==1.12.48
+botocore==1.15.48
 cachetools==4.1.0
 certifi==2020.4.5.1
 cffi==1.14.0
@@ -30,7 +30,7 @@ django-cors-headers==3.2.1
 django-debug-toolbar==2.2
 django-minio-storage==0.3.7
 django-prometheus==2.0.0
-django-redis-sessions==0.6.1
+django-redis==4.11.0
 django-silk==4.0.1
 django-storages==1.9.1
 django-uwsgi==0.2.2
@@ -69,7 +69,7 @@ Pygments==2.6.1
 pyOpenSSL==19.1.0
 python-dateutil==2.8.1
 python-json-logger==0.1.11
-pytz==2019.3
+pytz==2020.1
 pyxform==0.15.1
 redis==3.4.1
 requests==2.23.0

aether-odk-module/entrypoint.sh

@@ -112,6 +112,9 @@ function setup {
     # migrate data model if needed
     ./manage.py migrate --noinput
 
+    # clean out expired sessions
+    ./manage.py clearsessions
+
     # arguments: -u=admin -p=secretsecret [email protected] -t=01234656789abcdefghij
     ./manage.py setup_admin -u=$ADMIN_USERNAME -p=$ADMIN_PASSWORD -t=$ADMIN_TOKEN
 

aether-producer/conf/pip/requirements.txt

@@ -16,7 +16,7 @@ attrs==19.3.0
 certifi==2020.4.5.1
 cffi==1.14.0
 chardet==3.0.4
-click==7.1.1
+click==7.1.2
 confluent-kafka==1.4.1
 cryptography==2.9.2
 entrypoints==0.3

aether-ui/conf/pip/requirements.txt

@@ -12,10 +12,10 @@
 #
 ################################################################################
 
-aether.sdk==1.2.21
+aether.sdk==1.2.22
 autopep8==1.5.2
-boto3==1.12.45
-botocore==1.15.45
+boto3==1.12.48
+botocore==1.15.48
 cachetools==4.1.0
 certifi==2020.4.5.1
 cffi==1.14.0
@@ -31,7 +31,7 @@ django-debug-toolbar==2.2
 django-minio-storage==0.3.7
 django-model-utils==4.0.0
 django-prometheus==2.0.0
-django-redis-sessions==0.6.1
+django-redis==4.11.0
 django-silk==4.0.1
 django-storages==1.9.1
 django-uwsgi==0.2.2
@@ -68,7 +68,7 @@ Pygments==2.6.1
 pyOpenSSL==19.1.0
 python-dateutil==2.8.1
 python-json-logger==0.1.11
-pytz==2019.3
+pytz==2020.1
 redis==3.4.1
 requests==2.23.0
 rsa==4.0

aether-ui/entrypoint.sh

@@ -112,6 +112,9 @@ function setup {
     # migrate data model if needed
     ./manage.py migrate --noinput
 
+    # clean out expired sessions
+    ./manage.py clearsessions
+
     # arguments: -u=admin -p=secretsecret [email protected] -t=01234656789abcdefghij
     ./manage.py setup_admin -u=$ADMIN_USERNAME -p=$ADMIN_PASSWORD -t=${ADMIN_TOKEN:-}
 

docker-compose-base.yml

@@ -18,7 +18,7 @@ version: "2.1"
 services:
 
   # ---------------------------------
-  # Database + NGINX containers
+  # Auxiliary containers
   # ---------------------------------
 
   postgres-base:
@@ -128,6 +128,18 @@ services:
       PGPORT: 5432
       PGUSER: postgres
 
+      DJANGO_USE_CACHE: "true"
+      REDIS_DJANGO_CACHE: "true"
+      REDIS_SESSION_CACHE: "true"
+      REDIS_HOST: redis
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+      REDIS_PORT: 6379
+
+      REDIS_DB: 0
+      REDIS_DB_CACHEOPS: 1
+      REDIS_DB_DJANGO: 2
+      REDIS_DB_SESSION: 3
+
       EXPORT_CSV_ESCAPE: "\\"
       EXPORT_CSV_QUOTES: "\""
       EXPORT_CSV_SEPARATOR: ","
@@ -198,6 +210,18 @@ services:
       PGPORT: 5432
       PGUSER: postgres
 
+      DJANGO_USE_CACHE: "true"
+      REDIS_DJANGO_CACHE: "true"
+      REDIS_SESSION_CACHE: "true"
+      REDIS_HOST: redis
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+      REDIS_PORT: 6379
+
+      REDIS_DB: 4
+      REDIS_DB_CACHEOPS: 5
+      REDIS_DB_DJANGO: 6
+      REDIS_DB_SESSION: 7
+
       WEB_SERVER_PORT: 8102
     volumes:
       - ./aether-odk-module:/code
@@ -247,6 +271,18 @@ services:
       PGPORT: 5432
       PGUSER: postgres
 
+      DJANGO_USE_CACHE: "true"
+      REDIS_DJANGO_CACHE: "true"
+      REDIS_SESSION_CACHE: "true"
+      REDIS_HOST: redis
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+      REDIS_PORT: 6379
+
+      REDIS_DB: 8
+      REDIS_DB_CACHEOPS: 9
+      REDIS_DB_DJANGO: 10
+      REDIS_DB_SESSION: 11
+
       # use this value with "start_dev" command and webpack otherwise comment it out
       WEBPACK_STATS_FILE: /code/aether/ui/assets/bundles/webpack-stats.json
 

docker-compose-test.yml

@@ -42,6 +42,8 @@ services:
       BUCKET_NAME: test-kernel
       # use a different database when testing the integration with another container
       DB_NAME: ${TEST_KERNEL_DB_NAME:-test-kernel}
+      DEFAULT_REALM: ${CLIENT_REALM}
+      DJANGO_USE_CACHE: null
       MULTITENANCY: "true"
       PGHOST: db-test
       PROFILING_ENABLED: null
@@ -83,6 +85,8 @@ services:
       APP_URL: /
       BUCKET_NAME: test-odk
       DB_NAME: test-odk
+      DEFAULT_REALM: ${CLIENT_REALM}
+      DJANGO_USE_CACHE: null
       MULTITENANCY: "true"
       ODK_COLLECT_ENDPOINT: collect-test/
       PGHOST: db-test
@@ -106,6 +110,8 @@ services:
       AETHER_KERNEL_URL: http://kernel-test:9100
       APP_URL: /
       DB_NAME: test-ui
+      DJANGO_USE_CACHE: null
+      DEFAULT_REALM: ${CLIENT_REALM}
       MULTITENANCY: "true"
       PGHOST: db-test
       PROFILING_ENABLED: null

docker-compose.yml

@@ -94,12 +94,14 @@ services:
     depends_on:
       db:
         condition: service_healthy
-      nginx:
-        condition: service_healthy
       keycloak:
         condition: service_started
       minio:
         condition: service_started
+      nginx:
+        condition: service_healthy
+      redis:
+        condition: service_started
     networks:
       - internal
     extra_hosts:
@@ -129,13 +131,14 @@ services:
     depends_on:
       db:
         condition: service_healthy
+      keycloak:
+        condition: service_started
       nginx:
         condition: service_healthy
-      keycloak:
+      redis:
         condition: service_started
       kernel:
         condition: service_started
-      # use this dependency with "start_dev" command and webpack (optional)
       ui-assets:
         condition: service_started
     networks:
@@ -160,12 +163,14 @@ services:
     depends_on:
       db:
         condition: service_healthy
-      nginx:
-        condition: service_healthy
       keycloak:
         condition: service_started
       minio:
         condition: service_started
+      nginx:
+        condition: service_healthy
+      redis:
+        condition: service_started
       kernel:
         condition: service_started
     networks: