[FAL-2030] Updates kombu package to support multi-tenant redis authen…

…tication (openedx#397) * chore: constrain kombu to support redis6 auth Constrains kombu to https://github.com/open-craft/kombu/tree/v4.6.11.1 to allow us to use celery with multi-tenant redis. (cherry picked from commit 3bd3d1e) * feat: adds CELERY_BROKER_TRANSPORT_OPTIONS which can be configured from the lms/studio environment (cherry picked from commit 3c1a43e)
eduNEXT · Sep 10, 2021 · 04dbfa3 · 04dbfa3
1 parent 00c5165
commit 04dbfa3
Show file tree

Hide file tree

Showing 6 changed files with 12 additions and 3 deletions.
diff --git a/cms/envs/production.py b/cms/envs/production.py
@@ -412,6 +412,7 @@ def get_env_setting(setting):
 BROKER_TRANSPORT_OPTIONS = {
     'fanout_patterns': True,
     'fanout_prefix': True,
+    **ENV_TOKENS.get('CELERY_BROKER_TRANSPORT_OPTIONS', {})
 }
 
 # Message expiry time in seconds

diff --git a/lms/envs/production.py b/lms/envs/production.py
@@ -548,6 +548,7 @@ def get_env_setting(setting):
 BROKER_TRANSPORT_OPTIONS = {
     'fanout_patterns': True,
     'fanout_prefix': True,
+    **ENV_TOKENS.get('CELERY_BROKER_TRANSPORT_OPTIONS', {})
 }
 
 # Block Structures

diff --git a/requirements/constraints.txt b/requirements/constraints.txt
@@ -8,6 +8,13 @@
 # pin when possible.  Writing an issue against the offending project and
 # linking to it here is good.
 
+# kombu needs additional functionality to support multi-tenant redis
+# This branch backports the following PRs to kombu 4.6.11, because celery 4.4.7 has requirement kombu<4.7,>=4.6.10
+# https://github.com/celery/kombu/pull/1351
+# https://github.com/celery/kombu/pull/1349
+# https://github.com/celery/kombu/pull/1376
+git+https://github.com/open-craft/[email protected]#egg=kombu==4.6.11.1
+
 # celery 5.0 has dropped python3.5 support.
 celery<5.0
 

diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt
@@ -141,7 +141,7 @@ jmespath==0.10.0          # via boto3, botocore
 joblib==0.14.1            # via -c requirements/edx/../constraints.txt, -r requirements/edx/../edx-sandbox/shared.txt, nltk
 jsondiff==1.2.0           # via edx-enterprise
 jsonfield2==3.0.3         # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.in, edx-celeryutils, edx-enterprise, edx-proctoring, edx-submissions, ora2
-kombu==4.6.11             # via celery
+git+https://github.com/open-craft/kombu[email protected]#egg=kombu==4.6.11.1 # via -c requirements/edx/../constraints.txt
 laboratory==1.0.2         # via -r requirements/edx/base.in
 lazy==1.4                 # via -r requirements/edx/paver.txt, acid-xblock, lti-consumer-xblock, ora2
 libsass==0.10.0           # via -r requirements/edx/paver.txt, ora2

diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt
@@ -169,7 +169,7 @@ joblib==0.14.1            # via -c requirements/edx/../constraints.txt, -r requi
 jsondiff==1.2.0           # via -r requirements/edx/testing.txt, edx-enterprise
 jsonfield2==3.0.3         # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, edx-celeryutils, edx-enterprise, edx-proctoring, edx-submissions, ora2
 jsonschema==3.2.0         # via sphinxcontrib-openapi
-kombu==4.6.11             # via -r requirements/edx/testing.txt, celery
+git+https://github.com/open-craft/kombu[email protected]#egg=kombu==4.6.11.1 # via -c requirements/edx/../constraints.txt
 laboratory==1.0.2         # via -r requirements/edx/testing.txt
 lazy-object-proxy==1.4.3  # via -r requirements/edx/testing.txt, astroid
 lazy==1.4                 # via -r requirements/edx/testing.txt, acid-xblock, bok-choy, lti-consumer-xblock, ora2

diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt
@@ -163,7 +163,7 @@ jmespath==0.10.0          # via -r requirements/edx/base.txt, boto3, botocore
 joblib==0.14.1            # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, nltk
 jsondiff==1.2.0           # via -r requirements/edx/base.txt, edx-enterprise
 jsonfield2==3.0.3         # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, edx-celeryutils, edx-enterprise, edx-proctoring, edx-submissions, ora2
-kombu==4.6.11             # via -r requirements/edx/base.txt, celery
+git+https://github.com/open-craft/kombu[email protected]#egg=kombu==4.6.11.1 # via -c requirements/edx/../constraints.txt
 laboratory==1.0.2         # via -r requirements/edx/base.txt
 lazy-object-proxy==1.4.3  # via astroid
 lazy==1.4                 # via -r requirements/edx/base.txt, acid-xblock, bok-choy, lti-consumer-xblock, ora2