Make the index names configurable using env vars

.flake8

@@ -10,4 +10,6 @@ per-file-ignores =
     *__init__*:F401
     # https://www.flake8rules.com/rules/E402.html; patches are applied before all imports are finished
     *wsgi.py:E402
+    # https://www.flake8rules.com/rules/F401.html; django settings holds global values
+    *settings.py:F401
 max-line-length = 88

api/catalog/api/constants/media_types.py

@@ -5,4 +5,6 @@
 AUDIO_TYPE = "audio"
 IMAGE_TYPE = "image"
 
-MEDIA_TYPES = [(AUDIO_TYPE, "Audio"), (IMAGE_TYPE, "Image")]
+MEDIA_TYPES = [AUDIO_TYPE, IMAGE_TYPE]
+
+MEDIA_TYPE_CHOICES = [(AUDIO_TYPE, "Audio"), (IMAGE_TYPE, "Image")]

api/catalog/api/models/models.py

@@ -1,6 +1,6 @@
 from django.db import models
 
-from catalog.api.constants.media_types import MEDIA_TYPES
+from catalog.api.constants.media_types import MEDIA_TYPE_CHOICES
 from catalog.api.models.base import OpenLedgerModel
 
 
@@ -29,7 +29,7 @@ class ContentProvider(models.Model):
     notes = models.TextField(null=True)
     media_type = models.CharField(
         max_length=80,
-        choices=MEDIA_TYPES,
+        choices=MEDIA_TYPE_CHOICES,
     )
 
     class Meta:

api/catalog/api/views/audio_views.py

@@ -1,9 +1,11 @@
+from django.conf import settings
 from django.utils.decorators import method_decorator
 from rest_framework.decorators import action
 from rest_framework.response import Response
 
 from drf_yasg.utils import swagger_auto_schema
 
+from catalog.api.constants.media_types import AUDIO_TYPE
 from catalog.api.docs.audio_docs import (
     AudioComplain,
     AudioDetail,
@@ -39,7 +41,7 @@ class AudioViewSet(MediaViewSet):
 
     model_class = Audio
     query_serializer_class = AudioSearchRequestSerializer
-    default_index = "audio"
+    default_index = settings.MEDIA_INDEX_MAPPING[AUDIO_TYPE]
     qa_index = "search-qa-audio"
 
     serializer_class = AudioSerializer

api/catalog/api/views/image_views.py

@@ -11,6 +11,7 @@
 from drf_yasg.utils import swagger_auto_schema
 from PIL import Image as PILImage
 
+from catalog.api.constants.media_types import IMAGE_TYPE
 from catalog.api.docs.image_docs import (
     ImageComplain,
     ImageDetail,
@@ -51,7 +52,7 @@ class ImageViewSet(MediaViewSet):
 
     model_class = Image
     query_serializer_class = ImageSearchRequestSerializer
-    default_index = "image"
+    default_index = settings.MEDIA_INDEX_MAPPING[IMAGE_TYPE]
     qa_index = "search-qa-image"
 
     serializer_class = ImageSerializer

api/catalog/configuration/aws.py

@@ -0,0 +1,5 @@
+from decouple import config
+
+
+AWS_ACCESS_KEY_ID = config("AWS_ACCESS_KEY_ID", default="")
+AWS_SECRET_ACCESS_KEY = config("AWS_SECRET_ACCESS_KEY", default="")

api/catalog/configuration/elasticsearch.py

@@ -0,0 +1,52 @@
+from django.conf import settings
+
+from aws_requests_auth.aws_auth import AWSRequestsAuth
+from decouple import config
+from elasticsearch import Elasticsearch, RequestsHttpConnection
+from elasticsearch_dsl import connections
+
+from catalog.api.constants.media_types import MEDIA_TYPES
+
+
+def _elasticsearch_connect():
+    """
+    Connect to configured Elasticsearch domain.
+
+    :return: An Elasticsearch connection object.
+    """
+
+    es_url = config("ELASTICSEARCH_URL", default="localhost")
+    es_port = config("ELASTICSEARCH_PORT", default=9200, cast=int)
+    es_aws_region = config("ELASTICSEARCH_AWS_REGION", default="us-east-1")
+
+    auth = AWSRequestsAuth(
+        aws_access_key=settings.AWS_ACCESS_KEY_ID,
+        aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
+        aws_host=es_url,
+        aws_region=es_aws_region,
+        aws_service="es",
+    )
+    auth.encode = lambda x: bytes(x.encode("utf-8"))
+    _es = Elasticsearch(
+        host=es_url,
+        port=es_port,
+        connection_class=RequestsHttpConnection,
+        timeout=10,
+        max_retries=1,
+        retry_on_timeout=True,
+        http_auth=auth,
+        wait_for_status="yellow",
+    )
+    _es.info()
+    return _es
+
+
+ES = _elasticsearch_connect()
+"""Elasticsearch client, also aliased to connection 'default'"""
+connections.add_connection("default", ES)
+
+MEDIA_INDEX_MAPPING = {
+    media_type: config(f"{media_type.upper()}_INDEX_NAME", default=media_type)
+    for media_type in MEDIA_TYPES
+}
+"""mapping of media types to Elasticsearch index names"""

api/catalog/settings.py

@@ -14,13 +14,12 @@
 from socket import gethostbyname, gethostname
 
 import sentry_sdk
-from aws_requests_auth.aws_auth import AWSRequestsAuth
 from decouple import config
-from elasticsearch import Elasticsearch, RequestsHttpConnection
-from elasticsearch_dsl import connections
 from sentry_sdk.integrations.django import DjangoIntegration
 
-from catalog.logger import LOGGING as LOGGING_CONF
+from catalog.configuration.aws import AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY
+from catalog.configuration.elasticsearch import ES, MEDIA_INDEX_MAPPING
+from catalog.configuration.logging import LOGGING
 
 
 # Build paths inside the project like this: BASE_DIR.join('dir', 'subdir'...)
@@ -70,8 +69,6 @@
 
 USE_S3 = config("USE_S3", default=False, cast=bool)
 
-LOGGING = LOGGING_CONF
-
 # Application definition
 
 INSTALLED_APPS = [
@@ -298,14 +295,6 @@
 
 WATERMARK_ENABLED = config("WATERMARK_ENABLED", default=False, cast=bool)
 
-ELASTICSEARCH_URL = config("ELASTICSEARCH_URL", default="localhost")
-ELASTICSEARCH_PORT = config("ELASTICSEARCH_PORT", default=9200, cast=int)
-ELASTICSEARCH_AWS_REGION = config("ELASTICSEARCH_AWS_REGION", default="us-east-1")
-
-# Additional settings for dev/prod environments
-AWS_ACCESS_KEY_ID = config("AWS_ACCESS_KEY_ID", default="")
-AWS_SECRET_ACCESS_KEY = config("AWS_SECRET_ACCESS_KEY", default="")
-
 EMAIL_SENDER = config("EMAIL_SENDER", default="")
 EMAIL_HOST = config("EMAIL_HOST", default="")
 EMAIL_PORT = config("EMAIL_PORT", default=587, cast=int)
@@ -354,38 +343,3 @@
         send_default_pii=False,
         environment=ENVIRONMENT,
     )
-
-
-# Elasticsearch connection
-
-
-def _elasticsearch_connect():
-    """
-    Connect to configured Elasticsearch domain.
-
-    :return: An Elasticsearch connection object.
-    """
-    auth = AWSRequestsAuth(
-        aws_access_key=AWS_ACCESS_KEY_ID,
-        aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
-        aws_host=ELASTICSEARCH_URL,
-        aws_region=ELASTICSEARCH_AWS_REGION,
-        aws_service="es",
-    )
-    auth.encode = lambda x: bytes(x.encode("utf-8"))
-    _es = Elasticsearch(
-        host=ELASTICSEARCH_URL,
-        port=ELASTICSEARCH_PORT,
-        connection_class=RequestsHttpConnection,
-        timeout=10,
-        max_retries=1,
-        retry_on_timeout=True,
-        http_auth=auth,
-        wait_for_status="yellow",
-    )
-    _es.info()
-    return _es
-
-
-ES = _elasticsearch_connect()
-connections.add_connection("default", ES)

api/env.template

@@ -44,6 +44,9 @@ SEMANTIC_VERSION="1.0.0"
 #ELASTICSEARCH_PORT="9200"
 #ELASTICSEARCH_AWS_REGION="us-east-1"
 
+#IMAGE_INDEX_NAME="image"
+#AUDIO_INDEX_NAME="audio"
+
 #AWS_ACCESS_KEY_ID=""
 #AWS_SECRET_ACCESS_KEY=""