Merge pull request #3271 from bjester/minio-dev-only

Move minio to dev requirements, update urllib3

contentcuration/contentcuration/apps.py

@@ -1,7 +1,7 @@
 from django.apps import AppConfig
 from django.conf import settings
 
-from contentcuration.utils.minio_utils import ensure_storage_bucket_public
+from contentcuration.utils.storage_common import is_gcs_backend
 
 
 class ContentConfig(AppConfig):
@@ -10,5 +10,7 @@ class ContentConfig(AppConfig):
     def ready(self):
         # see note in the celery_signals.py file for why we import here.
         import contentcuration.utils.celery.signals  # noqa
-        if settings.AWS_AUTO_CREATE_BUCKET:
+
+        if settings.AWS_AUTO_CREATE_BUCKET and not is_gcs_backend():
+            from contentcuration.utils.minio_utils import ensure_storage_bucket_public
             ensure_storage_bucket_public()

contentcuration/contentcuration/management/commands/setup.py

@@ -23,8 +23,8 @@
 from contentcuration.utils.db_tools import create_topic
 from contentcuration.utils.db_tools import create_user
 from contentcuration.utils.files import duplicate_file
-from contentcuration.utils.minio_utils import ensure_storage_bucket_public
 from contentcuration.utils.publish import publish_channel
+from contentcuration.utils.storage_common import is_gcs_backend
 
 logmodule.basicConfig()
 logging = logmodule.getLogger(__name__)
@@ -56,7 +56,9 @@ def handle(self, *args, **options):
             sys.exit()
 
         # create the minio bucket
-        ensure_storage_bucket_public()
+        if not is_gcs_backend():
+            from contentcuration.utils.minio_utils import ensure_storage_bucket_public
+            ensure_storage_bucket_public()
 
         # create the cache table
         try:

contentcuration/contentcuration/utils/minio_utils.py

@@ -13,6 +13,8 @@
 from minio.error import BucketAlreadyOwnedByYou
 from minio.error import ResponseError
 
+from contentcuration.utils.storage_common import is_gcs_backend
+
 logger = logging.getLogger(__name__)
 
 
@@ -42,6 +44,12 @@ def stop_minio(p):
 
 
 def ensure_storage_bucket_public(bucket=None, will_sleep=True):
+    # GCS' S3 compatibility is broken, especially in bucket operations;
+    # skip bucket creation there and just bug Aron to create buckets with
+    # public-read access for you
+    if is_gcs_backend():
+        logging.info("Skipping storage creation on googleapis")
+        return
 
     # If true, sleep for 5 seconds to wait for minio to start
     if will_sleep:
@@ -53,14 +61,6 @@ def ensure_storage_bucket_public(bucket=None, will_sleep=True):
         bucketname = bucket
 
     host = urlparse(settings.AWS_S3_ENDPOINT_URL).netloc
-
-    # GCS' S3 compatibility is broken, especially in bucket operations;
-    # skip bucket creation there and just bug Aron to create buckets with
-    # public-read access for you
-    if "storage.googleapis.com" in host:
-        logging.info("Skipping storage creation on googleapis")
-        return
-
     c = minio.Minio(
         host,
         access_key=settings.AWS_ACCESS_KEY_ID,

contentcuration/contentcuration/utils/storage_common.py

@@ -1,6 +1,7 @@
 import mimetypes
 import os
 from datetime import timedelta
+from urllib.parse import urlparse
 
 from django.conf import settings
 from django.core.files.storage import default_storage
@@ -19,6 +20,15 @@ class UnknownStorageBackendError(Exception):
     pass
 
 
+def is_gcs_backend():
+    """
+    Determines if storage is GCS backend, which if not we can assume it is minio
+    :return: A bool
+    """
+    host = urlparse(settings.AWS_S3_ENDPOINT_URL).netloc
+    return "storage.googleapis.com" in host
+
+
 def determine_content_type(filename):
     """
     Guesses the content type of a filename. Returns the mimetype of a file.

requirements-dev.in

@@ -41,3 +41,4 @@ git+https://github.com/someshchaturvedi/customizable-django-profiler.git#customi
 tabulate==0.8.2
 fonttools
 flower==0.9.4
+minio==3.0.3

requirements-dev.txt

@@ -40,6 +40,7 @@ certifi==2020.12.5
     # via
     #   -c requirements.txt
     #   geventhttpclient
+    #   minio
     #   requests
 cfgv==3.3.0
     # via pre-commit
@@ -194,6 +195,8 @@ mccabe==0.6.1
     # via
     #   flake8
     #   pylint
+minio==3.0.3
+    # via -r requirements-dev.in
 mixer==6.1.3
     # via -r requirements-dev.in
 mock==4.0.3
@@ -301,6 +304,7 @@ pytz==2021.1
     #   celery
     #   django
     #   flower
+    #   minio
 pyyaml==5.4.1
     # via
     #   aspy.yaml
@@ -371,9 +375,10 @@ uritemplate==3.0.1
     #   -c requirements.txt
     #   coreapi
     #   drf-yasg
-urllib3==1.26.4
+urllib3==1.26.5
     # via
     #   -c requirements.txt
+    #   minio
     #   requests
 vine==1.3.0
     # via

requirements.in

@@ -14,7 +14,6 @@ newrelic>=2.86.3.70
 celery<5
 redis
 pycountry==17.5.14
-minio==3.0.3
 pathlib
 progressbar2==3.38.0
 python-postmark==0.5.0

requirements.txt

@@ -30,7 +30,6 @@ celery==4.4.7
     # via -r requirements.in
 certifi==2020.12.5
     # via
-    #   minio
     #   requests
     #   sentry-sdk
 cffi==1.14.5
@@ -168,8 +167,6 @@ kombu==4.6.11
     # via celery
 le-utils==0.1.31
     # via -r requirements.in
-minio==3.0.3
-    # via -r requirements.in
 newrelic==6.2.0.156
     # via -r requirements.in
 oauth2client==4.1.3
@@ -232,7 +229,6 @@ pytz==2021.1
     #   django
     #   django-postmark
     #   google-api-core
-    #   minio
 raven==6.10.0
     # via -r requirements.in
 redis==3.5.3
@@ -278,10 +274,9 @@ typing-extensions==3.10.0.0
     # via asgiref
 uritemplate==3.0.1
     # via google-api-python-client
-urllib3==1.26.4
+urllib3==1.26.5
     # via
     #   botocore
-    #   minio
     #   requests
     #   sentry-sdk
 vine==1.3.0