Backport v2.8.2: Pre-download file from S3 to TemporaryFile on File.o…

…pen (#699)
CERT-Polska · Oct 26, 2022 · f063214 · f063214
1 parent 15d7bf8
commit f063214
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 20 deletions.
diff --git a/docker-compose-dev-karton.yml b/docker-compose-dev-karton.yml
@@ -4,11 +4,12 @@ version: "3.3"
 services:
   minio:
     image: minio/minio
-    command: server /data
+    command: "server --address 0.0.0.0:9000 --console-address :8070 /data"
     volumes:
       - /tmp/minio:/data
     ports:
       - "127.0.0.1:9000:9000"
+      - "127.0.0.1:8070:8070"
     environment:
       - MINIO_ROOT_USER=mwdb-test-access
       - MINIO_ROOT_PASSWORD=mwdb-test-key
@@ -34,12 +35,12 @@ services:
       MWDB_ENABLE_REGISTRATION: 1
       MWDB_ENABLE_KARTON: 1
       # Uncomment if you want to test S3 functions
-      # MWDB_STORAGE_PROVIDER: s3
-      # MWDB_HASH_PATHING: 0
-      # MWDB_S3_STORAGE_ENDPOINT: "minio:9000"
-      # MWDB_S3_STORAGE_ACCESS_KEY: "mwdb-test-access"
-      # MWDB_S3_STORAGE_SECRET_KEY: "mwdb-test-key"
-      # MWDB_S3_STORAGE_BUCKET_NAME: "mwdb"
+      MWDB_STORAGE_PROVIDER: s3
+      MWDB_HASH_PATHING: 0
+      MWDB_S3_STORAGE_ENDPOINT: "minio:9000"
+      MWDB_S3_STORAGE_ACCESS_KEY: "mwdb-test-access"
+      MWDB_S3_STORAGE_SECRET_KEY: "mwdb-test-key"
+      MWDB_S3_STORAGE_BUCKET_NAME: "mwdb"
     volumes:
       - "./docker/mail_templates:/app/mail_templates"
       - "./mwdb:/app/mwdb"

diff --git a/mwdb/model/file.py b/mwdb/model/file.py
@@ -223,19 +223,27 @@ def open(self):
                 stream.seek(0, os.SEEK_SET)
                 return stream
         if app_config.mwdb.storage_provider == StorageProviderType.S3:
-            return get_s3_client(
-                app_config.mwdb.s3_storage_endpoint,
-                app_config.mwdb.s3_storage_access_key,
-                app_config.mwdb.s3_storage_secret_key,
-                app_config.mwdb.s3_storage_region_name,
-                app_config.mwdb.s3_storage_secure,
-                app_config.mwdb.s3_storage_iam_auth,
-            ).get_object(
-                Bucket=app_config.mwdb.s3_storage_bucket_name,
-                Key=self._calculate_path(),
-            )[
-                "Body"
-            ]
+            # Stream coming from Boto3 get_object is not buffered and not seekable.
+            # We need to download it to the temporary file first.
+            stream = tempfile.TemporaryFile(mode="w+b")
+            try:
+                get_s3_client(
+                    app_config.mwdb.s3_storage_endpoint,
+                    app_config.mwdb.s3_storage_access_key,
+                    app_config.mwdb.s3_storage_secret_key,
+                    app_config.mwdb.s3_storage_region_name,
+                    app_config.mwdb.s3_storage_secure,
+                    app_config.mwdb.s3_storage_iam_auth,
+                ).download_fileobj(
+                    Bucket=app_config.mwdb.s3_storage_bucket_name,
+                    Key=self._calculate_path(),
+                    Fileobj=stream,
+                )
+                stream.seek(0, io.SEEK_SET)
+                return stream
+            except Exception:
+                stream.close()
+                raise
         elif app_config.mwdb.storage_provider == StorageProviderType.DISK:
             return open(self._calculate_path(), "rb")
         else: