Feature/ephemeral storage (#1259)

* Adds ability to set ephemeral_storage

* Adds tests for ephemeral_storage settings

* Clarifies test data

* Adds readme description for ephemeral_storage setting

* Adds ephemeral_storage overwrite to test settings

README.md

@@ -974,6 +974,7 @@ to change Zappa's behavior. Use these at your own risk!
         "log_level": "DEBUG", // Set the Zappa log level. Can be one of CRITICAL, ERROR, WARNING, INFO and DEBUG. Default: DEBUG
         "manage_roles": true, // Have Zappa automatically create and define IAM execution roles and policies. Default true. If false, you must define your own IAM Role and role_name setting.
         "memory_size": 512, // Lambda function memory in MB. Default 512.
+        "ephemeral_storage": { "Size": 512 }, // Lambda function ephemeral_storage size in MB, Default 512, Max 10240
         "num_retained_versions":null, // Indicates the number of old versions to retain for the lambda. If absent, keeps all the versions of the function.
         "payload_compression": true, // Whether or not to enable API gateway payload compression (default: true)
         "payload_minimum_compression_size": 0, // The threshold size (in bytes) below which payload compression will not be applied (default: 0)
@@ -1060,7 +1061,7 @@ You can also simply handle CORS directly in your application. Your web framework
 
 ### Large Projects
 
-AWS currently limits Lambda zip sizes to 50 megabytes. If your project is larger than that, set `slim_handler: true` in your `zappa_settings.json`. In this case, your fat application package will be replaced with a small handler-only package. The handler file then pulls the rest of the large project down from S3 at run time! The initial load of the large project may add to startup overhead, but the difference should be minimal on a warm lambda function. Note that this will also eat into the storage space of your application function. Note that AWS currently [limits](https://docs.aws.amazon.com/lambda/latest/dg/limits.html) the `/tmp` directory storage to 512 MB, so your project must still be smaller than that.
+AWS currently limits Lambda zip sizes to 50 megabytes. If your project is larger than that, set `slim_handler: true` in your `zappa_settings.json`. In this case, your fat application package will be replaced with a small handler-only package. The handler file then pulls the rest of the large project down from S3 at run time! The initial load of the large project may add to startup overhead, but the difference should be minimal on a warm lambda function. Note that this will also eat into the storage space of your application function. Note that AWS [supports](https://aws.amazon.com/blogs/compute/using-larger-ephemeral-storage-for-aws-lambda/) custom `/tmp` directory storage size in a range of 512 - 10240 MB. Use `ephemeral_storage` in `zappa_settings.json` to adjust to your needs if your project is larger than default 512 MB.
 
 ### Enabling Bash Completion
 

test_settings.json

@@ -10,6 +10,9 @@
            "zip": "test_settings.callback"
        },
        "delete_local_zip": true,
+       "ephemeral_storage": {
+           "Size": 1024
+       },
        "debug": true,
        "parameter_depth": 2,
        "prebuild_script": "tests.test_app.prebuild_me",
@@ -107,6 +110,18 @@
           "EXTENDO": "You bet"
        }
     },
+    "invalid_ephemeral_storage_out_of_range": {
+        "extends": "ttt888",
+        "ephemeral_storage": {
+            "Size": 99999
+        }
+    },
+    "invalid_ephemeral_storage_missing_key": {
+        "extends": "ttt888",
+        "ephemeral_storage": {
+            "BadKey": 1024
+        }
+    },
     "build_package_only_delete_local_zip_false": {
       "delete_local_zip": false,
       "use_precompiled_packages": false,

tests/tests.py

@@ -1159,6 +1159,24 @@ def test_load_settings(self):
         zappa_cli.load_settings("test_settings.json")
         self.assertEqual(False, zappa_cli.stage_config["touch"])
 
+    def test_load_settings_ephemeral_storage_overwrite(self):
+        zappa_cli = ZappaCLI()
+        zappa_cli.api_stage = "ttt888"
+        zappa_cli.load_settings("test_settings.json")
+        self.assertEqual(zappa_cli.stage_config["ephemeral_storage"]["Size"], 1024)
+
+    def test_load_settings_ephemeral_storage_out_of_range(self):
+        zappa_cli = ZappaCLI()
+        zappa_cli.api_stage = "invalid_ephemeral_storage_out_of_range"
+        with self.assertRaises(ClickException) as err:
+            zappa_cli.load_settings("test_settings.json")
+
+    def test_load_settings_ephemeral_storage_missing_key(self):
+        zappa_cli = ZappaCLI()
+        zappa_cli.api_stage = "invalid_ephemeral_storage_missing_key"
+        with self.assertRaises(ClickException) as err:
+            zappa_cli.load_settings("test_settings.json")
+
     def test_load_extended_settings(self):
         zappa_cli = ZappaCLI()
         zappa_cli.api_stage = "extendo"

zappa/cli.py

@@ -108,6 +108,7 @@ class ZappaCLI:
     handler_path = None
     vpc_config = None
     memory_size = None
+    ephemeral_storage = None
     use_apigateway = None
     lambda_handler = None
     django_settings = None
@@ -810,6 +811,7 @@ def deploy(self, source_zip=None, docker_image_uri=None):
                 dead_letter_config=self.dead_letter_config,
                 timeout=self.timeout_seconds,
                 memory_size=self.memory_size,
+                ephemeral_storage=self.ephemeral_storage,
                 runtime=self.runtime,
                 aws_environment_variables=self.aws_environment_variables,
                 aws_kms_key_arn=self.aws_kms_key_arn,
@@ -1050,6 +1052,7 @@ def update(self, source_zip=None, no_upload=False, docker_image_uri=None):
             vpc_config=self.vpc_config,
             timeout=self.timeout_seconds,
             memory_size=self.memory_size,
+            ephemeral_storage=self.ephemeral_storage,
             runtime=self.runtime,
             aws_environment_variables=self.aws_environment_variables,
             aws_kms_key_arn=self.aws_kms_key_arn,
@@ -2228,6 +2231,14 @@ def load_settings(self, settings_file=None, session=None):
         )
         self.vpc_config = self.stage_config.get("vpc_config", {})
         self.memory_size = self.stage_config.get("memory_size", 512)
+        self.ephemeral_storage = self.stage_config.get("ephemeral_storage", {"Size": 512})
+
+        # Validate ephemeral storage structure and size
+        if "Size" not in self.ephemeral_storage:
+            raise ClickException("Please provide a valid Size for ephemeral_storage in your Zappa settings.")
+        elif not 512 <= self.ephemeral_storage["Size"] <= 10240:
+            raise ClickException("Please provide a valid ephemeral_storage size between 512 - 10240 in your Zappa settings.")
+
         self.app_function = self.stage_config.get("app_function", None)
         self.exception_handler = self.stage_config.get("exception_handler", None)
         self.aws_region = self.stage_config.get("aws_region", None)

zappa/core.py

@@ -1109,6 +1109,7 @@ def create_lambda_function(
         description="Zappa Deployment",
         timeout=30,
         memory_size=512,
+        ephemeral_storage={"Size": 512},
         publish=True,
         vpc_config=None,
         dead_letter_config=None,
@@ -1145,6 +1146,7 @@ def create_lambda_function(
             Description=description,
             Timeout=timeout,
             MemorySize=memory_size,
+            EphemeralStorage=ephemeral_storage,
             Publish=publish,
             VpcConfig=vpc_config,
             DeadLetterConfig=dead_letter_config,
@@ -1293,6 +1295,7 @@ def update_lambda_configuration(
         description="Zappa Deployment",
         timeout=30,
         memory_size=512,
+        ephemeral_storage={"Size": 512},
         publish=True,
         vpc_config=None,
         runtime="python3.7",
@@ -1337,6 +1340,7 @@ def update_lambda_configuration(
             "Description": description,
             "Timeout": timeout,
             "MemorySize": memory_size,
+            "EphemeralStorage": ephemeral_storage,
             "VpcConfig": vpc_config,
             "Environment": {"Variables": aws_environment_variables},
             "KMSKeyArn": aws_kms_key_arn,