diff --git a/README.md b/README.md index 6c801b0ec..8e9878c49 100644 --- a/README.md +++ b/README.md @@ -974,6 +974,7 @@ to change Zappa's behavior. Use these at your own risk! "log_level": "DEBUG", // Set the Zappa log level. Can be one of CRITICAL, ERROR, WARNING, INFO and DEBUG. Default: DEBUG "manage_roles": true, // Have Zappa automatically create and define IAM execution roles and policies. Default true. If false, you must define your own IAM Role and role_name setting. "memory_size": 512, // Lambda function memory in MB. Default 512. + "ephemeral_storage": { "Size": 512 }, // Lambda function ephemeral_storage size in MB, Default 512, Max 10240 "num_retained_versions":null, // Indicates the number of old versions to retain for the lambda. If absent, keeps all the versions of the function. "payload_compression": true, // Whether or not to enable API gateway payload compression (default: true) "payload_minimum_compression_size": 0, // The threshold size (in bytes) below which payload compression will not be applied (default: 0) @@ -1060,7 +1061,7 @@ You can also simply handle CORS directly in your application. Your web framework ### Large Projects -AWS currently limits Lambda zip sizes to 50 megabytes. If your project is larger than that, set `slim_handler: true` in your `zappa_settings.json`. In this case, your fat application package will be replaced with a small handler-only package. The handler file then pulls the rest of the large project down from S3 at run time! The initial load of the large project may add to startup overhead, but the difference should be minimal on a warm lambda function. Note that this will also eat into the storage space of your application function. Note that AWS currently [limits](https://docs.aws.amazon.com/lambda/latest/dg/limits.html) the `/tmp` directory storage to 512 MB, so your project must still be smaller than that. +AWS currently limits Lambda zip sizes to 50 megabytes. If your project is larger than that, set `slim_handler: true` in your `zappa_settings.json`. In this case, your fat application package will be replaced with a small handler-only package. The handler file then pulls the rest of the large project down from S3 at run time! The initial load of the large project may add to startup overhead, but the difference should be minimal on a warm lambda function. Note that this will also eat into the storage space of your application function. Note that AWS [supports](https://aws.amazon.com/blogs/compute/using-larger-ephemeral-storage-for-aws-lambda/) custom `/tmp` directory storage size in a range of 512 - 10240 MB. Use `ephemeral_storage` in `zappa_settings.json` to adjust to your needs if your project is larger than default 512 MB. ### Enabling Bash Completion diff --git a/test_settings.json b/test_settings.json index 2ee126a1f..178b98d8b 100644 --- a/test_settings.json +++ b/test_settings.json @@ -10,6 +10,9 @@ "zip": "test_settings.callback" }, "delete_local_zip": true, + "ephemeral_storage": { + "Size": 1024 + }, "debug": true, "parameter_depth": 2, "prebuild_script": "tests.test_app.prebuild_me", @@ -107,6 +110,18 @@ "EXTENDO": "You bet" } }, + "invalid_ephemeral_storage_out_of_range": { + "extends": "ttt888", + "ephemeral_storage": { + "Size": 99999 + } + }, + "invalid_ephemeral_storage_missing_key": { + "extends": "ttt888", + "ephemeral_storage": { + "BadKey": 1024 + } + }, "build_package_only_delete_local_zip_false": { "delete_local_zip": false, "use_precompiled_packages": false, diff --git a/tests/tests.py b/tests/tests.py index aaea595e8..a08d30697 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1159,6 +1159,24 @@ def test_load_settings(self): zappa_cli.load_settings("test_settings.json") self.assertEqual(False, zappa_cli.stage_config["touch"]) + def test_load_settings_ephemeral_storage_overwrite(self): + zappa_cli = ZappaCLI() + zappa_cli.api_stage = "ttt888" + zappa_cli.load_settings("test_settings.json") + self.assertEqual(zappa_cli.stage_config["ephemeral_storage"]["Size"], 1024) + + def test_load_settings_ephemeral_storage_out_of_range(self): + zappa_cli = ZappaCLI() + zappa_cli.api_stage = "invalid_ephemeral_storage_out_of_range" + with self.assertRaises(ClickException) as err: + zappa_cli.load_settings("test_settings.json") + + def test_load_settings_ephemeral_storage_missing_key(self): + zappa_cli = ZappaCLI() + zappa_cli.api_stage = "invalid_ephemeral_storage_missing_key" + with self.assertRaises(ClickException) as err: + zappa_cli.load_settings("test_settings.json") + def test_load_extended_settings(self): zappa_cli = ZappaCLI() zappa_cli.api_stage = "extendo" diff --git a/zappa/cli.py b/zappa/cli.py index e78707869..a4997d9ea 100755 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -108,6 +108,7 @@ class ZappaCLI: handler_path = None vpc_config = None memory_size = None + ephemeral_storage = None use_apigateway = None lambda_handler = None django_settings = None @@ -810,6 +811,7 @@ def deploy(self, source_zip=None, docker_image_uri=None): dead_letter_config=self.dead_letter_config, timeout=self.timeout_seconds, memory_size=self.memory_size, + ephemeral_storage=self.ephemeral_storage, runtime=self.runtime, aws_environment_variables=self.aws_environment_variables, aws_kms_key_arn=self.aws_kms_key_arn, @@ -1050,6 +1052,7 @@ def update(self, source_zip=None, no_upload=False, docker_image_uri=None): vpc_config=self.vpc_config, timeout=self.timeout_seconds, memory_size=self.memory_size, + ephemeral_storage=self.ephemeral_storage, runtime=self.runtime, aws_environment_variables=self.aws_environment_variables, aws_kms_key_arn=self.aws_kms_key_arn, @@ -2228,6 +2231,14 @@ def load_settings(self, settings_file=None, session=None): ) self.vpc_config = self.stage_config.get("vpc_config", {}) self.memory_size = self.stage_config.get("memory_size", 512) + self.ephemeral_storage = self.stage_config.get("ephemeral_storage", {"Size": 512}) + + # Validate ephemeral storage structure and size + if "Size" not in self.ephemeral_storage: + raise ClickException("Please provide a valid Size for ephemeral_storage in your Zappa settings.") + elif not 512 <= self.ephemeral_storage["Size"] <= 10240: + raise ClickException("Please provide a valid ephemeral_storage size between 512 - 10240 in your Zappa settings.") + self.app_function = self.stage_config.get("app_function", None) self.exception_handler = self.stage_config.get("exception_handler", None) self.aws_region = self.stage_config.get("aws_region", None) diff --git a/zappa/core.py b/zappa/core.py index 4a4d8b616..19a95821d 100644 --- a/zappa/core.py +++ b/zappa/core.py @@ -1109,6 +1109,7 @@ def create_lambda_function( description="Zappa Deployment", timeout=30, memory_size=512, + ephemeral_storage={"Size": 512}, publish=True, vpc_config=None, dead_letter_config=None, @@ -1145,6 +1146,7 @@ def create_lambda_function( Description=description, Timeout=timeout, MemorySize=memory_size, + EphemeralStorage=ephemeral_storage, Publish=publish, VpcConfig=vpc_config, DeadLetterConfig=dead_letter_config, @@ -1293,6 +1295,7 @@ def update_lambda_configuration( description="Zappa Deployment", timeout=30, memory_size=512, + ephemeral_storage={"Size": 512}, publish=True, vpc_config=None, runtime="python3.7", @@ -1337,6 +1340,7 @@ def update_lambda_configuration( "Description": description, "Timeout": timeout, "MemorySize": memory_size, + "EphemeralStorage": ephemeral_storage, "VpcConfig": vpc_config, "Environment": {"Variables": aws_environment_variables}, "KMSKeyArn": aws_kms_key_arn,