From 67b137784d4faa13da9d83a332b17d9e1d46aaad Mon Sep 17 00:00:00 2001 From: Denny Biasiolli Date: Tue, 15 Aug 2023 11:48:05 +0200 Subject: [PATCH 1/7] adding support for Python 3.11 (#1261) Ref. https://github.com/aws/aws-lambda-base-images/issues/62 --- .github/ISSUE_TEMPLATE.md | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/cd.yaml | 4 ++-- .github/workflows/ci.yaml | 2 +- README.md | 6 +++--- setup.py | 1 + tests/tests.py | 27 +++++++++++++++++++++++++++ zappa/__init__.py | 2 +- zappa/core.py | 4 +++- zappa/utilities.py | 4 +++- 10 files changed, 43 insertions(+), 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index d1fd903f5..cc4114a92 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,7 +1,7 @@ ## Context - + ## Expected Behavior diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a46c8a28c..79e454b3d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -20,7 +20,7 @@ Before you submit this PR, please make sure that you meet these criteria: * Did you **make sure this code actually works on Lambda**, as well as locally? -* Did you test this code with all of **Python 3.7**, **Python 3.8**, **Python 3.9** and **Python 3.10** ? +* Did you test this code with all of **Python 3.7**, **Python 3.8**, **Python 3.9**, **Python 3.10** and **Python 3.11** ? * Does this commit ONLY relate to the issue at hand and have your linter shit all over the code? diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index b22f50413..b19664b48 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -15,10 +15,10 @@ jobs: steps: - name: Checkout Code Repository uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Install `pypa/build` run: python -m pip install build - name: Build sdist and wheel diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9f1177bfe..acb688536 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: [3.7, 3.8, 3.9, "3.10"] + python: [3.7, 3.8, 3.9, "3.10", "3.11"] steps: - name: Checkout Code Repository uses: actions/checkout@v3 diff --git a/README.md b/README.md index 9af47f016..6c801b0ec 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ __Awesome!__ ## Installation and Configuration -_Before you begin, make sure you are running Python 3.7/3.8/3.9/3.10 and you have a valid AWS account and your [AWS credentials file](https://blogs.aws.amazon.com/security/post/Tx3D6U6WSFGOK2H/A-New-and-Standardized-Way-to-Manage-Credentials-in-the-AWS-SDKs) is properly installed._ +_Before you begin, make sure you are running Python 3.7/3.8/3.9/3.10/3.11 and you have a valid AWS account and your [AWS credentials file](https://blogs.aws.amazon.com/security/post/Tx3D6U6WSFGOK2H/A-New-and-Standardized-Way-to-Manage-Credentials-in-the-AWS-SDKs) is properly installed._ **Zappa** can easily be installed through pip, like so: @@ -443,7 +443,7 @@ For instance, suppose you have a basic application in a file called "my_app.py", Any remote print statements made and the value the function returned will then be printed to your local console. **Nifty!** -You can also invoke interpretable Python 3.7/3.8/3.9/3.10 strings directly by using `--raw`, like so: +You can also invoke interpretable Python 3.7/3.8/3.9/3.10/3.11 strings directly by using `--raw`, like so: $ zappa invoke production "print(1 + 2 + 3)" --raw @@ -984,7 +984,7 @@ to change Zappa's behavior. Use these at your own risk! "role_name": "MyLambdaRole", // Name of Zappa execution role. Default --ZappaExecutionRole. To use a different, pre-existing policy, you must also set manage_roles to false. "role_arn": "arn:aws:iam::12345:role/app-ZappaLambdaExecutionRole", // ARN of Zappa execution role. Default to None. To use a different, pre-existing policy, you must also set manage_roles to false. This overrides role_name. Use with temporary credentials via GetFederationToken. "route53_enabled": true, // Have Zappa update your Route53 Hosted Zones when certifying with a custom domain. Default true. - "runtime": "python3.10", // Python runtime to use on Lambda. Can be one of "python3.7", "python3.8", "python3.9", or "python3.10". Defaults to whatever the current Python being used is. + "runtime": "python3.11", // Python runtime to use on Lambda. Can be one of "python3.7", "python3.8", "python3.9", or "python3.10", or "python3.11". Defaults to whatever the current Python being used is. "s3_bucket": "dev-bucket", // Zappa zip bucket, "slim_handler": false, // Useful if project >50M. Set true to just upload a small handler to Lambda and load actual project from S3 at runtime. Default false. "settings_file": "~/Projects/MyApp/settings/dev_settings.py", // Server side settings file location, diff --git a/setup.py b/setup.py index e68926ac7..a0eb02667 100755 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Framework :: Django", "Framework :: Django :: 1.11", "Framework :: Django :: 2.0", diff --git a/tests/tests.py b/tests/tests.py index 07a38c41d..aaea595e8 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -197,6 +197,33 @@ def test_get_manylinux_python310(self): self.assertTrue(os.path.isfile(path)) os.remove(path) + def test_get_manylinux_python311(self): + z = Zappa(runtime="python3.11") + self.assertIsNotNone(z.get_cached_manylinux_wheel("psycopg2-binary", "2.9.7")) + self.assertIsNone(z.get_cached_manylinux_wheel("derp_no_such_thing", "0.0")) + + # mock with a known manylinux wheel package so that code for downloading them gets invoked + mock_installed_packages = {"psycopg2-binary": "2.9.7"} + with mock.patch( + "zappa.core.Zappa.get_installed_packages", + return_value=mock_installed_packages, + ): + z = Zappa(runtime="python3.11") + path = z.create_lambda_zip(handler_file=os.path.realpath(__file__)) + self.assertTrue(os.path.isfile(path)) + os.remove(path) + + # same, but with an ABI3 package + mock_installed_packages = {"cryptography": "2.8"} + with mock.patch( + "zappa.core.Zappa.get_installed_packages", + return_value=mock_installed_packages, + ): + z = Zappa(runtime="python3.11") + path = z.create_lambda_zip(handler_file=os.path.realpath(__file__)) + self.assertTrue(os.path.isfile(path)) + os.remove(path) + def test_getting_installed_packages(self, *args): z = Zappa(runtime="python3.7") diff --git a/zappa/__init__.py b/zappa/__init__.py index 7918f71e8..402cdaab5 100644 --- a/zappa/__init__.py +++ b/zappa/__init__.py @@ -12,7 +12,7 @@ def running_in_docker() -> bool: return running_in_docker_flag -SUPPORTED_VERSIONS = [(3, 7), (3, 8), (3, 9), (3, 10)] +SUPPORTED_VERSIONS = [(3, 7), (3, 8), (3, 9), (3, 10), (3, 11)] MINIMUM_SUPPORTED_MINOR_VERSION = 7 if not running_in_docker() and sys.version_info[:2] not in SUPPORTED_VERSIONS: diff --git a/zappa/core.py b/zappa/core.py index b01a03a9d..4a4d8b616 100644 --- a/zappa/core.py +++ b/zappa/core.py @@ -314,8 +314,10 @@ def __init__( self.manylinux_suffix_start = "cp38" elif self.runtime == "python3.9": self.manylinux_suffix_start = "cp39" - else: + elif self.runtime == "python3.10": self.manylinux_suffix_start = "cp310" + else: + self.manylinux_suffix_start = "cp311" # AWS Lambda supports manylinux1/2010, manylinux2014, and manylinux_2_24 manylinux_suffixes = ("_2_24", "2014", "2010", "1") diff --git a/zappa/utilities.py b/zappa/utilities.py index b1da8474c..7cd11d1c5 100644 --- a/zappa/utilities.py +++ b/zappa/utilities.py @@ -214,8 +214,10 @@ def get_runtime_from_python_version(): return "python3.8" elif sys.version_info[1] <= 9: return "python3.9" - else: + elif sys.version_info[1] <= 10: return "python3.10" + else: + return "python3.11" ## From 68ad9d2ebafcb0633c50a67c0212f35b8e377a3a Mon Sep 17 00:00:00 2001 From: James Lizamore Date: Wed, 16 Aug 2023 09:33:53 +0100 Subject: [PATCH 2/7] Feature/ephemeral storage (#1259) * Adds ability to set ephemeral_storage * Adds tests for ephemeral_storage settings * Clarifies test data * Adds readme description for ephemeral_storage setting * Adds ephemeral_storage overwrite to test settings --- README.md | 3 ++- test_settings.json | 15 +++++++++++++++ tests/tests.py | 18 ++++++++++++++++++ zappa/cli.py | 11 +++++++++++ zappa/core.py | 4 ++++ 5 files changed, 50 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6c801b0ec..8e9878c49 100644 --- a/README.md +++ b/README.md @@ -974,6 +974,7 @@ to change Zappa's behavior. Use these at your own risk! "log_level": "DEBUG", // Set the Zappa log level. Can be one of CRITICAL, ERROR, WARNING, INFO and DEBUG. Default: DEBUG "manage_roles": true, // Have Zappa automatically create and define IAM execution roles and policies. Default true. If false, you must define your own IAM Role and role_name setting. "memory_size": 512, // Lambda function memory in MB. Default 512. + "ephemeral_storage": { "Size": 512 }, // Lambda function ephemeral_storage size in MB, Default 512, Max 10240 "num_retained_versions":null, // Indicates the number of old versions to retain for the lambda. If absent, keeps all the versions of the function. "payload_compression": true, // Whether or not to enable API gateway payload compression (default: true) "payload_minimum_compression_size": 0, // The threshold size (in bytes) below which payload compression will not be applied (default: 0) @@ -1060,7 +1061,7 @@ You can also simply handle CORS directly in your application. Your web framework ### Large Projects -AWS currently limits Lambda zip sizes to 50 megabytes. If your project is larger than that, set `slim_handler: true` in your `zappa_settings.json`. In this case, your fat application package will be replaced with a small handler-only package. The handler file then pulls the rest of the large project down from S3 at run time! The initial load of the large project may add to startup overhead, but the difference should be minimal on a warm lambda function. Note that this will also eat into the storage space of your application function. Note that AWS currently [limits](https://docs.aws.amazon.com/lambda/latest/dg/limits.html) the `/tmp` directory storage to 512 MB, so your project must still be smaller than that. +AWS currently limits Lambda zip sizes to 50 megabytes. If your project is larger than that, set `slim_handler: true` in your `zappa_settings.json`. In this case, your fat application package will be replaced with a small handler-only package. The handler file then pulls the rest of the large project down from S3 at run time! The initial load of the large project may add to startup overhead, but the difference should be minimal on a warm lambda function. Note that this will also eat into the storage space of your application function. Note that AWS [supports](https://aws.amazon.com/blogs/compute/using-larger-ephemeral-storage-for-aws-lambda/) custom `/tmp` directory storage size in a range of 512 - 10240 MB. Use `ephemeral_storage` in `zappa_settings.json` to adjust to your needs if your project is larger than default 512 MB. ### Enabling Bash Completion diff --git a/test_settings.json b/test_settings.json index 2ee126a1f..178b98d8b 100644 --- a/test_settings.json +++ b/test_settings.json @@ -10,6 +10,9 @@ "zip": "test_settings.callback" }, "delete_local_zip": true, + "ephemeral_storage": { + "Size": 1024 + }, "debug": true, "parameter_depth": 2, "prebuild_script": "tests.test_app.prebuild_me", @@ -107,6 +110,18 @@ "EXTENDO": "You bet" } }, + "invalid_ephemeral_storage_out_of_range": { + "extends": "ttt888", + "ephemeral_storage": { + "Size": 99999 + } + }, + "invalid_ephemeral_storage_missing_key": { + "extends": "ttt888", + "ephemeral_storage": { + "BadKey": 1024 + } + }, "build_package_only_delete_local_zip_false": { "delete_local_zip": false, "use_precompiled_packages": false, diff --git a/tests/tests.py b/tests/tests.py index aaea595e8..a08d30697 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1159,6 +1159,24 @@ def test_load_settings(self): zappa_cli.load_settings("test_settings.json") self.assertEqual(False, zappa_cli.stage_config["touch"]) + def test_load_settings_ephemeral_storage_overwrite(self): + zappa_cli = ZappaCLI() + zappa_cli.api_stage = "ttt888" + zappa_cli.load_settings("test_settings.json") + self.assertEqual(zappa_cli.stage_config["ephemeral_storage"]["Size"], 1024) + + def test_load_settings_ephemeral_storage_out_of_range(self): + zappa_cli = ZappaCLI() + zappa_cli.api_stage = "invalid_ephemeral_storage_out_of_range" + with self.assertRaises(ClickException) as err: + zappa_cli.load_settings("test_settings.json") + + def test_load_settings_ephemeral_storage_missing_key(self): + zappa_cli = ZappaCLI() + zappa_cli.api_stage = "invalid_ephemeral_storage_missing_key" + with self.assertRaises(ClickException) as err: + zappa_cli.load_settings("test_settings.json") + def test_load_extended_settings(self): zappa_cli = ZappaCLI() zappa_cli.api_stage = "extendo" diff --git a/zappa/cli.py b/zappa/cli.py index e78707869..a4997d9ea 100755 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -108,6 +108,7 @@ class ZappaCLI: handler_path = None vpc_config = None memory_size = None + ephemeral_storage = None use_apigateway = None lambda_handler = None django_settings = None @@ -810,6 +811,7 @@ def deploy(self, source_zip=None, docker_image_uri=None): dead_letter_config=self.dead_letter_config, timeout=self.timeout_seconds, memory_size=self.memory_size, + ephemeral_storage=self.ephemeral_storage, runtime=self.runtime, aws_environment_variables=self.aws_environment_variables, aws_kms_key_arn=self.aws_kms_key_arn, @@ -1050,6 +1052,7 @@ def update(self, source_zip=None, no_upload=False, docker_image_uri=None): vpc_config=self.vpc_config, timeout=self.timeout_seconds, memory_size=self.memory_size, + ephemeral_storage=self.ephemeral_storage, runtime=self.runtime, aws_environment_variables=self.aws_environment_variables, aws_kms_key_arn=self.aws_kms_key_arn, @@ -2228,6 +2231,14 @@ def load_settings(self, settings_file=None, session=None): ) self.vpc_config = self.stage_config.get("vpc_config", {}) self.memory_size = self.stage_config.get("memory_size", 512) + self.ephemeral_storage = self.stage_config.get("ephemeral_storage", {"Size": 512}) + + # Validate ephemeral storage structure and size + if "Size" not in self.ephemeral_storage: + raise ClickException("Please provide a valid Size for ephemeral_storage in your Zappa settings.") + elif not 512 <= self.ephemeral_storage["Size"] <= 10240: + raise ClickException("Please provide a valid ephemeral_storage size between 512 - 10240 in your Zappa settings.") + self.app_function = self.stage_config.get("app_function", None) self.exception_handler = self.stage_config.get("exception_handler", None) self.aws_region = self.stage_config.get("aws_region", None) diff --git a/zappa/core.py b/zappa/core.py index 4a4d8b616..19a95821d 100644 --- a/zappa/core.py +++ b/zappa/core.py @@ -1109,6 +1109,7 @@ def create_lambda_function( description="Zappa Deployment", timeout=30, memory_size=512, + ephemeral_storage={"Size": 512}, publish=True, vpc_config=None, dead_letter_config=None, @@ -1145,6 +1146,7 @@ def create_lambda_function( Description=description, Timeout=timeout, MemorySize=memory_size, + EphemeralStorage=ephemeral_storage, Publish=publish, VpcConfig=vpc_config, DeadLetterConfig=dead_letter_config, @@ -1293,6 +1295,7 @@ def update_lambda_configuration( description="Zappa Deployment", timeout=30, memory_size=512, + ephemeral_storage={"Size": 512}, publish=True, vpc_config=None, runtime="python3.7", @@ -1337,6 +1340,7 @@ def update_lambda_configuration( "Description": description, "Timeout": timeout, "MemorySize": memory_size, + "EphemeralStorage": ephemeral_storage, "VpcConfig": vpc_config, "Environment": {"Variables": aws_environment_variables}, "KMSKeyArn": aws_kms_key_arn, From 26d6182129ecd737cbef95ea8d462132a8fb3c58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20H=C3=B8rl=C3=BCck=20Berg?= Date: Thu, 17 Aug 2023 07:09:04 +0200 Subject: [PATCH 3/7] Update permissions in deploy.json (#1119) Add required permissions to deploy.json Co-authored-by: monkut --- example/policy/deploy.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/example/policy/deploy.json b/example/policy/deploy.json index 39f0c1cfc..2130d20be 100644 --- a/example/policy/deploy.json +++ b/example/policy/deploy.json @@ -31,6 +31,8 @@ "lambda:AddPermission", "lambda:CreateFunction", "lambda:DeleteFunction", + "lambda:DeleteFunctionConcurrency", + "lambda:GetAlias", "lambda:GetFunction", "lambda:GetFunctionConfiguration", "lambda:GetPolicy", From f2f03ba8b58c9e2bdbbff7b5e8c905e3a8c08021 Mon Sep 17 00:00:00 2001 From: Sridhar <59284206+sridhar562345@users.noreply.github.com> Date: Sun, 20 Aug 2023 12:41:14 +0530 Subject: [PATCH 4/7] update manylinux wheels download logic (#1250) * update manylinux wheels download logic * add tests for updated manylinux logic * Remove incorrect return statement from get_manylinux_wheel_url * convert the manylinux file names to lowercase in get_manylinux_wheel_url * add test to verify manylinux filenames are lowered * change print to logger output * mock requests.get in test_verify_manylinux_filename_is_lowered * apply black * fix mock.patch reference * :wrench: add `ignore_cache` option for ease of testing. :white_check_mark: update testcase to use `ignore_cache` * :art: fix flake8 --------- Co-authored-by: monkut Co-authored-by: shane --- tests/tests.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++ zappa/core.py | 58 ++++++++++++++++++++++++++++++----------------- 2 files changed, 98 insertions(+), 21 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index a08d30697..0b415d18a 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -197,6 +197,67 @@ def test_get_manylinux_python310(self): self.assertTrue(os.path.isfile(path)) os.remove(path) + def test_verify_python37_does_not_download_2_24_manylinux_wheel(self): + z = Zappa(runtime="python3.7") + cached_wheels_dir = os.path.join(tempfile.gettempdir(), "cached_wheels") + expected_wheel_path = os.path.join( + cached_wheels_dir, "cryptography-35.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl" + ) + + # Check with known manylinux wheel package + actual_wheel_path = z.get_cached_manylinux_wheel("cryptography", "35.0.0") + self.assertEqual(actual_wheel_path, expected_wheel_path) + os.remove(actual_wheel_path) + + def test_verify_downloaded_manylinux_wheel(self): + z = Zappa(runtime="python3.10") + cached_wheels_dir = os.path.join(tempfile.gettempdir(), "cached_wheels") + expected_wheel_path = os.path.join( + cached_wheels_dir, + "pycryptodome-3.16.0-cp35-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", + ) + + # check with a known manylinux wheel package + actual_wheel_path = z.get_cached_manylinux_wheel("pycryptodome", "3.16.0") + self.assertEqual(actual_wheel_path, expected_wheel_path) + os.remove(actual_wheel_path) + + def test_verify_manylinux_filename_is_lowered(self): + z = Zappa(runtime="python3.10") + expected_filename = "markupsafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + + mock_package_data = { + "releases": { + "2.1.3": [ + { + "url": "https://files.pythonhosted.org/packages/a6/56/f1d4ee39e898a9e63470cbb7fae1c58cce6874f25f54220b89213a47f273/MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "filename": "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + }, + { + "url": "https://files.pythonhosted.org/packages/12/b3/d9ed2c0971e1435b8a62354b18d3060b66c8cb1d368399ec0b9baa7c0ee5/MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "filename": "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + }, + { + "url": "https://files.pythonhosted.org/packages/bf/b7/c5ba9b7ad9ad21fc4a60df226615cf43ead185d328b77b0327d603d00cc5/MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + "filename": "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + }, + ] + } + } + + with mock.patch("zappa.core.requests.get") as mock_get: + mock_get.return_value.json.return_value = mock_package_data + wheel_url, file_name = z.get_manylinux_wheel_url("markupsafe", "2.1.3", ignore_cache=True) + + self.assertEqual(file_name, expected_filename) + mock_get.assert_called_once_with( + "https://pypi.python.org/pypi/markupsafe/json", timeout=float(os.environ.get("PIP_TIMEOUT", 1.5)) + ) + + # Clean the generated files + cached_pypi_info_dir = os.path.join(tempfile.gettempdir(), "cached_pypi_info") + os.remove(os.path.join(cached_pypi_info_dir, "markupsafe-2.1.3.json")) + def test_get_manylinux_python311(self): z = Zappa(runtime="python3.11") self.assertIsNotNone(z.get_cached_manylinux_wheel("psycopg2-binary", "2.9.7")) diff --git a/zappa/core.py b/zappa/core.py index 19a95821d..ae80b77e1 100644 --- a/zappa/core.py +++ b/zappa/core.py @@ -24,6 +24,7 @@ from builtins import bytes, int from distutils.dir_util import copy_tree from io import open +from pathlib import Path from typing import Optional import boto3 @@ -320,13 +321,17 @@ def __init__( self.manylinux_suffix_start = "cp311" # AWS Lambda supports manylinux1/2010, manylinux2014, and manylinux_2_24 - manylinux_suffixes = ("_2_24", "2014", "2010", "1") + # Currently python3.7 lambda runtime does not support manylinux_2_24 + # See https://github.com/zappa/Zappa/issues/1249 for more details + if self.runtime == "python3.7": + self.manylinux_suffixes = ("2014", "2010", "1") + else: + self.manylinux_suffixes = ("_2_24", "2014", "2010", "1") + self.manylinux_wheel_file_match = re.compile( - rf'^.*{self.manylinux_suffix_start}-(manylinux_\d+_\d+_x86_64[.])?manylinux({"|".join(manylinux_suffixes)})_x86_64[.]whl$' # noqa: E501 - ) - self.manylinux_wheel_abi3_file_match = re.compile( - rf'^.*cp3.-abi3-manylinux({"|".join(manylinux_suffixes)})_x86_64.whl$' + rf'^.*{self.manylinux_suffix_start}-(manylinux_\d+_\d+_x86_64[.])?manylinux({"|".join(self.manylinux_suffixes)})_x86_64[.]whl$' # noqa: E501 ) + self.manylinux_wheel_abi3_file_match = re.compile(rf"^.*cp3.-abi3-manylinux.*_x86_64[.]whl$") self.endpoint_urls = endpoint_urls self.xray_tracing = xray_tracing @@ -922,11 +927,14 @@ def get_cached_manylinux_wheel(self, package_name, package_version, disable_prog wheel_path = os.path.join(cached_wheels_dir, wheel_file) for pathname in glob.iglob(wheel_path): - if re.match(self.manylinux_wheel_file_match, pathname) or re.match( - self.manylinux_wheel_abi3_file_match, pathname - ): - print(f" - {package_name}=={package_version}: Using locally cached manylinux wheel") + if re.match(self.manylinux_wheel_file_match, pathname): + logger.info(f" - {package_name}=={package_version}: Using locally cached manylinux wheel") return pathname + elif re.match(self.manylinux_wheel_abi3_file_match, pathname): + for manylinux_suffix in self.manylinux_suffixes: + if f"manylinux{manylinux_suffix}_x86_64" in pathname: + logger.info(f" - {package_name}=={package_version}: Using locally cached manylinux wheel") + return pathname # The file is not cached, download it. wheel_url, filename = self.get_manylinux_wheel_url(package_name, package_version) @@ -934,7 +942,7 @@ def get_cached_manylinux_wheel(self, package_name, package_version, disable_prog return None wheel_path = os.path.join(cached_wheels_dir, filename) - print(f" - {package_name}=={package_version}: Downloading") + logger.info(f" - {package_name}=={package_version}: Downloading") with open(wheel_path, "wb") as f: self.download_url_with_progress(wheel_url, f, disable_progress) @@ -943,7 +951,7 @@ def get_cached_manylinux_wheel(self, package_name, package_version, disable_prog return wheel_path - def get_manylinux_wheel_url(self, package_name, package_version): + def get_manylinux_wheel_url(self, package_name, package_version, ignore_cache: bool = False): """ For a given package name, returns a link to the download URL, else returns None. @@ -954,27 +962,31 @@ def get_manylinux_wheel_url(self, package_name, package_version): also caches the JSON file so that we don't have to poll Pypi every time. """ - cached_pypi_info_dir = os.path.join(tempfile.gettempdir(), "cached_pypi_info") - if not os.path.isdir(cached_pypi_info_dir): + cached_pypi_info_dir = Path(tempfile.gettempdir()) / "cached_pypi_info" + if not cached_pypi_info_dir.is_dir(): os.makedirs(cached_pypi_info_dir) + # Even though the metadata is for the package, we save it in a # filename that includes the package's version. This helps in # invalidating the cached file if the user moves to a different # version of the package. # Related: https://github.com/Miserlou/Zappa/issues/899 - json_file = "{0!s}-{1!s}.json".format(package_name, package_version) - json_file_path = os.path.join(cached_pypi_info_dir, json_file) - if os.path.exists(json_file_path): - with open(json_file_path, "rb") as metafile: + data = None + json_file_name = "{0!s}-{1!s}.json".format(package_name, package_version) + json_file_path = cached_pypi_info_dir / json_file_name + if json_file_path.exists(): + with json_file_path.open("rb") as metafile: data = json.load(metafile) - else: + + if not data or ignore_cache: url = "https://pypi.python.org/pypi/{}/json".format(package_name) try: res = requests.get(url, timeout=float(os.environ.get("PIP_TIMEOUT", 1.5))) data = res.json() except Exception: # pragma: no cover return None, None - with open(json_file_path, "wb") as metafile: + + with json_file_path.open("wb") as metafile: jsondata = json.dumps(data) metafile.write(bytes(jsondata, "utf-8")) @@ -984,9 +996,13 @@ def get_manylinux_wheel_url(self, package_name, package_version): for f in data["releases"][package_version]: if re.match(self.manylinux_wheel_file_match, f["filename"]): - return f["url"], f["filename"] + # Since we have already lowered package names in get_installed_packages + # manylinux caching is not working for packages with capital case in names like MarkupSafe + return f["url"], f["filename"].lower() elif re.match(self.manylinux_wheel_abi3_file_match, f["filename"]): - return f["url"], f["filename"] + for manylinux_suffix in self.manylinux_suffixes: + if f"manylinux{manylinux_suffix}_x86_64" in f["filename"]: + return f["url"], f["filename"].lower() return None, None ## From 7fba630a7c18885e6e5925a8bf32de92dc642e65 Mon Sep 17 00:00:00 2001 From: monkut Date: Thu, 7 Sep 2023 15:14:08 +0900 Subject: [PATCH 5/7] :bug: fix response time improperly configured for micro-seconds. (#1265) * :bug: fix response time improperly configured for micro-seconds. * :bug: fix missing import --------- Co-authored-by: shane --- zappa/handler.py | 4 ++-- zappa/utilities.py | 2 +- zappa/wsgi.py | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/zappa/handler.py b/zappa/handler.py index 58a12b593..c1a7ff5e6 100644 --- a/zappa/handler.py +++ b/zappa/handler.py @@ -613,9 +613,9 @@ def handler(self, event, context): # and log it in the Common Log format. time_end = datetime.datetime.now() delta = time_end - time_start - response_time_ms = delta.total_seconds() * 1000 + response_time_us = delta.total_seconds() * 1_000_000 # convert to microseconds response.content = response.data - common_log(environ, response, response_time=response_time_ms) + common_log(environ, response, response_time=response_time_us) return zappa_returndict except Exception as e: # pragma: no cover diff --git a/zappa/utilities.py b/zappa/utilities.py index 7cd11d1c5..635fdd5af 100644 --- a/zappa/utilities.py +++ b/zappa/utilities.py @@ -704,4 +704,4 @@ def validate_json_serializable(*args: Any, **kwargs: Any) -> None: try: json.dumps((args, kwargs)) except (TypeError, OverflowError): - raise UnserializableJsonError("Arguments to an asynchronous.task must be JSON serializable!") + raise UnserializableJsonError("Arguments to asynchronous.task must be JSON serializable!") diff --git a/zappa/wsgi.py b/zappa/wsgi.py index 70333e0ac..34e33313c 100644 --- a/zappa/wsgi.py +++ b/zappa/wsgi.py @@ -2,6 +2,7 @@ import logging import sys from io import BytesIO +from typing import Optional from urllib.parse import unquote, urlencode from .utilities import ApacheNCSAFormatter, merge_headers, titlecase_keys @@ -156,11 +157,12 @@ def create_wsgi_request( return environ -def common_log(environ, response, response_time=None): +def common_log(environ, response, response_time: Optional[int] = None): """ Given the WSGI environ and the response, log this event in Common Log Format. + response_time: response time in micro-seconds """ logger = logging.getLogger() From f0b51ac78bc75f72a6a17b18ef6ee6bcd8e1782f Mon Sep 17 00:00:00 2001 From: monkut Date: Thu, 7 Sep 2023 17:05:37 +0900 Subject: [PATCH 6/7] :bug: fix unquote issue with querystring handling. (#1264) * :bug: fix unquote issue with querystring handling. * :art: fix flake8 * :pencil: Add comment regarding query string handling for clarity. --------- Co-authored-by: shane --- tests/tests.py | 32 +++++++++++++++++++++++++++++++- zappa/wsgi.py | 5 ++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 0b415d18a..daba1ada2 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -2768,7 +2768,37 @@ def test_wsgi_query_string_unquoted(self): "requestContext": {}, } request = create_wsgi_request(event) - self.assertEqual(request["QUERY_STRING"], "a=A,B&b=C#D") + expected = "a=A%2CB&b=C%23D" # unencoded result: "a=A,B&b=C#D" + self.assertEqual(request["QUERY_STRING"], expected) + + def test_wsgi_query_string_ampersand_unencoded(self): + event = { + "body": None, + "headers": {}, + "pathParameters": {}, + "path": "/path/path1", + "httpMethod": "GET", + "queryStringParameters": { + "test": "M&M", + }, + "requestContext": {}, + } + request = create_wsgi_request(event) + self.assertEqual(request["QUERY_STRING"], "test=M%26M") + + def test_wsgi_query_string_with_encodechars(self): + event = { + "body": None, + "headers": {}, + "pathParameters": {}, + "path": "/path/path1", + "httpMethod": "GET", + "queryStringParameters": {"query": "Jane&John", "otherquery": "B", "test": "hello+m.te&how&are&you"}, + "requestContext": {}, + } + request = create_wsgi_request(event) + expected = "query=Jane%26John&otherquery=B&test=hello%2Bm.te%26how%26are%26you" + self.assertEqual(request["QUERY_STRING"], expected) if __name__ == "__main__": diff --git a/zappa/wsgi.py b/zappa/wsgi.py index 34e33313c..9f3051f04 100644 --- a/zappa/wsgi.py +++ b/zappa/wsgi.py @@ -36,13 +36,16 @@ def create_wsgi_request( # we have to check for the existence of one and then fall back to the # other. + # Assumes that the lambda event provides the unencoded string as + # the value in "queryStringParameters"/"multiValueQueryStringParameters" + # The QUERY_STRING value provided to WSGI expects the query string to be properly urlencoded. + # See https://github.com/zappa/Zappa/issues/1227 for discussion of this behavior. if "multiValueQueryStringParameters" in event_info: query = event_info["multiValueQueryStringParameters"] query_string = urlencode(query, doseq=True) if query else "" else: query = event_info.get("queryStringParameters", {}) query_string = urlencode(query) if query else "" - query_string = unquote(query_string) if context_header_mappings: for key, value in context_header_mappings.items(): From 8d68b54a7265ec1acd5b9ac6bb23c61e4e8e65f2 Mon Sep 17 00:00:00 2001 From: monkut Date: Fri, 15 Sep 2023 16:14:09 +0900 Subject: [PATCH 7/7] Prepare CHANGELOG/VERSION for 0.58.0 release (#1271) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✏️ update version 0.57.0 -> 0.58.0 ✏️ update CHANGELOG.md with issues addressed in 0.58.0 * :wrench: attempt to fix "AttributeError: 'String' object has no attribute 'update'" pip/pipenv error. * :wrench: attempt to fix "AttributeError: 'String' object has no attribute 'update'" pip/pipenv error" -> Upgrade pip/pipenv * :wrench: attempt to fix "AttributeError: 'String' object has no attribute 'update'" pip/pipenv error" -> Upgrade pip in Makefile * :wrench: attempt to fix "AttributeError: 'String' object has no attribute 'update'" pip/pipenv error" -> Set pipenv<2023.8.19 * :pencil: add python 3.11 support issue to CHANGELOG * :fire: remove unnecessary update pip/pipenv section * :fire: remove `pipenv` from Pipfile (as recommended by pipenv dev team (https://github.com/pypa/pipenv/issues/5927) --------- Co-authored-by: shane --- CHANGELOG.md | 10 ++++++++++ Makefile | 4 +++- Pipfile | 1 - zappa/__init__.py | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7484cecb5..6c46376c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Zappa Changelog +## 0.58.0 + +* Add Python 3.11 support (#1262) +* support new ephemeral storage feature in zappa_settings.json (#1120) +* Update permissions (PR #1119) +* Outdated manylinux wheels download logic (#1249) +* cryptography>=35.0, plus pip>=20.3 - downloads wrong cryptography anywheel package (GLIBC_2.18 error) (#1063) +* fix response time improperly configured for micro-seconds. (#1265) +* fix unquote issue with querystring handling. (#1264) + ## 0.57.0 * Python 3.10 support (#1124, #1160) diff --git a/Makefile b/Makefile index 0d201b91e..d84eb89cc 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,9 @@ clean: coverage erase requirements: - pip install pipenv>2021.11.15 + pip install pip --upgrade + pip install "pipenv>2021.11.15" + pipenv lock pipenv sync --dev diff --git a/Pipfile b/Pipfile index e1a35d0e1..f8d05fbd9 100644 --- a/Pipfile +++ b/Pipfile @@ -14,7 +14,6 @@ Flask = "*" isort = "*" mock = "*" mypy = "*" -pipenv = ">2021.11.15" packaging = "*" pytest = "*" pytest-cov = "*" diff --git a/zappa/__init__.py b/zappa/__init__.py index 402cdaab5..ced860fba 100644 --- a/zappa/__init__.py +++ b/zappa/__init__.py @@ -30,4 +30,4 @@ def running_in_docker() -> bool: ) raise RuntimeError(err_msg) -__version__ = "0.57.0" +__version__ = "0.58.0"