diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index d1fd903f5..cc4114a92 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,7 +1,7 @@ ## Context - + ## Expected Behavior diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a46c8a28c..79e454b3d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -20,7 +20,7 @@ Before you submit this PR, please make sure that you meet these criteria: * Did you **make sure this code actually works on Lambda**, as well as locally? -* Did you test this code with all of **Python 3.7**, **Python 3.8**, **Python 3.9** and **Python 3.10** ? +* Did you test this code with all of **Python 3.7**, **Python 3.8**, **Python 3.9**, **Python 3.10** and **Python 3.11** ? * Does this commit ONLY relate to the issue at hand and have your linter shit all over the code? diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index b22f50413..b19664b48 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -15,10 +15,10 @@ jobs: steps: - name: Checkout Code Repository uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Install `pypa/build` run: python -m pip install build - name: Build sdist and wheel diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9f1177bfe..acb688536 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: [3.7, 3.8, 3.9, "3.10"] + python: [3.7, 3.8, 3.9, "3.10", "3.11"] steps: - name: Checkout Code Repository uses: actions/checkout@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7484cecb5..6c46376c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Zappa Changelog +## 0.58.0 + +* Add Python 3.11 support (#1262) +* support new ephemeral storage feature in zappa_settings.json (#1120) +* Update permissions (PR #1119) +* Outdated manylinux wheels download logic (#1249) +* cryptography>=35.0, plus pip>=20.3 - downloads wrong cryptography anywheel package (GLIBC_2.18 error) (#1063) +* fix response time improperly configured for micro-seconds. (#1265) +* fix unquote issue with querystring handling. (#1264) + ## 0.57.0 * Python 3.10 support (#1124, #1160) diff --git a/Makefile b/Makefile index 0d201b91e..d84eb89cc 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,9 @@ clean: coverage erase requirements: - pip install pipenv>2021.11.15 + pip install pip --upgrade + pip install "pipenv>2021.11.15" + pipenv lock pipenv sync --dev diff --git a/Pipfile b/Pipfile index e1a35d0e1..f8d05fbd9 100644 --- a/Pipfile +++ b/Pipfile @@ -14,7 +14,6 @@ Flask = "*" isort = "*" mock = "*" mypy = "*" -pipenv = ">2021.11.15" packaging = "*" pytest = "*" pytest-cov = "*" diff --git a/README.md b/README.md index 9af47f016..8e9878c49 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ __Awesome!__ ## Installation and Configuration -_Before you begin, make sure you are running Python 3.7/3.8/3.9/3.10 and you have a valid AWS account and your [AWS credentials file](https://blogs.aws.amazon.com/security/post/Tx3D6U6WSFGOK2H/A-New-and-Standardized-Way-to-Manage-Credentials-in-the-AWS-SDKs) is properly installed._ +_Before you begin, make sure you are running Python 3.7/3.8/3.9/3.10/3.11 and you have a valid AWS account and your [AWS credentials file](https://blogs.aws.amazon.com/security/post/Tx3D6U6WSFGOK2H/A-New-and-Standardized-Way-to-Manage-Credentials-in-the-AWS-SDKs) is properly installed._ **Zappa** can easily be installed through pip, like so: @@ -443,7 +443,7 @@ For instance, suppose you have a basic application in a file called "my_app.py", Any remote print statements made and the value the function returned will then be printed to your local console. **Nifty!** -You can also invoke interpretable Python 3.7/3.8/3.9/3.10 strings directly by using `--raw`, like so: +You can also invoke interpretable Python 3.7/3.8/3.9/3.10/3.11 strings directly by using `--raw`, like so: $ zappa invoke production "print(1 + 2 + 3)" --raw @@ -974,6 +974,7 @@ to change Zappa's behavior. Use these at your own risk! "log_level": "DEBUG", // Set the Zappa log level. Can be one of CRITICAL, ERROR, WARNING, INFO and DEBUG. Default: DEBUG "manage_roles": true, // Have Zappa automatically create and define IAM execution roles and policies. Default true. If false, you must define your own IAM Role and role_name setting. "memory_size": 512, // Lambda function memory in MB. Default 512. + "ephemeral_storage": { "Size": 512 }, // Lambda function ephemeral_storage size in MB, Default 512, Max 10240 "num_retained_versions":null, // Indicates the number of old versions to retain for the lambda. If absent, keeps all the versions of the function. "payload_compression": true, // Whether or not to enable API gateway payload compression (default: true) "payload_minimum_compression_size": 0, // The threshold size (in bytes) below which payload compression will not be applied (default: 0) @@ -984,7 +985,7 @@ to change Zappa's behavior. Use these at your own risk! "role_name": "MyLambdaRole", // Name of Zappa execution role. Default --ZappaExecutionRole. To use a different, pre-existing policy, you must also set manage_roles to false. "role_arn": "arn:aws:iam::12345:role/app-ZappaLambdaExecutionRole", // ARN of Zappa execution role. Default to None. To use a different, pre-existing policy, you must also set manage_roles to false. This overrides role_name. Use with temporary credentials via GetFederationToken. "route53_enabled": true, // Have Zappa update your Route53 Hosted Zones when certifying with a custom domain. Default true. - "runtime": "python3.10", // Python runtime to use on Lambda. Can be one of "python3.7", "python3.8", "python3.9", or "python3.10". Defaults to whatever the current Python being used is. + "runtime": "python3.11", // Python runtime to use on Lambda. Can be one of "python3.7", "python3.8", "python3.9", or "python3.10", or "python3.11". Defaults to whatever the current Python being used is. "s3_bucket": "dev-bucket", // Zappa zip bucket, "slim_handler": false, // Useful if project >50M. Set true to just upload a small handler to Lambda and load actual project from S3 at runtime. Default false. "settings_file": "~/Projects/MyApp/settings/dev_settings.py", // Server side settings file location, @@ -1060,7 +1061,7 @@ You can also simply handle CORS directly in your application. Your web framework ### Large Projects -AWS currently limits Lambda zip sizes to 50 megabytes. If your project is larger than that, set `slim_handler: true` in your `zappa_settings.json`. In this case, your fat application package will be replaced with a small handler-only package. The handler file then pulls the rest of the large project down from S3 at run time! The initial load of the large project may add to startup overhead, but the difference should be minimal on a warm lambda function. Note that this will also eat into the storage space of your application function. Note that AWS currently [limits](https://docs.aws.amazon.com/lambda/latest/dg/limits.html) the `/tmp` directory storage to 512 MB, so your project must still be smaller than that. +AWS currently limits Lambda zip sizes to 50 megabytes. If your project is larger than that, set `slim_handler: true` in your `zappa_settings.json`. In this case, your fat application package will be replaced with a small handler-only package. The handler file then pulls the rest of the large project down from S3 at run time! The initial load of the large project may add to startup overhead, but the difference should be minimal on a warm lambda function. Note that this will also eat into the storage space of your application function. Note that AWS [supports](https://aws.amazon.com/blogs/compute/using-larger-ephemeral-storage-for-aws-lambda/) custom `/tmp` directory storage size in a range of 512 - 10240 MB. Use `ephemeral_storage` in `zappa_settings.json` to adjust to your needs if your project is larger than default 512 MB. ### Enabling Bash Completion diff --git a/example/policy/deploy.json b/example/policy/deploy.json index 39f0c1cfc..2130d20be 100644 --- a/example/policy/deploy.json +++ b/example/policy/deploy.json @@ -31,6 +31,8 @@ "lambda:AddPermission", "lambda:CreateFunction", "lambda:DeleteFunction", + "lambda:DeleteFunctionConcurrency", + "lambda:GetAlias", "lambda:GetFunction", "lambda:GetFunctionConfiguration", "lambda:GetPolicy", diff --git a/setup.py b/setup.py index e68926ac7..a0eb02667 100755 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Framework :: Django", "Framework :: Django :: 1.11", "Framework :: Django :: 2.0", diff --git a/test_settings.json b/test_settings.json index 2ee126a1f..178b98d8b 100644 --- a/test_settings.json +++ b/test_settings.json @@ -10,6 +10,9 @@ "zip": "test_settings.callback" }, "delete_local_zip": true, + "ephemeral_storage": { + "Size": 1024 + }, "debug": true, "parameter_depth": 2, "prebuild_script": "tests.test_app.prebuild_me", @@ -107,6 +110,18 @@ "EXTENDO": "You bet" } }, + "invalid_ephemeral_storage_out_of_range": { + "extends": "ttt888", + "ephemeral_storage": { + "Size": 99999 + } + }, + "invalid_ephemeral_storage_missing_key": { + "extends": "ttt888", + "ephemeral_storage": { + "BadKey": 1024 + } + }, "build_package_only_delete_local_zip_false": { "delete_local_zip": false, "use_precompiled_packages": false, diff --git a/tests/tests.py b/tests/tests.py index 2efe2ca2b..408422f7c 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -198,6 +198,94 @@ def test_get_manylinux_python310(self): self.assertTrue(os.path.isfile(path)) os.remove(path) + def test_verify_python37_does_not_download_2_24_manylinux_wheel(self): + z = Zappa(runtime="python3.7") + cached_wheels_dir = os.path.join(tempfile.gettempdir(), "cached_wheels") + expected_wheel_path = os.path.join( + cached_wheels_dir, "cryptography-35.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl" + ) + + # Check with known manylinux wheel package + actual_wheel_path = z.get_cached_manylinux_wheel("cryptography", "35.0.0") + self.assertEqual(actual_wheel_path, expected_wheel_path) + os.remove(actual_wheel_path) + + def test_verify_downloaded_manylinux_wheel(self): + z = Zappa(runtime="python3.10") + cached_wheels_dir = os.path.join(tempfile.gettempdir(), "cached_wheels") + expected_wheel_path = os.path.join( + cached_wheels_dir, + "pycryptodome-3.16.0-cp35-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", + ) + + # check with a known manylinux wheel package + actual_wheel_path = z.get_cached_manylinux_wheel("pycryptodome", "3.16.0") + self.assertEqual(actual_wheel_path, expected_wheel_path) + os.remove(actual_wheel_path) + + def test_verify_manylinux_filename_is_lowered(self): + z = Zappa(runtime="python3.10") + expected_filename = "markupsafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + + mock_package_data = { + "releases": { + "2.1.3": [ + { + "url": "https://files.pythonhosted.org/packages/a6/56/f1d4ee39e898a9e63470cbb7fae1c58cce6874f25f54220b89213a47f273/MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "filename": "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + }, + { + "url": "https://files.pythonhosted.org/packages/12/b3/d9ed2c0971e1435b8a62354b18d3060b66c8cb1d368399ec0b9baa7c0ee5/MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "filename": "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + }, + { + "url": "https://files.pythonhosted.org/packages/bf/b7/c5ba9b7ad9ad21fc4a60df226615cf43ead185d328b77b0327d603d00cc5/MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + "filename": "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + }, + ] + } + } + + with mock.patch("zappa.core.requests.get") as mock_get: + mock_get.return_value.json.return_value = mock_package_data + wheel_url, file_name = z.get_manylinux_wheel_url("markupsafe", "2.1.3", ignore_cache=True) + + self.assertEqual(file_name, expected_filename) + mock_get.assert_called_once_with( + "https://pypi.python.org/pypi/markupsafe/json", timeout=float(os.environ.get("PIP_TIMEOUT", 1.5)) + ) + + # Clean the generated files + cached_pypi_info_dir = os.path.join(tempfile.gettempdir(), "cached_pypi_info") + os.remove(os.path.join(cached_pypi_info_dir, "markupsafe-2.1.3.json")) + + def test_get_manylinux_python311(self): + z = Zappa(runtime="python3.11") + self.assertIsNotNone(z.get_cached_manylinux_wheel("psycopg2-binary", "2.9.7")) + self.assertIsNone(z.get_cached_manylinux_wheel("derp_no_such_thing", "0.0")) + + # mock with a known manylinux wheel package so that code for downloading them gets invoked + mock_installed_packages = {"psycopg2-binary": "2.9.7"} + with mock.patch( + "zappa.core.Zappa.get_installed_packages", + return_value=mock_installed_packages, + ): + z = Zappa(runtime="python3.11") + path = z.create_lambda_zip(handler_file=os.path.realpath(__file__)) + self.assertTrue(os.path.isfile(path)) + os.remove(path) + + # same, but with an ABI3 package + mock_installed_packages = {"cryptography": "2.8"} + with mock.patch( + "zappa.core.Zappa.get_installed_packages", + return_value=mock_installed_packages, + ): + z = Zappa(runtime="python3.11") + path = z.create_lambda_zip(handler_file=os.path.realpath(__file__)) + self.assertTrue(os.path.isfile(path)) + os.remove(path) + def test_getting_installed_packages(self, *args): z = Zappa(runtime="python3.7") @@ -1154,6 +1242,24 @@ def test_load_settings(self): zappa_cli.load_settings("test_settings.json") self.assertEqual(False, zappa_cli.stage_config["touch"]) + def test_load_settings_ephemeral_storage_overwrite(self): + zappa_cli = ZappaCLI() + zappa_cli.api_stage = "ttt888" + zappa_cli.load_settings("test_settings.json") + self.assertEqual(zappa_cli.stage_config["ephemeral_storage"]["Size"], 1024) + + def test_load_settings_ephemeral_storage_out_of_range(self): + zappa_cli = ZappaCLI() + zappa_cli.api_stage = "invalid_ephemeral_storage_out_of_range" + with self.assertRaises(ClickException) as err: + zappa_cli.load_settings("test_settings.json") + + def test_load_settings_ephemeral_storage_missing_key(self): + zappa_cli = ZappaCLI() + zappa_cli.api_stage = "invalid_ephemeral_storage_missing_key" + with self.assertRaises(ClickException) as err: + zappa_cli.load_settings("test_settings.json") + def test_load_extended_settings(self): zappa_cli = ZappaCLI() zappa_cli.api_stage = "extendo" @@ -2684,7 +2790,37 @@ def test_wsgi_query_string_unquoted(self): "requestContext": {}, } request = create_wsgi_request(event) - self.assertEqual(request["QUERY_STRING"], "a=A,B&b=C#D") + expected = "a=A%2CB&b=C%23D" # unencoded result: "a=A,B&b=C#D" + self.assertEqual(request["QUERY_STRING"], expected) + + def test_wsgi_query_string_ampersand_unencoded(self): + event = { + "body": None, + "headers": {}, + "pathParameters": {}, + "path": "/path/path1", + "httpMethod": "GET", + "queryStringParameters": { + "test": "M&M", + }, + "requestContext": {}, + } + request = create_wsgi_request(event) + self.assertEqual(request["QUERY_STRING"], "test=M%26M") + + def test_wsgi_query_string_with_encodechars(self): + event = { + "body": None, + "headers": {}, + "pathParameters": {}, + "path": "/path/path1", + "httpMethod": "GET", + "queryStringParameters": {"query": "Jane&John", "otherquery": "B", "test": "hello+m.te&how&are&you"}, + "requestContext": {}, + } + request = create_wsgi_request(event) + expected = "query=Jane%26John&otherquery=B&test=hello%2Bm.te%26how%26are%26you" + self.assertEqual(request["QUERY_STRING"], expected) if __name__ == "__main__": diff --git a/zappa/__init__.py b/zappa/__init__.py index 7918f71e8..ced860fba 100644 --- a/zappa/__init__.py +++ b/zappa/__init__.py @@ -12,7 +12,7 @@ def running_in_docker() -> bool: return running_in_docker_flag -SUPPORTED_VERSIONS = [(3, 7), (3, 8), (3, 9), (3, 10)] +SUPPORTED_VERSIONS = [(3, 7), (3, 8), (3, 9), (3, 10), (3, 11)] MINIMUM_SUPPORTED_MINOR_VERSION = 7 if not running_in_docker() and sys.version_info[:2] not in SUPPORTED_VERSIONS: @@ -30,4 +30,4 @@ def running_in_docker() -> bool: ) raise RuntimeError(err_msg) -__version__ = "0.57.0" +__version__ = "0.58.0" diff --git a/zappa/cli.py b/zappa/cli.py index fa4394754..3fd7354ad 100755 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -108,6 +108,7 @@ class ZappaCLI: handler_path = None vpc_config = None memory_size = None + ephemeral_storage = None use_apigateway = None lambda_handler = None django_settings = None @@ -810,6 +811,7 @@ def deploy(self, source_zip=None, docker_image_uri=None): dead_letter_config=self.dead_letter_config, timeout=self.timeout_seconds, memory_size=self.memory_size, + ephemeral_storage=self.ephemeral_storage, runtime=self.runtime, aws_environment_variables=self.aws_environment_variables, aws_kms_key_arn=self.aws_kms_key_arn, @@ -1050,6 +1052,7 @@ def update(self, source_zip=None, no_upload=False, docker_image_uri=None): vpc_config=self.vpc_config, timeout=self.timeout_seconds, memory_size=self.memory_size, + ephemeral_storage=self.ephemeral_storage, runtime=self.runtime, aws_environment_variables=self.aws_environment_variables, aws_kms_key_arn=self.aws_kms_key_arn, @@ -2229,6 +2232,14 @@ def load_settings(self, settings_file=None, session=None): ) self.vpc_config = self.stage_config.get("vpc_config", {}) self.memory_size = self.stage_config.get("memory_size", 512) + self.ephemeral_storage = self.stage_config.get("ephemeral_storage", {"Size": 512}) + + # Validate ephemeral storage structure and size + if "Size" not in self.ephemeral_storage: + raise ClickException("Please provide a valid Size for ephemeral_storage in your Zappa settings.") + elif not 512 <= self.ephemeral_storage["Size"] <= 10240: + raise ClickException("Please provide a valid ephemeral_storage size between 512 - 10240 in your Zappa settings.") + self.app_function = self.stage_config.get("app_function", None) self.exception_handler = self.stage_config.get("exception_handler", None) self.aws_region = self.stage_config.get("aws_region", None) diff --git a/zappa/core.py b/zappa/core.py index b01a03a9d..ae80b77e1 100644 --- a/zappa/core.py +++ b/zappa/core.py @@ -24,6 +24,7 @@ from builtins import bytes, int from distutils.dir_util import copy_tree from io import open +from pathlib import Path from typing import Optional import boto3 @@ -314,17 +315,23 @@ def __init__( self.manylinux_suffix_start = "cp38" elif self.runtime == "python3.9": self.manylinux_suffix_start = "cp39" - else: + elif self.runtime == "python3.10": self.manylinux_suffix_start = "cp310" + else: + self.manylinux_suffix_start = "cp311" # AWS Lambda supports manylinux1/2010, manylinux2014, and manylinux_2_24 - manylinux_suffixes = ("_2_24", "2014", "2010", "1") + # Currently python3.7 lambda runtime does not support manylinux_2_24 + # See https://github.com/zappa/Zappa/issues/1249 for more details + if self.runtime == "python3.7": + self.manylinux_suffixes = ("2014", "2010", "1") + else: + self.manylinux_suffixes = ("_2_24", "2014", "2010", "1") + self.manylinux_wheel_file_match = re.compile( - rf'^.*{self.manylinux_suffix_start}-(manylinux_\d+_\d+_x86_64[.])?manylinux({"|".join(manylinux_suffixes)})_x86_64[.]whl$' # noqa: E501 - ) - self.manylinux_wheel_abi3_file_match = re.compile( - rf'^.*cp3.-abi3-manylinux({"|".join(manylinux_suffixes)})_x86_64.whl$' + rf'^.*{self.manylinux_suffix_start}-(manylinux_\d+_\d+_x86_64[.])?manylinux({"|".join(self.manylinux_suffixes)})_x86_64[.]whl$' # noqa: E501 ) + self.manylinux_wheel_abi3_file_match = re.compile(rf"^.*cp3.-abi3-manylinux.*_x86_64[.]whl$") self.endpoint_urls = endpoint_urls self.xray_tracing = xray_tracing @@ -920,11 +927,14 @@ def get_cached_manylinux_wheel(self, package_name, package_version, disable_prog wheel_path = os.path.join(cached_wheels_dir, wheel_file) for pathname in glob.iglob(wheel_path): - if re.match(self.manylinux_wheel_file_match, pathname) or re.match( - self.manylinux_wheel_abi3_file_match, pathname - ): - print(f" - {package_name}=={package_version}: Using locally cached manylinux wheel") + if re.match(self.manylinux_wheel_file_match, pathname): + logger.info(f" - {package_name}=={package_version}: Using locally cached manylinux wheel") return pathname + elif re.match(self.manylinux_wheel_abi3_file_match, pathname): + for manylinux_suffix in self.manylinux_suffixes: + if f"manylinux{manylinux_suffix}_x86_64" in pathname: + logger.info(f" - {package_name}=={package_version}: Using locally cached manylinux wheel") + return pathname # The file is not cached, download it. wheel_url, filename = self.get_manylinux_wheel_url(package_name, package_version) @@ -932,7 +942,7 @@ def get_cached_manylinux_wheel(self, package_name, package_version, disable_prog return None wheel_path = os.path.join(cached_wheels_dir, filename) - print(f" - {package_name}=={package_version}: Downloading") + logger.info(f" - {package_name}=={package_version}: Downloading") with open(wheel_path, "wb") as f: self.download_url_with_progress(wheel_url, f, disable_progress) @@ -941,7 +951,7 @@ def get_cached_manylinux_wheel(self, package_name, package_version, disable_prog return wheel_path - def get_manylinux_wheel_url(self, package_name, package_version): + def get_manylinux_wheel_url(self, package_name, package_version, ignore_cache: bool = False): """ For a given package name, returns a link to the download URL, else returns None. @@ -952,27 +962,31 @@ def get_manylinux_wheel_url(self, package_name, package_version): also caches the JSON file so that we don't have to poll Pypi every time. """ - cached_pypi_info_dir = os.path.join(tempfile.gettempdir(), "cached_pypi_info") - if not os.path.isdir(cached_pypi_info_dir): + cached_pypi_info_dir = Path(tempfile.gettempdir()) / "cached_pypi_info" + if not cached_pypi_info_dir.is_dir(): os.makedirs(cached_pypi_info_dir) + # Even though the metadata is for the package, we save it in a # filename that includes the package's version. This helps in # invalidating the cached file if the user moves to a different # version of the package. # Related: https://github.com/Miserlou/Zappa/issues/899 - json_file = "{0!s}-{1!s}.json".format(package_name, package_version) - json_file_path = os.path.join(cached_pypi_info_dir, json_file) - if os.path.exists(json_file_path): - with open(json_file_path, "rb") as metafile: + data = None + json_file_name = "{0!s}-{1!s}.json".format(package_name, package_version) + json_file_path = cached_pypi_info_dir / json_file_name + if json_file_path.exists(): + with json_file_path.open("rb") as metafile: data = json.load(metafile) - else: + + if not data or ignore_cache: url = "https://pypi.python.org/pypi/{}/json".format(package_name) try: res = requests.get(url, timeout=float(os.environ.get("PIP_TIMEOUT", 1.5))) data = res.json() except Exception: # pragma: no cover return None, None - with open(json_file_path, "wb") as metafile: + + with json_file_path.open("wb") as metafile: jsondata = json.dumps(data) metafile.write(bytes(jsondata, "utf-8")) @@ -982,9 +996,13 @@ def get_manylinux_wheel_url(self, package_name, package_version): for f in data["releases"][package_version]: if re.match(self.manylinux_wheel_file_match, f["filename"]): - return f["url"], f["filename"] + # Since we have already lowered package names in get_installed_packages + # manylinux caching is not working for packages with capital case in names like MarkupSafe + return f["url"], f["filename"].lower() elif re.match(self.manylinux_wheel_abi3_file_match, f["filename"]): - return f["url"], f["filename"] + for manylinux_suffix in self.manylinux_suffixes: + if f"manylinux{manylinux_suffix}_x86_64" in f["filename"]: + return f["url"], f["filename"].lower() return None, None ## @@ -1107,6 +1125,7 @@ def create_lambda_function( description="Zappa Deployment", timeout=30, memory_size=512, + ephemeral_storage={"Size": 512}, publish=True, vpc_config=None, dead_letter_config=None, @@ -1143,6 +1162,7 @@ def create_lambda_function( Description=description, Timeout=timeout, MemorySize=memory_size, + EphemeralStorage=ephemeral_storage, Publish=publish, VpcConfig=vpc_config, DeadLetterConfig=dead_letter_config, @@ -1291,6 +1311,7 @@ def update_lambda_configuration( description="Zappa Deployment", timeout=30, memory_size=512, + ephemeral_storage={"Size": 512}, publish=True, vpc_config=None, runtime="python3.7", @@ -1335,6 +1356,7 @@ def update_lambda_configuration( "Description": description, "Timeout": timeout, "MemorySize": memory_size, + "EphemeralStorage": ephemeral_storage, "VpcConfig": vpc_config, "Environment": {"Variables": aws_environment_variables}, "KMSKeyArn": aws_kms_key_arn, diff --git a/zappa/handler.py b/zappa/handler.py index 58a12b593..c1a7ff5e6 100644 --- a/zappa/handler.py +++ b/zappa/handler.py @@ -613,9 +613,9 @@ def handler(self, event, context): # and log it in the Common Log format. time_end = datetime.datetime.now() delta = time_end - time_start - response_time_ms = delta.total_seconds() * 1000 + response_time_us = delta.total_seconds() * 1_000_000 # convert to microseconds response.content = response.data - common_log(environ, response, response_time=response_time_ms) + common_log(environ, response, response_time=response_time_us) return zappa_returndict except Exception as e: # pragma: no cover diff --git a/zappa/utilities.py b/zappa/utilities.py index b1da8474c..635fdd5af 100644 --- a/zappa/utilities.py +++ b/zappa/utilities.py @@ -214,8 +214,10 @@ def get_runtime_from_python_version(): return "python3.8" elif sys.version_info[1] <= 9: return "python3.9" - else: + elif sys.version_info[1] <= 10: return "python3.10" + else: + return "python3.11" ## @@ -702,4 +704,4 @@ def validate_json_serializable(*args: Any, **kwargs: Any) -> None: try: json.dumps((args, kwargs)) except (TypeError, OverflowError): - raise UnserializableJsonError("Arguments to an asynchronous.task must be JSON serializable!") + raise UnserializableJsonError("Arguments to asynchronous.task must be JSON serializable!") diff --git a/zappa/wsgi.py b/zappa/wsgi.py index 70333e0ac..9f3051f04 100644 --- a/zappa/wsgi.py +++ b/zappa/wsgi.py @@ -2,6 +2,7 @@ import logging import sys from io import BytesIO +from typing import Optional from urllib.parse import unquote, urlencode from .utilities import ApacheNCSAFormatter, merge_headers, titlecase_keys @@ -35,13 +36,16 @@ def create_wsgi_request( # we have to check for the existence of one and then fall back to the # other. + # Assumes that the lambda event provides the unencoded string as + # the value in "queryStringParameters"/"multiValueQueryStringParameters" + # The QUERY_STRING value provided to WSGI expects the query string to be properly urlencoded. + # See https://github.com/zappa/Zappa/issues/1227 for discussion of this behavior. if "multiValueQueryStringParameters" in event_info: query = event_info["multiValueQueryStringParameters"] query_string = urlencode(query, doseq=True) if query else "" else: query = event_info.get("queryStringParameters", {}) query_string = urlencode(query) if query else "" - query_string = unquote(query_string) if context_header_mappings: for key, value in context_header_mappings.items(): @@ -156,11 +160,12 @@ def create_wsgi_request( return environ -def common_log(environ, response, response_time=None): +def common_log(environ, response, response_time: Optional[int] = None): """ Given the WSGI environ and the response, log this event in Common Log Format. + response_time: response time in micro-seconds """ logger = logging.getLogger()