From 33de0bb558d69a545875fd40833c9dfaa5f8d648 Mon Sep 17 00:00:00 2001 From: Huon Wilson Date: Sun, 28 May 2023 21:07:43 +1000 Subject: [PATCH] Implement python_aws_lambda_target, and include_sources --- .../pants/backend/awslambda/python/rules.py | 65 ++++++++++++- .../backend/awslambda/python/rules_test.py | 97 ++++++++++++++++++- .../backend/awslambda/python/target_types.py | 72 +++++++++++--- 3 files changed, 215 insertions(+), 19 deletions(-) diff --git a/src/python/pants/backend/awslambda/python/rules.py b/src/python/pants/backend/awslambda/python/rules.py index d16389cabb7..a363489c95f 100644 --- a/src/python/pants/backend/awslambda/python/rules.py +++ b/src/python/pants/backend/awslambda/python/rules.py @@ -10,6 +10,9 @@ PythonAWSLambda, PythonAwsLambdaHandlerField, PythonAwsLambdaIncludeRequirements, + PythonAwsLambdaIncludeSources, + PythonAWSLambdaLayer, + PythonAwsLambdaLayerDependenciesField, PythonAwsLambdaRuntime, ) from pants.backend.python.subsystems.lambdex import Lambdex, LambdexLayout @@ -22,8 +25,10 @@ from pants.core.goals.package import BuiltPackage, OutputPathField, PackageFieldSet from pants.core.util_rules.environments import EnvironmentField from pants.engine.rules import Get, collect_rules, rule +from pants.engine.target import InvalidTargetException from pants.engine.unions import UnionRule from pants.util.logging import LogLevel +from pants.util.strutil import softwrap logger = logging.getLogger(__name__) @@ -34,6 +39,20 @@ class PythonAwsLambdaFieldSet(PackageFieldSet): handler: PythonAwsLambdaHandlerField include_requirements: PythonAwsLambdaIncludeRequirements + include_sources: PythonAwsLambdaIncludeSources + runtime: PythonAwsLambdaRuntime + complete_platforms: PythonFaaSCompletePlatforms + output_path: OutputPathField + environment: EnvironmentField + + +@dataclass(frozen=True) +class PythonAwsLambdaLayerFieldSet(PackageFieldSet): + required_fields = (PythonAwsLambdaLayerDependenciesField,) + + dependencies: PythonAwsLambdaLayerDependenciesField + include_requirements: PythonAwsLambdaIncludeRequirements + include_sources: PythonAwsLambdaIncludeSources runtime: PythonAwsLambdaRuntime complete_platforms: PythonFaaSCompletePlatforms output_path: OutputPathField @@ -74,12 +93,56 @@ async def package_python_awslambda( handler=field_set.handler, output_path=field_set.output_path, include_requirements=field_set.include_requirements.value, - include_sources=True, + include_sources=field_set.include_sources.value, reexported_handler_module=PythonAwsLambdaHandlerField.reexported_handler_module, ), ) +@rule(desc="Create Python AWS Lambda Layer", level=LogLevel.DEBUG) +async def package_python_aws_lambda_layer( + field_set: PythonAwsLambdaLayerFieldSet, + lambdex: Lambdex, +) -> BuiltPackage: + if lambdex.layout is LambdexLayout.LAMBDEX: + raise InvalidTargetException( + softwrap( + f""" + the `{PythonAWSLambdaLayer.alias}` target {field_set.address} cannot be used with + the old Lambdex layout (`[lambdex].layout = \"{LambdexLayout.LAMBDEX.value}\"` in + `pants.toml`), set that to `{LambdexLayout.ZIP.value}` or remove this target + """ + ) + ) + + return await Get( + BuiltPackage, + BuildPythonFaaSRequest( + address=field_set.address, + target_name=PythonAWSLambdaLayer.alias, + complete_platforms=field_set.complete_platforms, + runtime=field_set.runtime, + output_path=field_set.output_path, + include_requirements=field_set.include_requirements.value, + include_sources=field_set.include_sources.value, + # See + # https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path + # + # Runtime | Path + # ... + # Python | `python` + # | `python/lib/python3.10/site-packages` + # ... + # + # The one independent on the runtime-version is more convenient: + prefix_in_artifact="python", + # a layer doesn't have a handler, just pulls in things via `dependencies` + handler=None, + reexported_handler_module=None, + ), + ) + + def rules(): return [ *collect_rules(), diff --git a/src/python/pants/backend/awslambda/python/rules_test.py b/src/python/pants/backend/awslambda/python/rules_test.py index 8de3a9f88e6..4a8a87a3135 100644 --- a/src/python/pants/backend/awslambda/python/rules_test.py +++ b/src/python/pants/backend/awslambda/python/rules_test.py @@ -12,9 +12,12 @@ import pytest -from pants.backend.awslambda.python.rules import PythonAwsLambdaFieldSet +from pants.backend.awslambda.python.rules import ( + PythonAwsLambdaFieldSet, + PythonAwsLambdaLayerFieldSet, +) from pants.backend.awslambda.python.rules import rules as awslambda_python_rules -from pants.backend.awslambda.python.target_types import PythonAWSLambda +from pants.backend.awslambda.python.target_types import PythonAWSLambda, PythonAWSLambdaLayer from pants.backend.awslambda.python.target_types import rules as target_rules from pants.backend.python.goals import package_pex_binary from pants.backend.python.goals.package_pex_binary import PexBinaryFieldSet @@ -37,6 +40,8 @@ from pants.core.target_types import rules as core_target_types_rules from pants.engine.addresses import Address from pants.engine.fs import DigestContents +from pants.engine.internals.scheduler import ExecutionError +from pants.engine.target import FieldSet from pants.testutil.python_interpreter_selection import all_major_minor_python_versions from pants.testutil.python_rule_runner import PythonRuleRunner from pants.testutil.rule_runner import QueryRule @@ -54,12 +59,14 @@ def rule_runner() -> PythonRuleRunner: *target_rules(), *package.rules(), QueryRule(BuiltPackage, (PythonAwsLambdaFieldSet,)), + QueryRule(BuiltPackage, (PythonAwsLambdaLayerFieldSet,)), ], target_types=[ FileTarget, FilesGeneratorTarget, PexBinary, PythonAWSLambda, + PythonAWSLambdaLayer, PythonRequirementTarget, PythonRequirementTarget, PythonSourcesGeneratorTarget, @@ -77,13 +84,18 @@ def create_python_awslambda( *, expected_extra_log_lines: tuple[str, ...], extra_args: list[str] | None = None, + layer: bool = False, ) -> tuple[str, bytes]: rule_runner.set_options( ["--source-root-patterns=src/python", *(extra_args or ())], env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) target = rule_runner.get_target(addr) - built_asset = rule_runner.request(BuiltPackage, [PythonAwsLambdaFieldSet.create(target)]) + if layer: + field_set: type[FieldSet] = PythonAwsLambdaLayerFieldSet + else: + field_set = PythonAwsLambdaFieldSet + built_asset = rule_runner.request(BuiltPackage, [field_set.create(target)]) assert expected_extra_log_lines == built_asset.artifacts[0].extra_log_lines digest_contents = rule_runner.request(DigestContents, [built_asset.digest]) assert len(digest_contents) == 1 @@ -328,3 +340,82 @@ def handler(event, context): assert ( zipfile.read("lambda_function.py") == b"from foo.bar.hello_world import handler as handler" ) + + +def test_create_hello_world_layer(rule_runner: PythonRuleRunner) -> None: + rule_runner.write_files( + { + "src/python/foo/bar/hello_world.py": dedent( + """ + import mureq + + def handler(event, context): + print('Hello, World!') + """ + ), + "src/python/foo/bar/BUILD": dedent( + """ + python_requirement(name="mureq", requirements=["mureq==0.2"]) + python_sources() + + python_aws_lambda_layer( + name='lambda', + dependencies=["./hello_world.py"], + runtime="python3.7", + ) + python_aws_lambda_layer( + name='slimlambda', + include_sources=False, + dependencies=["./hello_world.py"], + runtime="python3.7", + ) + """ + ), + } + ) + + zip_file_relpath, content = create_python_awslambda( + rule_runner, + Address("src/python/foo/bar", target_name="lambda"), + expected_extra_log_lines=(), + layer=True, + ) + assert "src.python.foo.bar/lambda.zip" == zip_file_relpath + + zipfile = ZipFile(BytesIO(content)) + names = set(zipfile.namelist()) + assert "python/mureq/__init__.py" in names + assert "python/foo/bar/hello_world.py" in names + # nothing that looks like a synthesized handler in any of the names + assert "lambda_function.py" not in " ".join(names) + + zip_file_relpath, content = create_python_awslambda( + rule_runner, + Address("src/python/foo/bar", target_name="slimlambda"), + expected_extra_log_lines=(), + layer=True, + ) + assert "src.python.foo.bar/slimlambda.zip" == zip_file_relpath + + zipfile = ZipFile(BytesIO(content)) + names = set(zipfile.namelist()) + assert "python/mureq/__init__.py" in names + assert "python/foo/bar/hello_world.py" not in names + # nothing that looks like a synthesized handler in any of the names + assert "lambda_function.py" not in " ".join(names) + + +def test_layer_must_have_dependencies(rule_runner: PythonRuleRunner) -> None: + """A layer _must_ use 'dependencies', unlike most other targets.""" + rule_runner.write_files( + {"BUILD": "python_aws_lambda_layer(name='lambda', runtime='python3.7')"} + ) + with pytest.raises( + ExecutionError, match="The 'dependencies' field in target //:lambda must be defined" + ): + create_python_awslambda( + rule_runner, + Address("", target_name="lambda"), + expected_extra_log_lines=(), + layer=True, + ) diff --git a/src/python/pants/backend/awslambda/python/target_types.py b/src/python/pants/backend/awslambda/python/target_types.py index 3a8a54b4832..5a082b11343 100644 --- a/src/python/pants/backend/awslambda/python/target_types.py +++ b/src/python/pants/backend/awslambda/python/target_types.py @@ -1,9 +1,11 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + import re from dataclasses import dataclass -from typing import Match, Optional, Tuple, cast +from typing import ClassVar, Match, Optional, Tuple, cast from pants.backend.python.target_types import PexCompletePlatformsField, PythonResolveField from pants.backend.python.util_rules.faas import ( @@ -20,6 +22,7 @@ from pants.engine.target import ( COMMON_TARGET_FIELDS, BoolField, + Field, InvalidFieldException, InvalidTargetException, Target, @@ -63,8 +66,20 @@ class PythonAwsLambdaIncludeRequirements(BoolField): default = True help = help_text( """ - Whether to resolve requirements and include them in the Pex. This is most useful with Lambda - Layers to make code uploads smaller when deps are in layers. + Whether to resolve requirements and include them in the AWS Lambda artifact. This is most useful with Lambda + Layers to make code uploads smaller when third-party requirements are in layers. + https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html + """ + ) + + +class PythonAwsLambdaIncludeSources(BoolField): + alias = "include_sources" + default = True + help = help_text( + """ + Whether to resolve first party sources and include them in the AWS Lambda artifact. This is + most useful to allow creating a Lambda Layer with only third-party requirements. https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html """ ) @@ -109,26 +124,21 @@ def to_interpreter_version(self) -> Optional[Tuple[int, int]]: return int(mo.group("major")), int(mo.group("minor")) -class PythonAWSLambda(Target): - alias = "python_awslambda" - core_fields = ( +class PythonAwsLambdaLayerDependenciesField(PythonFaaSDependencies): + required = True + + +class _AWSLambdaBaseTarget(Target): + core_fields: ClassVar[tuple[type[Field], ...]] = ( *COMMON_TARGET_FIELDS, OutputPathField, - PythonFaaSDependencies, - PythonAwsLambdaHandlerField, PythonAwsLambdaIncludeRequirements, + PythonAwsLambdaIncludeSources, PythonAwsLambdaRuntime, PythonFaaSCompletePlatforms, PythonResolveField, EnvironmentField, ) - help = help_text( - f""" - A self-contained Python function suitable for uploading to AWS Lambda. - - See {doc_url('awslambda-python')}. - """ - ) def validate(self) -> None: if self[PythonAwsLambdaRuntime].value is None and not self[PexCompletePlatformsField].value: @@ -143,6 +153,38 @@ def validate(self) -> None: ) +class PythonAWSLambda(_AWSLambdaBaseTarget): + # TODO: rename to python_aws_lambda_function + alias = "python_awslambda" + core_fields = ( + *_AWSLambdaBaseTarget.core_fields, + PythonFaaSDependencies, + PythonAwsLambdaHandlerField, + ) + help = help_text( + f""" + A self-contained Python function suitable for uploading to AWS Lambda. + + See {doc_url('awslambda-python')}. + """ + ) + + +class PythonAWSLambdaLayer(_AWSLambdaBaseTarget): + alias = "python_aws_lambda_layer" + core_fields = ( + *_AWSLambdaBaseTarget.core_fields, + PythonAwsLambdaLayerDependenciesField, + ) + help = help_text( + f""" + A Python layer suitable for uploading to AWS Lambda. + + See {doc_url('awslambda-python')}. + """ + ) + + def rules(): return ( *collect_rules(),