Implement python_aws_lambda_target, and include_sources

src/python/pants/backend/awslambda/python/rules.py

@@ -10,6 +10,9 @@
     PythonAWSLambda,
     PythonAwsLambdaHandlerField,
     PythonAwsLambdaIncludeRequirements,
+    PythonAwsLambdaIncludeSources,
+    PythonAWSLambdaLayer,
+    PythonAwsLambdaLayerDependenciesField,
     PythonAwsLambdaRuntime,
 )
 from pants.backend.python.subsystems.lambdex import Lambdex, LambdexLayout
@@ -22,8 +25,10 @@
 from pants.core.goals.package import BuiltPackage, OutputPathField, PackageFieldSet
 from pants.core.util_rules.environments import EnvironmentField
 from pants.engine.rules import Get, collect_rules, rule
+from pants.engine.target import InvalidTargetException
 from pants.engine.unions import UnionRule
 from pants.util.logging import LogLevel
+from pants.util.strutil import softwrap
 
 logger = logging.getLogger(__name__)
 
@@ -34,6 +39,20 @@ class PythonAwsLambdaFieldSet(PackageFieldSet):
 
     handler: PythonAwsLambdaHandlerField
     include_requirements: PythonAwsLambdaIncludeRequirements
+    include_sources: PythonAwsLambdaIncludeSources
+    runtime: PythonAwsLambdaRuntime
+    complete_platforms: PythonFaaSCompletePlatforms
+    output_path: OutputPathField
+    environment: EnvironmentField
+
+
+@dataclass(frozen=True)
+class PythonAwsLambdaLayerFieldSet(PackageFieldSet):
+    required_fields = (PythonAwsLambdaLayerDependenciesField,)
+
+    dependencies: PythonAwsLambdaLayerDependenciesField
+    include_requirements: PythonAwsLambdaIncludeRequirements
+    include_sources: PythonAwsLambdaIncludeSources
     runtime: PythonAwsLambdaRuntime
     complete_platforms: PythonFaaSCompletePlatforms
     output_path: OutputPathField
@@ -74,12 +93,56 @@ async def package_python_awslambda(
             handler=field_set.handler,
             output_path=field_set.output_path,
             include_requirements=field_set.include_requirements.value,
-            include_sources=True,
+            include_sources=field_set.include_sources.value,
             reexported_handler_module=PythonAwsLambdaHandlerField.reexported_handler_module,
         ),
     )
 
 
+@rule(desc="Create Python AWS Lambda Layer", level=LogLevel.DEBUG)
+async def package_python_aws_lambda_layer(
+    field_set: PythonAwsLambdaLayerFieldSet,
+    lambdex: Lambdex,
+) -> BuiltPackage:
+    if lambdex.layout is LambdexLayout.LAMBDEX:
+        raise InvalidTargetException(
+            softwrap(
+                f"""
+                the `{PythonAWSLambdaLayer.alias}` target {field_set.address} cannot be used with
+                the old Lambdex layout (`[lambdex].layout = \"{LambdexLayout.LAMBDEX.value}\"` in
+                `pants.toml`), set that to `{LambdexLayout.ZIP.value}` or remove this target
+                """
+            )
+        )
+
+    return await Get(
+        BuiltPackage,
+        BuildPythonFaaSRequest(
+            address=field_set.address,
+            target_name=PythonAWSLambdaLayer.alias,
+            complete_platforms=field_set.complete_platforms,
+            runtime=field_set.runtime,
+            output_path=field_set.output_path,
+            include_requirements=field_set.include_requirements.value,
+            include_sources=field_set.include_sources.value,
+            # See
+            # https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path
+            #
+            # Runtime | Path
+            # ...
+            # Python  | `python`
+            #         | `python/lib/python3.10/site-packages`
+            # ...
+            #
+            # The one independent on the runtime-version is more convenient:
+            prefix_in_artifact="python",
+            # a layer doesn't have a handler, just pulls in things via `dependencies`
+            handler=None,
+            reexported_handler_module=None,
+        ),
+    )
+
+
 def rules():
     return [
         *collect_rules(),

src/python/pants/backend/awslambda/python/rules_test.py

@@ -12,9 +12,12 @@
 
 import pytest
 
-from pants.backend.awslambda.python.rules import PythonAwsLambdaFieldSet
+from pants.backend.awslambda.python.rules import (
+    PythonAwsLambdaFieldSet,
+    PythonAwsLambdaLayerFieldSet,
+)
 from pants.backend.awslambda.python.rules import rules as awslambda_python_rules
-from pants.backend.awslambda.python.target_types import PythonAWSLambda
+from pants.backend.awslambda.python.target_types import PythonAWSLambda, PythonAWSLambdaLayer
 from pants.backend.awslambda.python.target_types import rules as target_rules
 from pants.backend.python.goals import package_pex_binary
 from pants.backend.python.goals.package_pex_binary import PexBinaryFieldSet
@@ -37,6 +40,8 @@
 from pants.core.target_types import rules as core_target_types_rules
 from pants.engine.addresses import Address
 from pants.engine.fs import DigestContents
+from pants.engine.internals.scheduler import ExecutionError
+from pants.engine.target import FieldSet
 from pants.testutil.python_interpreter_selection import all_major_minor_python_versions
 from pants.testutil.python_rule_runner import PythonRuleRunner
 from pants.testutil.rule_runner import QueryRule
@@ -54,12 +59,14 @@ def rule_runner() -> PythonRuleRunner:
             *target_rules(),
             *package.rules(),
             QueryRule(BuiltPackage, (PythonAwsLambdaFieldSet,)),
+            QueryRule(BuiltPackage, (PythonAwsLambdaLayerFieldSet,)),
         ],
         target_types=[
             FileTarget,
             FilesGeneratorTarget,
             PexBinary,
             PythonAWSLambda,
+            PythonAWSLambdaLayer,
             PythonRequirementTarget,
             PythonRequirementTarget,
             PythonSourcesGeneratorTarget,
@@ -77,13 +84,18 @@ def create_python_awslambda(
     *,
     expected_extra_log_lines: tuple[str, ...],
     extra_args: list[str] | None = None,
+    layer: bool = False,
 ) -> tuple[str, bytes]:
     rule_runner.set_options(
         ["--source-root-patterns=src/python", *(extra_args or ())],
         env_inherit={"PATH", "PYENV_ROOT", "HOME"},
     )
     target = rule_runner.get_target(addr)
-    built_asset = rule_runner.request(BuiltPackage, [PythonAwsLambdaFieldSet.create(target)])
+    if layer:
+        field_set: type[FieldSet] = PythonAwsLambdaLayerFieldSet
+    else:
+        field_set = PythonAwsLambdaFieldSet
+    built_asset = rule_runner.request(BuiltPackage, [field_set.create(target)])
     assert expected_extra_log_lines == built_asset.artifacts[0].extra_log_lines
     digest_contents = rule_runner.request(DigestContents, [built_asset.digest])
     assert len(digest_contents) == 1
@@ -328,3 +340,82 @@ def handler(event, context):
     assert (
         zipfile.read("lambda_function.py") == b"from foo.bar.hello_world import handler as handler"
     )
+
+
+def test_create_hello_world_layer(rule_runner: PythonRuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "src/python/foo/bar/hello_world.py": dedent(
+                """
+                import mureq
+
+                def handler(event, context):
+                    print('Hello, World!')
+                """
+            ),
+            "src/python/foo/bar/BUILD": dedent(
+                """
+                python_requirement(name="mureq", requirements=["mureq==0.2"])
+                python_sources()
+
+                python_aws_lambda_layer(
+                    name='lambda',
+                    dependencies=["./hello_world.py"],
+                    runtime="python3.7",
+                )
+                python_aws_lambda_layer(
+                    name='slimlambda',
+                    include_sources=False,
+                    dependencies=["./hello_world.py"],
+                    runtime="python3.7",
+                )
+                """
+            ),
+        }
+    )
+
+    zip_file_relpath, content = create_python_awslambda(
+        rule_runner,
+        Address("src/python/foo/bar", target_name="lambda"),
+        expected_extra_log_lines=(),
+        layer=True,
+    )
+    assert "src.python.foo.bar/lambda.zip" == zip_file_relpath
+
+    zipfile = ZipFile(BytesIO(content))
+    names = set(zipfile.namelist())
+    assert "python/mureq/__init__.py" in names
+    assert "python/foo/bar/hello_world.py" in names
+    # nothing that looks like a synthesized handler in any of the names
+    assert "lambda_function.py" not in " ".join(names)
+
+    zip_file_relpath, content = create_python_awslambda(
+        rule_runner,
+        Address("src/python/foo/bar", target_name="slimlambda"),
+        expected_extra_log_lines=(),
+        layer=True,
+    )
+    assert "src.python.foo.bar/slimlambda.zip" == zip_file_relpath
+
+    zipfile = ZipFile(BytesIO(content))
+    names = set(zipfile.namelist())
+    assert "python/mureq/__init__.py" in names
+    assert "python/foo/bar/hello_world.py" not in names
+    # nothing that looks like a synthesized handler in any of the names
+    assert "lambda_function.py" not in " ".join(names)
+
+
+def test_layer_must_have_dependencies(rule_runner: PythonRuleRunner) -> None:
+    """A layer _must_ use 'dependencies', unlike most other targets."""
+    rule_runner.write_files(
+        {"BUILD": "python_aws_lambda_layer(name='lambda', runtime='python3.7')"}
+    )
+    with pytest.raises(
+        ExecutionError, match="The 'dependencies' field in target //:lambda must be defined"
+    ):
+        create_python_awslambda(
+            rule_runner,
+            Address("", target_name="lambda"),
+            expected_extra_log_lines=(),
+            layer=True,
+        )

src/python/pants/backend/awslambda/python/target_types.py

@@ -63,8 +63,20 @@ class PythonAwsLambdaIncludeRequirements(BoolField):
     default = True
     help = help_text(
         """
-        Whether to resolve requirements and include them in the Pex. This is most useful with Lambda
-        Layers to make code uploads smaller when deps are in layers.
+        Whether to resolve requirements and include them in the AWS Lambda artifact. This is most useful with Lambda
+        Layers to make code uploads smaller when third-party requirements are in layers.
+        https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html
+        """
+    )
+
+
+class PythonAwsLambdaIncludeSources(BoolField):
+    alias = "include_sources"
+    default = True
+    help = help_text(
+        """
+        Whether to resolve first party sources and include them in the AWS Lambda artifact. This is
+        most useful to allow creating a Lambda Layer with only third-party requirements.
         https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html
         """
     )
@@ -109,26 +121,21 @@ def to_interpreter_version(self) -> Optional[Tuple[int, int]]:
         return int(mo.group("major")), int(mo.group("minor"))
 
 
-class PythonAWSLambda(Target):
-    alias = "python_awslambda"
+class PythonAwsLambdaLayerDependenciesField(PythonFaaSDependencies):
+    required = True
+
+
+class _AWSLambdaBaseTarget(Target):
     core_fields = (
         *COMMON_TARGET_FIELDS,
         OutputPathField,
-        PythonFaaSDependencies,
-        PythonAwsLambdaHandlerField,
         PythonAwsLambdaIncludeRequirements,
+        PythonAwsLambdaIncludeSources,
         PythonAwsLambdaRuntime,
         PythonFaaSCompletePlatforms,
         PythonResolveField,
         EnvironmentField,
     )
-    help = help_text(
-        f"""
-        A self-contained Python function suitable for uploading to AWS Lambda.
-
-        See {doc_url('awslambda-python')}.
-        """
-    )
 
     def validate(self) -> None:
         if self[PythonAwsLambdaRuntime].value is None and not self[PexCompletePlatformsField].value:
@@ -143,6 +150,38 @@ def validate(self) -> None:
             )
 
 
+class PythonAWSLambda(_AWSLambdaBaseTarget):
+    # TODO: rename to python_aws_lambda_function
+    alias = "python_awslambda"
+    core_fields = (
+        *_AWSLambdaBaseTarget.core_fields,
+        PythonFaaSDependencies,
+        PythonAwsLambdaHandlerField,
+    )
+    help = help_text(
+        f"""
+        A self-contained Python function suitable for uploading to AWS Lambda.
+
+        See {doc_url('awslambda-python')}.
+        """
+    )
+
+
+class PythonAWSLambdaLayer(_AWSLambdaBaseTarget):
+    alias = "python_aws_lambda_layer"
+    core_fields = (
+        *_AWSLambdaBaseTarget.core_fields,
+        PythonAwsLambdaLayerDependenciesField,
+    )
+    help = help_text(
+        f"""
+        A Python layer suitable for uploading to AWS Lambda.
+
+        See {doc_url('awslambda-python')}.
+        """
+    )
+
+
 def rules():
     return (
         *collect_rules(),