Generate entry_points.txt for python_tests that require entry points from a python_distribution

docs/notes/2.23.x.md

@@ -76,6 +76,13 @@ The default version of the pex tool has been updated from 2.3.1 to 2.3.3.
 
 Fix running python source files that have dashes in them (bug introduced in 2.20). For example: `pants run path/to/some-executable.py`
 
+A new `entry_point_dependencies` field is now available for `python_tests` and `python_test` targets. This allows tests
+to depend on a subset (or all) of the `entry_points` defined on `python_distribution` targets. A dependency defined in
+`entry_point_dependencies` emulates an editable install of those `python_distribution` targets. Instead of including
+all of the `python_distribution`'s sources, only the specified entry points are made available. The entry_points metadata
+is also installed in the pytest sandbox so that tests (or the code under test) can load that metadata via `pkg_resources`.
+To use this, enable the `pants.backend.experimental.python` backend.
+
 #### Terraform
 
 The `tfsec` linter now works on all supported platforms without extra config.

src/python/pants/backend/experimental/python/framework/stevedore/register.py

@@ -10,6 +10,7 @@
 from pants.backend.python.framework.stevedore import rules as stevedore_rules
 from pants.backend.python.framework.stevedore.target_types import StevedoreNamespace
 from pants.backend.python.target_types_rules import rules as python_target_types_rules
+from pants.backend.python.util_rules.entry_points import rules as entry_points_rules
 from pants.build_graph.build_file_aliases import BuildFileAliases
 
 
@@ -19,6 +20,7 @@ def build_file_aliases():
 
 def rules():
     return [
+        *entry_points_rules(),
         *stevedore_rules.rules(),
         *python_target_dependencies.rules(),
         *python_target_types_rules(),

src/python/pants/backend/experimental/python/register.py

@@ -4,7 +4,8 @@
 from pants.backend.python.goals import debug_goals, publish
 from pants.backend.python.subsystems import setuptools_scm, twine
 from pants.backend.python.target_types import VCSVersion
-from pants.backend.python.util_rules import pex, vcs_versioning
+from pants.backend.python.target_types_rules import rules as target_types_rules
+from pants.backend.python.util_rules import entry_points, pex, vcs_versioning
 
 
 def rules():
@@ -15,6 +16,8 @@ def rules():
         *setuptools_scm.rules(),
         *twine.rules(),
         *debug_goals.rules(),
+        *target_types_rules(),
+        *entry_points.rules(),
     )
 
 

src/python/pants/backend/python/framework/stevedore/python_target_dependencies.py

@@ -7,10 +7,6 @@
 from dataclasses import dataclass
 from typing import Mapping
 
-from pants.backend.python.dependency_inference.module_mapper import (
-    PythonModuleOwners,
-    PythonModuleOwnersRequest,
-)
 from pants.backend.python.framework.stevedore.target_types import (
     AllStevedoreExtensionTargets,
     StevedoreExtensionTargets,
@@ -20,20 +16,18 @@
 )
 from pants.backend.python.target_types import (
     PythonDistribution,
-    PythonDistributionDependenciesField,
     PythonDistributionEntryPointsField,
     PythonTestsDependenciesField,
     PythonTestsGeneratorTarget,
     PythonTestTarget,
-    ResolvedPythonDistributionEntryPoints,
-    ResolvePythonDistributionEntryPointsRequest,
 )
-from pants.engine.addresses import Address, Addresses
-from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.backend.python.util_rules.entry_points import (
+    EntryPointDependencies,
+    GetEntryPointDependenciesRequest,
+)
+from pants.engine.rules import Get, collect_rules, rule
 from pants.engine.target import (
     AllTargets,
-    DependenciesRequest,
-    ExplicitlyProvidedDependencies,
     FieldSet,
     InferDependenciesRequest,
     InferredDependencies,
@@ -42,8 +36,6 @@
 from pants.engine.unions import UnionRule
 from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
-from pants.util.ordered_set import OrderedSet
-from pants.util.strutil import softwrap
 
 # -----------------------------------------------------------------------------------------------
 # Utility rules to analyze all StevedoreNamespace entry_points
@@ -154,65 +146,17 @@ async def infer_stevedore_namespaces_dependencies(
         StevedoreNamespacesProviderTargetsRequest(requested_namespaces),
     )
 
-    # This is based on pants.backend.python.target_type_rules.infer_python_distribution_dependencies,
-    # but handles multiple targets and filters the entry_points to just get the requested deps.
-    all_explicit_dependencies = await MultiGet(
-        Get(
-            ExplicitlyProvidedDependencies,
-            DependenciesRequest(tgt[PythonDistributionDependenciesField]),
-        )
-        for tgt in targets
-    )
-    all_resolved_entry_points = await MultiGet(
-        Get(
-            ResolvedPythonDistributionEntryPoints,
-            ResolvePythonDistributionEntryPointsRequest(tgt[PythonDistributionEntryPointsField]),
-        )
-        for tgt in targets
+    requested_namespaces_value = requested_namespaces.value
+    entry_point_dependencies = await Get(
+        EntryPointDependencies,
+        GetEntryPointDependenciesRequest(
+            targets,
+            lambda tgt, ns: ns in requested_namespaces_value,
+            lambda tgt, ns, ep_name: True,
+        ),
     )
 
-    all_module_entry_points = [
-        (tgt.address, namespace, name, entry_point, explicitly_provided_deps)
-        for tgt, distribution_entry_points, explicitly_provided_deps in zip(
-            targets, all_resolved_entry_points, all_explicit_dependencies
-        )
-        for namespace, entry_points in distribution_entry_points.explicit_modules.items()
-        for name, entry_point in entry_points.items()
-    ]
-    all_module_owners = await MultiGet(
-        Get(PythonModuleOwners, PythonModuleOwnersRequest(entry_point.module, resolve=None))
-        for _, _, _, entry_point, _ in all_module_entry_points
-    )
-    module_owners: OrderedSet[Address] = OrderedSet()
-    for (address, namespace, name, entry_point, explicitly_provided_deps), owners in zip(
-        all_module_entry_points, all_module_owners
-    ):
-        if namespace not in requested_namespaces.value:
-            continue
-
-        field_str = repr({namespace: {name: entry_point.spec}})
-        explicitly_provided_deps.maybe_warn_of_ambiguous_dependency_inference(
-            owners.ambiguous,
-            address,
-            import_reference="module",
-            context=softwrap(
-                f"""
-                The python_distribution target {address} has the field
-                `entry_points={field_str}`, which maps to the Python module
-                `{entry_point.module}`
-                """
-            ),
-        )
-        maybe_disambiguated = explicitly_provided_deps.disambiguated(owners.ambiguous)
-        unambiguous_owners = owners.unambiguous or (
-            (maybe_disambiguated,) if maybe_disambiguated else ()
-        )
-        module_owners.update(unambiguous_owners)
-
-    result: tuple[Address, ...] = Addresses(module_owners)
-    for distribution_entry_points in all_resolved_entry_points:
-        result += distribution_entry_points.pex_binary_addresses
-    return InferredDependencies(result)
+    return InferredDependencies(entry_point_dependencies.addresses)
 
 
 def rules():

src/python/pants/backend/python/framework/stevedore/python_target_dependencies_test.py

@@ -31,6 +31,7 @@
     PythonTestTarget,
 )
 from pants.backend.python.target_types_rules import rules as python_target_types_rules
+from pants.backend.python.util_rules.entry_points import rules as entry_points_rules
 from pants.engine.addresses import Address
 from pants.engine.target import InferredDependencies
 from pants.testutil.rule_runner import QueryRule, RuleRunner
@@ -74,6 +75,7 @@ def write_test_files(rule_runner: RuleRunner, extra_build_contents: str = ""):
 def rule_runner() -> RuleRunner:
     rule_runner = RuleRunner(
         rules=[
+            *entry_points_rules(),
             *python_target_types_rules(),
             *stevedore_dep_rules(),
             QueryRule(AllStevedoreExtensionTargets, ()),

src/python/pants/backend/python/framework/stevedore/rules.py

@@ -3,26 +3,21 @@
 
 from __future__ import annotations
 
-from collections import defaultdict
-
 from pants.backend.python.framework.stevedore.target_types import (
     StevedoreExtensionTargets,
     StevedoreNamespacesField,
     StevedoreNamespacesProviderTargetsRequest,
 )
 from pants.backend.python.goals.pytest_runner import PytestPluginSetup, PytestPluginSetupRequest
-from pants.backend.python.target_types import (
-    PythonDistribution,
-    PythonDistributionEntryPoint,
-    PythonDistributionEntryPointsField,
-    ResolvedPythonDistributionEntryPoints,
-    ResolvePythonDistributionEntryPointsRequest,
+from pants.backend.python.target_types import PythonDistribution
+from pants.backend.python.util_rules.entry_points import (
+    EntryPointsTxt,
+    GenerateEntryPointsTxtRequest,
 )
-from pants.engine.fs import EMPTY_DIGEST, CreateDigest, Digest, FileContent, PathGlobs, Paths
-from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.engine.fs import EMPTY_DIGEST
+from pants.engine.rules import Get, collect_rules, rule
 from pants.engine.target import Target
 from pants.engine.unions import UnionRule
-from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
 
 
@@ -52,63 +47,16 @@ async def generate_entry_points_txt_from_stevedore_extension(
         StevedoreNamespacesProviderTargetsRequest(requested_namespaces),
     )
 
-    all_resolved_entry_points = await MultiGet(
-        Get(
-            ResolvedPythonDistributionEntryPoints,
-            ResolvePythonDistributionEntryPointsRequest(tgt[PythonDistributionEntryPointsField]),
-        )
-        for tgt in stevedore_targets
-    )
-
-    possible_paths = [
-        {
-            f"{tgt.address.spec_path}/{ep.entry_point.module.split('.')[0]}"
-            for _, entry_points in (resolved_eps.val or {}).items()
-            for ep in entry_points.values()
-        }
-        for tgt, resolved_eps in zip(stevedore_targets, all_resolved_entry_points)
-    ]
-    resolved_paths = await MultiGet(
-        Get(Paths, PathGlobs(module_candidate_paths)) for module_candidate_paths in possible_paths
+    requested_namespaces_value = requested_namespaces.value
+    entry_points_txt = await Get(
+        EntryPointsTxt,
+        GenerateEntryPointsTxtRequest(
+            stevedore_targets,
+            lambda tgt, ns: ns in requested_namespaces_value,
+            lambda tgt, ns, ep_name: True,
+        ),
     )
-
-    # arrange in sibling groups
-    stevedore_extensions_by_path: dict[
-        str, list[ResolvedPythonDistributionEntryPoints]
-    ] = defaultdict(list)
-    for resolved_ep, paths in zip(all_resolved_entry_points, resolved_paths):
-        path = paths.dirs[0]  # just take the first match
-        stevedore_extensions_by_path[path].append(resolved_ep)
-
-    entry_points_txt_files = []
-    for module_path, resolved_eps in stevedore_extensions_by_path.items():
-        namespace_sections = {}
-
-        for resolved_ep in resolved_eps:
-            namespace: str
-            entry_points: FrozenDict[str, PythonDistributionEntryPoint]
-            for namespace, entry_points in resolved_ep.val.items():
-                if not entry_points or namespace not in requested_namespaces.value:
-                    continue
-
-                entry_points_txt_section = f"[{namespace}]\n"
-                for entry_point_name, ep in sorted(entry_points.items()):
-                    entry_points_txt_section += f"{entry_point_name} = {ep.entry_point.spec}\n"
-                entry_points_txt_section += "\n"
-                namespace_sections[namespace] = entry_points_txt_section
-
-        # consistent sorting
-        entry_points_txt_contents = "".join(
-            namespace_sections[ns] for ns in sorted(namespace_sections)
-        )
-
-        entry_points_txt_path = f"{module_path}.egg-info/entry_points.txt"
-        entry_points_txt_files.append(
-            FileContent(entry_points_txt_path, entry_points_txt_contents.encode("utf-8"))
-        )
-
-    digest = await Get(Digest, CreateDigest(entry_points_txt_files))
-    return PytestPluginSetup(digest)
+    return PytestPluginSetup(entry_points_txt.digest)
 
 
 def rules():

src/python/pants/backend/python/framework/stevedore/rules_test.py

@@ -25,6 +25,7 @@
     PythonTestTarget,
 )
 from pants.backend.python.target_types_rules import rules as python_target_types_rules
+from pants.backend.python.util_rules.entry_points import rules as entry_points_rules
 from pants.engine.addresses import Address
 from pants.engine.fs import EMPTY_DIGEST, CreateDigest, Digest, FileContent
 from pants.testutil.rule_runner import QueryRule, RuleRunner
@@ -37,6 +38,7 @@
 def rule_runner() -> RuleRunner:
     return RuleRunner(
         rules=[
+            *entry_points_rules(),
             *python_target_types_rules(),
             *stevedore_dep_rules(),
             *stevedore_rules(),

src/python/pants/backend/python/target_types.py

@@ -940,6 +940,47 @@ class PythonTestsDependenciesField(PythonDependenciesField):
     supports_transitive_excludes = True
 
 
+class PythonTestsEntryPointDependenciesField(DictStringToStringSequenceField):
+    alias = "entry_point_dependencies"
+    help = help_text(
+        lambda: f"""
+        Dependencies on entry point metadata of `{PythonDistribution.alias}` targets.
+
+        This is a dict where each key is a `{PythonDistribution.alias}` address
+        and the value is a list or tuple of entry point groups and/or entry points
+        on that target. The strings in the value list/tuple must be one of:
+        - "entry.point.group/entry-point-name" to depend on a named entry point
+        - "entry.point.group" (without a "/") to depend on an entry point group
+        - "*" to get all entry points on the target
+
+        For example:
+
+            {PythonTestsEntryPointDependenciesField.alias}={{
+                "//foo/address:dist_tgt": ["*"],  # all entry points
+                "bar:dist_tgt": ["console_scripts"],  # only from this group
+                "foo/bar/baz:dist_tgt": ["console_scripts/my-script"],  # a single entry point
+                "another:dist_tgt": [  # multiple entry points
+                    "console_scripts/my-script",
+                    "console_scripts/another-script",
+                    "entry.point.group/entry-point-name",
+                    "other.group",
+                    "gui_scripts",
+                ],
+            }}
+
+        Code for matching `entry_points` on `{PythonDistribution.alias}` targets
+        will be added as dependencies so that they are available on PYTHONPATH
+        during tests.
+
+        Plus, an `entry_points.txt` file will be generated in the sandbox so that
+        each of the `{PythonDistribution.alias}`s appear to be "installed". The
+        `entry_points.txt` file will only include the entry points requested on this
+        field. This allows the tests, or the code under test, to lookup entry points
+        metadata using something like `pkg_resources.iter_entry_points` from `setuptools`.
+        """
+    )
+
+
 # TODO This field class should extend from a core `TestTimeoutField` once the deprecated options in `pytest` get removed.
 class PythonTestsTimeoutField(IntField):
     alias = "timeout"
@@ -1014,6 +1055,8 @@ class SkipPythonTestsField(BoolField):
 
 _PYTHON_TEST_MOVED_FIELDS = (
     PythonTestsDependenciesField,
+    # This field is registered in the experimental backend for now.
+    # PythonTestsEntryPointDependenciesField,
     PythonResolveField,
     PythonRunGoalUseSandboxField,
     PythonTestsTimeoutField,
@@ -1641,7 +1684,7 @@ class LongDescriptionPathField(StringField):
 
 
 class PythonDistribution(Target):
-    alias = "python_distribution"
+    alias: ClassVar[str] = "python_distribution"
     core_fields = (
         *COMMON_TARGET_FIELDS,
         InterpreterConstraintsField,