Resolve pants_requirements from direct wheel links to GitHub Releas…

…e assets (#19432)

After #19431 is merged and we're
releasing wheels as GitHub Release assets, if/when we stop publishing to
PyPI we need an alternate solution for pants plugin requirements.

This change switches the `pants_requirements` target generator in two
ways:
- Allows for a version to be specified (defaulting to the current
running version) which is matched exactly
- Populates a private `_find_links` field with a value pointing to the
relevant assets list for the release

Then, the necessary plumbing is done for the new `_find_links` field,
and testing.

Local testing:

```console
josh@cephandrius:~/work/techlabs$ cat pants-plugins/BUILD.pants | head -n6
pants_requirements(
    name = "pants",
    resolve = "pants-plugins",
    version="2.16.0.dev5",
    testutil=False
)
josh@cephandrius:~/work/techlabs$ PANTS_SOURCE=../pants pants generate-lockfiles --resolve=pants-plugins                                                           
Pantsd has been turned off via Env.
21:13:13.20 [INFO] Completed: Generate lockfile for pants-plugins                                                                                                  
21:13:13.21 [INFO] Wrote lockfile for the resolve `pants-plugins` to 3rdparty/python/lockfiles/pants-plugins.lock
josh@cephandrius:~/work/techlabs$ grep -v '^//'  3rdparty/python/lockfiles/pants-plugins.lock | jq '.locked_resolves[0].locked_requirements[] | select(.project_name == "pantsbuild-pants")'
{
  "artifacts": [
    {
      "algorithm": "sha256",
      "hash": "ea66ac27d983d4c7b3252845fbea635f4ae49ebd5fd5d139a167eb173a5dc5e0",
      "url": "https://github.com/pantsbuild/pants/releases/download/release_2.16.0.dev5/pantsbuild.pants-2.16.0.dev5-cp39-cp39-manylinux2014_x86_64.whl"
    },
    {
      "algorithm": "sha256",
      "hash": "5428b46f33d824a8066a6d5c44379bce678cd3c504a07e06d214a6665fa8604a",
      "url": "https://github.com/pantsbuild/pants/releases/download/release_2.16.0.dev5/pantsbuild.pants-2.16.0.dev5-cp39-cp39-macosx_10_15_x86_64.whl"
    },
    {
      "algorithm": "sha256",
      "hash": "f97f91ecf9913f0b1d2898f03326ac567e0e5e7df5bd1638536e1e87a566cd47",
      "url": "https://github.com/pantsbuild/pants/releases/download/release_2.16.0.dev5/pantsbuild.pants-2.16.0.dev5-cp39-cp39-macosx_11_0_arm64.whl"
    },
    {
      "algorithm": "sha256",
      "hash": "c7e4e9a1a88f6ee1fac2d62fc52caebcb66029458fe1c1978b730b7a5bc8823a",
      "url": "https://github.com/pantsbuild/pants/releases/download/release_2.16.0.dev5/pantsbuild.pants-2.16.0.dev5-cp39-cp39-manylinux2014_aarch64.whl"
    }
  ],
  "project_name": "pantsbuild-pants",
  "requires_dists": [
    "PyYAML<7.0,>=6.0",
    "ansicolors==1.1.8",
    "chevron==0.14.0",
    "fasteners==0.16.3",
    "humbug==0.2.7",
    "ijson==3.1.4",
    "importlib-resources==5.0.*",
    "packaging==21.3",
    "pex==2.1.120",
    "psutil==5.9.0",
    "python-lsp-jsonrpc==1.0.0",
    "setproctitle==1.3.2",
    "setuptools<64.0,>=63.1.0",
    "toml==0.10.2",
    "types-PyYAML==6.0.3",
    "types-setuptools==62.6.1",
    "types-toml==0.10.8",
    "typing-extensions==4.3.0"
  ],
  "requires_python": "<3.10,>=3.7",
  "version": "2.16.0.dev5"
}
```

src/python/pants/backend/plugin_development/pants_requirements.py

@@ -2,6 +2,7 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from pants.backend.python.target_types import (
+    PythonRequirementFindLinksField,
     PythonRequirementModulesField,
     PythonRequirementResolveField,
     PythonRequirementsField,
@@ -13,11 +14,12 @@
     BoolField,
     GeneratedTargets,
     GenerateTargetsRequest,
+    StringField,
     TargetGenerator,
 )
 from pants.engine.unions import UnionMembership, UnionRule
 from pants.util.strutil import help_text
-from pants.version import MAJOR_MINOR, PANTS_SEMVER
+from pants.version import PANTS_SEMVER
 
 
 class PantsRequirementsTestutilField(BoolField):
@@ -26,21 +28,29 @@ class PantsRequirementsTestutilField(BoolField):
     help = "If true, include `pantsbuild.pants.testutil` to write tests for your plugin."
 
 
+class PantsRequirementsVersionSpecField(StringField):
+    alias = "version_spec"
+    default = f"== {PANTS_SEMVER.public}"
+    help = help_text(
+        """
+        The PEP 440 version specifier version of Pants to target.
+        E.g. `== 2.15.*`, or `>= 2.16.0, < 2.17.0`
+        """
+    )
+
+
 class PantsRequirementsTargetGenerator(TargetGenerator):
     alias = "pants_requirements"
     help = help_text(
-        f"""
+        """
         Generate `python_requirement` targets for Pants itself to use with Pants plugins.
 
         This is useful when writing plugins so that you can build and test your
-        plugin using Pants. The generated targets will have the correct version based on the
-        `version` in your `pants.toml`, and they will work with dependency inference.
+        plugin using Pants.
 
-        Because the Plugin API is not yet stable, the version is set automatically for you
-        to improve stability. If you're currently using a dev release, the version will be set to
-        that exact dev release. If you're using an alpha release, release candidate (rc), or stable
-        release, the version will allow any non-dev-release release within the release series, e.g.
-        `>={MAJOR_MINOR}.0rc0,<{PANTS_SEMVER.major}.{PANTS_SEMVER.minor + 1}`.
+        The generated targets will have the correct version based on the exact `version` in your
+        `pants.toml`, and they will work with dependency inference. They're pulled directly from
+        our GitHub releases, using the relevant platform markers.
 
         (If this versioning scheme does not work for you, you can directly create
         `python_requirement` targets for `pantsbuild.pants` and `pantsbuild.pants.testutil`. We
@@ -51,7 +61,9 @@ class PantsRequirementsTargetGenerator(TargetGenerator):
     generated_target_cls = PythonRequirementTarget
     core_fields = (
         *COMMON_TARGET_FIELDS,
+        PantsRequirementsVersionSpecField,
         PantsRequirementsTestutilField,
+        PythonRequirementFindLinksField,
     )
     copied_fields = COMMON_TARGET_FIELDS
     moved_fields = (PythonRequirementResolveField,)
@@ -61,40 +73,18 @@ class GenerateFromPantsRequirementsRequest(GenerateTargetsRequest):
     generate_from = PantsRequirementsTargetGenerator
 
 
-def determine_version() -> str:
-    # Because the Plugin API is not stable, it can have breaking changes in-between dev releases.
-    # Technically, it can also have breaking changes between rcs in the same release series, but
-    # this is much less likely.
-    #
-    # So, we require exact matches when developing against a dev release, but only require
-    # matching the release series if on an alpha release, rc, or stable release.
-    #
-    # If this scheme does not work for users, they can:
-    #
-    #    1. Use a `python_requirement` directly
-    #    2. Add a new `version` field to this target generator.
-    #    3. Fork this target generator.
-    return (
-        f"=={PANTS_SEMVER}"
-        if PANTS_SEMVER.is_devrelease
-        else (
-            f">={PANTS_SEMVER.major}.{PANTS_SEMVER.minor}.0a0,"
-            f"<{PANTS_SEMVER.major}.{PANTS_SEMVER.minor + 1}"
-        )
-    )
-
-
 @rule
 def generate_from_pants_requirements(
     request: GenerateFromPantsRequirementsRequest, union_membership: UnionMembership
 ) -> GeneratedTargets:
     generator = request.generator
-    version = determine_version()
+    version_spec = generator[PantsRequirementsVersionSpecField].value
 
     def create_tgt(dist: str, module: str) -> PythonRequirementTarget:
         return PythonRequirementTarget(
             {
-                PythonRequirementsField.alias: (f"{dist}{version}",),
+                PythonRequirementsField.alias: (f"{dist} {version_spec}",),
+                PythonRequirementFindLinksField.alias: ("https://wheels.pantsbuild.org/simple",),
                 PythonRequirementModulesField.alias: (module,),
                 **request.template,
             },

src/python/pants/backend/plugin_development/pants_requirements_test.py

@@ -1,15 +1,10 @@
 # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-import pytest
-from packaging.version import Version
-
 from pants.backend.plugin_development import pants_requirements
-from pants.backend.plugin_development.pants_requirements import (
-    PantsRequirementsTargetGenerator,
-    determine_version,
-)
+from pants.backend.plugin_development.pants_requirements import PantsRequirementsTargetGenerator
 from pants.backend.python.target_types import (
+    PythonRequirementFindLinksField,
     PythonRequirementModulesField,
     PythonRequirementResolveField,
     PythonRequirementsField,
@@ -18,19 +13,7 @@
 from pants.engine.internals.graph import _TargetParametrizations, _TargetParametrizationsRequest
 from pants.testutil.rule_runner import QueryRule, RuleRunner
 from pants.util.pip_requirement import PipRequirement
-
-
-@pytest.mark.parametrize(
-    "pants_version,expected",
-    (
-        ("2.4.0.dev1", "==2.4.0.dev1"),
-        ("2.4.0rc1", ">=2.4.0a0,<2.5"),
-        ("2.4.0", ">=2.4.0a0,<2.5"),
-    ),
-)
-def test_determine_version(monkeypatch, pants_version: str, expected: str) -> None:
-    monkeypatch.setattr(pants_requirements, "PANTS_SEMVER", Version(pants_version))
-    assert determine_version() == expected
+from pants.version import VERSION
 
 
 def test_target_generator() -> None:
@@ -46,6 +29,7 @@ def test_target_generator() -> None:
         {
             "BUILD": (
                 "pants_requirements(name='default')\n"
+                "pants_requirements(name='pants-2.16', version_spec='>= 2.16.0, < 2.17.0')\n"
                 "pants_requirements(\n"
                 "  name='no_testutil', testutil=False, resolve='a'\n"
                 ")"
@@ -69,11 +53,18 @@ def test_target_generator() -> None:
     assert pants_req[PythonRequirementModulesField].value == ("pants",)
     assert testutil_req[PythonRequirementModulesField].value == ("pants.testutil",)
     assert pants_req[PythonRequirementsField].value == (
-        PipRequirement.parse(f"pantsbuild.pants{determine_version()}"),
+        PipRequirement.parse(f"pantsbuild.pants=={VERSION}"),
     )
     assert testutil_req[PythonRequirementsField].value == (
-        PipRequirement.parse(f"pantsbuild.pants.testutil{determine_version()}"),
+        PipRequirement.parse(f"pantsbuild.pants.testutil=={VERSION}"),
+    )
+    assert pants_req[PythonRequirementFindLinksField].value == (
+        "https://wheels.pantsbuild.org/simple",
+    )
+    assert testutil_req[PythonRequirementFindLinksField].value == (
+        "https://wheels.pantsbuild.org/simple",
     )
+
     for t in (pants_req, testutil_req):
         assert not t[PythonRequirementResolveField].value
 
@@ -89,3 +80,28 @@ def test_target_generator() -> None:
     assert next(iter(result.keys())).generated_name == "pantsbuild.pants"
     pants_req = next(iter(result.values()))
     assert pants_req[PythonRequirementResolveField].value == "a"
+
+    result = rule_runner.request(
+        _TargetParametrizations,
+        [
+            _TargetParametrizationsRequest(
+                Address("", target_name="pants-2.16"), description_of_origin="tests"
+            )
+        ],
+    ).parametrizations
+    pants_req = next(t for t in result.values() if t.address.generated_name == "pantsbuild.pants")
+    testutil_req = next(
+        t for t in result.values() if t.address.generated_name == "pantsbuild.pants.testutil"
+    )
+    assert pants_req[PythonRequirementsField].value == (
+        PipRequirement.parse("pantsbuild.pants>=2.16.0,<2.17.0"),
+    )
+    assert testutil_req[PythonRequirementsField].value == (
+        PipRequirement.parse("pantsbuild.pants.testutil>=2.16.0,<2.17.0"),
+    )
+    assert pants_req[PythonRequirementFindLinksField].value == (
+        "https://wheels.pantsbuild.org/simple",
+    )
+    assert testutil_req[PythonRequirementFindLinksField].value == (
+        "https://wheels.pantsbuild.org/simple",
+    )

src/python/pants/backend/python/goals/lockfile.py

@@ -10,7 +10,11 @@
 from operator import itemgetter
 
 from pants.backend.python.subsystems.setup import PythonSetup
-from pants.backend.python.target_types import PythonRequirementResolveField, PythonRequirementsField
+from pants.backend.python.target_types import (
+    PythonRequirementFindLinksField,
+    PythonRequirementResolveField,
+    PythonRequirementsField,
+)
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.lockfile_diff import _generate_python_lockfile_diff
 from pants.backend.python.util_rules.lockfile_metadata import PythonLockfileMetadata
@@ -47,6 +51,7 @@
 @dataclass(frozen=True)
 class GeneratePythonLockfile(GenerateLockfile):
     requirements: FrozenOrderedSet[str]
+    find_links: FrozenOrderedSet[str]
     interpreter_constraints: InterpreterConstraints
 
     @property
@@ -131,6 +136,7 @@ async def generate_lockfile(
                 "mac",
                 # This makes diffs more readable when lockfiles change.
                 "--indent=2",
+                *(f"--find-links={link}" for link in req.find_links),
                 *pip_args_setup.args,
                 *req.interpreter_constraints.generate_pex_arg_list(),
                 *req.requirements,
@@ -220,17 +226,20 @@ async def setup_user_lockfile_requests(
         return UserGenerateLockfiles()
 
     resolve_to_requirements_fields = defaultdict(set)
+    find_links: set[str] = set()
     for tgt in all_targets:
         if not tgt.has_fields((PythonRequirementResolveField, PythonRequirementsField)):
             continue
         resolve = tgt[PythonRequirementResolveField].normalized_value(python_setup)
         resolve_to_requirements_fields[resolve].add(tgt[PythonRequirementsField])
+        find_links.update(tgt[PythonRequirementFindLinksField].value or ())
 
     return UserGenerateLockfiles(
         GeneratePythonLockfile(
             requirements=PexRequirements.req_strings_from_requirement_fields(
                 resolve_to_requirements_fields[resolve]
             ),
+            find_links=FrozenOrderedSet(find_links),
             interpreter_constraints=InterpreterConstraints(
                 python_setup.resolves_to_interpreter_constraints.get(
                     resolve, python_setup.interpreter_constraints

src/python/pants/backend/python/goals/lockfile_test.py

@@ -51,6 +51,7 @@ def _generate(
         [
             GeneratePythonLockfile(
                 requirements=FrozenOrderedSet([f"ansicolors{ansicolors_version}"]),
+                find_links=FrozenOrderedSet([]),
                 interpreter_constraints=InterpreterConstraints(),
                 resolve_name="test",
                 lockfile_dest="test.lock",
@@ -236,13 +237,15 @@ def test_multiple_resolves() -> None:
     assert set(result) == {
         GeneratePythonLockfile(
             requirements=FrozenOrderedSet(["a"]),
+            find_links=FrozenOrderedSet([]),
             interpreter_constraints=InterpreterConstraints(["CPython>=3.7,<3.10"]),
             resolve_name="a",
             lockfile_dest="a.lock",
             diff=False,
         ),
         GeneratePythonLockfile(
             requirements=FrozenOrderedSet(["b"]),
+            find_links=FrozenOrderedSet([]),
             interpreter_constraints=InterpreterConstraints(["==3.7.*"]),
             resolve_name="b",
             lockfile_dest="b.lock",

src/python/pants/backend/python/target_types.py

@@ -1271,6 +1271,14 @@ class PythonRequirementResolveField(PythonResolveField):
     )
 
 
+class PythonRequirementFindLinksField(StringSequenceField):
+    # NB: This is solely used for `pants_requirements` target generation
+    alias = "_find_links"
+    required = False
+    default = ()
+    help = "<Internal>"
+
+
 class PythonRequirementEntryPointField(EntryPointField):
     # Specialist subclass for matching `PythonRequirementTarget` when running.
     pass
@@ -1286,6 +1294,7 @@ class PythonRequirementTarget(Target):
         PythonRequirementTypeStubModulesField,
         PythonRequirementResolveField,
         PythonRequirementEntryPointField,
+        PythonRequirementFindLinksField,
     )
     help = help_text(
         f"""

src/python/pants/backend/python/util_rules/pex_test.py

@@ -505,6 +505,7 @@ def options(path_mappings_dir: Path) -> tuple[str, ...]:
             [
                 GeneratePythonLockfile(
                     requirements=FrozenOrderedSet([wheel_req_str]),
+                    find_links=FrozenOrderedSet([]),
                     interpreter_constraints=InterpreterConstraints([">=3.7,<4"]),
                     resolve_name="test",
                     lockfile_dest="test.lock",