replace pkg_resources with importlib_metadata and packaging

* Closes #5775
* Replace the pkg_resources library (deprecated) with newer alternatives.
* This resolves an efficiency issue due to the high import time of
  pkg_resources. This import time hits every single Cylc command
  including `cylc message`.

conda-environment.yml

@@ -15,8 +15,7 @@ dependencies:
   - psutil >=5.6.0
   - python
   - pyzmq >=22
-  - setuptools >=49,!=67.*
-  - importlib_metadata # [py<3.8]
+  - importlib_metadata >=5.0 # [py<3.12]
   - urwid >=2,<3
   - tomli >=2 # [py<3.11]
 

cylc/flow/__init__.py

@@ -58,11 +58,19 @@ def environ_init():
 
 def iter_entry_points(entry_point_name):
     """Iterate over Cylc entry points."""
-    import pkg_resources
+    import sys
+    if sys.version_info[:2] > (3, 11):
+        from importlib.metadata import entry_points
+    else:
+        # BACK COMPAT: importlib_metadata
+        #   importlib.metadata was added in Python 3.8. The required interfaces
+        #   were completed by 3.12. For lower versions we must use the
+        #   importlib_metadata backport.
+        # FROM: Python 3.7
+        # TO: Python: 3.12
+        from importlib_metadata import entry_points
     yield from (
         entry_point
-        for entry_point in pkg_resources.iter_entry_points(entry_point_name)
-        # Filter out the cylc namespace as it should be empty.
-        # All cylc packages should take the form cylc-<name>
-        if entry_point.dist.key != 'cylc'
+        # for entry_point in entry_points()[entry_point_name]
+        for entry_point in entry_points().select(group=entry_point_name)
     )

cylc/flow/cfgspec/globalcfg.py

@@ -22,7 +22,7 @@
 from typing import List, Optional, Tuple, Any, Union
 
 from contextlib import suppress
-from pkg_resources import parse_version
+from packaging.version import parse as parse_version, Version
 
 from cylc.flow import LOG
 from cylc.flow import __version__ as CYLC_VERSION
@@ -1866,8 +1866,7 @@ def get_version_hierarchy(version: str) -> List[str]:
         ['', '8', '8.0', '8.0.1', '8.0.1a2', '8.0.1a2.dev']
 
     """
-    smart_ver: Any = parse_version(version)
-    # (No type anno. yet for Version in pkg_resources.extern.packaging.version)
+    smart_ver: Version = parse_version(version)
     base = [str(i) for i in smart_ver.release]
     hierarchy = ['']
     hierarchy += ['.'.join(base[:i]) for i in range(1, len(base) + 1)]

cylc/flow/network/scan.py

@@ -50,10 +50,8 @@
 import re
 from typing import AsyncGenerator, Dict, Iterable, List, Optional, Tuple, Union
 
-from pkg_resources import (
-    parse_requirements,
-    parse_version
-)
+from packaging.version import parse as parse_version
+from packaging.specifiers import SpecifierSet
 
 from cylc.flow import LOG
 from cylc.flow.async_util import (
@@ -354,11 +352,7 @@ async def validate_contact_info(flow):
 
 def parse_requirement(requirement_string):
     """Parse a requirement from a requirement string."""
-    # we have to give the requirement a name but what we call it doesn't
-    # actually matter
-    for req in parse_requirements(f'x {requirement_string}'):
-        # there should only be one requirement
-        return (req,), {}
+    return (SpecifierSet(requirement_string),), {}
 
 
 @pipe(preproc=parse_requirement)
@@ -373,7 +367,7 @@ async def cylc_version(flow, requirement):
         flow (dict):
             Flow information dictionary, provided by scan through the pipe.
         requirement (str):
-            Requirement specifier in pkg_resources format e.g. ``> 8, < 9``
+            Requirement specifier in PEP 440 format e.g. ``> 8, < 9``
 
     """
     return parse_version(flow[ContactFileFields.VERSION]) in requirement
@@ -391,7 +385,7 @@ async def api_version(flow, requirement):
         flow (dict):
             Flow information dictionary, provided by scan through the pipe.
         requirement (str):
-            Requirement specifier in pkg_resources format e.g. ``> 8, < 9``
+            Requirement specifier in PEP 440 format e.g. ``> 8, < 9``
 
     """
     return parse_version(flow[ContactFileFields.API]) in requirement

cylc/flow/parsec/fileparse.py

@@ -277,7 +277,7 @@ def process_plugins(fpath, opts):
         try:
             # If you want it to work on sourcedirs you need to get the options
             # to here.
-            plugin_result = entry_point.resolve()(
+            plugin_result = entry_point.load()(
                 srcdir=fpath, opts=opts
             )
         except Exception as exc:

cylc/flow/scheduler_cli.py

@@ -25,7 +25,7 @@
 import sys
 from typing import TYPE_CHECKING
 
-from pkg_resources import parse_version
+from packaging.version import parse as parse_version
 
 from cylc.flow import LOG, __version__
 from cylc.flow.exceptions import (

cylc/flow/scripts/completion_server.py

@@ -46,10 +46,8 @@
 import sys
 import typing as t
 
-from pkg_resources import (
-    parse_requirements,
-    parse_version
-)
+from packaging.version import parse as parse_version
+from packaging.specifiers import SpecifierSet
 
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 from cylc.flow.id import tokenise, IDTokens, Tokens
@@ -74,7 +72,7 @@
 # set the compatibility range for completion scripts with this server
 # I.E. if we change the server interface, change this compatibility range.
 # User's will be presented with an upgrade notice if this happens.
-REQUIRED_SCRIPT_VERSION = 'completion-script >=1.0.0, <2.0.0'
+REQUIRED_SCRIPT_VERSION = '>=1.0.0, <2.0.0'
 
 # register the psudo "help" and "version" commands
 COMMAND_LIST = list(COMMANDS) + ['help', 'version']
@@ -317,7 +315,7 @@ def list_options(command: str) -> t.List[str]:
     if command in COMMAND_OPTION_MAP:
         return COMMAND_OPTION_MAP[command]
     try:
-        entry_point = COMMANDS[command].resolve()
+        entry_point = COMMANDS[command].load()
     except KeyError:
         return []
     parser = entry_point.parser_function()
@@ -637,15 +635,13 @@ def check_completion_script_compatibility(
 
     # check that the installed completion script is compabile with this
     # completion server version
-    for requirement in parse_requirements(REQUIRED_SCRIPT_VERSION):
-        # NOTE: there's only one requirement but we have to iterate to get it
-        if installed_version not in requirement:
-            is_compatible = False
-            print(
-                f'The Cylc {completion_lang} script needs to be updated to'
-                ' work with this version of Cylc.',
-                file=sys.stderr,
-            )
+    if installed_version not in SpecifierSet(REQUIRED_SCRIPT_VERSION):
+        is_compatible = False
+        print(
+            f'The Cylc {completion_lang} script needs to be updated to'
+            ' work with this version of Cylc.',
+            file=sys.stderr,
+        )
 
     # check for completion script updates
     if installed_version < current_version:

cylc/flow/scripts/cylc.py

@@ -43,12 +43,28 @@ def pythonpath_manip():
 
 pythonpath_manip()
 
+if sys.version_info[:2] > (3, 11):
+    from importlib.metadata import (
+        entry_points,
+        files,
+    )
+else:
+    # BACK COMPAT: importlib_metadata
+    #   importlib.metadata was added in Python 3.8. The required interfaces
+    #   were completed by 3.12. For lower versions we must use the
+    #   importlib_metadata backport.
+    # FROM: Python 3.7
+    # TO: Python: 3.12
+    from importlib_metadata import (
+        entry_points,
+        files,
+    )
+
 import argparse
 from contextlib import contextmanager
 from typing import Iterator, NoReturn, Optional, Tuple
 
 from ansimarkup import parse as cparse
-import pkg_resources
 
 from cylc.flow import __version__, iter_entry_points
 from cylc.flow.option_parsers import (
@@ -306,9 +322,9 @@ def execute_cmd(cmd: str, *args: str) -> NoReturn:
         args: Command line arguments to pass to that command.
 
     """
-    entry_point: pkg_resources.EntryPoint = COMMANDS[cmd]
+    entry_point = COMMANDS[cmd]
     try:
-        entry_point.resolve()(*args)
+        entry_point.load()(*args)
     except ModuleNotFoundError as exc:
         msg = handle_missing_dependency(entry_point, exc)
         print(msg, file=sys.stderr)
@@ -401,7 +417,7 @@ def iter_commands() -> Iterator[Tuple[str, Optional[str], Optional[str]]]:
     """
     for cmd, entry_point in sorted(COMMANDS.items()):
         try:
-            module = __import__(entry_point.module_name, fromlist=[''])
+            module = __import__(entry_point.module, fromlist=[''])
         except ModuleNotFoundError as exc:
             handle_missing_dependency(entry_point, exc)
             continue
@@ -417,18 +433,10 @@ def print_id_help():
 
 
 def print_license() -> None:
-    try:
-        from importlib.metadata import files
-    except ImportError:
-        # BACK COMPAT: importlib_metadata
-        #   importlib.metadata was added in Python 3.8
-        # FROM: Python 3.7
-        # TO: Python: 3.8
-        from importlib_metadata import files  # type: ignore[no-redef]
-    license_file = next(filter(
-        lambda f: f.name == 'COPYING', files('cylc-flow')
-    ))
-    print(license_file.read_text())
+    for file in files('cylc-flow') or []:
+        if file.name == 'COPYING':
+            print(file.read_text())
+            return
 
 
 def print_command_list(commands=None, indent=0):
@@ -467,54 +475,57 @@ def cli_version(long_fmt=False):
     """Wrapper for get_version."""
     print(get_version(long_fmt))
     if long_fmt:
-        print(list_plugins())
+        print(cparse(list_plugins()))
     sys.exit(0)
 
 
 def list_plugins():
-    entry_point_names = [
-        entry_point_name
-        for entry_point_name
-        in pkg_resources.get_entry_map('cylc-flow').keys()
-        if entry_point_name.startswith('cylc.')
-    ]
-
-    entry_point_groups = {
-        entry_point_name: [
-            entry_point
-            for entry_point
-            in iter_entry_points(entry_point_name)
-            if not entry_point.module_name.startswith('cylc.flow')
-        ]
-        for entry_point_name in entry_point_names
-    }
-
-    dists = {
-        entry_point.dist
-        for entry_points in entry_point_groups.values()
-        for entry_point in entry_points
-    }
-
-    lines = []
-    if dists:
-        lines.append('\nPlugins:')
-        maxlen1 = max(len(dist.project_name) for dist in dists) + 2
-        maxlen2 = max(len(dist.version) for dist in dists) + 2
-        for dist in dists:
-            lines.append(
-                f'  {dist.project_name.ljust(maxlen1)}'
-                f' {dist.version.ljust(maxlen2)}'
-                f' {dist.module_path}'
-            )
-
-        lines.append('\nEntry Points:')
-        for entry_point_name, entry_points in entry_point_groups.items():
-            if entry_points:
-                lines.append(f'  {entry_point_name}:')
-                for entry_point in entry_points:
-                    lines.append(f'    {entry_point}')
-
-    return '\n'.join(lines)
+    from cylc.flow.terminal import DIM, format_grid
+    # go through all Cylc entry points
+    _dists = set()
+    __entry_points = {}
+    for entry_point in entry_points():
+        if (
+            # all Cylc entry points are under the "cylc" namespace
+            entry_point.group.startswith('cylc.')
+            # don't list cylc-flow entry-points (i.e. built-ins)
+            and not entry_point.value.startswith('cylc.flow')
+        ):
+            _dists.add(entry_point.dist)
+            __entry_points.setdefault(
+                entry_point.group,
+                [],
+            ).append(entry_point)
+
+    # list all the distriutions which provide Cylc entry points
+    _plugins = []
+    for dist in _dists:
+        _plugins.append((
+            '',
+            f'<light-blue>{dist.name}</light-blue>',
+            dist.version,
+            f'<{DIM}>{dist.locate_file("__init__.py").parent}</{DIM}>',
+        ))
+
+    # list all of the entry points by "group" (e.g. "cylc.command")
+    _entry_points = []
+    for group, points in sorted(__entry_points.items()):
+        _entry_points.append((f'  {group}:', '', ''))
+        for entry_point in points:
+            _entry_points.append((
+                f'    {entry_point.name}',
+                f'<light-blue>{entry_point.dist.name}</light-blue>',
+                f'<{DIM}>{entry_point.value}</{DIM}>',
+            ))
+
+    return '\n'.join((
+        '\n<bold>Plugins:</bold>',
+        *format_grid(_plugins),
+        '\n<bold>Entry Points:</bold>',
+        *format_grid(
+            _entry_points
+        ),
+    ))
 
 
 @contextmanager
@@ -686,7 +697,7 @@ def main():
 
 
 def handle_missing_dependency(
-    entry_point: pkg_resources.EntryPoint,
+    entry_point,
     err: ModuleNotFoundError
 ) -> str:
     """Return a suitable error message for a missing optional dependency.
@@ -698,12 +709,8 @@ def handle_missing_dependency(
 
     Re-raises the given ModuleNotFoundError if it is unexpected.
     """
-    try:
-        # Check for missing optional dependencies
-        entry_point.require()
-    except pkg_resources.DistributionNotFound as exc:
-        # Confirmed missing optional dependencies
-        return f"cylc {entry_point.name}: {exc}"
-    else:
-        # Error not due to missing optional dependencies; this is unexpected
-        raise err
+    msg = f'"cylc {entry_point.name}" requires "{entry_point.dist.name}'
+    if entry_point.extras:
+        msg += f'[{",".join(entry_point.extras)}]'
+    msg += f'"\n\n{err.__class__.__name__}: {err}'
+    return msg

cylc/flow/scripts/install.py

@@ -301,7 +301,7 @@ def install(
         'cylc.pre_configure'
     ):
         try:
-            entry_point.resolve()(srcdir=source, opts=opts)
+            entry_point.load()(srcdir=source, opts=opts)
         except Exception as exc:
             # NOTE: except Exception (purposefully vague)
             # this is to separate plugin from core Cylc errors
@@ -329,7 +329,7 @@ def install(
         'cylc.post_install'
     ):
         try:
-            entry_point.resolve()(
+            entry_point.load()(
                 srcdir=source_dir,
                 opts=opts,
                 rundir=str(rundir)