Merge branch 't/30940/src_bin_sage_list_packages__make_it_work_if_sag…

…e_root_is_not_available' into t/29124/script-packages-prereq-toolchain-bootstrap

src/sage/misc/package.py

@@ -120,22 +120,35 @@ def pip_remote_version(pkg, pypi_url=DEFAULT_PYPI, ignore_URLError=False):
     stable_releases = [v for v in info['releases'] if 'a' not in v and 'b' not in v]
     return max(stable_releases)
 
-def pip_installed_packages():
+def pip_installed_packages(normalization=None):
     r"""
     Return a dictionary `name->version` of installed pip packages.
 
     This command returns *all* pip-installed packages. Not only Sage packages.
 
+    INPUT:
+
+    - ``normalization`` -- (optional, default: ``None``) according to which rule to
+      normalize the package name, either ``None`` (as is) or ``'spkg'`` (format
+      as in the Sage distribution in ``build/pkgs/``), i.e., lowercased and
+      dots and dashes replaced by underscores.
+
     EXAMPLES::
 
         sage: from sage.misc.package import pip_installed_packages
         sage: d = pip_installed_packages()  # optional - build
         sage: 'scipy' in d  # optional - build
         True
         sage: d['scipy']  # optional - build
-        u'...'
+        '...'
         sage: d['beautifulsoup4']   # optional - build beautifulsoup4
-        u'...'
+        '...'
+        sage: d['prompt-toolkit']   # optional - build
+        '...'
+        sage: d = pip_installed_packages(normalization='spkg')  # optional - build
+        sage: d['prompt_toolkit']   # optional - build
+        '...'
+
     """
     with open(os.devnull, 'w') as devnull:
         proc = subprocess.Popen(
@@ -144,8 +157,15 @@ def pip_installed_packages():
             stderr=devnull,
         )
         stdout = proc.communicate()[0].decode()
+        def normalize(name):
+            if normalization is None:
+                return name
+            elif normalization == 'spkg':
+                return name.lower().replace('-', '_').replace('.', '_')
+            else:
+                raise NotImplementedError(f'normalization {normalization} is not implemented')
         try:
-            return {package['name'].lower(): package['version']
+            return {normalize(package['name']): package['version']
                     for package in json.loads(stdout)}
         except json.decoder.JSONDecodeError:
             # Something went wrong while parsing the output from pip.
@@ -180,7 +200,8 @@ def list_packages(*pkg_types, **opts):
       ``'pip'`` type)
 
     - ``exclude_pip`` -- (optional, default: ``False``) if set to ``True``, then
-      pip packages are not considered.
+      pip packages are not considered.  This is the same as removing ``'pip'``
+      from ``pkg_sources``.
 
     - ``ignore_URLError`` -- (default: ``False``) if set to ``True``, then
       connection errors will be ignored
@@ -227,22 +248,27 @@ def list_packages(*pkg_types, **opts):
     local = opts.pop('local', False)
     ignore_URLError = opts.pop('ignore_URLError', False)
     exclude_pip = opts.pop('exclude_pip', False)
+    if exclude_pip:
+        pkg_sources = [s for s in pkg_sources if s != 'pip']
     if opts:
         raise ValueError("{} are not valid options".format(sorted(opts)))
 
-    installed = installed_packages(exclude_pip)
+    pkgs = {p: {'name': p, 'installed_version': v, 'installed': True,
+                'remote_version': None, 'source': None}
+            for p, v in installed_packages('pip' not in pkg_sources).items()}
 
+    lp = []
     SAGE_PKGS = sage.env.SAGE_PKGS
     if not SAGE_PKGS:
-        return {}
+        return pkgs
 
     try:
         lp = os.listdir(SAGE_PKGS)
     except FileNotFoundError:
-        return {}
+        return pkgs
 
-    pkgs = {}
     for p in lp:
+
         try:
             f = open(os.path.join(SAGE_PKGS, p, "type"))
         except IOError:
@@ -252,25 +278,28 @@ def list_packages(*pkg_types, **opts):
         with f:
             typ = f.read().strip()
 
-        if typ not in pkg_types:
-            continue
-
         if os.path.isfile(os.path.join(SAGE_PKGS, p, "requirements.txt")):
             src = 'pip'
         elif os.path.isfile(os.path.join(SAGE_PKGS, p, "checksums.ini")):
             src = 'normal'
         else:
             src = 'script'
 
-        if src not in pkg_sources:
+        pkg = pkgs.get(p, dict())
+        pkgs[p] = pkg
+
+        if typ not in pkg_types or src not in pkg_sources:
+            del pkgs[p]
             continue
 
-        pkg = {'name': p, 'type': typ, 'source': src, 'installed_version': installed.get(p)}
-        pkg['installed'] = pkg['installed_version'] is not None
+        pkg.update({'name': p, 'type': typ, 'source': src})
+        if pkg.get('installed_version', None):
+            pkg['installed'] = True
+        else:
+            pkg['installed'] = False
+            pkg['installed_version'] = None
 
         if pkg['source'] == 'pip':
-            if exclude_pip:
-                continue
             if not local:
                 pkg['remote_version'] = pip_remote_version(p, ignore_URLError=ignore_URLError)
             else:
@@ -281,12 +310,9 @@ def list_packages(*pkg_types, **opts):
             package_filename = os.path.join(SAGE_PKGS, p, "package-version.txt")
             with open(package_filename) as f:
                 pkg['remote_version'] = f.read().strip()
-            pkg['installed_version'] = installed.get(p)
         else:
             pkg['remote_version'] = 'none'
 
-        pkgs[p] = pkg
-
     return pkgs
 
 
@@ -314,13 +340,14 @@ def installed_packages(exclude_pip=True):
     """
     installed = {}
     if not exclude_pip:
-        installed.update(pip_installed_packages())
+        installed.update(pip_installed_packages(normalization='spkg'))
     # Sage packages should override pip packages (Trac #23997)
     SAGE_SPKG_INST = sage.env.SAGE_SPKG_INST
     if SAGE_SPKG_INST:
         try:
             lp = os.listdir(SAGE_SPKG_INST)
-            installed.update(pkgname_split(pkgname) for pkgname in lp)
+            installed.update(pkgname_split(pkgname) for pkgname in lp
+                             if not pkgname.startswith('.'))
         except FileNotFoundError:
             pass
     return installed
@@ -351,12 +378,6 @@ def is_package_installed(package, exclude_pip=True):
     Otherwise, installing "pillow" would cause this function to think
     that "pil" is installed, for example.
 
-    Check that the option ``exclude_pip`` is turned on by default::
-
-        sage: from sage.misc.package import list_packages
-        sage: for pkg in list_packages(pkg_sources=('pip'), local=True):  # optional - build
-        ....:     assert not is_package_installed(pkg), "pip package is installed: {}".format(pkg)
-
     .. NOTE::
 
         Do not use this function to check whether you can use a feature from an
@@ -366,7 +387,7 @@ def is_package_installed(package, exclude_pip=True):
         Use the framework provided by :mod:`sage.features` to check
         whether a library is installed and functional.
     """
-    return any(p.split('-')[0] == package for p in installed_packages(exclude_pip))
+    return any(p == package for p in installed_packages(exclude_pip))
 
 
 def package_versions(package_type, local=False):
@@ -424,8 +445,8 @@ def standard_packages():
         sage: from sage.misc.package import standard_packages
         sage: installed, not_installed = standard_packages()  # optional - build
         doctest:...: DeprecationWarning: ...
-        sage: installed[0], installed[-1]  # optional - build
-        ('alabaster', 'zn_poly')
+        sage: 'numpy' in installed                            # optional - build
+        True
     """
     from sage.misc.superseded import deprecation
     deprecation(30747,