gh-37857: Replace special handling of optional extensions (bliss, cox…

…eter3, ....)

    
<!-- ^ Please provide a concise and informative title. -->
<!-- ^ Don't put issue numbers in the title, do this in the PR
description below. -->
<!-- ^ For example, instead of "Fixes #12345" use "Introduce new method
to calculate 1 + 2". -->
<!-- v Describe your changes below in detail. -->
<!-- v Why is this change required? What problem does it solve? -->
<!-- v If this PR resolves an open issue, please link to it here. For
example, "Fixes #12345". -->

We relieve the distribution **sagemath-standard** (in both versions -
`SAGE_ROOT/pkgs/sagemath-standard` and `SAGE_ROOT/src`) from the duty to
build "optional extensions" based on what Sage packages are installed.

The installation is now done uniformly using the modularized
distributions **sagemath-bliss**, **sagemath-coxeter3** etc.

We introduce the missing features `coxeter3` and `sirocco` so that the
doctester does not have to rely on the sage-the-distro installation
records any more.

The wheels of the distributions now build correctly even when not going
through building an sdist first, which previously was required to apply
MANIFEST-based filtering. This is achieved using the new
`sage_setup.command.build_py`.

User-visible change:
- To install these options, use `./configure --enable-sagemath_bliss`
before building, or use `./sage -i sagemath_bliss` or `make
sagemath_bliss`.

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->

- [x] The title is concise and informative.
- [x] The description explains in detail what this PR is about.
- [ ] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation and checked the documentation
preview.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on. For example,
-->
<!-- - #12345: short description why this is a dependency -->
<!-- - #34567: ... -->

- Depends on #37737 (merged here)
- Depends on #37973 (merged here)
    
URL: #37857
Reported by: Matthias Köppe
Reviewer(s): François Bissey

build/pkgs/sagelib/spkg-install.in

@@ -43,14 +43,10 @@ unset SAGE_PKG_CONFIG_PATH
 
 SITEPACKAGESDIR=$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')
 
-export SAGE_OPTIONAL_PACKAGES_WITH_EXTENSIONS=""
-
+# Make sure that an installed old version of sagelib in which sage is an ordinary package
+# does not shadow the namespace package sage during the build.
+(cd "$SITEPACKAGESDIR" && rm -f sage/__init__.py)
 if [ "$SAGE_EDITABLE" = yes ]; then
-    # In an incremental build, we may need to uninstall old versions installed by distutils
-    # under the old distribution name "sage" (before #30912, which switched to setuptools
-    # and renamed the distribution to "sagemath-standard").  There is no clean way to uninstall
-    # them, so we just use rm.
-    (cd "$SITEPACKAGESDIR" && rm -rf sage sage-[1-9]*.egg-info sage-[1-9]*.dist-info)
     # Until https://github.com/sagemath/sage/issues/34209 switches us to PEP 660 editable wheels
     export SETUPTOOLS_ENABLE_FEATURES=legacy-editable
     sdh_pip_editable_install .
@@ -61,12 +57,8 @@ if [ "$SAGE_EDITABLE" = yes ]; then
     fi
 else
     # Now implied: "$SAGE_WHEELS" = yes
-    # Make sure that an installed old version of sagelib in which sage is an ordinary package
-    # does not shadow the namespace package sage during the build.
-    (cd "$SITEPACKAGESDIR" && rm -f sage/__init__.py)
-    # Likewise, we should remove the egg-link that may have been installed previously.
+    # We should remove the egg-link that may have been installed previously.
     (cd "$SITEPACKAGESDIR" && rm -f sagemath-standard.egg-link)
-
     # Use --no-build-isolation to avoid rebuilds because of dependencies:
     # Compiling sage/interfaces/sagespawn.pyx because it depends on /private/var/folders/38/wnh4gf1552g_crsjnv2vmmww0000gp/T/pip-build-env-609n5985/overlay/lib/python3.10/site-packages/Cython/Includes/posix/unistd.pxd
     sdh_pip_install --no-build-isolation .

build/pkgs/sagemath_bliss/spkg-install.in

@@ -3,13 +3,7 @@ cd src
 export PIP_NO_INDEX=true
 export PIP_FIND_LINKS="file://$SAGE_SPKG_WHEELS"
 
-if [ "$SAGE_EDITABLE" = yes ]; then
-    # SAGE_ROOT/src/setup.py installs everything, nothing to do...
-    if [ "$SAGE_WHEELS" = yes ]; then
-        # ... except we build the wheel if requested
-        sdh_setup_bdist_wheel && sdh_store_wheel .
-    fi
-else
-    # Modularized install via wheels. Now implied: "$SAGE_WHEELS" = yes
-    sdh_pip_install .
-fi
+# Modularized install via wheels
+# --no-build-isolation so that declared build dependencies,
+# in particular sagemath-environment do not have to be present as wheels.
+sdh_pip_install --no-build-isolation .

build/pkgs/sagemath_meataxe/dependencies

@@ -1 +1 @@
- meataxe | $(PYTHON_TOOLCHAIN) sage_setup sagemath_environment cython pkgconfig $(PYTHON)
+ meataxe cysignals | $(PYTHON_TOOLCHAIN) sage_setup sagemath_environment cython pkgconfig $(PYTHON)

pkgs/sagemath-bliss/setup.py

@@ -42,10 +42,12 @@
 
     from sage_setup.command.sage_build_cython import sage_build_cython
     from sage_setup.command.sage_build_ext import sage_build_ext
+    from sage_setup.command.sage_build_py import sage_build_py
     sage_build_cython.built_distributions = ['sagemath-bliss']
 
     cmdclass = dict(build_cython=sage_build_cython,
-                    build_ext=sage_build_ext)
+                    build_ext=sage_build_ext,
+                    build_py=sage_build_py)
 
 from sage_setup.find import find_python_sources
 python_packages, python_modules, cython_modules = find_python_sources(

pkgs/sagemath-coxeter3/setup.py

@@ -42,10 +42,12 @@
 
     from sage_setup.command.sage_build_cython import sage_build_cython
     from sage_setup.command.sage_build_ext import sage_build_ext
+    from sage_setup.command.sage_build_py import sage_build_py
     sage_build_cython.built_distributions = ['sagemath-coxeter3']
 
     cmdclass = dict(build_cython=sage_build_cython,
-                    build_ext=sage_build_ext)
+                    build_ext=sage_build_ext,
+                    build_py=sage_build_py)
 
 from sage_setup.find import find_python_sources
 python_packages, python_modules, cython_modules = find_python_sources(

pkgs/sagemath-mcqd/setup.py

@@ -42,10 +42,12 @@
 
     from sage_setup.command.sage_build_cython import sage_build_cython
     from sage_setup.command.sage_build_ext import sage_build_ext
+    from sage_setup.command.sage_build_py import sage_build_py
     sage_build_cython.built_distributions = ['sagemath-mcqd']
 
     cmdclass = dict(build_cython=sage_build_cython,
-                    build_ext=sage_build_ext)
+                    build_ext=sage_build_ext,
+                    build_py=sage_build_py)
 
 from sage_setup.find import find_python_sources
 python_packages, python_modules, cython_modules = find_python_sources(

pkgs/sagemath-meataxe/setup.py

@@ -42,10 +42,12 @@
 
     from sage_setup.command.sage_build_cython import sage_build_cython
     from sage_setup.command.sage_build_ext import sage_build_ext
+    from sage_setup.command.sage_build_py import sage_build_py
     sage_build_cython.built_distributions = ['sagemath-meataxe']
 
     cmdclass = dict(build_cython=sage_build_cython,
-                    build_ext=sage_build_ext)
+                    build_ext=sage_build_ext,
+                    build_py=sage_build_py)
 
 from sage_setup.find import find_python_sources
 python_packages, python_modules, cython_modules = find_python_sources(

pkgs/sagemath-sirocco/setup.py

@@ -42,10 +42,12 @@
 
     from sage_setup.command.sage_build_cython import sage_build_cython
     from sage_setup.command.sage_build_ext import sage_build_ext
+    from sage_setup.command.sage_build_py import sage_build_py
     sage_build_cython.built_distributions = ['sagemath-sirocco']
 
     cmdclass = dict(build_cython=sage_build_cython,
-                    build_ext=sage_build_ext)
+                    build_ext=sage_build_ext,
+                    build_py=sage_build_py)
 
 from sage_setup.find import find_python_sources
 python_packages, python_modules, cython_modules = find_python_sources(

pkgs/sagemath-standard/setup.py

@@ -72,12 +72,7 @@
 # TODO: This should be quiet by default
 print("Discovering Python/Cython source code....")
 t = time.time()
-from sage.misc.package import is_package_installed_and_updated
 distributions = ['']
-optional_packages_with_extensions = os.environ.get('SAGE_OPTIONAL_PACKAGES_WITH_EXTENSIONS', '').split(',')
-distributions += ['sagemath-{}'.format(pkg)
-                  for pkg in optional_packages_with_extensions
-                  if is_package_installed_and_updated(pkg)]
 log.warn('distributions = {0}'.format(distributions))
 from sage_setup.find import find_python_sources
 python_packages, python_modules, cython_modules = find_python_sources(

pkgs/sagemath-tdlib/setup.py

@@ -42,10 +42,12 @@
 
     from sage_setup.command.sage_build_cython import sage_build_cython
     from sage_setup.command.sage_build_ext import sage_build_ext
+    from sage_setup.command.sage_build_py import sage_build_py
     sage_build_cython.built_distributions = ['sagemath-tdlib']
 
     cmdclass = dict(build_cython=sage_build_cython,
-                    build_ext=sage_build_ext)
+                    build_ext=sage_build_ext,
+                    build_py=sage_build_py)
 
 from sage_setup.find import find_python_sources
 python_packages, python_modules, cython_modules = find_python_sources(

src/MANIFEST.in

@@ -56,7 +56,7 @@ exclude sage/graphs/mcqd.p*
 exclude sage/libs/meataxe.p*
 exclude sage/libs/sirocco.p*
 exclude sage/matrix/matrix_gfpn_dense.p*
-exclude sage/graphs/graph_decomposition/tdlib.p*
+exclude sage/graphs/graph_decompositions/tdlib.p*
 
 # Exclude all__*.py files belonging to distributions related to optional packages
 global-exclude all__sagemath_bliss.py

src/sage/all__sagemath_bliss.py

@@ -0,0 +1 @@
+# sage_setup: distribution = sagemath-bliss

src/sage/all__sagemath_coxeter3.py

@@ -0,0 +1 @@
+# sage_setup: distribution = sagemath-coxeter3

src/sage/all__sagemath_mcqd.py

@@ -0,0 +1 @@
+# sage_setup: distribution = sagemath-mcqd

src/sage/all__sagemath_meataxe.py

@@ -0,0 +1 @@
+# sage_setup: distribution = sagemath-meataxe

src/sage/all__sagemath_sirocco.py

@@ -0,0 +1 @@
+# sage_setup: distribution = sagemath-sirocco

src/sage/all__sagemath_tdlib.py

@@ -0,0 +1 @@
+# sage_setup: distribution = sagemath-tdlib

src/sage/doctest/control.py

@@ -475,6 +475,9 @@ def __init__(self, options, args):
                     for pkg in list_packages('optional', local=True).values():
                         if pkg.name in options.hide:
                             continue
+                        # Skip features for which we have a more specific runtime feature test.
+                        if pkg.name in ['bliss', 'coxeter3', 'mcqd', 'meataxe', 'sirocco', 'tdlib']:
+                            continue
                         if pkg.is_installed() and pkg.installed_version == pkg.remote_version:
                             options.optional.add(pkg.name)
 
@@ -1447,7 +1450,7 @@ def run(self):
             sage: with open(filename, 'w') as f:
             ....:     f.write(test_hide)
             ....:     f.close()
-            729
+            714
             sage: DF = DocTestDefaults(hide='buckygen,all')
             sage: DC = DocTestController(DF, [filename])
             sage: DC.run()
@@ -1670,22 +1673,22 @@ def stringify(x):
  ...
 FeatureNotPresentError: buckygen is not available.
 ...
-{prompt}: next(graphs.fullerenes(20))   # optional buckygen
+{prompt}: next(graphs.fullerenes(20))   # optional - buckygen
 Graph on 20 vertices
 
 {prompt}: len(list(graphs.fusenes(2)))
 Traceback (most recent call last):
  ...
 FeatureNotPresentError: benzene is not available.
 ...
-{prompt}: len(list(graphs.fusenes(2)))  # optional benzene
+{prompt}: len(list(graphs.fusenes(2)))  # optional - benzene
 1
 {prompt}: from sage.matrix.matrix_space import get_matrix_class
 {prompt}: get_matrix_class(GF(25,'x'), 4, 4, False, 'meataxe')
 Failed lazy import:
-sage.matrix.matrix_gfpn_dense is not available.
+meataxe is not available.
 ...
-{prompt}: get_matrix_class(GF(25,'x'), 4, 4, False, 'meataxe') # optional meataxe
+{prompt}: get_matrix_class(GF(25,'x'), 4, 4, False, 'meataxe')  # optional - meataxe
 <class 'sage.matrix.matrix_gfpn_dense.Matrix_gfpn_dense'>
 {quotmark}
 """.format(quotmark='"""', prompt='sage')  # using prompt to hide these lines from _test_enough_doctests

src/sage/features/bliss.py

@@ -75,10 +75,8 @@ def __init__(self):
             sage: Bliss()
             Feature('bliss')
         """
-        # Currently part of sagemath_standard, conditionally built.
-        # Will be changed to spkg='sagemath_bliss' later
         JoinFeature.__init__(self, "bliss",
-                             [PythonModule("sage.graphs.bliss", spkg="bliss",
+                             [PythonModule("sage.graphs.bliss", spkg="sagemath_bliss",
                                            url="http://www.tcs.hut.fi/Software/bliss/")])
 
 

src/sage/features/coxeter3.py

@@ -0,0 +1,45 @@
+# sage_setup: distribution = sagemath-environment
+r"""
+Features for testing the presence of ``coxeter3``
+"""
+
+# *****************************************************************************
+#       Copyright (C) 2016      Julian Rüth
+#                     2018      Jeroen Demeyer
+#                     2021-2024 Matthias Koeppe
+#
+#  Distributed under the terms of the GNU General Public License (GPL)
+#  as published by the Free Software Foundation; either version 2 of
+#  the License, or (at your option) any later version.
+#                  https://www.gnu.org/licenses/
+# *****************************************************************************
+
+from . import PythonModule
+from .join_feature import JoinFeature
+
+
+class Coxeter3(JoinFeature):
+    r"""
+    A :class:`~sage.features.Feature` which describes whether the :mod:`sage.libs.coxeter3`
+    module is available in this installation of Sage.
+
+    EXAMPLES::
+
+        sage: from sage.features.coxeter3 import Coxeter3
+        sage: Coxeter3().require()  # optional - coxeter3
+    """
+    def __init__(self):
+        r"""
+        TESTS::
+
+            sage: from sage.features.coxeter3 import Coxeter3
+            sage: Coxeter3()
+            Feature('coxeter3')
+        """
+        JoinFeature.__init__(self, "coxeter3",
+                             [PythonModule("sage.libs.coxeter3.coxeter",
+                                           spkg="sagemath_coxeter3")])
+
+
+def all_features():
+    return [Coxeter3()]

src/sage/features/mcqd.py

@@ -35,10 +35,9 @@ def __init__(self):
             sage: isinstance(Mcqd(), Mcqd)
             True
         """
-        # Currently part of sagemath_standard, conditionally built.
-        # Will be changed to spkg='sagemath_mcqd' later
         JoinFeature.__init__(self, 'mcqd',
-                             [PythonModule('sage.graphs.mcqd', spkg='mcqd')])
+                             [PythonModule('sage.graphs.mcqd',
+                                           spkg='sagemath_mcqd')])
 
 
 def all_features():

src/sage/features/meataxe.py

@@ -36,10 +36,9 @@ def __init__(self):
             sage: isinstance(Meataxe(), Meataxe)
             True
         """
-        # Currently part of sagemath_standard, conditionally built.
-        # Will be changed to spkg='sagemath_meataxe' later
         JoinFeature.__init__(self, 'meataxe',
-                             [PythonModule('sage.matrix.matrix_gfpn_dense', spkg='meataxe')])
+                             [PythonModule('sage.matrix.matrix_gfpn_dense',
+                                           spkg='sagemath_meataxe')])
 
 
 def all_features():

src/sage/features/sirocco.py

@@ -0,0 +1,45 @@
+# sage_setup: distribution = sagemath-environment
+r"""
+Features for testing the presence of ``sirocco``
+"""
+
+# *****************************************************************************
+#       Copyright (C) 2016      Julian Rüth
+#                     2018      Jeroen Demeyer
+#                     2021-2024 Matthias Koeppe
+#
+#  Distributed under the terms of the GNU General Public License (GPL)
+#  as published by the Free Software Foundation; either version 2 of
+#  the License, or (at your option) any later version.
+#                  https://www.gnu.org/licenses/
+# *****************************************************************************
+
+from . import PythonModule
+from .join_feature import JoinFeature
+
+
+class Sirocco(JoinFeature):
+    r"""
+    A :class:`~sage.features.Feature` which describes whether the :mod:`sage.libs.sirocco`
+    module is available in this installation of Sage.
+
+    EXAMPLES::
+
+        sage: from sage.features.sirocco import Sirocco
+        sage: Sirocco().require()  # optional - sirocco
+    """
+    def __init__(self):
+        r"""
+        TESTS::
+
+            sage: from sage.features.sirocco import Sirocco
+            sage: Sirocco()
+            Feature('sirocco')
+        """
+        JoinFeature.__init__(self, "sirocco",
+                             [PythonModule("sage.libs.sirocco",
+                                           spkg="sagemath_sirocco")])
+
+
+def all_features():
+    return [Sirocco()]

src/sage/features/tdlib.py

@@ -28,10 +28,9 @@ def __init__(self):
             sage: isinstance(Tdlib(), Tdlib)
             True
         """
-        # Currently part of sagemath_standard, conditionally built.
-        # Will be changed to spkg='sagemath_tdlib' later
         JoinFeature.__init__(self, 'tdlib',
-                             [PythonModule('sage.graphs.graph_decompositions.tdlib', spkg='tdlib')])
+                             [PythonModule('sage.graphs.graph_decompositions.tdlib',
+                                           spkg='sagemath_tdlib')])
 
 
 def all_features():

src/sage/graphs/all__sagemath_bliss.py

@@ -0,0 +1 @@
+# sage_setup: distribution = sagemath-bliss

src/sage/graphs/all__sagemath_mcqd.py

@@ -0,0 +1 @@
+# sage_setup: distribution = sagemath-mcqd

src/sage/graphs/all__sagemath_tdlib.py

@@ -0,0 +1 @@
+# sage_setup: distribution = sagemath-tdlib

src/sage/graphs/graph.py

@@ -426,9 +426,9 @@
 from sage.parallel.decorate import parallel
 
 from sage.misc.lazy_import import lazy_import, LazyImport
-from sage.features import PythonModule
+from sage.features.mcqd import Mcqd
 lazy_import('sage.graphs.mcqd', ['mcqd'],
-            feature=PythonModule('sage.graphs.mcqd', spkg='mcqd'))
+            feature=Mcqd())
 
 
 class Graph(GenericGraph):

src/sage/graphs/graph_decompositions/all__sagemath_tdlib.py

@@ -0,0 +1 @@
+# sage_setup: distribution = sagemath-tdlib

src/sage/graphs/graph_decompositions/tree_decomposition.pyx

@@ -455,7 +455,7 @@ def treewidth(g, k=None, kmin=None, certificate=False, algorithm=None, nice=Fals
       tree-decomposition itself.
 
     - ``algorithm`` -- whether to use ``"sage"`` or ``"tdlib"`` (requires the
-      installation of the 'tdlib' package). The default behaviour is to use
+      installation of the :ref:`spkg_sagemath_tdlib` package). The default behaviour is to use
       'tdlib' if it is available, and Sage's own algorithm when it is not.
 
     - ``nice`` -- boolean (default: ``False``); whether or not to return the
@@ -671,8 +671,8 @@ def treewidth(g, k=None, kmin=None, certificate=False, algorithm=None, nice=Fals
     if algorithm == 'tdlib':
         if not tdlib_found:
             from sage.features import FeatureNotPresentError
-            raise FeatureNotPresentError(PythonModule('sage.graphs.graph_decompositions.tdlib',
-                                                      spkg='tdlib'))
+            from sage.features.tdlib import Tdlib
+            raise FeatureNotPresentError(Tdlib())
 
         tree_decomp = tdlib.treedecomposition_exact(g, -1 if k is None else k)
         width = tdlib.get_width(tree_decomp)