Trac #33812: Refactor distributions sagemath-{objects,categories} thr…

…ough sagemath-{environment,repl}

Currently '''sagemath-categories''' distributes a superset of
'''sagemath-objects''', and both distributions include a copy of the
doctester for their doctesting. We remove this duplication by declaring
install-requires and extras-require.

The dependencies are illustrated in the updated developer's guide -
https://e019add9a8540b8213dfc563e8e4bda04777282c--sagemath-
tobias.netlify.app/developer/packaging_sage_library.html#hierarchy-of-
distribution-packages

As #33817 is a dependency, this ticket can be tested by running
{{{
./bootstrap && make SAGE_CHECK=yes pypi-wheels
}}}
This test is also run as part of the "Build&Test" workflow.

URL: https://trac.sagemath.org/33812
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): Kwankyu Lee

build/make/Makefile.in

@@ -430,7 +430,7 @@ define SET_SAGE_CHECK
 $(eval SAGE_CHECK_$(1) := $(2))
 endef
 # Set defaults
-$(foreach pkgname, $(NORMAL_PACKAGES),\
+$(foreach pkgname, $(NORMAL_PACKAGES) $(SCRIPT_PACKAGES),\
 	$(eval $(call SET_SAGE_CHECK,$(pkgname),$(SAGE_CHECK))))
 
 # Parsing the SAGE_CHECK_PACKAGES variable:
@@ -460,7 +460,7 @@ $(foreach clause, $(SAGE_CHECK_PACKAGES_sep),					\
 	       $(eval $(call SET_SAGE_CHECK,$(subst ?,,$(clause)),warn)),	\
 	       $(eval $(call SET_SAGE_CHECK,$(clause),yes)))))
 debug-check:
-	@echo $(foreach pkgname, $(NORMAL_PACKAGES), SAGE_CHECK_$(pkgname) = $(SAGE_CHECK_$(pkgname)))
+	@echo $(foreach pkgname, $(NORMAL_PACKAGES) $(SCRIPT_PACKAGES), SAGE_CHECK_$(pkgname) = $(SAGE_CHECK_$(pkgname)))
 
 
 #==============================================================================
@@ -676,7 +676,7 @@ $(1)-$(4)-no-deps:
 		. '$$(SAGE_ROOT)/build/bin/sage-build-env' && \
 		SAGE_SPKG_WHEELS=$$($(4))/var/lib/sage/wheels \
 		SAGE_INST_LOCAL=$$($(4)) \
-		sage-logger -p '$$(SAGE_ROOT)/build/pkgs/$(1)/spkg-install' '$$(SAGE_LOGS)/$(1)-$(2).log' && \
+		sage-logger -p 'SAGE_CHECK=$$(SAGE_CHECK_$(1)) $$(SAGE_ROOT)/build/pkgs/$(1)/spkg-install' '$$(SAGE_LOGS)/$(1)-$(2).log' && \
 	    rm -f "$$($(4))/$(SPKG_INST_RELDIR)/$(1)-*" && \
 	    touch "$$($(4))/$(SPKG_INST_RELDIR)/$(1)-$(2)"; \
 	else ( \

build/pkgs/sagemath_categories/dependencies

@@ -0,0 +1 @@
+$(PYTHON) sagemath_objects | $(PYTHON_TOOLCHAIN) sagemath_environment sage_setup cython pkgconfig python_build

build/pkgs/sagemath_categories/dependencies_check

@@ -0,0 +1 @@
+tox sagemath_repl

build/pkgs/sagemath_environment/dependencies

@@ -0,0 +1 @@
+$(PYTHON) | $(PYTHON_TOOLCHAIN) python_build

build/pkgs/sagemath_environment/dependencies_check

@@ -0,0 +1 @@
+tox

build/pkgs/sagemath_objects/dependencies

@@ -1,4 +1,3 @@
-FORCE $(PYTHON) cysignals gmpy2 ipython | $(PYTHON_TOOLCHAIN) sage_setup cython pkgconfig python_build
+FORCE $(PYTHON) cysignals gmpy2 | $(PYTHON_TOOLCHAIN) sagemath_environment sage_setup cython pkgconfig python_build
 
 # FORCE: Always run the spkg-install script
-# ipython - for the doctester

build/pkgs/sagemath_objects/spkg-install

@@ -19,15 +19,11 @@ export PIP_FIND_LINKS="file://$SAGE_SPKG_WHEELS"
 # (Important because sagemath-objects uses MANIFEST.in for filtering.)
 # Do not install the wheel.
 DIST_DIR="$(mktemp -d)"
-if ! python3 -m build --outdir "$DIST_DIR"/dist .; then
-    # This happens on Debian without python3-venv installed - "ensurepip" is missing
-    echo "Falling back to --no-isolation"
-    python3 -m build --no-isolation --outdir "$DIST_DIR"/dist . || sdh_die "Failure building sdist and wheel"
-fi
+python3 -m build --outdir "$DIST_DIR"/dist . || sdh_die "Failure building sdist and wheel"
 
 wheel=$(cd "$DIST_DIR" && sdh_store_wheel . && echo $wheel)
 ls -l "$wheel"
 
 if [ "$SAGE_CHECK" != no ]; then
-    tox -v -e sagepython-norequirements --installpkg "$wheel"
+    tox -r -v -e sagepython-sagewheels-nopypi-norequirements --installpkg $wheel
 fi

build/pkgs/sagemath_repl/dependencies

@@ -0,0 +1 @@
+$(PYTHON) sagemath_objects sagemath_environment ipython ipywidgets | $(PYTHON_TOOLCHAIN) python_build

build/pkgs/sagemath_repl/dependencies_check

@@ -0,0 +1 @@
+tox

build/pkgs/sagemath_repl/install-requires.txt

@@ -0,0 +1,2 @@
+# This file is updated on every release by the sage-update-version script
+sagemath-repl ~= 9.7b6

pkgs/sagemath-categories/MANIFEST.in.m4

@@ -1,13 +1,36 @@
 dnl MANIFEST.in is generated from this file by SAGE_ROOT/bootstrap via m4.
+prune sage
 
-dnl Include all from sagemath-objects (via m4 include)
-include(`../sagemath_objects/src/MANIFEST.in')
+include VERSION.txt
 
-# Extra in sagemath-categories:
 global-include all__sagemath_categories.py
 graft sage/categories
+# Exclude what is already shipped in sagemath-objects
+exclude sage/categories/action.*
+exclude sage/categories/algebra_functor.*
+exclude sage/categories/basic.*
+exclude sage/categories/cartesian_product.*
+exclude sage/categories/category*.*
+exclude sage/categories/covariant_functorial_construction.*
+exclude sage/categories/functor.*
+exclude sage/categories/homset.*
+exclude sage/categories/homsets.*
+exclude sage/categories/map.*
+exclude sage/categories/morphism.*
+exclude sage/categories/isomorphic_objects.*
+exclude sage/categories/objects.*
+exclude sage/categories/primer.*
+exclude sage/categories/pushout.*
+exclude sage/categories/quotients.*
+exclude sage/categories/realizations.*
+exclude sage/categories/sets_cat.*
+exclude sage/categories/sets_with_partial_maps.*
+exclude sage/categories/subobjects.*
+exclude sage/categories/subquotients.*
+exclude sage/categories/with_realizations.*
+# Exclude to make it a namespace package
 exclude sage/categories/__init__.py
-include sage/misc/prandom.*              # dep of sage/rings/ring
+
 include sage/rings/ideal.*
 include sage/rings/ring.*
 graft sage/typeset                     # dep of sage.categories.tensor
@@ -17,10 +40,12 @@ graft sage/typeset                     # dep of sage.categories.tensor
 
 global-exclude *.c
 global-exclude *.cpp
-include sage/cpython/debugimpl.c
-include sage/misc/inherit_comparison_impl.c
 
 global-exclude __pycache__
 global-exclude *.py[co]
 global-exclude *.bak
 global-exclude *.so
+global-exclude *~
+prune .tox
+prune build
+prune dist

pkgs/sagemath-categories/README.rst

@@ -12,15 +12,27 @@ About SageMath
 
    https://www.sagemath.org
 
-SageMath fully supports all major Linux distributions, recent versions of macOS, and Windows (using Cygwin or Windows Subsystem for Linux).
+SageMath fully supports all major Linux distributions, recent versions of
+macOS, and Windows (using Cygwin or Windows Subsystem for Linux).
 
-The traditional and recommended way to install SageMath is from source via Sage-the-distribution (https://www.sagemath.org/download-source.html).  Sage-the-distribution first builds a large number of open source packages from source (unless it finds suitable versions installed in the system) and then installs the Sage Library (sagelib, implemented in Python and Cython).
+The traditional and recommended way to install SageMath is from source via
+Sage-the-distribution (https://www.sagemath.org/download-source.html).
+Sage-the-distribution first builds a large number of open source packages from
+source (unless it finds suitable versions installed in the system) and then
+installs the Sage Library (sagelib, implemented in Python and Cython).
 
 
 About this experimental pip-installable source distribution
 -----------------------------------------------------------
 
-This pip-installable source distribution `sagemath-categories` is an experimental distribution of a small part of the Sage Library.  Use at your own risk.  It provides a small subset of the modules of the Sage library ("sagelib", `sagemath-standard`).  It is a superset of the `sagemath-objects` (providing Sage objects, the element/parent framework, categories, the coercion system and the related metaclasses), making various additional categories available without introducing dependencies on additional mathematical libraries.
+This pip-installable source distribution `sagemath-categories` is an
+experimental distribution of a small part of the Sage Library.  Use at your own
+risk.  It provides a small subset of the modules of the Sage library
+("sagelib", `sagemath-standard`).  It is a superset of the `sagemath-objects`
+(providing Sage objects, the element/parent framework, categories, the coercion
+system and the related metaclasses), making various additional categories
+available without introducing dependencies on additional mathematical
+libraries.
 
 
 Dependencies

pkgs/sagemath-categories/pyproject.toml.m4

@@ -0,0 +1,14 @@
+[build-system]
+# Minimum requirements for the build system to execute.
+requires = [
+    esyscmd(`sage-get-system-packages install-requires-toml \
+        setuptools     \
+        wheel          \
+        sage_setup     \
+        sagemath_environment \
+        sagemath_objects \
+        cython         \
+        gmpy2          \
+        cysignals      \
+                    ')]
+build-backend = "setuptools.build_meta"

pkgs/sagemath-categories/setup.cfg.m4

@@ -28,21 +28,8 @@ classifiers =
 python_requires = >=3.8, <3.11
 install_requires =
     esyscmd(`sage-get-system-packages install-requires \
-        cython         \
-        pkgconfig      \
-        ipython        \
-        gmpy2          \
-        cysignals      \
+        sagemath_objects \
         | sed "2,\$s/^/    /;"')dnl
 
-scripts =
-    bin/sage
-    bin/sage-env
-    bin/sage-eval
-    bin/sage-fixdoctests
-    bin/sage-ipython
-    bin/sage-python
-    bin/sage-run
-    bin/sage-runtests
-    bin/sage-venv-config
-    bin/sage-version.sh
+[options.extras_require]
+test = sagemath-repl

pkgs/sagemath-categories/tox.ini

@@ -13,17 +13,31 @@ envlist =
 [testenv]
 deps =
     !norequirements: -rrequirements.txt
+    # tox 3.x does not handle extras when using --installpkg. https://github.com/tox-dev/tox/issues/1576
+    sagemath-repl
 
-setenv =
-    # Sage scripts such as sage-runtests like to use $HOME/.sage
-    HOME={envdir}
+extras = test
 
 passenv =
+    # Variables set by .homebrew-build-env
+                             CPATH
+                             LIBRARY_PATH
+                             PKG_CONFIG_PATH
     # Parallel build
                              SAGE_NUM_THREADS
                              SAGE_NUM_THREADS_PARALLEL
-    # SAGE_VENV only for referring to the basepython
-    sagepython:              SAGE_VENV
+    # SAGE_VENV only for referring to the basepython or finding the wheels
+    sagepython, sagewheels:  SAGE_VENV
+    # Location of the wheels
+    sagewheels:              SAGE_SPKG_WHEELS
+
+setenv =
+    # Sage scripts such as sage-runtests like to use $HOME/.sage
+                             HOME={envdir}
+    # We supply pip options by environment variables so that they
+    # apply both to the installation of the dependencies and of the package
+    sagewheels:              PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_VENV:{toxinidir}/../../../../venv}/var/lib/sage/wheels}
+    nopypi:                  PIP_NO_INDEX=true
 
 whitelist_externals =
     bash
@@ -35,10 +49,19 @@ commands =
     # Test that importing sage.categories.all initializes categories
     {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()'
 
-    bash -c 'cd bin && SAGE_SRC=$(python -c "from sage.env import SAGE_SRC; print(SAGE_SRC)") && sage-runtests --environment=sage.all__sagemath_categories --optional=sage $SAGE_SRC/sage/structure  || echo "(lots of doctest failures are expected)"'
+    bash -c 'cd {temp_dir} && SAGE_SRC=$(python -c "from sage.env import SAGE_SRC; print(SAGE_SRC)") && sage-runtests --initial --environment=sage.all__sagemath_categories --optional=sage $SAGE_SRC/sage/structure  || echo "(lots of doctest failures are expected)"'
 
 [testenv:sagepython]
 basepython = {env:SAGE_VENV}/bin/python3
 
+[testenv:sagepython-sagewheels-nopypi]
+basepython = {env:SAGE_VENV}/bin/python3
+
+[testenv:sagepython-sagewheels-nopypi-norequirements]
+basepython = {env:SAGE_VENV}/bin/python3
+
+[testenv:sagepython-sagewheels]
+basepython = {env:SAGE_VENV}/bin/python3
+
 [testenv:sagepython-norequirements]
 basepython = {env:SAGE_VENV}/bin/python3

pkgs/sagemath-environment/README.rst

@@ -8,16 +8,27 @@ About SageMath
    "Creating a Viable Open Source Alternative to
     Magma, Maple, Mathematica, and MATLAB"
 
-   Copyright (C) 2005-2020 The Sage Development Team
+   Copyright (C) 2005-2022 The Sage Development Team
 
    https://www.sagemath.org
 
-SageMath fully supports all major Linux distributions, recent versions of macOS, and Windows (using Cygwin or Windows Subsystem for Linux).
+SageMath fully supports all major Linux distributions, recent versions of
+macOS, and Windows (using Cygwin or Windows Subsystem for Linux).
 
-The traditional and recommended way to install SageMath is from source via Sage-the-distribution (https://www.sagemath.org/download-source.html).  Sage-the-distribution first builds a large number of open source packages from source (unless it finds suitable versions installed in the system) and then installs the Sage Library (sagelib, implemented in Python and Cython).
+The traditional and recommended way to install SageMath is from source via
+Sage-the-distribution (https://www.sagemath.org/download-source.html).
+Sage-the-distribution first builds a large number of open source packages from
+source (unless it finds suitable versions installed in the system) and then
+installs the Sage Library (sagelib, implemented in Python and Cython).
 
 
 About this experimental pip-installable source distribution
 -----------------------------------------------------------
 
-This pip-installable source distribution `sagemath-environment` is an experimental distribution of a small part of the Sage Library.  Use at your own risk.  It provides a small, fundamental subset of the modules of the Sage library ("sagelib", `sagemath-standard`), providing the connection to the system and software environment.
+This pip-installable source distribution `sagemath-environment` is an
+experimental distribution of a small part of the Sage Library.  Use at your own
+risk.  It provides a small, fundamental subset of the modules of the Sage
+library ("sagelib", `sagemath-standard`), providing the connection to the
+system and software environment.  It also includes the `sage` script for
+launching the Sage REPL and accessing various developer tools (see `sage
+--help`).

pkgs/sagemath-environment/tox.ini

@@ -16,16 +16,22 @@ isolated_build = True
 deps =
     !norequirements: -rrequirements.txt
 
-setenv =
-    # Sage scripts such as sage-runtests like to use $HOME/.sage
-    HOME={envdir}
-
 passenv =
     # Parallel build
                              SAGE_NUM_THREADS
                              SAGE_NUM_THREADS_PARALLEL
-    # SAGE_VENV only for referring to the basepython
-    sagepython:              SAGE_VENV
+    # SAGE_VENV only for referring to the basepython or finding the wheels
+    sagepython, sagewheels:  SAGE_VENV
+    # Location of the wheels
+    sagewheels:              SAGE_SPKG_WHEELS
+
+setenv =
+    # Sage scripts such as sage-runtests like to use $HOME/.sage
+                             HOME={envdir}
+    # We supply pip options by environment variables so that they
+    # apply both to the installation of the dependencies and of the package
+    sagewheels:              PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_VENV:{toxinidir}/../../../../venv}/var/lib/sage/wheels}
+    nopypi:                  PIP_NO_INDEX=true
 
 whitelist_externals =
     bash
@@ -37,5 +43,14 @@ commands =
 [testenv:sagepython]
 basepython = {env:SAGE_VENV}/bin/python3
 
+[testenv:sagepython-sagewheels-nopypi]
+basepython = {env:SAGE_VENV}/bin/python3
+
+[testenv:sagepython-sagewheels-nopypi-norequirements]
+basepython = {env:SAGE_VENV}/bin/python3
+
+[testenv:sagepython-sagewheels]
+basepython = {env:SAGE_VENV}/bin/python3
+
 [testenv:sagepython-norequirements]
 basepython = {env:SAGE_VENV}/bin/python3