Merge #28925

.gitignore

@@ -158,6 +158,14 @@ build/bin/sage-build-env-config
 /pkgs/*/*.egg-info
 /pkgs/*/.tox
 
+/pkgs/sagemath-objects/setup.cfg
+/pkgs/sagemath-categories/setup.cfg
+/pkgs/sagemath-objects/pyproject.toml
+/pkgs/sagemath-categories/pyproject.toml
+/pkgs/sagemath-objects/requirements.txt
+/pkgs/sagemath-categories/requirements.txt
+/pkgs/sagemath-categories/MANIFEST.in
+
 # same for old locations - before Trac #31577
 /build/pkgs/*/src/build
 /build/pkgs/*/src/dist

Makefile

@@ -79,6 +79,17 @@ download:
 dist: build/make/Makefile
 	./sage --sdist
 
+pypi-sdists: sage_setup
+	./sage --sh build/pkgs/sage_conf/spkg-src
+	./sage --sh build/pkgs/sage_setup/spkg-src
+	./sage --sh build/pkgs/sage_sws2rst/spkg-src
+	./sage --sh build/pkgs/sage_docbuild/spkg-src
+	./sage --sh build/pkgs/sage_setup/spkg-src
+	./sage --sh build/pkgs/sagelib/spkg-src
+	./sage --sh build/pkgs/sagemath_objects/spkg-src
+	./sage --sh build/pkgs/sagemath_categories/spkg-src
+	@echo "Built sdists are in upstream/"
+
 # ssl: build Sage, and also install pyOpenSSL. This is necessary for
 # running the secure notebook. This make target requires internet
 # access. Note that this requires that your system have OpenSSL

README.md

@@ -349,54 +349,72 @@ Simplified directory layout (only essential files/directories):
 SAGE_ROOT                 Root directory (sage-x.y.z in Sage tarball)
 ├── build
 │   └── pkgs              Every package is a subdirectory here
-│       ├── 4ti2
+│       ├── 4ti2/
 │       …
-│       └── zn_poly
+│       └── zn_poly/
 ├── configure             Top-level configure script
 ├── COPYING.txt           Copyright information
 ├── pkgs                  Source trees of Python distribution packages
 │   ├── sage-conf
 │   │   ├── sage_conf.py
 │   │   └── setup.py
 │   ├── sage-docbuild
-│   │   ├── sage_docbuild
+│   │   ├── sage_docbuild/
+│   │   └── setup.py
+│   ├── sage-setup
+│   │   ├── sage_setup/
 │   │   └── setup.py
 │   ├── sage-sws2rst
-│   │   ├── sage_sws2rst
+│   │   ├── sage_sws2rst/
 │   │   └── setup.py
 │   └── sagemath-standard
-│       ├── bin
-│       ├── sage
+│       ├── bin/
+│       ├── sage -> ../../src/sage
 │       └── setup.py
-├── local  (SAGE_LOCAL)   Compiled packages are installed here
+├── local  (SAGE_LOCAL)   Installation hierarchy for non-Python packages
 │   ├── bin               Executables
 │   ├── include           C/C++ headers
-│   ├── lib               Shared libraries
+│   ├── lib               Shared libraries, architecture-dependent data
 │   ├── share             Databases, architecture-independent data, docs
 │   │   └── doc           Viewable docs of Sage and of some components
 │   └── var
-│       ├── lib/sage      List of installed packages
-│       └── tmp/sage      Temporary files when building Sage
+│       ├── lib/sage
+│       │   ├── installed/
+│       │   │             Records of installed non-Python packages
+│       │   ├── scripts/  Scripts for uninstalling installed packages
+│       │   └── venv-python3.9  (SAGE_VENV)
+│       │       │         Installation hierarchy (virtual environment)
+│       │       │         for Python packages
+│       │       ├── bin/  Executables and installed scripts
+│       │       ├── lib/python3.9/site-packages/
+│       │       │         Python modules/packages are installed here
+│       │       └── var/lib/sage/
+│       │           └── wheels/
+│       │                 Python wheels for all installed Python packages
+│       │
+│       └── tmp/sage/     Temporary files when building Sage
 ├── logs
 │   ├── dochtml.log       Log of the documentation build
 │   ├── install.log       Full install log
 │   └── pkgs              Build logs of individual packages
 │       ├── alabaster-0.7.12.log
 │       …
 │       └── zn_poly-0.9.2.log
-├── m4                    M4 macros for configure
+├── m4                    M4 macros for generating the configure script
 │   └── *.m4
 ├── Makefile              Running "make" uses this file
+├── prefix -> SAGE_LOCAL  Convenience symlink to the installation tree
 ├── README.md             This file
 ├── sage                  Script to start Sage
 ├── src                   Monolithic Sage library source tree
-│   ├── bin               Scripts that Sage uses internally
-│   ├── doc               Sage documentation sources
-│   └── sage              The Sage library source code
+│   ├── bin/              Scripts that Sage uses internally
+│   ├── doc/              Sage documentation sources
+│   └── sage/             The Sage library source code
 ├── upstream              Source tarballs of packages
 │   ├── Babel-2.9.1.tar.gz
 │   …
 │   └── zn_poly-0.9.2.tar.gz
+├── venv -> SAGE_VENV     Convenience symlink to the virtual environment
 └── VERSION.txt
 ```
 For more details see [our Developer's Guide](https://doc.sagemath.org/html/en/developer/coding_basics.html#files-and-directory-structure).

build/bin/write-dockerfile.sh

@@ -187,7 +187,7 @@ FROM with-system-packages as bootstrapped
 #:bootstrapping:
 RUN mkdir -p /sage
 WORKDIR /sage
-ADD Makefile VERSION.txt README.md bootstrap configure.ac sage ./
+ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 ./
 ADD src/doc/bootstrap src/doc/bootstrap
 ADD src/bin src/bin
 ADD m4 ./m4

build/make/Makefile.in

@@ -234,7 +234,6 @@ base-toolchain: _clean-broken-gcc base
 
 # All targets except for the base packages and except the documentation
 all-sage: \
-		sagelib \
 		$(SAGE_LOCAL_INSTALLED_PACKAGE_INSTS) $(SAGE_LOCAL_UNINSTALLED_PACKAGES_CLEANS) \
 		$(SAGE_VENV_INSTALLED_PACKAGE_INSTS)  $(SAGE_VENV_UNINSTALLED_PACKAGES_CLEANS)
 

build/pkgs/pplpy/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) $(MP_LIBRARY) gmpy2 cysignals mpfr mpc ppl | $(PYTHON_TOOLCHAIN) sphinx
+$(PYTHON) $(MP_LIBRARY) gmpy2 cysignals mpfr mpc ppl | $(PYTHON_TOOLCHAIN) $(if $(findstring no,$(SAGE_SPKG_INSTALL_DOCS)),,sphinx)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/sage_conf/spkg-src

@@ -15,7 +15,5 @@ fi
 # Exit on failure
 set -e
 
-cd build/pkgs/sage_conf
-
-cd src
+cd pkgs/sage-conf_pypi
 python3 -u setup.py --no-user-cfg sdist --dist-dir "$SAGE_DISTFILES"

build/pkgs/sage_setup/install-requires.txt

@@ -1 +1 @@
-sage-setup
+sage-setup ~= 9.5.b6

build/pkgs/sagelib/bootstrap

@@ -4,7 +4,7 @@ for infile in src/*.m4; do
     if [ -f "$infile" ]; then
         outfile="src/$(basename $infile .m4)"
         if [ "${BOOTSTRAP_QUIET}" = "no" ]; then
-            echo "$0: installing build/pkgs/sagelib/$outfile"
+            echo "$0: installing $(pwd)/$outfile"
         fi
         m4 "$infile" > "$outfile"
     fi

build/pkgs/sagelib/install-cleaner/LICENSE.txt

@@ -0,0 +1 @@
+../src/LICENSE.txt

build/pkgs/sagelib/install-cleaner/VERSION.txt

@@ -0,0 +1 @@
+../src/VERSION.txt

build/pkgs/sagelib/install-cleaner/build

@@ -0,0 +1 @@
+../src/build

build/pkgs/sagelib/install-cleaner/sage

@@ -0,0 +1 @@
+../src/sage

build/pkgs/sagelib/install-cleaner/setup.cfg

@@ -0,0 +1 @@
+../src/setup.cfg

build/pkgs/sagelib/install-cleaner/setup.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import time
+# Import setuptools before importing distutils, so that setuptools
+# can replace distutils by its own vendored copy.
+import setuptools
+from distutils import log
+from setuptools import setup
+
+#########################################################
+### Set source directory
+#########################################################
+
+import sage.env
+sage.env.SAGE_SRC = os.getcwd()
+from sage.env import *
+
+#########################################################
+### Configuration
+#########################################################
+
+if len(sys.argv) > 1 and (sys.argv[1] == "sdist" or sys.argv[1] == "egg_info"):
+    sdist = True
+else:
+    sdist = False
+
+if sdist:
+    cmdclass = {}
+else:
+    from sage_setup.excepthook import excepthook
+    sys.excepthook = excepthook
+
+    from sage_setup.setenv import setenv
+    setenv()
+
+    from sage_setup.command.sage_build import sage_build
+    from sage_setup.command.sage_build_cython import sage_build_cython
+    from sage_setup.command.sage_build_ext import sage_build_ext
+    from sage_setup.command.sage_install import sage_clean
+
+    cmdclass = dict(build=sage_build,
+                    build_cython=sage_build_cython,
+                    build_ext=sage_build_ext,
+                    install=sage_clean)
+
+#########################################################
+### Discovering Sources
+#########################################################
+
+if sdist:
+    # No need to compute distributions.  This avoids a dependency on Cython
+    # just to make an sdist.
+    distributions = None
+    python_packages = []
+    python_modules = []
+    cython_modules = []
+else:
+    # TODO: This should be quiet by default
+    print("Discovering Python/Cython source code....")
+    t = time.time()
+    from sage_setup.optional_extension import is_package_installed_and_updated
+    distributions = ['']
+    optional_packages_with_extensions = ['mcqd', 'bliss', 'tdlib', 'primecount',
+                                         'coxeter3', 'fes', 'sirocco', 'meataxe']
+    distributions += ['sagemath-{}'.format(pkg)
+                      for pkg in optional_packages_with_extensions
+                      if is_package_installed_and_updated(pkg)]
+    log.warn('distributions = {0}'.format(distributions))
+    from sage_setup.find import find_python_sources
+    python_packages, python_modules, cython_modules = find_python_sources(
+        SAGE_SRC, ['sage'], distributions=distributions)
+
+    log.debug('python_packages = {0}'.format(python_packages))
+    print("Discovered Python/Cython sources, time: %.2f seconds." % (time.time() - t))
+
+
+#########################################################
+### Distutils
+#########################################################
+
+code = setup(
+      packages = python_packages,
+      cmdclass = cmdclass,
+      ext_modules = cython_modules)

build/pkgs/sagelib/spkg-install

@@ -53,7 +53,10 @@ if [ "$SAGE_EDITABLE" = yes ]; then
 else
     # Likewise, we should remove the egg-link that may have been installed previously.
     (cd "$SITEPACKAGESDIR" && rm -f sagemath-standard.egg-link)
-    time python3 -u setup.py --no-user-cfg build install || exit 1
+    time python3 -u setup.py --no-user-cfg install || exit 1
+    # Run the installation cleaner (no longer part of the sagemath-standard install procedure.)
+    # FIXME: It needs to clean the right build directory!
+    (cd ../install-cleaner && python3 -u setup.py --no-user-cfg install)
 fi
 
 if [ "$UNAME" = "CYGWIN" ]; then

build/pkgs/sagelib/spkg-src

@@ -17,5 +17,6 @@ set -e
 
 cd build/pkgs/sagelib
 
-cd src
-python3 -u setup.py --no-user-cfg sdist --dist-dir "$SAGE_DISTFILES"
+for src in src*; do
+    (cd $src && python3 -u setup.py --no-user-cfg sdist --dist-dir "$SAGE_DISTFILES")
+done

build/pkgs/sagemath_categories/SPKG.rst

@@ -0,0 +1 @@
+src/README.rst

build/pkgs/sagemath_categories/bootstrap

@@ -0,0 +1 @@
+../sagelib/bootstrap

build/pkgs/sagemath_categories/dependencies

@@ -0,0 +1 @@
+../sagemath_objects/dependencies

build/pkgs/sagemath_categories/package-version.txt

@@ -0,0 +1 @@
+../sagelib/package-version.txt

build/pkgs/sagemath_categories/spkg-install

@@ -0,0 +1 @@
+../sagemath_objects/spkg-install

build/pkgs/sagemath_categories/spkg-src

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Script to prepare an sdist tarball for sagemath-categories
+# This script is not used during build.
+#
+# HOW TO MAKE THE TARBALL:
+# ./sage --sh build/pkgs/sagemath_categories/spkg-src
+
+if [ -z "$SAGE_ROOT" ] ; then
+    echo >&2 "Error - SAGE_ROOT undefined ... exiting"
+    echo >&2 "Maybe run 'sage -sh'?"
+    exit 1
+fi
+
+# Exit on failure
+set -e
+
+cd build/pkgs/sagemath_categories
+
+cd src
+python3 -u setup.py --no-user-cfg sdist --dist-dir "$SAGE_DISTFILES"

build/pkgs/sagemath_categories/src

@@ -0,0 +1 @@
+../../../pkgs/sagemath-categories

build/pkgs/sagemath_categories/type

@@ -0,0 +1 @@
+experimental

build/pkgs/sagemath_objects/SPKG.rst

@@ -0,0 +1 @@
+src/README.rst

build/pkgs/sagemath_objects/bootstrap

@@ -0,0 +1 @@
+../sagelib/bootstrap

build/pkgs/sagemath_objects/dependencies

@@ -0,0 +1,4 @@
+FORCE $(PYTHON) cysignals gmpy2 ipython | $(PYTHON_TOOLCHAIN) cython pkgconfig $(and $(filter-out no,$(SAGE_CHECK)), tox)
+
+# FORCE: Always run the spkg-install script
+# ipython - for the doctester

build/pkgs/sagemath_objects/package-version.txt

@@ -0,0 +1 @@
+../sagelib/package-version.txt

build/pkgs/sagemath_objects/spkg-install

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+cd src
+
+if [ "$SAGE_CHECK" != no ]; then
+    tox
+fi
+# We skip the install for now.
+exit 0

build/pkgs/sagemath_objects/spkg-src

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Script to prepare an sdist tarball for sagemath-objects
+# This script is not used during build.
+#
+# HOW TO MAKE THE TARBALL:
+# 1) ./sage --sh build/pkgs/sagemath_objects/spkg-src
+
+if [ -z "$SAGE_ROOT" ] ; then
+    echo >&2 "Error - SAGE_ROOT undefined ... exiting"
+    echo >&2 "Maybe run 'sage -sh'?"
+    exit 1
+fi
+
+# Exit on failure
+set -e
+
+cd build/pkgs/sagemath_objects
+
+cd src
+python3 -u setup.py --no-user-cfg sdist --dist-dir "$SAGE_DISTFILES"

build/pkgs/sagemath_objects/src

@@ -0,0 +1 @@
+../../../pkgs/sagemath-objects

build/pkgs/sagemath_objects/type

@@ -0,0 +1 @@
+experimental