Trac #29053: Add debian/fedora/arch/conda package information to buil…

…d/pkgs, generate Dockerfiles and installation help; add tox.ini

https://doc.sagemath.org/html/en/installation/source.html#linux-
recommended-installation
gives `apt-get install` and `yum install` command lines to install
system packages that will be recognized by `build/pkgs/SPKG/spkg-
configure.m4`.

Several packages are missing, see comments in
https://trac.sagemath.org/attachment/ticket/27824/Dockerfile-ubuntu-
minimal

This ticket puts this information on a per-package basis into
`build/pkgs/SPKG/distros/debian.txt`, `.../fedora.txt`, `.../arch.txt`
for the following purposes:
- `sage -info SPKG` prints out this information.
- Provide a `tox.ini` that generates and runs `Dockerfile`s that test
the installation on various systems. Example (running 3 builds in
parallel):
   {{{
   $ tox -p auto -e archlinux-latest,ubuntu-focal-minimal,debian-buster-
maximal
⠸
   }}}

Incidental change on this ticket:
- Mark `gfortran` as a `type=standard` package. It is a prereq of
standard package `numpy`. Its status as a `type=optional` package
predates the clarifications brought by the development of the `spkg-
configure.m4` mechanism (#27330).

----------

Possible follow-up and related tickets:
- #29087 - Add a GitHub workflow that runs this automatically, with many
parallel jobs.
- Also the `apt-cyg` command lines at
https://trac.sagemath.org/wiki/Cygwin64Port could be generated in the
same way.
- Aggregate and format it (during `./bootstrap`) to produce the command
lines shown in the manual. This ticket already prepares it by moving
these command lines into separate `.txt` files.
- #29041: at `./bootstrap` time, generate `src/requirements.txt`,
`src/constraints.txt`, `src/setup.cfg` `[install_requires]` from
`build/pkgs`
- #28745: Add `SAGE_ROOT/environment.yml` for `conda env create -f`.
- Add to tox.ini a conda run (either with a fresh environment or fresh
conda install) without docker - to test with macOS conda
- Add to tox.ini a homebrew run (with a fresh install not in /usr/local)

Resources:
- https://repology.org/

URL: https://trac.sagemath.org/29053
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): Dima Pasechnik

build/bin/sage-spkg

@@ -349,10 +349,72 @@ fi
 if [ $INFO -ne 0 -a "$USE_LOCAL_SCRIPTS" = yes ]; then
     cat "$PKG_SCRIPTS/SPKG.txt"
     if [ -r "$PKG_SCRIPTS/type" ] ; then
+        echo
         echo "== Type =="
         echo
         cat "$PKG_SCRIPTS/type"
+        echo
+    fi
+    echo "== Equivalent System Packages =="
+    echo
+    PKG_DISTROS="$PKG_SCRIPTS"/distros
+    for system_package_file in "$PKG_DISTROS"/debian*.txt "$PKG_DISTROS"/fedora*.txt "$PKG_DISTROS"/*conda*.txt "$PKG_DISTROS"/homebrew*.txt  "$PKG_DISTROS"/arch*.txt ; do
+        if [ -f "$system_package_file" ]; then
+            system=$(basename "$system_package_file" .txt)
+            system_packages="$(echo $(sed 's/#.*//;' $system_package_file))"
+            case $system in
+                debian)
+                    # Generic
+                    echo "Debian/Ubuntu:"
+                    echo "  sudo apt-get install $system_packages"
+                    ;;
+                debian*|ubuntu*)
+                    # Specific distribution
+                    echo "$system:"
+                    echo "  sudo apt-get install $system_packages"
+                    ;;
+                fedora)
+                    # Generic
+                    echo "Fedora/Redhat/CentOS:"
+                    echo "  sudo yum install $system_packages"
+                    ;;
+                fedora*|redhat*|centos*)
+                    # Specific distribution
+                    echo "$system:"
+                    echo "  sudo yum install $system_packages"
+                    ;;
+                arch*)
+                    echo "$system:"
+                    echo "  sudo pacman -S $system_packages"
+                    ;;
+                *conda*)
+                    echo "$system:"
+                    echo "  conda install $system_packages"
+                    ;;
+                homebrew*)
+                    echo "$system:"
+                    echo "  brew install $system_packages"
+                    ;;
+                *)
+                    echo "$system:"
+                    echo "  install the following packages: $system_packages"
+                    ;;
+            esac
+        fi
+    done
+    if [ -z "$system" ]; then
+        echo "(none known)"
+    else
+        echo
+        if [ -f "$PKG_SCRIPTS"/spkg-configure.m4 ]; then
+            echo "If the system package is installed, ./configure will check whether it can be used."
+        else
+            echo "However, these system packages will not be used for building Sage"
+            echo "because spkg-configure.m4 has not been written for this package;"
+            echo "see https://trac.sagemath.org/ticket/27330"
+        fi
     fi
+    echo
     exit 0
 fi
 

build/bin/write-dockerfile.sh

@@ -0,0 +1,168 @@
+#! /usr/bin/env bash
+## Write a Dockerfile to stdout that tests that the packages listed in the debian.txt/fedora.txt files of standard spkg exist
+## and satisfy the requirements tested by spkg-configure.m4
+## This is called by $SAGE_ROOT/tox.ini
+set -e
+SYSTEM="${1:-debian}"
+shopt -s extglob
+TYPE_PATTERN="${2:-standard}"
+WITH_SYSTEM_SPKG="${3:-yes}"
+IGNORE_MISSING_SYSTEM_PACKAGES="${4:-no}"
+#
+STRIP_COMMENTS="sed s/#.*//;"
+SAGE_ROOT=.
+SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SAGE_ROOT/build/pkgs/$SYSTEM{,-bootstrap}.txt))
+CONFIGURE_ARGS="--enable-option-checking "
+for PKG_SCRIPTS in build/pkgs/*; do
+    if [ -d $PKG_SCRIPTS ]; then
+        PKG_BASE=$(basename $PKG_SCRIPTS)
+        SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/$SYSTEM.txt
+        PKG_TYPE=$(cat $PKG_SCRIPTS/type)
+        if [ -f $SYSTEM_PACKAGES_FILE -a -f $PKG_SCRIPTS/spkg-configure.m4 ]; then
+           PKG_SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE))
+           if [ -n "PKG_SYSTEM_PACKAGES" ]; then
+               case "$PKG_TYPE" in
+                   $TYPE_PATTERN)
+                       SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                       if [ -f $PKG_SCRIPTS/spkg-configure.m4 ]; then
+                           CONFIGURE_ARGS+="--with-system-$PKG_BASE=${WITH_SYSTEM_SPKG} "
+                       fi
+                       ;;
+               esac
+           fi
+        fi
+    fi
+done
+echo "# Automatically generated by SAGE_ROOT/build/bin/write-dockerfile.sh"
+echo "# the :comments: separate the generated file into sections"
+echo "# to simplify writing scripts that customize this file"
+RUN=RUN
+case $SYSTEM in
+    debian*|ubuntu*)
+        cat <<EOF
+ARG BASE_IMAGE=ubuntu:latest
+FROM \${BASE_IMAGE}
+EOF
+        UPDATE="apt-get update &&"
+        INSTALL="DEBIAN_FRONTEND=noninteractive apt-get install -qqq --no-install-recommends --yes"
+        CLEAN="&& apt-get clean"
+        ;;
+    fedora*|redhat*|centos*)
+        cat <<EOF
+ARG BASE_IMAGE=fedora:latest
+FROM \${BASE_IMAGE}
+EOF
+        INSTALL="yum install -y"
+        ;;
+    arch*)
+        # https://hub.docker.com/_/archlinux/
+        cat <<EOF
+ARG BASE_IMAGE=archlinux:latest
+FROM \${BASE_IMAGE}
+EOF
+        INSTALL="pacman -Syu --noconfirm"
+        ;;
+    conda*)
+        cat <<EOF
+ARG BASE_IMAGE=continuumio/miniconda3:latest
+FROM \${BASE_IMAGE}
+ARG USE_CONDARC=condarc.yml
+ADD *condarc*.yml /tmp/
+RUN echo \${CONDARC}; cd /tmp && conda config --stdin < \${USE_CONDARC}
+RUN conda update -n base conda
+RUN ln -sf /bin/bash /bin/sh
+EOF
+        # On this image, /bin/sh -> /bin/dash;
+        # but some of the scripts in /opt/conda/etc/conda/activate.d
+        # from conda-forge (as of 2020-01-27) contain bash-isms:
+        # /bin/sh: 5: /opt/conda/etc/conda/activate.d/activate-binutils_linux-64.sh: Syntax error: "(" unexpected
+        # The command '/bin/sh -c . /opt/conda/etc/profile.d/conda.sh; conda activate base;  ./bootstrap' returned a non-zero code
+        # We just change the link to /bin/bash.
+        INSTALL="conda install --update-all --quiet --yes"
+        EXISTS="2>/dev/null >/dev/null conda search -f"
+        #EXISTS="conda search -f"
+        CLEAN="&& conda info; conda list"
+        RUN="RUN . /opt/conda/etc/profile.d/conda.sh; conda activate base; "  # to activate the conda env
+        ;;
+    *)
+        echo "Not implemented: package installation for SYSTEM=$SYSTEM" >&2
+        exit 1
+        ;;
+esac
+cat <<EOF
+#:packages:
+ENV PACKAGES="$SYSTEM_PACKAGES"
+EOF
+case "$IGNORE_MISSING_SYSTEM_PACKAGES" in
+    no)
+        cat <<EOF
+RUN $UPDATE $INSTALL $SYSTEM_PACKAGES $CLEAN
+EOF
+        ;;
+    yes)
+        if [ -n "$EXISTS" ]; then
+            # Filter by existing packages, try to install these in one shot; fall back to one by one.
+            cat <<EOF
+RUN $UPDATE EXISTING_PACKAGES=""; for pkg in \$PACKAGES; do echo -n .; if $EXISTS \$pkg; then EXISTING_PACKAGES="\$EXISTING_PACKAGES \$pkg"; echo -n "\$pkg"; fi; done; $INSTALL \$EXISTING_PACKAGES || (echo "Trying again one by one:"; for pkg in \$EXISTING_PACKAGES; do echo "Trying to install \$pkg"; $INSTALL \$pkg || echo "(ignoring error)"; done); : $CLEAN
+EOF
+        else
+            # Try in one shot, fall back to one by one.  Separate "RUN" commands
+            # for caching by docker.
+            cat <<EOF
+RUN $UPDATE $INSTALL \${PACKAGES} || echo "(ignoring error)"
+EOF
+            for pkg in $SYSTEM_PACKAGES; do
+                cat <<EOF
+RUN $INSTALL $pkg || echo "(ignoring error)"
+EOF
+            done
+            if [ -n "$CLEAN" ]; then
+                cat <<EOF
+RUN : $CLEAN
+EOF
+            fi
+        fi
+        ;;
+    *)
+        echo "Argument IGNORE_MISSING_SYSTEM_PACKAGES must be yes or no"
+        ;;
+esac
+cat <<EOF
+#:bootstrapping:
+RUN mkdir -p /sage
+WORKDIR /sage
+ADD Makefile VERSION.txt README.md bootstrap configure.ac sage ./
+ADD m4 ./m4
+ADD build ./build
+ADD src/bin/sage-version.sh src/bin/sage-version.sh
+$RUN ./bootstrap
+#:configuring:
+ADD src/ext src/ext
+ADD src/bin src/bin
+ADD src/Makefile.in src/Makefile.in
+ARG EXTRA_CONFIGURE_ARGS=""
+EOF
+if [ ${WITH_SYSTEM_SPKG} = "force" ]; then
+    cat <<EOF
+$RUN echo "****** Configuring: ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} *******"; ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} || (echo "********** configuring without forcing ***********"; cat config.log; ./configure --enable-build-as-root; cat config.log; exit 1)
+EOF
+else
+    cat <<EOF
+$RUN echo "****** Configuring: ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} *******"; ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} || (cat config.log; exit 1)
+EOF
+fi
+cat <<EOF
+# We first compile base-toolchain because otherwise lots of packages are missing their dependency on 'patch'
+ARG NUMPROC=8
+ENV MAKE="make -j\${NUMPROC}"
+ARG USE_MAKEFLAGS="-k"
+#:toolchain:
+$RUN make \${USE_MAKEFLAGS} base-toolchain
+#:make:
+# Avoid running the lengthy testsuite of the following.
+$RUN make \${USE_MAKEFLAGS} cython
+# By default, compile something tricky but that does not take too long. scipy uses BLAS.
+ARG TARGETS="scipy"
+$RUN SAGE_CHECK=yes SAGE_CHECK_PACKAGES="!r,!python3,!python2,!nose" make \${USE_MAKEFLAGS} \${TARGETS}
+#:end:
+EOF

build/pkgs/arb/distros/arch.txt

@@ -0,0 +1 @@
+arb

build/pkgs/arb/distros/conda.txt

@@ -0,0 +1 @@
+arb

build/pkgs/arb/distros/debian.txt

@@ -0,0 +1 @@
+libflint-arb-dev

build/pkgs/arch-bootstrap.txt

@@ -0,0 +1,4 @@
+# Packages needed for ./bootstrap
+gettext autoconf automake libtool
+# Otherwise: Error: could not locate the pkg-config autoconf macros. These are
+pkg-config

build/pkgs/arch.txt

@@ -0,0 +1,8 @@
+binutils
+make
+m4
+perl
+python # system python for bootstrapping the build
+tar
+bc
+gcc

build/pkgs/benzene/distros/arch.txt

@@ -0,0 +1 @@
+benzene

build/pkgs/bliss/distros/arch.txt

@@ -0,0 +1 @@
+bliss

build/pkgs/bliss/distros/conda.txt

@@ -0,0 +1 @@
+bliss

build/pkgs/boost/distros/arch.txt

@@ -0,0 +1 @@
+boost

build/pkgs/boost/distros/conda.txt

@@ -0,0 +1 @@
+boost-cpp

build/pkgs/buckygen/distros/arch.txt

@@ -0,0 +1 @@
+buckygen

build/pkgs/bzip2/distros/conda.txt

@@ -0,0 +1 @@
+bzip2

build/pkgs/bzip2/distros/debian.txt

@@ -0,0 +1,3 @@
+libbz2-dev
+# Debian bullseye does not have bzip2 as a dependency of above
+bzip2

build/pkgs/bzip2/distros/fedora.txt

@@ -0,0 +1 @@
+bzip2 bzip2-devel

build/pkgs/cbc/distros/arch.txt

@@ -0,0 +1 @@
+coin-or-cbc

build/pkgs/cbc/distros/conda.txt

@@ -0,0 +1 @@
+coincbc

build/pkgs/cbc/distros/debian.txt

@@ -0,0 +1,2 @@
+coinor-cbc
+coinor-libcbc-dev

build/pkgs/cliquer/distros/conda.txt

@@ -0,0 +1 @@
+cliquer

build/pkgs/cliquer/distros/debian.txt

@@ -0,0 +1 @@
+cliquer libcliquer-dev

build/pkgs/cmake/distros/conda.txt

@@ -0,0 +1 @@
+cmake

build/pkgs/cmake/distros/debian.txt

@@ -0,0 +1 @@
+cmake

build/pkgs/cmake/distros/fedora.txt

@@ -0,0 +1 @@
+cmake

build/pkgs/combinatorial_designs/distros/arch.txt

@@ -0,0 +1 @@
+sage-data-combinatorial_designs

build/pkgs/conda-bootstrap.txt

@@ -0,0 +1,2 @@
+# Packages needed for ./bootstrap
+gettext autoconf automake libtool pkg-config

build/pkgs/conda.txt

@@ -0,0 +1,7 @@
+compilers
+make
+m4
+perl
+python
+tar
+bc

build/pkgs/conway_polynomials/distros/arch.txt

@@ -0,0 +1 @@
+sage-data-conway_polynomials

build/pkgs/coxeter3/distros/arch.txt

@@ -0,0 +1 @@
+coxeter

build/pkgs/csdp/distros/arch.txt

@@ -0,0 +1 @@
+coin-or-csdp

build/pkgs/curl/distros/conda.txt

@@ -0,0 +1 @@
+curl

build/pkgs/curl/distros/debian.txt

@@ -0,0 +1 @@
+curl libcurl4-openssl-dev

build/pkgs/curl/distros/fedora.txt

@@ -0,0 +1 @@
+libcurl-devel curl

build/pkgs/debian-bootstrap.txt

@@ -0,0 +1,2 @@
+# Packages needed for ./bootstrap
+gettext autoconf automake libtool pkg-config

build/pkgs/debian.txt

@@ -0,0 +1,27 @@
+# This file, build/pkgs/debian.txt, contains names of Debian/Ubuntu packages
+# needed for installation of Sage from source.
+#
+# In addition, the files build/pkgs/SPKG/debian.txt contain the names
+# of packages that provide the equivalent of SPKG.
+#
+# If distinctions between different distributions need to be made,
+# files named debian....txt or ubuntu.....txt can be used.
+#
+# See build/bin/sage-spkg, where this information is processed
+# for use in "sage -info SPKG".
+#
+# Everything on a line after a # character is ignored.
+binutils
+make
+m4
+perl
+# python3-minimal is not enough on debian buster, ubuntu bionic - it does not have urllib
+python3    # system python for bootstrapping the build
+tar
+bc
+gcc
+# On debian buster, need C++ even to survive 'configure'. Otherwise:
+# checking how to run the C++ preprocessor... /lib/cpp
+# configure: error: in `/sage':
+# configure: error: C++ preprocessor "/lib/cpp" fails sanity check
+g++

build/pkgs/dot2tex/distros/arch.txt

@@ -0,0 +1 @@
+dot2tex

build/pkgs/eclib/distros/arch.txt

@@ -0,0 +1 @@
+eclib

build/pkgs/eclib/distros/conda.txt

@@ -0,0 +1 @@
+eclib

build/pkgs/eclib/distros/debian.txt

@@ -0,0 +1 @@
+libec-dev

build/pkgs/eclib/distros/fedora.txt

@@ -0,0 +1 @@
+eclib-devel

build/pkgs/ecm/distros/conda.txt

@@ -0,0 +1 @@
+ecm

build/pkgs/ecm/distros/debian.txt

@@ -0,0 +1 @@
+gmp-ecm libecm-dev

build/pkgs/ecm/distros/fedora.txt

@@ -0,0 +1 @@
+gmp-ecm-devel

build/pkgs/elliptic_curves/arch.txt

@@ -0,0 +1 @@
+sage-data-elliptic_curves

build/pkgs/fedora-bootstrap.txt

@@ -0,0 +1,4 @@
+# Packages needed for ./bootstrap
+gettext-devel autoconf automake libtool
+# Fedora 26 needs:
+pkg-config