Merge branch 't/31227/accept__usr_bin_python3_from_xcode_12_3_on_maco…

…s_10_15__catalina_' into t/31335/homebrew__unused_packages__singular__pari_______in__usr_local_leak_into_sagelib_build
sagemath · Feb 4, 2021 · 32dfadd · 32dfadd
2 parents 13b4090 + fc8b676
commit 32dfadd
Show file tree

Hide file tree

Showing 15 changed files with 164 additions and 56 deletions.
diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml
@@ -105,7 +105,7 @@ jobs:
         os: [ macos-10.15, macos-11.0 ]
         tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, homebrew-macos-python3_pythonorg, homebrew-macos-python3_xcode-gcc_spkg, conda-forge-macos]
         tox_packages_factor: [minimal, standard]
-        xcode_version_factor: [11.7, default, 12.3]
+        xcode_version_factor: [11.7, 12.2, default, 12.4]
     env:
       TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
       LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
@@ -223,7 +223,7 @@ jobs:
         os: [ macos-10.15, macos-11.0 ]
         tox_system_factor: [macos-nobootstrap, macos-nobootstrap-python3_pythonorg]
         tox_packages_factor: [minimal]
-        xcode_version_factor: [default, 12]
+        xcode_version_factor: [11.7, 12.2, default, 12.4]
     env:
       TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
       LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-xcode_${{ matrix.xcode_version_factor }}

diff --git a/build/pkgs/cmake/spkg-install.in b/build/pkgs/cmake/spkg-install.in
@@ -1,6 +1,8 @@
 if [ "$UNAME" = Darwin ]; then
     CC=clang
     CXX=clang++
+    export CFLAGS="$CFLAGS_NON_NATIVE"
+    export CXXFLAGS="$CXXFLAGS_NON_NATIVE"
 fi
 
 cd src

diff --git a/build/pkgs/curl/spkg-install.in b/build/pkgs/curl/spkg-install.in
@@ -10,6 +10,7 @@ if [ "$UNAME" = "Darwin" ]; then
     if [ $MACOSX_VERSION -ge 16 ]; then
         echo "OS X 10.$[$MACOSX_VERSION-4] Building with clang and --with-darwinssl."
         CC=clang
+        export CFLAGS="$CFLAGS_NON_NATIVE"
         CURL_CONFIGURE="--with-darwinssl $CURL_CONFIGURE"
     fi
 fi

diff --git a/build/pkgs/pari/spkg-install.in b/build/pkgs/pari/spkg-install.in
@@ -105,7 +105,6 @@ unset dfltlibdir dfltmandir dfltsysdatadir dfltobjdir
 
 # Avoid segmentation fault on macOS Catalina with XCode 11.4
 # #29451
-MACOSX_VERSION=`uname -r | awk -F. '{print $1}'`
 if [ $MACOSX_VERSION -ge 14 ]; then
     # various packages have still have issues with
     # two digit OS X versions

diff --git a/build/pkgs/pip/checksums.ini b/build/pkgs/pip/checksums.ini
@@ -1,5 +1,5 @@
 tarball=pip-VERSION.tar.gz
-sha1=5e708de15d0959386595bea819d75fbc5d4480b3
-md5=efd9dc5c7405452c72c0e3cd410a2018
-cksum=3748660343
+sha1=a8c85abfdc0bf15dd7360865f59e4234be00d1e2
+md5=3d5d0639042c829bba411d3735267546
+cksum=4219824548
 upstream_url=https://pypi.io/packages/source/p/pip/pip-VERSION.tar.gz
diff --git a/build/pkgs/pip/package-version.txt b/build/pkgs/pip/package-version.txt
@@ -1 +1 @@
-20.2.3
+20.3.3
diff --git a/build/pkgs/psutil/spkg-install.in b/build/pkgs/psutil/spkg-install.in
@@ -1,6 +1,7 @@
 if [ "$UNAME" = "Darwin" ] && [ $MACOSX_VERSION -ge 16 ]; then
     echo "OS X 10.$[$MACOSX_VERSION-4] Building with clang."
     CC=clang
+    export CFLAGS="$CFLAGS_NON_NATIVE"
 fi
 
 cd src && sdh_pip_install .
diff --git a/build/pkgs/python3/checksums.ini b/build/pkgs/python3/checksums.ini
@@ -1,5 +1,5 @@
 tarball=Python-VERSION.tar.xz
-sha1=68d6c7f948801cc755905162f5ee7589595edee4
-md5=35b5a3d0254c1c59be9736373d429db7
-cksum=2309344767
+sha1=77f4105846f6740297e50d7535a42c02d6b8e7db
+md5=61981498e75ac8f00adcb908281fadb6
+cksum=4032849512
 upstream_url=https://www.python.org/ftp/python/VERSION/Python-VERSION.tar.xz
diff --git a/build/pkgs/python3/package-version.txt b/build/pkgs/python3/package-version.txt
@@ -1 +1 @@
-3.8.5
+3.9.1
diff --git a/build/pkgs/python3/spkg-build.in b/build/pkgs/python3/spkg-build.in
@@ -20,11 +20,8 @@ if [ "$SAGE_VALGRIND" = "yes" ]; then
     PYTHON_CONFIGURE="$PYTHON_CONFIGURE --without-pymalloc"
 fi
 
-# Use EXTRA_CFLAGS for user-defined CFLAGS since Python puts its own
-# default flags like -O3 after CFLAGS but before EXTRA_CFLAGS.
-# We also disable warnings about unused variables/functions which are
-# common in Cython-generated code.
-export EXTRA_CFLAGS="`testcflags.sh -Wno-unused` $CFLAGS"
+# We will use EXTRA_CFLAGS for user-defined CFLAGS below.
+OLD_CFLAGS="$CFLAGS"
 unset CFLAGS
 
 if [ "$UNAME" = Darwin ]; then
@@ -58,6 +55,12 @@ elif [ "$UNAME" = SunOS ]; then
     export CFLAGS="-D__C99FEATURES__ $CFLAGS"
 fi
 
+# Use EXTRA_CFLAGS for user-defined CFLAGS since Python puts its own
+# default flags like -O3 after CFLAGS but before EXTRA_CFLAGS.
+# We also disable warnings about unused variables/functions which are
+# common in Cython-generated code.
+export EXTRA_CFLAGS="`testcflags.sh -Wno-unused $OLD_CFLAGS`"
+
 # Remove old symbolic link: it is not needed and its presence can
 # interfere with the Python build.
 rm -f "$SAGE_LOCAL/lib/python"

diff --git a/build/pkgs/python3/spkg-configure.m4 b/build/pkgs/python3/spkg-configure.m4
@@ -29,6 +29,9 @@ SAGE_SPKG_CONFIGURE([python3], [
            SAGE_CHECK_PYTHON_FOR_VENV([$ac_path_PYTHON3],
                                            MIN_VERSION, LT_VERSION,
                                            $check_modules, [
+                    AS_IF([[conftest_venv/bin/python3 -m sysconfig | grep '^\sw*\(C\|LD\)FLAGS *=.*[" ]-[IL]' ]] [>& AS_MESSAGE_LOG_FD 2>&1 ], [
+                        AC_MSG_WARN([this is a misconfigured Python whose sysconfig compiler/linker flags contain -I or -L options, which may cause wrong versions of libraries to leak into the build of Python packages - see https://trac.sagemath.org/ticket/31132])
+                    ])
                     dnl It is good
                     ac_cv_path_PYTHON3="$ac_path_PYTHON3"
                     ac_path_PYTHON3_found=:
@@ -45,27 +48,59 @@ SAGE_SPKG_CONFIGURE([python3], [
                 SAGE_CHECK_PYTHON_FOR_VENV([$ac_path_PYTHON3],
                                            MIN_VERSION, LT_VERSION,
                                            $check_modules, [
-                    dnl It is good
-                    ac_cv_path_PYTHON3="$ac_path_PYTHON3"
-                    ac_path_PYTHON3_found=:
-                    AC_MSG_RESULT([yes])
-                    dnl introduction for AC_MSG_RESULT printed by AC_CACHE_CHECK
-                    AC_MSG_CHECKING([for python3 >= ]MIN_VERSION[, < ]LT_VERSION[ with modules $check_modules])
+                    AS_IF([[conftest_venv/bin/python3 -m sysconfig | grep '^\sw*\(C\|LD\)FLAGS *=.*[" ]-[IL]' ]] [>& AS_MESSAGE_LOG_FD 2>&1 ], [
+                        AC_MSG_RESULT([no, this is a misconfigured Python whose sysconfig compiler/linker flags contain -I or -L options, which may cause wrong versions of libraries to leak into the build of Python packages - see https://trac.sagemath.org/ticket/31132; to use it anyway, use ./configure --with-python=$ac_path_PYTHON3])
+                    ], [
+                        dnl It is good
+                        ac_cv_path_PYTHON3="$ac_path_PYTHON3"
+                        ac_path_PYTHON3_found=:
+                        AC_MSG_RESULT([yes])
+                        dnl introduction for AC_MSG_RESULT printed by AC_CACHE_CHECK
+                        AC_MSG_CHECKING([for python3 >= ]MIN_VERSION[, < ]LT_VERSION[ with modules $check_modules])
+                    ])
                 ])
             ])
 	])
       ])
-      AS_IF([test -z "$ac_cv_path_PYTHON3"], [sage_spkg_install_python3=yes])
+      AS_IF([test -z "$ac_cv_path_PYTHON3"], [
+          AC_MSG_NOTICE([to try to use a different system python, use ./configure --with-python=/path/to/python])
+          sage_spkg_install_python3=yes
+      ])
       m4_popdef([MIN_VERSION])
       m4_popdef([LT_VERSION])
     ])
 ],, [
     dnl PRE
 ], [
     dnl POST
-    AS_IF([test x$sage_spkg_install_python3 = xno],
-          [PYTHON_FOR_VENV="$ac_cv_path_PYTHON3"],
-          [SAGE_MACOSX_DEPLOYMENT_TARGET=legacy])
+    AS_IF([test x$sage_spkg_install_python3 = xno], [
+        PYTHON_FOR_VENV="$ac_cv_path_PYTHON3"
+        AS_IF([test "$SAGE_ARCHFLAGS" = "unset"], [
+           AC_MSG_CHECKING([whether $PYTHON_FOR_VENV is configured to build multiarch extensions])
+           AS_IF([[CC="$CC" CXX="$CXX" conftest_venv/bin/python3 -m sysconfig | grep '^\sw*\(C\|LD\)FLAGS *=.*[" ]-arch.* -arch' ]] [>& AS_MESSAGE_LOG_FD 2>&1 ], [
+               AC_MSG_RESULT([yes; disabling it by setting ARCHFLAGS])
+               SAGE_ARCHFLAGS=""
+           ], [
+               AC_MSG_RESULT([no])
+           ])
+        ])
+        AS_IF([test "$SAGE_ARCHFLAGS" != "unset"], [
+            ARCHFLAGS="$SAGE_ARCHFLAGS"
+            export ARCHFLAGS
+        ])
+        AS_IF([test -n "$CFLAGS_MARCH"], [
+            dnl Trac #31228
+            AC_MSG_CHECKING([whether "$CFLAGS_MARCH" works with the C/C++ compilers configured for building extensions for $PYTHON_FOR_VENV])
+            SAGE_PYTHON_CHECK_DISTUTILS([CC="$CC" CXX="$CXX" CFLAGS="$CFLAGS_MARCH" conftest_venv/bin/python3], [
+                AC_MSG_RESULT([yes])
+            ], [
+                AC_MSG_RESULT([no, with these flags, $reason; disabling use of "$CFLAGS_MARCH"])
+                CFLAGS_MARCH=""
+            ])
+        ])
+    ], [
+        SAGE_MACOSX_DEPLOYMENT_TARGET=legacy
+    ])
     AC_SUBST([PYTHON_FOR_VENV])
     AC_SUBST([SAGE_MACOSX_DEPLOYMENT_TARGET])
 

diff --git a/m4/sage_check_python_for_venv.m4 b/m4/sage_check_python_for_venv.m4
@@ -7,6 +7,8 @@ AC_DEFUN([SAGE_CHECK_PYTHON_FOR_VENV], [
                 m4_pushdef([REQUIRED_MODULES], [$4])
                 m4_pushdef([COMMANDS_IF_GOOD], [$5])
 
+                AC_SUBST([SAGE_ARCHFLAGS])
+
                 AC_MSG_CHECKING([... whether ]PYTHON_EXE[ is good])
                 python3_version=`"PYTHON_EXE" --version 2>&1 \
                     | $SED -n -e 's/\([[0-9]]*\.[[0-9]]*\.[[0-9]]*\).*/\1/p'`
@@ -19,6 +21,80 @@ AC_DEFUN([SAGE_CHECK_PYTHON_FOR_VENV], [
                             dnl m4_define([conftest_venv], [config-venv]) .... for debugging only
                             rm -rf conftest_venv
                             AS_IF(["]PYTHON_EXE[" build/bin/sage-venv conftest_venv && conftest_venv/bin/python3 -c "import ]REQUIRED_MODULES["], [
+                                AS_VAR_SET([python3_result], [yes])
+                                SAGE_PYTHON_CHECK_DISTUTILS([CC="$CC" CXX="$CXX" conftest_venv/bin/python3], [
+                                    SAGE_ARCHFLAGS="unset"
+                                ], [
+                                    AS_CASE([$host],
+                                        [*-*-darwin*], [
+                                            dnl #31227: Try if setting ARCHFLAGS to empty fixes it
+                                            SAGE_PYTHON_CHECK_DISTUTILS([CC="$CC" CXX="$CXX" ARCHFLAGS="" conftest_venv/bin/python3], [
+                                                SAGE_ARCHFLAGS=""
+                                            ], [
+                                                AS_VAR_SET([python3_result],
+                                                           ["no, the version is in the supported range, and the modules can be imported, but $reason (even with ARCHFLAGS set to empty)"])
+                                            ])
+                                        ], [
+                                            AS_VAR_SET([python3_result],
+                                                       ["no, the version is in the supported range, and the modules can be imported, but $reason"])
+                                        ])
+                                ])
+                                AS_VAR_IF([python3_result], [yes], [
+                                    dnl these commands are expected to call AC_MSG_RESULT
+                                    COMMANDS_IF_GOOD
+                                ], [
+                                    AC_MSG_RESULT([$python3_result])
+                                ])
+                            ], [
+                                AC_MSG_RESULT([no, the version is in the supported range but cannot import one of the required modules: ]REQUIRED_MODULES)
+                            ])
+                        ], [
+                            AC_MSG_RESULT([no, $python3_version is too recent])
+                        ])
+                    ], [
+                        AC_MSG_RESULT([no, $python3_version is too old])
+                    ])
+                ], [
+                    AC_MSG_RESULT([no, "]PYTHON_EXE[ --version" does not work])
+                ])
+
+
+                m4_popdef([PYTHON_EXE])
+                m4_popdef([MIN_VERSION])
+                m4_popdef([LT_VERSION])
+                m4_popdef([REQUIRED_MODULES])
+                m4_popdef([COMMANDS_IF_GOOD])
+
+])
+
+dnl distutils test
+AC_DEFUN([SAGE_PYTHON_CHECK_DISTUTILS], [
+    m4_pushdef([PYTHON_EXE], [$1])
+    m4_pushdef([COMMANDS_IF_DISTUTILS_GOOD], [$2])
+    m4_pushdef([COMMANDS_IF_DISTUTILS_NOT_GOOD], [$3])
+    SAGE_PYTHON_DISTUTILS_C_CONFTEST
+    dnl (echo "***ENV***:"; env; echo "***SYSCONFIG***"; conftest_venv/bin/python3 -m sysconfig) >& AS_MESSAGE_LOG_FD
+    echo PYTHON_EXE conftest.py --verbose build --build-base=conftest.dir >& AS_MESSAGE_LOG_FD
+    AS_IF([PYTHON_EXE conftest.py --verbose build --build-base=conftest.dir >& AS_MESSAGE_LOG_FD 2>&1 ], [
+        SAGE_PYTHON_DISTUTILS_CXX_CONFTEST
+        echo PYTHON_EXE conftest.py --verbose build --build-base=conftest.dir >& AS_MESSAGE_LOG_FD 2>&1
+        AS_IF([PYTHON_EXE conftest.py --verbose build --build-base=conftest.dir >& AS_MESSAGE_LOG_FD 2>&1 ], [
+            COMMANDS_IF_DISTUTILS_GOOD], [
+            reason="distutils cannot build a C++ 11 extension"
+            COMMANDS_IF_DISTUTILS_NOT_GOOD
+        ])
+    ], [
+       reason="distutils cannot build a C extension"
+       COMMANDS_IF_DISTUTILS_NOT_GOOD
+    ])
+    m4_popdef([PYTHON_EXE])
+    m4_popdef([COMMANDS_IF_DISTUTILS_GOOD])
+    m4_popdef([COMMANDS_IF_DISTUTILS_NOT_GOOD])
+])
+
+dnl Write conftest.py and conftest.c
+AC_DEFUN([SAGE_PYTHON_DISTUTILS_C_CONFTEST], [
+                                rm -rf conftest.*
                                 AC_LANG_PUSH([C])
                                 AC_LANG_CONFTEST([
                                     AC_LANG_SOURCE([[
@@ -54,9 +130,10 @@ modules = list((Extension("config_check_distutils", list(("conftest.c",))),))
 setup(name="config_check_distutils", ext_modules=modules)
 exit(0)
 EOF
-                                dnl (echo "***ENV***:"; env; echo "***SYSCONFIG***"; conftest_venv/bin/python3 -m sysconfig) >& AS_MESSAGE_LOG_FD
-                                echo CC="$CC" CXX="$CXX" conftest_venv/bin/python3 conftest.py --verbose build --build-base=conftest.dir >& AS_MESSAGE_LOG_FD
-                                AS_IF([CC="$CC" CXX="$CXX" conftest_venv/bin/python3 conftest.py --verbose build --build-base=conftest.dir >& AS_MESSAGE_LOG_FD 2>&1 ], [
+])
+
+dnl Write conftest.py and conftest.cpp
+AC_DEFUN([SAGE_PYTHON_DISTUTILS_CXX_CONFTEST], [
                                     rm -rf conftest.*
                                     AC_LANG_PUSH([C++])
                                     AC_LANG_CONFTEST([
@@ -107,31 +184,4 @@ modules = list((Extension("config_check_distutils_cxx", list(("conftest.cpp",)),
 setup(name="config_check_distutils_cxx", ext_modules=modules)
 exit(0)
 EOF
-                                    AS_IF([CC="$CC" CXX="$CXX" conftest_venv/bin/python3 conftest.py --verbose build --build-base=conftest.dir >& AS_MESSAGE_LOG_FD 2>&1 ],
-                                      [COMMANDS_IF_GOOD], [
-                                        AC_MSG_RESULT([no, the version is in the supported range, and the modules can be imported, but distutils cannot build a C++ 11 extension])
-                                    ])
-                                ], [
-                                    AC_MSG_RESULT([no, the version is in the supported range, and the modules can be imported, but distutils cannot build a C extension])
-                                ])
-                            ], [
-                                AC_MSG_RESULT([no, the version is in the supported range but cannot import one of the required modules: ]REQUIRED_MODULES)
-                            ])
-                        ], [
-                            AC_MSG_RESULT([no, $python3_version is too recent])
-                        ])
-                    ], [
-                        AC_MSG_RESULT([no, $python3_version is too old])
-                    ])
-                ], [
-                    AC_MSG_RESULT([no, "]PYTHON_EXE[ --version" does not work])
-                ])
-
-
-                m4_popdef([PYTHON_EXE])
-                m4_popdef([MIN_VERSION])
-                m4_popdef([LT_VERSION])
-                m4_popdef([REQUIRED_MODULES])
-                m4_popdef([COMMANDS_IF_GOOD])
-
 ])
diff --git a/src/bin/sage-env b/src/bin/sage-env
@@ -202,6 +202,16 @@ export F77="$FC"
 export F90="$FC"   # Needed for SciPy
 export F95="$FC"
 
+# For ARCHFLAGS (#31227) we need to distinguish unset and empty.
+# If the environment defines ARCHFLAGS, even when empty, then take that.
+# Otherwise, use the configured value; but if that is "unset", do not set
+# the variable at all.
+if [ "${ARCHFLAGS-unset}" = "unset" ]; then
+    if [ "${SAGE_ARCHFLAGS-unset}" != "unset" ]; then
+        export ARCHFLAGS="${SAGE_ARCHFLAGS}"
+    fi
+fi
+
 # Call with: contains_spaces X${VAR}X
 # i.e., WITHOUT quotes but some character(s) around the environment variable to test.
 # (This function does return false for empty/unset variables.)

diff --git a/src/bin/sage-env-config.in b/src/bin/sage-env-config.in
@@ -50,5 +50,6 @@ CONFIGURED_OBJCXX="@OBJCXX@"
 #######################################
 # Other configuration (not exported, only used in sage-env)
 #######################################
+SAGE_ARCHFLAGS="@SAGE_ARCHFLAGS@"
 SAGE_PKG_CONFIG_PATH="@SAGE_PKG_CONFIG_PATH@"
 SAGE_MACOSX_DEPLOYMENT_TARGET="@SAGE_MACOSX_DEPLOYMENT_TARGET@"
diff --git a/tox.ini b/tox.ini
@@ -372,6 +372,10 @@ setenv =
     macos-python3_xcode:      CONFIG_CONFIGURE_ARGS_1=--with-python=/usr/bin/python3
     # Must manually download and install from https://www.python.org/ftp/python/3.7.7/python-3.7.7-macosx10.9.pkg
     macos-python3_pythonorg:  CONFIG_CONFIGURE_ARGS_1=--with-python=/Library/Frameworks/Python.framework/Versions/3.7/bin/python3
+    # Homebrew keg installs
+    homebrew-python3.7:       CONFIG_CONFIGURE_ARGS_1=--with-python={env:HOMEBREW}/opt/[email protected]/bin/python3
+    homebrew-python3.8:       CONFIG_CONFIGURE_ARGS_1=--with-python={env:HOMEBREW}/opt/[email protected]/bin/python3
+    homebrew-python3.9:       CONFIG_CONFIGURE_ARGS_1=--with-python={env:HOMEBREW}/opt/[email protected]/bin/python3
     # https://github.com/pypa/manylinux
     manylinux-standard:       CONFIG_CONFIGURE_ARGS_1=--with-system-python3=force --with-python=/opt/python/cp38-cp38/bin/python3
     manylinux-python3.6:      CONFIG_CONFIGURE_ARGS_1=--with-system-python3=force --with-python=/opt/python/cp36-cp36m/bin/python3
@@ -383,6 +387,8 @@ setenv =
     #
     gcc_spkg:                 CONFIG_CONFIGURE_ARGS_2=--without-system-gcc
     gcc_9:                    CONFIG_CONFIGURE_ARGS_2=CC=gcc-9 CXX=g++-9 FC=gfortran-9
+    gcc_10:                   CONFIG_CONFIGURE_ARGS_2=CC=gcc-10 CXX=g++-10 FC=gfortran-10
+    gcc_11:                   CONFIG_CONFIGURE_ARGS_2=CC=gcc-11 CXX=g++-11 FC=gfortran-11
     #
     # Resulting full configuration args, including EXTRA_CONFIGURE_ARGS from the user environment
     #