CMake maintenance

.github/workflows/test_build.yml

@@ -77,11 +77,11 @@ jobs:
         run: |
           brew config
           brew tap macaulay2/tap
-          brew install autoconf automake bison ccache cmake gnu-tar libtool make ninja pkg-config yasm
-          brew install bdw-gc boost eigen gdbm readline libatomic_ops libxml2 libomp tbb
-          brew install gmp mpfr ntl [email protected] [email protected] mpsolve glpk
-          brew install frobby memtailor mathic mathicgb givaro fflas-ffpack
-          brew install 4ti2 cohomcalg csdp gfan lrs nauty normaliz topcom
+          brew install autoconf automake bison ccache cmake ctags gnu-tar libtool make ninja pkg-config yasm
+          brew install --with-tbb --only-dependencies macaulay2/tap/M2
+          HOMEBREW_DEPS=`brew deps --1 --include-optional macaulay2/tap/M2 | tr '\n' ';'`
+          HOMEBREW_PREFIX=`brew --prefix`
+          echo "OPT_PREFIX=$HOMEBREW_PREFIX/opt/${HOMEBREW_DEPS//;/;$HOMEBREW_PREFIX/opt/}" >> $GITHUB_ENV
 
 # ----------------------
 #   Install missing tools and libraries for Linux
@@ -138,11 +138,9 @@ jobs:
       - name: Configure Macaulay2 using CMake
         if: matrix.build-system == 'cmake'
         run: |
-          deps=`brew deps --1 --include-optional macaulay2/tap/M2 | tr '\n' ';'`
-          paths=$HOMEBREW_PREFIX/opt/${deps//;/;$HOMEBREW_PREFIX/opt/}
           cmake -S../.. -B. -GNinja \
             -DCMAKE_BUILD_TYPE=MinSizeRel -DBUILD_NATIVE=OFF \
-            -DCMAKE_PREFIX_PATH=$paths \
+            -DCMAKE_PREFIX_PATH=$OPT_PREFIX \
             -DCMAKE_INSTALL_PREFIX=/usr
 
       - name: Build libraries using Ninja
@@ -183,15 +181,17 @@ jobs:
 # ----------------------
 
       - name: Run Tests using CTest
-        if: matrix.build-system == 'cmake' && matrix.os == 'ubuntu-latest' && matrix.compiler == 'default'
+        if: matrix.build-system == 'cmake' && matrix.os == 'ubuntu-latest' && matrix.compiler == 'clang10'
         run: |
+          ./M2 -q --check 1 -e 'exit 0'
+          # TODO: uncomment these when https://github.com/Macaulay2/M2/issues/1849 is fixed
+          #./M2 -q --check 2 -e 'exit 0'
+          #./M2 -q --check 3 -e 'exit 0'
           cmake --build . --target M2-tests
           cmake --build . --target M2-unit-tests
           ctest -j1 --output-on-failure -R "unit-tests"
-          # TODO: remove the excluded normal tests
-          ctest -j4 --output-on-failure -R "normal/" -E "command|program|threads"
           ctest -j4 --output-on-failure -R "ComputationsBook"
-          # TODO: add engine tests
+          # TODO: add Macaulay2/tests/engine when https://github.com/Macaulay2/M2/issues/1213 is fixed
 
       - name: Run Tests using Autotools
         if: matrix.build-system == 'autotools' && matrix.os == 'ubuntu-latest' && matrix.compiler == 'default'

M2/BUILD/docker/README.md

@@ -1,6 +1,6 @@
 # Using Macaulay2 with Docker
 
-Docker is a containerization program for building, shipping, or running applications on different platforms.
+[Docker](https://docs.docker.com/get-docker/) is a containerization program for building, shipping, or running applications on different platforms.
 
 The `Dockerfile` in this directory creates a container image based on Ubuntu 20.04.1 LTS with Macaulay2 installed from the repository. Each subdirectory contains a `Dockerfile` built for other use cases:
 

M2/BUILD/docker/debian/Makefile

@@ -5,6 +5,6 @@ TAG = m2-debian-build
 BUILD_DIR = M2/BUILD/build-docker
 BUILD_OPT = -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr
 
-deb:;       docker run $(VOLUME) -it --entrypoint "" $(TAG) bash -c "cd $(BUILD_DIR); cpack -G DEB"
+deb:;       docker run $(VOLUME) -it --entrypoint "" $(TAG) bash -c "cd M2/$(BUILD_DIR); cpack -G DEB"
 
-lintian:;   docker run $(VOLUME) -it --entrypoint "" $(TAG) bash -c "lintian -I -E --pedantic $(BUILD_DIR)/Macaulay2-*.deb"
+lintian:;   docker run $(VOLUME) -it --entrypoint "" $(TAG) bash -c "lintian -I -E --pedantic M2/$(BUILD_DIR)/Macaulay2-*.deb"

M2/BUILD/docker/fedora/Makefile

@@ -5,6 +5,6 @@ TAG = m2-fedora-build
 BUILD_DIR = M2/BUILD/build-docker
 BUILD_OPT = -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr
 
-rpm:;       docker run $(VOLUME) -it --entrypoint "" $(TAG) bash -c "cd $(BUILD_DIR); cpack -G RPM"
+rpm:;       docker run $(VOLUME) -it --entrypoint "" $(TAG) bash -c "cd M2/$(BUILD_DIR); cpack -G RPM"
 
-rpmlint:;   docker run $(VOLUME) -it --entrypoint "" $(TAG) bash -c "rpmlint $(BUILD_DIR)/Macaulay2-*.rpm"
+rpmlint:;   docker run $(VOLUME) -it --entrypoint "" $(TAG) bash -c "rpmlint M2/$(BUILD_DIR)/Macaulay2-*.rpm"

M2/BUILD/docker/ubuntu/Makefile

@@ -13,13 +13,13 @@ git config --global user.email "macaulay@docker"
 git config --global user.name "Macaulay"
 
 ## Full build
-cmake -GNinja -S M2/M2 -B $(BUILD_DIR) $(BUILD_OPT)
-cmake --build $(BUILD_DIR) --target build-libraries # build-programs
-cmake --build $(BUILD_DIR) --target M2-core # install-packages
-cmake --build $(BUILD_DIR) --target M2-emacs
+cmake -GNinja -S M2/M2 -B M2/$(BUILD_DIR) $(BUILD_OPT)
+cmake --build M2/$(BUILD_DIR) --target build-libraries # build-programs
+cmake --build M2/$(BUILD_DIR) --target M2-core # install-packages
+cmake --build M2/$(BUILD_DIR) --target M2-emacs
 ## Changes to the docker container after it is created are ephemeral,
 ## but this is how one would install Macaulay2:
-# cmake --install $(BUILD_DIR)
+# cmake --install M2/$(BUILD_DIR)
 endef
 export M2_BUILD_SCRIPT
 
@@ -29,4 +29,4 @@ build: build-image
 	docker run $(VOLUME) -it --entrypoint="" $(TAG) bash -c "$$M2_BUILD_SCRIPT"
 
 # For anything else, run it in the build directory within the container
-%:;	docker run $(VOLUME) -it --entrypoint="" $(TAG) cmake --build $(M2_HOME)/$(BUILD_DIR) --target $@
+%:;	docker run $(VOLUME) -it --entrypoint="" $(TAG) cmake --build $(M2_HOME)/M2/$(BUILD_DIR) --target $@

M2/INSTALL-CMake.md

@@ -6,12 +6,17 @@ Building Macaulay2 from Source using CMake
 ## Why CMake?
 CMake is a cross-platform system for generating build environments using native tools such as Makefiles and Ninja or IDEs such as Xcode and Visual Studio. See this article on [why the KDE project switched to CMake](https://lwn.net/Articles/188693/) and this list of [cool CMake features](https://gitlab.kitware.com/cmake/community/-/wikis/doc/cmake/Really-Cool-CMake-Features).
 
+Also see [this guide](BUILD/docker/README.md) for building and packaging Macaulay2 in a Docker container.
+
 ## Getting started
 [Download](https://cmake.org/download/) the latest CMake for your platform.
+CMake is availeble through [Homebrew](https://brew.sh/) for both Mac OS X and Linux distributions.
 If using a packaged distribution, confirm using `cmake --version` that you have version at least 3.15.
+This build system is tested on GCC 6+, Clang 6+, and Xcode 9+ compilers.
 
-**NOTE**: This build system is tested on GCC, Clang, and AppleClang compilers. Optionally, install `ccache` for caching compiler artifacts and `ninja-build` (`ninja` on Brew) for optimized parallel builds.
+**TIP**: install `ccache` for caching compiler artifacts and `ninja-build` (`ninja` on Brew) for optimized parallel builds.
 
+#### Requirements
 There are various tools needed to compile Macaulay2 dependencies.
 - On Debian/Ubuntu, install `autoconf build-essential bison libtool pkg-config yasm`.
 - On Fedora/CentOS, install `autoconf automake bison libtool pkg-config yasm`.
@@ -27,8 +32,14 @@ Finally, there are 2 optional libraries that help with building other requiremen
 - On Fedora/CentOS, install `libomp-devel tbb-devel`.
 - On Mac OS X, using Homebrew, install `libomp tbb`.
 
-**NOTE**: the source directory must not contain any build artifacts from an in-source build. If you have built Macaulay2 in-source before, clean the build artifacts first by running `make clean distclean` in the source directory.
+**TIP**: x86_64 binary packages for all dependencies on Mac OS X 10.15+ and Linux distributions are available through the [Macaulay2 tap](https://github.com/Macaulay2/homebrew-tap/) for Homebrew. To download the dependencies this way run:
+```
+brew tap Macaulay2/tap
+brew install --only-dependencies macaulay2/tap/M2
+```
+and append ``-DCMAKE_PREFIX_PATH=`brew --prefix` `` to an invocation of CMake prior to starting the build so that CMake can find the dependencies installed through Homebrew.
 
+#### Quick build
 A quick build involves the following steps:
 ```
 git clone https://github.com/Macaulay2/M2.git
@@ -40,6 +51,8 @@ cmake --install M2/M2/BUILD/build
 ```
 Each step is explained separately in the next section.
 
+**NOTE**: the source directory must not contain any build artifacts from an in-source build. If you have built Macaulay2 in-source before, clean the build artifacts first by running `make clean distclean` in the source directory.
+
 ### Building Macaulay2
 1. Clone Macaulay2:
 ```
@@ -117,6 +130,10 @@ cpack -G DEB	# requires dpkg
 cpack -G RPM	# requires rpmbuild
 ```
 
+**TIP**: see [this guide](BUILD/docker/README.md) for packaging Macaulay2 for other Linux distributions using Docker.
+
+**NOTE**: Macaulay2 is packaged for Mac OS X as a Homebrew bottle available through the [Macaulay2 tap](https://github.com/Macaulay2/homebrew-tap).
+
 
 ## Advanced cached flags
 Within the build environment, you can:
@@ -261,13 +278,12 @@ Note: rerun the corresponding `build-[library]-install` target after the targets
 The main targets for building Macaulay2 are:
 - `M2-engine`: build the `libM2-engine` library
 - `M2-binary`: build the Macaulay2 executable
-- `M2-core`: build (and install) the core
-- `M2-emacs`: build the M2-mode package for Emacs
+- `M2-core`: generate and copy the Core package
+- `M2-emacs`: generate the M2-mode package for Emacs
 
 In addition, the following targets are available:
 - `scc1`: build the Safe C Compiler
 - `M2-interpreter`: translate `.d` and `.dd` sources into C and C++ sources
-- `M2-regex`: build the vendored GNU regex library
 - `M2-supervisor`: build the Macaulay2 multithreading supervisor
 - `M2-tests`: build various miscellaneous tests (currently `ComputationsBook`)
 
@@ -286,7 +302,7 @@ For specific packages, the following targets are also available:
 
 ### Targets for Generating Macaulay2 Syntax Highlighters
 - `M2-emacs`: generate the [M2.el](https://github.com/Macaulay2/M2-emacs) package for Emacs
-- `M2-editors`: generate various syntax highlighting grammar files
+- `M2-editors`: generate various syntax highlighting grammar files, such as [this one](https://github.com/Macaulay2/language-macaulay2)
 
 
 ## FAQ
@@ -302,19 +318,6 @@ cmake -DCMAKE_SYSTEM_PREFIX_PATH=`brew --prefix` .
 ```
 </details>
 
-<details>
-<summary>Building from the source code tarfile</summary>
-
-When building from a source code tarfile (e.g. downloaded from GitHub, rather than a git clone), you may see an error like this:
-```
-  No download info given for 'build-bdwgc' and its source directory: ...
-```
-To resolve this, you should download and extract the source tarfiles for each git submodule in the `M2/submodules` directory. Also, you may need to manually turn off submodule updates:
-```
-cmake -DGIT_SUBMODULE=off .
-```
-</details>
-
 <details>
 <summary><code>/usr/include/c++/10.1.0/bits/unique_ptr.h:594:9: error: no matching function for call to std::__uniq_ptr_data</code> when using GCC 10 or Clang 10</summary>
 

M2/Macaulay2/editors/CMakeLists.txt

@@ -33,9 +33,14 @@ add_custom_command(OUTPUT ${GRAMMAR_FILES}
 ###############################################################################
 ## Generating the M2-mode package for Emacs
 
+if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/emacs/make-M2-emacs-help.m2)
+  message(WARNING "Macaulay2/editors/emacs submodule is missing; skipping M2-emacs target")
+  return()
+endif()
+
 # TODO: only the emacs target is installed in usr-dist by default
 set(M2_INSTALL_EMACSDIR
-  ${M2_DIST_PREFIX}/${M2_INSTALL_DATAROOTDIR}/emacs/site-lisp/Macaulay2)
+  ${M2_DIST_PREFIX}/${M2_INSTALL_DATAROOTDIR}/emacs/site-lisp/macaulay2)
 file(COPY emacs/ DESTINATION ${M2_INSTALL_EMACSDIR}
   FILES_MATCHING PATTERN "M2*" PATTERN "*.in" EXCLUDE)
 

M2/Macaulay2/m2/CMakeLists.txt

@@ -2,6 +2,7 @@
 ## This directory contains the rules for building M2-core
 
 set(CORE_DIR ${M2_DIST_PREFIX}/${M2_INSTALL_DATADIR}/Core)
+set(CORE_TESTS ${M2_DIST_PREFIX}/${M2_INSTALL_DOCDIR}/Core/tests)
 set(M2_ARGS -q --silent --stop -e errorDepth=0 --no-preload --no-tvalues)
 
 # List of Core sources
@@ -29,4 +30,8 @@ foreach(filename IN LISTS M2LIST ITEMS
   file(COPY ${filename} DESTINATION ${CORE_DIR})
 endforeach()
 
+## Copy Core tests from Macaulay2/tests/normal to share/doc/Macaulay2/Core/tests
+file(COPY ${CMAKE_SOURCE_DIR}/Macaulay2/tests/normal/ DESTINATION ${CORE_TESTS}
+  FILES_MATCHING PATTERN "*.m2")
+
 # TODO: handle Style here.

M2/Macaulay2/m2/help.m2

@@ -443,8 +443,10 @@ viewHelp Thing  := key -> (
         else error("missing documentation index: ", frontpage, ". Run makePackageIndex() or start M2 without -q"))
     else viewHelp makeDocumentTag key)
 viewHelp DocumentTag := tag -> (
-    docpage := concatenate htmlFilename ( tag = getPrimaryTag tag );
-    if fileExists docpage then show URL { docpage } else show help tag)
+    rawdoc := fetchAnyRawDocumentation tag;
+    if ( tag' := getOption(rawdoc, symbol DocumentTag) ) =!= null
+    and fileExists( docpage := concatenate htmlFilename tag' )
+    then show URL { docpage } else show help tag)
 viewHelp ZZ := i -> seeAbout(viewHelp, i)
 
 viewHelp = new Command from viewHelp

M2/Macaulay2/m2/testing.m2

@@ -37,10 +37,14 @@ captureTestResult := (desc, teststring, pkg, usermode) -> (
     if match("no-check-flag", teststring) then (
 	checkmsg("skipping", desc);
 	return true);
+    -- TODO: remove this when capture uses ArgQ
+    if usermode === not noinitfile then
     -- try capturing in the same process
     if isCapturable(teststring, pkg, true) then (
 	checkmsg("capturing", desc);
-	(err, output) := capture(teststring, PackageExports => pkg, UserMode => usermode);
+	-- TODO: adjust and pass argumentMode, instead. This can be done earlier, too.
+	-- Note: UserMode option of capture is not related to UserMode option of check
+	(err, output) := capture(teststring, PackageExports => pkg, UserMode => false);
 	if err then printerr "capture failed; retrying ..." else return true);
     -- fallback to using an external process
     checkmsg("running", desc);

M2/Macaulay2/packages/Text.m2

@@ -566,7 +566,7 @@ document {
 	 ( BLOCKQUOTE PRE s, value s )),
      PARA {
 	 TEX ///The macro can be used at any point after:
-	 $$ 0 \to 2\OO_{\\P^3}(-3) \to 3\OO_{\\P^3}(-2) \to \OO_{\\P^3} \to \OO_C \to 0 $$///},
+	 $$ 0 \to 2\OO_{\PP^3}(-3) \to 3\OO_{\PP^3}(-2) \to \OO_{\PP^3} \to \OO_C \to 0 $$///},
      SeeAlso => {tex, texMath, (show, TEX)}
      }
 

M2/cmake/FindAtomicOps.cmake

@@ -11,12 +11,12 @@
 # Redistribution and use is allowed according to the terms of the BSD license.
 
 find_path(ATOMICOPS_INCLUDE_DIR NAMES atomic_ops.h
-  HINTS /usr/local/opt/libatomic_ops
+  HINTS ${HOMEBREW_PREFIX}/opt/libatomic_ops/include
   PATHS ${INCLUDE_INSTALL_DIR} ${CMAKE_INSTALL_PREFIX}/include
   )
 
 find_library(ATOMICOPS_LIBRARY NAMES atomic_ops
-  HINTS /usr/local/opt/libatomic_ops
+  HINTS ${HOMEBREW_PREFIX}/opt/libatomic_ops/lib
   PATHS ${LIB_INSTALL_DIR} ${CMAKE_INSTALL_PREFIX}/lib
   )
 

M2/cmake/FindGDBM.cmake

@@ -11,12 +11,12 @@
 # Redistribution and use is allowed according to the terms of the BSD license.
 
 find_path(GDBM_INCLUDE_DIR NAMES gdbm.h
-  HINTS /usr/local/opt/gdbm
+  HINTS ${HOMEBREW_PREFIX}/opt/gdbm/include
   PATHS ${INCLUDE_INSTALL_DIR} ${CMAKE_INSTALL_PREFIX}/include
   )
 
 find_library(GDBM_LIBRARY NAMES gdbm
-  HINTS /usr/local/opt/gdbm
+  HINTS ${HOMEBREW_PREFIX}/opt/gdbm/lib
   PATHS ${LIB_INSTALL_DIR} ${CMAKE_INSTALL_PREFIX}/lib
   )
 

M2/cmake/FindHistory.cmake

@@ -1,13 +1,11 @@
-# Locale GNU history library
+# Locate GNU history library
 
-find_path(HISTORY_INCLUDE_DIR
-  NAMES readline/history.h
-  PATHS ${READLINE_ROOT_DIR}/include ${READLINE_INCLUDE_DIR}
+find_path(HISTORY_INCLUDE_DIR NAMES readline/history.h
+  HINTS ${READLINE_ROOT_DIR}/include
   )
 
-find_library(HISTORY_LIBRARY
-  NAMES history
-  PATHS ${READLINE_ROOT_DIR}/lib
+find_library(HISTORY_LIBRARY NAMES history
+  HINTS ${READLINE_ROOT_DIR}/lib
   )
 
 # handle the QUIETLY and REQUIRED arguments and set XXX_FOUND to TRUE if all listed variables are TRUE

M2/cmake/FindReadline.cmake

@@ -1,4 +1,4 @@
-# - Try to find readline include dirs and libraries 
+# Try to find readline include dirs and libraries
 #
 # Usage of this module as follows:
 #
@@ -23,7 +23,7 @@
 
 find_path(READLINE_ROOT_DIR
     NAMES include/readline/readline.h
-    PATHS /usr/local/opt/readline/ /opt/local/ /usr/local/ /usr/
+    PATHS ${HOMEBREW_PREFIX}/opt/readline /opt/local /usr/local /usr
     NO_DEFAULT_PATH
 )
 

M2/cmake/FindTBB.cmake

@@ -57,6 +57,7 @@ FIND_PATH (TBB_ROOT_DIR
         ENV TBB22_INSTALL_DIR
         ENV TBB21_INSTALL_DIR
         ENV TBB_ROOT_DIR
+        ${HOMEBREW_PREFIX}/opt/tbb
   DOC "TBB root directory")
 
 FIND_PATH (TBB_INCLUDE_DIR

M2/cmake/build-libraries.cmake

@@ -118,7 +118,7 @@ if(GIT_FOUND AND EXISTS "${CMAKE_SOURCE_DIR}/../.git")
       WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
       RESULT_VARIABLE GIT_SUBMOD_RESULT)
     if(NOT GIT_SUBMOD_RESULT EQUAL "0")
-      message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
+      message(WARNING "git submodule update failed, please checkout submodules manually")
     endif()
   endif()
 endif()

M2/cmake/check-libraries.cmake

@@ -16,11 +16,14 @@ message(CHECK_START " Checking for existing libraries and programs")
 ###############################################################################
 ## Program requirements: (git and bison are checked for elsewhere)
 find_package(PkgConfig	REQUIRED QUIET)
-find_package(Doxygen)
-find_package(Sphinx)
-find_program(MAKE  NAMES make gmake nmake)
+find_program(MAKE  NAMES gmake make)
 find_program(ETAGS NAMES etags)
 
+if(BUILD_DOCS)
+  find_package(Doxygen)
+  find_package(Sphinx)
+endif()
+
 ###############################################################################
 ## Requirement	Debian package	RPM package	Homebrew package
 #   Threads	libc6-dev	glibc-headers	N/A