From 30e624abbc52b5a5a060a4eca801e8fda303937f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 15 Nov 2020 16:07:18 -0800 Subject: [PATCH 01/13] tox.ini: Add local-sudo --- tox.ini | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tox.ini b/tox.ini index fee460bd22d..754c4fb306f 100644 --- a/tox.ini +++ b/tox.ini @@ -409,6 +409,14 @@ commands = local-conda-standard: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/conda.txt`; {env:SETENV} && {env:CONDA_PREFIX}/bin/conda install --yes --quiet $PACKAGES' local-conda-maximal: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/conda.txt`; {env:SETENV} && {env:CONDA_PREFIX}/bin/conda install --yes --quiet $PACKAGES' # + # local-sudo + # + local-sudo: bash -c '$(build/bin/sage-print-system-package-command {env:SYSTEM} update) #' + local-sudo: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install $PACKAGES)' + # -standard and -maximal are for now the same. This should be fixed when refactoring write-dockerfile. + local-sudo-standard: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install $PACKAGES)' + local-sudo-maximal: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install $PACKAGES)' + # # All "local" environments # # Install symbolic links "config.log" and "logs" in SAGE_ROOT so that log files are written into the tox log directory. From 9472d09685c8f64f2bd15030edfadd94d3e85fbf Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Mon, 16 Nov 2020 01:27:47 +0100 Subject: [PATCH 02/13] Use tox to install packages --- .github/workflows/ci-wsl.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-wsl.yml b/.github/workflows/ci-wsl.yml index 9808b919bf4..5b6533d6a5c 100644 --- a/.github/workflows/ci-wsl.yml +++ b/.github/workflows/ci-wsl.yml @@ -31,7 +31,7 @@ jobs: & wsl sudo add-apt-repository ppa:deadsnakes/ppa # Install & wsl sudo apt-get update -y - & wsl sudo apt-get install -y @packages + & wsl sudo apt-get install -y tox - name: Build run: wsl tox -e local -- SAGE_NUM_THREADS=4 build env: @@ -42,7 +42,7 @@ jobs: MAKE: "make -j2" WSLENV: EXTRA_CONFIGURE_ARGS:MAKE - name: Test - run: wsl tox -e local -- SAGE_NUM_THREADS=4 ptest + run: wsl tox -e local-sudo-ubuntu-standard -- SAGE_NUM_THREADS=4 ptest - name: Prepare logs artifact run: mkdir -p "artifacts/logs"; cp -r .tox/*/log "artifacts/logs" shell: bash From 765b840d68c0a4637039e10f180c6685116c3a67 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Mon, 16 Nov 2020 01:30:26 +0100 Subject: [PATCH 03/13] Also for the build --- .github/workflows/ci-wsl.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-wsl.yml b/.github/workflows/ci-wsl.yml index 5b6533d6a5c..190e587f060 100644 --- a/.github/workflows/ci-wsl.yml +++ b/.github/workflows/ci-wsl.yml @@ -33,7 +33,7 @@ jobs: & wsl sudo apt-get update -y & wsl sudo apt-get install -y tox - name: Build - run: wsl tox -e local -- SAGE_NUM_THREADS=4 build + run: wsl tox -e local-sudo-ubuntu-standard -- SAGE_NUM_THREADS=4 build env: # WSL runs everything as root, so we have to enable build as root user EXTRA_CONFIGURE_ARGS: "--enable-build-as-root" From 942149ee76a97628046042b7255a2d31630e7721 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Mon, 16 Nov 2020 01:39:16 +0100 Subject: [PATCH 04/13] Add confirmation --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 754c4fb306f..aebc9b37943 100644 --- a/tox.ini +++ b/tox.ini @@ -412,10 +412,10 @@ commands = # local-sudo # local-sudo: bash -c '$(build/bin/sage-print-system-package-command {env:SYSTEM} update) #' - local-sudo: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install $PACKAGES)' + local-sudo: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install -y $PACKAGES)' # -standard and -maximal are for now the same. This should be fixed when refactoring write-dockerfile. - local-sudo-standard: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install $PACKAGES)' - local-sudo-maximal: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install $PACKAGES)' + local-sudo-standard: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install -y $PACKAGES)' + local-sudo-maximal: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install -y $PACKAGES)' # # All "local" environments # From 75ecd11e878e34a4c508d2f983b6d7527e689f94 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 15 Nov 2020 16:57:09 -0800 Subject: [PATCH 05/13] tox.ini (local-sudo): Also use ...-bootstrap.txt --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 754c4fb306f..16650ede36d 100644 --- a/tox.ini +++ b/tox.ini @@ -412,7 +412,7 @@ commands = # local-sudo # local-sudo: bash -c '$(build/bin/sage-print-system-package-command {env:SYSTEM} update) #' - local-sudo: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install $PACKAGES)' + local-sudo: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/{env:SYSTEM}*.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install $PACKAGES)' # -standard and -maximal are for now the same. This should be fixed when refactoring write-dockerfile. local-sudo-standard: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install $PACKAGES)' local-sudo-maximal: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install $PACKAGES)' From 0dc79eb4b542a11aa8c226ca8cf9f0e2306698eb Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 15 Nov 2020 17:35:51 -0800 Subject: [PATCH 06/13] build/bin/sage-print-system-package-command: Handle --no-install-recommends, --yes for systems for which write-dockerfile.sh knows these flags --- build/bin/sage-print-system-package-command | 28 +++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/build/bin/sage-print-system-package-command b/build/bin/sage-print-system-package-command index 839085be093..2117a26b094 100755 --- a/build/bin/sage-print-system-package-command +++ b/build/bin/sage-print-system-package-command @@ -21,6 +21,12 @@ do --prompt) PROMPT=' $ ' ;; + --no-install-recommends) + NO_INSTALL_RECOMMENDS=yes + ;; + --yes) + YES=yes + ;; -*) echo >&2 "$0: unknown option $1" exit 1 @@ -37,6 +43,7 @@ if [ -z "$system" -o -z "$command" ]; then exit 1 fi system_packages="$*" +options= shopt -s extglob case $system:$command in homebrew*:setup-build-env) @@ -56,22 +63,33 @@ case $system:$command in echo "${PROMPT}${SUDO}apt-get $command $system_packages" ;; @(debian*|ubuntu*):*) - [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}apt-get $command $system_packages" + [ "$NO_INSTALL_RECOMMENDS" = yes ] && options="$options --no-install-recommends" + [ "$YES" = yes ] && options="$options --yes" + [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}apt-get $command $options $system_packages" ;; @(fedora*|redhat*|centos*):install) - [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}yum install $system_packages" + [ "$YES" = yes ] && options="$options -y" + [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}yum install $options $system_packages" ;; gentoo*:install) [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}emerge $system_packages" ;; + arch*:update) + echo "${PROMPT}${SUDO}pacman -Sy" + ;; arch*:install) - [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}pacman -S $system_packages" + [ "$YES" = yes ] && options="$options --noconfirm" + [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}pacman -S $options $system_packages" + ;; + void*:update) + echo "${PROMPT}${SUDO}xbps-install -Su" ;; - void*:install) - [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}xbps-install $system_packages" + [ "$YES" = yes ] && options="$options --yes" + [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}xbps-install $options $system_packages" ;; *conda*:install) + [ "$YES" = yes ] && options="$options --yes" [ -n "$system_packages" ] && echo "${PROMPT}conda install $system_packages" ;; homebrew*:install) From 050dcb8275b3a7e0db4c7ae3503428107bb5bf17 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 15 Nov 2020 17:38:47 -0800 Subject: [PATCH 07/13] tox.ini (local-sudo): Use --yes --no-install-recommends --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 16650ede36d..8e55c945182 100644 --- a/tox.ini +++ b/tox.ini @@ -412,10 +412,10 @@ commands = # local-sudo # local-sudo: bash -c '$(build/bin/sage-print-system-package-command {env:SYSTEM} update) #' - local-sudo: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/{env:SYSTEM}*.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install $PACKAGES)' + local-sudo: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/{env:SYSTEM}*.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo --yes --no-install-recommends install $PACKAGES)' # -standard and -maximal are for now the same. This should be fixed when refactoring write-dockerfile. - local-sudo-standard: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install $PACKAGES)' - local-sudo-maximal: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo install $PACKAGES)' + local-sudo-standard: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo --yes --no-install-recommends install $PACKAGES)' + local-sudo-maximal: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo --yes --no-install-recommends install $PACKAGES)' # # All "local" environments # From c8fbe0beaec3a404d330aaebdf37186e9c6becec Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 15 Nov 2020 17:57:28 -0800 Subject: [PATCH 08/13] tox.ini (local-sudo): Ignore errors when IGNORE_MISSING_SYSTEM_PACKAGES=yes --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 8e55c945182..8476dde809a 100644 --- a/tox.ini +++ b/tox.ini @@ -412,10 +412,10 @@ commands = # local-sudo # local-sudo: bash -c '$(build/bin/sage-print-system-package-command {env:SYSTEM} update) #' - local-sudo: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/{env:SYSTEM}*.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo --yes --no-install-recommends install $PACKAGES)' + local-sudo: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/{env:SYSTEM}*.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo --yes --no-install-recommends install $PACKAGES) || [ "$IGNORE_MISSING_SYSTEM_PACKAGES" = yes ] && echo "(ignoring errors)" ' # -standard and -maximal are for now the same. This should be fixed when refactoring write-dockerfile. - local-sudo-standard: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo --yes --no-install-recommends install $PACKAGES)' - local-sudo-maximal: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo --yes --no-install-recommends install $PACKAGES)' + local-sudo-standard: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo --yes --no-install-recommends install $PACKAGES) || [ "$IGNORE_MISSING_SYSTEM_PACKAGES" = yes ] && echo "(ignoring errors)" ' + local-sudo-maximal: bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo --yes --no-install-recommends install $PACKAGES) || [ "$IGNORE_MISSING_SYSTEM_PACKAGES" = yes ] && echo "(ignoring errors)" ' # # All "local" environments # From ff34897cac35061afd348cc08a6559f95f406f14 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 15 Nov 2020 19:00:18 -0800 Subject: [PATCH 09/13] tox.ini (local): Guess the package system if it is not provided as a factor --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 8476dde809a..ce11cdc35f6 100644 --- a/tox.ini +++ b/tox.ini @@ -132,6 +132,8 @@ setenv = maximal: TYPE_PATTERN=@(standard|optional) # local envs need HOME set, also Docker 19.03 needs HOME {local,docker}: HOME={envdir} + # for local envs we can guess the package system if it is not provided as a factor + local: SYSTEM=$(build/bin/sage-guess-package-system) # # default tag is "latest" # From 6ff45c41ce97531cc845456941a982c9b583d5f7 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Mon, 16 Nov 2020 11:14:22 +0100 Subject: [PATCH 10/13] Cleanup workflow --- .github/workflows/ci-wsl.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-wsl.yml b/.github/workflows/ci-wsl.yml index 190e587f060..fc605c4dacf 100644 --- a/.github/workflows/ci-wsl.yml +++ b/.github/workflows/ci-wsl.yml @@ -18,20 +18,8 @@ jobs: Ubuntu\ubuntu2004.exe install --root - name: Install dependencies run: | - Function ExtractPackages - { - param($path) - @(Get-Content $path | Where-Object { !$_.StartsWith("#") }) - } - $packages = Get-ChildItem build/pkgs -recurse -Include debian.txt, debian-bootstrap.txt | foreach-object { ExtractPackages $_.FullName } | sort -Unique - # Add tox which is use to build - $packages += 'tox' - echo "Install the following packages:" @packages - # Add deadsnakes so that libpython3.7-dev is found - & wsl sudo add-apt-repository ppa:deadsnakes/ppa - # Install - & wsl sudo apt-get update -y - & wsl sudo apt-get install -y tox + wsl sudo apt-get update -y + wsl sudo apt-get install -y tox - name: Build run: wsl tox -e local-sudo-ubuntu-standard -- SAGE_NUM_THREADS=4 build env: From 53481440a24a3914430dcbf1cf4e19e4aab5267c Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Mon, 16 Nov 2020 11:31:54 +0100 Subject: [PATCH 11/13] Replace texlive-generic-extra --- build/pkgs/_recommended/distros/debian.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/pkgs/_recommended/distros/debian.txt b/build/pkgs/_recommended/distros/debian.txt index 8cd6a185445..11a7927b3f2 100644 --- a/build/pkgs/_recommended/distros/debian.txt +++ b/build/pkgs/_recommended/distros/debian.txt @@ -1,7 +1,7 @@ # From https://wiki.sagemath.org/prerequisitesUbuntu # to generate pdf documentation -texlive-generic-extra +texlive-plain-generic # to convert Jupyter notebooks to pdf texlive-xetex # to generate pdf documentation From 0fd029f1e2b1e8384d14995c361f90e331b784b9 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Mon, 16 Nov 2020 11:36:31 +0100 Subject: [PATCH 12/13] Narrow down events that trigger the workflow --- .github/workflows/ci-wsl.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-wsl.yml b/.github/workflows/ci-wsl.yml index fc605c4dacf..7c5cce70cb3 100644 --- a/.github/workflows/ci-wsl.yml +++ b/.github/workflows/ci-wsl.yml @@ -1,11 +1,19 @@ name: Build & Test WSL -on: [push, pull_request] +on: + pull_request: + types: [opened, synchronize] + push: + tags: + - '*' + branches: + - 'public/build/**wsl**' + workflow_dispatch: jobs: windows: runs-on: windows-latest - name: Windows (using WSL) + name: Ubuntu 20.04 # Following https://trac.sagemath.org/ticket/25206#comment:63 steps: - name: Configure git From 3ccae0258309b682d6429cafbc656e90894d6ebe Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 16 Nov 2020 18:27:04 -0800 Subject: [PATCH 13/13] build/pkgs/_recommended/distros/debian.txt: Update latex package --- build/pkgs/_recommended/distros/debian.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/pkgs/_recommended/distros/debian.txt b/build/pkgs/_recommended/distros/debian.txt index 11a7927b3f2..8041435c58d 100644 --- a/build/pkgs/_recommended/distros/debian.txt +++ b/build/pkgs/_recommended/distros/debian.txt @@ -1,7 +1,7 @@ # From https://wiki.sagemath.org/prerequisitesUbuntu # to generate pdf documentation -texlive-plain-generic +texlive-latex-extra # to convert Jupyter notebooks to pdf texlive-xetex # to generate pdf documentation