From 3dde52bea9cdc5cfcbfa94d20a28e134907f1a1a Mon Sep 17 00:00:00 2001 From: AlexAndorra Date: Mon, 4 May 2020 19:14:59 +0200 Subject: [PATCH 01/11] Pinned MPL to avoid mutiprocessing MacOS error --- RELEASE-NOTES.md | 3 ++- requirements.txt | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 25fadaaeda..ac6d425ebb 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -27,7 +27,8 @@ - Moved argument division out of `NegativeBinomial` `random` method. Fixes [#3864](https://github.com/pymc-devs/pymc3/issues/3864) in the style of [#3509](https://github.com/pymc-devs/pymc3/pull/3509). - The Dirichlet distribution now raises a ValueError when it's initialized with <= 0 values (see [#3853](https://github.com/pymc-devs/pymc3/pull/3853)). - End of sampling report now uses `arviz.InferenceData` internally and avoids storing - pointwise log likelihood (see [#3883](https://github.com/pymc-devs/pymc3/pull/3883)) + pointwise log likelihood (see [#3883](https://github.com/pymc-devs/pymc3/pull/3883)). +- To avoid crashing multiprocessing on MacOS, matplotlib dependency is pinned to versions < 3.1.1 until [the issue](https://github.com/matplotlib/matplotlib/issues/15410) is resolved (see [#3849](https://github.com/pymc-devs/pymc3/issues/3849)). ## PyMC3 3.8 (November 29 2019) diff --git a/requirements.txt b/requirements.txt index 4b10495e21..378c58e2d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,11 @@ arviz>=0.7.0 -theano>=1.0.4 +fastprogress>=0.2.0 +h5py>=2.7.0 +matplotlib<=3.1.0 numpy>=1.13.0 -scipy>=0.18.1 pandas>=0.18.0 patsy>=0.5.1 -fastprogress>=0.2.0 -h5py>=2.7.0 +scipy>=0.18.1 +theano>=1.0.4 typing-extensions>=3.7.4 contextvars; python_version < '3.7' From 77c1cd0670e40d76c6b8e67aefede86225bc150e Mon Sep 17 00:00:00 2001 From: AlexAndorra Date: Tue, 5 May 2020 16:43:47 +0200 Subject: [PATCH 02/11] Force reinstall MPL in test env to override global ignore-installed flag --- scripts/create_testenv.sh | 49 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/scripts/create_testenv.sh b/scripts/create_testenv.sh index 8970f0e675..a298b2834c 100755 --- a/scripts/create_testenv.sh +++ b/scripts/create_testenv.sh @@ -2,52 +2,51 @@ set -ex # fail on first error, print commands -while test $# -gt 0 -do - case "$1" in - --global) - GLOBAL=1 - ;; - --no-setup) - NO_SETUP=1 - ;; - esac - shift +while test $# -gt 0; do + case "$1" in + --global) + GLOBAL=1 + ;; + --no-setup) + NO_SETUP=1 + ;; + esac + shift done command -v conda >/dev/null 2>&1 || { - echo "Requires conda but it is not installed. Run install_miniconda.sh." >&2; - exit 1; + echo "Requires conda but it is not installed. Run install_miniconda.sh." >&2 + exit 1 } -ENVNAME="${ENVNAME:-testenv}" # if no ENVNAME is specified, use testenv +ENVNAME="${ENVNAME:-testenv}" # if no ENVNAME is specified, use testenv PYTHON_VERSION=${PYTHON_VERSION:-3.6} # if no python specified, use 3.6 -if [ -z ${GLOBAL} ] -then - if conda env list | grep -q ${ENVNAME} - then - echo "Environment ${ENVNAME} already exists, keeping up to date" - else - conda create -n ${ENVNAME} --yes pip python=${PYTHON_VERSION} - fi - source activate ${ENVNAME} +if [ -z ${GLOBAL} ]; then + if conda env list | grep -q ${ENVNAME}; then + echo "Environment ${ENVNAME} already exists, keeping up to date" + else + conda create -n ${ENVNAME} --yes pip python=${PYTHON_VERSION} + fi + source activate ${ENVNAME} fi pip install --upgrade pip conda install --yes mkl-service conda install --yes -c conda-forge python-graphviz - # Install editable using the setup.py # Travis env is unable to import cached mpl sometimes https://github.com/pymc-devs/pymc3/issues/3423 pip install --no-cache-dir --ignore-installed -e . pip install --no-cache-dir --ignore-installed -r requirements-dev.txt +# Pin MPL until https://github.com/matplotlib/matplotlib/issues/15410 is resolved +pip install --no-cache-dir --force-reinstall "matplotlib<3.1.1" + # Install untested, non-required code (linter fails without them) pip install ipython ipywidgets if [ -z ${NO_SETUP} ]; then - python setup.py build_ext --inplace + python setup.py build_ext --inplace fi From 53717d478305e4785fb53ae7f690b105db8da960 Mon Sep 17 00:00:00 2001 From: AlexAndorra Date: Tue, 5 May 2020 16:56:38 +0200 Subject: [PATCH 03/11] Install MPL before parsing requirements --- scripts/create_testenv.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/create_testenv.sh b/scripts/create_testenv.sh index a298b2834c..859f72a0a0 100755 --- a/scripts/create_testenv.sh +++ b/scripts/create_testenv.sh @@ -35,18 +35,17 @@ pip install --upgrade pip conda install --yes mkl-service conda install --yes -c conda-forge python-graphviz -# Install editable using the setup.py +# Pin MPL until https://github.com/matplotlib/matplotlib/issues/15410 is resolved +pip install --no-cache-dir --force-reinstall "matplotlib<3.1.1" # Travis env is unable to import cached mpl sometimes https://github.com/pymc-devs/pymc3/issues/3423 pip install --no-cache-dir --ignore-installed -e . pip install --no-cache-dir --ignore-installed -r requirements-dev.txt -# Pin MPL until https://github.com/matplotlib/matplotlib/issues/15410 is resolved -pip install --no-cache-dir --force-reinstall "matplotlib<3.1.1" - # Install untested, non-required code (linter fails without them) pip install ipython ipywidgets +# Install editable using the setup.py if [ -z ${NO_SETUP} ]; then python setup.py build_ext --inplace fi From 57c535728aa9bd6710cefae3726cd7374fd68606 Mon Sep 17 00:00:00 2001 From: AlexAndorra Date: Tue, 5 May 2020 18:35:16 +0200 Subject: [PATCH 04/11] Specify exact MPL version for tests --- scripts/create_testenv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create_testenv.sh b/scripts/create_testenv.sh index 859f72a0a0..0edfda3eff 100755 --- a/scripts/create_testenv.sh +++ b/scripts/create_testenv.sh @@ -36,7 +36,7 @@ conda install --yes mkl-service conda install --yes -c conda-forge python-graphviz # Pin MPL until https://github.com/matplotlib/matplotlib/issues/15410 is resolved -pip install --no-cache-dir --force-reinstall "matplotlib<3.1.1" +pip install --no-cache-dir --force-reinstall matplotlib==3.1.0 # Travis env is unable to import cached mpl sometimes https://github.com/pymc-devs/pymc3/issues/3423 pip install --no-cache-dir --ignore-installed -e . From babd3f0263921f2649f47936b84180242ea12753 Mon Sep 17 00:00:00 2001 From: AlexAndorra Date: Tue, 5 May 2020 19:20:31 +0200 Subject: [PATCH 05/11] New try for tests, with quotes for matplotlib --- scripts/create_testenv.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/create_testenv.sh b/scripts/create_testenv.sh index 0edfda3eff..ebd834b8d4 100755 --- a/scripts/create_testenv.sh +++ b/scripts/create_testenv.sh @@ -36,7 +36,8 @@ conda install --yes mkl-service conda install --yes -c conda-forge python-graphviz # Pin MPL until https://github.com/matplotlib/matplotlib/issues/15410 is resolved -pip install --no-cache-dir --force-reinstall matplotlib==3.1.0 +pip uninstall matplotlib +pip install "matplotlib==3.1.0" # Travis env is unable to import cached mpl sometimes https://github.com/pymc-devs/pymc3/issues/3423 pip install --no-cache-dir --ignore-installed -e . From f88f74cdcdd62c2b20aaa8c042d5114e57288493 Mon Sep 17 00:00:00 2001 From: AlexAndorra Date: Tue, 5 May 2020 20:40:56 +0200 Subject: [PATCH 06/11] =?UTF-8?q?Don=E2=80=99t=20ask=20for=20confirmation?= =?UTF-8?q?=20of=20uninstall=20deletions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/create_testenv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create_testenv.sh b/scripts/create_testenv.sh index ebd834b8d4..0c113b8503 100755 --- a/scripts/create_testenv.sh +++ b/scripts/create_testenv.sh @@ -36,7 +36,7 @@ conda install --yes mkl-service conda install --yes -c conda-forge python-graphviz # Pin MPL until https://github.com/matplotlib/matplotlib/issues/15410 is resolved -pip uninstall matplotlib +pip uninstall matplotlib --yes pip install "matplotlib==3.1.0" # Travis env is unable to import cached mpl sometimes https://github.com/pymc-devs/pymc3/issues/3423 From 7423f0dc8752c49f4f043fb5803bc4d9268a285f Mon Sep 17 00:00:00 2001 From: AlexAndorra Date: Wed, 6 May 2020 09:24:13 +0200 Subject: [PATCH 07/11] New try installing matplotlib to pass tests --- scripts/create_testenv.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/create_testenv.sh b/scripts/create_testenv.sh index 0c113b8503..90a0bc4780 100755 --- a/scripts/create_testenv.sh +++ b/scripts/create_testenv.sh @@ -35,9 +35,8 @@ pip install --upgrade pip conda install --yes mkl-service conda install --yes -c conda-forge python-graphviz -# Pin MPL until https://github.com/matplotlib/matplotlib/issues/15410 is resolved +# Uninstall then pin Matplotlib until https://github.com/matplotlib/matplotlib/issues/15410 is resolved pip uninstall matplotlib --yes -pip install "matplotlib==3.1.0" # Travis env is unable to import cached mpl sometimes https://github.com/pymc-devs/pymc3/issues/3423 pip install --no-cache-dir --ignore-installed -e . From 05d25c5c290fc03190970e639a7f0c02fa410973 Mon Sep 17 00:00:00 2001 From: AlexAndorra Date: Wed, 6 May 2020 10:04:00 +0200 Subject: [PATCH 08/11] Also remove potential conda matplotlib installation --- scripts/create_testenv.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/create_testenv.sh b/scripts/create_testenv.sh index 90a0bc4780..088dc26512 100755 --- a/scripts/create_testenv.sh +++ b/scripts/create_testenv.sh @@ -37,6 +37,7 @@ conda install --yes -c conda-forge python-graphviz # Uninstall then pin Matplotlib until https://github.com/matplotlib/matplotlib/issues/15410 is resolved pip uninstall matplotlib --yes +conda uninstall matplotlib --yes # Travis env is unable to import cached mpl sometimes https://github.com/pymc-devs/pymc3/issues/3423 pip install --no-cache-dir --ignore-installed -e . From 28080996edef5b42e8957988086bbf32ae37817c Mon Sep 17 00:00:00 2001 From: AlexAndorra Date: Wed, 6 May 2020 10:38:43 +0200 Subject: [PATCH 09/11] Moved MPL from requirements to requirements-dev --- requirements-dev.txt | 1 + requirements.txt | 1 - scripts/create_testenv.sh | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index ec6d74ac00..99aa54aa2e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,6 +3,7 @@ coverage<5.0 graphviz>=0.8.3 ipython Keras>=2.0.8 +matplotlib<=3.1.0 nbsphinx>=0.4.2 nose>=1.3.7 nose-parameterized==0.6.0 diff --git a/requirements.txt b/requirements.txt index 378c58e2d7..738349529c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ arviz>=0.7.0 fastprogress>=0.2.0 h5py>=2.7.0 -matplotlib<=3.1.0 numpy>=1.13.0 pandas>=0.18.0 patsy>=0.5.1 diff --git a/scripts/create_testenv.sh b/scripts/create_testenv.sh index 088dc26512..90a0bc4780 100755 --- a/scripts/create_testenv.sh +++ b/scripts/create_testenv.sh @@ -37,7 +37,6 @@ conda install --yes -c conda-forge python-graphviz # Uninstall then pin Matplotlib until https://github.com/matplotlib/matplotlib/issues/15410 is resolved pip uninstall matplotlib --yes -conda uninstall matplotlib --yes # Travis env is unable to import cached mpl sometimes https://github.com/pymc-devs/pymc3/issues/3423 pip install --no-cache-dir --ignore-installed -e . From 1340758b26b6b929f408ab2301b58611d09e4392 Mon Sep 17 00:00:00 2001 From: AlexAndorra Date: Wed, 6 May 2020 16:02:19 +0200 Subject: [PATCH 10/11] Reintegrate matplotlib pin in requirements file --- requirements-dev.txt | 1 - requirements.txt | 1 + scripts/create_testenv.sh | 4 +++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 99aa54aa2e..ec6d74ac00 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,7 +3,6 @@ coverage<5.0 graphviz>=0.8.3 ipython Keras>=2.0.8 -matplotlib<=3.1.0 nbsphinx>=0.4.2 nose>=1.3.7 nose-parameterized==0.6.0 diff --git a/requirements.txt b/requirements.txt index 738349529c..f8366437c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ arviz>=0.7.0 fastprogress>=0.2.0 h5py>=2.7.0 +matplotlib==3.1.0 numpy>=1.13.0 pandas>=0.18.0 patsy>=0.5.1 diff --git a/scripts/create_testenv.sh b/scripts/create_testenv.sh index 90a0bc4780..58148714f5 100755 --- a/scripts/create_testenv.sh +++ b/scripts/create_testenv.sh @@ -35,7 +35,9 @@ pip install --upgrade pip conda install --yes mkl-service conda install --yes -c conda-forge python-graphviz -# Uninstall then pin Matplotlib until https://github.com/matplotlib/matplotlib/issues/15410 is resolved +# Uninstall then pin matplotlib until https://github.com/matplotlib/matplotlib/issues/15410 is resolved +pip uninstall matplotlib --yes +# Uninstall second matplotlib present on conda venv pip uninstall matplotlib --yes # Travis env is unable to import cached mpl sometimes https://github.com/pymc-devs/pymc3/issues/3423 From 0cf008e43159dd18a26c223b3d509769799aefcd Mon Sep 17 00:00:00 2001 From: AlexAndorra Date: Wed, 6 May 2020 16:29:26 +0200 Subject: [PATCH 11/11] Uninstall third matplotlib on conda env --- scripts/create_testenv.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/create_testenv.sh b/scripts/create_testenv.sh index 58148714f5..8b0ee16a15 100755 --- a/scripts/create_testenv.sh +++ b/scripts/create_testenv.sh @@ -35,10 +35,12 @@ pip install --upgrade pip conda install --yes mkl-service conda install --yes -c conda-forge python-graphviz -# Uninstall then pin matplotlib until https://github.com/matplotlib/matplotlib/issues/15410 is resolved +# Uninstall the three matplotlibs present on conda env pip uninstall matplotlib --yes -# Uninstall second matplotlib present on conda venv pip uninstall matplotlib --yes +pip uninstall matplotlib --yes +# Pin matplotlib until https://github.com/matplotlib/matplotlib/issues/15410 is resolved +pip install --force-reinstall "matplotlib==3.1.0" # Travis env is unable to import cached mpl sometimes https://github.com/pymc-devs/pymc3/issues/3423 pip install --no-cache-dir --ignore-installed -e .