From 9d799c4b3e7cb6791ba1cc00c1cb31f42a38355c Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 9 Oct 2024 12:27:01 -0500 Subject: [PATCH 01/11] sync README with pyproject.toml (run with --no-verify) --- .pre-commit-config.yaml | 11 ++++ README.rst | 2 + tools/hooks/sync_dependencies.py | 87 ++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100755 tools/hooks/sync_dependencies.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 951ae12d839..67f7f1ab2be 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -63,6 +63,17 @@ repos: - id: toml-sort-fix files: pyproject.toml + # dependencies + - repo: local + hooks: + - id: dependency-sync + name: Sync dependency list between pyproject.toml and README.rst + language: python + entry: ./tools/hooks/sync_dependencies.py + files: README\.rst$ + additional_dependencies: ["mne"] + + # The following are too slow to run on local commits, so let's only run on CIs: # # - repo: https://github.com/pre-commit/mirrors-mypy diff --git a/README.rst b/README.rst index 745e74849c3..295503eb547 100644 --- a/README.rst +++ b/README.rst @@ -73,6 +73,7 @@ Dependencies The minimum required dependencies to run MNE-Python are: +.. ↓↓↓ BEGIN CORE DEPS LIST. DO NOT EDIT! HANDLED BY PRE-COMMIT HOOK ↓↓↓ - `Python `__ ≥ 3.9 - `NumPy `__ ≥ 1.24 - `SciPy `__ ≥ 1.10 @@ -82,6 +83,7 @@ The minimum required dependencies to run MNE-Python are: - `Jinja2 `__ - `decorator `__ - `lazy_loader `__ +.. ↑↑↑ END CORE DEPS LIST. DO NOT EDIT! HANDLED BY PRE-COMMIT HOOK ↑↑↑ For full functionality, some functions require: diff --git a/tools/hooks/sync_dependencies.py b/tools/hooks/sync_dependencies.py new file mode 100755 index 00000000000..cc5bb979355 --- /dev/null +++ b/tools/hooks/sync_dependencies.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python + +import re +from importlib.metadata import metadata +from pathlib import Path + +from mne.utils import _pl, warn + +README_PATH = Path(__file__).parents[2] / "README.rst" +BEGIN = ".. ↓↓↓ BEGIN CORE DEPS LIST. DO NOT EDIT! HANDLED BY PRE-COMMIT HOOK ↓↓↓" +END = ".. ↑↑↑ END CORE DEPS LIST. DO NOT EDIT! HANDLED BY PRE-COMMIT HOOK ↑↑↑" + +CORE_DEPS_URLS = { + "Python": "https://www.python.org", + "NumPy": "https://numpy.org", + "SciPy": "https://scipy.org", + "Matplotlib": "https://matplotlib.org", + "Pooch": "https://www.fatiando.org/pooch/latest/", + "tqdm": "https://tqdm.github.io", + "Jinja2": "https://palletsprojects.com/p/jinja/", + "decorator": "https://github.com/micheles/decorator", + "lazy-loader": "https://pypi.org/project/lazy_loader", + "packaging": "https://packaging.pypa.io/en/stable/", +} + + +def _prettify_pin(pin): + if pin is None: + return "" + pins = pin.split(",") + replacements = { + "<=": " ≤ ", + ">=": " ≥ ", + "<": " < ", + ">": " > ", + } + for old, new in replacements.items(): + pins = [p.replace(old, new) for p in pins] + pins = reversed(pins) + return ",".join(pins) + + +# get the dependency info +py_pin = metadata("mne").get("Requires-Python") +all_deps = metadata("mne").get_all("Requires-Dist") +core_deps = [f"python{py_pin}", *[dep for dep in all_deps if "extra ==" not in dep]] +pattern = re.compile(r"(?P[A-Za-z_\-\d]+)(?P[<>=]+.*)?") +core_deps_pins = { + dep["name"]: _prettify_pin(dep["pin"]) for dep in map(pattern.match, core_deps) +} + + +# make sure our URLs dict is minimal and complete +missing_urls = set(core_deps_pins) - {dep.lower() for dep in CORE_DEPS_URLS} +extra_urls = {dep.lower() for dep in CORE_DEPS_URLS} - set(core_deps_pins) +update_msg = ( + "please update `CORE_DEPS_URLS` mapping in `tools/hooks/sync_dependencies.py`." +) +if missing_urls: + _s = _pl(missing_urls) + raise RuntimeError( + f"Missing URL{_s} for package{_s} {', '.join(missing_urls)}; {update_msg}" + ) +if extra_urls: + _s = _pl(extra_urls) + warn(f"Superfluous URL{_s} for package{_s} {', '.join(extra_urls)}; {update_msg}") +# construct the rST +core_deps_bullets = [ + f"- `{key} <{url}>`__{core_deps_pins[key.lower()]}" + for key, url in CORE_DEPS_URLS.items() +] +core_deps_rst = "\n".join(core_deps_bullets) + "\n" +# rewrite the README file +skip = False +with open(README_PATH, "r+") as fid: + lines = fid.readlines() + fid.seek(0) + for line in lines: + if line.strip() == BEGIN: + skip = True + fid.write(line) + fid.write(core_deps_rst) + if line.strip() == END: + skip = False + if not skip: + fid.write(line) + fid.truncate() From 4693515c75196eb923274e6265a81a7e9b90a87f Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 9 Oct 2024 12:55:56 -0500 Subject: [PATCH 02/11] run pre-commit --- README.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 295503eb547..ce5f4f066bf 100644 --- a/README.rst +++ b/README.rst @@ -75,14 +75,15 @@ The minimum required dependencies to run MNE-Python are: .. ↓↓↓ BEGIN CORE DEPS LIST. DO NOT EDIT! HANDLED BY PRE-COMMIT HOOK ↓↓↓ - `Python `__ ≥ 3.9 -- `NumPy `__ ≥ 1.24 -- `SciPy `__ ≥ 1.10 +- `NumPy `__ ≥ 1.23, < 3 +- `SciPy `__ ≥ 1.9 - `Matplotlib `__ ≥ 3.6 - `Pooch `__ ≥ 1.5 - `tqdm `__ - `Jinja2 `__ - `decorator `__ -- `lazy_loader `__ +- `lazy-loader `__ ≥ 0.3 +- `packaging `__ .. ↑↑↑ END CORE DEPS LIST. DO NOT EDIT! HANDLED BY PRE-COMMIT HOOK ↑↑↑ For full functionality, some functions require: From a5dbf823a409cb75a838b4764fec653a1a219d0a Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 9 Oct 2024 12:59:33 -0500 Subject: [PATCH 03/11] remove extras dependencies from README --- README.rst | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/README.rst b/README.rst index ce5f4f066bf..62ecb8ffd33 100644 --- a/README.rst +++ b/README.rst @@ -86,33 +86,6 @@ The minimum required dependencies to run MNE-Python are: - `packaging `__ .. ↑↑↑ END CORE DEPS LIST. DO NOT EDIT! HANDLED BY PRE-COMMIT HOOK ↑↑↑ -For full functionality, some functions require: - -- `scikit-learn `__ ≥ 1.2 -- `Joblib `__ ≥ 1.2 (for parallelization) -- `mne-qt-browser `__ ≥ 0.5 (for fast raw data visualization) -- `Qt `__ ≥ 5.15 via one of the following bindings (for fast raw data visualization and interactive 3D visualization): - - - `PySide6 `__ ≥ 6.0 - - `PyQt6 `__ ≥ 6.0 - - `PyQt5 `__ ≥ 5.15 - -- `Numba `__ ≥ 0.56.4 -- `NiBabel `__ ≥ 3.2.1 -- `OpenMEEG `__ ≥ 2.5.6 -- `pandas `__ ≥ 1.5.2 -- `Picard `__ ≥ 0.3 -- `CuPy `__ ≥ 9.0.0 (for NVIDIA CUDA acceleration) -- `DIPY `__ ≥ 1.4.0 -- `imageio `__ ≥ 2.8.0 -- `PyVista `__ ≥ 0.37 (for 3D visualization) -- `PyVistaQt `__ ≥ 0.9 (for 3D visualization) -- `mffpy `__ ≥ 0.5.7 -- `h5py `__ -- `h5io `__ -- `pymatreader `__ - - Contributing ^^^^^^^^^^^^ From c04d350a4c98dd9f4d44d3de42feb768e94421a2 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 9 Oct 2024 13:00:43 -0500 Subject: [PATCH 04/11] remove sentence about python version --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index 62ecb8ffd33..d9db47ab851 100644 --- a/README.rst +++ b/README.rst @@ -43,9 +43,6 @@ only, use pip_ in a terminal: $ pip install --upgrade mne -The current MNE-Python release requires Python 3.9 or higher. MNE-Python 0.17 -was the last release to support Python 2.7. - For more complete instructions, including our standalone installers and more advanced installation methods, please refer to the `installation guide`_. From 893fd9e5cf8e9484f03ac8c666df276f15d0d2c0 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:03:56 +0000 Subject: [PATCH 05/11] [autofix.ci] apply automated fixes --- tools/hooks/sync_dependencies.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/hooks/sync_dependencies.py b/tools/hooks/sync_dependencies.py index cc5bb979355..add848fd920 100755 --- a/tools/hooks/sync_dependencies.py +++ b/tools/hooks/sync_dependencies.py @@ -1,5 +1,9 @@ #!/usr/bin/env python +# Authors: The MNE-Python contributors. +# License: BSD-3-Clause +# Copyright the MNE-Python contributors. + import re from importlib.metadata import metadata from pathlib import Path From da9459d7ddbd382a18b751c250f80566cbf1adef Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 9 Oct 2024 14:05:12 -0500 Subject: [PATCH 06/11] hide NumPy upper pin; cleaner file writing; whitespace --- tools/hooks/sync_dependencies.py | 34 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/tools/hooks/sync_dependencies.py b/tools/hooks/sync_dependencies.py index add848fd920..ff00bb6aa54 100755 --- a/tools/hooks/sync_dependencies.py +++ b/tools/hooks/sync_dependencies.py @@ -52,7 +52,10 @@ def _prettify_pin(pin): core_deps_pins = { dep["name"]: _prettify_pin(dep["pin"]) for dep in map(pattern.match, core_deps) } - +# don't show upper pin on NumPy (not important for users, just devs) +new_pin = core_deps_pins["numpy"].split(",") +new_pin.remove(" < 3") +core_deps_pins["numpy"] = new_pin[0] # make sure our URLs dict is minimal and complete missing_urls = set(core_deps_pins) - {dep.lower() for dep in CORE_DEPS_URLS} @@ -68,24 +71,25 @@ def _prettify_pin(pin): if extra_urls: _s = _pl(extra_urls) warn(f"Superfluous URL{_s} for package{_s} {', '.join(extra_urls)}; {update_msg}") + # construct the rST core_deps_bullets = [ f"- `{key} <{url}>`__{core_deps_pins[key.lower()]}" for key, url in CORE_DEPS_URLS.items() ] -core_deps_rst = "\n".join(core_deps_bullets) + "\n" +core_deps_rst = "\n".join(core_deps_bullets) + # rewrite the README file +lines = README_PATH.read_text("utf-8").splitlines() +out_lines = list() skip = False -with open(README_PATH, "r+") as fid: - lines = fid.readlines() - fid.seek(0) - for line in lines: - if line.strip() == BEGIN: - skip = True - fid.write(line) - fid.write(core_deps_rst) - if line.strip() == END: - skip = False - if not skip: - fid.write(line) - fid.truncate() +for line in lines: + if line.strip() == BEGIN: + skip = True + out_lines.append(line) + out_lines.append(core_deps_rst) + if line.strip() == END: + skip = False + if not skip: + out_lines.append(line) +README_PATH.write_text("\n".join(out_lines) + "\n", encoding="utf-8") From 3aade5dfedfacacd18046b6ab1a74917889faf0c Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 9 Oct 2024 14:12:38 -0500 Subject: [PATCH 07/11] fix hook conditon --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 67f7f1ab2be..61f80005949 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -70,7 +70,7 @@ repos: name: Sync dependency list between pyproject.toml and README.rst language: python entry: ./tools/hooks/sync_dependencies.py - files: README\.rst$ + files: pyproject.toml additional_dependencies: ["mne"] From 2dd111e34cbea119c98ae5c3e49496064b1751eb Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 9 Oct 2024 14:20:29 -0500 Subject: [PATCH 08/11] don't run it on CIs (exceeds free tier space alloc) --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 61f80005949..b3f5191c438 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -74,6 +74,10 @@ repos: additional_dependencies: ["mne"] +# these should *not* be run on CIs: +ci: + skip: [dependency-sync] # needs MNE to work, which exceeds the free tier space alloc. + # The following are too slow to run on local commits, so let's only run on CIs: # # - repo: https://github.com/pre-commit/mirrors-mypy From 98f97eed172e00191799994eacbdefc8afcf94d4 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 9 Oct 2024 14:22:00 -0500 Subject: [PATCH 09/11] trivial change to ensure it works --- README.rst | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index d9db47ab851..9d9b89bea13 100644 --- a/README.rst +++ b/README.rst @@ -72,7 +72,7 @@ The minimum required dependencies to run MNE-Python are: .. ↓↓↓ BEGIN CORE DEPS LIST. DO NOT EDIT! HANDLED BY PRE-COMMIT HOOK ↓↓↓ - `Python `__ ≥ 3.9 -- `NumPy `__ ≥ 1.23, < 3 +- `NumPy `__ ≥ 1.23 - `SciPy `__ ≥ 1.9 - `Matplotlib `__ ≥ 3.6 - `Pooch `__ ≥ 1.5 diff --git a/pyproject.toml b/pyproject.toml index d58793a6e45..9bdf8244b99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ scripts = {mne = "mne.commands.utils:main"} [project.optional-dependencies] # Leave this one here for backward-compat data = [] -dev = ["mne[test,doc]", "rcssmin"] +dev = ["mne[doc,test]", "rcssmin"] # Dependencies for building the documentation doc = [ "graphviz", From ab6cc514bdb30fb8045835c332faead46606c73e Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 9 Oct 2024 14:29:25 -0500 Subject: [PATCH 10/11] fix for PyPI long description --- tools/hooks/sync_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/hooks/sync_dependencies.py b/tools/hooks/sync_dependencies.py index ff00bb6aa54..0878a5f56eb 100755 --- a/tools/hooks/sync_dependencies.py +++ b/tools/hooks/sync_dependencies.py @@ -77,7 +77,7 @@ def _prettify_pin(pin): f"- `{key} <{url}>`__{core_deps_pins[key.lower()]}" for key, url in CORE_DEPS_URLS.items() ] -core_deps_rst = "\n".join(core_deps_bullets) +core_deps_rst = "\n" + "\n".join(core_deps_bullets) + "\n" # rewrite the README file lines = README_PATH.read_text("utf-8").splitlines() From 38e1aa35439686b1629feb2d6ba144a6f9339701 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 9 Oct 2024 14:30:04 -0500 Subject: [PATCH 11/11] rerun hook --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 9d9b89bea13..50e0daaa52c 100644 --- a/README.rst +++ b/README.rst @@ -71,6 +71,7 @@ Dependencies The minimum required dependencies to run MNE-Python are: .. ↓↓↓ BEGIN CORE DEPS LIST. DO NOT EDIT! HANDLED BY PRE-COMMIT HOOK ↓↓↓ + - `Python `__ ≥ 3.9 - `NumPy `__ ≥ 1.23 - `SciPy `__ ≥ 1.9 @@ -81,6 +82,7 @@ The minimum required dependencies to run MNE-Python are: - `decorator `__ - `lazy-loader `__ ≥ 0.3 - `packaging `__ + .. ↑↑↑ END CORE DEPS LIST. DO NOT EDIT! HANDLED BY PRE-COMMIT HOOK ↑↑↑ Contributing