From a43d743f6fd677a4bf3128c12b9655a22b738ad5 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Thu, 21 Dec 2023 17:09:43 -0300 Subject: [PATCH 1/3] add colab missing dependencies script --- .../content/add_colab_install_missing_deps.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 jupyterbook/content/add_colab_install_missing_deps.py diff --git a/jupyterbook/content/add_colab_install_missing_deps.py b/jupyterbook/content/add_colab_install_missing_deps.py new file mode 100644 index 00000000..d54e5d33 --- /dev/null +++ b/jupyterbook/content/add_colab_install_missing_deps.py @@ -0,0 +1,105 @@ +from pathlib import Path + +from depfinder import notebook_path_to_dependencies + +notebook_path = Path('.') +notebooks = list(notebook_path.glob('**/*.ipynb')) + +## Get modules that are absent on GoogleColab +# required = [] + +# for notebook in notebooks: +# try: +# ret = notebook_path_to_dependencies(notebook) +# except Exception as err: +# print(f"Could not parse {notebook=}. {err}") +# required.extend(ret["required"]) + +# modules = sorted(set(required)) + +## go to colab with this list and run to obtain the absent list + +# """ +# import importlib +# for module in modules: +# if importlib.util.find_spec(module) is None: +# print(module) +# """ + +## update notebooks +import nbformat + +absent = [ + 'bagit', + 'cartopy', + 'cf-units', + 'cf_xarray', + 'compliance-checker', + 'erddapy', + 'geoplot', + 'gridgeo', + 'ioos-tools', + 'ioos_qc', + 'ipyleaflet', + 'iris', + 'netcdf4', + 'oceans', + 'odvc', + 'owslib', + 'palettable', + 'pocean-core', + 'pyobis', + 'pyoos', + 'pysgrid', + 'pyugrid', + 'pyworms', + 'retrying', + 'seawater', + 'zarr', +] + + +code = """\ +import subprocess +import sys +COLAB = "google.colab" in sys.modules + +def _install(package): + if COLAB: + ans = input(f"Install {{ package }}? [y/n]:") + if ans.lower() in ["y", "yes"]: + subprocess.check_call([sys.executable, "-m", "pip", "install", "--quiet", package]) + print(f"{{ package }} installed!") + +def _colab_install_missing_deps(deps): + import importlib + for dep in deps: + if importlib.util.find_spec(dep) is None: + if dep == "iris": + dep = "scitools-iris" + _install(dep) + +deps = {} +_colab_install_missing_deps(deps)""" + + +def update_notebook(notebook, code): + nb = nbformat.read(notebook, as_version=4) + + new_cell = nbformat.v4.new_code_cell(code) + new_cell.pop('id') # cannot save mod nb with this + new_cell['metadata'] = {'tags': ['remove-cell']} # won't render on jupyterbook + # nb.cells.insert(0, new_cell) + nb.cells[0] = new_cell + nbformat.write(nb, notebook, version=4) + + +for notebook in notebooks: + if 'archived' not in str(notebook): + try: + required = notebook_path_to_dependencies(notebook)['required'] + missing_from_colab = [module for module in absent if module in required] + update_notebook(notebook, code.format(missing_from_colab)) + except Exception as err: + print(f"Could not parse {notebook=}. {err}") + continue From 56284a221e772c5842729cd96a1ccb0977fe54f3 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Thu, 21 Dec 2023 17:22:08 -0300 Subject: [PATCH 2/3] update and remove double quote fix --- .pre-commit-config.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 15098404..d9a7a0c6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,17 +14,16 @@ repos: - id: check-docstring-first - id: check-json - id: check-yaml - - id: double-quote-string-fixer - repo: https://github.com/pre-commit/mirrors-prettier - rev: v4.0.0-alpha.3 + rev: v4.0.0-alpha.8 hooks: - id: prettier types_or: [html] exclude: "_templates/layout.html" - repo: https://github.com/psf/black - rev: 23.11.0 + rev: 23.12.0 hooks: - id: black language_version: python3 @@ -35,7 +34,7 @@ repos: - id: add-trailing-comma - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.6 + rev: v0.1.9 hooks: - id: ruff From d97ca03c8ff66491ddf6fe8708b382bb274ecfea Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Thu, 21 Dec 2023 17:22:19 -0300 Subject: [PATCH 3/3] run pre-commit --- .../content/add_colab_install_missing_deps.py | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/jupyterbook/content/add_colab_install_missing_deps.py b/jupyterbook/content/add_colab_install_missing_deps.py index d54e5d33..122ea94e 100644 --- a/jupyterbook/content/add_colab_install_missing_deps.py +++ b/jupyterbook/content/add_colab_install_missing_deps.py @@ -2,8 +2,8 @@ from depfinder import notebook_path_to_dependencies -notebook_path = Path('.') -notebooks = list(notebook_path.glob('**/*.ipynb')) +notebook_path = Path(".") +notebooks = list(notebook_path.glob("**/*.ipynb")) ## Get modules that are absent on GoogleColab # required = [] @@ -30,32 +30,32 @@ import nbformat absent = [ - 'bagit', - 'cartopy', - 'cf-units', - 'cf_xarray', - 'compliance-checker', - 'erddapy', - 'geoplot', - 'gridgeo', - 'ioos-tools', - 'ioos_qc', - 'ipyleaflet', - 'iris', - 'netcdf4', - 'oceans', - 'odvc', - 'owslib', - 'palettable', - 'pocean-core', - 'pyobis', - 'pyoos', - 'pysgrid', - 'pyugrid', - 'pyworms', - 'retrying', - 'seawater', - 'zarr', + "bagit", + "cartopy", + "cf-units", + "cf_xarray", + "compliance-checker", + "erddapy", + "geoplot", + "gridgeo", + "ioos-tools", + "ioos_qc", + "ipyleaflet", + "iris", + "netcdf4", + "oceans", + "odvc", + "owslib", + "palettable", + "pocean-core", + "pyobis", + "pyoos", + "pysgrid", + "pyugrid", + "pyworms", + "retrying", + "seawater", + "zarr", ] @@ -87,17 +87,17 @@ def update_notebook(notebook, code): nb = nbformat.read(notebook, as_version=4) new_cell = nbformat.v4.new_code_cell(code) - new_cell.pop('id') # cannot save mod nb with this - new_cell['metadata'] = {'tags': ['remove-cell']} # won't render on jupyterbook + new_cell.pop("id") # cannot save mod nb with this + new_cell["metadata"] = {"tags": ["remove-cell"]} # won't render on jupyterbook # nb.cells.insert(0, new_cell) nb.cells[0] = new_cell nbformat.write(nb, notebook, version=4) for notebook in notebooks: - if 'archived' not in str(notebook): + if "archived" not in str(notebook): try: - required = notebook_path_to_dependencies(notebook)['required'] + required = notebook_path_to_dependencies(notebook)["required"] missing_from_colab = [module for module in absent if module in required] update_notebook(notebook, code.format(missing_from_colab)) except Exception as err: