From 5bc5c40219d4ab34acbc04378776a181d68c2e68 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Thu, 26 Jul 2018 02:00:25 +0200 Subject: [PATCH 1/4] pylint code improvements --- nbsrc/__init__.py | 12 ++++++------ nbsrc/cli.py | 22 +++++++++++++++++----- nbsrc/nbsrc.py | 12 ++++++------ nbsrc/srcexporter.py | 6 +++++- tests/test_cli.py | 2 +- tests/test_nbconvert.py | 8 ++++---- tests/utils.py | 2 +- 7 files changed, 40 insertions(+), 24 deletions(-) diff --git a/nbsrc/__init__.py b/nbsrc/__init__.py index b7b419e..6a8a2b9 100644 --- a/nbsrc/__init__.py +++ b/nbsrc/__init__.py @@ -1,10 +1,10 @@ """Jupyter notebooks as python or R script -Use this module to read or write Jupyter notebooks as R or python scripts -(methods 'read', 'reads', 'write', 'writes') - Use the 'nbsrc' conversion script to convert Jupyter notebooks from/to R or Python scripts + +NB: read, write methods, as well as ContentsManager, are to be found in the +nbrmd package """ from .nbsrc import readme @@ -12,6 +12,6 @@ try: from .srcexporter import PyNotebookExporter from .srcexporter import RNotebookExporter -except ImportError as e: - PyNotebookExporter = str(e) - RNotebookExporter = str(e) +except ImportError as error: + PyNotebookExporter = str(error) + RNotebookExporter = str(error) diff --git a/nbsrc/cli.py b/nbsrc/cli.py index 964ecb3..d734a65 100644 --- a/nbsrc/cli.py +++ b/nbsrc/cli.py @@ -1,11 +1,14 @@ +"""Command line conversion tool `nbsrc` +""" + import os +import argparse from nbformat import writes as ipynb_writes +from nbformat.reader import NotJSONError from nbrmd import readf, writef from nbrmd import writes from nbrmd.languages import get_default_language from nbrmd.combine import combine_inputs_with_outputs -from nbformat.reader import NotJSONError -import argparse def convert(nb_files, in_place=True, combine=True): @@ -27,7 +30,7 @@ def convert(nb_files, in_place=True, combine=True): nb = readf(nb_file) main_language = get_default_language(nb) - ext_dest = '.R' if main_language is 'R' else '.py' + ext_dest = '.R' if main_language == 'R' else '.py' if in_place: if ext == '.ipynb': @@ -42,8 +45,8 @@ def convert(nb_files, in_place=True, combine=True): nb_outputs = readf(nb_dest) combine_inputs_with_outputs(nb, nb_outputs) msg = '(outputs were preserved)' - except (IOError, NotJSONError) as e: - msg = '(outputs could not be preserved: {})'.format(e) + except (IOError, NotJSONError) as error: + msg = '(outputs were not preserved: {})'.format(error) print('R Markdown {} being converted to ' 'Jupyter notebook {} {}' .format(nb_file, nb_dest, msg)) @@ -56,6 +59,11 @@ def convert(nb_files, in_place=True, combine=True): def cli(args=None): + """ + Command line parser + :param args: + :return: + """ parser = argparse.ArgumentParser(description='Jupyter notebook ' 'from/to R or Python script') parser.add_argument('notebooks', @@ -72,5 +80,9 @@ def cli(args=None): def main(): + """ + Entry point for the nbsrc script + :return: + """ args = cli() convert(args.notebooks, args.in_place, args.preserve_outputs) diff --git a/nbsrc/nbsrc.py b/nbsrc/nbsrc.py index 4b82f57..f26efa9 100644 --- a/nbsrc/nbsrc.py +++ b/nbsrc/nbsrc.py @@ -1,12 +1,12 @@ +"""Methods for the nbsrc package. Note that methods for notebooks are +found in the nbrmd package""" + import os def readme(): - """ - Contents of README.md - :return: - """ + """Contents of README.md""" readme_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'README.md') - with open(readme_path) as fh: - return fh.read() + with open(readme_path) as readme_file: + return readme_file.read() diff --git a/nbsrc/srcexporter.py b/nbsrc/srcexporter.py index 82f1e76..6853aea 100644 --- a/nbsrc/srcexporter.py +++ b/nbsrc/srcexporter.py @@ -1,6 +1,10 @@ -import nbrmd +""" +R and Py notebook exporters for nbconvert +""" + from traitlets import default from nbconvert.exporters import Exporter +import nbrmd class PyNotebookExporter(Exporter): diff --git a/tests/test_cli.py b/tests/test_cli.py index 3be75c1..6b48afb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,6 @@ -import pytest import os from shutil import copyfile +import pytest import nbrmd from nbsrc.cli import convert, cli from .utils import list_all_notebooks, remove_outputs diff --git a/tests/test_nbconvert.py b/tests/test_nbconvert.py index 6ea94b3..fee0d5b 100644 --- a/tests/test_nbconvert.py +++ b/tests/test_nbconvert.py @@ -1,9 +1,9 @@ -import nbsrc -import nbrmd +import os +import subprocess import pytest +import nbrmd +import nbsrc from .utils import list_all_notebooks -import subprocess -import os @pytest.mark.skipif(isinstance(nbsrc.PyNotebookExporter, str), diff --git a/tests/utils.py b/tests/utils.py index bb8f97a..d6f0a03 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -11,7 +11,7 @@ def list_all_notebooks(ext): nb_path = os.path.dirname(os.path.abspath(__file__)) notebooks = [] for nb_file in os.listdir(nb_path): - file, nb_ext = os.path.splitext(nb_file) + _, nb_ext = os.path.splitext(nb_file) if nb_ext.lower() == ext.lower(): notebooks.append(os.path.join(nb_path, nb_file)) return notebooks From aae6421534f1e4b5206674e328293aaa35aec309 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Thu, 26 Jul 2018 02:00:37 +0200 Subject: [PATCH 2/4] Version 0.4.3 --- HISTORY.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index e1e07ce..9ff21c4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,14 @@ Release History dev +++ +0.4.3 (2018-07-25) ++++++++++++++++++++ + +**Improvements** + +- Readme refactored, notebook demos available on binder #23 +- Based on nbrmd 0.4.3 (multiline comments now supported) + 0.4.0 (2018-07-18) +++++++++++++++++++ diff --git a/setup.py b/setup.py index 0488144..68a6edf 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name='nbsrc', - version='0.4.0', + version='0.4.3', author='Marc Wouts', author_email='marc.wouts@gmail.com', description='Jupyter notebooks from/to python and R scripts', From 1246a89c31d1a5777d5e261a7f3d6d4953a1fe03 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Thu, 26 Jul 2018 02:00:51 +0200 Subject: [PATCH 3/4] Readme updated --- README.md | 85 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 91df534..7e0b305 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,55 @@ -# Python and R scripts as Jupyter notebooks +# Jupyter notebooks from/to Python or R scripts [![Pypi](https://img.shields.io/pypi/v/nbsrc.svg)](https://pypi.python.org/pypi/nbsrc) [![Pypi](https://img.shields.io/pypi/l/nbsrc.svg)](https://pypi.python.org/pypi/nbsrc) [![Build Status](https://travis-ci.com/mwouts/nbsrc.svg?branch=master)](https://travis-ci.com/mwouts/nbsrc) [![codecov.io](https://codecov.io/github/mwouts/nbsrc/coverage.svg?branch=master)](https://codecov.io/github/mwouts/nbsrc?branch=master) +![pylint Score](https://mperlet.github.io/pybadge/badges/9.6.svg) [![pyversions](https://img.shields.io/pypi/pyversions/nbsrc.svg)](https://pypi.python.org/pypi/nbsrc) +[![Binder](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/mwouts/nbrmd/master?filepath=demo) -This package provides companion scripts (`.py` or `.R` extension) -to your Jupyter notebooks, that are always *synchronized* -with the notebook. With this you will be able to -- set the `.py` or `.R` script under version control -- modify the script outside of Jupyter, and easily merge multiple contributions -to the notebook using standard, text merge tools -- reload the latest version of the notebook from the `.py` or `.R` script. -Outputs for the cells with unchanged input are taken from the `.ipynb` file. +Jupyter notebooks are complex files, that contain source code, metadata, and +rich outputs. Here we offer a simple and complementary format for Jupyter +notebooks, as pure python (or R) companion scripts. + +The resulting python scripts are perfect candidates for +keeping notebooks under *version control*. They can be +*edited* outside of Jupyter, using +your favorite text editor, or even standard merge tools if you wish to merge +*multiple contributions* to a notebook. + +With the `nbsrc` package, any python or R script can be loaded as a notebook +in Jupyter. If a classical `ipynb` notebook with a matching name exists, +outputs for matching inputs are reconstructed. And, if you associate python +and jupyter files as recommended below, when a `ipynb` notebook opens, the +corresponding inputs are taken from the `py` file, which you may have updated +outside of Jupyter. + +## Can I have a demo? + +Sure. Try our package on [binder](https://mybinder.org/v2/gh/mwouts/nbrmd/master?filepath=demo)! + +There, you will be able +- to open and execute arbitrary python files as notebooks (give a try to +the matplotlib demo named `filled_step.py`, for instance) +- to open a notebook, then edit the companion python script, and reload the notebook, +to find up-to-date inputs in Jupyter. + +## How does the python version look like? + +Below is an example of a Jupyter notebook, together with its python representation. + +We have a battery of tests that ensure that +- Round trip conversion: python to notebook to python, is *identity* +- Round trip conversion, starting from a Jupyter notebook, preserve *source* +and *metadata*, not outputs. In some occasions (consecutive blank lines in +code cells), cells may be splitted into smaller ones. + +Python [notebook](https://mybinder.org/v2/gh/mwouts/nbrmd/master?filepath=tests/python_notebook_sample.py) in Jupyter | Python [script](https://github.com/mwouts/nbrmd/blob/master/tests/python_notebook_sample.py) +:--------------------------:|:-----------------------: +![](https://raw.githubusercontent.com/mwouts/nbsrc/master/img/python_notebook.png) | ![](https://raw.githubusercontent.com/mwouts/nbsrc/master/img/python_source.png) + +The representation of notebooks as R scripts follows the [standard](https://rmarkdown.rstudio.com/articles_report_from_r_script.html) for that language. ## How do I activate the companion script? @@ -24,7 +60,7 @@ c.NotebookApp.contents_manager_class = 'nbrmd.RmdFileContentsManager' c.ContentsManager.default_nbrmd_formats = 'ipynb,py' ``` -Then, make sure you have the `nbrmd` packages installed, and re-start jupyter, i.e. run +Then, make sure you have the `nbrmd` package up-to-date, and re-start jupyter, i.e. run ```bash pip install nbrmd --upgrade jupyter notebook @@ -56,25 +92,15 @@ In case you want both `.py` and `.Rmd`, please note that the order matters: the first non-`.ipynb` extension is the one used as the reference source for notebook inputs. -## Can I edit the python file? - -Yes, please! That's the precise purpose for the `nbsrc` package. When you're done, please _reload_ the notebook, i.e. refresh your notebook in the browser. Note that the url should have only the notebook name (no additional #), like -`http://localhost:8888/notebooks/GitHub/nbrmd/tests/python_notebook_sample.ipynb`. - -As mentioned above, reloading the `.ipynb` with actually load updated inputs from the python script. +## What is the difference between `nbsrc` and `nbrmd`? -It is not required to _restart_ the current kernel. Reloading may remove a few outputs (those corresponding to inputs you have changed), but it will preserve the python variables. - -Python notebook in Jupyter | Python script -:--------------------------:|:-----------------------: -![](https://raw.githubusercontent.com/mwouts/nbsrc/master/img/python_notebook.png) | ![](https://raw.githubusercontent.com/mwouts/nbsrc/master/img/python_source.png) +`nbrmd` is a python package that represents Jupyter notebooks as R markdown +files. It is also where notebooks as python scripts are implemented. But +I felt notebooks as scripts deserved a standalone documentation, and +that's the main reason for having the `nbsrc` package. - -## How do you represent notebooks as scripts? - -`.R` scripts follow the [standard](https://rmarkdown.rstudio.com/articles_report_from_r_script.html) for that language. - -Designing a comfortable standard for `.py` scripts is not trivial. The current format is documented [here](https://github.com/mwouts/nbrmd/blob/master/tests/python_notebook_sample.py). +You don't actually need the `nbsrc` package unless you want the command line +conversion tools. ## Command line conversion @@ -100,8 +126,3 @@ nbconvert jupyter.ipynb --to pynotebook nbconvert jupyter.ipynb --to rnotebook ``` -## And if I convert twice? - -Round trip conversion of scripts is identity. -Round trip conversion of Jupyter notebooks preserves the source, not outputs. - From a9a4650db95714cd5593d7dbe82dd6abb45d48da Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Thu, 26 Jul 2018 02:06:54 +0200 Subject: [PATCH 4/4] Less formating --- README.md | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 7e0b305..c51c856 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,10 @@ rich outputs. Here we offer a simple and complementary format for Jupyter notebooks, as pure python (or R) companion scripts. The resulting python scripts are perfect candidates for -keeping notebooks under *version control*. They can be -*edited* outside of Jupyter, using +keeping notebooks under version control. They can be +edited outside of Jupyter, using your favorite text editor, or even standard merge tools if you wish to merge -*multiple contributions* to a notebook. +multiple contributions to a notebook. With the `nbsrc` package, any python or R script can be loaded as a notebook in Jupyter. If a classical `ipynb` notebook with a matching name exists, @@ -28,10 +28,9 @@ outside of Jupyter. ## Can I have a demo? Sure. Try our package on [binder](https://mybinder.org/v2/gh/mwouts/nbrmd/master?filepath=demo)! - There, you will be able - to open and execute arbitrary python files as notebooks (give a try to -the matplotlib demo named `filled_step.py`, for instance) +the matplotlib demo named `filled_step.py`) - to open a notebook, then edit the companion python script, and reload the notebook, to find up-to-date inputs in Jupyter. @@ -39,10 +38,10 @@ to find up-to-date inputs in Jupyter. Below is an example of a Jupyter notebook, together with its python representation. -We have a battery of tests that ensure that -- Round trip conversion: python to notebook to python, is *identity* -- Round trip conversion, starting from a Jupyter notebook, preserve *source* -and *metadata*, not outputs. In some occasions (consecutive blank lines in +We have hundreds of tests that ensure that +- Round trip conversion: python to notebook to python, is identity +- Round trip conversion, starting from a Jupyter notebook, preserves source +and metadata, not outputs. In some occasions (consecutive blank lines in code cells), cells may be splitted into smaller ones. Python [notebook](https://mybinder.org/v2/gh/mwouts/nbrmd/master?filepath=tests/python_notebook_sample.py) in Jupyter | Python [script](https://github.com/mwouts/nbrmd/blob/master/tests/python_notebook_sample.py) @@ -60,12 +59,25 @@ c.NotebookApp.contents_manager_class = 'nbrmd.RmdFileContentsManager' c.ContentsManager.default_nbrmd_formats = 'ipynb,py' ``` -Then, make sure you have the `nbrmd` package up-to-date, and re-start jupyter, i.e. run +Then, make sure you have the [`nbrmd`](https://github.com/mwouts/nbrmd) +package up-to-date, and re-start jupyter, i.e. run ```bash pip install nbrmd --upgrade jupyter notebook ``` +With the above configuration, every Jupyter notebook will have a companion +`.py` script. And every `.py` script that you edit in Jupyter +will have a companion `.ipynb` notebook. + +If you prefer the `.ipynb` notebook not to be created by Jupyter when a `.py` +script is edited, set +``` +c.ContentsManager.default_nbrmd_formats = '' +``` +(as the default value is `ipynb`). Outputs for scripts, however, +will not be saved any more. + ## Per notebook configuration With the above configuration, every notebook will have a companion `.py` file. @@ -94,7 +106,8 @@ is the one used as the reference source for notebook inputs. ## What is the difference between `nbsrc` and `nbrmd`? -`nbrmd` is a python package that represents Jupyter notebooks as R markdown +[`nbrmd`](https://github.com/mwouts/nbrmd) +is a python package that represents Jupyter notebooks as R markdown files. It is also where notebooks as python scripts are implemented. But I felt notebooks as scripts deserved a standalone documentation, and that's the main reason for having the `nbsrc` package.