From c30dce7d78c182d54df8fb030c3b1c8a1869bc09 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Thu, 23 May 2019 14:05:13 -0700 Subject: [PATCH 1/7] adding documentation --- .gitignore | 1 + docs/Makefile | 20 +++++++ docs/advanced.md | 20 +++++++ docs/conf.py | 72 ++++++++++++++++++++++++ docs/doc-requirements.txt | 4 ++ docs/examples.md | 47 ++++++++++++++++ docs/formats.md | 113 ++++++++++++++++++++++++++++++++++++++ docs/index.md | 46 ++++++++++++++++ docs/install.md | 56 +++++++++++++++++++ docs/using-cli.md | 93 +++++++++++++++++++++++++++++++ docs/using-server.md | 106 +++++++++++++++++++++++++++++++++++ 11 files changed, 578 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/advanced.md create mode 100644 docs/conf.py create mode 100644 docs/doc-requirements.txt create mode 100644 docs/examples.md create mode 100644 docs/formats.md create mode 100644 docs/index.md create mode 100644 docs/install.md create mode 100644 docs/using-cli.md create mode 100644 docs/using-server.md diff --git a/.gitignore b/.gitignore index 4c8dd3c0b..b0b2aafb4 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ future *.pyc gits* .ipynb_checkpoints +docs/_build \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..9764015b4 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = Binder +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/advanced.md b/docs/advanced.md new file mode 100644 index 000000000..38bd08768 --- /dev/null +++ b/docs/advanced.md @@ -0,0 +1,20 @@ +# Advanced functionality + +Below are some advanced functionality and tips. + +## Fine tuning + +Jupyter magic commands are commented when exporting the notebook to text, except for the `markdown` and the `hydrogen` format. If you want to change this for a single line, add a `#escape` or `#noescape` flag on the same line as the magic, or a `"comment_magics": true` or `false` entry in the notebook metadata, in the `"jupytext"` section. Or set your preference globally on the contents manager by adding this line to `.jupyter/jupyter_notebook_config.py`: +```python +c.ContentsManager.comment_magics = True # or False +``` + +Also, you may want some cells to be active only in the Python, or R Markdown representation. For this, use the `active` cell metadata. Set `"active": "ipynb"` if you want that cell to be active only in the Jupyter notebook. And `"active": "py"` if you want it to be active only in the Python script. And `"active": "ipynb,py"` if you want it to be active in both, but not in the R Markdown representation... + +## Extending the `light` and `percent` formats to more languages + +You want to extend the `light` and `percent` format to another language? In principle that is easy, and you will only have to: +- document the language extension and comment by adding one line to `_SCRIPT_EXTENSIONS` in `languages.py`. +- contribute a sample notebook in `tests\notebooks\ipynb_[language]`. +- add two tests in `test_mirror.py`: one for the `light` format, and another one for the `percent` format. +- Make sure that the tests pass, and that the text representations of your notebook, found in `tests\notebooks\mirror\ipynb_to_script` and `tests\notebooks\mirror\ipynb_to_percent`, are valid scripts. \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 000000000..de30c741c --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os +import requests +from recommonmark.transform import AutoStructify + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ["sphinx_copybutton", "recommonmark"] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: + +source_suffix = [".rst", ".md"] + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "Jupytext" +copyright = "2017, The Jupytext Team" +author = "The Jupytext Team" + + +def setup(app): + app.add_config_value( + "recommonmark_config", {"auto_toc_tree_section": "Site Contents"}, True + ) + app.add_transform(AutoStructify) + + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# html_sidebars = { +# "**": ["globaltoc.html", "relations.html", "sourcelink.html", "searchbox.html"] +# } + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +html_theme = "sphinx_rtd_theme" +html_static_path = ["_static"] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = "jupytext" diff --git a/docs/doc-requirements.txt b/docs/doc-requirements.txt new file mode 100644 index 000000000..6b9187fa5 --- /dev/null +++ b/docs/doc-requirements.txt @@ -0,0 +1,4 @@ +sphinx +sphinx_rtd_theme +sphinx_copybutton +jupytext \ No newline at end of file diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 000000000..fd9226865 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,47 @@ +# Example usage + +## Writing notebooks as plain text + +You like to work with scripts? The good news is that plain scripts, which you can draft and test in your favorite IDE, open transparently as notebooks in Jupyter when using Jupytext. Run the notebook in Jupyter to generate the outputs, [associate](#paired-notebooks) an `.ipynb` representation, save and share your research as either a plain script or as a traditional Jupyter notebook with outputs. + +## Collaborating on Jupyter Notebooks + +With Jupytext, collaborating on Jupyter notebooks with Git becomes as easy as collaborating on text files. + +The setup is straightforward: +- Open your favorite notebook in Jupyter notebook +- [Associate](#paired-notebooks) a `.py` representation (for instance) to that notebook +- Save the notebook, and put the Python script under Git control. Sharing the `.ipynb` file is possible, but not required. + +Collaborating then works as follows: +- Your collaborator pulls your script. +- The script opens as a notebook in Jupyter, with no outputs (in JupyterLab right-click the script and use the open-with context menu). +- They run the notebook and save it. Outputs are regenerated, and a local `.ipynb` file is created. +- Note that, alternatively, the `.ipynb` file could have been regenerated with `jupytext --sync notebook.py`. +- They change the notebook, and push their updated script. The diff is nothing else than a standard diff on a Python script. +- You pull the changed script, and refresh your browser. Input cells are updated. The outputs from cells that were changed are removed. Your variables are untouched, so you have the option to run only the modified cells to get the new outputs. + +## Code refactoring + +In the animation below we propose a quick demo of Jupytext. While the example remains simple, it shows how your favorite text editor or IDE can be used to edit your Jupyter notebooks. IDEs are more convenient than Jupyter for navigating through code, editing and executing cells or fractions of cells, and debugging. + +- We start with a Jupyter notebook. +- The notebook includes a plot of the world population. The plot legend is not in order of decreasing population, we'll fix this. +- We want the notebook to be saved as both a `.ipynb` and a `.py` file: we select _Pair Notebook with a light Script_ in either the [Jupytext menu](#jupytext-menu-in-jupyter-notebook) in Jupyter Notebook, or in the [Jupytext commands](#jupytext-commands-in-jupyterlab) in JupyterLab. This has the effect of adding a `"jupytext": {"formats": "ipynb,py:light"},` entry to the notebook metadata. +- The Python script can be opened with PyCharm: + - Navigating in the code and documentation is easier than in Jupyter. + - The console is convenient for quick tests. We don't need to create cells for this. + - We find out that the columns of the data frame were not in the correct order. We update the corresponding cell, and get the correct plot. +- The Jupyter notebook is refreshed in the browser. Modified inputs are loaded from the Python script. Outputs and variables are preserved. We finally rerun the code and get the correct plot. + +![](https://gist.githubusercontent.com/mwouts/13de42d8bb514e4acf6481c580feffd0/raw/b8dd28f44678f8c91f262da2381276fc4d03b00a/JupyterPyCharm.gif) + +## Importing Jupyter Notebooks as modules + +Jupytext allows to import code from other Jupyter notebooks in a very simple manner. Indeed, all you need to do is to pair the notebook that you wish to import with a script, and import the resulting script. + +If the notebook contains demos and plots that you don't want to import, mark those cells as either +- _active_ only in the `ipynb` format, with the `{"active": "ipynb"}` cell metadata, or with an `active-ipynb` tag (you may use the [jupyterlab-celltags](https://github.com/jupyterlab/jupyterlab-celltags) extension for this). +- _frozen_, using the [freeze extension](https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/nbextensions/freeze/readme.html) for Jupyter notebook. + +Inactive cells will be commented in the paired script, and consequently will not be executed when the script is imported. \ No newline at end of file diff --git a/docs/formats.md b/docs/formats.md new file mode 100644 index 000000000..5e26ec844 --- /dev/null +++ b/docs/formats.md @@ -0,0 +1,113 @@ +# Supported formats + +Jupytext supports conversion between the `.ipynb` format and many different formats. This page describes +each format, as well as some considerations for each. + +## Markdown and R Markdown + +Save Jupyter notebooks as [Markdown](https://daringfireball.net/projects/markdown/syntax) documents and edit them in one of the many editors with good Markdown support. + +[R Markdown](https://rmarkdown.rstudio.com/authoring_quick_tour.html) is [RStudio](https://www.rstudio.com/)'s format for notebooks, with support for R, Python, and many [other languages](https://bookdown.org/yihui/rmarkdown/language-engines.html). + + +Jupytext's implementation for Jupyter notebooks as [Markdown](https://daringfireball.net/projects/markdown/syntax) or [R Markdown](https://rmarkdown.rstudio.com/authoring_quick_tour.html) documents is as follows: +- The notebook metadata (Jupyter kernel, etc) goes to a YAML header +- Code and raw cells are encoded as Markdown code blocks with triple backticks. In a Python notebook, a code cell starts with ` ```python` and ends with ` ``` `. Cell metadata are found after the language information, with a `key=value` syntax, where `value` is encoded in JSON format (Markdown) or R format (R Markdown). R Markdown [code cell options](https://yihui.name/knitr/options/) are mapped to the corresponding Jupyter cell metadata options, when available. +- Markdown cells are inserted verbatim and separated with two blank lines. When required (cells with metadata, cells that contain two blank lines or code blocks), Jupytext protects the cell boundary with HTML comments: `` and ``. Cells with explicit boundaries are [foldable](https://code.visualstudio.com/docs/editor/codebasics#_folding) in vscode, and can accept both a title and/or metadata in JSON format: ``. + +See how our `World population.ipynb` notebook in the [demo folder](https://github.com/mwouts/jupytext/tree/master/demo) is represented in [Markdown](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.md) or [R Markdown](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.Rmd). + +When you open a plain Markdown file in Jupytext, the Markdown text is rendered in Markdown cells. Cells breaks occur when the text contains two consecutive blank lines (or code cells). If you want to also split cells on Markdown headers, so that headers prefixed by one blank line appear at the top of a new cell, use the `split_at_heading` option. Set the option either on the command line, or by adding `"split_at_heading": true` to the jupytext section in the notebook metadata, or on Jupytext's content manager: + +```python +c.ContentsManager.split_at_heading = True +``` + +## Pandoc's Markdown + +Pandoc, the _Universal document converter_, can now read and write Jupyter notebooks - see [Pandoc's documentation](https://pandoc.org/MANUAL.html#creating-jupyter-notebooks-with-pandoc). + +Pandoc's Markdown format is available in Jupytext as `md:pandoc`. This requires `pandoc` in version 2.7.2 or above - please check Pandoc's version number with `pandoc -v` before running either `jupytext` command line, or before you start your Jupyter notebook or Lab server. Note that you can get the latest version of `pandoc` in a conda environment with +``` +conda install pandoc -c conda-forge +``` + +Pandoc's format uses Pandoc divs (`:::`) as explicit cell markers. See how our `World population.ipynb` notebook is [represented](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.pandoc.md) in that format. Please also note that `pandoc`, while preserving the HTML rendering, may reformat the text in some of the Markdown cells. If that is an issue for you, please wait until [jgm/pandoc#5408](https://github.com/jgm/pandoc/issues/5408) gets implemented. + +Jupytext currently strips the output cells before calling `pandoc`. As for the other formats, outputs cells can be preserved in paired notebooks. In the case of the pandoc format, paired notebooks are available with the metadata `"jupytext": {"formats": "md:pandoc,ipynb"}`. + +## The `light` format for notebooks as scripts + +The `light` format was created for this project. It is the default format for scripts. That format can read any script as a Jupyter notebook, even scripts which were never prepared to become a notebook. When a notebook is written as a script using this format, only a few cells markers are introduced—none if possible. + +The `light` format has: +- A (commented) YAML header, that contains the notebook metadata. +- Markdown cells are commented, and separated from other cells with a blank line. +- Code cells are exported verbatim (except for Jupyter magics, which are commented), and separated with blank lines. Code cells are reconstructed from consistent Python paragraphs (no function, class or multiline comment will be broken). +- Cells that contain more than one Python paragraphs need an explicit start-of-cell delimiter that is, by default, `# +` (`// +` in C++, etc). Cells that have explicit metadata have a cell header `# + {JSON}` where the metadata is represented, in JSON format. The default end of cell delimiter is `# -`, and is omitted when followed by another explicit start of cell marker. + +The `light` format is currently available for Python, Julia, R, Bash, Scheme, Clojure, Matlab, Octave, C++, q/kdb+ and IDL. Open our sample notebook in the `light` format [here](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.lgt.py). + +A variation of the `light` format is the `bare` format, with no cell marker at all. Please note that this format will split your code cells on code paragraphs. By default, this format still includes a YAML header - if you prefer to also remove the header, set `"notebook_metadata_filter": "-all"` in the jupytext section of your notebook metadata. + +The `light` format can use custom cell markers instead of `# +` or `# -`. If you prefer to mark cells with VScode/PyCharm (resp. Vim) folding markers, set `"cell_markers": "region,endregion"` (resp. `"{{{,}}}"`) in the jupytext section of the notebook metadata. If you want to configure this as a global default, add either +```python +c.ContentsManager.default_cell_markers = "region,endregion" # Use VScode/PyCharm region folding delimiters +``` +or +```python +c.ContentsManager.default_cell_markers = "{{{,}}}" # Use Vim region folding delimiters +``` +to your `.jupyter/jupyter_notebook_config.py` file. + + +## The `percent` format + +The `percent` format is a representation of Jupyter notebooks as scripts, in which cells are delimited with a commented double percent sign `# %%`. The format was introduced by Spyder five years ago, and is now supported by many editors, including +- [Spyder IDE](https://docs.spyder-ide.org/editor.html#defining-code-cells), +- [Hydrogen](https://atom.io/packages/hydrogen), a package for Atom, +- [VS Code](https://code.visualstudio.com/) with the [vscodeJupyter](https://marketplace.visualstudio.com/items?itemName=donjayamanne.jupyter) extension, +- [Python Tools for Visual Studio](https://github.com/Microsoft/PTVS), +- and [PyCharm Professional](https://www.jetbrains.com/pycharm/). + +Our implementation of the `percent` format is compatible with the original specifications by Spyder. We extended the format to allow markdown cells and cell metadata. Cell headers have the following structure: +```python +# %% Optional text [cell type] {optional JSON metadata} +``` +where cell type is either omitted (code cells), or `[markdown]` or `[raw]`. The content of markdown and raw cells is commented out in the resulting script. + +Percent scripts created by Jupytext have a header with an explicit format information. The format of scripts with no header is inferred automatically: scripts with at least one `# %%` cell are identified as `percent` scripts. Scripts with at least one double percent cell, and an uncommented Jupyter magic command, are identified as `hydrogen` scripts. + +The `percent` format is currently available for Python, Julia, R, Bash, Scheme, Clojure, Matlab, Octave, C++, q/kdb+ and IDL. Open our sample notebook in the `percent` format [here](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.pct.py). + +If the `percent` format is your favorite one, add the following to your `.jupyter/jupyter_notebook_config.py` file: +```python +c.ContentsManager.preferred_jupytext_formats_save = "py:percent" # or "auto:percent" +``` +Then, Jupytext's content manager will understand `"jupytext": {"formats": "ipynb,py"},` as an instruction to create the paired Python script in the `percent` format. + +By default, Jupyter magics are commented in the `percent` representation. If you run the percent scripts in Hydrogen, use instead the `hydrogen` format, a variant of the `percent` format that does not comment Jupyter magic commands. + +## Sphinx-gallery scripts + +Another popular notebook-like format for Python scripts is the Sphinx-gallery [format](https://sphinx-gallery.readthedocs.io/en/latest/tutorials/plot_notebook.html). Scripts that contain at least two lines with more than twenty hash signs are classified as Sphinx-Gallery notebooks by Jupytext. + +Comments in Sphinx-Gallery scripts are formatted using reStructuredText rather than markdown. They can be converted to markdown for a nicer display in Jupyter by adding a `c.ContentsManager.sphinx_convert_rst2md = True` line to your Jupyter configuration file. Please note that this is a non-reversible transformation—use this only with Binder. Revert to the default value `sphinx_convert_rst2md = False` when you edit Sphinx-Gallery files with Jupytext. + +Turn a GitHub repository containing Sphinx-Gallery scripts into a live notebook repository with [Binder](https://mybinder.org/) and Jupytext by adding only two files to the repo: +- `binder/requirements.txt`, a list of the required packages (including `jupytext`) +- `.jupyter/jupyter_notebook_config.py` with the following contents: +```python +c.NotebookApp.contents_manager_class = "jupytext.TextFileContentsManager" +c.ContentsManager.preferred_jupytext_formats_read = "py:sphinx" +c.ContentsManager.sphinx_convert_rst2md = True +``` + +Our sample notebook is also represented in `sphinx` format [here](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.spx.py). + +## R knitr::spin scripts + +The `spin` format is specific to R scripts. These scripts can be compiled into reports using either `knitr::spin` or [RStudio](https://rmarkdown.rstudio.com/articles_report_from_r_script.html). The implementation of the format is as follows: +- Jupyter metadata are in YAML format, in a `#' `-commented header. +- Markdown cells are commented with `#' `. +- Code cells are exported verbatim. Cell metadata are signalled with `#+`. Cells end with a blank line, an explicit start of cell marker, or a markdown cell. \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..5de012749 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,46 @@ +# Jupyter notebooks as Markdown documents, Julia, Python or R scripts + +[![](https://travis-ci.com/mwouts/jupytext.svg?branch=master)](https://travis-ci.com/mwouts/jupytext) +[![](https://codecov.io/github/mwouts/jupytext/coverage.svg?branch=master)](https://codecov.io/github/mwouts/jupytext?branch=master) +[![](https://img.shields.io/badge/lgtm-A+-brightgreen.svg)](https://lgtm.com/projects/g/mwouts/jupytext/context:python) + +Have you always wished Jupyter notebooks were plain text documents? Wished you could edit them in your favorite IDE? And get clear and meaningful diffs when doing version control? Then... Jupytext may well be the tool you're looking for! + +Jupytext can save Jupyter notebooks as +- Markdown and R Markdown documents, +- Julia, Python, R, Bash, Scheme, Clojure, Matlab, Octave, C++ and q/kdb+ scripts. + +The plain text formatting of the notebook cell divisions in scripts is configurable. Popular choices include the `percent` format [`# %%`] used by Spyder, VSCode, and others, and the `light` format which was developed to support this project. `light` uses as few cell markers as possible and is particularly suited for importing a pre-existing python script as a notebook with cell divisions automatically inferred from paragraph breaks in the source code. + +## Site Contents + +* [Install](install.md) +* [Using Jupytext from the Jupyter Server](using-server.md) +* [Using Jupytext from the Command Line](using-cli.md) +* [Formats supported in Jupytext](formats.md) +* [Examples](examples.md) +* [Advanced Usage](advanced.md) + +## Quickstart + +There are multiple ways to use `jupytext`: +- **Directly from Jupyter Notebook or JupyterLab.** Jupytext provides a _contents manager_ that allows Jupyter to save your notebook to your favorite format (`.py`, `.R`, `.jl`, `.md`, `.Rmd`...) in addition to (or in place of) the traditional `.ipynb` file. The text representation can be edited in your favorite editor. When you're done, refresh the notebook in Jupyter: inputs cells are loaded from the text file, while output cells are reloaded from the `.ipynb` file if present. Refreshing preserves kernel variables, so you can resume your work in the notebook and run the modified cells without having to rerun the notebook in full. +- **On the [command line](using-cli.html)**. `jupytext` converts Jupyter notebooks to their text representation, and back. The command line tool can act on notebooks in many ways. It can synchronize multiple representations of a notebook, pipe a notebook into a reformatting tool like `black`, etc... It can also work as a [pre-commit hook](#jupytext-as-a-git-pre-commit-hook) if you wish to automatically update the text representation when you commit the `.ipynb` file. +- **in Vim**: edit your Jupyter notebooks, represented as a Markdown document, or a Python script, with [jupytext.vim](https://github.com/goerz/jupytext.vim). + +## Demo time + +[![](https://img.shields.io/badge/TDS-Introducing%20Jupytext-blue.svg)](https://towardsdatascience.com/introducing-jupytext-9234fdff6c57) +[![](https://img.shields.io/badge/YouTube-PyParis-red.svg)](https://www.youtube.com/watch?v=y-VEZenk824) +[![](https://img.shields.io/badge/Binder-Try%20it!-blue.svg)](https://mybinder.org/v2/gh/mwouts/jupytext/master?filepath=demo) + +Looking for a demo? +- Read the original [announcement](https://towardsdatascience.com/introducing-jupytext-9234fdff6c57) in Towards Data Science, +- Watch the [PyParis talk](https://github.com/mwouts/jupytext_pyparis_2018/blob/master/README.md), +- or, try Jupytext online with [binder](https://mybinder.org/v2/gh/mwouts/jupytext/master?filepath=demo)! + + + +## Want to contribute? + +Contributions are welcome. Please let us know how you use `jupytext` and how we could improve it. You think the documentation could be improved? Go ahead! And stay tuned for more demos on [medium](https://medium.com/@marc.wouts) and [twitter](https://twitter.com/marcwouts)! diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 000000000..13180c69d --- /dev/null +++ b/docs/install.md @@ -0,0 +1,56 @@ +# Installation + +[![Conda Version](https://img.shields.io/conda/vn/conda-forge/jupytext.svg)](https://anaconda.org/conda-forge/jupytext) +[![Pypi](https://img.shields.io/pypi/v/jupytext.svg)](https://pypi.python.org/pypi/jupytext) +[![pyversions](https://img.shields.io/pypi/pyversions/jupytext.svg)](https://pypi.python.org/pypi/jupytext) + +Jupytext is available on pypi and on conda-forge. Run either of +```bash +pip install jupytext --upgrade +``` +or +```bash +conda install -c conda-forge jupytext +``` + +If you want to use Jupytext within Jupyter Notebook or JupyterLab, make sure you install Jupytext in the Python environment where the Jupyter server runs. If that environment is read-only, for instance if your server is started using JupyterHub, install Jupytext in user mode with: +``` +/path_to_your_jupyter_environment/python -m pip install jupytext --upgrade --user +``` + +## Jupytext's contents manager + +Jupytext includes a contents manager for Jupyter that allows Jupyter to open and save notebooks as text files. When Jupytext's content manager is active in Jupyter, scripts and Markdown documents have a notebook icon. + +If you don't have the notebook icon on text documents after a fresh restart of your Jupyter server, install the contents manager manually. Append +```python +c.NotebookApp.contents_manager_class = "jupytext.TextFileContentsManager" +``` +to your `.jupyter/jupyter_notebook_config.py` file (generate a Jupyter config, if you don't have one yet, with `jupyter notebook --generate-config`). Our contents manager accepts a few options: default formats, default metadata filter, etc — read more on this [below](#global-configuration). Then, restart Jupyter Notebook or JupyterLab, either from the JupyterHub interface or from the command line with +```bash +jupyter notebook # or lab +``` + +## Jupytext menu in Jupyter Notebook + +Jupytext includes an extensions for Jupyter Notebook that adds a Jupytext section in the File menu. + +![Jupyter notebook extension](https://raw.githubusercontent.com/mwouts/jupytext_nbextension/master/jupytext_menu.png) + +If the extension was not automatically installed, install and activate it with +``` +jupyter nbextension install --py jupytext [--user] +jupyter nbextension enable --py jupytext [--user] +``` + +## Jupytext commands in JupyterLab + +In JupyterLab, Jupytext adds a set of commands to the command palette: + +![JupyterLab extension](https://raw.githubusercontent.com/mwouts/jupyterlab-jupytext/master/jupytext_commands.png) + +If you don't see these commands, install the extension manually with +``` +jupyter labextension install jupyterlab-jupytext +``` +(the above requires `npm`, run `conda install nodejs` first if you don't have `npm`). \ No newline at end of file diff --git a/docs/using-cli.md b/docs/using-cli.md new file mode 100644 index 000000000..2bf344888 --- /dev/null +++ b/docs/using-cli.md @@ -0,0 +1,93 @@ +# Using Jupytext with the command line + +## Command line conversion + +The package provides a `jupytext` script for command line conversion between the various notebook extensions: + +```bash +jupytext --to py notebook.ipynb # create a notebook.py file in the light format +jupytext --to py:percent notebook.ipynb # create a notebook.py file in the double percent format +jupytext --to py:percent --comment-magics false notebook.ipynb # create a notebook.py file in the double percent format, and do not comment magic commands +jupytext --to markdown notebook.ipynb # create a notebook.md file +jupytext --output script.py notebook.ipynb # create a script.py file + +jupytext --to notebook notebook.py # overwrite notebook.ipynb (remove outputs) +jupytext --to notebook --update notebook.py # update notebook.ipynb (preserve outputs) +jupytext --to ipynb notebook1.md notebook2.py # overwrite notebook1.ipynb and notebook2.ipynb + +jupytext --to md --test notebook.ipynb # Test round trip conversion + +jupytext --to md --output - notebook.ipynb # display the markdown version on screen +jupytext --from ipynb --to py:percent # read ipynb from stdin and write double percent script on stdout +``` + +Jupytext has a `--sync` mode that updates all the paired representations of a notebook based on the file that was last modified. You may also find useful to `--pipe` the text representation of a notebook into tools like `black`: +```bash +jupytext --sync --pipe black notebook.ipynb # read most recent version of notebook, reformat with black, save +``` + +The `jupytext` command accepts many arguments. Use the `--set-formats` and the `--update-metadata` arguments to edit the pairing information or more generally the notebook metadata. Execute `jupytext --help` to access the documentation. + +## Jupytext as a Git pre-commit hook + +Jupytext is also available as a Git pre-commit hook. Use this if you want Jupytext to create and update the `.py` (or `.md`...) representation of the staged `.ipynb` notebooks. All you need is to create an executable `.git/hooks/pre-commit` file with the following content: +```bash +#!/bin/sh +# For every ipynb file in the git index, add a Python representation +jupytext --from ipynb --to py:light --pre-commit +``` + +```bash +#!/bin/sh +# For every ipynb file in the git index: +# - apply black and flake8 +# - export the notebook to a Python script in folder 'python' +# - and add it to the git index +jupytext --from ipynb --pipe black --check flake8 --pre-commit +jupytext --from ipynb --to python//py:light --pre-commit +``` + +If you don't want notebooks to be committed (and only commit the representations), you can ask the pre-commit hook to unstage notebooks after conversion by adding the following line: +```bash +git reset HEAD **/*.ipynb +``` +Note that these hooks do not update the `.ipynb` notebook when you pull. Make sure to either run `jupytext` in the other direction, or to use our paired notebook and our contents manager for Jupyter. Also, Jupytext does not offer a merge driver. If a conflict occurs, solve it on the text representation and then update or recreate the `.ipynb` notebook. Or give a try to nbdime and its [merge driver](https://nbdime.readthedocs.io/en/stable/vcs.html#merge-driver). + +## Testing the round-trip conversion + +Representing Jupyter notebooks as scripts requires a solid round trip conversion. You don't want your notebooks (nor your scripts) to be modified because you are converting them to the other form. Our test suite includes a few hundred tests to ensure that round trip conversion is safe. + +You can easily test that the round trip conversion preserves your Jupyter notebooks and scripts. Run for instance: +```bash +# Test the ipynb -> py:percent -> ipynb round trip conversion +jupytext --test notebook.ipynb --to py:percent + +# Test the ipynb -> (py:percent + ipynb) -> ipynb (à la paired notebook) conversion +jupytext --test --update notebook.ipynb --to py:percent +``` + +Note that `jupytext --test` compares the resulting notebooks according to its expectations. If you wish to proceed to a strict comparison of the two notebooks, use `jupytext --test-strict`, and use the flag `-x` to report with more details on the first difference, if any. + +Please note that +- Scripts opened with Jupyter have a default [metadata filter](#default-metadata-filtering) that prevents additional notebook or cell +metadata to be added back to the script. Remove the filter if you want to store Jupytext's settings, or the kernel information, in the text file. +- Cell metadata are available in the `light` and `percent` formats, as well as in the Markdown and R Markdown formats. R scripts in `spin` format support cell metadata for code cells only. Sphinx Gallery scripts in `sphinx` format do not support cell metadata. +- By default, a few cell metadata are not included in the text representation of the notebook. And only the most standard notebook metadata are exported. Learn more on this in the sections for [notebook specific](#-per-notebook-configuration) and [global settings](#default-metadata-filtering) for metadata filtering. + +## Reading notebooks in Python + +You can also manipulate notebooks in a Python shell or script using Jupytext's main functions: + +```python +# Read a notebook from a file. Format can be any of 'py', 'md', 'jl:percent', ... +readf(nb_file, fmt=None) + +# Read a notebook from a string. Here, format should contain at least the file extension. +reads(text, fmt) + +# Return the text representation for a notebook in the desired format. +writes(notebook, fmt) + +# Write a notebook to a file in the desired format. +writef(notebook, nb_file, fmt=None) +``` diff --git a/docs/using-server.md b/docs/using-server.md new file mode 100644 index 000000000..7ce1c2e66 --- /dev/null +++ b/docs/using-server.md @@ -0,0 +1,106 @@ +# Using Jupytext from within Jupyter + +## Paired notebooks + +Jupytext can write a given notebook to multiple files. In addition to the original notebook file, Jupytext can save the input cells to a text file — either a script or a Markdown document. Put the text file under version control for a clear commit history. Or refactor the paired script, and reimport the updated input cells by simply refreshing the notebook in Jupyter. + +## Per-notebook configuration + +Select the pairing for a given notebook using either the [Jupytext menu](#jupytext-menu-in-jupyter-notebook) in Jupyter Notebook, or the [Jupytext commands](#jupytext-commands-in-jupyterlab) in JupyterLab. + +These command simply add a `"jupytext": {"formats": "ipynb,md"}`-like entry in the notebook metadata. You could also set that metadata yourself with _Edit/Edit Notebook Metadata_ in Jupyter Notebook. In JupyterLab, use [this extension](https://github.com/yuvipanda/jupyterlab-nbmetadata). + +The pairing information for one or multiple notebooks can be set on the command line: +``` +jupytext --set-formats ipynb,py [--sync] notebook.ipynb +``` +You can pair a notebook to as many text representations as you want (see our _World population_ notebook in the demo folder). Format specifications are of the form +``` +[[path/][prefix]/][suffix.]ext[:format_name] +``` +where +- `ext` is one of `ipynb`, `md`, `Rmd`, `jl`, `py`, `R`, `sh`, `cpp`, `q`. Use the `auto` extension to have the script extension chosen according to the Jupyter kernel. +- `format_name` (optional) is either `light` (default for scripts), `bare`, `percent`, `hydrogen`, `sphinx` (Python only), `spin` (R only) — see below for the [format specifications](#Format-specifications). +- `path`, `prefix` and `suffix` allow to save the text representation to files with different names, or in a different folder. + +If you want to pair a notebook to a python script in a subfolder named `scripts`, set the formats metadata to `ipynb,scripts//py`. If the notebook is in a `notebooks` folder and you want the text representation to be in a `scripts` folder at the same level, set the Jupytext formats to `notebooks//ipynb,scripts//py`. + +Jupytext accepts a few additional options. These options should be added to the `"jupytext"` section in the metadata — use either the metadata editor or the `--opt/--format-options` argument on the command line. +- `comment_magics`: By default, Jupyter magics are commented when notebooks are exported to any other format than markdown. If you prefer otherwise, use this boolean option, or is global counterpart (see below). +- `notebook_metadata_filter`: By default, Jupytext only exports the `kernelspec` and `jupytext` metadata to the text files. Set `"jupytext": {"notebook_metadata_filter": "-all"}` if you want that the script has no notebook metadata at all. The value for `notebook_metadata_filter` is a comma separated list of additional/excluded (negated) entries, with `all` a keyword that allows to exclude all entries. +- `cell_metadata_filter`: By default, cell metadata `autoscroll`, `collapsed`, `scrolled`, `trusted` and `ExecuteTime` are not included in the text representation. Add or exclude more cell metadata with this option. + +## Global configuration + +Jupytext's contents manager also accepts global options. These options have to be set on Jupytext's contents manager, so please first include the following line in your `.jupyter/jupyter_notebook_config.py` file: +```python +c.NotebookApp.contents_manager_class = "jupytext.TextFileContentsManager" +``` + +We start with the default format pairing. Say you want to always associate every `.ipynb` notebook with a `.md` file (and reciprocally). This is simply done by adding the following to your Jupyter configuration file: +```python +# Always pair ipynb notebooks to md files +c.ContentsManager.default_jupytext_formats = "ipynb,md" +``` +(and similarly for the other formats). + +In case the [`percent`](#the-percent-format) format is your favorite, add the following to your `.jupyter/jupyter_notebook_config.py` file: +```python +# Use the percent format when saving as py +c.ContentsManager.preferred_jupytext_formats_save = "py:percent" +``` +and then, Jupytext will understand `"jupytext": {"formats": "ipynb,py"}` as an instruction to create the paired Python script in the `percent` format. + +To disable global pairing for an individual notebook, set formats to a single format, e.g.: +`"jupytext": {"formats": "ipynb"}` + +## Metadata filtering + +You can specify which metadata to include or exclude in the text files created by Jupytext by setting `c.ContentsManager.default_notebook_metadata_filter` (notebook metadata) and `c.ContentsManager.default_cell_metadata_filter` (cell metadata). They accept a string of comma separated keywords. A minus sign `-` in font of a keyword means exclusion. + +Suppose you want to keep all the notebook metadata but `widgets` and `varInspector` in the YAML header. For cell metadata, you want to allow `ExecuteTime` and `autoscroll`, but not `hide_output`. You can set +```python +c.ContentsManager.default_notebook_metadata_filter = "all,-widgets,-varInspector" +c.ContentsManager.default_cell_metadata_filter = "ExecuteTime,autoscroll,-hide_output" +``` + +If you want that the text files created by Jupytext have no metadata, you may use the global metadata filters below. Please note that with this setting, the metadata is only preserved in the `.ipynb` file. +```python +c.ContentsManager.default_notebook_metadata_filter = "-all" +c.ContentsManager.default_cell_metadata_filter = "-all" +``` + +NB: All these global options (and more) are documented [here](https://github.com/mwouts/jupytext/blob/master/jupytext/contentsmanager.py). + +## Can I edit a notebook simultaneously in Jupyter and in a text editor? + +When saving a paired notebook using Jupytext's contents manager, Jupyter updates both the `.ipynb` and its text representation. The text representation can be edited outside of Jupyter. When the notebook is refreshed in Jupyter, the input cells are read from the text file, and the output cells from the `.ipynb` file. + +It is possible (and convenient) to leave the notebook open in Jupyter while you edit its text representation. However, you don't want that the two editors save the notebook simultaneously. To avoid this: +- deactivate Jupyter's autosave, by toggling the `"Autosave notebook"` menu entry (or run `%autosave 0` in a cell of the notebook) +- and refresh the notebook when you switch back from the editor to Jupyter. + +In case you forgot to refresh, and saved the Jupyter notebook while the text representation had changed, no worries: Jupyter will ask you which version you want to keep: +![Notebook changed](https://gist.githubusercontent.com/mwouts/13de42d8bb514e4acf6481c580feffd0/raw/fcbcd3c3fc1ec4a74669381b53753f9f783e10da/notebook_changed.png) + +When that occurs, please choose the version in which you made the latest changes. And give a second look to our advice to deactivate the autosaving of notebooks in Jupyter. + +## How to open scripts with either the text or notebook view in Jupyter? + +With Jupytext's contents manager for Jupyter, scripts and Markdown documents gain a notebook icon. If you don't see the notebook icon, double check the [contents manager configuration](https://github.com/mwouts/jupytext/blob/master/README.md#jupytexts-contents-manager). + +By default, Jupyter Notebook open scripts and Markdown documents as notebooks. If you want to open them with the text editor, select the document and click on _edit_: + +![Open as text](https://user-images.githubusercontent.com/29915202/53228364-42265400-3681-11e9-812d-46168c6e398c.png) + +In JupyterLab this is slightly different. Scripts and Markdown document also have a notebook icon. But they open as text by default. Open them as notebooks with the _Open With -> Notebook_ context menu (available in JupyterLab 0.35 and above): + +![](https://gist.githubusercontent.com/mwouts/13de42d8bb514e4acf6481c580feffd0/raw/403b53ac5097446a15ea664579ba44cd1badcc57/ContextMenuLab.png) + +If do not want to classify scripts or Markdown documents as notebooks, please use the `notebook_extension` option. For instance, if you want to get the notebook icon only for `.ipynb` and `.Rmd` files, set + +```python +c.ContentsManager.notebook_extensions = "ipynb,Rmd" +``` + +Please note that, with the above setting, Jupyter will not let you open scripts as notebooks. If you still want to do so, use Jupytext command line (see below) to first convert or pair the script to an `.ipynb` notebook. \ No newline at end of file From 4607703140fe6c359c4f68d7be41c0baa423cf3e Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Thu, 30 May 2019 00:38:12 +0200 Subject: [PATCH 2/7] First steps with sphinx... - Make.bat generated with sphinx-quickstart - Explicit TOC in index.rst - index.md moved to intro.md - 'Edit on Github' --- docs/Makefile | 3 +- docs/conf.py | 89 ++++++++++++++++--------------------- docs/doc-requirements.txt | 3 +- docs/index.rst | 26 +++++++++++ docs/{index.md => intro.md} | 13 +----- docs/make.bat | 35 +++++++++++++++ 6 files changed, 105 insertions(+), 64 deletions(-) create mode 100644 docs/index.rst rename docs/{index.md => intro.md} (79%) create mode 100644 docs/make.bat diff --git a/docs/Makefile b/docs/Makefile index 9764015b4..298ea9e21 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -4,7 +4,6 @@ # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build -SPHINXPROJ = Binder SOURCEDIR = . BUILDDIR = _build @@ -17,4 +16,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index de30c741c..8ceb5e576 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,72 +1,61 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import os -import requests -from recommonmark.transform import AutoStructify - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. +# Configuration file for the Sphinx documentation builder. # -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = ["sphinx_copybutton", "recommonmark"] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# http://www.sphinx-doc.org/en/master/config -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: +# -- Path setup -------------------------------------------------------------- -source_suffix = [".rst", ".md"] +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) -# The master toctree document. -master_doc = "index" +# -- Project information ----------------------------------------------------- -# General information about the project. project = "Jupytext" -copyright = "2017, The Jupytext Team" +copyright = "2018-2019, The Jupytext Team" author = "The Jupytext Team" -def setup(app): - app.add_config_value( - "recommonmark_config", {"auto_toc_tree_section": "Site Contents"}, True - ) - app.add_transform(AutoStructify) +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ["sphinx_copybutton", "recommonmark"] +html_context = { + "display_github": True, # Integrate GitHub + "github_user": "mwouts", # Username + "github_repo": "jupytext", # Repo name + "github_version": "master", # Version + "conf_py_path": "/docs/", # Path in the checkout to the docs root +} -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path +# This pattern also affects html_static_path and html_extra_path. exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] -# html_sidebars = { -# "**": ["globaltoc.html", "relations.html", "sourcelink.html", "searchbox.html"] -# } - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False +# -- Options for HTML output ------------------------------------------------- +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# html_theme = "sphinx_rtd_theme" -html_static_path = ["_static"] - +pygments_style = "sphinx" -# -- Options for HTMLHelp output ------------------------------------------ +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] # Output file base name for HTML help builder. htmlhelp_basename = "jupytext" diff --git a/docs/doc-requirements.txt b/docs/doc-requirements.txt index 6b9187fa5..769925a1a 100644 --- a/docs/doc-requirements.txt +++ b/docs/doc-requirements.txt @@ -1,4 +1,5 @@ sphinx sphinx_rtd_theme sphinx_copybutton -jupytext \ No newline at end of file +jupytext +recommonmark diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 000000000..8b7cb970e --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,26 @@ +.. Jupytext documentation master file, created by + sphinx-quickstart on Thu May 30 00:07:50 2019. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Jupytext's documentation! +==================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + intro + examples + install + using-server + using-cli + formats + advanced + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/index.md b/docs/intro.md similarity index 79% rename from docs/index.md rename to docs/intro.md index 5de012749..fdc58f708 100644 --- a/docs/index.md +++ b/docs/intro.md @@ -1,4 +1,4 @@ -# Jupyter notebooks as Markdown documents, Julia, Python or R scripts +# Introduction [![](https://travis-ci.com/mwouts/jupytext.svg?branch=master)](https://travis-ci.com/mwouts/jupytext) [![](https://codecov.io/github/mwouts/jupytext/coverage.svg?branch=master)](https://codecov.io/github/mwouts/jupytext?branch=master) @@ -12,20 +12,11 @@ Jupytext can save Jupyter notebooks as The plain text formatting of the notebook cell divisions in scripts is configurable. Popular choices include the `percent` format [`# %%`] used by Spyder, VSCode, and others, and the `light` format which was developed to support this project. `light` uses as few cell markers as possible and is particularly suited for importing a pre-existing python script as a notebook with cell divisions automatically inferred from paragraph breaks in the source code. -## Site Contents - -* [Install](install.md) -* [Using Jupytext from the Jupyter Server](using-server.md) -* [Using Jupytext from the Command Line](using-cli.md) -* [Formats supported in Jupytext](formats.md) -* [Examples](examples.md) -* [Advanced Usage](advanced.md) - ## Quickstart There are multiple ways to use `jupytext`: - **Directly from Jupyter Notebook or JupyterLab.** Jupytext provides a _contents manager_ that allows Jupyter to save your notebook to your favorite format (`.py`, `.R`, `.jl`, `.md`, `.Rmd`...) in addition to (or in place of) the traditional `.ipynb` file. The text representation can be edited in your favorite editor. When you're done, refresh the notebook in Jupyter: inputs cells are loaded from the text file, while output cells are reloaded from the `.ipynb` file if present. Refreshing preserves kernel variables, so you can resume your work in the notebook and run the modified cells without having to rerun the notebook in full. -- **On the [command line](using-cli.html)**. `jupytext` converts Jupyter notebooks to their text representation, and back. The command line tool can act on notebooks in many ways. It can synchronize multiple representations of a notebook, pipe a notebook into a reformatting tool like `black`, etc... It can also work as a [pre-commit hook](#jupytext-as-a-git-pre-commit-hook) if you wish to automatically update the text representation when you commit the `.ipynb` file. +- **On the [command line](using-cli.md)**. `jupytext` converts Jupyter notebooks to their text representation, and back. The command line tool can act on notebooks in many ways. It can synchronize multiple representations of a notebook, pipe a notebook into a reformatting tool like `black`, etc... It can also work as a [pre-commit hook](#jupytext-as-a-git-pre-commit-hook) if you wish to automatically update the text representation when you commit the `.ipynb` file. - **in Vim**: edit your Jupyter notebooks, represented as a Markdown document, or a Python script, with [jupytext.vim](https://github.com/goerz/jupytext.vim). ## Demo time diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 000000000..7893348a1 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd From b78862876b3270b10927bf453cbe1a0b42dab9f8 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Fri, 31 May 2019 07:38:29 -0700 Subject: [PATCH 3/7] streamlining readme --- README.md | 381 ++---------------------------------------------------- 1 file changed, 12 insertions(+), 369 deletions(-) diff --git a/README.md b/README.md index 13b55be06..bc7a891bd 100755 --- a/README.md +++ b/README.md @@ -10,20 +10,13 @@ Jupytext can save Jupyter notebooks as - Markdown and R Markdown documents, - Julia, Python, R, Bash, Scheme, Clojure, Matlab, Octave, C++ and q/kdb+ scripts. -The plain text formatting of the notebook cell divisions in scripts is configurable. Popular choices include the `percent` format [`# %%`] used by Spyder, VSCode, and others, and the `light` format which was developed to support this project. `light` uses as few cell markers as possible and is particularly suited for importing a pre-existing python script as a notebook with cell divisions automatically inferred from paragraph breaks in the source code. +It can also convert these documents **into** Jupyter +Notebooks, allowing you to synchronize content in both +directions. -There are multiple ways to use `jupytext`: -- Directly from Jupyter Notebook or JupyterLab. Jupytext provides a _contents manager_ that allows Jupyter to save your notebook to your favorite format (`.py`, `.R`, `.jl`, `.md`, `.Rmd`...) in addition to (or in place of) the traditional `.ipynb` file. The text representation can be edited in your favorite editor. When you're done, refresh the notebook in Jupyter: inputs cells are loaded from the text file, while output cells are reloaded from the `.ipynb` file if present. Refreshing preserves kernel variables, so you can resume your work in the notebook and run the modified cells without having to rerun the notebook in full. -- On the [command line](#command-line-conversion). `jupytext` converts Jupyter notebooks to their text representation, and back. The command line tool can act on notebooks in many ways. It can synchronize multiple representations of a notebook, pipe a notebook into a reformatting tool like `black`, etc... It can also work as a [pre-commit hook](#jupytext-as-a-git-pre-commit-hook) if you wish to automatically update the text representation when you commit the `.ipynb` file. -- in Vim: edit your Jupyter notebooks, represented as a Markdown document, or a Python script, with [jupytext.vim](https://github.com/goerz/jupytext.vim). +**For more complete information try [the jupytext documentation](https://jupytext.readthedocs.io)!** ---- - -*Contents:* [Demo time](#demo-time) | [Example usage](#example|usage) | [Installation](#installation) | [Using Jupytext from within Jupyter](#using-jupytext-from-within-jupyter) | [Scripting Jupytext](#scripting-jupytext) | [Format specifications](#format-specifications) | [Want to contribute?](#want-to-contribute) - ---- - -## Demo time +## Try Jupytext [![Introducing Jupytext](https://img.shields.io/badge/TDS-Introducing%20Jupytext-blue.svg)](https://towardsdatascience.com/introducing-jupytext-9234fdff6c57) [![PyParis](https://img.shields.io/badge/YouTube-PyParis-red.svg)](https://www.youtube.com/watch?v=y-VEZenk824) @@ -34,54 +27,6 @@ Looking for a demo? - Watch the [PyParis talk](https://github.com/mwouts/jupytext_pyparis_2018/blob/master/README.md), - or, try Jupytext online with [binder](https://mybinder.org/v2/gh/mwouts/jupytext/master?filepath=demo)! -## Example usage - -### Writing notebooks as plain text - -You like to work with scripts? The good news is that plain scripts, which you can draft and test in your favorite IDE, open transparently as notebooks in Jupyter when using Jupytext. Run the notebook in Jupyter to generate the outputs, [associate](#paired-notebooks) an `.ipynb` representation, save and share your research as either a plain script or as a traditional Jupyter notebook with outputs. - -### Collaborating on Jupyter Notebooks - -With Jupytext, collaborating on Jupyter notebooks with Git becomes as easy as collaborating on text files. - -The setup is straightforward: -- Open your favorite notebook in Jupyter notebook -- [Associate](#paired-notebooks) a `.py` representation (for instance) to that notebook -- Save the notebook, and put the Python script under Git control. Sharing the `.ipynb` file is possible, but not required. - -Collaborating then works as follows: -- Your collaborator pulls your script. -- The script opens as a notebook in Jupyter, with no outputs (in JupyterLab right-click the script and use the open-with context menu). -- They run the notebook and save it. Outputs are regenerated, and a local `.ipynb` file is created. -- Note that, alternatively, the `.ipynb` file could have been regenerated with `jupytext --sync notebook.py`. -- They change the notebook, and push their updated script. The diff is nothing else than a standard diff on a Python script. -- You pull the changed script, and refresh your browser. Input cells are updated. The outputs from cells that were changed are removed. Your variables are untouched, so you have the option to run only the modified cells to get the new outputs. - -### Code refactoring - -In the animation below we propose a quick demo of Jupytext. While the example remains simple, it shows how your favorite text editor or IDE can be used to edit your Jupyter notebooks. IDEs are more convenient than Jupyter for navigating through code, editing and executing cells or fractions of cells, and debugging. - -- We start with a Jupyter notebook. -- The notebook includes a plot of the world population. The plot legend is not in order of decreasing population, we'll fix this. -- We want the notebook to be saved as both a `.ipynb` and a `.py` file: we select _Pair Notebook with a light Script_ in either the [Jupytext menu](#jupytext-menu-in-jupyter-notebook) in Jupyter Notebook, or in the [Jupytext commands](#jupytext-commands-in-jupyterlab) in JupyterLab. This has the effect of adding a `"jupytext": {"formats": "ipynb,py:light"},` entry to the notebook metadata. -- The Python script can be opened with PyCharm: - - Navigating in the code and documentation is easier than in Jupyter. - - The console is convenient for quick tests. We don't need to create cells for this. - - We find out that the columns of the data frame were not in the correct order. We update the corresponding cell, and get the correct plot. -- The Jupyter notebook is refreshed in the browser. Modified inputs are loaded from the Python script. Outputs and variables are preserved. We finally rerun the code and get the correct plot. - -![](https://gist.githubusercontent.com/mwouts/13de42d8bb514e4acf6481c580feffd0/raw/b8dd28f44678f8c91f262da2381276fc4d03b00a/JupyterPyCharm.gif) - -### Importing Jupyter Notebooks as modules - -Jupytext allows to import code from other Jupyter notebooks in a very simple manner. Indeed, all you need to do is to pair the notebook that you wish to import with a script, and import the resulting script. - -If the notebook contains demos and plots that you don't want to import, mark those cells as either -- _active_ only in the `ipynb` format, with the `{"active": "ipynb"}` cell metadata, or with an `active-ipynb` tag (you may use the [jupyterlab-celltags](https://github.com/jupyterlab/jupyterlab-celltags) extension for this). -- _frozen_, using the [freeze extension](https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/nbextensions/freeze/readme.html) for Jupyter notebook. - -Inactive cells will be commented in the paired script, and consequently will not be executed when the script is imported. - ## Installation [![Conda Version](https://img.shields.io/conda/vn/conda-forge/jupytext.svg)](https://anaconda.org/conda-forge/jupytext) @@ -110,7 +55,7 @@ If you don't have the notebook icon on text documents after a fresh restart of y ```python c.NotebookApp.contents_manager_class = "jupytext.TextFileContentsManager" ``` -to your `.jupyter/jupyter_notebook_config.py` file (generate a Jupyter config, if you don't have one yet, with `jupyter notebook --generate-config`). Our contents manager accepts a few options: default formats, default metadata filter, etc — read more on this [below](#global-configuration). Then, restart Jupyter Notebook or JupyterLab, either from the JupyterHub interface or from the command line with +to your `.jupyter/jupyter_notebook_config.py` file (generate a Jupyter config, if you don't have one yet, with `jupyter notebook --generate-config`). Our contents manager accepts a few options: default formats, default metadata filter, etc. Then, restart Jupyter Notebook or JupyterLab, either from the JupyterHub interface or from the command line with ```bash jupyter notebook # or lab ``` @@ -139,114 +84,22 @@ jupyter labextension install jupyterlab-jupytext ``` (the above requires `npm`, run `conda install nodejs` first if you don't have `npm`). -## Using Jupytext from within Jupyter +## Using Jupytext -### Paired notebooks +### Paired notebooks in the Jupyter Server Jupytext can write a given notebook to multiple files. In addition to the original notebook file, Jupytext can save the input cells to a text file — either a script or a Markdown document. Put the text file under version control for a clear commit history. Or refactor the paired script, and reimport the updated input cells by simply refreshing the notebook in Jupyter. -### Per-notebook configuration +### Configuring notebooks to use Jupytext Select the pairing for a given notebook using either the [Jupytext menu](#jupytext-menu-in-jupyter-notebook) in Jupyter Notebook, or the [Jupytext commands](#jupytext-commands-in-jupyterlab) in JupyterLab. -These command simply add a `"jupytext": {"formats": "ipynb,md"}`-like entry in the notebook metadata. You could also set that metadata yourself with _Edit/Edit Notebook Metadata_ in Jupyter Notebook. In JupyterLab, use [this extension](https://github.com/yuvipanda/jupyterlab-nbmetadata). - -The pairing information for one or multiple notebooks can be set on the command line: +Alternatively, the pairing information for one or multiple notebooks can be set on the command line: ``` jupytext --set-formats ipynb,py [--sync] notebook.ipynb ``` -You can pair a notebook to as many text representations as you want (see our _World population_ notebook in the demo folder). Format specifications are of the form -``` -[[path/][prefix]/][suffix.]ext[:format_name] -``` -where -- `ext` is one of `ipynb`, `md`, `Rmd`, `jl`, `py`, `R`, `sh`, `cpp`, `q`. Use the `auto` extension to have the script extension chosen according to the Jupyter kernel. -- `format_name` (optional) is either `light` (default for scripts), `bare`, `percent`, `hydrogen`, `sphinx` (Python only), `spin` (R only) — see below for the [format specifications](#Format-specifications). -- `path`, `prefix` and `suffix` allow to save the text representation to files with different names, or in a different folder. - -If you want to pair a notebook to a python script in a subfolder named `scripts`, set the formats metadata to `ipynb,scripts//py`. If the notebook is in a `notebooks` folder and you want the text representation to be in a `scripts` folder at the same level, set the Jupytext formats to `notebooks//ipynb,scripts//py`. - -Jupytext accepts a few additional options. These options should be added to the `"jupytext"` section in the metadata — use either the metadata editor or the `--opt/--format-options` argument on the command line. -- `comment_magics`: By default, Jupyter magics are commented when notebooks are exported to any other format than markdown. If you prefer otherwise, use this boolean option, or is global counterpart (see below). -- `notebook_metadata_filter`: By default, Jupytext only exports the `kernelspec` and `jupytext` metadata to the text files. Set `"jupytext": {"notebook_metadata_filter": "-all"}` if you want that the script has no notebook metadata at all. The value for `notebook_metadata_filter` is a comma separated list of additional/excluded (negated) entries, with `all` a keyword that allows to exclude all entries. -- `cell_metadata_filter`: By default, cell metadata `autoscroll`, `collapsed`, `scrolled`, `trusted` and `ExecuteTime` are not included in the text representation. Add or exclude more cell metadata with this option. - -### Global configuration -Jupytext's contents manager also accepts global options. These options have to be set on Jupytext's contents manager, so please first include the following line in your `.jupyter/jupyter_notebook_config.py` file: -```python -c.NotebookApp.contents_manager_class = "jupytext.TextFileContentsManager" -``` - -We start with the default format pairing. Say you want to always associate every `.ipynb` notebook with a `.md` file (and reciprocally). This is simply done by adding the following to your Jupyter configuration file: -```python -# Always pair ipynb notebooks to md files -c.ContentsManager.default_jupytext_formats = "ipynb,md" -``` -(and similarly for the other formats). - -In case the [`percent`](#the-percent-format) format is your favorite, add the following to your `.jupyter/jupyter_notebook_config.py` file: -```python -# Use the percent format when saving as py -c.ContentsManager.preferred_jupytext_formats_save = "py:percent" -``` -and then, Jupytext will understand `"jupytext": {"formats": "ipynb,py"}` as an instruction to create the paired Python script in the `percent` format. - -To disable global pairing for an individual notebook, set formats to a single format, e.g.: -`"jupytext": {"formats": "ipynb"}` - -### Metadata filtering - -You can specify which metadata to include or exclude in the text files created by Jupytext by setting `c.ContentsManager.default_notebook_metadata_filter` (notebook metadata) and `c.ContentsManager.default_cell_metadata_filter` (cell metadata). They accept a string of comma separated keywords. A minus sign `-` in font of a keyword means exclusion. - -Suppose you want to keep all the notebook metadata but `widgets` and `varInspector` in the YAML header. For cell metadata, you want to allow `ExecuteTime` and `autoscroll`, but not `hide_output`. You can set -```python -c.ContentsManager.default_notebook_metadata_filter = "all,-widgets,-varInspector" -c.ContentsManager.default_cell_metadata_filter = "ExecuteTime,autoscroll,-hide_output" -``` - -If you want that the text files created by Jupytext have no metadata, you may use the global metadata filters below. Please note that with this setting, the metadata is only preserved in the `.ipynb` file. -```python -c.ContentsManager.default_notebook_metadata_filter = "-all" -c.ContentsManager.default_cell_metadata_filter = "-all" -``` - -NB: All these global options (and more) are documented [here](https://github.com/mwouts/jupytext/blob/master/jupytext/contentsmanager.py). - -### Can I edit a notebook simultaneously in Jupyter and in a text editor? - -When saving a paired notebook using Jupytext's contents manager, Jupyter updates both the `.ipynb` and its text representation. The text representation can be edited outside of Jupyter. When the notebook is refreshed in Jupyter, the input cells are read from the text file, and the output cells from the `.ipynb` file. - -It is possible (and convenient) to leave the notebook open in Jupyter while you edit its text representation. However, you don't want that the two editors save the notebook simultaneously. To avoid this: -- deactivate Jupyter's autosave, by toggling the `"Autosave notebook"` menu entry (or run `%autosave 0` in a cell of the notebook) -- and refresh the notebook when you switch back from the editor to Jupyter. - -In case you forgot to refresh, and saved the Jupyter notebook while the text representation had changed, no worries: Jupyter will ask you which version you want to keep: -![Notebook changed](https://gist.githubusercontent.com/mwouts/13de42d8bb514e4acf6481c580feffd0/raw/fcbcd3c3fc1ec4a74669381b53753f9f783e10da/notebook_changed.png) - -When that occurs, please choose the version in which you made the latest changes. And give a second look to our advice to deactivate the autosaving of notebooks in Jupyter. - -### How to open scripts with either the text or notebook view in Jupyter? - -With Jupytext's contents manager for Jupyter, scripts and Markdown documents gain a notebook icon. If you don't see the notebook icon, double check the [contents manager configuration](https://github.com/mwouts/jupytext/blob/master/README.md#jupytexts-contents-manager). - -By default, Jupyter Notebook open scripts and Markdown documents as notebooks. If you want to open them with the text editor, select the document and click on _edit_: - -![Open as text](https://user-images.githubusercontent.com/29915202/53228364-42265400-3681-11e9-812d-46168c6e398c.png) - -In JupyterLab this is slightly different. Scripts and Markdown document also have a notebook icon. But they open as text by default. Open them as notebooks with the _Open With -> Notebook_ context menu (available in JupyterLab 0.35 and above): - -![](https://gist.githubusercontent.com/mwouts/13de42d8bb514e4acf6481c580feffd0/raw/403b53ac5097446a15ea664579ba44cd1badcc57/ContextMenuLab.png) - -If do not want to classify scripts or Markdown documents as notebooks, please use the `notebook_extension` option. For instance, if you want to get the notebook icon only for `.ipynb` and `.Rmd` files, set - -```python -c.ContentsManager.notebook_extensions = "ipynb,Rmd" -``` - -Please note that, with the above setting, Jupyter will not let you open scripts as notebooks. If you still want to do so, use Jupytext command line (see below) to first convert or pair the script to an `.ipynb` notebook. - -## Scripting Jupytext +For more information see [the jupytext documentation](https://jupytext.readthedocs.io). ### Command line conversion @@ -254,220 +107,10 @@ The package provides a `jupytext` script for command line conversion between the ```bash jupytext --to py notebook.ipynb # create a notebook.py file in the light format -jupytext --to py:percent notebook.ipynb # create a notebook.py file in the double percent format -jupytext --to py:percent --comment-magics false notebook.ipynb # create a notebook.py file in the double percent format, and do not comment magic commands -jupytext --to markdown notebook.ipynb # create a notebook.md file -jupytext --output script.py notebook.ipynb # create a script.py file - jupytext --to notebook notebook.py # overwrite notebook.ipynb (remove outputs) -jupytext --to notebook --update notebook.py # update notebook.ipynb (preserve outputs) -jupytext --to ipynb notebook1.md notebook2.py # overwrite notebook1.ipynb and notebook2.ipynb - -jupytext --to md --test notebook.ipynb # Test round trip conversion - -jupytext --to md --output - notebook.ipynb # display the markdown version on screen -jupytext --from ipynb --to py:percent # read ipynb from stdin and write double percent script on stdout -``` - -Jupytext has a `--sync` mode that updates all the paired representations of a notebook based on the file that was last modified. You may also find useful to `--pipe` the text representation of a notebook into tools like `black`: -```bash -jupytext --sync --pipe black notebook.ipynb # read most recent version of notebook, reformat with black, save -``` - -The `jupytext` command accepts many arguments. Use the `--set-formats` and the `--update-metadata` arguments to edit the pairing information or more generally the notebook metadata. Execute `jupytext --help` to access the documentation. - -### Jupytext as a Git pre-commit hook - -Jupytext is also available as a Git pre-commit hook. Use this if you want Jupytext to create and update the `.py` (or `.md`...) representation of the staged `.ipynb` notebooks. All you need is to create an executable `.git/hooks/pre-commit` file with the following content: -```bash -#!/bin/sh -# For every ipynb file in the git index, add a Python representation -jupytext --from ipynb --to py:light --pre-commit ``` -```bash -#!/bin/sh -# For every ipynb file in the git index: -# - apply black and flake8 -# - export the notebook to a Python script in folder 'python' -# - and add it to the git index -jupytext --from ipynb --pipe black --check flake8 --pre-commit -jupytext --from ipynb --to python//py:light --pre-commit -``` - -If you don't want notebooks to be committed (and only commit the representations), you can ask the pre-commit hook to unstage notebooks after conversion by adding the following line: -```bash -git reset HEAD **/*.ipynb -``` -Note that these hooks do not update the `.ipynb` notebook when you pull. Make sure to either run `jupytext` in the other direction, or to use our paired notebook and our contents manager for Jupyter. Also, Jupytext does not offer a merge driver. If a conflict occurs, solve it on the text representation and then update or recreate the `.ipynb` notebook. Or give a try to nbdime and its [merge driver](https://nbdime.readthedocs.io/en/stable/vcs.html#merge-driver). - -### Testing the round-trip conversion - -Representing Jupyter notebooks as scripts requires a solid round trip conversion. You don't want your notebooks (nor your scripts) to be modified because you are converting them to the other form. Our test suite includes a few hundred tests to ensure that round trip conversion is safe. - -You can easily test that the round trip conversion preserves your Jupyter notebooks and scripts. Run for instance: -```bash -# Test the ipynb -> py:percent -> ipynb round trip conversion -jupytext --test notebook.ipynb --to py:percent - -# Test the ipynb -> (py:percent + ipynb) -> ipynb (à la paired notebook) conversion -jupytext --test --update notebook.ipynb --to py:percent -``` - -Note that `jupytext --test` compares the resulting notebooks according to its expectations. If you wish to proceed to a strict comparison of the two notebooks, use `jupytext --test-strict`, and use the flag `-x` to report with more details on the first difference, if any. - -Please note that -- Scripts opened with Jupyter have a default [metadata filter](#default-metadata-filtering) that prevents additional notebook or cell -metadata to be added back to the script. Remove the filter if you want to store Jupytext's settings, or the kernel information, in the text file. -- Cell metadata are available in the `light` and `percent` formats, as well as in the Markdown and R Markdown formats. R scripts in `spin` format support cell metadata for code cells only. Sphinx Gallery scripts in `sphinx` format do not support cell metadata. -- By default, a few cell metadata are not included in the text representation of the notebook. And only the most standard notebook metadata are exported. Learn more on this in the sections for [notebook specific](#-per-notebook-configuration) and [global settings](#default-metadata-filtering) for metadata filtering. - -### Reading notebooks in Python - -You can also manipulate notebooks in a Python shell or script using Jupytext's main functions: - -```python -# Read a notebook from a file. Format can be any of 'py', 'md', 'jl:percent', ... -readf(nb_file, fmt=None) - -# Read a notebook from a string. Here, format should contain at least the file extension. -reads(text, fmt) - -# Return the text representation for a notebook in the desired format. -writes(notebook, fmt) - -# Write a notebook to a file in the desired format. -writef(notebook, nb_file, fmt=None) -``` - -## Format specifications - -### Markdown and R Markdown - -Save Jupyter notebooks as [Markdown](https://daringfireball.net/projects/markdown/syntax) documents and edit them in one of the many editors with good Markdown support. - -[R Markdown](https://rmarkdown.rstudio.com/authoring_quick_tour.html) is [RStudio](https://www.rstudio.com/)'s format for notebooks, with support for R, Python, and many [other languages](https://bookdown.org/yihui/rmarkdown/language-engines.html). - - -Jupytext's implementation for Jupyter notebooks as [Markdown](https://daringfireball.net/projects/markdown/syntax) or [R Markdown](https://rmarkdown.rstudio.com/authoring_quick_tour.html) documents is as follows: -- The notebook metadata (Jupyter kernel, etc) goes to a YAML header -- Code and raw cells are encoded as Markdown code blocks with triple backticks. In a Python notebook, a code cell starts with ` ```python` and ends with ` ``` `. Cell metadata are found after the language information, with a `key=value` syntax, where `value` is encoded in JSON format (Markdown) or R format (R Markdown). R Markdown [code cell options](https://yihui.name/knitr/options/) are mapped to the corresponding Jupyter cell metadata options, when available. -- Markdown cells are inserted verbatim and separated with two blank lines. When required (cells with metadata, cells that contain two blank lines or code blocks), Jupytext protects the cell boundary with HTML comments: `` and ``. Cells with explicit boundaries are [foldable](https://code.visualstudio.com/docs/editor/codebasics#_folding) in vscode, and can accept both a title and/or metadata in JSON format: ``. - -See how our `World population.ipynb` notebook in the [demo folder](https://github.com/mwouts/jupytext/tree/master/demo) is represented in [Markdown](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.md) or [R Markdown](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.Rmd). - -When you open a plain Markdown file in Jupytext, the Markdown text is rendered in Markdown cells. Cells breaks occur when the text contains two consecutive blank lines (or code cells). If you want to also split cells on Markdown headers, so that headers prefixed by one blank line appear at the top of a new cell, use the `split_at_heading` option. Set the option either on the command line, or by adding `"split_at_heading": true` to the jupytext section in the notebook metadata, or on Jupytext's content manager: - -```python -c.ContentsManager.split_at_heading = True -``` - -### Pandoc's Markdown - -Pandoc, the _Universal document converter_, can now read and write Jupyter notebooks - see [Pandoc's documentation](https://pandoc.org/MANUAL.html#creating-jupyter-notebooks-with-pandoc). - -Pandoc's Markdown format is available in Jupytext as `md:pandoc`. This requires `pandoc` in version 2.7.2 or above - please check Pandoc's version number with `pandoc -v` before running either `jupytext` command line, or before you start your Jupyter notebook or Lab server. Note that you can get the latest version of `pandoc` in a conda environment with -``` -conda install pandoc -c conda-forge -``` - -Pandoc's format uses Pandoc divs (`:::`) as explicit cell markers. See how our `World population.ipynb` notebook is [represented](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.pandoc.md) in that format. Please also note that `pandoc`, while preserving the HTML rendering, may reformat the text in some of the Markdown cells. If that is an issue for you, please wait until [jgm/pandoc#5408](https://github.com/jgm/pandoc/issues/5408) gets implemented. - -Jupytext currently strips the output cells before calling `pandoc`. As for the other formats, outputs cells can be preserved in paired notebooks. In the case of the pandoc format, paired notebooks are available with the metadata `"jupytext": {"formats": "md:pandoc,ipynb"}`. - -### The `light` format for notebooks as scripts - -The `light` format was created for this project. It is the default format for scripts. That format can read any script as a Jupyter notebook, even scripts which were never prepared to become a notebook. When a notebook is written as a script using this format, only a few cells markers are introduced—none if possible. - -The `light` format has: -- A (commented) YAML header, that contains the notebook metadata. -- Markdown cells are commented, and separated from other cells with a blank line. -- Code cells are exported verbatim (except for Jupyter magics, which are commented), and separated with blank lines. Code cells are reconstructed from consistent Python paragraphs (no function, class or multiline comment will be broken). -- Cells that contain more than one Python paragraphs need an explicit start-of-cell delimiter that is, by default, `# +` (`// +` in C++, etc). Cells that have explicit metadata have a cell header `# + {JSON}` where the metadata is represented, in JSON format. The default end of cell delimiter is `# -`, and is omitted when followed by another explicit start of cell marker. - -The `light` format is currently available for Python, Julia, R, Bash, Scheme, Clojure, Matlab, Octave, C++, q/kdb+ and IDL. Open our sample notebook in the `light` format [here](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.lgt.py). - -A variation of the `light` format is the `bare` format, with no cell marker at all. Please note that this format will split your code cells on code paragraphs. By default, this format still includes a YAML header - if you prefer to also remove the header, set `"notebook_metadata_filter": "-all"` in the jupytext section of your notebook metadata. - -The `light` format can use custom cell markers instead of `# +` or `# -`. If you prefer to mark cells with VScode/PyCharm (resp. Vim) folding markers, set `"cell_markers": "region,endregion"` (resp. `"{{{,}}}"`) in the jupytext section of the notebook metadata. If you want to configure this as a global default, add either -```python -c.ContentsManager.default_cell_markers = "region,endregion" # Use VScode/PyCharm region folding delimiters -``` -or -```python -c.ContentsManager.default_cell_markers = "{{{,}}}" # Use Vim region folding delimiters -``` -to your `.jupyter/jupyter_notebook_config.py` file. - - -### The `percent` format - -The `percent` format is a representation of Jupyter notebooks as scripts, in which cells are delimited with a commented double percent sign `# %%`. The format was introduced by Spyder five years ago, and is now supported by many editors, including -- [Spyder IDE](https://docs.spyder-ide.org/editor.html#defining-code-cells), -- [Hydrogen](https://atom.io/packages/hydrogen), a package for Atom, -- [VS Code](https://code.visualstudio.com/) with the [vscodeJupyter](https://marketplace.visualstudio.com/items?itemName=donjayamanne.jupyter) extension, -- [Python Tools for Visual Studio](https://github.com/Microsoft/PTVS), -- and [PyCharm Professional](https://www.jetbrains.com/pycharm/). - -Our implementation of the `percent` format is compatible with the original specifications by Spyder. We extended the format to allow markdown cells and cell metadata. Cell headers have the following structure: -```python -# %% Optional text [cell type] {optional JSON metadata} -``` -where cell type is either omitted (code cells), or `[markdown]` or `[raw]`. The content of markdown and raw cells is commented out in the resulting script. - -Percent scripts created by Jupytext have a header with an explicit format information. The format of scripts with no header is inferred automatically: scripts with at least one `# %%` cell are identified as `percent` scripts. Scripts with at least one double percent cell, and an uncommented Jupyter magic command, are identified as `hydrogen` scripts. - -The `percent` format is currently available for Python, Julia, R, Bash, Scheme, Clojure, Matlab, Octave, C++, q/kdb+ and IDL. Open our sample notebook in the `percent` format [here](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.pct.py). - -If the `percent` format is your favorite one, add the following to your `.jupyter/jupyter_notebook_config.py` file: -```python -c.ContentsManager.preferred_jupytext_formats_save = "py:percent" # or "auto:percent" -``` -Then, Jupytext's content manager will understand `"jupytext": {"formats": "ipynb,py"},` as an instruction to create the paired Python script in the `percent` format. - -By default, Jupyter magics are commented in the `percent` representation. If you run the percent scripts in Hydrogen, use instead the `hydrogen` format, a variant of the `percent` format that does not comment Jupyter magic commands. - -### Sphinx-gallery scripts - -Another popular notebook-like format for Python scripts is the Sphinx-gallery [format](https://sphinx-gallery.readthedocs.io/en/latest/tutorials/plot_notebook.html). Scripts that contain at least two lines with more than twenty hash signs are classified as Sphinx-Gallery notebooks by Jupytext. - -Comments in Sphinx-Gallery scripts are formatted using reStructuredText rather than markdown. They can be converted to markdown for a nicer display in Jupyter by adding a `c.ContentsManager.sphinx_convert_rst2md = True` line to your Jupyter configuration file. Please note that this is a non-reversible transformation—use this only with Binder. Revert to the default value `sphinx_convert_rst2md = False` when you edit Sphinx-Gallery files with Jupytext. - -Turn a GitHub repository containing Sphinx-Gallery scripts into a live notebook repository with [Binder](https://mybinder.org/) and Jupytext by adding only two files to the repo: -- `binder/requirements.txt`, a list of the required packages (including `jupytext`) -- `.jupyter/jupyter_notebook_config.py` with the following contents: -```python -c.NotebookApp.contents_manager_class = "jupytext.TextFileContentsManager" -c.ContentsManager.preferred_jupytext_formats_read = "py:sphinx" -c.ContentsManager.sphinx_convert_rst2md = True -``` - -Our sample notebook is also represented in `sphinx` format [here](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.spx.py). - -### R knitr::spin scripts - -The `spin` format is specific to R scripts. These scripts can be compiled into reports using either `knitr::spin` or [RStudio](https://rmarkdown.rstudio.com/articles_report_from_r_script.html). The implementation of the format is as follows: -- Jupyter metadata are in YAML format, in a `#' `-commented header. -- Markdown cells are commented with `#' `. -- Code cells are exported verbatim. Cell metadata are signalled with `#+`. Cells end with a blank line, an explicit start of cell marker, or a markdown cell. - - -## Fine tuning - -Jupyter magic commands are commented when exporting the notebook to text, except for the `markdown` and the `hydrogen` format. If you want to change this for a single line, add a `#escape` or `#noescape` flag on the same line as the magic, or a `"comment_magics": true` or `false` entry in the notebook metadata, in the `"jupytext"` section. Or set your preference globally on the contents manager by adding this line to `.jupyter/jupyter_notebook_config.py`: -```python -c.ContentsManager.comment_magics = True # or False -``` - -Also, you may want some cells to be active only in the Python, or R Markdown representation. For this, use the `active` cell metadata. Set `"active": "ipynb"` if you want that cell to be active only in the Jupyter notebook. And `"active": "py"` if you want it to be active only in the Python script. And `"active": "ipynb,py"` if you want it to be active in both, but not in the R Markdown representation... - -## Extending the `light` and `percent` formats to more languages - -You want to extend the `light` and `percent` format to another language? In principle that is easy, and you will only have to: -- document the language extension and comment by adding one line to `_SCRIPT_EXTENSIONS` in `languages.py`. -- contribute a sample notebook in `tests\notebooks\ipynb_[language]`. -- add two tests in `test_mirror.py`: one for the `light` format, and another one for the `percent` format. -- Make sure that the tests pass, and that the text representations of your notebook, found in `tests\notebooks\mirror\ipynb_to_script` and `tests\notebooks\mirror\ipynb_to_percent`, are valid scripts. +For more examples, see the [jupytext documentation](https://jupytext.readthedocs.io) ## Want to contribute? From 5a625bf64dd6b35fcb1823287d48b6f440292090 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Fri, 31 May 2019 07:55:31 -0700 Subject: [PATCH 4/7] updating docs --- docs/conf.py | 13 +++++++++++-- docs/{intro.md => index.md} | 16 ++++++++++++++-- docs/index.rst | 26 -------------------------- 3 files changed, 25 insertions(+), 30 deletions(-) rename docs/{intro.md => index.md} (84%) delete mode 100644 docs/index.rst diff --git a/docs/conf.py b/docs/conf.py index 8ceb5e576..5c95400ad 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -37,7 +37,7 @@ } # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -55,7 +55,16 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Output file base name for HTML help builder. htmlhelp_basename = "jupytext" + +from recommonmark.transform import AutoStructify + + +def setup(app): + app.add_config_value( + "recommonmark_config", {"auto_toc_tree_section": "Contents"}, True + ) + app.add_transform(AutoStructify) diff --git a/docs/intro.md b/docs/index.md similarity index 84% rename from docs/intro.md rename to docs/index.md index fdc58f708..c75004745 100644 --- a/docs/intro.md +++ b/docs/index.md @@ -1,3 +1,5 @@ +# Jupytext + # Introduction [![](https://travis-ci.com/mwouts/jupytext.svg?branch=master)](https://travis-ci.com/mwouts/jupytext) @@ -10,9 +12,12 @@ Jupytext can save Jupyter notebooks as - Markdown and R Markdown documents, - Julia, Python, R, Bash, Scheme, Clojure, Matlab, Octave, C++ and q/kdb+ scripts. -The plain text formatting of the notebook cell divisions in scripts is configurable. Popular choices include the `percent` format [`# %%`] used by Spyder, VSCode, and others, and the `light` format which was developed to support this project. `light` uses as few cell markers as possible and is particularly suited for importing a pre-existing python script as a notebook with cell divisions automatically inferred from paragraph breaks in the source code. +Jupytext can also convert these formats **into Jupyter Notebooks**, allowing +for two-directional syncing between formats. See below for a quick demo. + +![](https://gist.githubusercontent.com/mwouts/13de42d8bb514e4acf6481c580feffd0/raw/b8dd28f44678f8c91f262da2381276fc4d03b00a/JupyterPyCharm.gif) -## Quickstart +## How to use Jupytext There are multiple ways to use `jupytext`: - **Directly from Jupyter Notebook or JupyterLab.** Jupytext provides a _contents manager_ that allows Jupyter to save your notebook to your favorite format (`.py`, `.R`, `.jl`, `.md`, `.Rmd`...) in addition to (or in place of) the traditional `.ipynb` file. The text representation can be edited in your favorite editor. When you're done, refresh the notebook in Jupyter: inputs cells are loaded from the text file, while output cells are reloaded from the `.ipynb` file if present. Refreshing preserves kernel variables, so you can resume your work in the notebook and run the modified cells without having to rerun the notebook in full. @@ -30,7 +35,14 @@ Looking for a demo? - Watch the [PyParis talk](https://github.com/mwouts/jupytext_pyparis_2018/blob/master/README.md), - or, try Jupytext online with [binder](https://mybinder.org/v2/gh/mwouts/jupytext/master?filepath=demo)! +## Contents +* [Install Jupytext](install) +* [Usecases for Jupytext](examples) +* [Using with the Jupyter server](using-server) +* [Using at the command line](using-cli) +* [Supported document formats](formats) +* [Advanced configuration and use](advanced) ## Want to contribute? diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 8b7cb970e..000000000 --- a/docs/index.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. Jupytext documentation master file, created by - sphinx-quickstart on Thu May 30 00:07:50 2019. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to Jupytext's documentation! -==================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - intro - examples - install - using-server - using-cli - formats - advanced - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` From 5ff796e336b997e928cb332b55b18b97c8e74ab8 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Sun, 2 Jun 2019 09:11:05 +0200 Subject: [PATCH 5/7] Mention --update command And support of IDL language --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bc7a891bd..06905ed43 100755 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Have you always wished Jupyter notebooks were plain text documents? Wished you c Jupytext can save Jupyter notebooks as - Markdown and R Markdown documents, -- Julia, Python, R, Bash, Scheme, Clojure, Matlab, Octave, C++ and q/kdb+ scripts. +- Julia, Python, R, Bash, Scheme, Clojure, Matlab, Octave, C++, q/kdb+ and IDL scripts. It can also convert these documents **into** Jupyter Notebooks, allowing you to synchronize content in both @@ -108,6 +108,7 @@ The package provides a `jupytext` script for command line conversion between the ```bash jupytext --to py notebook.ipynb # create a notebook.py file in the light format jupytext --to notebook notebook.py # overwrite notebook.ipynb (remove outputs) +jupytext --update --to notebook notebook.py # update notebook.ipynb (preserve outputs and metadata) ``` For more examples, see the [jupytext documentation](https://jupytext.readthedocs.io) From 57be924563dce316307c7c12786bd096085f956b Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Tue, 4 Jun 2019 14:03:28 +0200 Subject: [PATCH 6/7] Introduction and Index are separate files --- docs/index.md | 46 ++++---------------------------------------- docs/introduction.md | 46 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 42 deletions(-) create mode 100644 docs/introduction.md diff --git a/docs/index.md b/docs/index.md index c75004745..e0ceb29d9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,49 +1,11 @@ -# Jupytext - -# Introduction - -[![](https://travis-ci.com/mwouts/jupytext.svg?branch=master)](https://travis-ci.com/mwouts/jupytext) -[![](https://codecov.io/github/mwouts/jupytext/coverage.svg?branch=master)](https://codecov.io/github/mwouts/jupytext?branch=master) -[![](https://img.shields.io/badge/lgtm-A+-brightgreen.svg)](https://lgtm.com/projects/g/mwouts/jupytext/context:python) - -Have you always wished Jupyter notebooks were plain text documents? Wished you could edit them in your favorite IDE? And get clear and meaningful diffs when doing version control? Then... Jupytext may well be the tool you're looking for! - -Jupytext can save Jupyter notebooks as -- Markdown and R Markdown documents, -- Julia, Python, R, Bash, Scheme, Clojure, Matlab, Octave, C++ and q/kdb+ scripts. - -Jupytext can also convert these formats **into Jupyter Notebooks**, allowing -for two-directional syncing between formats. See below for a quick demo. - -![](https://gist.githubusercontent.com/mwouts/13de42d8bb514e4acf6481c580feffd0/raw/b8dd28f44678f8c91f262da2381276fc4d03b00a/JupyterPyCharm.gif) - -## How to use Jupytext - -There are multiple ways to use `jupytext`: -- **Directly from Jupyter Notebook or JupyterLab.** Jupytext provides a _contents manager_ that allows Jupyter to save your notebook to your favorite format (`.py`, `.R`, `.jl`, `.md`, `.Rmd`...) in addition to (or in place of) the traditional `.ipynb` file. The text representation can be edited in your favorite editor. When you're done, refresh the notebook in Jupyter: inputs cells are loaded from the text file, while output cells are reloaded from the `.ipynb` file if present. Refreshing preserves kernel variables, so you can resume your work in the notebook and run the modified cells without having to rerun the notebook in full. -- **On the [command line](using-cli.md)**. `jupytext` converts Jupyter notebooks to their text representation, and back. The command line tool can act on notebooks in many ways. It can synchronize multiple representations of a notebook, pipe a notebook into a reformatting tool like `black`, etc... It can also work as a [pre-commit hook](#jupytext-as-a-git-pre-commit-hook) if you wish to automatically update the text representation when you commit the `.ipynb` file. -- **in Vim**: edit your Jupyter notebooks, represented as a Markdown document, or a Python script, with [jupytext.vim](https://github.com/goerz/jupytext.vim). - -## Demo time - -[![](https://img.shields.io/badge/TDS-Introducing%20Jupytext-blue.svg)](https://towardsdatascience.com/introducing-jupytext-9234fdff6c57) -[![](https://img.shields.io/badge/YouTube-PyParis-red.svg)](https://www.youtube.com/watch?v=y-VEZenk824) -[![](https://img.shields.io/badge/Binder-Try%20it!-blue.svg)](https://mybinder.org/v2/gh/mwouts/jupytext/master?filepath=demo) - -Looking for a demo? -- Read the original [announcement](https://towardsdatascience.com/introducing-jupytext-9234fdff6c57) in Towards Data Science, -- Watch the [PyParis talk](https://github.com/mwouts/jupytext_pyparis_2018/blob/master/README.md), -- or, try Jupytext online with [binder](https://mybinder.org/v2/gh/mwouts/jupytext/master?filepath=demo)! +# Jupyter notebooks as Markdown documents, Julia, Python or R scripts ## Contents -* [Install Jupytext](install) +* [Introduction](introduction) * [Usecases for Jupytext](examples) -* [Using with the Jupyter server](using-server) +* [Installation](install) +* [Using Jupytext in Jupyter](using-server) * [Using at the command line](using-cli) * [Supported document formats](formats) * [Advanced configuration and use](advanced) - -## Want to contribute? - -Contributions are welcome. Please let us know how you use `jupytext` and how we could improve it. You think the documentation could be improved? Go ahead! And stay tuned for more demos on [medium](https://medium.com/@marc.wouts) and [twitter](https://twitter.com/marcwouts)! diff --git a/docs/introduction.md b/docs/introduction.md new file mode 100644 index 000000000..634a0bebe --- /dev/null +++ b/docs/introduction.md @@ -0,0 +1,46 @@ +# Introduction + +[![](https://travis-ci.com/mwouts/jupytext.svg?branch=master)](https://travis-ci.com/mwouts/jupytext) +[![](https://codecov.io/github/mwouts/jupytext/coverage.svg?branch=master)](https://codecov.io/github/mwouts/jupytext?branch=master) +[![](https://img.shields.io/badge/lgtm-A+-brightgreen.svg)](https://lgtm.com/projects/g/mwouts/jupytext/context:python) + +Have you always wished Jupyter notebooks were plain text documents? Wished you could edit them in your favorite IDE? And get clear and meaningful diffs when doing version control? Then... Jupytext may well be the tool you're looking for! + +Jupytext can save Jupyter notebooks as +- Markdown and R Markdown documents, +- Julia, Python, R, Bash, Scheme, Clojure, Matlab, Octave, C++, q/kdb+ and IDL scripts. + +Jupytext can also convert these formats **into Jupyter Notebooks**, allowing +for two-directional syncing between formats. See below for a quick demo. + +![](https://gist.githubusercontent.com/mwouts/13de42d8bb514e4acf6481c580feffd0/raw/b8dd28f44678f8c91f262da2381276fc4d03b00a/JupyterPyCharm.gif) + +## How to use Jupytext + +There are multiple ways to use `jupytext`: +- **Directly from Jupyter Notebook or JupyterLab.** Jupytext provides a _contents manager_ that allows Jupyter to save your notebook to your favorite format (`.py`, `.R`, `.jl`, `.md`, `.Rmd`...) in addition to (or in place of) the traditional `.ipynb` file. The text representation can be edited in your favorite editor. When you're done, refresh the notebook in Jupyter: inputs cells are loaded from the text file, while output cells are reloaded from the `.ipynb` file if present. Refreshing preserves kernel variables, so you can resume your work in the notebook and run the modified cells without having to rerun the notebook in full. +- **On the [command line](using-cli.md)**. `jupytext` converts Jupyter notebooks to their text representation, and back. The command line tool can act on notebooks in many ways. It can synchronize multiple representations of a notebook, pipe a notebook into a reformatting tool like `black`, etc... It can also work as a [pre-commit hook](using-cli.html#jupytext-as-a-git-pre-commit-hook) if you wish to automatically update the text representation when you commit the `.ipynb` file. +- **in Vim**: edit your Jupyter notebooks, represented as a Markdown document, or a Python script, with [jupytext.vim](https://github.com/goerz/jupytext.vim). + +## Jupytext formats + +Jupytext implements a series of text formats for notebooks, which are documented [here](formats.md). + +In short: the Markdown representation of notebooks fits well the notebooks that contain narratives, while notebooks that mostly contain code are conveniently saved as scripts. The most popular formats for notebooks as scripts are: +- the `percent` format (in which cells are delimited with `# %%`) also used by Spyder, VSCode, PyCharm, and others, +- and the `light` format which was developed to support this project. `light` uses as few cell markers as possible and is particularly suited for importing a pre-existing python script as a notebook with cell divisions automatically inferred from paragraph breaks in the source code. + +## Demo time + +[![](https://img.shields.io/badge/TDS-Introducing%20Jupytext-blue.svg)](https://towardsdatascience.com/introducing-jupytext-9234fdff6c57) +[![](https://img.shields.io/badge/YouTube-PyParis-red.svg)](https://www.youtube.com/watch?v=y-VEZenk824) +[![](https://img.shields.io/badge/Binder-Try%20it!-blue.svg)](https://mybinder.org/v2/gh/mwouts/jupytext/master?filepath=demo) + +Looking for a demo? +- Read the original [announcement](https://towardsdatascience.com/introducing-jupytext-9234fdff6c57) in Towards Data Science, +- Watch the [PyParis talk](https://github.com/mwouts/jupytext_pyparis_2018/blob/master/README.md), +- or, try Jupytext online with [binder](https://mybinder.org/v2/gh/mwouts/jupytext/master?filepath=demo)! + +## Want to contribute? + +Contributions are welcome. Please let us know how you use `jupytext` and how we could improve it. You think the documentation could be improved? Go ahead! And stay tuned for more demos on [medium](https://medium.com/@marc.wouts) and [twitter](https://twitter.com/marcwouts)! From cfb59753b0713bb2f0a7daec6849ddf280e4f853 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Tue, 4 Jun 2019 14:03:44 +0200 Subject: [PATCH 7/7] Fixed links --- docs/examples.md | 8 ++++---- docs/install.md | 2 +- docs/using-cli.md | 6 +++--- docs/using-server.md | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index fd9226865..5ea5c6f27 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,8 +1,8 @@ -# Example usage +# Usecases for Jupytext ## Writing notebooks as plain text -You like to work with scripts? The good news is that plain scripts, which you can draft and test in your favorite IDE, open transparently as notebooks in Jupyter when using Jupytext. Run the notebook in Jupyter to generate the outputs, [associate](#paired-notebooks) an `.ipynb` representation, save and share your research as either a plain script or as a traditional Jupyter notebook with outputs. +You like to work with scripts? The good news is that plain scripts, which you can draft and test in your favorite IDE, open transparently as notebooks in Jupyter when using Jupytext. Run the notebook in Jupyter to generate the outputs, [associate](using-server.html#paired-notebooks) an `.ipynb` representation, save and share your research as either a plain script or as a traditional Jupyter notebook with outputs. ## Collaborating on Jupyter Notebooks @@ -10,7 +10,7 @@ With Jupytext, collaborating on Jupyter notebooks with Git becomes as easy as co The setup is straightforward: - Open your favorite notebook in Jupyter notebook -- [Associate](#paired-notebooks) a `.py` representation (for instance) to that notebook +- [Associate](using-server.html#paired-notebooks) a `.py` representation (for instance) to that notebook - Save the notebook, and put the Python script under Git control. Sharing the `.ipynb` file is possible, but not required. Collaborating then works as follows: @@ -27,7 +27,7 @@ In the animation below we propose a quick demo of Jupytext. While the example re - We start with a Jupyter notebook. - The notebook includes a plot of the world population. The plot legend is not in order of decreasing population, we'll fix this. -- We want the notebook to be saved as both a `.ipynb` and a `.py` file: we select _Pair Notebook with a light Script_ in either the [Jupytext menu](#jupytext-menu-in-jupyter-notebook) in Jupyter Notebook, or in the [Jupytext commands](#jupytext-commands-in-jupyterlab) in JupyterLab. This has the effect of adding a `"jupytext": {"formats": "ipynb,py:light"},` entry to the notebook metadata. +- We want the notebook to be saved as both a `.ipynb` and a `.py` file: we select _Pair Notebook with a light Script_ in either the [Jupytext menu](install.html#jupytext-menu-in-jupyter-notebook) in Jupyter Notebook, or in the [Jupytext commands](install.html#jupytext-commands-in-jupyterlab) in JupyterLab. This has the effect of adding a `"jupytext": {"formats": "ipynb,py:light"},` entry to the notebook metadata. - The Python script can be opened with PyCharm: - Navigating in the code and documentation is easier than in Jupyter. - The console is convenient for quick tests. We don't need to create cells for this. diff --git a/docs/install.md b/docs/install.md index 13180c69d..83857741d 100644 --- a/docs/install.md +++ b/docs/install.md @@ -26,7 +26,7 @@ If you don't have the notebook icon on text documents after a fresh restart of y ```python c.NotebookApp.contents_manager_class = "jupytext.TextFileContentsManager" ``` -to your `.jupyter/jupyter_notebook_config.py` file (generate a Jupyter config, if you don't have one yet, with `jupyter notebook --generate-config`). Our contents manager accepts a few options: default formats, default metadata filter, etc — read more on this [below](#global-configuration). Then, restart Jupyter Notebook or JupyterLab, either from the JupyterHub interface or from the command line with +to your `.jupyter/jupyter_notebook_config.py` file (generate a Jupyter config, if you don't have one yet, with `jupyter notebook --generate-config`). Our contents manager accepts a few options: default formats, default metadata filter, etc — read more on this [below](using-server.html#global-configuration). Then, restart Jupyter Notebook or JupyterLab, either from the JupyterHub interface or from the command line with ```bash jupyter notebook # or lab ``` diff --git a/docs/using-cli.md b/docs/using-cli.md index 2bf344888..d5a34f4de 100644 --- a/docs/using-cli.md +++ b/docs/using-cli.md @@ -1,4 +1,4 @@ -# Using Jupytext with the command line +# Using Jupytext at the command line ## Command line conversion @@ -69,10 +69,10 @@ jupytext --test --update notebook.ipynb --to py:percent Note that `jupytext --test` compares the resulting notebooks according to its expectations. If you wish to proceed to a strict comparison of the two notebooks, use `jupytext --test-strict`, and use the flag `-x` to report with more details on the first difference, if any. Please note that -- Scripts opened with Jupyter have a default [metadata filter](#default-metadata-filtering) that prevents additional notebook or cell +- Scripts opened with Jupyter have a default [metadata filter](using-server.html#metadata-filtering) that prevents additional notebook or cell metadata to be added back to the script. Remove the filter if you want to store Jupytext's settings, or the kernel information, in the text file. - Cell metadata are available in the `light` and `percent` formats, as well as in the Markdown and R Markdown formats. R scripts in `spin` format support cell metadata for code cells only. Sphinx Gallery scripts in `sphinx` format do not support cell metadata. -- By default, a few cell metadata are not included in the text representation of the notebook. And only the most standard notebook metadata are exported. Learn more on this in the sections for [notebook specific](#-per-notebook-configuration) and [global settings](#default-metadata-filtering) for metadata filtering. +- By default, a few cell metadata are not included in the text representation of the notebook. And only the most standard notebook metadata are exported. Learn more on this in the sections for [notebook specific](using-server.html#per-notebook-configuration) and [global settings](using-server.html#metadata-filtering) for metadata filtering. ## Reading notebooks in Python diff --git a/docs/using-server.md b/docs/using-server.md index 7ce1c2e66..632e5da72 100644 --- a/docs/using-server.md +++ b/docs/using-server.md @@ -1,4 +1,4 @@ -# Using Jupytext from within Jupyter +# Using Jupytext in Jupyter ## Paired notebooks @@ -6,7 +6,7 @@ Jupytext can write a given notebook to multiple files. In addition to the origin ## Per-notebook configuration -Select the pairing for a given notebook using either the [Jupytext menu](#jupytext-menu-in-jupyter-notebook) in Jupyter Notebook, or the [Jupytext commands](#jupytext-commands-in-jupyterlab) in JupyterLab. +Select the pairing for a given notebook using either the [Jupytext menu](install.html#jupytext-menu-in-jupyter-notebook) in Jupyter Notebook, or the [Jupytext commands](install.html#jupytext-commands-in-jupyterlab) in JupyterLab. These command simply add a `"jupytext": {"formats": "ipynb,md"}`-like entry in the notebook metadata. You could also set that metadata yourself with _Edit/Edit Notebook Metadata_ in Jupyter Notebook. In JupyterLab, use [this extension](https://github.com/yuvipanda/jupyterlab-nbmetadata). @@ -20,7 +20,7 @@ You can pair a notebook to as many text representations as you want (see our _Wo ``` where - `ext` is one of `ipynb`, `md`, `Rmd`, `jl`, `py`, `R`, `sh`, `cpp`, `q`. Use the `auto` extension to have the script extension chosen according to the Jupyter kernel. -- `format_name` (optional) is either `light` (default for scripts), `bare`, `percent`, `hydrogen`, `sphinx` (Python only), `spin` (R only) — see below for the [format specifications](#Format-specifications). +- `format_name` (optional) is either `light` (default for scripts), `bare`, `percent`, `hydrogen`, `sphinx` (Python only), `spin` (R only) — see the [format specifications](formats.md). - `path`, `prefix` and `suffix` allow to save the text representation to files with different names, or in a different folder. If you want to pair a notebook to a python script in a subfolder named `scripts`, set the formats metadata to `ipynb,scripts//py`. If the notebook is in a `notebooks` folder and you want the text representation to be in a `scripts` folder at the same level, set the Jupytext formats to `notebooks//ipynb,scripts//py`. @@ -44,7 +44,7 @@ c.ContentsManager.default_jupytext_formats = "ipynb,md" ``` (and similarly for the other formats). -In case the [`percent`](#the-percent-format) format is your favorite, add the following to your `.jupyter/jupyter_notebook_config.py` file: +In case the [`percent`](formats.html#the-percent-format) format is your favorite, add the following to your `.jupyter/jupyter_notebook_config.py` file: ```python # Use the percent format when saving as py c.ContentsManager.preferred_jupytext_formats_save = "py:percent"