Skip to content

Commit

Permalink
Cell and notebook metadata filters in the content manager
Browse files Browse the repository at this point in the history
  • Loading branch information
mwouts committed Oct 23, 2018
1 parent dceade8 commit 7f981e9
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 14 deletions.
40 changes: 27 additions & 13 deletions jupytext/contentsmanager.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,18 @@ def all_nb_extensions(self):
'scripts as Sphinx gallery scripts.',
config=True)

default_notebook_metadata_filter = Unicode(
u'',
help="Cell metadata that should be save in the text representations. "
"Examples: 'all', '-all', 'widgets,nteract', 'kernelspec,jupytext-all'",
config=True)

default_cell_metadata_filter = Unicode(
u'',
help="Notebook metadata that should be saved in the text representations. "
"Examples: 'all', 'hide_input,hide_output'",
config=True)

comment_magics = Enum(
values=[True, False],
allow_none=True,
Expand Down Expand Up @@ -215,11 +227,8 @@ def _read_notebook(self, os_path, as_version=4):
"""Read a notebook from an os path."""
_, fmt, ext = file_fmt_ext(os_path)
if ext in self.nb_extensions:
format_name = self.preferred_format(
fmt, self.preferred_jupytext_formats_read)
with mock.patch('nbformat.reads',
_jupytext_reads(fmt, format_name,
self.sphinx_convert_rst2md)):
format_name = self.preferred_format(fmt, self.preferred_jupytext_formats_read)
with mock.patch('nbformat.reads', _jupytext_reads(fmt, format_name, self.sphinx_convert_rst2md)):
return super(TextFileContentsManager, self)._read_notebook(os_path, as_version)
else:
return super(TextFileContentsManager, self)._read_notebook(os_path, as_version)
Expand Down Expand Up @@ -283,18 +292,14 @@ def get(self, path, content=True, type=None, format=None,
break

if source_format != fmt:
self.log.info(u'Reading SOURCE from {}'.format(
os.path.basename(nb_file + source_format)))
self.log.info(u'Reading SOURCE from {}'.format(os.path.basename(nb_file + source_format)))
model_outputs = model
model = self.get(nb_file + source_format, content=content,
type=type, format=format,
load_alternative_format=False)
type=type, format=format, load_alternative_format=False)
elif outputs_format != fmt:
self.log.info(u'Reading OUTPUTS from {}'.format(
os.path.basename(nb_file + outputs_format)))
self.log.info(u'Reading OUTPUTS from {}'.format(os.path.basename(nb_file + outputs_format)))
model_outputs = self.get(nb_file + outputs_format, content=content,
type=type, format=format,
load_alternative_format=False)
type=type, format=format, load_alternative_format=False)
else:
model_outputs = None

Expand Down Expand Up @@ -326,6 +331,15 @@ def get(self, path, content=True, type=None, format=None,
except OverflowError:
pass

if self.default_notebook_metadata_filter:
(model['content'].metadata.setdefault('jupytext', {})
.setdefault('metadata_filter', {})
.setdefault('notebook', self.default_notebook_metadata_filter))
if self.default_cell_metadata_filter:
(model['content'].metadata.setdefault('jupytext', {})
.setdefault('metadata_filter', {})
.setdefault('cells', self.default_cell_metadata_filter))

if model_outputs:
combine_inputs_with_outputs(model['content'], model_outputs['content'])
elif not fmt.endswith('.ipynb'):
Expand Down
40 changes: 40 additions & 0 deletions tests/test_contentsmanager.py
Original file line number Diff line number Diff line change
Expand Up @@ -532,3 +532,43 @@ def test_save_in_pct_and_lgt_auto_extensions(nb_file, tmpdir):
# check that text representation exists in light format
with open(str(tmpdir.join(tmp_lgt_script))) as stream:
assert read_format_from_metadata(stream.read(), '.lgt' + auto_ext) == 'light'


@pytest.mark.parametrize('nb_file', list_notebooks('ipynb'))
def test_metadata_filter_is_effective(nb_file, tmpdir):
nb = jupytext.readf(nb_file)
tmp_ipynb = 'notebook.ipynb'
tmp_script = 'notebook.py'

# create contents manager
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)

# save notebook to tmpdir
cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb)

# set config
cm.default_jupytext_formats = 'ipynb,py'
cm.default_notebook_metadata_filter = 'jupytext-all'
cm.default_cell_metadata_filter = '-all'

# load notebook
nb = cm.get(tmp_ipynb)['content']

# save notebook again
with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', True):
cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb)

# read text version
nb2 = jupytext.readf(str(tmpdir.join(tmp_script)))

# test no metadata
assert nb2.metadata.keys() == {'jupytext'}
for cell in nb2.cells:
assert not cell.metadata

# read paired notebook
with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', True):
nb3 = cm.get(tmp_script)['content']

compare_notebooks(nb, nb3)
41 changes: 40 additions & 1 deletion tests/test_trust_notebook.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ def test_ipynb_notebooks_can_be_trusted(nb_file, tmpdir):
cm.default_jupytext_formats = 'ipynb,py'
cm.root_dir = str(tmpdir)
model = cm.get(file)
nb = model['content']
cm.save(model, py_file)

# Unsign and test notebook
Expand Down Expand Up @@ -69,3 +68,43 @@ def test_ipynb_notebooks_can_be_trusted(nb_file, tmpdir):
assert cell.metadata.get('trusted', True)

assert model['content'] == nb2['content']


@pytest.mark.parametrize('nb_file', list_notebooks('ipynb_py'))
def test_ipynb_notebooks_can_be_trusted_even_with_metadata_filter(nb_file, tmpdir):
cm = TextFileContentsManager()
root, file = os.path.split(nb_file)
tmp_ipynb = str(tmpdir.join(file))
py_file = file.replace('.ipynb', '.py')
tmp_py = str(tmpdir.join(py_file))
shutil.copy(nb_file, tmp_ipynb)

cm.default_jupytext_formats = 'ipynb,py'
cm.default_notebook_metadata_filter = 'all'
cm.default_cell_metadata_filter = '-all'
cm.root_dir = str(tmpdir)
model = cm.get(file)
cm.save(model, py_file)

# Unsign notebook
nb = model['content']
for cell in nb.cells:
if 'trusted' in cell.metadata:
cell.metadata.pop('trusted')

cm.notary.unsign(nb)

# Trust and reload
cm.trust_notebook(py_file)

model = cm.get(file)
for cell in model['content'].cells:
assert cell.metadata.get('trusted', True)

# Remove py file, content should be the same
os.remove(tmp_py)
nb2 = cm.get(file)
for cell in nb2['content'].cells:
assert cell.metadata.get('trusted', True)

assert model['content'] == nb2['content']

0 comments on commit 7f981e9

Please sign in to comment.