Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use config in read and write functions #753

Merged
merged 4 commits into from
Mar 16, 2021
Merged

Use config in read and write functions #753

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9f0e269
Version 1.10.4-dev
mwouts Mar 14, 2021
4bc7875
jupytext.read and write accept a config argument
mwouts Mar 14, 2021
88725af
use the local configuration in the pre-commit mode
mwouts Mar 14, 2021
4a474ea
Settings from the config file have precedence over those in the noteb…
mwouts Mar 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 16 additions & 26 deletions jupytext/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

from .combine import combine_inputs_with_outputs
from .compare import NotebookDifference, compare, test_round_trip_conversion
from .config import load_jupytext_config, prepare_notebook_for_save
from .config import load_jupytext_config, notebook_formats
from .formats import (
_BINARY_FORMAT_OPTIONS,
_VALID_FORMAT_OPTIONS,
Expand Down Expand Up @@ -481,8 +481,6 @@ def jupytext_single_file(nb_file, args, log):
if ext:
fmt = {"extension": ext}
if fmt:
if config:
config.set_default_format_options(fmt)
set_format_options(fmt, args.format_options)
log(
"[jupytext] Reading {}{}".format(
Expand All @@ -493,23 +491,16 @@ def jupytext_single_file(nb_file, args, log):
)
)

notebook = read(nb_file, fmt=fmt)
notebook = read(nb_file, fmt=fmt, config=config)
if "extension" in fmt and "format_name" not in fmt:
text_representation = notebook.metadata.get("jupytext", {}).get(
"text_representation", {}
)
if text_representation.get("extension") == fmt["extension"]:
fmt["format_name"] = text_representation["format_name"]

if config and "formats" not in notebook.metadata.get("jupytext", {}):
default_formats = config.default_formats(nb_file)
if default_formats:
notebook.metadata.setdefault("jupytext", {})["formats"] = default_formats

# Compute actual extension when using script/auto, and update nb_dest if necessary
dest_fmt = args.output_format
if dest_fmt and config:
config.set_default_format_options(dest_fmt)
if dest_fmt and dest_fmt["extension"] == ".auto":
dest_fmt = check_auto_ext(dest_fmt, notebook.metadata, "--to")
if not args.output and nb_file != "-":
Expand Down Expand Up @@ -583,11 +574,14 @@ def jupytext_single_file(nb_file, args, log):
# Read paired notebooks, except if the pair is being created
nb_files = [nb_file, nb_dest]
if args.sync:
set_prefix_and_suffix(fmt, notebook, nb_file)
formats = notebook_formats(
notebook, config, nb_file, fallback_on_current_fmt=False
)
set_prefix_and_suffix(fmt, formats, nb_file)
if args.set_formats is None:
try:
notebook, inputs_nb_file, outputs_nb_file = load_paired_notebook(
notebook, fmt, nb_file, log, args.pre_commit_mode
notebook, fmt, config, formats, nb_file, log, args.pre_commit_mode
)
nb_files = [inputs_nb_file, outputs_nb_file]
except NotAPairedNotebook as err:
Expand Down Expand Up @@ -693,7 +687,7 @@ def jupytext_single_file(nb_file, args, log):
dest_text = writes(notebook, fmt=dest_fmt)
notebook = reads(dest_text, fmt=dest_fmt)

text = writes(notebook, fmt=fmt)
text = writes(notebook, fmt=fmt, config=config)

if args.test_strict:
compare(text, org_text)
Expand Down Expand Up @@ -736,7 +730,7 @@ def lazy_write(path, fmt=None, action=None, update_timestamp_only=False):
_, ext = os.path.splitext(path)
fmt = copy(fmt or {})
fmt = long_form_one_format(fmt, update={"extension": ext})
new_content = writes(notebook, fmt=fmt)
new_content = writes(notebook, fmt=fmt, config=config)
diff = None
if not new_content.endswith("\n"):
new_content += "\n"
Expand Down Expand Up @@ -830,7 +824,7 @@ def write_function(path, fmt):
update_timestamp_only=(path == inputs_nb_file),
)

formats = prepare_notebook_for_save(notebook, config, nb_file)
formats = notebook_formats(notebook, config, nb_file)
write_pair(nb_file, formats, write_function)
return untracked_files

Expand Down Expand Up @@ -918,11 +912,9 @@ def set_format_options(fmt, format_options):
fmt[key] = value


def set_prefix_and_suffix(fmt, notebook, nb_file):
def set_prefix_and_suffix(fmt, formats, nb_file):
"""Add prefix and suffix information from jupytext.formats if format and path matches"""
for alt_fmt in long_form_multiple_formats(
notebook.metadata.get("jupytext", {}).get("formats")
):
for alt_fmt in long_form_multiple_formats(formats):
if alt_fmt["extension"] == fmt["extension"] and fmt.get(
"format_name"
) == alt_fmt.get("format_name"):
Expand All @@ -942,10 +934,8 @@ class InconsistentVersions(ValueError):
"""An error raised when two paired files in the git index contain inconsistent representations"""


def load_paired_notebook(notebook, fmt, nb_file, log, pre_commit_mode):
def load_paired_notebook(notebook, fmt, config, formats, nb_file, log, pre_commit_mode):
"""Update the notebook with the inputs and outputs of the most recent paired files"""
formats = notebook.metadata.get("jupytext", {}).get("formats")

if not formats:
raise NotAPairedNotebook(f"{shlex.quote(nb_file)} is not a paired notebook")

Expand All @@ -971,7 +961,7 @@ def read_one_file(path, fmt):
return notebook

log(f"[jupytext] Loading {shlex.quote(path)}")
return read(path, fmt=fmt)
return read(path, fmt=fmt, config=config)

if use_git_index_rather_than_timestamp:
# We raise an error if two representations of this notebook in the git index are inconsistent
Expand All @@ -989,8 +979,8 @@ def read_one_file(path, fmt):
with open(path0) as fp:
text0 = fp.read()
for alt_path, alt_fmt in nb_files_in_git_index[1:]:
nb = read(alt_path, fmt=alt_fmt)
alt_text = writes(nb, fmt=fmt0)
nb = read(alt_path, fmt=alt_fmt, config=config)
alt_text = writes(nb, fmt=fmt0, config=config)
if alt_text != text0:
diff = compare(alt_text, text0, alt_path, path0, return_diff=True)
raise InconsistentVersions(
Expand Down
17 changes: 12 additions & 5 deletions jupytext/combine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,19 @@ def combine_inputs_with_outputs(nb_source, nb_outputs, fmt=None):
ext = fmt.get("extension") or text_repr.get("extension")
format_name = fmt.get("format_name") or text_repr.get("format_name")

nb_metadata = restore_filtered_metadata(
nb_source.metadata,
nb_outputs.metadata,
nb_source.metadata.get("jupytext", {}).get("notebook_metadata_filter"),
_DEFAULT_NOTEBOOK_METADATA,
notebook_metadata_filter = nb_source.metadata.get("jupytext", {}).get(
"notebook_metadata_filter"
)
if notebook_metadata_filter == "-all":
nb_metadata = nb_outputs.metadata

else:
nb_metadata = restore_filtered_metadata(
nb_source.metadata,
nb_outputs.metadata,
notebook_metadata_filter,
_DEFAULT_NOTEBOOK_METADATA,
)

source_is_md_version_one = (
ext in [".md", ".markdown", ".Rmd"] and text_repr.get("format_version") == "1.0"
Expand Down
24 changes: 9 additions & 15 deletions jupytext/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
NOTEBOOK_EXTENSIONS,
long_form_multiple_formats,
long_form_one_format,
rearrange_jupytext_metadata,
)


Expand Down Expand Up @@ -351,16 +350,19 @@ def validate_jupytext_configuration_file(config_file, config_dict):
return config


def prepare_notebook_for_save(nbk, config, path):
"""Apply the given configuration to the notebook before saving it"""
def notebook_formats(nbk, config, path, fallback_on_current_fmt=True):
"""Return the list of formats for the current notebook"""
metadata = nbk.get("metadata")
rearrange_jupytext_metadata(metadata)
jupytext_metadata = metadata.setdefault("jupytext", {})
formats = jupytext_metadata.get("formats") or (
config.default_formats(path) if config else None
jupytext_metadata = metadata.get("jupytext", {})
formats = (
jupytext_metadata.get("formats")
or metadata.get("jupytext_formats")
or (config.default_formats(path) if config else None)
)

if not formats:
if not fallback_on_current_fmt:
return None
text_repr = jupytext_metadata.get("text_representation", {})
ext = os.path.splitext(path)[1]
fmt = {"extension": ext}
Expand All @@ -379,13 +381,5 @@ def prepare_notebook_for_save(nbk, config, path):
formats = [
preferred_format(f, config.preferred_jupytext_formats_save) for f in formats
]
config.set_default_format_options(jupytext_metadata)

# Don't keep the formats if they are equal to the default
if config and jupytext_metadata.get("formats") == config.default_formats(path):
jupytext_metadata.pop("formats", None)

if not jupytext_metadata:
metadata.pop("jupytext")

return formats
14 changes: 7 additions & 7 deletions jupytext/contentsmanager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
JupytextConfigurationError,
find_global_jupytext_configuration_file,
load_jupytext_configuration_file,
notebook_formats,
preferred_format,
prepare_notebook_for_save,
validate_jupytext_configuration_file,
)
from .formats import (
Expand Down Expand Up @@ -122,7 +122,7 @@ def save(self, model, path=""):
nbk = model["content"]
try:
config = self.get_config(path)
jupytext_formats = prepare_notebook_for_save(nbk, config, path)
jupytext_formats = notebook_formats(nbk, config, path)
self.update_paired_notebooks(path, jupytext_formats)

def save_one_file(path, fmt):
Expand Down Expand Up @@ -161,7 +161,7 @@ def save_one_file(path, fmt):
type="file",
format="text",
content=jupytext.writes(
nbformat.from_dict(model["content"]), fmt=fmt
nbformat.from_dict(model["content"]), fmt=fmt, config=config
),
)

Expand Down Expand Up @@ -204,14 +204,15 @@ def get(
else:
model = self.super.get(path, content, type="file", format=format)
model["type"] = "notebook"
config.set_default_format_options(fmt, read=True)
if content:
# We may need to update these keys, inherited from text files formats
# Cf. https://github.com/mwouts/jupytext/issues/659
model["format"] = "json"
model["mimetype"] = None
try:
model["content"] = jupytext.reads(model["content"], fmt=fmt)
model["content"] = jupytext.reads(
model["content"], fmt=fmt, config=config
)
except Exception as err:
self.log.error(
u"Error while reading file: %s %s", path, err, exc_info=True
Expand Down Expand Up @@ -289,11 +290,10 @@ def read_one_file(alt_path, alt_fmt):
)["content"]

self.log.info(u"Reading SOURCE from {}".format(alt_path))
config.set_default_format_options(alt_fmt, read=True)
text = self.super.get(
alt_path, content=True, type="file", format=format
)["content"]
return jupytext.reads(text, fmt=alt_fmt)
return jupytext.reads(text, fmt=alt_fmt, config=config)

inputs, outputs = latest_inputs_and_outputs(
path, fmt, formats, get_timestamp, contents_manager_mode=True
Expand Down
17 changes: 7 additions & 10 deletions jupytext/header.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,12 @@ def encoding_and_executable(notebook, metadata, ext):
return lines


def insert_jupytext_info_and_filter_metadata(metadata, ext, text_format):
def insert_jupytext_info_and_filter_metadata(metadata, fmt, text_format):
"""Update the notebook metadata to include Jupytext information, and filter
the notebook metadata according to the default or user filter"""
if insert_or_test_version_number():
metadata.setdefault("jupytext", {})["text_representation"] = {
"extension": ext,
"extension": fmt["extension"],
"format_name": text_format.format_name,
"format_version": text_format.current_version_number,
"jupytext_version": __version__,
Expand All @@ -79,17 +79,13 @@ def insert_jupytext_info_and_filter_metadata(metadata, ext, text_format):
if "jupytext" in metadata and not metadata["jupytext"]:
del metadata["jupytext"]

notebook_metadata_filter = metadata.get("jupytext", {}).get(
"notebook_metadata_filter"
)
notebook_metadata_filter = fmt.get("notebook_metadata_filter")
return filter_metadata(
metadata, notebook_metadata_filter, _DEFAULT_NOTEBOOK_METADATA
)


def metadata_and_cell_to_header(
notebook, metadata, text_format, ext, root_level_metadata_as_raw_cell=True
):
def metadata_and_cell_to_header(notebook, metadata, text_format, fmt):
"""
Return the text header corresponding to a notebook, and remove the
first cell of the notebook if it contained the header
Expand All @@ -99,6 +95,7 @@ def metadata_and_cell_to_header(

lines_to_next_cell = None
root_level_metadata = {}
root_level_metadata_as_raw_cell = fmt.get("root_level_metadata_as_raw_cell", True)

if not root_level_metadata_as_raw_cell:
root_level_metadata = metadata.get("jupytext", {}).pop(
Expand All @@ -117,7 +114,7 @@ def metadata_and_cell_to_header(
lines_to_next_cell = cell.metadata.get("lines_to_next_cell")
notebook.cells = notebook.cells[1:]

metadata = insert_jupytext_info_and_filter_metadata(metadata, ext, text_format)
metadata = insert_jupytext_info_and_filter_metadata(metadata, fmt, text_format)

if metadata:
root_level_metadata["jupyter"] = metadata
Expand All @@ -131,7 +128,7 @@ def metadata_and_cell_to_header(
header = ["---"] + header + ["---"]

if (
metadata.get("jupytext", {}).get("hide_notebook_metadata", False)
fmt.get("hide_notebook_metadata", False)
and text_format.format_name == "markdown"
):
header = ["<!--", ""] + header + ["", "-->"]
Expand Down
Loading