From 754e615fc1d818ac7064887de43daf3a46dc4bed Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Thu, 24 Feb 2022 09:54:17 +0200 Subject: [PATCH] Emit chunks of lines instead of individual lines This should allow us to get smaller diff chunks from Black than by diffing and analyzing Black's string output for whole files. --- src/darker/black_diff.py | 6 ++-- src/darker/linewise_black.py | 47 ++++++++++++++------------- src/darker/tests/test_black_diff.py | 6 ++-- src/darker/tests/test_command_line.py | 8 ++--- 4 files changed, 35 insertions(+), 32 deletions(-) diff --git a/src/darker/black_diff.py b/src/darker/black_diff.py index 7edba9332..51a7f44d4 100644 --- a/src/darker/black_diff.py +++ b/src/darker/black_diff.py @@ -49,8 +49,8 @@ from black.const import DEFAULT_EXCLUDES, DEFAULT_INCLUDES from black.files import gen_python_files from black.report import Report -from darker.linewise_black import format_str_to_lines +from darker.linewise_black import format_str_to_chunks from darker.utils import TextDocument if sys.version_info >= (3, 8): @@ -181,8 +181,8 @@ def run_black(src_contents: TextDocument, black_config: BlackConfig) -> TextDocu # https://github.com/psf/black/pull/2484 lands in Black. contents_for_black = src_contents.string_with_newline("\n") if contents_for_black.strip(): - dst_lines = format_str_to_lines(contents_for_black, mode=Mode(**mode)) - dst_contents = "".join(dst_lines) + dst_chunks = format_str_to_chunks(contents_for_black, mode=Mode(**mode)) + dst_contents = "".join(line for chunk in dst_chunks for line in chunk) else: dst_contents = "\n" if "\n" in src_contents.string else "" return TextDocument.from_str( diff --git a/src/darker/linewise_black.py b/src/darker/linewise_black.py index a0fd8b4fc..10afb3408 100644 --- a/src/darker/linewise_black.py +++ b/src/darker/linewise_black.py @@ -1,22 +1,23 @@ """Re-implementation of :func:`black.format_str` as a line generator""" -from typing import Generator -from black import get_future_imports, detect_target_versions, decode_bytes -from black.lines import Line, EmptyLineTracker -from black.linegen import transform_line, LineGenerator +from typing import Generator, List + +from black import decode_bytes, detect_target_versions, get_future_imports from black.comments import normalize_fmt_off -from black.mode import Mode -from black.mode import Feature, supports_feature +from black.linegen import LineGenerator, transform_line +from black.lines import EmptyLineTracker, Line +from black.mode import Feature, Mode, supports_feature from black.parsing import lib2to3_parse -def format_str_to_lines( +def format_str_to_chunks( # pylint: disable=too-many-locals src_contents: str, *, mode: Mode -) -> Generator[str, None, None]: # pylint: disable=too-many-locals +) -> Generator[List[str], None, None]: """Reformat a string and yield each line of new contents This is a re-implementation of :func:`black.format_str` modified to be a generator - which yields each resulting line instead of concatenating them into a single string. + which yields each resulting chunk as a list of lines instead of concatenating them + into a single string. """ src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions) @@ -42,20 +43,22 @@ def format_str_to_lines( } num_chars = 0 for current_line in lines.visit(src_node): - for _ in range(after): - yield empty_line - num_chars += after * empty_line_len + if after: + yield after * [empty_line] + num_chars += after * empty_line_len before, after = elt.maybe_empty_lines(current_line) - for _ in range(before): - yield empty_line - num_chars += before * empty_line_len - for line in transform_line( - current_line, mode=mode, features=split_line_features - ): - line_str = str(line) - yield line_str - num_chars += len(line_str) + if before: + yield before * [empty_line] + num_chars += before * empty_line_len + lines = [ + str(line) + for line in transform_line( + current_line, mode=mode, features=split_line_features + ) + ] + yield lines + num_chars += sum(len(line) for line in lines) if not num_chars: normalized_content, _, newline = decode_bytes(src_contents.encode("utf-8")) if "\n" in normalized_content: - yield newline + yield [newline] diff --git a/src/darker/tests/test_black_diff.py b/src/darker/tests/test_black_diff.py index f06909bb2..8738bc9c6 100644 --- a/src/darker/tests/test_black_diff.py +++ b/src/darker/tests/test_black_diff.py @@ -196,12 +196,12 @@ def test_run_black(encoding, newline): def test_run_black_always_uses_unix_newlines(newline): """Content is always passed to Black with Unix newlines""" src = TextDocument.from_str(f"print ( 'touché' ){newline}") - with patch.object(black_diff, "format_str_to_lines") as format_str_to_lines: - format_str_to_lines.return_value = ['print("touché")\n'] + with patch.object(black_diff, "format_str_to_chunks") as format_str_to_chunks: + format_str_to_chunks.return_value = [['print("touché")\n']] _ = run_black(src, BlackConfig()) - format_str_to_lines.assert_called_once_with("print ( 'touché' )\n", mode=ANY) + format_str_to_chunks.assert_called_once_with("print ( 'touché' )\n", mode=ANY) def test_run_black_ignores_excludes(): diff --git a/src/darker/tests/test_command_line.py b/src/darker/tests/test_command_line.py index c177f1865..7d6097220 100644 --- a/src/darker/tests/test_command_line.py +++ b/src/darker/tests/test_command_line.py @@ -470,9 +470,9 @@ def test_black_options_skip_string_normalization(git_repo, config, options, expe added_files["main.py"].write_bytes(b"bar") mode_class_mock = Mock(wraps=black_diff.Mode) # Speed up tests by mocking `format_str` to skip running Black - format_str_to_lines = Mock(return_value=["bar"]) + format_str_to_chunks = Mock(return_value=[["bar"]]) with patch.multiple( - black_diff, Mode=mode_class_mock, format_str_to_lines=format_str_to_lines + black_diff, Mode=mode_class_mock, format_str_to_chunks=format_str_to_chunks ): main(options + [str(path) for path in added_files.values()]) @@ -498,9 +498,9 @@ def test_black_options_skip_magic_trailing_comma(git_repo, config, options, expe added_files["main.py"].write_bytes(b"a = [1, 2,]") mode_class_mock = Mock(wraps=black_diff.Mode) # Speed up tests by mocking `format_str` to skip running Black - format_str_to_lines = Mock(return_value=["a = [1, 2,]"]) + format_str_to_chunks = Mock(return_value=[["a = [1, 2,]"]]) with patch.multiple( - black_diff, Mode=mode_class_mock, format_str_to_lines=format_str_to_lines + black_diff, Mode=mode_class_mock, format_str_to_chunks=format_str_to_chunks ): main(options + [str(path) for path in added_files.values()])