Refactor and fix assert_expected_matched_actual (#65)

* Refactor and fix assert_expected_matched_actual

This PR:

- Refactors assert_expected_matched_actual function to avoid repeated
  matching between expected and actual output
- Fixes #63, #64

* Reorder imports

* Add test file

* Add back  leading or trainling ... in case of matching input

* Fix and group leading / trailing ... logic

* Update test-expect-fail.yaml file

* Update tests

* Express error_message prefix in terms of actual and expected

* Drop data test and add unit tests

* Add return type annotation

* Make exact match on the error_message

* Reformat utils.py

pytest_mypy_plugins/tests/test_utils.py

@@ -1,5 +1,21 @@
 # encoding=utf-8
+from typing import List, NamedTuple
+
+import pytest
+
 from pytest_mypy_plugins import utils
+from pytest_mypy_plugins.utils import (
+    OutputMatcher,
+    TypecheckAssertionError,
+    assert_expected_matched_actual,
+    extract_output_matchers_from_comments,
+)
+
+
+class ExpectMatchedActualTestData(NamedTuple):
+    source_lines: List[str]
+    actual_lines: List[str]
+    expected_message_lines: List[str]
 
 
 def test_render_template_with_None_value() -> None:
@@ -12,3 +28,159 @@ def test_render_template_with_None_value() -> None:
 
     # Then
     assert actual == "None 99"
+
+
+expect_matched_actual_data = [
+    ExpectMatchedActualTestData(
+        [
+            '''reveal_type(42)  # N: Revealed type is "Literal['foo']?"''',
+            '''reveal_type("foo")  # N: Revealed type is "Literal[42]?"''',
+        ],
+        ['''main:1: note: Revealed type is "Literal[42]?"''', '''main:2: note: Revealed type is "Literal['foo']?"'''],
+        [
+            """Invalid output: """,
+            """Actual:""",
+            """  main:1: note: Revealed type is "Literal[42]?" (diff)""",
+            """  main:2: note: Revealed type is "Literal['foo']?" (diff)""",
+            """Expected:""",
+            """  main:1: note: Revealed type is "Literal['foo']?" (diff)""",
+            """  main:2: note: Revealed type is "Literal[42]?" (diff)""",
+            """Alignment of first line difference:""",
+            '''  E: ...ed type is "Literal['foo']?"''',
+            '''  A: ...ed type is "Literal[42]?"''',
+            """                            ^""",
+        ],
+    ),
+    ExpectMatchedActualTestData(
+        [
+            """reveal_type(42)""",
+            '''reveal_type("foo")  # N: Revealed type is "Literal['foo']?"''',
+        ],
+        ['''main:1: note: Revealed type is "Literal[42]?"''', '''main:2: note: Revealed type is "Literal['foo']?"'''],
+        [
+            """Invalid output: """,
+            """Actual:""",
+            """  main:1: note: Revealed type is "Literal[42]?" (diff)""",
+            """  main:2: note: Revealed type is "Literal['foo']?" (diff)""",
+            """Expected:""",
+            """  main:2: note: Revealed type is "Literal['foo']?" (diff)""",
+            """Alignment of first line difference:""",
+            '''  E: main:2: note: Revealed type is "Literal['foo']?"''',
+            '''  A: main:1: note: Revealed type is "Literal[42]?"''',
+            """          ^""",
+        ],
+    ),
+    ExpectMatchedActualTestData(
+        ['''reveal_type(42)  # N: Revealed type is "Literal[42]?"''', """reveal_type("foo")"""],
+        ['''main:1: note: Revealed type is "Literal[42]?"''', '''main:2: note: Revealed type is "Literal['foo']?"'''],
+        [
+            """Invalid output: """,
+            """Actual:""",
+            """  main:2: note: Revealed type is "Literal['foo']?" (diff)""",
+            """Expected:""",
+            """  (empty)""",
+        ],
+    ),
+    ExpectMatchedActualTestData(
+        ['''42 + "foo"'''],
+        ["""main:1: error: Unsupported operand types for + ("int" and "str")"""],
+        [
+            """Output is not expected: """,
+            """Actual:""",
+            """  main:1: error: Unsupported operand types for + ("int" and "str") (diff)""",
+            """Expected:""",
+            """  (empty)""",
+        ],
+    ),
+    ExpectMatchedActualTestData(
+        [""" 1 + 1  # E: Unsupported operand types for + ("int" and "int")"""],
+        [],
+        [
+            """Invalid output: """,
+            """Actual:""",
+            """  (empty)""",
+            """Expected:""",
+            """  main:1: error: Unsupported operand types for + ("int" and "int") (diff)""",
+        ],
+    ),
+    ExpectMatchedActualTestData(
+        [
+            '''reveal_type(42)  # N: Revealed type is "Literal[42]?"''',
+            '''reveal_type("foo")  # N: Revealed type is "builtins.int"''',
+        ],
+        ['''main:1: note: Revealed type is "Literal[42]?"''', '''main:2: note: Revealed type is "Literal['foo']?"'''],
+        [
+            """Invalid output: """,
+            """Actual:""",
+            """  ...""",
+            """  main:2: note: Revealed type is "Literal['foo']?" (diff)""",
+            """Expected:""",
+            """  ...""",
+            """  main:2: note: Revealed type is "builtins.int" (diff)""",
+            """Alignment of first line difference:""",
+            '''  E: ...te: Revealed type is "builtins.int"''',
+            '''  A: ...te: Revealed type is "Literal['foo']?"''',
+            """                              ^""",
+        ],
+    ),
+    ExpectMatchedActualTestData(
+        [
+            '''reveal_type(42)  # N: Revealed type is "Literal[42]?"''',
+            '''reveal_type("foo")  # N: Revealed type is "builtins.int"''',
+        ],
+        ['''main:1: note: Revealed type is "Literal[42]?"''', '''main:2: note: Revealed type is "Literal['foo']?"'''],
+        [
+            """Invalid output: """,
+            """Actual:""",
+            """  ...""",
+            """  main:2: note: Revealed type is "Literal['foo']?" (diff)""",
+            """Expected:""",
+            """  ...""",
+            """  main:2: note: Revealed type is "builtins.int" (diff)""",
+            """Alignment of first line difference:""",
+            '''  E: ...te: Revealed type is "builtins.int"''',
+            '''  A: ...te: Revealed type is "Literal['foo']?"''',
+            """                              ^""",
+        ],
+    ),
+    ExpectMatchedActualTestData(
+        [
+            '''reveal_type(42.0)  # N: Revealed type is "builtins.float"''',
+            '''reveal_type("foo")  # N: Revealed type is "builtins.int"''',
+            '''reveal_type(42)  # N: Revealed type is "Literal[42]?"''',
+        ],
+        [
+            '''main:1: note: Revealed type is "builtins.float"''',
+            '''main:2: note: Revealed type is "Literal['foo']?"''',
+            '''main:3: note: Revealed type is "Literal[42]?"''',
+        ],
+        [
+            """Invalid output: """,
+            """Actual:""",
+            """  ...""",
+            """  main:2: note: Revealed type is "Literal['foo']?" (diff)""",
+            """  ...""",
+            """Expected:""",
+            """  ...""",
+            """  main:2: note: Revealed type is "builtins.int" (diff)""",
+            """  ...""",
+            """Alignment of first line difference:""",
+            '''  E: ...te: Revealed type is "builtins.int"''',
+            '''  A: ...te: Revealed type is "Literal['foo']?"''',
+            """                              ^""",
+        ],
+    ),
+]
+
+
+@pytest.mark.parametrize("source_lines,actual_lines,expected_message_lines", expect_matched_actual_data)
+def test_assert_expected_matched_actual_failures(
+    source_lines: List[str], actual_lines: List[str], expected_message_lines: List[str]
+) -> None:
+    expected: List[OutputMatcher] = extract_output_matchers_from_comments("main", source_lines, False)
+    expected_error_message = "\n".join(expected_message_lines)
+
+    with pytest.raises(TypecheckAssertionError) as e:
+        assert_expected_matched_actual(expected, actual_lines)
+
+    assert e.value.error_message.strip() == expected_error_message.strip()

pytest_mypy_plugins/utils.py

@@ -7,10 +7,12 @@
 import re
 import sys
 from dataclasses import dataclass
+from itertools import zip_longest
 from pathlib import Path
 from typing import (
     Any,
     Callable,
+    Dict,
     Iterator,
     List,
     Mapping,
@@ -129,20 +131,6 @@ def remove_common_prefix(lines: List[str]) -> List[str]:
     return cleaned_lines
 
 
-def _num_skipped_prefix_lines(a1: List[OutputMatcher], a2: List[str]) -> int:
-    num_eq = 0
-    while num_eq < min(len(a1), len(a2)) and a1[num_eq].matches(a2[num_eq]):
-        num_eq += 1
-    return max(0, num_eq - 4)
-
-
-def _num_skipped_suffix_lines(a1: List[OutputMatcher], a2: List[str]) -> int:
-    num_eq = 0
-    while num_eq < min(len(a1), len(a2)) and a1[-num_eq - 1].matches(a2[-num_eq - 1]):
-        num_eq += 1
-    return max(0, num_eq - 4)
-
-
 def _add_aligned_message(s1: str, s2: str, error_message: str) -> str:
     """Align s1 and s2 so that the their first difference is highlighted.
 
@@ -224,78 +212,78 @@ def assert_expected_matched_actual(expected: List[OutputMatcher], actual: List[s
 
     Display any differences in a human-readable form.
     """
+
+    def format_mismatched_line(line: str) -> str:
+        return "  {:<45} (diff)".format(str(line))
+
+    def format_matched_line(line: str, width: int = 100) -> str:
+        return " {}...".format(line[:width]) if len(line) > width else " {}".format(line)
+
+    def format_error_lines(lines: List[str]) -> str:
+        return "\n".join(lines) if lines else "  (empty)"
+
     expected = sorted(expected, key=lambda om: (om.fname, om.lnum))
     actual = sorted_by_file_and_line(remove_empty_lines(actual))
 
     actual = remove_common_prefix(actual)
-    error_message = ""
 
-    if not all(e.matches(a) for e, a in zip(expected, actual)):
-        num_skip_start = _num_skipped_prefix_lines(expected, actual)
-        num_skip_end = _num_skipped_suffix_lines(expected, actual)
+    diff_lines: Dict[int, Tuple[OutputMatcher, str]] = {
+        i: (e, a)
+        for i, (e, a) in enumerate(zip_longest(expected, actual))
+        if e is None or a is None or not e.matches(a)
+    }
 
-        error_message += "Expected:\n"
+    if diff_lines:
+        first_diff_line = min(diff_lines.keys())
+        last_diff_line = max(diff_lines.keys())
 
-        # If omit some lines at the beginning, indicate it by displaying a line
-        # with '...'.
-        if num_skip_start > 0:
-            error_message += "  ...\n"
+        expected_message_lines = []
+        actual_message_lines = []
 
-        # Keep track of the first different line.
-        first_diff = -1
+        for i in range(first_diff_line, last_diff_line + 1):
+            if i in diff_lines:
+                expected_line, actual_line = diff_lines[i]
+                if expected_line:
+                    expected_message_lines.append(format_mismatched_line(str(expected_line)))
+                if actual_line:
+                    actual_message_lines.append(format_mismatched_line(actual_line))
 
-        # Display only this many first characters of identical lines.
-        width = 100
-
-        for i in range(num_skip_start, len(expected) - num_skip_end):
-            if i >= len(actual) or not expected[i].matches(actual[i]):
-                if first_diff < 0:
-                    first_diff = i
-                error_message += "  {:<45} (diff)".format(expected[i])
-            else:
-                e = expected[i]
-                error_message += "  " + str(e)[:width]
-                if len(e) > width:
-                    error_message += "..."
-            error_message += "\n"
-        if num_skip_end > 0:
-            error_message += "  ...\n"
-
-        error_message += "Actual:\n"
-
-        if num_skip_start > 0:
-            error_message += "  ...\n"
-
-        for j in range(num_skip_start, len(actual) - num_skip_end):
-            if j >= len(expected) or not expected[j].matches(actual[j]):
-                error_message += "  {:<45} (diff)".format(actual[j])
             else:
-                a = actual[j]
-                error_message += "  " + a[:width]
-                if len(a) > width:
-                    error_message += "..."
-            error_message += "\n"
-        if not actual:
-            error_message += "  (empty)\n"
-        if num_skip_end > 0:
-            error_message += "  ...\n"
-
-        error_message += "\n"
-
-        if 0 <= first_diff < len(actual) and (
-            len(expected[first_diff]) >= MIN_LINE_LENGTH_FOR_ALIGNMENT
-            or len(actual[first_diff]) >= MIN_LINE_LENGTH_FOR_ALIGNMENT
-        ):
-            # Display message that helps visualize the differences between two
-            # long lines.
-            error_message = _add_aligned_message(str(expected[first_diff]), actual[first_diff], error_message)
+                expected_line, actual_line = expected[i], actual[i]
+                actual_message_lines.append(format_matched_line(actual_line))
+                expected_message_lines.append(format_matched_line(str(expected_line)))
+
+        first_diff_expected, first_diff_actual = diff_lines[first_diff_line]
+
+        failure_reason = "Output is not expected" if actual and not expected else "Invalid output"
+
+        if actual_message_lines and expected_message_lines:
+            if first_diff_line > 0:
+                expected_message_lines.insert(0, "  ...")
+                actual_message_lines.insert(0, "  ...")
+
+            if last_diff_line < len(actual) - 1 and last_diff_line < len(expected) - 1:
+                expected_message_lines.append("  ...")
+                actual_message_lines.append("  ...")
 
-        if len(expected) == 0:
-            raise TypecheckAssertionError(f"Output is not expected: \n{error_message}")
+        error_message = "Actual:\n{}\nExpected:\n{}\n".format(
+            format_error_lines(actual_message_lines), format_error_lines(expected_message_lines)
+        )
 
-        first_failure = expected[first_diff]
-        if first_failure:
-            raise TypecheckAssertionError(error_message=f"Invalid output: \n{error_message}", lineno=first_failure.lnum)
+        if (
+            first_diff_actual is not None
+            and first_diff_expected is not None
+            and (
+                len(first_diff_actual) >= MIN_LINE_LENGTH_FOR_ALIGNMENT
+                or len(str(first_diff_expected)) >= MIN_LINE_LENGTH_FOR_ALIGNMENT
+            )
+        ):
+            error_message = _add_aligned_message(str(first_diff_expected), first_diff_actual, error_message)
+
+        raise TypecheckAssertionError(
+            error_message=f"{failure_reason}: \n{error_message}",
+            lineno=first_diff_expected.lnum if first_diff_expected else 0,
+        )
 
 
 def extract_output_matchers_from_comments(fname: str, input_lines: List[str], regex: bool) -> List[OutputMatcher]: