Add syntax to compare just mypy error codes and not full messages

Author: David Fritzsche

README.md

@@ -64,6 +64,7 @@ the file:
 * `# N: <msg>` - we expect a mypy note message
 * `# W: <msg>` - we expect a mypy warning message
 * `# E: <msg>` - we expect a mypy error message
+* `# F: <msg>` - we expect a mypy fatal error message
 * `# R: <msg>` - we expect a mypy note message `Revealed type is
   '<msg>'`. This is useful to easily check `reveal_type` output:
      ```python
@@ -73,13 +74,15 @@ the file:
          reveal_type(456)  # R: Literal[456]?
      ```
 
-## mypy Error Codes
+## mypy Error Code Matching
 
 The algorithm matching messages parses mypy error code both in the
-output generated by mypy and in the Python comments. If both the mypy
-output and the Python comment contain an error code, then the codes
-must match. So the following test case expects that mypy writes out an
-``assignment`` error code:
+output generated by mypy and in the Python comments.
+
+If both the mypy output and the Python comment contain an error code
+and a full message, then the messages and the error codes must
+match. The following test case expects that mypy writes out an
+``assignment`` error code and a specific error message:
 
 ``` python
 @pytest.mark.mypy_testing
@@ -89,7 +92,27 @@ def mypy_test_invalid_assignment() -> None:
 ```
 
 If the Python comment does not contain an error code, then the error
-code written out by mypy (if any) is simply ignored.
+code written out by mypy (if any) is ignored. The following test case
+expects a specific error message from mypy, but ignores the error code
+produced by mypy:
+
+``` python
+@pytest.mark.mypy_testing
+def mypy_test_invalid_assignment() -> None:
+    foo = "abc"
+    foo = 123  # E: Incompatible types in assignment (expression has type "int", variable has type "str")
+```
+
+If the Python comment specifies only an error code, then the message
+written out by mypy is ignored, i.e., the following test case checks
+that mypy reports an `assignment` error:
+
+``` python
+@pytest.mark.mypy_testing
+def mypy_test_invalid_assignment() -> None:
+    foo = "abc"
+    foo = 123  # E: [assignment]
+```
 
 
 ## Skipping and Expected Failures
@@ -114,9 +137,15 @@ decorators are extracted from the ast.
 
 # Changelog
 
+## v0.1.1
+
+* Compare just mypy error codes if given and no error message is given
+  in the test case Python comment ([#36][i36], [#41][p43])
+
 ## v0.1.0
 
-* Implement support for flexible matching of mypy error codes (towards [#36][i36], [#41][p41])
+* Implement support for flexible matching of mypy error codes (towards
+  [#36][i36], [#41][p41])
 * Add support for pytest 7.2.x ([#42][p42])
 * Add support for mypy 1.0.x ([#42][p42])
 * Add support for Python 3.11 ([#42][p42])
@@ -192,3 +221,4 @@ decorators are extracted from the ast.
 [p40]: https://github.com/davidfritzsche/pytest-mypy-testing/pull/40
 [p41]: https://github.com/davidfritzsche/pytest-mypy-testing/pull/41
 [p42]: https://github.com/davidfritzsche/pytest-mypy-testing/pull/42
+[p43]: https://github.com/davidfritzsche/pytest-mypy-testing/pull/43

pytest.ini

@@ -2,9 +2,12 @@
 # SPDX-License-Identifier: CC0-1.0
 [pytest]
 testpaths =
-    tests mypy_tests
+    tests mypy_tests pytest_mypy_testing
 addopts =
+    --doctest-continue-on-failure
+    --doctest-modules
     --failed-first
+    --pyargs
     --showlocals
     -p no:mypy-testing
 doctest_optionflags = NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL ELLIPSIS

src/pytest_mypy_testing/message.py

@@ -22,6 +22,7 @@ class Severity(enum.Enum):
     NOTE = 1
     WARNING = 2
     ERROR = 3
+    FATAL = 4
 
     @classmethod
     def from_string(cls, string: str) -> "Severity":
@@ -39,6 +40,7 @@ def __repr__(self) -> str:
     "N": Severity.NOTE,
     "W": Severity.WARNING,
     "E": Severity.ERROR,
+    "F": Severity.FATAL,
 }
 
 _COMMENT_MESSAGES = frozenset(
@@ -55,11 +57,11 @@ def __repr__(self) -> str:
 class Message:
     """Mypy message"""
 
-    filename: str
-    lineno: int
-    colno: Optional[int]
-    severity: Severity
-    message: str
+    filename: str = ""
+    lineno: int = 0
+    colno: Optional[int] = None
+    severity: Severity = Severity.ERROR
+    message: str = ""
     revealed_type: Optional[str] = None
     error_code: Optional[str] = None
 
@@ -72,19 +74,22 @@ class Message:
     COMMENT_RE = re.compile(
         r"^(?:# *type: *ignore *)?(?:# *)?"
         r"(?P<severity>[RENW]):"
-        r"((?P<colno>\d+):)? *"
-        r"(?P<message>[^#]*?)"
-        r"(?: +\[(?P<error_code>[^\]]*)\])?"
+        r"((?P<colno>\d+):)?"
+        r" *"
+        r"(?P<message_and_error_code>[^#]*)"
         r"(?:#.*?)?$"
     )
 
+    MESSAGE_AND_ERROR_CODE = re.compile(
+        r"(?P<message>[^\[][^#]*?)" r" +" r"\[(?P<error_code>[^\]]*)\]"
+    )
+
     OUTPUT_RE = re.compile(
         r"^(?P<fname>([a-zA-Z]:)?[^:]+):"
         r"(?P<lineno>[0-9]+):"
         r"((?P<colno>[0-9]+):)?"
         r" *(?P<severity>(error|note|warning)):"
-        r"(?P<message>.*?)"
-        r"(?: +\[(?P<error_code>[^\]]*)\])?"
+        r"(?P<message_and_error_code>.*?)"
         r"$"
     )
 
@@ -128,7 +133,7 @@ def astuple(self, *, normalized: bool = False) -> "Message.TupleType":
 
         >>> m = Message("foo.py", 1, 1, Severity.NOTE, 'Revealed type is "float"')
         >>> m.astuple()
-        ('foo.py', 1, 1, Severity.NOTE, 'Revealed type is "float"', 'float')
+        ('foo.py', 1, 1, Severity.NOTE, 'Revealed type is "float"', 'float', None)
         """
         return (
             self.filename,
@@ -144,7 +149,11 @@ def is_comment(self) -> bool:
         return (self.severity, self.message) in _COMMENT_MESSAGES
 
     def _as_short_tuple(
-        self, *, normalized: bool = False, default_error_code: Optional[str] = None
+        self,
+        *,
+        normalized: bool = False,
+        default_message: str = "",
+        default_error_code: Optional[str] = None,
     ) -> "Message.TupleType":
         if normalized:
             message = self.normalized_message
@@ -155,32 +164,73 @@ def _as_short_tuple(
             self.lineno,
             None,
             self.severity,
-            message,
+            message or default_message,
             self.revealed_type,
             self.error_code or default_error_code,
         )
 
+    def __hash__(self) -> int:
+        t = (self.filename, self.lineno, self.severity, self.revealed_type)
+        return hash(t)
+
     def __eq__(self, other):
+        """Compare if *self* and *other* are equal.
+
+        Returns `True` if *other* is a :obj:`Message:` object
+        considered to be equal to *self*.
+
+        >>> Message() == Message()
+        True
+        >>> Message(error_code="baz") == Message(message="some text", error_code="baz")
+        True
+        >>> Message(message="some text") == Message(message="some text", error_code="baz")
+        True
+
+        >>> Message() == Message(message="some text", error_code="baz")
+        False
+        >>> Message(error_code="baz") == Message(error_code="bax")
+        False
+        """
         if isinstance(other, Message):
             default_error_code = self.error_code or other.error_code
-            if self.colno is None or other.colno is None:
-                a = self._as_short_tuple(
-                    normalized=True, default_error_code=default_error_code
-                )
-                b = other._as_short_tuple(
-                    normalized=True, default_error_code=default_error_code
+            if self.error_code and other.error_code:
+                default_message = self.normalized_message or other.normalized_message
+            else:
+                default_message = ""
+
+            def to_tuple(m: Message):
+                return m._as_short_tuple(
+                    normalized=True,
+                    default_message=default_message,
+                    default_error_code=default_error_code,
                 )
-                return a == b
+
+            if self.colno is None or other.colno is None:
+                return to_tuple(self) == to_tuple(other)
             else:
                 return self.astuple(normalized=True) == other.astuple(normalized=True)
         else:
             return NotImplemented
 
-    def __hash__(self) -> int:
-        return hash(self._as_short_tuple(normalized=True))
-
     def __str__(self) -> str:
-        return f"{self._prefix} {self.severity.name.lower()}: {self.message}"
+        return self.to_string(prefix=f"{self._prefix} ")
+
+    def to_string(self, prefix: Optional[str] = None) -> str:
+        prefix = prefix or f"{self._prefix} "
+        error_code = f"  [{self.error_code}]" if self.error_code else ""
+        return f"{prefix}{self.severity.name.lower()}: {self.message}{error_code}"
+
+    @classmethod
+    def __split_message_and_error_code(cls, msg: str) -> Tuple[str, Optional[str]]:
+        msg = msg.strip()
+        if msg.startswith("[") and msg.endswith("]"):
+            return "", msg[1:-1]
+        else:
+            m = cls.MESSAGE_AND_ERROR_CODE.fullmatch(msg)
+            if m:
+                return m.group("message"), m.group("error_code")
+            else:
+                return msg, None
 
     @classmethod
     def from_comment(
@@ -189,13 +239,17 @@ def from_comment(
         """Create message object from Python *comment*.
 
         >>> Message.from_comment("foo.py", 1, "R: foo")
-        Message(filename='foo.py', lineno=1, colno=None, severity=Severity.NOTE, message="Revealed type is 'foo'", revealed_type='foo')
+        Message(filename='foo.py', lineno=1, colno=None, severity=Severity.NOTE, message="Revealed type is 'foo'", revealed_type='foo', error_code=None)
+        >>> Message.from_comment("foo.py", 1, "E: [assignment]")
+        Message(filename='foo.py', lineno=1, colno=None, severity=Severity.ERROR, message='', revealed_type=None, error_code='assignment')
         """
         m = cls.COMMENT_RE.match(comment.strip())
         if not m:
             raise ValueError("Not a valid mypy message comment")
         colno = int(m.group("colno")) if m.group("colno") else None
-        message = m.group("message").strip()
+        message, error_code = cls.__split_message_and_error_code(
+            m.group("message_and_error_code")
+        )
         if m.group("severity") == "R":
             revealed_type = message
             message = "Revealed type is {!r}".format(message)
@@ -208,37 +262,45 @@ def from_comment(
             severity=Severity.from_string(m.group("severity")),
             message=message,
             revealed_type=revealed_type,
-            error_code=m.group("error_code") or None,
+            error_code=error_code,
         )
 
     @classmethod
     def from_output(cls, line: str) -> "Message":
         """Create message object from mypy output line.
 
         >>> m = Message.from_output("z.py:1: note: bar")
-        >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type)
-        (1, None, Severity.NOTE, 'bar', None)
+        >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type, m.error_code)
+        (1, None, Severity.NOTE, 'bar', None, None)
 
         >>> m = Message.from_output("z.py:1:13: note: bar")
-        >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type)
-        (1, 13, Severity.NOTE, 'bar', None)
+        >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type, m.error_code)
+        (1, 13, Severity.NOTE, 'bar', None, None)
 
         >>> m = Message.from_output("z.py:1: note: Revealed type is 'bar'")
-        >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type)
-        (1, None, Severity.NOTE, "Revealed type is 'bar'", 'bar')
+        >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type, m.error_code)
+        (1, None, Severity.NOTE, "Revealed type is 'bar'", 'bar', None)
 
         >>> m = Message.from_output('z.py:1: note: Revealed type is "bar"')
-        >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type)
-        (1, None, Severity.NOTE, 'Revealed type is "bar"', 'bar')
+        >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type, m.error_code)
+        (1, None, Severity.NOTE, 'Revealed type is "bar"', 'bar', None)
+
+        >>> m = Message.from_output("z.py:1:13: error: bar [baz]")
+        >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type, m.error_code)
+        (1, 13, Severity.ERROR, 'bar', None, 'baz')
 
         """
         m = cls.OUTPUT_RE.match(line)
         if not m:
             raise ValueError("Not a valid mypy message")
+        message, error_code = cls.__split_message_and_error_code(
+            m.group("message_and_error_code")
+        )
         return cls(
             os.path.abspath(m.group("fname")),
             lineno=int(m.group("lineno")),
             colno=int(m.group("colno")) if m.group("colno") else None,
             severity=Severity[m.group("severity").upper()],
-            message=m.group("message").strip(),
+            message=message,
+            error_code=error_code,
         )

src/pytest_mypy_testing/output_processing.py

@@ -46,9 +46,7 @@ def __post_init__(self) -> None:
         def _fmt(msg: Message, actual_expected: str = "", *, indent: str = "  ") -> str:
             if actual_expected:
                 actual_expected += ": "
-            return (
-                f"{indent}{actual_expected}{msg.severity.name.lower()}: {msg.message}"
-            )
+            return msg.to_string(prefix=f"{indent}{actual_expected}")
 
         if not any([self.actual, self.expected]):
             raise ValueError("At least one of actual and expected must be given")

src/pytest_mypy_testing/plugin.py

@@ -172,6 +172,7 @@ def _run_mypy(self, filename: Union[pathlib.Path, os.PathLike, str]) -> MypyResu
                 "--no-silence-site-packages",
                 "--no-warn-unused-configs",
                 "--show-column-numbers",
+                "--show-error-codes",
                 "--show-traceback",
                 str(filename),
             ]

tests/test_basics.mypy-testing

@@ -11,6 +11,39 @@ def mypy_test_invalid_assginment():
     foo = 123  # E: Incompatible types in assignment (expression has type "int", variable has type "str")
 
 
+@pytest.mark.mypy_testing
+def mypy_test_invalid_assginment_with_error_code():
+    foo = "abc"
+    foo = 123  # E: Incompatible types in assignment (expression has type "int", variable has type "str") [assignment]
+
+
+@pytest.mark.xfail
+@pytest.mark.mypy_testing
+def mypy_test_invalid_assginment_with_error_code__message_does_not_match():
+    foo = "abc"
+    foo = 123  # E: Invalid assignment [assignment]
+
+
+@pytest.mark.mypy_testing
+def mypy_test_invalid_assginment_only_error_code():
+    foo = "abc"
+    foo = 123  # E: [assignment]
+
+
+@pytest.mark.xfail
+@pytest.mark.mypy_testing
+def mypy_test_invalid_assginment_only_error_code__error_code_does_not_match():
+    foo = "abc"
+    foo = 123  # E: [baz]
+
+
+@pytest.mark.xfail
+@pytest.mark.mypy_testing
+def mypy_test_invalid_assginment_no_message_and_no_error_code():
+    foo = "abc"
+    foo = 123  # E:
+
+
 @pytest.mark.mypy_testing
 def mypy_test_use_reveal_type():
     reveal_type(123)  # N: Revealed type is 'Literal[123]?'