Fix directive bug

CHANGELOG.md

@@ -8,6 +8,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 ### Fixed
 * Can now handle basic versions of the new `__editable__` package finder mechanism.
+* Parsing bug where directives were incorrectly flagged as inline if they were
+  directly followed by a function with a decorator.
 
 
 ### Removed

src/xdoctest/__init__.py

@@ -314,7 +314,7 @@ def fib(n):
 mkinit xdoctest --nomods
 '''
 
-__version__ = '1.0.2'
+__version__ = '1.1.0'
 
 
 # Expose only select submodules

src/xdoctest/directive.py

@@ -157,7 +157,6 @@
 from xdoctest import utils
 from collections import OrderedDict
 from collections import namedtuple
-# from xdoctest import exceptions
 
 
 def named(key, pattern):
@@ -463,7 +462,8 @@ def extract(cls, text):
             >>> any(Directive.extract(' # badprefix: not-a-directive'))
             False
         """
-        # Flag extracted directives as inline iff the text is only comments
+        # Flag extracted directives as inline iff the text contains non-comments
+        print(f'text={text}')
         inline = not all(line.strip().startswith('#')
                          for line in text.splitlines())
         #

src/xdoctest/doctest_example.py

@@ -687,18 +687,25 @@ def run(self, verbose=None, on_error=None):
 
         needs_capture = True
 
+        DEBUG = global_state.DEBUG_DOCTEST
+
         # Use the same capture object for all parts in the test
         cap = utils.CaptureStdout(suppress=self._suppressed_stdout,
                                   enabled=needs_capture)
         with warnings.catch_warnings(record=True) as self.warn_list:
             for partx, part in enumerate(self._parts):
 
+                if DEBUG:
+                    print(f'part[{partx}] checking')
+
                 # Prepare to capture stdout and evaluated values
                 self.failed_part = part  # Assume part will fail (it may not)
                 got_eval = constants.NOT_EVALED
 
                 # Extract directives and and update runtime state
                 part_directive = part.directives
+                if DEBUG:
+                    print(f'part[{partx}] directives: {part_directive}')
                 try:
                     try:
                         runstate.update(part_directive)
@@ -715,18 +722,29 @@ def run(self, verbose=None, on_error=None):
                         raise
                     break
 
+                if DEBUG:
+                    print(f'part[{partx}] runstate={runstate}')
+                    print(f'runstate._inline_state={runstate._inline_state}')
+                    print(f'runstate._global_state={runstate._global_state}')
+
                 # Handle runtime actions
                 if runstate['SKIP'] or len(runstate['REQUIRES']) > 0:
+                    if DEBUG:
+                        print(f'part[{partx}] runstate requests skipping')
                     self._skipped_parts.append(part)
                     continue
 
                 if not part.has_any_code():
+                    if DEBUG:
+                        print(f'part[{partx}] No code, skipping')
                     self._skipped_parts.append(part)
                     continue
 
                 if not did_pre_import:
                     # Execute the pre-import before the first run of
                     # non-skipped code.
+                    if DEBUG:
+                        print(f'part[{partx}] Importing parent module')
                     try:
                         self._import_module()
                     except Exception:

src/xdoctest/parser.py

@@ -269,6 +269,7 @@ def _package_chunk(self, raw_source_lines, raw_want_lines, lineno=0):
         if global_state.DEBUG_PARSER > 1:
             print('mode_hint = {!r}'.format(mode_hint))
             print(' * located ps1 lines')
+            print(f'ps1_linenos={ps1_linenos}')
 
         # Find all directives here:
         # A directive necessarily will split a doctest into multiple parts
@@ -290,6 +291,9 @@ def _package_chunk(self, raw_source_lines, raw_want_lines, lineno=0):
                     if s2 is not None:
                         break_linenos.append(s2)
 
+        if global_state.DEBUG_PARSER > 3:
+            print(f'break_linenos={break_linenos}')
+
         def slice_example(s1, s2, want_lines=None):
             exec_lines = exec_source_lines[s1:s2]
             orig_lines = source_lines[s1:s2]
@@ -386,6 +390,7 @@ def _group_labeled_lines(self, labeled_lines):
         if current:
             groups.append((state, current))
 
+        print(f'global_state.DEBUG_PARSER={global_state.DEBUG_PARSER}')
         if global_state.DEBUG_PARSER > 4:
             print('groups = {!r}'.format(groups))
 
@@ -491,6 +496,20 @@ def _locate_ps1_linenos(self, source_lines):
             >>> linenos, mode_hint = self._locate_ps1_linenos(source_lines)
             >>> assert linenos == [0]
             >>> assert mode_hint == 'single'
+
+        Example:
+            >>> # We should ensure that decorators are PS1 lines
+            >>> from xdoctest.parser import *  # NOQA
+            >>> self = DoctestParser()
+            >>> source_lines = [
+            >>>    '>>> # foo',
+            >>>    '>>> @foo',
+            >>>    '... def bar():',
+            >>>    '...     ...',
+            >>> ]
+            >>> linenos, mode_hint = self._locate_ps1_linenos(source_lines)
+            >>> print(f'linenos={linenos}')
+            >>> assert linenos == [0, 1]
         """
         # Strip indentation (and PS1 / PS2 from source)
         exec_source_lines = [p[4:] for p in source_lines]
@@ -555,6 +574,17 @@ def balanced_intervals(lines):
 
         statement_nodes = pt.body
         ps1_linenos = [node.lineno - 1 for node in statement_nodes]
+
+        if 1:
+            # Get PS1 line numbers of statements accounting for decorators
+            ps1_linenos = []
+            for node in statement_nodes:
+                if hasattr(node, 'decorator_list') and node.decorator_list:
+                    lineno = node.decorator_list[0].lineno - 1
+                else:
+                    lineno = node.lineno - 1
+                ps1_linenos.append(lineno)
+
         # NEED_16806_WORKAROUND = 1
         if NEED_16806_WORKAROUND:  # pragma: nobranch
             ps1_linenos = self._workaround_16806(

tests/test_cases.py

@@ -113,14 +113,20 @@ def test(self, s):
     assert 'running 0 test' in text
 
 
-def WIP_test_torch_dispatch_case():
+def test_correct_skipping_on_decorators():
     """
     This is a weird case similar to the torch dispatch doctest
 
     ~/code/pytorch/torch/fx/experimental/unification/multipledispatch/core.py
 
     Something about it causes the skip directive not to be applied to the
     entire thing. Not quite sure what's going on yet.
+
+    The issue was that decorator line numbers were returning as the line of the
+    function itself. This mean that the PS1 grouping put the directive in a
+    group with logic, which made the parser think it was inline, which meant
+    the skip state was cleared after it was executed, so it executed the bad
+    code. This fixes that.
     """
     import xdoctest
     from xdoctest import runner
@@ -182,13 +188,14 @@ def dispatch(*types, **kwargs):
     }
 
     xdoctest.global_state.DEBUG = 1
-    xdoctest.global_state.DEBUG_PARSER = 1
+    xdoctest.global_state.DEBUG_PARSER = 10
     xdoctest.global_state.DEBUG_CORE = 1
     xdoctest.global_state.DEBUG_RUNNER = 1
     xdoctest.global_state.DEBUG_DOCTEST = 1
 
-    with utils.TempDir() as temp:
-        dpath = temp.dpath
+    temp = utils.TempDir()
+    dpath = temp.ensure()
+    with temp as temp:
         modpath = join(dpath, 'test_example_run.py')
 
         with open(modpath, 'w') as file:
@@ -199,6 +206,11 @@ def dispatch(*types, **kwargs):
 
         with utils.CaptureStdout() as cap:
             runner.doctest_module(modpath, 'all', argv=[''], config=config)
-    print(cap.text)
-    # assert 'running 0 test' in cap.text
-    print(z.format_src())
+        print(cap.text)
+        assert '1 skipped' in cap.text
+
+        # example = examples[0]
+        # print(example.format_src())
+        # example.run()
+
+    # temp.cleanup()