pdb: fix capturing with recursive debugging and pdb++

While I think that pdb++ should be fixed in this regard (by using
`pdb.Pdb`, and not `self.__class__` maybe), this ensures that custom
debuggers like this are working.

src/_pytest/debugging.py

@@ -75,6 +75,7 @@ class pytestPDB(object):
     _config = None
     _pdb_cls = pdb.Pdb
     _saved = []
+    _recursive = 0
 
     @classmethod
     def set_trace(cls, set_break=True):
@@ -88,25 +89,33 @@ def set_trace(cls, set_break=True):
                 capman.suspend_global_capture(in_=True)
             tw = _pytest.config.create_terminal_writer(cls._config)
             tw.line()
-            if capman and capman.is_globally_capturing():
-                tw.sep(">", "PDB set_trace (IO-capturing turned off)")
-            else:
-                tw.sep(">", "PDB set_trace")
+            if cls._recursive == 0:
+                if capman and capman.is_globally_capturing():
+                    tw.sep(">", "PDB set_trace (IO-capturing turned off)")
+                else:
+                    tw.sep(">", "PDB set_trace")
 
             class _PdbWrapper(cls._pdb_cls, object):
                 _pytest_capman = capman
                 _continued = False
 
+                def do_debug(self, arg):
+                    cls._recursive += 1
+                    ret = super(_PdbWrapper, self).do_debug(arg)
+                    cls._recursive -= 1
+                    return ret
+
                 def do_continue(self, arg):
                     ret = super(_PdbWrapper, self).do_continue(arg)
                     if self._pytest_capman:
                         tw = _pytest.config.create_terminal_writer(cls._config)
                         tw.line()
-                        if self._pytest_capman.is_globally_capturing():
-                            tw.sep(">", "PDB continue (IO-capturing resumed)")
-                        else:
-                            tw.sep(">", "PDB continue")
-                        self._pytest_capman.resume_global_capture()
+                        if cls._recursive == 0:
+                            if self._pytest_capman.is_globally_capturing():
+                                tw.sep(">", "PDB continue (IO-capturing resumed)")
+                            else:
+                                tw.sep(">", "PDB continue")
+                            self._pytest_capman.resume_global_capture()
                     cls._pluginmanager.hook.pytest_leave_pdb(
                         config=cls._config, pdb=self
                     )

testing/test_pdb.py

@@ -518,6 +518,76 @@ def test_1():
         assert "1 failed" in rest
         self.flush(child)
 
+    def test_pdb_interaction_continue_recursive(self, testdir):
+        p1 = testdir.makepyfile(
+            mytest="""
+            import pdb
+            import pytest
+
+            count_continue = 0
+
+            # Simulates pdbpp, which injects Pdb into do_debug, and uses
+            # self.__class__ in do_continue.
+            class CustomPdb(pdb.Pdb):
+                def do_debug(self, arg):
+                    import sys
+                    import types
+
+                    newglobals = {
+                        'Pdb': self.__class__,  # NOTE: different with pdb.Pdb
+                        'sys': sys,
+                    }
+                    if sys.version_info < (3, ):
+                        do_debug_func = pdb.Pdb.do_debug.im_func
+                    else:
+                        do_debug_func = pdb.Pdb.do_debug
+
+                    orig_do_debug = types.FunctionType(
+                        do_debug_func.__code__, newglobals,
+                        do_debug_func.__name__, do_debug_func.__defaults__,
+                    )
+                    return orig_do_debug(self, arg)
+                do_debug.__doc__ = pdb.Pdb.do_debug.__doc__
+
+                def do_continue(self, *args, **kwargs):
+                    global count_continue
+                    count_continue += 1
+                    return super(CustomPdb, self).do_continue(*args, **kwargs)
+
+            def foo():
+                print("print_from_foo")
+
+            def test_1():
+                i = 0
+                print("hello17")
+                pytest.set_trace()
+                x = 3
+                print("hello18")
+
+                assert count_continue == 2, "unexpected_failure: %d != 2" % count_continue
+                pytest.fail("expected_failure")
+        """
+        )
+        child = testdir.spawn_pytest("--pdbcls=mytest:CustomPdb %s" % str(p1))
+        child.expect(r"PDB set_trace \(IO-capturing turned off\)")
+        child.expect(r"\n\(Pdb")
+        child.sendline("debug foo()")
+        child.expect("ENTERING RECURSIVE DEBUGGER")
+        child.expect(r"\n\(\(Pdb")
+        child.sendline("c")
+        child.expect("LEAVING RECURSIVE DEBUGGER")
+        assert b"PDB continue" not in child.before
+        assert b"print_from_foo" in child.before
+        child.sendline("c")
+        child.expect(r"PDB continue \(IO-capturing resumed\)")
+        rest = child.read().decode("utf8")
+        assert "hello17" in rest  # out is captured
+        assert "hello18" in rest  # out is captured
+        assert "1 failed" in rest
+        assert "Failed: expected_failure" in rest
+        assert "AssertionError: unexpected_failure" not in rest
+        self.flush(child)
+
     def test_pdb_without_capture(self, testdir):
         p1 = testdir.makepyfile(
             """