Trac #29922: sage.doctest: Make imports more specific; make global en…

…vironment for tests customizable

We prepare the Sage doctesting module for running in environments where
`sage.all` is not available (#29705).

1. An explicit import in `sage.doctest.parsing`:
{{{
# We need to import from sage.all to avoid circular imports.
from sage.all import RealIntervalField
RIFtol = RealIntervalField(1044)
}}}

2. We make the global environment for tests customizable in
`sage.doctest.forker`: in `init_sage`:
{{{
    import sage.repl.ipython_kernel.all_jupyter
}}}
   and again in `DocTestTask._run`:
{{{
            # Import Jupyter globals to doctest the Jupyter
            # implementation of widgets and interacts
            import sage.repl.ipython_kernel.all_jupyter as sage_all
}}}
   ... which is
{{{
"""
All imports for Jupyter
"""

from sage.all_cmdline import *

from .widgets_sagenb import (input_box, text_control, slider,
        range_slider, checkbox, selector, input_grid, color_selector)
from .interact import interact
}}}

   This is exposed by the new `sage-runtest` option `--environment`.

   For example, #29865 defines modules such as `sage.all__sage_objects`.
We would invoke `sage -t --environment=sage.all__sage_objects` to test
against this global environment.

3. We ignore errors importing `sage.interfaces`.

URL: https://trac.sagemath.org/29922
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): Jonathan Kliem

src/bin/sage-runtests

@@ -73,6 +73,7 @@ if __name__ == "__main__":
     parser.add_option("--random-seed", dest="random_seed", type=int, metavar="SEED", help="random seed for fuzzing doctests")
     parser.add_option("--global-iterations", "--global_iterations", type=int, default=0, help="repeat the whole testing process this many times")
     parser.add_option("--file-iterations", "--file_iterations", type=int, default=0, help="repeat each file this many times, stopping on the first failure")
+    parser.add_option("--environment", type=str, default="sage.repl.ipython_kernel.all_jupyter", help="name of a module that provides the global environment for tests")
 
     parser.add_option("-i", "--initial", action="store_true", default=False, help="only show the first failure in each file")
     parser.add_option("--exitfirst", action="store_true", default=False, help="end the test run immediately after the first failure or unexpected exception")

src/sage/doctest/control.py

@@ -39,7 +39,6 @@
 from .reporting import DocTestReporter
 from .util import Timer, count_noun, dict_difference
 from .external import external_software, available_software
-from sage.features import PythonModule
 
 nodoctest_regex = re.compile(r'\s*(#+|%+|r"+|"+|\.\.)\s*nodoctest')
 optionaltag_regex = re.compile(r'^\w+$')
@@ -100,6 +99,7 @@ def __init__(self, **kwds):
         self.random_seed = 0
         self.global_iterations = 1  # sage-runtests default is 0
         self.file_iterations = 1    # sage-runtests default is 0
+        self.environment = "sage.repl.ipython_kernel.all_jupyter"
         self.initial = False
         self.exitfirst = False
         self.force_lib = False
@@ -501,6 +501,25 @@ def _repr_(self):
         """
         return "DocTest Controller"
 
+    def load_environment(self):
+        """
+        Return the module that provides the global environment.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults, DocTestController
+            sage: DC = DocTestController(DocTestDefaults(), [])
+            sage: 'BipartiteGraph' in DC.load_environment().__dict__
+            True
+            sage: DC = DocTestController(DocTestDefaults(environment='sage.doctest.all'), [])
+            sage: 'BipartiteGraph' in  DC.load_environment().__dict__
+            False
+            sage: 'run_doctests' in DC.load_environment().__dict__
+            True
+        """
+        from importlib import import_module
+        return import_module(self.options.environment)
+
     def load_stats(self, filename):
         """
         Load stats from the most recent run(s).

src/sage/doctest/forker.py

@@ -115,7 +115,7 @@ def inner(obj, p, cycle):
     return inner
 
 
-def init_sage():
+def init_sage(controller=None):
     """
     Import the Sage library.
 
@@ -195,8 +195,16 @@ def init_sage():
     from sage.cpython._py2_random import Random
     sage.misc.randstate.DEFAULT_PYTHON_RANDOM = Random
 
-    import sage.repl.ipython_kernel.all_jupyter
-    sage.interfaces.quit.invalidate_all()
+    if controller is None:
+        import sage.repl.ipython_kernel.all_jupyter
+    else:
+        controller.load_environment()
+
+    try:
+        from sage.interfaces.quit import invalidate_all
+        invalidate_all()
+    except ModuleNotFoundError:
+        pass
 
     # Disable cysignals debug messages in doctests: this is needed to
     # make doctests pass when cysignals was built with debugging enabled
@@ -1653,7 +1661,7 @@ def __init__(self, controller):
             <sage.doctest.forker.DocTestDispatcher object at ...>
         """
         self.controller = controller
-        init_sage()
+        init_sage(controller)
 
     def serial_dispatch(self):
         """
@@ -2534,7 +2542,8 @@ def _run(self, runner, options, results):
         """
         # Import Jupyter globals to doctest the Jupyter
         # implementation of widgets and interacts
-        import sage.repl.ipython_kernel.all_jupyter as sage_all
+        from importlib import import_module
+        sage_all = import_module(options.environment)
         dict_all = sage_all.__dict__
         # Remove '__package__' item from the globals since it is not
         # always in the globals in an actual Sage session.

src/sage/doctest/parsing.py

@@ -53,21 +53,39 @@
 backslash_replacer = re.compile(r"""(\s*)sage:(.*)\\\ *
 \ *(((\.){4}:)|((\.){3}))?\ *""")
 
-# Use this real interval field for doctest tolerances. It allows large
-# numbers like 1e1000, it parses strings with spaces like RIF(" - 1 ")
-# out of the box and it carries a lot of precision. The latter is
-# useful for testing libraries using arbitrary precision but not
-# guaranteed rounding such as PARI. We use 1044 bits of precision,
-# which should be good to deal with tolerances on numbers computed with
-# 1024 bits of precision.
-#
-# The interval approach also means that we do not need to worry about
-# rounding errors and it is also very natural to see a number with
-# tolerance as an interval.
-# We need to import from sage.all to avoid circular imports.
-from sage.all import RealIntervalField
-RIFtol = RealIntervalField(1044)
+_RIFtol = None
+
+def RIFtol(*args):
+    """
+    Create an element of the real interval field used for doctest tolerances.
+
+    It allows large numbers like 1e1000, it parses strings with spaces
+    like ``RIF(" - 1 ")`` out of the box and it carries a lot of
+    precision. The latter is useful for testing libraries using
+    arbitrary precision but not guaranteed rounding such as PARI. We use
+    1044 bits of precision, which should be good to deal with tolerances
+    on numbers computed with 1024 bits of precision.
 
+    The interval approach also means that we do not need to worry about
+    rounding errors and it is also very natural to see a number with
+    tolerance as an interval.
+
+    EXAMPLES::
+
+        sage: from sage.doctest.parsing import RIFtol
+        sage: RIFtol(-1, 1)
+        0.?
+        sage: RIFtol(" - 1 ")
+        -1
+        sage: RIFtol("1e1000")
+        1.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000?e1000
+    """
+    global _RIFtol
+    if _RIFtol is None:
+        # We need to import from sage.all to avoid circular imports.
+        from sage.all import RealIntervalField
+        _RIFtol = RealIntervalField(1044)
+    return _RIFtol(*args)
 
 # This is the correct pattern to match ISO/IEC 6429 ANSI escape sequences:
 #

src/sage/misc/package.py

@@ -234,9 +234,17 @@ def list_packages(*pkg_types, **opts):
 
     installed = installed_packages(exclude_pip)
 
-    pkgs = {}
     SAGE_PKGS = sage.env.SAGE_PKGS
-    for p in os.listdir(SAGE_PKGS):
+    if not SAGE_PKGS:
+        return {}
+
+    try:
+        lp = os.listdir(SAGE_PKGS)
+    except FileNotFoundError:
+        return {}
+
+    pkgs = {}
+    for p in lp:
         try:
             f = open(os.path.join(SAGE_PKGS, p, "type"))
         except IOError:
@@ -310,8 +318,13 @@ def installed_packages(exclude_pip=True):
     if not exclude_pip:
         installed.update(pip_installed_packages())
     # Sage packages should override pip packages (Trac #23997)
-    installed.update(pkgname_split(pkgname)
-                     for pkgname in os.listdir(sage.env.SAGE_SPKG_INST))
+    SAGE_SPKG_INST = sage.env.SAGE_SPKG_INST
+    if SAGE_SPKG_INST:
+        try:
+            lp = os.listdir(SAGE_SPKG_INST)
+            installed.update(pkgname_split(pkgname) for pkgname in lp)
+        except FileNotFoundError:
+            pass
     return installed
 
 

src/sage/repl/display/fancy_repr.py

@@ -199,7 +199,10 @@ def __call__(self, obj, p, cycle):
         if not p.toplevel():
             # Do not print the help for matrices inside containers
             return False
-        from sage.matrix.matrix1 import Matrix
+        try:
+            from sage.matrix.matrix1 import Matrix
+        except ModuleNotFoundError:
+            return False
         if not isinstance(obj, Matrix):
             return False
         from sage.matrix.matrix0 import max_rows, max_cols