Merge branch 't/32174/doctests__detect__safe__external_features_even_…

…if____optional_external__is_not_used' into t/32881/feature_for_sage_rings_padics

src/sage/doctest/control.py

@@ -38,7 +38,7 @@
 from .forker import DocTestDispatcher
 from .reporting import DocTestReporter
 from .util import Timer, count_noun, dict_difference
-from .external import external_software, available_software
+from .external import available_software
 from .parsing import parse_optional_tags
 
 nodoctest_regex = re.compile(r'\s*(#+|%+|r"+|"+|\.\.)\s*nodoctest')
@@ -409,11 +409,6 @@ def __init__(self, options, args):
                     options.optional.update(system.name
                                             for system in package_systems())
 
-                    logger = sys.stderr if options.verbose else None
-                    from sage.features.sagemath import sage_features
-                    options.optional.update(feature.name
-                                            for feature in sage_features(logger=logger))
-
                 # Check that all tags are valid
                 for o in options.optional:
                     if not optionaltag_regex.search(o):
@@ -928,7 +923,7 @@ def run_doctests(self):
             ----------------------------------------------------------------------
             Total time for all tests: ... seconds
                 cpu time: ... seconds
-                cumulative wall time: ... seconds
+                cumulative wall time: ... seconds...
         """
         nfiles = 0
         nother = 0
@@ -1004,6 +999,7 @@ def cleanup(self, final=True):
             Total time for all tests: ... seconds
                 cpu time: ... seconds
                 cumulative wall time: ... seconds
+            Features detected...
             0
             sage: DC.cleanup()
         """
@@ -1194,6 +1190,7 @@ def run(self):
             Total time for all tests: ... seconds
                 cpu time: ... seconds
                 cumulative wall time: ... seconds
+            Features detected...
             0
 
         We check that :trac:`25378` is fixed (testing external packages
@@ -1207,7 +1204,7 @@ def run(self):
             sage: DC.run()
             Running doctests with ID ...
             Using --optional=external,sage
-            External software to be detected: ...
+            Features to be detected: ...
             Doctesting 1 file.
             sage -t ....py
                 [0 tests, ... s]
@@ -1217,7 +1214,7 @@ def run(self):
             Total time for all tests: ... seconds
                 cpu time: ... seconds
                 cumulative wall time: ... seconds
-            External software detected for doctesting:...
+            Features detected...
             0
 
         """
@@ -1247,18 +1244,16 @@ def run(self):
                     pass
 
             self.log("Using --optional=" + self._optional_tags_string())
-            if self.options.optional is True or 'external' in self.options.optional:
-                self.log("External software to be detected: " + ','.join(external_software))
-
+            available_software._allow_external = self.options.optional is True or 'external' in self.options.optional
+            self.log("Features to be detected: " + ','.join(available_software.detectable()))
             self.add_files()
             self.expand_files_into_sources()
             self.filter_sources()
             self.sort_sources()
             self.run_doctests()
 
-            if self.options.optional is True or 'external' in self.options.optional:
-                self.log("External software detected for doctesting: "
-                         + ','.join(available_software.seen()))
+            self.log("Features detected for doctesting: "
+                     + ','.join(available_software.seen()))
             self.cleanup()
             return self.reporter.error_status
 
@@ -1286,6 +1281,7 @@ def run_doctests(module, options=None):
         Total time for all tests: ... seconds
             cpu time: ... seconds
             cumulative wall time: ... seconds
+        Features detected...
     """
     import sys
     sys.stdout.flush()

src/sage/doctest/external.py

@@ -117,12 +117,8 @@ def has_magma():
         sage: has_magma() # random, optional - magma
         True
     """
-    from sage.interfaces.magma import magma
-    try:
-        magma('2+3')
-        return True
-    except Exception:
-        return False
+    from sage.features.interfaces import Magma
+    return Magma().is_present()
 
 def has_matlab():
     """
@@ -134,12 +130,8 @@ def has_matlab():
         sage: has_matlab() # random, optional - matlab
         True
     """
-    from sage.interfaces.matlab import matlab
-    try:
-        matlab('2+3')
-        return True
-    except Exception:
-        return False
+    from sage.features.interfaces import Matlab
+    return Matlab().is_present()
 
 def has_mathematica():
     """
@@ -151,12 +143,8 @@ def has_mathematica():
         sage: has_mathematica() # random, optional - mathematica
         True
     """
-    from sage.interfaces.mathematica import mathematica
-    try:
-        mathematica('2+3')
-        return True
-    except Exception:
-        return False
+    from sage.features.interfaces import Mathematica
+    return Mathematica().is_present()
 
 def has_maple():
     """
@@ -168,12 +156,8 @@ def has_maple():
         sage: has_maple() # random, optional - maple
         True
     """
-    from sage.interfaces.maple import maple
-    try:
-        maple('2+3')
-        return True
-    except Exception:
-        return False
+    from sage.features.interfaces import Maple
+    return Maple().is_present()
 
 def has_macaulay2():
     """
@@ -185,12 +169,8 @@ def has_macaulay2():
         sage: has_macaulay2() # random, optional - macaulay2
         True
     """
-    from sage.interfaces.macaulay2 import macaulay2
-    try:
-        macaulay2('2+3')
-        return True
-    except Exception:
-        return False
+    from sage.features.interfaces import Macaulay2
+    return Macaulay2().is_present()
 
 def has_octave():
     """
@@ -202,12 +182,8 @@ def has_octave():
         sage: has_octave() # random, optional - octave
         True
     """
-    from sage.interfaces.octave import octave
-    try:
-        octave('2+3')
-        return True
-    except Exception:
-        return False
+    from sage.features.interfaces import Octave
+    return Octave().is_present()
 
 def has_pandoc():
     """
@@ -357,6 +333,24 @@ def has_4ti2():
     from sage.features.four_ti_2 import FourTi2
     return FourTi2().is_present()
 
+def external_features():
+    r"""
+    Generate the features that are only to be tested if ``--optional=external`` is used.
+
+    EXAMPLES::
+
+        sage: from sage.doctest.external import external_features
+        sage: next(external_features())
+        Feature('internet')
+    """
+    from sage.features.internet import Internet
+    yield Internet()
+    import sage.features.interfaces
+    yield from sage.features.interfaces.all_features()
+    from sage.features.mip_backends import CPLEX, Gurobi
+    yield CPLEX()
+    yield Gurobi()
+
 def external_software():
     """
     Return the alphabetical list of external software supported by this module.
@@ -367,28 +361,10 @@ def external_software():
         sage: sorted(external_software) == external_software
         True
     """
-    supported = list()
-    for func in globals():
-        if func.startswith(prefix):
-            supported.append(func[len(prefix):])
-    return sorted(supported)
+    return sorted(f.name for f in external_features())
 
 external_software = external_software()
 
-def _lookup(software):
-    """
-    Test if the software is available on the system.
-
-    EXAMPLES::
-
-        sage: sage.doctest.external._lookup('internet') # random, optional - internet
-        True
-    """
-    if software in external_software:
-        func = globals().get(prefix + software)
-        return func()
-    else:
-        return False
 
 class AvailableSoftware(object):
     """
@@ -399,28 +375,16 @@ class AvailableSoftware(object):
 
         sage: from sage.doctest.external import external_software, available_software
         sage: external_software
-        ['4ti2',
-         'cplex',
-         'dvipng',
-         'ffmpeg',
-         'graphviz',
+        ['cplex',
          'gurobi',
-         'imagemagick',
          'internet',
-         'latex',
-         'lualatex',
          'macaulay2',
          'magma',
          'maple',
          'mathematica',
          'matlab',
          'octave',
-         'pandoc',
-         'pdf2svg',
-         'pdflatex',
-         'rubiks',
-         'scilab',
-         'xelatex']
+         'scilab']
         sage: 'internet' in available_software # random, optional - internet
         True
         sage: available_software.issuperset(set(['internet','latex'])) # random, optional - internet latex
@@ -437,10 +401,17 @@ def __init__(self):
             sage: S.seen() # random
             []
         """
+        self._allow_external = True
         # For multiprocessing of doctests, the data self._seen should be
         # shared among subprocesses. Thus we use Array class from the
         # multiprocessing module.
-        self._seen = Array('i', len(external_software)) # initialized to zeroes
+        from sage.features.all import all_features
+        self._external_features = set(external_features())
+        features = set(self._external_features)
+        features.update(all_features())
+        self._features = sorted(features, key=lambda feature: feature.name)
+        self._indices = {feature.name: idx for idx, feature in enumerate(self._features)}
+        self._seen = Array('i', len(self._features)) # initialized to zeroes
 
     def __contains__(self, item):
         """
@@ -453,11 +424,13 @@ def __contains__(self, item):
             True
         """
         try:
-            idx = external_software.index(item)
-        except Exception:
+            idx = self._indices[item]
+        except KeyError:
             return False
         if not self._seen[idx]:
-            if _lookup(item):
+            if not self._allow_external and self._features[idx] in self._external_features:
+                self._seen[idx] = -1 # not available
+            elif self._features[idx].is_present():
                 self._seen[idx] = 1 # available
             else:
                 self._seen[idx] = -1 # not available
@@ -483,6 +456,14 @@ def issuperset(self, other):
                 return False
         return True
 
+    def detectable(self):
+        """
+        Return the list of names of those features for which testing their presence is allowed.
+        """
+        return [feature.name
+                for feature in self._features
+                if self._allow_external or feature not in self._external_features]
+
     def seen(self):
         """
         Return the list of detected external software.
@@ -493,6 +474,9 @@ def seen(self):
             sage: available_software.seen() # random
             ['internet', 'latex', 'magma']
         """
-        return [external_software[i] for i in range(len(external_software)) if self._seen[i] > 0]
+        return [feature.name
+                for feature, seen in zip(self._features, self._seen)
+                if seen > 0]
+
 
 available_software = AvailableSoftware()

src/sage/doctest/forker.py

@@ -2056,6 +2056,7 @@ def __init__(self, source, options, funclist=[]):
             Total time for all tests: ... seconds
                 cpu time: ... seconds
                 cumulative wall time: ... seconds
+            Features detected...
         """
         multiprocessing.Process.__init__(self)
 
@@ -2101,6 +2102,7 @@ def run(self):
             Total time for all tests: ... seconds
                 cpu time: ... seconds
                 cumulative wall time: ... seconds
+            Features detected...
         """
         os.setpgid(os.getpid(), os.getpid())
 

src/sage/doctest/parsing.py

@@ -798,8 +798,7 @@ def parse(self, string, *args):
                     if self.optional_tags is not True:
                         extra = optional_tags - self.optional_tags # set difference
                         if extra:
-                            if not('external' in self.optional_tags
-                                   and available_software.issuperset(extra)):
+                            if not available_software.issuperset(extra):
                                 continue
                 elif self.optional_only:
                     self.optionals['sage'] += 1

src/sage/features/all.py

@@ -0,0 +1,24 @@
+
+def all_features():
+    r"""
+    Return an iterable of all features.
+
+    EXAMPLES::
+
+        sage: from sage.features.all import all_features
+        sage: sorted(all_features(), key=lambda f: f.name)  # random
+        [...Feature('sage.combinat')...]
+    """
+    import pkgutil
+    import importlib
+    import sage.features
+    # Following https://packaging.python.org/guides/creating-and-discovering-plugins/#using-namespace-packages
+    for finder, name, ispkg in pkgutil.iter_modules(sage.features.__path__, sage.features.__name__ + "."):
+        module = importlib.import_module(name)
+        try:
+            af = module.all_features
+        except AttributeError:
+            pass
+        else:
+            if af != all_features:
+                yield from af()

src/sage/features/bliss.py

@@ -69,3 +69,8 @@ def __init__(self):
         JoinFeature.__init__(self, "bliss",
                              [PythonModule("sage.graphs.bliss", spkg="bliss",
                                            url="http://www.tcs.hut.fi/Software/bliss/")])
+
+
+def all_features():
+    return [BlissLibrary(),
+            Bliss()]