gh-35620: doctester: Check for consistency of # optional annotations

    
<!-- Please provide a concise, informative and self-explanatory title.
-->
<!-- Don't put issue numbers in the title. Put it in the Description
below. -->
<!-- For example, instead of "Fixes #12345", use "Add a new method to
multiply two integers" -->

### 📚 Description

<!-- Describe your changes here in detail. -->
For example:
```
sage -t --long --random-seed=52927571004392838159040103596363721016
src/sage/rings/function_field/function_field.py
**********************************************************************
File "src/sage/rings/function_field/function_field.py", line 40, in
sage.rings.function_field.function_field
Warning: Variable 'L' referenced here was set only in doctest marked '#
optional - sage.rings.finite_rings sage.rings.function_field'
    S.<t> = L[]
# optional - sage.rings.finite_rings
```
<!-- Why is this change required? What problem does it solve? -->
<!-- If this PR resolves an open issue, please link to it here. For
example "Fixes #12345". -->
Resolves  #35401
<!-- If your change requires a documentation PR, please link it
appropriately. -->


### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. It should be `[x]` not `[x
]`. -->

- [x] The title is concise, informative, and self-explanatory.
- [x] The description explains in detail what this PR is about.
- [x] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation accordingly.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on
- #12345: short description why this is a dependency
- #34567: ...
-->

<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
    
URL: #35620
Reported by: Matthias Köppe
Reviewer(s): Kwankyu Lee

src/sage/doctest/forker.py

@@ -47,6 +47,7 @@
 import doctest
 import traceback
 import tempfile
+from collections import defaultdict
 from dis import findlinestarts
 from queue import Empty
 import gc
@@ -56,7 +57,7 @@
 from sage.misc.misc import walltime
 from .util import Timer, RecordingDict, count_noun
 from .sources import DictAsObject
-from .parsing import OriginalSource, reduce_hex
+from .parsing import OriginalSource, reduce_hex, unparse_optional_tags
 from sage.structure.sage_object import SageObject
 from .parsing import SageOutputChecker, pre_hash, get_source
 from sage.repl.user_globals import set_globals
@@ -527,7 +528,7 @@ def __init__(self, *args, **kwds):
             self.msgfile = self._fakeout.real_stdout
         self.history = []
         self.references = []
-        self.setters = {}
+        self.setters = defaultdict(dict)
         self.running_global_digest = hashlib.md5()
         self.total_walltime_skips = 0
         self.total_performed_tests = 0
@@ -772,6 +773,9 @@ def compiler(example):
                 if self.options.warn_long > 0 and example.walltime + check_duration > self.options.warn_long:
                     self.report_overtime(out, test, example, got,
                                          check_duration=check_duration)
+                elif example.warnings:
+                    for warning in example.warnings:
+                        out(self._failure_header(test, example, f'Warning: {warning}'))
                 elif not quiet:
                     self.report_success(out, test, example, got,
                                         check_duration=check_duration)
@@ -831,14 +835,15 @@ def run(self, test, compileflags=0, out=None, clear_globs=True):
             sage: from sage.doctest.control import DocTestDefaults; DD = DocTestDefaults()
             sage: from sage.env import SAGE_SRC
             sage: import doctest, sys, os
-            sage: DTR = SageDocTestRunner(SageOutputChecker(), verbose=False, sage_options=DD, optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS)
+            sage: DTR = SageDocTestRunner(SageOutputChecker(), verbose=False, sage_options=DD,
+            ....:                         optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS)
             sage: filename = os.path.join(SAGE_SRC,'sage','doctest','forker.py')
             sage: FDS = FileDocTestSource(filename,DD)
             sage: doctests, extras = FDS.create_doctests(globals())
             sage: DTR.run(doctests[0], clear_globs=False)
             TestResults(failed=0, attempted=4)
         """
-        self.setters = {}
+        self.setters = defaultdict(dict)
         randstate.set_random_seed(self.options.random_seed)
         warnings.showwarning = showwarning_with_traceback
         self.running_doctest_digest = hashlib.md5()
@@ -1037,17 +1042,20 @@ def compile_and_execute(self, example, compiler, globs):
             sage: from sage.doctest.control import DocTestDefaults; DD = DocTestDefaults()
             sage: from sage.env import SAGE_SRC
             sage: import doctest, sys, os, hashlib
-            sage: DTR = SageDocTestRunner(SageOutputChecker(), verbose=False, sage_options=DD, optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS)
+            sage: DTR = SageDocTestRunner(SageOutputChecker(), verbose=False, sage_options=DD,
+            ....:           optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS)
             sage: DTR.running_doctest_digest = hashlib.md5()
-            sage: filename = os.path.join(SAGE_SRC,'sage','doctest','forker.py')
-            sage: FDS = FileDocTestSource(filename,DD)
+            sage: filename = os.path.join(SAGE_SRC, 'sage', 'doctest', 'forker.py')
+            sage: FDS = FileDocTestSource(filename, DD)
             sage: globs = RecordingDict(globals())
             sage: 'doctest_var' in globs
             False
             sage: doctests, extras = FDS.create_doctests(globs)
             sage: ex0 = doctests[0].examples[0]
             sage: flags = 32768 if sys.version_info.minor < 8 else 524288
-            sage: compiler = lambda ex: compile(ex.source, '<doctest sage.doctest.forker[0]>', 'single', flags, 1)
+            sage: def compiler(ex):
+            ....:     return compile(ex.source, '<doctest sage.doctest.forker[0]>',
+            ....:                    'single', flags, 1)
             sage: DTR.compile_and_execute(ex0, compiler, globs)
             1764
             sage: globs['doctest_var']
@@ -1060,7 +1068,9 @@ def compile_and_execute(self, example, compiler, globs):
         Now we can execute some more doctests to see the dependencies. ::
 
             sage: ex1 = doctests[0].examples[1]
-            sage: compiler = lambda ex:compile(ex.source, '<doctest sage.doctest.forker[1]>', 'single', flags, 1)
+            sage: def compiler(ex):
+            ....:     return compile(ex.source, '<doctest sage.doctest.forker[1]>',
+            ....:                    'single', flags, 1)
             sage: DTR.compile_and_execute(ex1, compiler, globs)
             sage: sorted(list(globs.set))
             ['R', 'a']
@@ -1072,7 +1082,9 @@ def compile_and_execute(self, example, compiler, globs):
         ::
 
             sage: ex2 = doctests[0].examples[2]
-            sage: compiler = lambda ex:compile(ex.source, '<doctest sage.doctest.forker[2]>', 'single', flags, 1)
+            sage: def compiler(ex):
+            ....:     return compile(ex.source, '<doctest sage.doctest.forker[2]>',
+            ....:                    'single', flags, 1)
             sage: DTR.compile_and_execute(ex2, compiler, globs)
             a + 42
             sage: list(globs.set)
@@ -1085,6 +1097,7 @@ def compile_and_execute(self, example, compiler, globs):
         if isinstance(globs, RecordingDict):
             globs.start()
         example.sequence_number = len(self.history)
+        example.warnings = []
         self.history.append(example)
         timer = Timer().start()
         try:
@@ -1096,11 +1109,20 @@ def compile_and_execute(self, example, compiler, globs):
             if isinstance(globs, RecordingDict):
                 example.predecessors = []
                 for name in globs.got:
-                    ref = self.setters.get(name)
-                    if ref is not None:
-                        example.predecessors.append(ref)
+                    setters_dict = self.setters.get(name)  # setter_optional_tags -> setter
+                    if setters_dict:
+                        for setter_optional_tags, setter in setters_dict.items():
+                            if setter_optional_tags.issubset(example.optional_tags):
+                                example.predecessors.append(setter)
+                        if not example.predecessors:
+                            f_setter_optional_tags = "; ".join("'"
+                                                               + unparse_optional_tags(setter_optional_tags)
+                                                               + "'"
+                                                               for setter_optional_tags in setters_dict)
+                            example.warnings.append(f"Variable '{name}' referenced here "
+                                                    f"was set only in doctest marked {f_setter_optional_tags}")
                 for name in globs.set:
-                    self.setters[name] = example
+                    self.setters[name][example.optional_tags] = example
             else:
                 example.predecessors = None
             self.update_digests(example)

src/sage/doctest/parsing.py

@@ -79,6 +79,10 @@ def fake_RIFtol(*args):
 # This is the correct pattern to match ISO/IEC 6429 ANSI escape sequences:
 ansi_escape_sequence = re.compile(r'(\x1b[@-Z\\-~]|\x1b\[.*?[@-~]|\x9b.*?[@-~])')
 
+special_optional_regex = 'arb216|arb218|py2|long time|not implemented|not tested|known bug'
+optional_regex = re.compile(fr'({special_optional_regex})|([^ a-z]\s*optional\s*[:-]*((\s|\w|[.])*))')
+special_optional_regex = re.compile(special_optional_regex)
+
 
 def parse_optional_tags(string):
     """
@@ -137,8 +141,6 @@ def parse_optional_tags(string):
     # strip_string_literals replaces comments
     comment = "#" + (literals[comment]).lower()
 
-    optional_regex = re.compile(r'(arb216|arb218|py2|long time|not implemented|not tested|known bug)|([^ a-z]\s*optional\s*[:-]*((\s|\w|[.])*))')
-
     tags = []
     for m in optional_regex.finditer(comment):
         cmd = m.group(1)
@@ -151,8 +153,40 @@ def parse_optional_tags(string):
     return set(tags)
 
 
-def parse_tolerance(source, want):
+def unparse_optional_tags(tags):
+    r"""
+    Return a comment string that sets ``tags``.
+
+    INPUT:
+
+    - ``tags`` -- iterable of tags, as output by :func:`parse_optional_tags`
+
+    EXAMPLES::
+
+        sage: from sage.doctest.parsing import unparse_optional_tags
+        sage: unparse_optional_tags(set())
+        ''
+        sage: unparse_optional_tags({'magma'})
+        '# optional - magma'
+        sage: unparse_optional_tags(['zipp', 'sage.rings.number_field', 'foo'])
+        '# optional - foo zipp sage.rings.number_field'
+        sage: unparse_optional_tags(['long time', 'not tested', 'p4cka9e'])
+        '# long time, not tested, optional - p4cka9e'
     """
+    tags = set(tags)
+    special_tags = set(tag for tag in tags if special_optional_regex.fullmatch(tag))
+    optional_tags = sorted(tags - special_tags,
+                           key=lambda tag: (tag.startswith('sage.'), tag))
+    tags = sorted(special_tags)
+    if optional_tags:
+        tags.append('optional - ' + " ".join(optional_tags))
+    if tags:
+        return '# ' + ', '.join(tags)
+    return ''
+
+
+def parse_tolerance(source, want):
+    r"""
     Return a version of ``want`` marked up with the tolerance tags
     specified in ``source``.
 
@@ -163,8 +197,8 @@ def parse_tolerance(source, want):
 
     OUTPUT:
 
-    - ``want`` if there are no tolerance tags specified; a
-      :class:`MarkedOutput` version otherwise.
+    ``want`` if there are no tolerance tags specified; a
+    :class:`MarkedOutput` version otherwise.
 
     EXAMPLES::
 
@@ -633,6 +667,7 @@ def parse(self, string, *args):
         for item in res:
             if isinstance(item, doctest.Example):
                 optional_tags = parse_optional_tags(item.source)
+                item.optional_tags = frozenset(optional_tags)
                 if optional_tags:
                     for tag in optional_tags:
                         self.optionals[tag] += 1

src/sage/doctest/sources.py

@@ -224,6 +224,7 @@ def _process_doc(self, doctests, doc, namespace, start):
                 # Line number refers to the end of the docstring
                 sigon = doctest.Example(sig_on_count_doc_doctest, "0\n", lineno=docstring.count("\n"))
                 sigon.sage_source = sig_on_count_doc_doctest
+                sigon.optional_tags = frozenset()
                 dt.examples.append(sigon)
             doctests.append(dt)
 
@@ -787,7 +788,7 @@ def _test_enough_doctests(self, check_extras=True, verbose=True):
             ....:             filename = os.path.join(path, F)
             ....:             FDS = FileDocTestSource(filename, DocTestDefaults(long=True, optional=True, force_lib=True))
             ....:             FDS._test_enough_doctests(verbose=False)
-            There are 3 unexpected tests being run in sage/doctest/parsing.py
+            There are 4 unexpected tests being run in sage/doctest/parsing.py
             There are 1 unexpected tests being run in sage/doctest/reporting.py
             sage: os.chdir(cwd)
         """

src/sage/rings/big_oh.py

@@ -94,7 +94,7 @@ def O(*x, **kwds):
         sage: A.<n> = AsymptoticRing(growth_group='QQ^n * n^QQ * log(n)^QQ',            # optional - sage.symbolic
         ....:                        coefficient_ring=QQ); A
         Asymptotic Ring <QQ^n * n^QQ * log(n)^QQ * Signs^n> over Rational Field
-        sage: O(n)
+        sage: O(n)                                                                      # optional - sage.symbolic
         O(n)
 
     Application with Puiseux series::
@@ -108,17 +108,17 @@ def O(*x, **kwds):
 
     TESTS::
 
-        sage: var('x, y')
+        sage: var('x, y')                                                               # optional - sage.symbolic
         (x, y)
-        sage: O(x)
+        sage: O(x)                                                                      # optional - sage.symbolic
         Traceback (most recent call last):
         ...
         ArithmeticError: O(x) not defined
-        sage: O(y)
+        sage: O(y)                                                                      # optional - sage.symbolic
         Traceback (most recent call last):
         ...
         ArithmeticError: O(y) not defined
-        sage: O(x, y)
+        sage: O(x, y)                                                                   # optional - sage.symbolic
         Traceback (most recent call last):
         ...
         ArithmeticError: O(x, y) not defined