cylc-rose: cylc list compatibility

cylc/flow/scripts/config.py

@@ -56,7 +56,8 @@
 from cylc.flow.option_parsers import CylcOptionParser as COP
 from cylc.flow.pathutil import get_workflow_run_dir
 from cylc.flow.workflow_files import WorkflowFiles, parse_workflow_arg
-from cylc.flow.templatevars import load_template_vars
+from cylc.flow.scripts.install import add_cylc_rose_options
+from cylc.flow.templatevars import get_template_vars
 from cylc.flow.terminal import cli_function
 
 
@@ -97,6 +98,8 @@ def get_option_parser():
             "overrides any settings it shares with those higher up."),
         action="store_true", default=False, dest="print_hierarchy")
 
+    parser = add_cylc_rose_options(parser)
+
     return parser
 
 
@@ -129,7 +132,7 @@ def main(parser, options, reg=None):
         workflow,
         flow_file,
         options,
-        load_template_vars(options.templatevars, options.templatevars_file))
+        get_template_vars(options, flow_file, [reg, workflow]))
 
     config.pcfg.idump(
         options.item,

cylc/flow/scripts/graph.py

@@ -37,8 +37,9 @@
 from cylc.flow.exceptions import UserInputError
 from cylc.flow.option_parsers import CylcOptionParser as COP
 from cylc.flow.workflow_files import parse_workflow_arg
-from cylc.flow.templatevars import load_template_vars
+from cylc.flow.templatevars import get_template_vars
 from cylc.flow.terminal import cli_function
+from cylc.flow.scripts.install import add_cylc_rose_options
 
 
 def sort_integer_node(item):
@@ -209,6 +210,8 @@ def get_option_parser():
         action='store',
     )
 
+    parser = add_cylc_rose_options(parser)
+
     return parser
 
 
@@ -222,8 +225,7 @@ def main(parser, opts, workflow=None, start=None, stop=None):
             'Only the --reference and --diff use cases are supported'
         )
 
-    template_vars = load_template_vars(
-        opts.templatevars, opts.templatevars_file)
+    template_vars = get_template_vars(opts, workflow)
 
     write = print
     flows = [(workflow, [])]

cylc/flow/scripts/install.py

@@ -83,6 +83,51 @@
     from optparse import Values
 
 
+def add_cylc_rose_options(parser):
+    """Add extra options for cylc-rose plugin if it is installed.
+
+    Args:
+        parser: An option parser object
+    """
+    try:
+        __import__('cylc.rose')
+        parser.add_option(
+            "--opt-conf-key", "-O",
+            help=(
+                "Use optional Rose Config Setting "
+                "(If Cylc-Rose is installed)"
+            ),
+            action="append",
+            default=[],
+            dest="opt_conf_keys"
+        )
+        parser.add_option(
+            "--define", '-D',
+            help=(
+                "Each of these overrides the `[SECTION]KEY` setting in a "
+                "`rose-suite.conf` file. "
+                "Can be used to disable a setting using the syntax "
+                "`--define=[SECTION]!KEY` or even `--define=[!SECTION]`."
+            ),
+            action="append",
+            default=[],
+            dest="defines"
+        )
+        parser.add_option(
+            "--rose-template-variable", '-S',
+            help=(
+                "As `--define`, but with an implicit `[SECTION]` for "
+                "workflow variables."
+            ),
+            action="append",
+            default=[],
+            dest="rose_template_vars"
+        )
+    except ImportError:
+        pass
+    return parser
+
+
 def get_option_parser():
     parser = COP(
         __doc__, comms=True, prep=True,
@@ -127,43 +172,7 @@ def get_option_parser():
         default=False,
         dest="no_symlinks")
 
-    # If cylc-rose plugin is available ad the --option/-O config
-    try:
-        __import__('cylc.rose')
-        parser.add_option(
-            "--opt-conf-key", "-O",
-            help=(
-                "Use optional Rose Config Setting"
-                "(If Cylc-Rose is installed)"
-            ),
-            action="append",
-            default=[],
-            dest="opt_conf_keys"
-        )
-        parser.add_option(
-            "--define", '-D',
-            help=(
-                "Each of these overrides the `[SECTION]KEY` setting in a "
-                "`rose-suite.conf` file. "
-                "Can be used to disable a setting using the syntax "
-                "`--define=[SECTION]!KEY` or even `--define=[!SECTION]`."
-            ),
-            action="append",
-            default=[],
-            dest="defines"
-        )
-        parser.add_option(
-            "--rose-template-variable", '-S',
-            help=(
-                "As `--define`, but with an implicit `[SECTION]` for "
-                "workflow variables."
-            ),
-            action="append",
-            default=[],
-            dest="rose_template_vars"
-        )
-    except ImportError:
-        pass
+    parser = add_cylc_rose_options(parser)
 
     return parser
 

cylc/flow/scripts/list.py

@@ -35,7 +35,8 @@
 from cylc.flow.config import WorkflowConfig
 from cylc.flow.option_parsers import CylcOptionParser as COP
 from cylc.flow.workflow_files import parse_workflow_arg
-from cylc.flow.templatevars import load_template_vars
+from cylc.flow.scripts.install import add_cylc_rose_options
+from cylc.flow.templatevars import get_template_vars
 from cylc.flow.terminal import cli_function
 
 
@@ -81,12 +82,14 @@ def get_option_parser():
         "initial cycle point, by default). Use '-p , ' for the default range.",
         metavar="[START],[STOP]", action="store", default=None, dest="prange")
 
+    parser = add_cylc_rose_options(parser)
     return parser
 
 
 @cli_function(get_option_parser)
 def main(parser, options, reg):
     workflow, flow_file = parse_workflow_arg(options, reg)
+    template_vars = get_template_vars(options, flow_file, [reg, flow_file])
 
     if options.all_tasks and options.all_namespaces:
         parser.error("Choose either -a or -n")
@@ -125,7 +128,7 @@ def main(parser, options, reg):
         workflow,
         flow_file,
         options,
-        load_template_vars(options.templatevars, options.templatevars_file))
+        template_vars)
     if options.tree:
         config.print_first_parent_tree(
             pretty=options.box, titles=options.titles)

cylc/flow/scripts/validate.py

@@ -38,12 +38,13 @@
 from cylc.flow.task_proxy import TaskProxy
 from cylc.flow.task_pool import FlowLabelMgr
 from cylc.flow.loggingutil import CylcLogFormatter
-from cylc.flow.templatevars import load_template_vars
+from cylc.flow.templatevars import get_template_vars
 from cylc.flow.option_parsers import (
     CylcOptionParser as COP,
     Options
 )
 from cylc.flow.workflow_files import parse_workflow_arg
+from cylc.flow.scripts.install import add_cylc_rose_options
 
 
 def get_option_parser():
@@ -71,6 +72,8 @@ def get_option_parser():
         default="live", dest="run_mode",
         choices=['live', 'dummy', 'dummy-local', 'simulation'])
 
+    parser = add_cylc_rose_options(parser)
+
     parser.set_defaults(is_validate=True)
 
     return parser
@@ -104,7 +107,7 @@ def main(_, options, reg):
         workflow,
         flow_file,
         options,
-        load_template_vars(options.templatevars, options.templatevars_file),
+        get_template_vars(options, flow_file, [reg, workflow]),
         output_fname=options.output, mem_log_func=profiler.log_memory)
 
     # Check bounds of sequences

cylc/flow/templatevars.py

@@ -16,8 +16,13 @@
 """Load custom variables for template processor."""
 
 from ast import literal_eval
+from optparse import Values
+from os import PathLike
+from pathlib import Path
+from typing import Union, Dict, Tuple, Optional
 
-from cylc.flow.exceptions import UserInputError
+from cylc.flow import iter_entry_points
+from cylc.flow.exceptions import UserInputError, PluginError
 
 
 def eval_var(var):
@@ -68,3 +73,58 @@ def load_template_vars(template_vars=None, template_vars_file=None):
             key, val = pair.split("=", 1)
             res[key.strip()] = eval_var(val.strip())
     return res
+
+
+def get_template_vars(
+    options: Values,
+    flow_file: Union[str, 'PathLike[str]'],
+    names: Optional[Tuple[str, str]] = None
+) -> Dict:
+    """Get Template Vars from either an uninstalled or installed flow.
+
+    Designed to allow a Cylc Script to be run on an installed workflow where
+    template variables have been processed and saved to file, but fallback to
+    evaluating templating if run on an uninstalled workflow.
+
+    Args:
+        options: Options passed to the Cylc script which is using this
+            function.
+        flow_file: Path to the ``flow.cylc`` (or ``suite.rc``) file defining
+            this workflow.
+        names: reg and flow file name from
+            `cylc.flow.workflow_filesparse_workflow_arg`: Used to determine
+            whether flow is installed.
+
+    Returns:
+        template_vars: Template variables to give to a Cylc config.
+    """
+    # We are operating on an installed workflow.
+    if (
+        names
+        and names[0] == names[1]            # reg == flow_file name
+        and not Path(names[0]).is_file()    # reg is not a path
+    ):
+        template_vars = load_template_vars(
+            options.templatevars, options.templatevars_file)
+    # Else we act as if we might be looking at a cylc-src dir.
+    else:
+        template_vars = load_template_vars(
+            options.templatevars, options.templatevars_file)
+        source = Path(flow_file).parent
+        for entry_point in iter_entry_points(
+            'cylc.pre_configure'
+        ):
+            try:
+                ep_result = entry_point.resolve()(
+                    srcdir=source, opts=options
+                )
+                template_vars.update(ep_result['template_variables'])
+            except Exception as exc:
+                # NOTE: except Exception (purposefully vague)
+                # this is to separate plugin from core Cylc errors
+                raise PluginError(
+                    'cylc.pre_configure',
+                    entry_point.name,
+                    exc
+                ) from None
+    return template_vars

tests/unit/test_templatevars.py

@@ -14,11 +14,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import pytest
 import tempfile
 import unittest
 
-from cylc.flow.templatevars import load_template_vars
+from types import SimpleNamespace
 
+from cylc.flow.exceptions import PluginError
+from cylc.flow.templatevars import get_template_vars, load_template_vars
 
 class TestTemplatevars(unittest.TestCase):
 
@@ -102,3 +105,77 @@ def test_load_template_vars_from_string_and_file_2(self):
 
 if __name__ == '__main__':
     unittest.main()
+
+
+def test_get_template_vars_installed_flow(monkeypatch):
+    """It works on an installed flow.
+
+    n.b. Does not attempt to test ``load_template_vars``
+    """
+    monkeypatch.setattr(
+        'cylc.flow.templatevars.load_template_vars',
+        lambda templatevars, templatevars_file: {'foo': 'bar'}
+    )
+    opts = SimpleNamespace(templatevars='', templatevars_file='')
+    result = get_template_vars(opts, '', names=('eg/runN', 'eg/runN'))
+    assert result == {'foo': 'bar'}
+
+
+@pytest.fixture(scope='module')
+def provide_opts():
+    """Provide a fake opts"""
+    return SimpleNamespace(
+        templatevars='', templatevars_file=''
+    )
+
+
+@pytest.fixture
+def monkeypatch_load_template_vars(monkeypatch):
+    monkeypatch.setattr(
+        'cylc.flow.templatevars.load_template_vars',
+        lambda templatevars, templatevars_file: {}
+    )
+
+
+def test_get_template_vars_src_flow(
+    monkeypatch, provide_opts, monkeypatch_load_template_vars):
+    """It works on a flow which hasn't been installed.
+    """
+    def fake_iter_entry_points(_):
+        class fake_ep:
+            name = 'Zaphod'
+            def resolve():
+                def _inner(srcdir, opts):
+                    return {'template_variables': {'MYVAR': 'foo'}}
+                return _inner
+        return [fake_ep]
+
+    monkeypatch.setattr(
+        'cylc.flow.templatevars.iter_entry_points',
+        fake_iter_entry_points
+    )
+    assert get_template_vars(provide_opts, '') == {'MYVAR': 'foo'}
+
+
+def test_get_template_vars_src_flow_fails(
+    monkeypatch, provide_opts, monkeypatch_load_template_vars):
+    """It fails if there is a plugin error.
+    """
+    def fake_iter_entry_points(_):
+        class fake_ep:
+            name = 'Zaphod'
+            def resolve():
+                def _inner(srcdir, opts):
+                    raise TypeError('Utter Drivel.')
+                return _inner
+        return [fake_ep]
+
+    monkeypatch.setattr(
+        'cylc.flow.templatevars.iter_entry_points',
+        fake_iter_entry_points
+    )
+
+    with pytest.raises(PluginError) as exc:
+        get_template_vars(provide_opts, '')
+    assert exc.match(
+        'Error in plugin cylc.pre_configure.Zaphod\nUtter Drivel.')