diff --git a/pyproject.toml b/pyproject.toml index 72d8c1f9..02a0ea79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "stimela" -version = "2.0rc9" +version = "2.0rc10" description = "Framework for system agnostic pipelines for (not just) radio interferometry" authors = ["Sphesihle Makhathini ", "Oleg Smirnov and RATT "] readme = "README.rst" diff --git a/scabha/configuratt/__init__.py b/scabha/configuratt/__init__.py index cf94bcad..c2d0b847 100644 --- a/scabha/configuratt/__init__.py +++ b/scabha/configuratt/__init__.py @@ -18,6 +18,7 @@ def load(path: str, use_sources: Optional[List[DictConfig]] = [], name: Optional includes: bool=True, selfrefs: bool=True, include_path: str=None, use_cache: bool = True, no_toplevel_cache = False, + include_stack = [], verbose: bool = False): """Loads config file, using a previously loaded config to resolve _use references. @@ -30,6 +31,7 @@ def load(path: str, use_sources: Optional[List[DictConfig]] = [], name: Optional includes (bool, optional): If True (default), "_include" references will be processed selfrefs (bool, optional): If False, "_use" references will only be looked up in existing config. If True (default), they'll also be looked up within the loaded config. + include_stack: list of paths which have been included. Used to catch recursive includes. include_path (str, optional): if set, path to each config file will be included in the section as element 'include_path' @@ -50,7 +52,8 @@ def load(path: str, use_sources: Optional[List[DictConfig]] = [], name: Optional if use_sources is not None and selfrefs: use_sources = [subconf] + list(use_sources) conf, deps = resolve_config_refs(subconf, pathname=path, location=location, name=name, - includes=includes, use_cache=use_cache, use_sources=use_sources, include_path=include_path) + includes=includes, use_cache=use_cache, use_sources=use_sources, include_path=include_path, + include_stack=include_stack + [path]) # update overall dependencies dependencies.update(deps) diff --git a/scabha/configuratt/resolvers.py b/scabha/configuratt/resolvers.py index 10a823a9..6f84e847 100644 --- a/scabha/configuratt/resolvers.py +++ b/scabha/configuratt/resolvers.py @@ -124,7 +124,8 @@ def _scrub_subsections(conf: DictConfig, scrubs: Union[str, List[str]]): def resolve_config_refs(conf, pathname: str, location: str, name: str, includes: bool, use_sources: Optional[List[DictConfig]], use_cache = True, - include_path: Optional[str]=None): + include_path: Optional[str]=None, + include_stack=[]): """Resolves cross-references ("_use" and "_include" statements) in config object Parameters @@ -143,6 +144,8 @@ def resolve_config_refs(conf, pathname: str, location: str, name: str, includes: one or more config object(s) in which to look up "_use" references. None to disable _use statements include_path (str, optional): if set, path to each config file will be included in the section as element 'include_path' + include_stack (list, optional): + stack of files from which this one was included. Used to catch recursion. Returns ------- @@ -222,29 +225,38 @@ def load_include_files(keyword): else: flags = {} - # check for (module)filename.yaml or (module)/filename.yaml style - match = re.match("^\\((.+)\\)/?(.+)$", incl) + # check for (location)filename.yaml or (location)/filename.yaml style + match= re.match("^\\((.+)\\)/?(.+)$", incl) if match: modulename, filename = match.groups() - try: - mod = importlib.import_module(modulename) - except ImportError as exc: - if 'optional' in flags: - dependencies.add_fail(FailRecord(incl, pathname, modulename=modulename, fname=filename)) - if 'warn' in flags: - print(f"Warning: unable to import module for optional include {incl}") - continue - raise ConfigurattError(f"{errloc}: {keyword} {incl}: can't import {modulename} ({exc})") - - filename = os.path.join(os.path.dirname(mod.__file__), filename) - if not os.path.exists(filename): - if 'optional' in flags: - dependencies.add_fail(FailRecord(incl, pathname, modulename=modulename, fname=filename)) - if 'warn' in flags: - print(f"Warning: unable to find optional include {incl}") - continue - raise ConfigurattError(f"{errloc}: {keyword} {incl}: {filename} does not exist") - + if modulename.startswith("."): + filename = os.path.join(os.path.dirname(pathname), modulename, filename) + if not os.path.exists(filename): + if 'optional' in flags: + dependencies.add_fail(FailRecord(filename, pathname)) + if 'warn' in flags: + print(f"Warning: unable to find optional include {incl} ({filename})") + continue + raise ConfigurattError(f"{errloc}: {keyword} {incl} does not exist") + else: + try: + mod = importlib.import_module(modulename) + except ImportError as exc: + if 'optional' in flags: + dependencies.add_fail(FailRecord(incl, pathname, modulename=modulename, fname=filename)) + if 'warn' in flags: + print(f"Warning: unable to import module for optional include {incl}") + continue + raise ConfigurattError(f"{errloc}: {keyword} {incl}: can't import {modulename} ({exc})") + + filename = os.path.join(os.path.dirname(mod.__file__), filename) + if not os.path.exists(filename): + if 'optional' in flags: + dependencies.add_fail(FailRecord(incl, pathname, modulename=modulename, fname=filename)) + if 'warn' in flags: + print(f"Warning: unable to find optional include {incl}") + continue + raise ConfigurattError(f"{errloc}: {keyword} {incl}: {filename} does not exist") # absolute path -- one candidate elif os.path.isabs(incl): if not os.path.exists(incl): @@ -270,10 +282,15 @@ def load_include_files(keyword): continue raise ConfigurattError(f"{errloc}: {keyword} {incl} not found in {':'.join(paths)}") + # check for recursion + for path in include_stack: + if os.path.samefile(path, filename): + raise ConfigurattError(f"{errloc}: {filename} is included recursively") # load included file incl_conf, deps = load(filename, location=location, name=f"{filename}, included from {name}", includes=True, + include_stack=include_stack, use_cache=use_cache, use_sources=None) # do not expand _use statements in included files, this is done below @@ -351,6 +368,7 @@ def load_use_sections(keyword): value1, deps = resolve_config_refs(value, pathname=pathname, name=name, location=f"{location}.{key}" if location else key, includes=includes, + include_stack=include_stack, use_sources=use_sources, use_cache=use_cache, include_path=include_path) dependencies.update(deps) @@ -366,6 +384,7 @@ def load_use_sections(keyword): value1, deps = resolve_config_refs(value, pathname=pathname, name=name, location=f"{location or ''}[{i}]", includes=includes, + include_stack=include_stack, use_sources=use_sources, use_cache=use_cache, include_path=include_path) dependencies.update(deps) diff --git a/scabha/validate.py b/scabha/validate.py index ff862476..e195e2c9 100644 --- a/scabha/validate.py +++ b/scabha/validate.py @@ -287,6 +287,11 @@ def validate_parameters(params: Dict[str, Any], schemas: Dict[str, Any], schema = schemas[name] if schema.choices and value not in schema.choices: raise ParameterValidationError(f"{mkname(name)}: invalid value '{value}'") + if schema.element_choices: + listval = value if isinstance(value, (list, tuple, ListConfig)) else [value] + for elem in listval: + if elem not in schema.element_choices: + raise ParameterValidationError(f"{mkname(name)}: invalid list element '{elem}'") # check for mkdir directives if create_dirs: diff --git a/stimela/backends/kube/infrastructure.py b/stimela/backends/kube/infrastructure.py index 9fed6a4f..49b1ea3c 100644 --- a/stimela/backends/kube/infrastructure.py +++ b/stimela/backends/kube/infrastructure.py @@ -6,7 +6,8 @@ from typing import Optional, Dict, List from stimela.backends import StimelaBackendOptions -from stimela.stimelogging import log_exception, declare_chapter, update_process_status +from stimela.stimelogging import log_exception, declare_chapter +from stimela.task_stats import update_process_status from scabha.basetypes import EmptyListDefault from stimela.exceptions import BackendError @@ -55,6 +56,11 @@ def init(backend: StimelaBackendOptions, log: logging.Logger, cleanup: bool = Fa klog = log.getChild("kube") kube = backend.kube + if not kube.namespace: + klog.error("kube.namespace not configured, kube backend will not be available") + return False + + if cleanup: klog.info("cleaning up backend") else: diff --git a/stimela/backends/kube/pod_proxy.py b/stimela/backends/kube/pod_proxy.py index cb31c6f1..cdf793f0 100644 --- a/stimela/backends/kube/pod_proxy.py +++ b/stimela/backends/kube/pod_proxy.py @@ -6,7 +6,7 @@ import threading from stimela.exceptions import BackendError from stimela.utils.xrun_asyncio import dispatch_to_log -from stimela.stimelogging import update_process_status +from stimela.task_stats import update_process_status from stimela.kitchen.cab import Cab, Parameter from kubernetes.client import ApiException diff --git a/stimela/backends/kube/run_kube.py b/stimela/backends/kube/run_kube.py index e1c6ed4c..e805c095 100644 --- a/stimela/backends/kube/run_kube.py +++ b/stimela/backends/kube/run_kube.py @@ -13,7 +13,7 @@ from stimela.utils.xrun_asyncio import dispatch_to_log from stimela.exceptions import StimelaCabParameterError, StimelaCabRuntimeError, BackendError from stimela.stimelogging import log_exception -from stimela.stimelogging import declare_subcommand, declare_subtask, update_process_status +from stimela.task_stats import declare_subcommand, declare_subtask, update_process_status from stimela.backends import StimelaBackendOptions from stimela.kitchen.cab import Cab diff --git a/stimela/commands/cleanup.py b/stimela/commands/cleanup.py index 02fcd8a5..4b7e93a5 100644 --- a/stimela/commands/cleanup.py +++ b/stimela/commands/cleanup.py @@ -9,51 +9,33 @@ from stimela.kitchen.recipe import Recipe from stimela.exceptions import RecipeValidationError, BackendError -from .run import load_recipe_file +from .run import load_recipe_files @cli.command("cleanup", help=""" Cleans up backend resources associated with recipe(s). """) -@click.argument("items", nargs=-1, metavar="filename.yml|recipe name|...") +@click.argument("items", nargs=-1, metavar="filename.yml...") def cleanup(items: List[str] = []): log = logger() - for item in items: - # a filename -- treat it as a config - if os.path.isfile(item): - log.info(f"loading recipe/config {item}") + # load all recipe/config files + # load config and recipes from all given files + load_recipe_files(items) - # if file contains a recipe entry, treat it as a full config (that can include cabs etc.) - conf, recipe_deps = load_recipe_file(item) - - # anything that is not a standard config section will be treated as a recipe - recipes = [name for name in conf if name not in stimela.CONFIG] - - if len(recipes) == 1: - default_recipe = recipes[0] - - for name in recipes: - try: - # cast section to Recipe and remove from loaded conf - recipe = OmegaConf.structured(Recipe) - recipe = OmegaConf.unsafe_merge(recipe, conf[name]) - except Exception as exc: - log.error(f"recipe '{name}': {exc}") - sys.exit(2) - del conf[name] - # add to global namespace - stimela.CONFIG.lib.recipes[name] = recipe - - # the rest is safe to merge into config as is - stimela.CONFIG = OmegaConf.unsafe_merge(stimela.CONFIG, conf) - # now cleanup backend - try: - backend = OmegaConf.to_object(stimela.CONFIG.opts.backend) - stimela.backends.cleanup_backends(backend, log) - except BackendError as exc: - log_exception(exc) - return 1 + backends_list = stimela.CONFIG.opts.backend.select + if type(backends_list) is str: + backends_list = [backends_list] + if not backends_list: + log.info(f"configuration does not specify any backends, nothing to clean up") + else: + log.info(f"invoking cleanup procedure, selected backends: {', '.join(backends_list)}") + try: + backend = OmegaConf.to_object(stimela.CONFIG.opts.backend) + stimela.backends.cleanup_backends(backend, log) + except BackendError as exc: + log_exception(exc) + return 1 diff --git a/stimela/commands/doc.py b/stimela/commands/doc.py index 9a862c1c..70c05a84 100644 --- a/stimela/commands/doc.py +++ b/stimela/commands/doc.py @@ -18,7 +18,7 @@ from stimela.config import ConfigExceptionTypes from stimela.exceptions import RecipeValidationError -from .run import load_recipe_file +from .run import load_recipe_files @cli.command("doc", help=""" @@ -66,56 +66,39 @@ def load_recipe(name: str, section: Dict): log_exception(exc) sys.exit(2) - + # load all recipe/config files + files_to_load = [] + names_to_document = [] for item in items: - # a filename -- treat it as a config - if os.path.isfile(item): - log.info(f"loading recipe/config {item}") - - # if file contains a recipe entry, treat it as a full config (that can include cabs etc.) - conf, recipe_deps = load_recipe_file(item) - - # anything that is not a standard config section will be treated as a recipe - recipes = [name for name in conf if name not in stimela.CONFIG] - - if len(recipes) == 1: - default_recipe = recipes[0] - - for name in recipes: - try: - # cast section to Recipe and remove from loaded conf - recipe = OmegaConf.structured(Recipe) - recipe = OmegaConf.unsafe_merge(recipe, conf[name]) - except Exception as exc: - log.error(f"recipe '{name}': {exc}") - sys.exit(2) - del conf[name] - # add to global namespace - stimela.CONFIG.lib.recipes[name] = recipe - - # the rest is safe to merge into config as is - stimela.CONFIG = OmegaConf.unsafe_merge(stimela.CONFIG, conf) - - # else treat as a wildcard for recipe names or cab names + if os.path.isfile(item) and os.path.splitext(item)[1].lower() in (".yml", ".yaml"): + files_to_load.append(item) + log.info(f"will load recipe/config file '{item}'") else: - recipe_names = fnmatch.filter(stimela.CONFIG.lib.recipes.keys(), item) - cab_names = fnmatch.filter(stimela.CONFIG.cabs.keys(), item) - if not recipe_names and not cab_names: - log.error(f"'{item}' does not match any files, recipes or cab names. Try -l/--list") - sys.exit(2) + names_to_document.append(item) + + # load config and recipes from all given files + if files_to_load: + load_recipe_files(files_to_load) + + for item in names_to_document: + recipe_names = fnmatch.filter(stimela.CONFIG.lib.recipes.keys(), item) + cab_names = fnmatch.filter(stimela.CONFIG.cabs.keys(), item) + if not recipe_names and not cab_names: + log.error(f"'{item}' does not match any files, recipes or cab names. Try -l/--list") + sys.exit(2) - for name in recipe_names: - recipe = load_recipe(name, stimela.CONFIG.lib.recipes[name]) - tree = top_tree.add(f"Recipe: [bold]{name}[/bold]") - recipe.rich_help(tree, max_category=max_category) - - for name in cab_names: - cab = Cab(**stimela.CONFIG.cabs[name]) - cab.finalize(config=stimela.CONFIG) - tree = top_tree.add(f"Cab: [bold]{name}[/bold]") - cab.rich_help(tree, max_category=max_category) + for name in recipe_names: + recipe = load_recipe(name, stimela.CONFIG.lib.recipes[name]) + tree = top_tree.add(f"Recipe: [bold]{name}[/bold]") + recipe.rich_help(tree, max_category=max_category) + + for name in cab_names: + cab = Cab(**stimela.CONFIG.cabs[name]) + cab.finalize(config=stimela.CONFIG) + tree = top_tree.add(f"Cab: [bold]{name}[/bold]") + cab.rich_help(tree, max_category=max_category) - found_something = True + found_something = True if do_list or (not found_something and not default_recipe): diff --git a/stimela/commands/run.py b/stimela/commands/run.py index f21d94d2..34e3925d 100644 --- a/stimela/commands/run.py +++ b/stimela/commands/run.py @@ -19,38 +19,65 @@ import stimela.config from stimela.config import ConfigExceptionTypes from stimela import logger, log_exception -from stimela.exceptions import RecipeValidationError, StimelaRuntimeError, StepSelectionError, BackendError +from stimela.exceptions import RecipeValidationError, StimelaRuntimeError, StepSelectionError from stimela.main import cli from stimela.kitchen.recipe import Recipe, Step, RecipeSchema, join_quote from stimela import task_stats import stimela.backends -def load_recipe_file(filename: str): - dependencies = stimela.config.get_initial_deps() +def load_recipe_files(filenames: List[str]): - # if file contains a recipe entry, treat it as a full config (that can include cabs etc.) - try: - conf, deps = configuratt.load(filename, use_sources=[stimela.CONFIG], no_toplevel_cache=True) - except ConfigExceptionTypes as exc: - log_exception(f"error loading {filename}", exc) - sys.exit(2) + full_conf = OmegaConf.create() + full_deps = configuratt.ConfigDependencies() + for filename in filenames: + try: + conf, deps = configuratt.load(filename, use_sources=[stimela.CONFIG, full_conf], no_toplevel_cache=True) + except ConfigExceptionTypes as exc: + log_exception(f"error loading {filename}", exc) + sys.exit(2) + # accumulate loaded config + full_conf.merge_with(conf) + full_deps.update(deps) # warn user if any includes failed - if deps.fails: - logger().warning(f"{len(deps.fails)} optional includes were not found, some cabs may not be available") - for path, dep in deps.fails.items(): + if full_deps.fails: + logger().warning(f"{len(full_deps.fails)} optional includes were not found, some cabs may not be available") + for path, dep in full_deps.fails.items(): logger().warning(f" {path} (from {dep.origin})") - dependencies.update(deps) + # merge into full config dependencies + dependencies = stimela.config.get_initial_deps() + dependencies.update(full_deps) + stimela.config.CONFIG_DEPS.update(dependencies) # check for missing requirements - missing = configuratt.check_requirements(conf, [stimela.CONFIG], strict=True) + missing = configuratt.check_requirements(full_conf, [stimela.CONFIG], strict=True) for (loc, name, _) in missing: logger().warning(f"optional config section '{loc}' omitted due to missing requirement '{name}'") + # split content into config sections, and recipes: + # config secions are merged into the config namespace, while recipes go under + # lib.recipes + recipe_names = [] + update_conf = OmegaConf.create() + for name, value in full_conf.items(): + if name in stimela.CONFIG: + update_conf[name] = value + else: + try: + stimela.CONFIG.lib.recipes[name] = OmegaConf.merge(RecipeSchema, value) + except Exception as exc: + log_exception(f"error in definition of recipe '{name}'", exc) + sys.exit(2) + recipe_names.append(name) + + try: + stimela.CONFIG.merge_with(update_conf) + except Exception as exc: + log_exception(f"error applying configuration from {' ,'.join(filenames)}", exc) + sys.exit(2) - return conf, dependencies - + return recipe_names @cli.command("run", help=""" @@ -76,24 +103,26 @@ def load_recipe_file(filename: str): @click.option("-a", "--assign", metavar="PARAM VALUE", nargs=2, multiple=True, help="""assigns values to parameters: equivalent to PARAM=VALUE, but plays nicer with the shell's tab completion.""") +@click.option("-C", "--config", "config_assign", metavar="X.Y.Z VALUE", nargs=2, multiple=True, + help="""tweak configuration sections.""") @click.option("-l", "--last-recipe", is_flag=True, help="""if multiple recipes are defined, selects the last one for execution.""") @click.option("-d", "--dry-run", is_flag=True, help="""Doesn't actually run anything, only prints the selected steps.""") @click.option("-p", "--profile", metavar="DEPTH", type=int, help="""Print per-step profiling stats to this depth. 0 disables.""") -@click.argument("what", metavar="filename.yml|cab name") -@click.argument("parameters", nargs=-1, metavar="[recipe name] [PARAM=VALUE] [X.Y.Z=FOO] ...", required=False) -def run(what: str, parameters: List[str] = [], dry_run: bool = False, last_recipe: bool = False, profile: Optional[int] = None, +@click.argument("parameters", nargs=-1, metavar="(cab name | filename.yml [filename.yml...] [recipe name]) [PARAM=VALUE] [X.Y.Z=VALUE] ...", required=True) +def run(parameters: List[str] = [], dry_run: bool = False, last_recipe: bool = False, profile: Optional[int] = None, assign: List[Tuple[str, str]] = [], + config_assign: List[Tuple[str, str]] = [], step_ranges: List[str] = [], tags: List[str] = [], skip_tags: List[str] = [], enable_steps: List[str] = [], build=False, rebuild=False, build_skips=False): log = logger() params = OrderedDict() - dotlist = OrderedDict() errcode = 0 - recipe_name = None + recipe_name = cab_name = None + files_to_load = [] def convert_value(value): if value == "=UNSET": @@ -112,12 +141,7 @@ def convert_value(value): # parse arguments as recipe name, parameter assignments, or dotlist for OmegaConf for pp in parameters: - if "=" not in pp: - if recipe_name is not None: - log_exception(f"multiple recipe names given") - errcode = 2 - recipe_name = pp - else: + if "=" in pp: key, value = pp.split("=", 1) # parse string as yaml value try: @@ -125,32 +149,47 @@ def convert_value(value): except Exception as exc: log_exception(f"error parsing {pp}", exc) errcode = 2 + elif os.path.isfile(pp) and os.path.splitext(pp)[1].lower() in (".yml", ".yaml"): + files_to_load.append(pp) + log.info(f"will load recipe/config file '{pp}'") + elif pp in stimela.CONFIG.cabs: + if recipe_name is not None or cab_name is not None: + log_exception(f"multiple recipe and/or cab names given") + errcode = 2 + else: + log.info(f"treating '{pp}' as a cab name") + cab_name = pp + else: + if recipe_name is not None or cab_name is not None: + log_exception(f"multiple recipe and/or cab names given") + errcode = 2 + else: + recipe_name = pp + log.info(f"treating '{pp}' as a recipe name") if errcode: sys.exit(errcode) - # load extra config settigs from dotkey arguments, to be merged in below - # (when loading a recipe file, we want to merge these in AFTER the recipe is loaded, because the arguments - # might apply to the recipe) + # load config and recipes from all given files + if files_to_load: + available_recipes = load_recipe_files(files_to_load) + else: + available_recipes = [] + + # load config settigs from --config arguments try: - extra_config = OmegaConf.from_dotlist(list(dotlist.values())) if dotlist else OmegaConf.create() + dotlist = [f"{key}={value}" for key, value in config_assign] + stimela.CONFIG.merge_with(OmegaConf.from_dotlist(dotlist)) except OmegaConfBaseException as exc: - log_exception(f"error loading command-line dotlist", exc) + log_exception(f"error loading --config assignments", exc) sys.exit(2) - if what in stimela.CONFIG.cabs: - cabname = what - - try: - stimela.CONFIG = OmegaConf.unsafe_merge(stimela.CONFIG, extra_config) - except OmegaConfBaseException as exc: - log_exception(f"error applying command-line dotlist", exc) - sys.exit(2) - - log.info(f"setting up cab {cabname}") + # run a cab + if cab_name is not None: + log.info(f"setting up cab {cab_name}") # create step config by merging in settings (var=value pairs from the command line) - outer_step = Step(cab=cabname, params=params) + outer_step = Step(cab=cab_name, params=params) # prevalidate() is done by run() automatically if not already done, but it does set up the recipe's logger, so do it anyway try: @@ -158,55 +197,26 @@ def convert_value(value): except ScabhaBaseException as exc: log_exception(exc) sys.exit(1) - + # run a recipe else: - if not os.path.isfile(what): - log_exception(f"'{what}' is neither a recipe file nor a known stimela cab") - sys.exit(2) - - log.info(f"loading recipe/config {what}") - - conf, recipe_deps = load_recipe_file(what) - conf.merge_with(extra_config) - - # anything that is not a standard config section will be treated as a recipe - all_recipe_names = [name for name in conf if name not in stimela.CONFIG] - if not all_recipe_names: - log_exception(f"{what} does not contain any recipes") - sys.exit(2) - - # split content into config sections, and recipes: - # config secions are merged into the config namespace, while recipes go under - # lib.recipes - update_conf = OmegaConf.create() - for name, value in conf.items(): - if name in stimela.CONFIG: - update_conf[name] = value - else: - try: - stimela.CONFIG.lib.recipes[name] = OmegaConf.merge(RecipeSchema, value) - except Exception as exc: - log_exception(f"error in definition of recipe '{name}'", exc) - sys.exit(2) - - try: - stimela.CONFIG = OmegaConf.unsafe_merge(stimela.CONFIG, update_conf) - except Exception as exc: - log_exception(f"error applying configuration from {what}", exc) + if not stimela.CONFIG.lib.recipes: + log_exception(f"no recipes were specified") sys.exit(2) - log.info(f"{what} contains the following recipe sections: {join_quote(all_recipe_names)}") - if recipe_name: - if recipe_name not in conf: - log_exception(f"{what} does not contain recipe '{recipe_name}'") + if recipe_name not in stimela.CONFIG.lib.recipes: + log_exception(f"recipe '{recipe_name}' not found") + sys.exit(2) + else: + if len(available_recipes) == 0: + log_exception(f"no top-level recipes were found") + sys.exit(2) + elif last_recipe or len(available_recipes) == 1: + recipe_name = available_recipes[-1] + else: + logger().info(f"found multiple top-level recipes: {', '.join(available_recipes)}") + log_exception(f"please specify a recipe on the command line, or use -l/--last-recipe") sys.exit(2) - elif last_recipe or len(all_recipe_names) == 1: - recipe_name = all_recipe_names[-1] - else: - print(f"This file contains the following recipes: {', '.join(all_recipe_names)}") - log_exception(f"multiple recipes found, please specify one on the command line, or use -l/--last-recipe") - sys.exit(2) log.info(f"selected recipe is '{recipe_name}'") @@ -246,7 +256,7 @@ def convert_value(value): stimelogging.declare_chapter("prevalidation") log.info("pre-validating the recipe") - outer_step = Step(recipe=recipe, name=f"{recipe_name}", info=what, params=params) + outer_step = Step(recipe=recipe, name=f"{recipe_name}", info=recipe_name, params=params) try: params = outer_step.prevalidate(root=True) except Exception as exc: @@ -272,7 +282,6 @@ def convert_value(value): log.info(f"recipe logs will be saved under {logdir}") filename = os.path.join(logdir, "stimela.recipe.deps") - stimela.config.CONFIG_DEPS.update(recipe_deps) stimela.config.CONFIG_DEPS.save(filename) log.info(f"saved recipe dependencies to {filename}") diff --git a/tests/scabha_tests/test_configuratt.py b/tests/scabha_tests/test_configuratt.py index 4c2609a7..951a6e5f 100644 --- a/tests/scabha_tests/test_configuratt.py +++ b/tests/scabha_tests/test_configuratt.py @@ -18,6 +18,7 @@ def test_includes(): path = "testconf.yaml" conf, deps = configuratt.load(path, use_sources=[], verbose=True, use_cache=False) assert conf.x.y2.z1 == 1 + assert conf.relative.x.y == 'a' missing = configuratt.check_requirements(conf, [], strict=False) assert len(missing) == 2 # 2 failed reqs @@ -32,6 +33,12 @@ def test_includes(): except configuratt.ConfigurattError as exc: print(f"Exception as expected: {exc}") + try: + conf, deps = configuratt.load("test_recursive_include.yml", use_sources=[], verbose=True, use_cache=False) + raise RuntimeError("Error was expected here!") + except configuratt.ConfigurattError as exc: + print(f"Exception as expected: {exc}") + nested = ["test_nest_a.yml", "test_nest_b.yml", "test_nest_c.yml"] nested = [os.path.join(os.path.dirname(path), name) for name in nested] @@ -46,4 +53,4 @@ def test_includes(): if __name__ == "__main__": - test_includes(sys.argv[1] if len(sys.argv) > 1 else None) \ No newline at end of file + test_includes() \ No newline at end of file diff --git a/tests/scabha_tests/test_include2.yaml b/tests/scabha_tests/test_include2.yaml index 02bb43eb..b39bfc9b 100644 --- a/tests/scabha_tests/test_include2.yaml +++ b/tests/scabha_tests/test_include2.yaml @@ -1,4 +1,5 @@ x: y: a z: b - _include: test_include.yaml \ No newline at end of file + _include: test_include.yaml + \ No newline at end of file diff --git a/tests/scabha_tests/test_recursive_include.yml b/tests/scabha_tests/test_recursive_include.yml new file mode 100644 index 00000000..b6f8730d --- /dev/null +++ b/tests/scabha_tests/test_recursive_include.yml @@ -0,0 +1,2 @@ +a: + _include: test_recursive_include2.yml \ No newline at end of file diff --git a/tests/scabha_tests/test_recursive_include2.yml b/tests/scabha_tests/test_recursive_include2.yml new file mode 100644 index 00000000..2ad31df7 --- /dev/null +++ b/tests/scabha_tests/test_recursive_include2.yml @@ -0,0 +1,2 @@ +b: + _include: (.)test_recursive_include.yml \ No newline at end of file diff --git a/tests/scabha_tests/testconf.yaml b/tests/scabha_tests/testconf.yaml index bba389ed..b0d34280 100644 --- a/tests/scabha_tests/testconf.yaml +++ b/tests/scabha_tests/testconf.yaml @@ -13,6 +13,9 @@ hierarchical: bar: _use: bar +relative: + _include: (.)test_include2.yaml + flat: _use: hierarchical _flatten: true diff --git a/tests/stimela_tests/test_recipe.py b/tests/stimela_tests/test_recipe.py index dc756b41..6c71d801 100644 --- a/tests/stimela_tests/test_recipe.py +++ b/tests/stimela_tests/test_recipe.py @@ -39,6 +39,9 @@ def test_test_aliasing(): retcode, _ = run(f"stimela -v exec test_aliasing.yml") assert retcode != 0 + print("===== expecting no errors now =====") + retcode, output = run("stimela -v doc test_aliasing.yml") + print("===== expecting no errors now =====") retcode, output = run("stimela -v exec test_aliasing.yml a=1 s3.a=1 s4.a=1 e=e f=f") assert retcode == 0 @@ -60,8 +63,20 @@ def test_test_recipe(): retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar") assert retcode != 0 + print("===== expecting an error due to elem-xyz choices wrong =====") + retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=t elemlist-xyz=[x,y]") + assert retcode != 0 + + print("===== expecting an error due to elem-xyz choices wrong =====") + retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=[x,t]") + assert retcode != 0 + + print("===== expecting no errors now =====") + retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=[x,y]") + assert retcode == 0 + print("===== expecting no errors now =====") - retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar msname=foo") + retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=x") assert retcode == 0 def test_test_loop_recipe(): diff --git a/tests/stimela_tests/test_recipe.yml b/tests/stimela_tests/test_recipe.yml index 6a3a7948..e58a4072 100644 --- a/tests/stimela_tests/test_recipe.yml +++ b/tests/stimela_tests/test_recipe.yml @@ -96,6 +96,13 @@ recipe: band: choices: [L, UHF, K] default: L + elem-xyz: + default: x + choices: [x,y,z] + elemlist-xyz: + dtype: Union[str, List[str]] + element_choices: [x,y,z] + assign: foo: bar