diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dd55c0ef..36f872dd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,12 +17,14 @@ Added - Automatic Optional for arguments with default None #30. - CLI now supports running methods from classes. +- Signature arguments can now be loaded from independent config files #32. Changed ^^^^^^^ - Improved description of parser used as standalone and for ActionParser #34. +- Removed :code:`__cwd__` and top level :code:`__path__` that were not needed. Fixed diff --git a/README.rst b/README.rst index bddfde02..c4daf830 100644 --- a/README.rst +++ b/README.rst @@ -180,7 +180,9 @@ If multiple classes or a mixture of functions and classes is given to :func:`.CLI`, to execute a method of a class, two levels of :ref:`sub-commands` are required. The first sub-command would be name of the class and the second the name of the method, i.e. :code:`python example.py class [init_arguments] -method [arguments]`. +method [arguments]`. For more details about the automatic adding of arguments +from classes and functions and the use of configuration files refer to section +:ref:`classes-methods-functions`. This simple way of usage is similar and inspired by `Fire `__. However, there are fundamental differences. @@ -191,9 +193,9 @@ values will be validated according to these. Third, the return values of the functions are not automatically printed. :func:`.CLI` returns its value and it is up to the developer to decide what to do with it. Finally, jsonargparse has many features designed to help in creating convenient argument parsers such as: -:ref:`nested-namespaces`, :ref:`configuration-files`, arguments from -:ref:`classes-methods-functions`, additional type hints (:ref:`parsing-paths`, -:ref:`restricted-numbers`, :ref:`restricted-strings`) and much more. +:ref:`nested-namespaces`, :ref:`configuration-files`, additional type hints +(:ref:`parsing-paths`, :ref:`restricted-numbers`, :ref:`restricted-strings`) and +much more. The next section explains how to create an argument parser in a very low level argparse-style. However, as parsers get more complex, being able to define them @@ -446,6 +448,27 @@ since the init has the :code:`**kwargs` argument, the keyword arguments from :code:`value` as a required float and :code:`flag` as an optional boolean with default value false. +Instead of using :func:`namespace_to_dict` to convert the namespaces to a +dictionary, the :class:`.ArgumentParser` object can be instantiated with +:code:`parse_as_dict=True` to get directly a dictionary from the parsing +methods. + +When parsing from a configuration file (see :ref:`configuration-files`) all the +values can be given in a single config file. However, for convenience it is also +possible that the values for each of the groups created by the calls to the add +signature methods can be parsed from independent files. This means that for the +example above there could be one general config file with contents: + +.. code-block:: yaml + + myclass: + init: myclass.yaml + method: mymethod.yaml + +Then the files :code:`myclass.yaml` and :code:`mymethod.yaml` would only include +the settings for each of the instantiation of the class and the call to the +method respectively. + A wide range of type hints are supported. For exact details go to section :ref:`type-hints`. Some notes about the support for automatic adding of arguments are: diff --git a/jsonargparse/actions.py b/jsonargparse/actions.py index 0253c3c5..510e8f78 100644 --- a/jsonargparse/actions.py +++ b/jsonargparse/actions.py @@ -17,6 +17,7 @@ namespace_to_dict, dict_to_namespace, Path, + _load_config, _flat_namespace_to_dict, _dict_to_flat_namespace, _check_unknown_kwargs, @@ -109,10 +110,7 @@ def _apply_config(parser, namespace, dest, value): cfg_file = _dict_to_flat_namespace(namespace_to_dict(cfg_file)) getattr(namespace, dest).append(cfg_path) for key, val in vars(cfg_file).items(): - if key == '__cwd__' and hasattr(namespace, '__cwd__'): - setattr(namespace, key, getattr(namespace, key)+val) - else: - setattr(namespace, key, val) + setattr(namespace, key, val) class _ActionPrintConfig(Action): @@ -131,6 +129,28 @@ def __call__(self, parser, *args, **kwargs): parser._print_config = True +class _ActionConfigLoad(Action): + + def __init__(self, **kwargs): + kwargs['help'] = SUPPRESS + kwargs['default'] = SUPPRESS + super().__init__(**kwargs) + + def __call__(self, parser, namespace, value, option_string=None): + cfg_file = self._load_config(value) + for key, val in vars(cfg_file).items(): + setattr(namespace, self.dest+'.'+key, val) + + def _load_config(self, value): + try: + return _load_config(value) + except (TypeError, yamlParserError, yamlScannerError) as ex: + raise TypeError('Parser key "'+self.dest+'": '+str(ex)) from ex + + def _check_type(self, value, cfg=None): + return self._load_config(value) + + class ActionYesNo(Action): """Paired options --{yes_prefix}opt, --{no_prefix}opt to set True or False respectively.""" @@ -535,7 +555,7 @@ def _check_type(self, value, cfg=None, islist=None): val = Path(val, mode=self._mode, skip_check=self._skip_check) value[num] = val except TypeError as ex: - raise TypeError('Parser key "'+self.dest+'": '+str(ex)) + raise TypeError('Parser key "'+self.dest+'": '+str(ex)) from ex return value if islist else value[0] @@ -598,7 +618,7 @@ def _check_type(self, value, cfg=None): with sys.stdin if path_list_file == '-' else open(path_list_file, 'r') as f: path_list = [x.strip() for x in f.readlines()] except FileNotFoundError as ex: - raise TypeError('Problems reading path list: '+path_list_file+' :: '+str(ex)) + raise TypeError('Problems reading path list: '+path_list_file+' :: '+str(ex)) from ex cwd = os.getcwd() if self._rel == 'list' and path_list_file != '-': os.chdir(os.path.abspath(os.path.join(path_list_file, os.pardir))) @@ -607,7 +627,7 @@ def _check_type(self, value, cfg=None): try: path_list[num] = Path(val, mode=self._mode) except TypeError as ex: - raise TypeError('Path number '+str(num+1)+' in list '+path_list_file+', '+str(ex)) + raise TypeError('Path number '+str(num+1)+' in list '+path_list_file+', '+str(ex)) from ex finally: os.chdir(cwd) value += path_list diff --git a/jsonargparse/core.py b/jsonargparse/core.py index 6b21a4dd..b419d88d 100644 --- a/jsonargparse/core.py +++ b/jsonargparse/core.py @@ -282,13 +282,6 @@ def _parse_common( if not skip_check: self.check_config(cfg_ns) - if with_meta or (with_meta is None and self._default_meta): - if hasattr(cfg_ns, '__cwd__'): - if os.getcwd() not in cfg_ns.__cwd__: - cfg_ns.__cwd__.insert(0, os.getcwd()) - else: - cfg_ns.__cwd__ = [os.getcwd()] - if log_message is not None: self._logger.info(log_message) @@ -303,6 +296,7 @@ def parse_args( # type: ignore[override] defaults: bool = True, nested: bool = True, with_meta: bool = None, + _skip_check: bool = False, ) -> Union[Namespace, Dict[str, Any]]: """Parses command line argument strings. @@ -339,7 +333,7 @@ def parse_args( # type: ignore[override] defaults=defaults, nested=nested, with_meta=with_meta, - skip_check=False, + skip_check=_skip_check, cfg_base=namespace, log_message='Parsed command line arguments.', ) @@ -517,11 +511,6 @@ def parse_path( cfg_str = fpath.get_content() parsed_cfg = self.parse_string(cfg_str, cfg_path, ext_vars, env, defaults, nested, with_meta=with_meta, _skip_logging=True, _skip_check=_skip_check, _base=_base) - if with_meta or (with_meta is None and self._default_meta): - if self._parse_as_dict: - parsed_cfg['__path__'] = fpath - else: - parsed_cfg.__path__ = fpath # type: ignore finally: if not fpath.is_url: os.chdir(cwd) diff --git a/jsonargparse/jsonschema.py b/jsonargparse/jsonschema.py index 919c6c7f..159b358a 100644 --- a/jsonargparse/jsonschema.py +++ b/jsonargparse/jsonschema.py @@ -18,6 +18,7 @@ ParserError, strip_meta, import_object, + _load_config, _check_unknown_kwargs, _issubclass, ) @@ -26,7 +27,6 @@ jsonschemaValidationError, jsonschema_support, import_jsonschema, - get_config_read_mode, files_completer, argcomplete_warn_redraw_prompt, ) @@ -90,7 +90,7 @@ def __init__(self, **kwargs): try: schema = yaml.safe_load(schema) except (yamlParserError, yamlScannerError) as ex: - raise ValueError('Problems parsing schema :: '+str(ex)) + raise ValueError('Problems parsing schema :: '+str(ex)) from ex jsonvalidator.check_schema(schema) self._validator = self._extend_jsonvalidator_with_default(jsonvalidator)(schema) self._enable_path = kwargs.get('enable_path', True) @@ -136,18 +136,7 @@ def _check_type(self, value, cfg=None): value = [value] for num, val in enumerate(value): try: - fpath = None - if isinstance(val, str) and val.strip() != '': - parsed_val = yaml.safe_load(val) - if not isinstance(parsed_val, str): - val = parsed_val - if self._enable_path and isinstance(val, str): - try: - fpath = Path(val, mode=get_config_read_mode()) - except TypeError: - pass - else: - val = yaml.safe_load(fpath.get_content()) + val, fpath = _load_config(val, enable_path=self._enable_path, flat_namespace=False) if isinstance(val, Namespace): val = namespace_to_dict(val) val = self._adapt_types(val, self._annotation, self._subschemas, reverse=True) @@ -161,7 +150,7 @@ def _check_type(self, value, cfg=None): value[num] = val except (TypeError, yamlParserError, yamlScannerError, jsonschemaValidationError) as ex: elem = '' if not islist else ' element '+str(num+1) - raise TypeError('Parser key "'+self.dest+'"'+elem+': '+str(ex)) + raise TypeError('Parser key "'+self.dest+'"'+elem+': '+str(ex)) from ex return value if islist else value[0] @@ -236,7 +225,7 @@ def validate_adapt(v, subschema): init_args = parser.instantiate_subclasses(val['init_args']) val = val_class(**init_args) # pylint: disable=not-a-mapping except (ImportError, ModuleNotFound, AttributeError, AssertionError, ParserError) as ex: - raise ParserError('Problem with given class_path "'+val['class_path']+'" :: '+str(ex)) + raise ParserError('Problem with given class_path "'+val['class_path']+'" :: '+str(ex)) from ex return val elif annotation.__origin__ == Union: diff --git a/jsonargparse/signatures.py b/jsonargparse/signatures.py index ca559d8c..3aac1e74 100644 --- a/jsonargparse/signatures.py +++ b/jsonargparse/signatures.py @@ -5,7 +5,7 @@ from typing import Union, Optional, List, Container, Type, Callable from .util import _issubclass -from .actions import ActionEnum +from .actions import ActionEnum, _ActionConfigLoad from .typing import is_optional from .jsonschema import ActionJsonSchema from .optionals import docstring_parser_support, import_docstring_parse, dataclasses_support, import_dataclasses @@ -214,6 +214,8 @@ def update_has_args_kwargs(base, has_args=True, has_kwargs=True): doc_group = str(objects[0]) name = objects[0].__name__ if nested_key is None else nested_key group = self.add_argument_group(doc_group, name=name) # type: ignore + if nested_key is not None: + group.add_argument('--'+nested_key, action=_ActionConfigLoad) # type: ignore ## Add objects arguments ## added_args = set() diff --git a/jsonargparse/util.py b/jsonargparse/util.py index e398feb7..282efe8b 100644 --- a/jsonargparse/util.py +++ b/jsonargparse/util.py @@ -4,6 +4,7 @@ import re import sys import stat +import yaml import inspect import logging from copy import deepcopy @@ -13,7 +14,13 @@ from yaml.parser import ParserError as yamlParserError from yaml.scanner import ScannerError as yamlScannerError -from .optionals import ModuleNotFound, url_support, import_requests, import_url_validator +from .optionals import ( + ModuleNotFound, + url_support, + import_requests, + import_url_validator, + get_config_read_mode, +) __all__ = [ @@ -32,7 +39,7 @@ null_logger = logging.Logger('jsonargparse_null_logger') null_logger.addHandler(logging.NullHandler()) -meta_keys = {'__cwd__', '__path__', '__default_config__'} +meta_keys = {'__path__', '__default_config__'} class ParserError(Exception): @@ -40,6 +47,30 @@ class ParserError(Exception): pass +def _load_config(value, enable_path=True, flat_namespace=True): + """Parses yaml config in a string or a path""" + cfg_path = None + if isinstance(value, str) and value.strip() != '': + parsed_val = yaml.safe_load(value) + if not isinstance(parsed_val, str): + value = parsed_val + if enable_path and isinstance(value, str): + try: + cfg_path = Path(value, mode=get_config_read_mode()) + except TypeError: + pass + else: + value = yaml.safe_load(cfg_path.get_content()) + + if flat_namespace: + value = _dict_to_flat_namespace(value) + if cfg_path is not None: + setattr(value, '__path__', cfg_path) + return value + + return value, cfg_path + + def _get_key_value(cfg, key, parent=False): """Gets the value for a given key in a config object (dict or argparse.Namespace).""" def key_in_cfg(cfg, key): diff --git a/jsonargparse_tests/core_tests.py b/jsonargparse_tests/core_tests.py index a5a0abe9..014c5e37 100755 --- a/jsonargparse_tests/core_tests.py +++ b/jsonargparse_tests/core_tests.py @@ -94,8 +94,6 @@ def test_parse_path(self): self.assertEqual(cfg2, parser.parse_path(yaml_file, defaults=False)) self.assertNotEqual(cfg2, parser.parse_path(yaml_file, defaults=True)) self.assertNotEqual(cfg1, parser.parse_path(yaml_file, defaults=False)) - self.assertTrue(hasattr(parser.parse_path(yaml_file, with_meta=True), '__cwd__')) - self.assertFalse(hasattr(parser.parse_path(yaml_file), '__cwd__')) with open(yaml_file, 'w') as output_file: output_file.write(example_yaml+' val2: eight\n') @@ -191,7 +189,6 @@ def test_parse_as_dict(self): with open('config.json', 'w') as f: f.write('{}') parser = ArgumentParser(parse_as_dict=True, default_meta=True) - self.assertEqual({'__path__', '__cwd__'}, set(parser.parse_path('config.json').keys())) class ArgumentFeaturesTests(unittest.TestCase): @@ -576,7 +573,6 @@ def test_save(self): cfg2 = parser.parse_path(main_file, with_meta=True) self.assertEqual(namespace_to_dict(cfg1), strip_meta(cfg2)) - self.assertEqual(cfg2.__path__(), main_file) self.assertEqual(cfg2.parser.__path__(absolute=False), 'parser.yaml') if jsonschema_support: self.assertEqual(cfg2.schema.__path__(absolute=False), 'schema.yaml') @@ -683,9 +679,6 @@ def test_ActionConfigFile_and_ActionPath(self): self.assertEqual(abs_yaml_file, os.path.realpath(cfg.file(absolute=True))) self.assertRaises(ParserError, lambda: parser.parse_args(['--cfg', abs_yaml_file+'~'])) - cfg = parser.parse_args(['--cfg', abs_yaml_file, '--cfg', abs_yaml_file]) - self.assertEqual(3, len(cfg.__cwd__)) - cfg = parser.parse_args(['--cfg', 'file: '+abs_yaml_file+'\ndir: '+self.tmpdir+'\n']) self.assertEqual(self.tmpdir, os.path.realpath(cfg.dir(absolute=True))) self.assertEqual(None, cfg.cfg[0]) diff --git a/jsonargparse_tests/signatures_tests.py b/jsonargparse_tests/signatures_tests.py index 7b9a655c..3d8a6635 100755 --- a/jsonargparse_tests/signatures_tests.py +++ b/jsonargparse_tests/signatures_tests.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import yaml from enum import Enum from io import StringIO from typing import Dict, List, Tuple, Optional, Union, Any @@ -420,5 +421,56 @@ def __init__(self, a1: int = 1): self.assertNotIn('a1 description', help_str.getvalue()) +@unittest.skipIf(not jsonschema_support, 'jsonschema package is required') +class SignaturesConfigTests(TempDirTestCase): + + def test_add_function_arguments_config(self): + + def func(a1 = '1', + a2: float = 2.0, + a3: bool = False): + return a1 + + parser = ArgumentParser(error_handler=None, default_meta=False) + parser.add_function_arguments(func, 'func') + + cfg_path = 'config.yaml' + with open(cfg_path, 'w') as f: + f.write(yaml.dump({'a1': 'one', 'a3': True})) + + cfg = parser.parse_args(['--func', cfg_path]) + self.assertEqual(cfg.func, Namespace(a1='one', a2=2.0, a3=True)) + + cfg = parser.parse_args(['--func={"a1": "ONE"}']) + self.assertEqual(cfg.func, Namespace(a1='ONE', a2=2.0, a3=False)) + + self.assertRaises(ParserError, lambda: parser.parse_args(['--func="""'])) + + + def test_config_within_config(self): + + def func(a1 = '1', + a2: float = 2.0, + a3: bool = False): + return a1 + + parser = ArgumentParser(error_handler=None) + parser.add_argument('--cfg', action=ActionConfigFile) + parser.add_function_arguments(func, 'func') + + cfg_path = 'subdir/config.yaml' + subcfg_path = 'subsubdir/func_config.yaml' + os.mkdir('subdir') + os.mkdir('subdir/subsubdir') + with open(cfg_path, 'w') as f: + f.write('func: '+subcfg_path+'\n') + with open(os.path.join('subdir', subcfg_path), 'w') as f: + f.write(yaml.dump({'a1': 'one', 'a3': True})) + + cfg = parser.parse_args(['--cfg', cfg_path]) + self.assertEqual(str(cfg.func.__path__), subcfg_path) + self.assertEqual(strip_meta(cfg.func), {'a1': 'one', 'a2': 2.0, 'a3': True}) + + if __name__ == '__main__': unittest.main(verbosity=2)