diff --git a/changelogs/fragments/1.2.0.yml b/changelogs/fragments/1.2.0.yml new file mode 100644 index 0000000..512efc9 --- /dev/null +++ b/changelogs/fragments/1.2.0.yml @@ -0,0 +1 @@ +release_summary: Feature release. diff --git a/changelogs/fragments/7-config.yml b/changelogs/fragments/7-config.yml new file mode 100644 index 0000000..98d145b --- /dev/null +++ b/changelogs/fragments/7-config.yml @@ -0,0 +1,2 @@ +minor_changes: + - "Make config file management more flexible to allow project-specific config file format extensions for the explicitly passed configuration files (https://github.com/ansible-community/antsibull-core/pull/7)." diff --git a/pyproject.toml b/pyproject.toml index 7fcd59f..1fe137c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "antsibull-core" -version = "1.1.0.post0" +version = "1.2.0" description = "Tools for building the Ansible Distribution" authors = ["Toshio Kuratomi ", "Felix Fontein "] license = "GPL-3.0-or-later" diff --git a/src/antsibull_core/app_context.py b/src/antsibull_core/app_context.py index 68b4451..0420ad6 100644 --- a/src/antsibull_core/app_context.py +++ b/src/antsibull_core/app_context.py @@ -29,31 +29,80 @@ Setup ===== -Importing antsibull.app_context will setup a default context with default values for the library to -use. The application should initialize a new context with user overriding values by calling -:func:`antsibull.app_context.create_contexts` with command line args and configuration data. The -data from those will be used to initialize a new app_ctx and new lib_ctx. The application can then -use the context managers to utilize these contexts before calling any further antsibull code. An -example: +Importing antsibull_core.app_context will setup a default context with default values for the +library to use. The application should initialize a new context with user overriding values by +calling :func:`antsibull_core.app_context.create_contexts` with command line args and +configuration data. The data from those will be used to initialize a new ``app_ctx`` and new +``lib_ctx``. The application can then use the context managers to utilize these contexts before +calling any further antsibull code. An example: .. code-block:: python + from antsibull_core import app_context + from antsibull_core.config import load_config + + def do_something(): app_ctx = app_context.app_ctx.get() core_filename = download_python_package('ansible-core', server_url=app_ctx.pypi_url) return core_filename def run(args): - args = parsre_args(args) + args = parse_args(args) cfg = load_config(args.config_file) context_data = app_context.create_contexts(args=args, cfg=cfg) + with app_context.app_and_lib_context(context_data): + do_something() + +Extending AppContext +==================== + +Since antsibull-core 1.2.0, applications using antsibull-core can use their own extension +of AppContext to better handle command line arguments, or handle additional configuration +values in explicitly specified configuration files. (The implicitly specified configuration +files, ``/etc/antsibull.cfg`` and ``~/.antsibull.cfg``, cannot have extra keys to prevent +incompatibility with other antsibull-core based applications.) + +For this, the application needs to create a derived class of :obj:`AppContext`, and pass +it to :func:`antsibull_core.config.load_config` when loading the configuration. Please +note that :func:`antsibull_core.app_context.app_ctx.get` always returns the :obj:`AppContext` +view of that extended app context, so applications should create their own ``app_context`` +module that provides itself a way to obtain the extended app context. For example in +antsibull-docs this is done as follows: + +.. code-block:: python + + from antsibull_core.app_context import AppContextWrapper + from antsibull_docs.schemas.app_context import DocsAppContext + + app_ctx: AppContextWrapper[DocsAppContext] = AppContextWrapper() + +In antsibull-docs, the extended app context (``DocsAppContext``) is then used as follows: + +.. code-block:: python + + from antsibull_core.app_context import app_and_lib_context, create_contexts + from antsibull_core.config import load_config + + from antsibull_docs import app_context + from antsibull_docs.schemas.app_context import DocsAppContext + + + def do_something(): + app_ctx = app_context.app_ctx.get() + # collection_url is an option in DocsAppContext + print(f'collection_url configuration: {app_ctx.collection_url}') + + def run(args): + args = parse_args(args) + cfg = load_config(args.config_file, app_context_model=DocsAppContext) + context_data = create_contexts(args=args, cfg=cfg) with app_and_lib_context(context_data): do_something() """ import argparse import contextvars -import functools import sys import typing as t from contextlib import contextmanager @@ -67,13 +116,8 @@ def run(args): import aiocontextvars # noqa: F401, pylint:disable=unused-import -#: Field names in the args and config whose value will be added to the app_ctx -_FIELDS_IN_APP_CTX = frozenset(('ansible_base_url', 'breadcrumbs', 'galaxy_url', 'indexes', - 'logging_cfg', 'pypi_url', 'use_html_blobs', 'collection_cache')) +AppContextT = t.TypeVar('AppContextT', bound=AppContext) -#: Field names in the args and config whose value will be added to the lib_ctx -_FIELDS_IN_LIB_CTX = frozenset( - ('chunksize', 'doc_parsing_backend', 'max_retries', 'process_max', 'thread_max')) #: lib_ctx should be restricted to things which do not belong in the API but an application or #: user might want to tweak. Global, internal, incidental values are good to store here. Things @@ -109,9 +153,9 @@ def run(args): app_ctx: 'contextvars.ContextVar[AppContext]' = contextvars.ContextVar('app_ctx') -class ContextReturn(t.NamedTuple): +class ContextReturn(t.Generic[AppContextT]): """ - NamedTuple for the return value of :func:`create_contexts`. + NamedTuple-like object for the return value of :func:`create_contexts`. The :func:`create_contexts` returns quite a bit of information. This data structure organizes the information. @@ -122,13 +166,40 @@ class ContextReturn(t.NamedTuple): :ivar args: An :python:obj:`argparse.Namespace` containing command line arguments that were not used to construct the contexts :ivar cfg: Configuration values which were not used to construct the contexts. + + .. note:: unfortunately generic ``NamedTuple`` objects are not possible, so this is a generic + class that tries to behave as close as possible to a named tuple. Right now it does + not support comparisons though, if that is needed please create an issue in the + antsibull-core repository. """ - app_ctx: AppContext + app_ctx: AppContextT lib_ctx: LibContext args: argparse.Namespace cfg: t.Dict + # pylint: disable-next=redefined-outer-name + def __init__(self, app_ctx: AppContextT, lib_ctx: LibContext, + args: argparse.Namespace, cfg: t.Dict): + self.app_ctx = app_ctx + self.lib_ctx = lib_ctx + self.args = args + self.cfg = cfg + + def __getitem__(self, index: int) -> t.Any: + if index == 0: + return self.app_ctx + if index == 1: + return self.lib_ctx + if index == 2: + return self.args + if index == 3: + return self.cfg + raise IndexError('tuple index out of range') + + def __iter__(self) -> t.Iterable: + return (self.app_ctx, self.lib_ctx, self.args, self.cfg).__iter__() + def _extract_context_values(known_fields, args: t.Optional[argparse.Namespace], cfg: t.Mapping = ImmutableDict()) -> t.Dict: @@ -152,13 +223,30 @@ def _extract_context_values(known_fields, args: t.Optional[argparse.Namespace], return context_values -_extract_lib_context_values = functools.partial(_extract_context_values, _FIELDS_IN_LIB_CTX) -_extract_app_context_values = functools.partial(_extract_context_values, _FIELDS_IN_APP_CTX) +@t.overload +def create_contexts(args: t.Optional[argparse.Namespace] = None, + cfg: t.Mapping = ImmutableDict(), + use_extra: bool = True, + ) -> ContextReturn[AppContext]: + ... +@t.overload def create_contexts(args: t.Optional[argparse.Namespace] = None, cfg: t.Mapping = ImmutableDict(), - use_extra: bool = True) -> ContextReturn: + use_extra: bool = True, + *, + app_context_model: t.Type[AppContextT], + ) -> ContextReturn[AppContextT]: + ... + + +def create_contexts(args: t.Optional[argparse.Namespace] = None, + cfg: t.Mapping = ImmutableDict(), + use_extra: bool = True, + *, + app_context_model=AppContext, + ) -> ContextReturn: """ Create new contexts appropriate for setting the app and lib context. @@ -172,7 +260,9 @@ def create_contexts(args: t.Optional[argparse.Namespace] = None, :kwarg use_extra: When True, the default, all extra arguments and config values will be set as fields in ``app_ctx.extra``. When False, the extra arguments and config values will be returned as part of the ContextReturn. - :returns: A ContextReturn NamedTuple. + :kwarg app_context_model: The model to use for the app context. Must be derived from + :obj:`AppContext`. If not provided, will use :obj:`AppContext` itself. + :returns: A ``ContextReturn`` object. .. warning:: We cannot tell whether a user set a value via the command line if :python:mod:`argparse` @@ -185,13 +275,16 @@ def create_contexts(args: t.Optional[argparse.Namespace] = None, If the field is only used via the :attr:`AppContext.extra` mechanism (not explictly set), then you should ignore this section and use :python:mod:`argparse`'s default mechanism. """ - lib_values = _extract_lib_context_values(args, cfg) - app_values = _extract_app_context_values(args, cfg) + fields_in_lib_ctx = set(LibContext.__fields__) + fields_in_app_ctx = set(app_context_model.__fields__) + known_fields = fields_in_app_ctx.union(fields_in_lib_ctx) + + lib_values = _extract_context_values(fields_in_lib_ctx, args, cfg) + app_values = _extract_context_values(fields_in_app_ctx, args, cfg) # # Save the unused values # - known_fields = _FIELDS_IN_APP_CTX.union(_FIELDS_IN_LIB_CTX) unused_cfg = {} if cfg: @@ -211,7 +304,7 @@ def create_contexts(args: t.Optional[argparse.Namespace] = None, unused_args_ns = argparse.Namespace(**unused_args) # create new app and lib ctxt from the application's arguments and config. - local_app_ctx = AppContext(**app_values) + local_app_ctx = app_context_model(**app_values) local_lib_ctx = LibContext(**lib_values) return ContextReturn( @@ -255,13 +348,24 @@ def lib_context(new_context: t.Optional[LibContext] = None) -> t.Generator[LibCo lib_ctx.reset(reset_token) +@t.overload @contextmanager -def app_context(new_context: t.Optional[AppContext] = None) -> t.Generator[AppContext, None, None]: +def app_context() -> t.ContextManager[AppContext]: + ... + + +@t.overload +@contextmanager +def app_context(new_context: AppContextT) -> t.ContextManager[AppContextT]: + ... + + +@contextmanager +def app_context(new_context=None): """ Set up a new app_context. - :kwarg new_context: New app context to setup. If this is None, the context is set to a copy of - the old context. + :kwarg new_context: New app context to setup. """ if new_context is None: new_context = _copy_app_context() @@ -273,8 +377,8 @@ def app_context(new_context: t.Optional[AppContext] = None) -> t.Generator[AppCo @contextmanager -def app_and_lib_context(context_data: ContextReturn - ) -> t.Generator[t.Tuple[AppContext, LibContext], None, None]: +def app_and_lib_context(context_data: ContextReturn[AppContextT] + ) -> t.Generator[t.Tuple[AppContextT, LibContext], None, None]: """ Set the app and lib context at the same time. @@ -293,6 +397,19 @@ def app_and_lib_context(context_data: ContextReturn yield (new_app_ctx, new_lib_ctx) +class AppContextWrapper(t.Generic[AppContextT]): + def __repr__(self): + return "" + + @property + def name(self): + return 'app_ctx' + + @staticmethod + def get() -> AppContextT: + return t.cast(AppContextT, app_ctx.get()) + + # # Set initial contexts with default values # diff --git a/src/antsibull_core/config.py b/src/antsibull_core/config.py index cd49dd7..829a1af 100644 --- a/src/antsibull_core/config.py +++ b/src/antsibull_core/config.py @@ -5,14 +5,15 @@ # SPDX-FileCopyrightText: 2020, Ansible Project """Functions to handle config files.""" -import itertools import os.path import typing as t import perky # type: ignore[import] +import pydantic as p from .logging import log from .schemas.config import ConfigModel +from .schemas.context import AppContext, LibContext mlog = log.fields(mod=__name__) @@ -24,6 +25,10 @@ USER_CONFIG_FILE = '~/.antsibull.cfg' +class ConfigError(Exception): + pass + + def find_config_files(conf_files: t.Iterable[str]) -> t.List[str]: """ Find all config files that exist. @@ -70,7 +75,49 @@ def read_config(filename: str) -> ConfigModel: return raw_config_data -def load_config(conf_files: t.Union[t.Iterable[str], str, None] = None) -> t.Dict: +def validate_config(config: t.Mapping, filenames: t.List[str], + app_context_model: t.Type[AppContext]) -> None: + """ + Validate configuration. + + Given the configuration loaded from one or more configuration files and the app context model, + splits up the config into lib context and app context part and validates both parts with the + given model. Raises a :obj:`ConfigError` if validation fails. + """ + lib_fields = set(LibContext.__fields__) + lib = {} + app = {} + for key, value in config.items(): + if key in lib_fields: + lib[key] = value + else: + app[key] = value + # Note: We parse the object but discard the model because we want to validate the config but let + # the context handle all setting of defaults + try: + LibContext.parse_obj(lib) + app_context_model.parse_obj(app) + except p.ValidationError as exc: + raise ConfigError( + f"Error while parsing configuration from {', '.join(filenames)}:\n{exc}") from exc + + +def _load_config_file(filename: str) -> t.Mapping: + """ + Load configuration from one file and return the raw data. + """ + try: + return perky.load(filename) + except OSError as exc: + raise ConfigError( + f"Error while loading configuration from {filename}: {exc}") from exc + except perky.PerkyFormatError as exc: + raise ConfigError( + f"Error while parsing configuration from {filename}:\n{exc}") from exc + + +def load_config(conf_files: t.Union[t.Iterable[str], str, None] = None, + app_context_model: t.Type[AppContext] = AppContext) -> t.Dict: """ Load configuration. @@ -80,6 +127,8 @@ def load_config(conf_files: t.Union[t.Iterable[str], str, None] = None) -> t.Dic those same keys in earlier files. :arg conf_files: An iterable of conf_files to load configuration information from. + :kwarg app_context_model: The model to use for the app context. Must be derived from + :obj:`AppContext`. If not provided, will use :obj:`AppContext` itself. :returns: A dict containing the configuration. """ flog = mlog.fields(func='load_config') @@ -90,29 +139,26 @@ def load_config(conf_files: t.Union[t.Iterable[str], str, None] = None) -> t.Dic elif conf_files is None: conf_files = () - user_config_file = os.path.expanduser(USER_CONFIG_FILE) - available_files = find_config_files(itertools.chain((SYSTEM_CONFIG_FILE, user_config_file), - conf_files)) + implicit_files = find_config_files((SYSTEM_CONFIG_FILE, os.path.expanduser(USER_CONFIG_FILE))) + explicit_files = find_config_files(conf_files) - includes = list(available_files) + flog.fields(implicit_files=implicit_files, + explicit_files=explicit_files).debug('found config files') - flog.debug('loading config files') - # Perky has some bugs that prevent this simple way from working: - # https://github.com/ansible-community/antsibull/pull/118 - # cfg = {'includes': includes} - # cfg = perky.includes(cfg, recursive=True) + flog.debug('loading implicit config files') + cfg: t.Dict = {} + for filename in implicit_files: + cfg.update(_load_config_file(filename)) - # Workaround for above bug. Note that includes specified in the config files will not work - # but we can just add that as a new feature when perky gets it working. - cfg = {} - for filename in includes: - new_cfg = perky.load(filename) - cfg.update(new_cfg) + flog.debug('validating implicit configuration') + validate_config(cfg, implicit_files, AppContext) - flog.debug('validating configuration') - # Note: We parse the object but discard the model because we want to validate the config but let - # the context handle all setting of defaults - ConfigModel.parse_obj(cfg) + flog.debug('loading explicit config files') + for filename in explicit_files: + cfg.update(_load_config_file(filename)) + + flog.debug('validating combined configuration') + validate_config(cfg, implicit_files + explicit_files, app_context_model) flog.fields(config=cfg).debug('Leave') return cfg diff --git a/src/antsibull_core/schemas/config.py b/src/antsibull_core/schemas/config.py index a7ed589..9332fa2 100644 --- a/src/antsibull_core/schemas/config.py +++ b/src/antsibull_core/schemas/config.py @@ -116,6 +116,7 @@ class LoggingModel(BaseModel): }) +# This class is no longer needed, it will eventually be removed class ConfigModel(BaseModel): # pyre-ignore[8]: https://github.com/samuelcolvin/pydantic/issues/1684 ansible_base_url: p.HttpUrl = 'https://github.com/ansible/ansible' # type: ignore[assignment] diff --git a/src/antsibull_core/schemas/context.py b/src/antsibull_core/schemas/context.py index d99e93a..673119c 100644 --- a/src/antsibull_core/schemas/context.py +++ b/src/antsibull_core/schemas/context.py @@ -73,10 +73,12 @@ class AppContext(BaseModel): use_html_blobs: p.StrictBool = False collection_cache: t.Optional[str] = None - _convert_bools = p.validator('breadcrumbs', 'indexes', 'use_html_blobs', - pre=True, allow_reuse=True)(convert_bool) - _convert_paths = p.validator('collection_cache', - pre=True, allow_reuse=True)(convert_path) + # pylint: disable-next=unused-private-member + __convert_bools = p.validator('breadcrumbs', 'indexes', 'use_html_blobs', + pre=True, allow_reuse=True)(convert_bool) + # pylint: disable-next=unused-private-member + __convert_paths = p.validator('collection_cache', + pre=True, allow_reuse=True)(convert_path) class LibContext(BaseModel): @@ -106,4 +108,5 @@ class LibContext(BaseModel): thread_max: int = 8 file_check_content: int = 262144 - _convert_nones = p.validator('process_max', pre=True, allow_reuse=True)(convert_none) + # pylint: disable-next=unused-private-member + __convert_nones = p.validator('process_max', pre=True, allow_reuse=True)(convert_none)