From 1da1d5641534d1b2f3914185e60f388baa1ef319 Mon Sep 17 00:00:00 2001 From: gmuloc Date: Wed, 27 Sep 2023 11:37:41 +0200 Subject: [PATCH 01/53] Feat: Add check command --- anta/cli/__init__.py | 22 +++++++++++----- anta/cli/check/__init__.py | 3 +++ anta/cli/check/commands.py | 53 ++++++++++++++++++++++++++++++++++++++ anta/cli/utils.py | 4 +++ 4 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 anta/cli/check/__init__.py create mode 100644 anta/cli/check/commands.py diff --git a/anta/cli/__init__.py b/anta/cli/__init__.py index b725d56ca..8d486c95a 100644 --- a/anta/cli/__init__.py +++ b/anta/cli/__init__.py @@ -15,10 +15,11 @@ import click from anta import __version__ +from anta.cli.check import commands as check_commands from anta.cli.debug import commands as debug_commands from anta.cli.exec import commands as exec_commands from anta.cli.get import commands as get_commands -from anta.cli.nrfu import commands as check_commands +from anta.cli.nrfu import commands as nrfu_commands from anta.cli.utils import AliasedGroup, IgnoreRequiredWithHelp, parse_catalog, parse_inventory from anta.loader import setup_logging from anta.result_manager import ResultManager @@ -133,6 +134,7 @@ def anta( ctx.ensure_object(dict) ctx.obj["inventory"] = parse_inventory(ctx, inventory) + ctx.obj["inventory_path"] = ctx.params["inventory"] @anta.group("nrfu", cls=IgnoreRequiredWithHelp) @@ -152,6 +154,11 @@ def _nrfu(ctx: click.Context, catalog: list[tuple[Callable[..., TestResult], dic ctx.obj["result_manager"] = ResultManager() +@anta.group("check", cls=AliasedGroup) +def _check() -> None: + """Check commands for building ANTA""" + + @anta.group("exec", cls=AliasedGroup) def _exec() -> None: """Execute commands to inventory devices""" @@ -170,11 +177,14 @@ def _debug() -> None: # Load group commands # Prefixing with `_` for avoiding the confusion when importing anta.cli.debug.commands as otherwise the debug group has # a commands attribute. +_check.add_command(check_commands.catalog) +# Inventory cannot be implemented for now as main 'anta' CLI is already parsing it +# _check.add_command(check_commands.inventory) + _exec.add_command(exec_commands.clear_counters) _exec.add_command(exec_commands.snapshot) _exec.add_command(exec_commands.collect_tech_support) - _get.add_command(get_commands.from_cvp) _get.add_command(get_commands.from_ansible) _get.add_command(get_commands.inventory) @@ -183,10 +193,10 @@ def _debug() -> None: _debug.add_command(debug_commands.run_cmd) _debug.add_command(debug_commands.run_template) -_nrfu.add_command(check_commands.table) -_nrfu.add_command(check_commands.json) -_nrfu.add_command(check_commands.text) -_nrfu.add_command(check_commands.tpl_report) +_nrfu.add_command(nrfu_commands.table) +_nrfu.add_command(nrfu_commands.json) +_nrfu.add_command(nrfu_commands.text) +_nrfu.add_command(nrfu_commands.tpl_report) # ANTA CLI Execution diff --git a/anta/cli/check/__init__.py b/anta/cli/check/__init__.py new file mode 100644 index 000000000..c460d5493 --- /dev/null +++ b/anta/cli/check/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2023 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. diff --git a/anta/cli/check/commands.py b/anta/cli/check/commands.py new file mode 100644 index 000000000..eba1a763e --- /dev/null +++ b/anta/cli/check/commands.py @@ -0,0 +1,53 @@ +# Copyright (c) 2023 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +# pylint: disable = redefined-outer-name + +""" +Commands for Anta CLI to run check commands. +""" +from __future__ import annotations + +import logging + +import click + +from anta.cli.console import console +from anta.cli.utils import parse_catalog +from anta.device import AsyncEOSDevice +from anta.models import AntaTest +from anta.result_manager import ResultManager + +logger = logging.getLogger(__name__) + + +@click.command(no_args_is_help=True) +@click.pass_context +@click.option( + "--catalog", + "-c", + show_envvar=True, + help="Path to the tests catalog YAML file", + type=click.Path(file_okay=True, dir_okay=False, exists=True, readable=True, resolve_path=True), + required=True, + callback=parse_catalog, +) +def catalog(ctx: click.Context, catalog: list[tuple[AntaTest, AntaTest.Input]]) -> None: + """ + Check that the catalog is valid + """ + logger.info(f"Checking syntax of catalog {ctx.obj['catalog_path']}") + mock_device = AsyncEOSDevice(name="mock", host="127.0.0.1", username="mock", password="mock") + manager = ResultManager() + # Instantiate each test to verify the Inputs are correct + for test_class, test_inputs in catalog: + # TODO - this is the same code with typing as in runner.py but somehow mypy complains that test_class + # ot type AntaTest is not callable + test_instance = test_class(device=mock_device, inputs=test_inputs) # type: ignore[operator] + manager.add_test_result(test_instance.result) + if manager.error_status: + console.print(f"[bold][red]Catalog {ctx.obj['catalog_path']} is invalid") + # TODO print nice report + ctx.exit(1) + else: + console.print(f"[bold][green]Catalog {ctx.obj['catalog_path']} is valid") diff --git a/anta/cli/utils.py b/anta/cli/utils.py index 4c16793b2..97bb033c4 100644 --- a/anta/cli/utils.py +++ b/anta/cli/utils.py @@ -86,6 +86,8 @@ def parse_catalog(ctx: click.Context, param: Option, value: str) -> list[tuple[A # pylint: disable=unused-argument """ Click option callback to parse an ANTA tests catalog YAML file + + Store the orignal value (catalog path) in the ctx.obj """ if ctx.obj.get("_anta_help"): # Currently looking for help for a subcommand so no @@ -101,6 +103,8 @@ def parse_catalog(ctx: click.Context, param: Option, value: str) -> list[tuple[A anta_log_exception(e, message, logger) ctx.fail(message) + # Storing catalog path + ctx.obj["catalog_path"] = value return anta.loader.parse_catalog(data) From cd6c2224cbb827a6231ff9149e4b53af2483166f Mon Sep 17 00:00:00 2001 From: gmuloc Date: Wed, 27 Sep 2023 11:46:47 +0200 Subject: [PATCH 02/53] Feat: Check catalog funcionality --- anta/catalog.py | 32 ++++++++++++++++++++++++++++++++ anta/cli/check/commands.py | 12 ++---------- 2 files changed, 34 insertions(+), 10 deletions(-) create mode 100644 anta/catalog.py diff --git a/anta/catalog.py b/anta/catalog.py new file mode 100644 index 000000000..ea5926c18 --- /dev/null +++ b/anta/catalog.py @@ -0,0 +1,32 @@ +# Copyright (c) 2023 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +""" +Catalog related functions +""" +from __future__ import annotations + +import logging + +from anta.device import AsyncEOSDevice +from anta.models import AntaTest +from anta.result_manager import ResultManager + +logger = logging.getLogger(__name__) + + +def is_catalog_valid(catalog: list[tuple[AntaTest, AntaTest.Input]]) -> ResultManager: + """ + TODO - for now a test requires a device but this may be revisited in the future + """ + # Mock device + mock_device = AsyncEOSDevice(name="mock", host="127.0.0.1", username="mock", password="mock") + + manager = ResultManager() + # Instantiate each test to verify the Inputs are correct + for test_class, test_inputs in catalog: + # TODO - this is the same code with typing as in runner.py but somehow mypy complains that test_class + # ot type AntaTest is not callable + test_instance = test_class(device=mock_device, inputs=test_inputs) # type: ignore[operator] + manager.add_test_result(test_instance.result) + return manager diff --git a/anta/cli/check/commands.py b/anta/cli/check/commands.py index eba1a763e..070699f00 100644 --- a/anta/cli/check/commands.py +++ b/anta/cli/check/commands.py @@ -12,11 +12,10 @@ import click +from anta.catalog import is_catalog_valid from anta.cli.console import console from anta.cli.utils import parse_catalog -from anta.device import AsyncEOSDevice from anta.models import AntaTest -from anta.result_manager import ResultManager logger = logging.getLogger(__name__) @@ -37,14 +36,7 @@ def catalog(ctx: click.Context, catalog: list[tuple[AntaTest, AntaTest.Input]]) Check that the catalog is valid """ logger.info(f"Checking syntax of catalog {ctx.obj['catalog_path']}") - mock_device = AsyncEOSDevice(name="mock", host="127.0.0.1", username="mock", password="mock") - manager = ResultManager() - # Instantiate each test to verify the Inputs are correct - for test_class, test_inputs in catalog: - # TODO - this is the same code with typing as in runner.py but somehow mypy complains that test_class - # ot type AntaTest is not callable - test_instance = test_class(device=mock_device, inputs=test_inputs) # type: ignore[operator] - manager.add_test_result(test_instance.result) + manager = is_catalog_valid(catalog) if manager.error_status: console.print(f"[bold][red]Catalog {ctx.obj['catalog_path']} is invalid") # TODO print nice report From 2bb0c65c13bd678241ca8960d0841705e62370de Mon Sep 17 00:00:00 2001 From: gmuloc Date: Mon, 9 Oct 2023 15:38:07 +0200 Subject: [PATCH 03/53] WIP --- anta/catalog.py | 146 +++++++++++++++++++++++++++++++++---- anta/cli/__init__.py | 6 +- anta/cli/check/commands.py | 7 +- anta/cli/utils.py | 20 ++--- anta/loader.py | 88 +--------------------- anta/runner.py | 11 +-- 6 files changed, 154 insertions(+), 124 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index ea5926c18..ccb3c7fb0 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -6,7 +6,11 @@ """ from __future__ import annotations +import importlib import logging +from typing import Any, Optional, cast + +from yaml import safe_load from anta.device import AsyncEOSDevice from anta.models import AntaTest @@ -15,18 +19,134 @@ logger = logging.getLogger(__name__) -def is_catalog_valid(catalog: list[tuple[AntaTest, AntaTest.Input]]) -> ResultManager: +class AntaCatalog: """ - TODO - for now a test requires a device but this may be revisited in the future + Class representing an ANTA Catalog + + Attributes: + name: Catalog name + filename Optional[str]: The path from which the catalog was loaded + tests: list[tuple[AntaTest, AntaTest.Input]]: A list of Tuple containing an AntaTest and the associated input """ - # Mock device - mock_device = AsyncEOSDevice(name="mock", host="127.0.0.1", username="mock", password="mock") - - manager = ResultManager() - # Instantiate each test to verify the Inputs are correct - for test_class, test_inputs in catalog: - # TODO - this is the same code with typing as in runner.py but somehow mypy complains that test_class - # ot type AntaTest is not callable - test_instance = test_class(device=mock_device, inputs=test_inputs) # type: ignore[operator] - manager.add_test_result(test_instance.result) - return manager + + def __init__(self, name: str, filename: Optional[str] = None) -> None: + """ + Constructor of AntaCatalog + + Args: + name: Device name + filname: Optional name - if provided tests are loaded + """ + self.name: str = name + self.filename: Optional[str] = filename + self.tests: list[tuple[AntaTest, AntaTest.Input]] = [] + + def parse_catalog_file(self: AntaCatalog) -> None: + """ + Parse a file + """ + if self.filename is None: + return + try: + with open(self.filename, "r", encoding="UTF-8") as file: + data = safe_load(file) + self.parse_catalog(data) + # pylint: disable-next=broad-exception-caught + except Exception: + logger.critical(f"Something went wrong while parsing {self.filename}") + raise + + def parse_catalog(self: AntaCatalog, test_catalog: dict[str, Any], package: str | None = None) -> None: + """ + Function to parse the catalog and return a list of tests with their inputs + + A valid test catalog must follow the following structure: + : + - : + + + Example: + anta.tests.connectivity: + - VerifyReachability: + hosts: + - dst: 8.8.8.8 + src: 172.16.0.1 + - dst: 1.1.1.1 + src: 172.16.0.1 + result_overwrite: + categories: + - "Overwritten category 1" + description: "Test with overwritten description" + custom_field: "Test run by John Doe" + + Also supports nesting for Python module definition: + anta.tests: + connectivity: + - VerifyReachability: + hosts: + - dst: 8.8.8.8 + src: 172.16.0.1 + - dst: 1.1.1.1 + src: 172.16.0.1 + result_overwrite: + categories: + - "Overwritten category 1" + description: "Test with overwritten description" + custom_field: "Test run by John Doe" + + Args: + test_catalog: Python dictionary representing the test catalog YAML file + + """ + # pylint: disable=broad-exception-raised + if not test_catalog: + return + + for key, value in test_catalog.items(): + # Required to manage iteration within a tests module + if package is not None: + key = ".".join([package, key]) + try: + module = importlib.import_module(f"{key}") + except ModuleNotFoundError: + logger.critical(f"No test module named '{key}'") + raise + + if isinstance(value, list): + # This is a list of tests + for test in value: + for test_name, inputs in test.items(): + # A test must be a subclass of AntaTest as defined in the Python module + try: + test = getattr(module, test_name) + except AttributeError: + logger.critical(f"Wrong test name '{test_name}' in '{module.__name__}'") + raise + if not issubclass(test, AntaTest): + logger.critical(f"'{test.__module__}.{test.__name__}' is not an AntaTest subclass") + raise Exception() + # Test inputs can be either None or a dictionary + if inputs is None or isinstance(inputs, dict): + self.tests.append((cast(AntaTest, test), cast(AntaTest.Input, inputs))) + else: + logger.critical(f"'{test.__module__}.{test.__name__}' inputs must be a dictionary") + raise Exception() + if isinstance(value, dict): + # This is an inner Python module + self.parse_catalog(value, package=module.__name__) + + def check(self: AntaCatalog) -> ResultManager: + """ + TODO - for now a test requires a device but this may be revisited in the future + """ + # Mock device + mock_device = AsyncEOSDevice(name="mock", host="127.0.0.1", username="mock", password="mock") + + manager = ResultManager() + # Instantiate each test to verify the Inputs are correct + for test_class, test_inputs in self.tests: + # TODO - this is the same code with typing as in runner.py but somehow mypy complains that test_class + # ot type AntaTest is not callable + test_instance = test_class(device=mock_device, inputs=test_inputs) # type: ignore[operator] + manager.add_test_result(test_instance.result) + return manager diff --git a/anta/cli/__init__.py b/anta/cli/__init__.py index 8d486c95a..c26866462 100644 --- a/anta/cli/__init__.py +++ b/anta/cli/__init__.py @@ -10,11 +10,12 @@ import logging import pathlib -from typing import Any, Callable, Literal +from typing import Any, Literal import click from anta import __version__ +from anta.catalog import AntaCatalog from anta.cli.check import commands as check_commands from anta.cli.debug import commands as debug_commands from anta.cli.exec import commands as exec_commands @@ -23,7 +24,6 @@ from anta.cli.utils import AliasedGroup, IgnoreRequiredWithHelp, parse_catalog, parse_inventory from anta.loader import setup_logging from anta.result_manager import ResultManager -from anta.result_manager.models import TestResult @click.group(cls=IgnoreRequiredWithHelp) @@ -148,7 +148,7 @@ def anta( required=True, callback=parse_catalog, ) -def _nrfu(ctx: click.Context, catalog: list[tuple[Callable[..., TestResult], dict[Any, Any]]]) -> None: +def _nrfu(ctx: click.Context, catalog: AntaCatalog) -> None: """Run NRFU against inventory devices""" ctx.obj["catalog"] = catalog ctx.obj["result_manager"] = ResultManager() diff --git a/anta/cli/check/commands.py b/anta/cli/check/commands.py index 070699f00..f542658e3 100644 --- a/anta/cli/check/commands.py +++ b/anta/cli/check/commands.py @@ -12,10 +12,9 @@ import click -from anta.catalog import is_catalog_valid +from anta.catalog import AntaCatalog from anta.cli.console import console from anta.cli.utils import parse_catalog -from anta.models import AntaTest logger = logging.getLogger(__name__) @@ -31,12 +30,12 @@ required=True, callback=parse_catalog, ) -def catalog(ctx: click.Context, catalog: list[tuple[AntaTest, AntaTest.Input]]) -> None: +def catalog(ctx: click.Context, catalog: AntaCatalog) -> None: """ Check that the catalog is valid """ logger.info(f"Checking syntax of catalog {ctx.obj['catalog_path']}") - manager = is_catalog_valid(catalog) + manager = catalog.check() if manager.error_status: console.print(f"[bold][red]Catalog {ctx.obj['catalog_path']} is invalid") # TODO print nice report diff --git a/anta/cli/utils.py b/anta/cli/utils.py index 97bb033c4..98d6c3302 100644 --- a/anta/cli/utils.py +++ b/anta/cli/utils.py @@ -14,9 +14,8 @@ from typing import TYPE_CHECKING, Any import click -from yaml import safe_load -import anta.loader +from anta.catalog import AntaCatalog from anta.inventory import AntaInventory from anta.tools.misc import anta_log_exception @@ -25,8 +24,6 @@ if TYPE_CHECKING: from click import Option - from anta.models import AntaTest - class ExitCode(enum.IntEnum): """ @@ -82,7 +79,7 @@ def parse_tags(ctx: click.Context, param: Option, value: str) -> list[str] | Non return None -def parse_catalog(ctx: click.Context, param: Option, value: str) -> list[tuple[AntaTest, dict[str, Any] | None]]: +def parse_catalog(ctx: click.Context, param: Option, value: str) -> AntaCatalog: # pylint: disable=unused-argument """ Click option callback to parse an ANTA tests catalog YAML file @@ -91,11 +88,12 @@ def parse_catalog(ctx: click.Context, param: Option, value: str) -> list[tuple[A """ if ctx.obj.get("_anta_help"): # Currently looking for help for a subcommand so no - # need to parse the Catalog - return an empty list - return [] + # need to parse the Catalog - return an empty catalog + return AntaCatalog("dummy") + # Storing catalog path + ctx.obj["catalog_path"] = value try: - with open(value, "r", encoding="UTF-8") as file: - data = safe_load(file) + catalog = AntaCatalog("cli", value) # TODO catch proper exception # pylint: disable-next=broad-exception-caught except Exception as e: @@ -103,9 +101,7 @@ def parse_catalog(ctx: click.Context, param: Option, value: str) -> list[tuple[A anta_log_exception(e, message, logger) ctx.fail(message) - # Storing catalog path - ctx.obj["catalog_path"] = value - return anta.loader.parse_catalog(data) + return catalog def exit_with_code(ctx: click.Context) -> None: diff --git a/anta/loader.py b/anta/loader.py index 95d4e9347..da8b75abd 100644 --- a/anta/loader.py +++ b/anta/loader.py @@ -6,16 +6,12 @@ """ from __future__ import annotations -import importlib import logging -import sys from pathlib import Path -from typing import Any from rich.logging import RichHandler from anta import __DEBUG__ -from anta.models import AntaTest logger = logging.getLogger(__name__) @@ -42,7 +38,7 @@ def setup_logging(level: str = logging.getLevelName(logging.INFO), file: Path | # Init root logger root = logging.getLogger() # In ANTA debug mode, level is overriden to DEBUG - loglevel = getattr(logging, level.upper()) if not __DEBUG__ else logging.DEBUG + loglevel = logging.DEBUG if __DEBUG__ else getattr(logging, level.upper()) root.setLevel(loglevel) # Silence the logging of chatty Python modules when level is INFO if loglevel == logging.INFO: @@ -73,85 +69,3 @@ def setup_logging(level: str = logging.getLevelName(logging.INFO), file: Path | if __DEBUG__: logger.debug("ANTA Debug Mode enabled") - - -def parse_catalog(test_catalog: dict[str, Any], package: str | None = None) -> list[tuple[AntaTest, dict[str, Any] | None]]: - """ - Function to parse the catalog and return a list of tests with their inputs - - A valid test catalog must follow the following structure: - : - - : - - - Example: - anta.tests.connectivity: - - VerifyReachability: - hosts: - - dst: 8.8.8.8 - src: 172.16.0.1 - - dst: 1.1.1.1 - src: 172.16.0.1 - result_overwrite: - categories: - - "Overwritten category 1" - description: "Test with overwritten description" - custom_field: "Test run by John Doe" - - Also supports nesting for Python module definition: - anta.tests: - connectivity: - - VerifyReachability: - hosts: - - dst: 8.8.8.8 - src: 172.16.0.1 - - dst: 1.1.1.1 - src: 172.16.0.1 - result_overwrite: - categories: - - "Overwritten category 1" - description: "Test with overwritten description" - custom_field: "Test run by John Doe" - - Args: - test_catalog: Python dictionary representing the test catalog YAML file - - Returns: - tests: List of tuples (test, inputs) where test is a reference of an AntaTest subclass - and inputs is a dictionary - """ - tests: list[tuple[AntaTest, dict[str, Any] | None]] = [] - if not test_catalog: - return tests - for key, value in test_catalog.items(): - # Required to manage iteration within a tests module - if package is not None: - key = ".".join([package, key]) - try: - module = importlib.import_module(f"{key}") - except ModuleNotFoundError: - logger.critical(f"No test module named '{key}'") - sys.exit(1) - if isinstance(value, list): - # This is a list of tests - for test in value: - for test_name, inputs in test.items(): - # A test must be a subclass of AntaTest as defined in the Python module - try: - test = getattr(module, test_name) - except AttributeError: - logger.critical(f"Wrong test name '{test_name}' in '{module.__name__}'") - sys.exit(1) - if not issubclass(test, AntaTest): - logger.critical(f"'{test.__module__}.{test.__name__}' is not an AntaTest subclass") - sys.exit(1) - # Test inputs can be either None or a dictionary - if inputs is None or isinstance(inputs, dict): - tests.append((test, inputs)) - else: - logger.critical(f"'{test.__module__}.{test.__name__}' inputs must be a dictionary") - sys.exit(1) - if isinstance(value, dict): - # This is an inner Python module - tests.extend(parse_catalog(value, package=module.__name__)) - return tests diff --git a/anta/runner.py b/anta/runner.py index d9f9e961c..cd5918a8d 100644 --- a/anta/runner.py +++ b/anta/runner.py @@ -12,6 +12,7 @@ import logging from typing import Union +from anta.catalog import AntaCatalog from anta.inventory import AntaInventory from anta.models import AntaTest from anta.result_manager import ResultManager @@ -28,8 +29,8 @@ def filter_tags(tags_cli: Union[list[str], None], tags_device: list[str], tags_t async def main( manager: ResultManager, inventory: AntaInventory, - tests: list[tuple[type[AntaTest], AntaTest.Input]], - tags: list[str], + catalog: AntaCatalog, + tags: list[str] | None = None, established_only: bool = True, ) -> None: """ @@ -39,7 +40,7 @@ async def main( Args: manager: ResultManager object to populate with the test results. inventory: AntaInventory object that includes the device(s). - tests: ANTA test catalog. Output of anta.loader.parse_catalog(). + catalog: AntaCatalog object that includes the list of tests. tags: List of tags to filter devices from the inventory. Defaults to None. established_only: Include only established device(s). Defaults to True. @@ -47,7 +48,7 @@ async def main( any: ResultManager object gets updated with the test results. """ - if not tests: + if not catalog.tests: logger.info("The list of tests is empty, exiting") return @@ -70,7 +71,7 @@ async def main( coros = [] - for device, test in itertools.product(devices, tests): + for device, test in itertools.product(devices, catalog.tests): test_class = test[0] test_inputs = test[1] test_filters = test[1].get("filters", None) if test[1] is not None else None From ad873f760ac74d45885662b6d56210c98b2bf99a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Fri, 27 Oct 2023 12:04:00 +0200 Subject: [PATCH 04/53] feat: Introduce AntaCatalogFile --- anta/catalog.py | 209 +++++++++++++++++++++---------------- anta/cli/check/commands.py | 14 ++- anta/cli/utils.py | 4 +- 3 files changed, 130 insertions(+), 97 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index ccb3c7fb0..75efd58af 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -8,64 +8,104 @@ import importlib import logging -from typing import Any, Optional, cast +from types import ModuleType +from typing import Any +from pydantic import RootModel, model_validator +from pydantic.types import ImportString from yaml import safe_load -from anta.device import AsyncEOSDevice from anta.models import AntaTest -from anta.result_manager import ResultManager logger = logging.getLogger(__name__) -class AntaCatalog: - """ - Class representing an ANTA Catalog +class AntaTestDefinition(RootModel[dict[type[AntaTest], AntaTest.Input]]): + root: dict[type[AntaTest], AntaTest.Input] - Attributes: - name: Catalog name - filename Optional[str]: The path from which the catalog was loaded - tests: list[tuple[AntaTest, AntaTest.Input]]: A list of Tuple containing an AntaTest and the associated input - """ + @model_validator(mode="after") + def check_single_entry(self) -> AntaTestDefinition: + assert len(self.root) == 1, "AntaTestDefinition is a dictionary with a single entry" + return self - def __init__(self, name: str, filename: Optional[str] = None) -> None: - """ - Constructor of AntaCatalog + @model_validator(mode="after") + def check_inputs(self) -> AntaTestDefinition: + definition: tuple[type[AntaTest], AntaTest.Input] = list(self.root.items())[0] + assert isinstance(definition[1], definition[0].Input), f"{definition[1]} object must be a instance of {definition[0].Input}" + return self - Args: - name: Device name - filname: Optional name - if provided tests are loaded - """ - self.name: str = name - self.filename: Optional[str] = filename - self.tests: list[tuple[AntaTest, AntaTest.Input]] = [] - def parse_catalog_file(self: AntaCatalog) -> None: - """ - Parse a file +class AntaCatalogFile(RootModel[dict[ImportString, list[AntaTestDefinition]]]): + root: dict[ImportString, list[AntaTestDefinition]] + + @model_validator(mode="before") + @classmethod + def check_tests(cls, data: Any) -> Any: """ - if self.filename is None: - return - try: - with open(self.filename, "r", encoding="UTF-8") as file: - data = safe_load(file) - self.parse_catalog(data) - # pylint: disable-next=broad-exception-caught - except Exception: - logger.critical(f"Something went wrong while parsing {self.filename}") - raise - - def parse_catalog(self: AntaCatalog, test_catalog: dict[str, Any], package: str | None = None) -> None: + Allow the user to provide a Python data structure that only has string values. + This validator will try to flatten and import Python modules, check if the tests classes + are actually defined in their respective Python module and instantiate Input instances + with provided value to validate test inputs. """ - Function to parse the catalog and return a list of tests with their inputs - A valid test catalog must follow the following structure: - : - - : - + def flatten_modules(data: dict[str, Any], package: str | None = None) -> dict[ModuleType, list[Any]]: + """ + Allow the user to provide a data structure with nested Python modules. + + Example: + ``` + anta.tests.routing: + generic: + - + bgp: + - + ``` + `anta.tests.routing.generic` and `anta.tests.routing.bgp` are importable Python modules. + """ + modules: dict[ModuleType, list[Any]] = {} + for module_name, tests in data.items(): + if package and not module_name.startswith("."): + module_name = f".{module_name}" + module: ModuleType = importlib.import_module(name=module_name, package=package) + if isinstance(tests, dict): + # This is an inner Python module + modules.update(flatten_modules(data=tests, package=module.__name__)) + else: + assert isinstance(tests, list), f"{tests} must be a list of AntaTestDefinition" + # This is a list of AntaTestDefinition + modules[module] = tests + return modules + + if isinstance(data, dict): + typed_data: dict[ModuleType, list[Any]] = flatten_modules(data) + for module, tests in typed_data.items(): + for test_definition in tests: + assert isinstance(test_definition, dict), "AntaTestDefinition must be a dictionary" + for test_name, test_inputs in test_definition.copy().items(): + test: type[AntaTest] | None = getattr(module, test_name, None) + assert test, f"{test_name} is not defined in Python module {module}" + del test_definition[test_name] + if test_inputs: + test_definition[test] = test.Input(**test_inputs) + else: + test_definition[test] = test.Input() + return typed_data + + +class AntaCatalog: + """ + Class representing an ANTA Catalog. + + It can be defined programmatically by providing the `tests` argument to the constructor + or it can be loaded from a file using the `filename` argument. + + A valid test catalog file must follow the following structure: + : + - : + Example: + ``` anta.tests.connectivity: - VerifyReachability: hosts: @@ -78,8 +118,10 @@ def parse_catalog(self: AntaCatalog, test_catalog: dict[str, Any], package: str - "Overwritten category 1" description: "Test with overwritten description" custom_field: "Test run by John Doe" + ``` Also supports nesting for Python module definition: + ``` anta.tests: connectivity: - VerifyReachability: @@ -93,60 +135,47 @@ def parse_catalog(self: AntaCatalog, test_catalog: dict[str, Any], package: str - "Overwritten category 1" description: "Test with overwritten description" custom_field: "Test run by John Doe" + ``` - Args: - test_catalog: Python dictionary representing the test catalog YAML file + Attributes: + filename: The path from which the catalog is loaded. + file: The AntaCatalogFile model representinf the catalog file. + tests: A list of tuple containing an AntaTest class and the associated input. + """ + + def __init__(self, filename: str | None = None, tests: list[tuple[type[AntaTest], AntaTest.Input]] = []) -> None: + """ + Constructor of AntaCatalog + Args: + filename: The path from which the catalog is loaded. Use this argument if you want to load the catalog from a file. + tests: A list of tuple containing an AntaTest class and the associated input. Use this argument if you want to define the catalog programmatically. + """ + if filename is not None and tests: + raise RuntimeError("'filename' and 'tests' arguments cannot be provided at the same time") + self.filename: str | None = filename + self.file: AntaCatalogFile | None = None + self._data = None + if self.filename: + self._parse_file() + self.tests: list[tuple[type[AntaTest], AntaTest.Input]] = tests + + def _parse_file(self) -> None: """ - # pylint: disable=broad-exception-raised - if not test_catalog: - return - - for key, value in test_catalog.items(): - # Required to manage iteration within a tests module - if package is not None: - key = ".".join([package, key]) + Parse the catalog YAML file + """ + if self.filename: try: - module = importlib.import_module(f"{key}") - except ModuleNotFoundError: - logger.critical(f"No test module named '{key}'") + with open(file=self.filename, mode="r", encoding="UTF-8") as file: + self._data = safe_load(file) + # pylint: disable-next=broad-exception-caught + except Exception: + logger.critical(f"Something went wrong while parsing {self.filename}") raise - if isinstance(value, list): - # This is a list of tests - for test in value: - for test_name, inputs in test.items(): - # A test must be a subclass of AntaTest as defined in the Python module - try: - test = getattr(module, test_name) - except AttributeError: - logger.critical(f"Wrong test name '{test_name}' in '{module.__name__}'") - raise - if not issubclass(test, AntaTest): - logger.critical(f"'{test.__module__}.{test.__name__}' is not an AntaTest subclass") - raise Exception() - # Test inputs can be either None or a dictionary - if inputs is None or isinstance(inputs, dict): - self.tests.append((cast(AntaTest, test), cast(AntaTest.Input, inputs))) - else: - logger.critical(f"'{test.__module__}.{test.__name__}' inputs must be a dictionary") - raise Exception() - if isinstance(value, dict): - # This is an inner Python module - self.parse_catalog(value, package=module.__name__) - - def check(self: AntaCatalog) -> ResultManager: + def check(self: AntaCatalog) -> None: """ - TODO - for now a test requires a device but this may be revisited in the future + Check if the data in the catalog file is valid """ - # Mock device - mock_device = AsyncEOSDevice(name="mock", host="127.0.0.1", username="mock", password="mock") - - manager = ResultManager() - # Instantiate each test to verify the Inputs are correct - for test_class, test_inputs in self.tests: - # TODO - this is the same code with typing as in runner.py but somehow mypy complains that test_class - # ot type AntaTest is not callable - test_instance = test_class(device=mock_device, inputs=test_inputs) # type: ignore[operator] - manager.add_test_result(test_instance.result) - return manager + if self._data is not None: + self.file = AntaCatalogFile(self._data) diff --git a/anta/cli/check/commands.py b/anta/cli/check/commands.py index f542658e3..2ce0bab3f 100644 --- a/anta/cli/check/commands.py +++ b/anta/cli/check/commands.py @@ -11,10 +11,13 @@ import logging import click +from pydantic import ValidationError +from rich.pretty import pretty_repr from anta.catalog import AntaCatalog from anta.cli.console import console from anta.cli.utils import parse_catalog +from anta.tools.misc import anta_log_exception logger = logging.getLogger(__name__) @@ -35,10 +38,11 @@ def catalog(ctx: click.Context, catalog: AntaCatalog) -> None: Check that the catalog is valid """ logger.info(f"Checking syntax of catalog {ctx.obj['catalog_path']}") - manager = catalog.check() - if manager.error_status: + try: + catalog.check() + console.print(f"[bold][green]Catalog {ctx.obj['catalog_path']} is valid") + console.print(pretty_repr(catalog.file)) + except ValidationError as e: console.print(f"[bold][red]Catalog {ctx.obj['catalog_path']} is invalid") - # TODO print nice report + anta_log_exception(e) ctx.exit(1) - else: - console.print(f"[bold][green]Catalog {ctx.obj['catalog_path']} is valid") diff --git a/anta/cli/utils.py b/anta/cli/utils.py index 98d6c3302..2af3fad66 100644 --- a/anta/cli/utils.py +++ b/anta/cli/utils.py @@ -89,11 +89,11 @@ def parse_catalog(ctx: click.Context, param: Option, value: str) -> AntaCatalog: if ctx.obj.get("_anta_help"): # Currently looking for help for a subcommand so no # need to parse the Catalog - return an empty catalog - return AntaCatalog("dummy") + return AntaCatalog() # Storing catalog path ctx.obj["catalog_path"] = value try: - catalog = AntaCatalog("cli", value) + catalog = AntaCatalog(filename=value) # TODO catch proper exception # pylint: disable-next=broad-exception-caught except Exception as e: From ffb9c920b81083a8d5deb77a3386004f857ed03a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Fri, 27 Oct 2023 13:03:25 +0200 Subject: [PATCH 05/53] refactor: simplify AntaTestDefinition model --- anta/catalog.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index 75efd58af..6375a286f 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -11,7 +11,7 @@ from types import ModuleType from typing import Any -from pydantic import RootModel, model_validator +from pydantic import BaseModel, RootModel, model_validator, model_serializer from pydantic.types import ImportString from yaml import safe_load @@ -20,18 +20,17 @@ logger = logging.getLogger(__name__) -class AntaTestDefinition(RootModel[dict[type[AntaTest], AntaTest.Input]]): - root: dict[type[AntaTest], AntaTest.Input] +class AntaTestDefinition(BaseModel): + test: type[AntaTest] + inputs: AntaTest.Input - @model_validator(mode="after") - def check_single_entry(self) -> AntaTestDefinition: - assert len(self.root) == 1, "AntaTestDefinition is a dictionary with a single entry" - return self + @model_serializer + def ser_model(self) -> dict[str, AntaTest.Input]: + return {self.test.__name__: self.inputs} @model_validator(mode="after") - def check_inputs(self) -> AntaTestDefinition: - definition: tuple[type[AntaTest], AntaTest.Input] = list(self.root.items())[0] - assert isinstance(definition[1], definition[0].Input), f"{definition[1]} object must be a instance of {definition[0].Input}" + def check_inputs(self) -> 'AntaTestDefinition': + assert isinstance(self.inputs, self.test.Input), f"{self.inputs} object must be a instance of {self.test.Input}" return self @@ -79,16 +78,16 @@ def flatten_modules(data: dict[str, Any], package: str | None = None) -> dict[Mo if isinstance(data, dict): typed_data: dict[ModuleType, list[Any]] = flatten_modules(data) for module, tests in typed_data.items(): + test_definitions: list[AntaTestDefinition] = [] for test_definition in tests: assert isinstance(test_definition, dict), "AntaTestDefinition must be a dictionary" + assert len(test_definition) == 1, "AntaTestDefinition must be a dictionary with a single entry" for test_name, test_inputs in test_definition.copy().items(): test: type[AntaTest] | None = getattr(module, test_name, None) assert test, f"{test_name} is not defined in Python module {module}" - del test_definition[test_name] - if test_inputs: - test_definition[test] = test.Input(**test_inputs) - else: - test_definition[test] = test.Input() + inputs: AntaTest.Input = test.Input(**test_inputs) if test_inputs else test.Input() + test_definitions.append(AntaTestDefinition(test=test, inputs=inputs)) + typed_data[module] = test_definitions return typed_data From 5cfaae468088618f7139e96bb0ecdb15eae181df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Fri, 27 Oct 2023 13:03:40 +0200 Subject: [PATCH 06/53] New VSCode settings --- .vscode/settings.json | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 3b3ba1da7..3ef1c3768 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,23 +1,22 @@ { - "python.linting.enabled": true, - "python.linting.flake8Enabled": true, - "python.linting.mypyEnabled": true, - "python.formatting.provider": "black", + "black-formatter.importStrategy": "fromEnvironment", "pylint.importStrategy": "fromEnvironment", + "pylint.args": [ + "--load-plugins pylint_pydantic", + "--rcfile=pylintrc" + ], "flake8.importStrategy": "fromEnvironment", - "black-formatter.importStrategy": "fromEnvironment", - "python.linting.flake8Args": [ + "flake8.args": [ "--config=/dev/null", "--max-line-length=165" ], - "python.linting.mypyArgs": [ + "mypy-type-checker.importStrategy": "fromEnvironment", + "mypy-type-checker.args": [ "--config-file=pyproject.toml" ], - "pylint.severity": { - "refactor": "Warning" + "isort.importStrategy": "fromEnvironment", + "isort.check": true, + "files.associations": { + "*.yaml": "home-assistant" }, - "pylint.args": [ - "--load-plugins pylint_pydantic", - "--rcfile=pylintrc" - ] } \ No newline at end of file From 2161ee7b883bb4b314eccaf40964f1ee478f5295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Fri, 27 Oct 2023 13:04:17 +0200 Subject: [PATCH 07/53] linting --- anta/catalog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index 6375a286f..076e49a6e 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -11,7 +11,7 @@ from types import ModuleType from typing import Any -from pydantic import BaseModel, RootModel, model_validator, model_serializer +from pydantic import BaseModel, RootModel, model_serializer, model_validator from pydantic.types import ImportString from yaml import safe_load @@ -29,7 +29,7 @@ def ser_model(self) -> dict[str, AntaTest.Input]: return {self.test.__name__: self.inputs} @model_validator(mode="after") - def check_inputs(self) -> 'AntaTestDefinition': + def check_inputs(self) -> "AntaTestDefinition": assert isinstance(self.inputs, self.test.Input), f"{self.inputs} object must be a instance of {self.test.Input}" return self From d141cdee28cfa1288a7c50f7c9c59631bf11b509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Fri, 27 Oct 2023 13:36:10 +0200 Subject: [PATCH 08/53] Rename loader module to logger module --- anta/cli/__init__.py | 2 +- anta/{loader.py => logger.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename anta/{loader.py => logger.py} (100%) diff --git a/anta/cli/__init__.py b/anta/cli/__init__.py index c26866462..6cf076ea4 100644 --- a/anta/cli/__init__.py +++ b/anta/cli/__init__.py @@ -22,7 +22,7 @@ from anta.cli.get import commands as get_commands from anta.cli.nrfu import commands as nrfu_commands from anta.cli.utils import AliasedGroup, IgnoreRequiredWithHelp, parse_catalog, parse_inventory -from anta.loader import setup_logging +from anta.logger import setup_logging from anta.result_manager import ResultManager diff --git a/anta/loader.py b/anta/logger.py similarity index 100% rename from anta/loader.py rename to anta/logger.py From c2ae21fb2cc530aa8c365f2b8e33fc7905564360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Fri, 27 Oct 2023 14:16:20 +0200 Subject: [PATCH 09/53] feat: add tests property to AntaCatalog --- anta/catalog.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index 076e49a6e..efa039573 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -142,7 +142,7 @@ class AntaCatalog: tests: A list of tuple containing an AntaTest class and the associated input. """ - def __init__(self, filename: str | None = None, tests: list[tuple[type[AntaTest], AntaTest.Input]] = []) -> None: + def __init__(self, filename: str | None = None, tests: list[AntaTestDefinition] = []) -> None: """ Constructor of AntaCatalog @@ -157,7 +157,18 @@ def __init__(self, filename: str | None = None, tests: list[tuple[type[AntaTest] self._data = None if self.filename: self._parse_file() - self.tests: list[tuple[type[AntaTest], AntaTest.Input]] = tests + self.tests: list[AntaTestDefinition] = tests + + @property + def tests(self) -> list[AntaTestDefinition]: + return self._tests + + @tests.setter + def tests(self, value: list[AntaTestDefinition]) -> None: + assert isinstance(value, list), "The catalog must contain a list of tests" + for t in value: + assert isinstance(t, AntaTestDefinition), "A test in the catalog must be an AntaTestDefinition instance" + self._tests = value def _parse_file(self) -> None: """ @@ -175,6 +186,16 @@ def _parse_file(self) -> None: def check(self: AntaCatalog) -> None: """ Check if the data in the catalog file is valid + and populate `tests` instance attribute. """ if self._data is not None: - self.file = AntaCatalogFile(self._data) + self.file = AntaCatalogFile(**self._data) + else: + logger.critical("Catalog file has not been parsed thus cannot be checked") + # TODO: custom exception + raise Exception() + if self._tests: + logger.warning(f'Overriding AntaCatalog data from file {self.filename}') + self._tests = [] + for tests in self.file.root.values(): + self._tests.extend(tests) From bbcbf4d0de6e2d4cbcf756108b747e3421d0ab11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Fri, 27 Oct 2023 14:27:06 +0200 Subject: [PATCH 10/53] doc: update docstring --- anta/catalog.py | 98 +++++++++++++++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 39 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index efa039573..f83d12330 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -14,6 +14,7 @@ from pydantic import BaseModel, RootModel, model_serializer, model_validator from pydantic.types import ImportString from yaml import safe_load +from anta.device import AntaDevice from anta.models import AntaTest @@ -35,6 +36,48 @@ def check_inputs(self) -> "AntaTestDefinition": class AntaCatalogFile(RootModel[dict[ImportString, list[AntaTestDefinition]]]): + """ + This model represents an ANTA Test Catalog File. + + A valid test catalog file must follow the following structure: + : + - : + + + Example: + ``` + anta.tests.connectivity: + - VerifyReachability: + hosts: + - dst: 8.8.8.8 + src: 172.16.0.1 + - dst: 1.1.1.1 + src: 172.16.0.1 + result_overwrite: + categories: + - "Overwritten category 1" + description: "Test with overwritten description" + custom_field: "Test run by John Doe" + ``` + + Also supports nesting for Python module definition: + ``` + anta.tests: + connectivity: + - VerifyReachability: + hosts: + - dst: 8.8.8.8 + src: 172.16.0.1 + - dst: 1.1.1.1 + src: 172.16.0.1 + result_overwrite: + categories: + - "Overwritten category 1" + description: "Test with overwritten description" + custom_field: "Test run by John Doe" + ``` + """ + root: dict[ImportString, list[AntaTestDefinition]] @model_validator(mode="before") @@ -46,7 +89,6 @@ def check_tests(cls, data: Any) -> Any: are actually defined in their respective Python module and instantiate Input instances with provided value to validate test inputs. """ - def flatten_modules(data: dict[str, Any], package: str | None = None) -> dict[ModuleType, list[Any]]: """ Allow the user to provide a data structure with nested Python modules. @@ -98,44 +140,6 @@ class AntaCatalog: It can be defined programmatically by providing the `tests` argument to the constructor or it can be loaded from a file using the `filename` argument. - A valid test catalog file must follow the following structure: - : - - : - - - Example: - ``` - anta.tests.connectivity: - - VerifyReachability: - hosts: - - dst: 8.8.8.8 - src: 172.16.0.1 - - dst: 1.1.1.1 - src: 172.16.0.1 - result_overwrite: - categories: - - "Overwritten category 1" - description: "Test with overwritten description" - custom_field: "Test run by John Doe" - ``` - - Also supports nesting for Python module definition: - ``` - anta.tests: - connectivity: - - VerifyReachability: - hosts: - - dst: 8.8.8.8 - src: 172.16.0.1 - - dst: 1.1.1.1 - src: 172.16.0.1 - result_overwrite: - categories: - - "Overwritten category 1" - description: "Test with overwritten description" - custom_field: "Test run by John Doe" - ``` - Attributes: filename: The path from which the catalog is loaded. file: The AntaCatalogFile model representinf the catalog file. @@ -199,3 +203,19 @@ def check(self: AntaCatalog) -> None: self._tests = [] for tests in self.file.root.values(): self._tests.extend(tests) + + def get_tests_by_tags(self, tags: list[str], strict: bool = False) -> list[AntaTestDefinition]: + """ + Return all the tests that have matching tags in their input filters. + If strict=True, returns only tests that match all the tags provided as input. + If strict=False, return all the tests that match at least one tag provided as input. + """ + raise NotImplementedError() + + def get_tests_by_device(self, devices: list[AntaDevice], strict: bool = False) -> list[AntaTestDefinition]: + """ + Return all the tests that have matching devices in their input filters. + If strict=True, returns only tests that match all the devices provided as input. + If strict=False, return all the tests that match at least one device provided as input. + """ + raise NotImplementedError() From 6aed48e054b83bb39cfe277314fc85f3948415c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Fri, 27 Oct 2023 14:56:51 +0200 Subject: [PATCH 11/53] feat: update CLI to support AntaCatalog --- anta/catalog.py | 15 ++++++--------- anta/cli/__init__.py | 3 ++- anta/cli/check/commands.py | 14 +++----------- anta/cli/nrfu/utils.py | 2 +- anta/cli/utils.py | 12 ++++++++++++ anta/models.py | 23 ++++++++++++++--------- anta/runner.py | 11 ++++------- 7 files changed, 42 insertions(+), 38 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index f83d12330..293242960 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -14,8 +14,8 @@ from pydantic import BaseModel, RootModel, model_serializer, model_validator from pydantic.types import ImportString from yaml import safe_load -from anta.device import AntaDevice +from anta.device import AntaDevice from anta.models import AntaTest logger = logging.getLogger(__name__) @@ -89,6 +89,7 @@ def check_tests(cls, data: Any) -> Any: are actually defined in their respective Python module and instantiate Input instances with provided value to validate test inputs. """ + def flatten_modules(data: dict[str, Any], package: str | None = None) -> dict[ModuleType, list[Any]]: """ Allow the user to provide a data structure with nested Python modules. @@ -194,15 +195,11 @@ def check(self: AntaCatalog) -> None: """ if self._data is not None: self.file = AntaCatalogFile(**self._data) - else: - logger.critical("Catalog file has not been parsed thus cannot be checked") - # TODO: custom exception - raise Exception() - if self._tests: - logger.warning(f'Overriding AntaCatalog data from file {self.filename}') + if self._tests: + logger.warning(f"Overriding AntaCatalog data from file {self.filename}") self._tests = [] - for tests in self.file.root.values(): - self._tests.extend(tests) + for tests in self.file.root.values(): + self._tests.extend(tests) def get_tests_by_tags(self, tags: list[str], strict: bool = False) -> list[AntaTestDefinition]: """ diff --git a/anta/cli/__init__.py b/anta/cli/__init__.py index 6cf076ea4..f2e259470 100644 --- a/anta/cli/__init__.py +++ b/anta/cli/__init__.py @@ -21,7 +21,7 @@ from anta.cli.exec import commands as exec_commands from anta.cli.get import commands as get_commands from anta.cli.nrfu import commands as nrfu_commands -from anta.cli.utils import AliasedGroup, IgnoreRequiredWithHelp, parse_catalog, parse_inventory +from anta.cli.utils import AliasedGroup, IgnoreRequiredWithHelp, check_catalog, parse_catalog, parse_inventory from anta.logger import setup_logging from anta.result_manager import ResultManager @@ -150,6 +150,7 @@ def anta( ) def _nrfu(ctx: click.Context, catalog: AntaCatalog) -> None: """Run NRFU against inventory devices""" + check_catalog(ctx, catalog) ctx.obj["catalog"] = catalog ctx.obj["result_manager"] = ResultManager() diff --git a/anta/cli/check/commands.py b/anta/cli/check/commands.py index 2ce0bab3f..ca4339625 100644 --- a/anta/cli/check/commands.py +++ b/anta/cli/check/commands.py @@ -11,13 +11,11 @@ import logging import click -from pydantic import ValidationError from rich.pretty import pretty_repr from anta.catalog import AntaCatalog from anta.cli.console import console -from anta.cli.utils import parse_catalog -from anta.tools.misc import anta_log_exception +from anta.cli.utils import check_catalog, parse_catalog logger = logging.getLogger(__name__) @@ -38,11 +36,5 @@ def catalog(ctx: click.Context, catalog: AntaCatalog) -> None: Check that the catalog is valid """ logger.info(f"Checking syntax of catalog {ctx.obj['catalog_path']}") - try: - catalog.check() - console.print(f"[bold][green]Catalog {ctx.obj['catalog_path']} is valid") - console.print(pretty_repr(catalog.file)) - except ValidationError as e: - console.print(f"[bold][red]Catalog {ctx.obj['catalog_path']} is invalid") - anta_log_exception(e) - ctx.exit(1) + check_catalog(ctx, catalog) + console.print(pretty_repr(catalog.file)) diff --git a/anta/cli/nrfu/utils.py b/anta/cli/nrfu/utils.py index 7275530d0..493651d40 100644 --- a/anta/cli/nrfu/utils.py +++ b/anta/cli/nrfu/utils.py @@ -26,7 +26,7 @@ def print_settings(context: click.Context, report_template: pathlib.Path | None = None, report_output: pathlib.Path | None = None) -> None: """Print ANTA settings before running tests""" - message = f"Running ANTA tests:\n- {context.obj['inventory']}\n- Tests catalog contains {len(context.obj['catalog'])} tests" + message = f"Running ANTA tests:\n- {context.obj['inventory']}\n- Tests catalog contains {len(context.obj['catalog'].tests)} tests" if report_template: message += f"\n- Report template: {report_template}" if report_output: diff --git a/anta/cli/utils.py b/anta/cli/utils.py index 2af3fad66..120f8df34 100644 --- a/anta/cli/utils.py +++ b/anta/cli/utils.py @@ -14,8 +14,10 @@ from typing import TYPE_CHECKING, Any import click +from pydantic import ValidationError from anta.catalog import AntaCatalog +from anta.cli.console import console from anta.inventory import AntaInventory from anta.tools.misc import anta_log_exception @@ -43,6 +45,16 @@ class ExitCode(enum.IntEnum): USAGE_ERROR = 4 +def check_catalog(ctx: click.Context, catalog: AntaCatalog) -> None: + try: + catalog.check() + console.print(f"[bold][green]Catalog {catalog.filename} is valid") + except ValidationError as e: + console.print(f"[bold][red]Catalog {catalog.filename} is invalid") + anta_log_exception(e) + ctx.exit(1) + + def parse_inventory(ctx: click.Context, path: Path) -> AntaInventory: """ Helper function parse an ANTA inventory YAML file diff --git a/anta/models.py b/anta/models.py index 13823496a..3f7f172a1 100644 --- a/anta/models.py +++ b/anta/models.py @@ -317,7 +317,7 @@ class Filters(BaseModel): def __init__( self, device: AntaDevice, - inputs: Optional[dict[str, Any]], + inputs: dict[str, Any] | AntaTest.Input | None, eos_data: Optional[list[dict[Any, Any] | str]] = None, ): """AntaTest Constructor @@ -337,19 +337,24 @@ def __init__( if self.result.result == "unset": self._init_commands(eos_data) - def _init_inputs(self, inputs: Optional[dict[str, Any]]) -> None: + def _init_inputs(self, inputs: dict[str, Any] | AntaTest.Input | None) -> None: """Instantiate the `inputs` instance attribute with an `AntaTest.Input` instance to validate test inputs from defined model. Overwrite result fields based on `ResultOverwrite` input definition. Any input validation error will set this test result status as 'error'.""" - try: - self.inputs = self.Input(**inputs) if inputs is not None else self.Input() - except ValidationError as e: - message = f"{self.__module__}.{self.__class__.__name__}: Inputs are not valid\n{e}" - self.logger.error(message) - self.result.is_error(message=message, exception=e) - return + if inputs is None: + self.inputs = self.Input() + elif isinstance(inputs, AntaTest.Input): + self.inputs = inputs + elif isinstance(inputs, dict): + try: + self.inputs = self.Input(**inputs) + except ValidationError as e: + message = f"{self.__module__}.{self.__class__.__name__}: Inputs are not valid\n{e}" + self.logger.error(message) + self.result.is_error(message=message, exception=e) + return if res_ow := self.inputs.result_overwrite: if res_ow.categories: self.result.categories = res_ow.categories diff --git a/anta/runner.py b/anta/runner.py index cd5918a8d..4ad236385 100644 --- a/anta/runner.py +++ b/anta/runner.py @@ -48,6 +48,7 @@ async def main( any: ResultManager object gets updated with the test results. """ + catalog.check() if not catalog.tests: logger.info("The list of tests is empty, exiting") return @@ -71,15 +72,11 @@ async def main( coros = [] - for device, test in itertools.product(devices, catalog.tests): - test_class = test[0] - test_inputs = test[1] - test_filters = test[1].get("filters", None) if test[1] is not None else None - test_tags = test_filters.get("tags", []) if test_filters is not None else [] - if len(test_tags) == 0 or filter_tags(tags_cli=tags, tags_device=device.tags, tags_test=test_tags): + for device, test_definition in itertools.product(devices, catalog.tests): + if len(test_definition.inputs.filters.tags) == 0 or filter_tags(tags_cli=tags, tags_device=device.tags, tags_test=test_definition.inputs.filters.tags): try: # Instantiate AntaTest object - test_instance = test_class(device=device, inputs=test_inputs) + test_instance = test_definition.test(device=device, inputs=test_definition.inputs) coros.append(test_instance.test(eos_data=None)) except Exception as e: # pylint: disable=broad-exception-caught message = "Error when creating ANTA tests" From dedf6336dd837810dd144110a4b28291478c3b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Fri, 27 Oct 2023 15:14:43 +0200 Subject: [PATCH 12/53] rebase onto main --- anta/runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/anta/runner.py b/anta/runner.py index 4ad236385..978a4ad94 100644 --- a/anta/runner.py +++ b/anta/runner.py @@ -73,7 +73,8 @@ async def main( coros = [] for device, test_definition in itertools.product(devices, catalog.tests): - if len(test_definition.inputs.filters.tags) == 0 or filter_tags(tags_cli=tags, tags_device=device.tags, tags_test=test_definition.inputs.filters.tags): + test_tags: list[str] = test_definition.inputs.filters.tags if test_definition.inputs.filters is not None else None + if not test_tags or filter_tags(tags_cli=tags, tags_device=device.tags, tags_test=test_tags): try: # Instantiate AntaTest object test_instance = test_definition.test(device=device, inputs=test_definition.inputs) From 93507330c1ab6683bf952adac035d68cae3583b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Fri, 27 Oct 2023 15:22:28 +0200 Subject: [PATCH 13/53] Thanks HA --- .vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 3ef1c3768..421e61908 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -16,7 +16,4 @@ ], "isort.importStrategy": "fromEnvironment", "isort.check": true, - "files.associations": { - "*.yaml": "home-assistant" - }, } \ No newline at end of file From 77e517749d49597736a2499c5296cbbbb2ffa231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Fri, 27 Oct 2023 17:30:36 +0200 Subject: [PATCH 14/53] Update runner to use AntaCatalog methods --- anta/catalog.py | 22 +++++++++++++++------- anta/models.py | 6 +++--- anta/runner.py | 49 +++++++++++++++++++++++++++++-------------------- 3 files changed, 47 insertions(+), 30 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index 293242960..544cd0809 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -207,12 +207,20 @@ def get_tests_by_tags(self, tags: list[str], strict: bool = False) -> list[AntaT If strict=True, returns only tests that match all the tags provided as input. If strict=False, return all the tests that match at least one tag provided as input. """ - raise NotImplementedError() - - def get_tests_by_device(self, devices: list[AntaDevice], strict: bool = False) -> list[AntaTestDefinition]: + result: list[AntaTestDefinition] = [] + for test in self.tests: + if test.inputs.filters and (filter := test.inputs.filters.tags): + if (strict and all(t in tags for t in filter)) or (not strict and any(t in tags for t in filter)): + result.append(test) + return result + + def get_tests_by_device(self, device: AntaDevice) -> list[AntaTestDefinition]: """ - Return all the tests that have matching devices in their input filters. - If strict=True, returns only tests that match all the devices provided as input. - If strict=False, return all the tests that match at least one device provided as input. + Return all the tests that have the provided device in their input filters. """ - raise NotImplementedError() + result: list[AntaTestDefinition] = [] + for test in self.tests: + if test.inputs.filters and (filter := test.inputs.filters.devices): + if device.name in filter: + result.append(test) + return result diff --git a/anta/models.py b/anta/models.py index 3f7f172a1..8a083c0af 100644 --- a/anta/models.py +++ b/anta/models.py @@ -317,8 +317,8 @@ class Filters(BaseModel): def __init__( self, device: AntaDevice, - inputs: dict[str, Any] | AntaTest.Input | None, - eos_data: Optional[list[dict[Any, Any] | str]] = None, + inputs: dict[str, Any] | AntaTest.Input | None = None, + eos_data: list[dict[Any, Any] | str] | None = None, ): """AntaTest Constructor @@ -472,7 +472,7 @@ def anta_test(function: F) -> Callable[..., Coroutine[Any, Any, TestResult]]: @wraps(function) async def wrapper( self: AntaTest, - eos_data: Optional[list[dict[Any, Any] | str]] = None, + eos_data: list[dict[Any, Any] | str] | None = None, **kwargs: Any, ) -> TestResult: """ diff --git a/anta/runner.py b/anta/runner.py index 978a4ad94..3db62a530 100644 --- a/anta/runner.py +++ b/anta/runner.py @@ -8,11 +8,11 @@ from __future__ import annotations import asyncio -import itertools import logging -from typing import Union +from typing import Tuple -from anta.catalog import AntaCatalog +from anta.catalog import AntaCatalog, AntaTestDefinition +from anta.device import AntaDevice from anta.inventory import AntaInventory from anta.models import AntaTest from anta.result_manager import ResultManager @@ -20,10 +20,7 @@ logger = logging.getLogger(__name__) - -def filter_tags(tags_cli: Union[list[str], None], tags_device: list[str], tags_test: list[str]) -> bool: - """Implement filtering logic for tags""" - return (tags_cli is None or any(t for t in tags_cli if t in tags_device)) and any(t for t in tags_device if t in tags_test) +AntaTestRunner = Tuple[AntaTestDefinition, AntaDevice] async def main( @@ -59,9 +56,7 @@ async def main( await inventory.connect_inventory() - # asyncio.gather takes an iterator of the function to run concurrently. - # we get the cross product of the devices and tests to build that iterator. - devices = inventory.get_inventory(established_only=established_only, tags=tags).values() + devices: list[AntaDevice] = list(inventory.get_inventory(established_only=established_only, tags=tags).values()) if len(devices) == 0: logger.info( @@ -72,16 +67,30 @@ async def main( coros = [] - for device, test_definition in itertools.product(devices, catalog.tests): - test_tags: list[str] = test_definition.inputs.filters.tags if test_definition.inputs.filters is not None else None - if not test_tags or filter_tags(tags_cli=tags, tags_device=device.tags, tags_test=test_tags): - try: - # Instantiate AntaTest object - test_instance = test_definition.test(device=device, inputs=test_definition.inputs) - coros.append(test_instance.test(eos_data=None)) - except Exception as e: # pylint: disable=broad-exception-caught - message = "Error when creating ANTA tests" - anta_log_exception(e, message, logger) + tests: list[AntaTestRunner] = [] + if tags: + for device in devices: + for test in catalog.get_tests_by_tags(tags): + tests.append((test, device)) + else: + # If there is no CLI tags, execute all tests without filters on all devices + for device in devices: + tests.extend([(t, device) for t in catalog.tests if t.inputs.filters is None or t.inputs.filters.tags is None]) + # Also execute tests with filters conditionally + if device.tags: + # Execute tests on devices with matching tags + tests.extend(list(map(lambda t: (t, device), catalog.get_tests_by_tags(device.tags)))) + # Also execute tests with filters on devices with matching name + tests.extend(list(map(lambda t: (t, device), catalog.get_tests_by_device(device)))) + + for test_definition, device in tests: + try: + # Instantiate AntaTest object + test_instance = test_definition.test(device=device, inputs=test_definition.inputs) + coros.append(test_instance.test(eos_data=None)) + except Exception as e: # pylint: disable=broad-exception-caught + message = "Error when creating ANTA tests" + anta_log_exception(e, message, logger) if AntaTest.progress is not None: AntaTest.nrfu_task = AntaTest.progress.add_task("Running NRFU Tests...", total=len(coros)) From 83c6ca815af55f9cba66a038ede9ecdf0340503d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Fri, 27 Oct 2023 17:36:23 +0200 Subject: [PATCH 15/53] comments --- anta/runner.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/anta/runner.py b/anta/runner.py index 3db62a530..4dd3cc689 100644 --- a/anta/runner.py +++ b/anta/runner.py @@ -68,19 +68,19 @@ async def main( coros = [] tests: list[AntaTestRunner] = [] - if tags: - for device in devices: - for test in catalog.get_tests_by_tags(tags): - tests.append((test, device)) - else: - # If there is no CLI tags, execute all tests without filters on all devices - for device in devices: + for device in devices: + if tags: + # If there are CLI tags, only execute tests with matching tags + for test in catalog.get_tests_by_tags(tags): + tests.append((test, device)) + else: + # If there is no CLI tags, execute all tests without filters tests.extend([(t, device) for t in catalog.tests if t.inputs.filters is None or t.inputs.filters.tags is None]) # Also execute tests with filters conditionally if device.tags: - # Execute tests on devices with matching tags + # If the device has tags, execute tests with matching tags tests.extend(list(map(lambda t: (t, device), catalog.get_tests_by_tags(device.tags)))) - # Also execute tests with filters on devices with matching name + # Also execute tests with filters on this device name tests.extend(list(map(lambda t: (t, device), catalog.get_tests_by_device(device)))) for test_definition, device in tests: From 81a84182020fef041b3c7f7f6d2610ef70046d6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Fri, 27 Oct 2023 18:51:41 +0200 Subject: [PATCH 16/53] linting --- .vscode/settings.json | 1 - anta/catalog.py | 34 +++++++++++++++++++++++++--------- anta/cli/utils.py | 4 ++++ anta/runner.py | 12 +++++++----- tests/units/test_runner.py | 20 +++++++++----------- 5 files changed, 45 insertions(+), 26 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 421e61908..d688a64d6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,7 +2,6 @@ "black-formatter.importStrategy": "fromEnvironment", "pylint.importStrategy": "fromEnvironment", "pylint.args": [ - "--load-plugins pylint_pydantic", "--rcfile=pylintrc" ], "flake8.importStrategy": "fromEnvironment", diff --git a/anta/catalog.py b/anta/catalog.py index 544cd0809..1fe740376 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -22,20 +22,33 @@ class AntaTestDefinition(BaseModel): + """ + Define a test with its associated inputs. + + test: An AntaTest concrete subclass + inputs: The associated AntaTest.Input subclass instance + """ + test: type[AntaTest] inputs: AntaTest.Input @model_serializer def ser_model(self) -> dict[str, AntaTest.Input]: + """ + Serialize an AntaTestDefinition as it is defined in a test catalog YAML file. + """ return {self.test.__name__: self.inputs} @model_validator(mode="after") def check_inputs(self) -> "AntaTestDefinition": + """ + The `inputs` class attribute needs to be an instance of the AntaTest.Input subclass defined in the class `test`. + """ assert isinstance(self.inputs, self.test.Input), f"{self.inputs} object must be a instance of {self.test.Input}" return self -class AntaCatalogFile(RootModel[dict[ImportString, list[AntaTestDefinition]]]): +class AntaCatalogFile(RootModel[dict[ImportString[Any], list[AntaTestDefinition]]]): # pylint: disable=too-few-public-methods """ This model represents an ANTA Test Catalog File. @@ -78,7 +91,7 @@ class AntaCatalogFile(RootModel[dict[ImportString, list[AntaTestDefinition]]]): ``` """ - root: dict[ImportString, list[AntaTestDefinition]] + root: dict[ImportString[Any], list[AntaTestDefinition]] @model_validator(mode="before") @classmethod @@ -143,11 +156,11 @@ class AntaCatalog: Attributes: filename: The path from which the catalog is loaded. - file: The AntaCatalogFile model representinf the catalog file. tests: A list of tuple containing an AntaTest class and the associated input. + file: The AntaCatalogFile model representinf the catalog file. """ - def __init__(self, filename: str | None = None, tests: list[AntaTestDefinition] = []) -> None: + def __init__(self, filename: str | None = None, tests: list[AntaTestDefinition] | None = None) -> None: """ Constructor of AntaCatalog @@ -158,14 +171,17 @@ def __init__(self, filename: str | None = None, tests: list[AntaTestDefinition] if filename is not None and tests: raise RuntimeError("'filename' and 'tests' arguments cannot be provided at the same time") self.filename: str | None = filename + self.tests: list[AntaTestDefinition] = [] + if tests is not None: + self.tests = tests self.file: AntaCatalogFile | None = None self._data = None if self.filename: self._parse_file() - self.tests: list[AntaTestDefinition] = tests @property def tests(self) -> list[AntaTestDefinition]: + """List of AntaTestDefinition in this catalog""" return self._tests @tests.setter @@ -209,8 +225,8 @@ def get_tests_by_tags(self, tags: list[str], strict: bool = False) -> list[AntaT """ result: list[AntaTestDefinition] = [] for test in self.tests: - if test.inputs.filters and (filter := test.inputs.filters.tags): - if (strict and all(t in tags for t in filter)) or (not strict and any(t in tags for t in filter)): + if test.inputs.filters and (f := test.inputs.filters.tags): + if (strict and all(t in tags for t in f)) or (not strict and any(t in tags for t in f)): result.append(test) return result @@ -220,7 +236,7 @@ def get_tests_by_device(self, device: AntaDevice) -> list[AntaTestDefinition]: """ result: list[AntaTestDefinition] = [] for test in self.tests: - if test.inputs.filters and (filter := test.inputs.filters.devices): - if device.name in filter: + if test.inputs.filters and (f := test.inputs.filters.devices): + if device.name in f: result.append(test) return result diff --git a/anta/cli/utils.py b/anta/cli/utils.py index 120f8df34..83e780895 100644 --- a/anta/cli/utils.py +++ b/anta/cli/utils.py @@ -46,6 +46,10 @@ class ExitCode(enum.IntEnum): def check_catalog(ctx: click.Context, catalog: AntaCatalog) -> None: + """ + Helper function to check test catalog file and print + output using console. + """ try: catalog.check() console.print(f"[bold][green]Catalog {catalog.filename} is valid") diff --git a/anta/runner.py b/anta/runner.py index 4dd3cc689..bd664ed37 100644 --- a/anta/runner.py +++ b/anta/runner.py @@ -71,23 +71,25 @@ async def main( for device in devices: if tags: # If there are CLI tags, only execute tests with matching tags - for test in catalog.get_tests_by_tags(tags): - tests.append((test, device)) + for test in catalog.get_tests_by_tags(tags): + tests.append((test, device)) else: # If there is no CLI tags, execute all tests without filters tests.extend([(t, device) for t in catalog.tests if t.inputs.filters is None or t.inputs.filters.tags is None]) # Also execute tests with filters conditionally if device.tags: # If the device has tags, execute tests with matching tags - tests.extend(list(map(lambda t: (t, device), catalog.get_tests_by_tags(device.tags)))) + for t in catalog.get_tests_by_tags(device.tags): + tests.append((t, device)) # Also execute tests with filters on this device name - tests.extend(list(map(lambda t: (t, device), catalog.get_tests_by_device(device)))) + for t in catalog.get_tests_by_device(device): + tests.append((t, device)) for test_definition, device in tests: try: # Instantiate AntaTest object test_instance = test_definition.test(device=device, inputs=test_definition.inputs) - coros.append(test_instance.test(eos_data=None)) + coros.append(test_instance.test()) except Exception as e: # pylint: disable=broad-exception-caught message = "Error when creating ANTA tests" anta_log_exception(e, message, logger) diff --git a/tests/units/test_runner.py b/tests/units/test_runner.py index d1b2f9fd3..cccce1838 100644 --- a/tests/units/test_runner.py +++ b/tests/units/test_runner.py @@ -11,14 +11,18 @@ import pytest +from anta.catalog import AntaCatalog, AntaTestDefinition from anta.inventory import AntaInventory -from anta.models import AntaTest from anta.result_manager import ResultManager from anta.runner import main +from .test_models import FakeTest + if TYPE_CHECKING: from pytest import LogCaptureFixture +FAKE_CATALOG = AntaCatalog(tests=[AntaTestDefinition(test=FakeTest, inputs=FakeTest.Input())]) + @pytest.mark.asyncio async def test_runner_empty_tests(caplog: LogCaptureFixture, test_inventory: AntaInventory) -> None: @@ -29,7 +33,7 @@ async def test_runner_empty_tests(caplog: LogCaptureFixture, test_inventory: Ant test_inventory is a fixture that gives a default inventory for tests """ manager = ResultManager() - await main(manager, test_inventory, [], tags=[]) + await main(manager, test_inventory, AntaCatalog()) assert len(caplog.record_tuples) == 1 assert "The list of tests is empty, exiting" in caplog.records[0].message @@ -44,10 +48,7 @@ async def test_runner_empty_inventory(caplog: LogCaptureFixture) -> None: """ manager = ResultManager() inventory = AntaInventory() - # This is not vaidated in this test - tests: list[tuple[type[AntaTest], AntaTest.Input]] = [(AntaTest, {})] # type: ignore[type-abstract] - await main(manager, inventory, tests, tags=[]) - + await main(manager, inventory, FAKE_CATALOG) assert len(caplog.record_tuples) == 1 assert "The inventory is empty, exiting" in caplog.records[0].message @@ -61,16 +62,13 @@ async def test_runner_no_selected_device(caplog: LogCaptureFixture, test_invento test_inventory is a fixture that gives a default inventory for tests """ manager = ResultManager() - # This is not vaidated in this test - tests: list[tuple[type[AntaTest], AntaTest.Input]] = [(AntaTest, {})] # type: ignore[type-abstract] - - await main(manager, test_inventory, tests, tags=[]) + await main(manager, test_inventory, FAKE_CATALOG) assert "No device in the established state 'True' was found. There is no device to run tests against, exiting" in [record.message for record in caplog.records] # Reset logs and run with tags caplog.clear() - await main(manager, test_inventory, tests, tags=["toto"]) + await main(manager, test_inventory, FAKE_CATALOG, tags=["toto"]) assert "No device in the established state 'True' matching the tags ['toto'] was found. There is no device to run tests against, exiting" in [ record.message for record in caplog.records From c46c362a12f632b0414294dc555e9173fe05a1cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Fri, 27 Oct 2023 19:10:51 +0200 Subject: [PATCH 17/53] fix tests --- anta/models.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/anta/models.py b/anta/models.py index 8a083c0af..7cb98d0ee 100644 --- a/anta/models.py +++ b/anta/models.py @@ -343,18 +343,18 @@ def _init_inputs(self, inputs: dict[str, Any] | AntaTest.Input | None) -> None: Overwrite result fields based on `ResultOverwrite` input definition. Any input validation error will set this test result status as 'error'.""" - if inputs is None: - self.inputs = self.Input() - elif isinstance(inputs, AntaTest.Input): - self.inputs = inputs - elif isinstance(inputs, dict): - try: + try: + if inputs is None: + self.inputs = self.Input() + elif isinstance(inputs, AntaTest.Input): + self.inputs = inputs + elif isinstance(inputs, dict): self.inputs = self.Input(**inputs) - except ValidationError as e: - message = f"{self.__module__}.{self.__class__.__name__}: Inputs are not valid\n{e}" - self.logger.error(message) - self.result.is_error(message=message, exception=e) - return + except ValidationError as e: + message = f"{self.__module__}.{self.__class__.__name__}: Inputs are not valid\n{e}" + self.logger.error(message) + self.result.is_error(message=message, exception=e) + return if res_ow := self.inputs.result_overwrite: if res_ow.categories: self.result.categories = res_ow.categories From feba64c24e75b1123b283e173ee5f4865c3393eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Mon, 30 Oct 2023 12:16:07 +0100 Subject: [PATCH 18/53] fix: Update AntaTestDefinition and AntaCatalogFile for Python3.8 compatibility --- anta/catalog.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index 1fe740376..2a2599b51 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -9,7 +9,7 @@ import importlib import logging from types import ModuleType -from typing import Any +from typing import Any, Type, Dict, List from pydantic import BaseModel, RootModel, model_serializer, model_validator from pydantic.types import ImportString @@ -29,11 +29,11 @@ class AntaTestDefinition(BaseModel): inputs: The associated AntaTest.Input subclass instance """ - test: type[AntaTest] + test: Type[AntaTest] inputs: AntaTest.Input @model_serializer - def ser_model(self) -> dict[str, AntaTest.Input]: + def ser_model(self) -> Dict[str, AntaTest.Input]: """ Serialize an AntaTestDefinition as it is defined in a test catalog YAML file. """ @@ -48,7 +48,7 @@ def check_inputs(self) -> "AntaTestDefinition": return self -class AntaCatalogFile(RootModel[dict[ImportString[Any], list[AntaTestDefinition]]]): # pylint: disable=too-few-public-methods +class AntaCatalogFile(RootModel[Dict[ImportString[Any], List[AntaTestDefinition]]]): # pylint: disable=too-few-public-methods """ This model represents an ANTA Test Catalog File. @@ -91,7 +91,7 @@ class AntaCatalogFile(RootModel[dict[ImportString[Any], list[AntaTestDefinition] ``` """ - root: dict[ImportString[Any], list[AntaTestDefinition]] + root: Dict[ImportString[Any], List[AntaTestDefinition]] @model_validator(mode="before") @classmethod From 77935f97265d955c004bd5dc5e7517a97b4cc797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Mon, 30 Oct 2023 18:47:51 +0100 Subject: [PATCH 19/53] tests: add unit tests for anta.catalog --- anta/catalog.py | 61 ++++-- tests/data/test_catalog_with_tags.yml | 28 +++ .../test_catalog_with_undefined_module.yml | 3 + ...t_catalog_with_undefined_module_nested.yml | 4 + .../test_catalog_with_undefined_tests.yml | 3 + tests/units/test_catalog.py | 181 ++++++++++++++++++ tests/units/test_runner.py | 4 +- 7 files changed, 268 insertions(+), 16 deletions(-) create mode 100644 tests/data/test_catalog_with_tags.yml create mode 100644 tests/data/test_catalog_with_undefined_module.yml create mode 100644 tests/data/test_catalog_with_undefined_module_nested.yml create mode 100644 tests/data/test_catalog_with_undefined_tests.yml create mode 100644 tests/units/test_catalog.py diff --git a/anta/catalog.py b/anta/catalog.py index 2a2599b51..9f4ae9d29 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -8,8 +8,9 @@ import importlib import logging +from inspect import isclass from types import ModuleType -from typing import Any, Type, Dict, List +from typing import Any, Dict, List, Type from pydantic import BaseModel, RootModel, model_serializer, model_validator from pydantic.types import ImportString @@ -39,12 +40,32 @@ def ser_model(self) -> Dict[str, AntaTest.Input]: """ return {self.test.__name__: self.inputs} + @model_validator(mode="before") + @classmethod + def instantiate_inputs(cls, data: Any) -> Any: + """ + If the test has no inputs, allow the user to omit providing the `inputs` field. + If the test has inputs, allow the user to provide a valid dictionary of the input fields. + This model validator will instantiate an Input class from the `test` class field. + + TODO: AntaTestDefinition typing suggest the user that `inputs` MUST be an AntaTest.Input instance + but this validator allow other types. Is there any way to change mypy behaviour without changing the + `inputs` typing that will change the pydantic validation logic ? + """ + if isinstance(data, dict) and isclass(data["test"]) and issubclass(data["test"], AntaTest): + if data["inputs"] is None: + data["inputs"] = data["test"].Input() + if isinstance(data["inputs"], dict): + data["inputs"] = data["test"].Input(**data["inputs"]) + return data + @model_validator(mode="after") def check_inputs(self) -> "AntaTestDefinition": """ The `inputs` class attribute needs to be an instance of the AntaTest.Input subclass defined in the class `test`. """ - assert isinstance(self.inputs, self.test.Input), f"{self.inputs} object must be a instance of {self.test.Input}" + if not isinstance(self.inputs, self.test.Input): + raise ValueError(f"Test input has type {self.inputs.__class__.__qualname__} but expected type {self.test.Input.__qualname__}") return self @@ -121,12 +142,19 @@ def flatten_modules(data: dict[str, Any], package: str | None = None) -> dict[Mo for module_name, tests in data.items(): if package and not module_name.startswith("."): module_name = f".{module_name}" - module: ModuleType = importlib.import_module(name=module_name, package=package) + try: + module: ModuleType = importlib.import_module(name=module_name, package=package) + except ModuleNotFoundError as e: + module_str = module_name if not module_name.startswith(".") else module_name[1:] + if package: + module_str += f" from package {package}" + raise ValueError(f"Module named {module_str} cannot be imported") from e if isinstance(tests, dict): # This is an inner Python module modules.update(flatten_modules(data=tests, package=module.__name__)) else: - assert isinstance(tests, list), f"{tests} must be a list of AntaTestDefinition" + if not isinstance(tests, list): + raise ValueError(f"{tests} must be a list of AntaTestDefinition") # This is a list of AntaTestDefinition modules[module] = tests return modules @@ -136,13 +164,15 @@ def flatten_modules(data: dict[str, Any], package: str | None = None) -> dict[Mo for module, tests in typed_data.items(): test_definitions: list[AntaTestDefinition] = [] for test_definition in tests: - assert isinstance(test_definition, dict), "AntaTestDefinition must be a dictionary" - assert len(test_definition) == 1, "AntaTestDefinition must be a dictionary with a single entry" + if not isinstance(test_definition, dict): + raise ValueError("AntaTestDefinition must be a dictionary") + if not len(test_definition) == 1: + raise ValueError("AntaTestDefinition must be a dictionary with a single entry") for test_name, test_inputs in test_definition.copy().items(): test: type[AntaTest] | None = getattr(module, test_name, None) - assert test, f"{test_name} is not defined in Python module {module}" - inputs: AntaTest.Input = test.Input(**test_inputs) if test_inputs else test.Input() - test_definitions.append(AntaTestDefinition(test=test, inputs=inputs)) + if test is None: + raise ValueError(f"{test_name} is not defined in Python module {module}") + test_definitions.append(AntaTestDefinition(test=test, inputs=test_inputs)) typed_data[module] = test_definitions return typed_data @@ -157,10 +187,10 @@ class AntaCatalog: Attributes: filename: The path from which the catalog is loaded. tests: A list of tuple containing an AntaTest class and the associated input. - file: The AntaCatalogFile model representinf the catalog file. + file: The AntaCatalogFile model representing the catalog file. """ - def __init__(self, filename: str | None = None, tests: list[AntaTestDefinition] | None = None) -> None: + def __init__(self, filename: str | None = None, tests: list[tuple[type[AntaTest], AntaTest.Input | dict[str, Any] | None]] | None = None) -> None: """ Constructor of AntaCatalog @@ -173,7 +203,8 @@ def __init__(self, filename: str | None = None, tests: list[AntaTestDefinition] self.filename: str | None = filename self.tests: list[AntaTestDefinition] = [] if tests is not None: - self.tests = tests + for test, inputs in tests: + self.tests.append(AntaTestDefinition(test=test, inputs=inputs)) # type: ignore self.file: AntaCatalogFile | None = None self._data = None if self.filename: @@ -186,9 +217,11 @@ def tests(self) -> list[AntaTestDefinition]: @tests.setter def tests(self, value: list[AntaTestDefinition]) -> None: - assert isinstance(value, list), "The catalog must contain a list of tests" + if not isinstance(value, list): + raise ValueError("The catalog must contain a list of tests") for t in value: - assert isinstance(t, AntaTestDefinition), "A test in the catalog must be an AntaTestDefinition instance" + if not isinstance(t, AntaTestDefinition): + raise ValueError("A test in the catalog must be an AntaTestDefinition instance") self._tests = value def _parse_file(self) -> None: diff --git a/tests/data/test_catalog_with_tags.yml b/tests/data/test_catalog_with_tags.yml new file mode 100644 index 000000000..30f90e20e --- /dev/null +++ b/tests/data/test_catalog_with_tags.yml @@ -0,0 +1,28 @@ +--- +anta.tests.system: + - VerifyUptime: + minimum: 10 + filters: + tags: ['fabric'] + - VerifyReloadCause: + filters: + tags: ['leaf', 'spine'] + - VerifyCoredump: + - VerifyAgentLogs: + - VerifyCPUUtilization: + filters: + tags: ['leaf'] + - VerifyMemoryUtilization: + filters: + devices: ['testdevice'] + - VerifyFileSystemUtilization: + - VerifyNTP: + +anta.tests.mlag: + - VerifyMlagStatus: + +anta.tests.interfaces: + - VerifyL3MTU: + mtu: 1500 + filters: + tags: ['demo'] diff --git a/tests/data/test_catalog_with_undefined_module.yml b/tests/data/test_catalog_with_undefined_module.yml new file mode 100644 index 000000000..bd847dc98 --- /dev/null +++ b/tests/data/test_catalog_with_undefined_module.yml @@ -0,0 +1,3 @@ +--- +anta.tests.undefined: + - MyTest: \ No newline at end of file diff --git a/tests/data/test_catalog_with_undefined_module_nested.yml b/tests/data/test_catalog_with_undefined_module_nested.yml new file mode 100644 index 000000000..a3019c9a7 --- /dev/null +++ b/tests/data/test_catalog_with_undefined_module_nested.yml @@ -0,0 +1,4 @@ +--- +anta.tests: + undefined: + - MyTest: \ No newline at end of file diff --git a/tests/data/test_catalog_with_undefined_tests.yml b/tests/data/test_catalog_with_undefined_tests.yml new file mode 100644 index 000000000..88b477615 --- /dev/null +++ b/tests/data/test_catalog_with_undefined_tests.yml @@ -0,0 +1,3 @@ +--- +anta.tests.software: + - FakeTest: \ No newline at end of file diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py new file mode 100644 index 000000000..71c3971a1 --- /dev/null +++ b/tests/units/test_catalog.py @@ -0,0 +1,181 @@ +# Copyright (c) 2023 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +""" +test anta.device.py +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +from pydantic import ValidationError + +from anta.catalog import AntaCatalog, AntaTestDefinition +from anta.device import AntaDevice +from anta.models import AntaTest + +# Test classes used as expected values +from anta.tests.configuration import VerifyZeroTouch +from anta.tests.hardware import VerifyTemperature +from anta.tests.interfaces import VerifyL3MTU +from anta.tests.mlag import VerifyMlagStatus +from anta.tests.software import VerifyEOSVersion +from anta.tests.system import ( + VerifyAgentLogs, + VerifyCoredump, + VerifyCPUUtilization, + VerifyFileSystemUtilization, + VerifyMemoryUtilization, + VerifyNTP, + VerifyReloadCause, + VerifyUptime, +) +from tests.lib.utils import generate_test_ids_list +from tests.units.test_models import FakeTestWithInput + +DATA_DIR: Path = Path(__file__).parent.parent.resolve() / "data" + +INIT_CATALOG_DATA: list[dict[str, Any]] = [ + { + "name": "test_catalog", + "filename": "test_catalog.yml", + "tests": [ + (VerifyZeroTouch, None), + (VerifyTemperature, None), + (VerifyEOSVersion, VerifyEOSVersion.Input(versions=["4.25.4M", "4.26.1F"])), + (VerifyUptime, {"minimum": 86400}), + ], + }, + { + "name": "test_catalog_with_tags", + "filename": "test_catalog_with_tags.yml", + "tests": [ + ( + VerifyUptime, + VerifyUptime.Input( + minimum=10, + filters=VerifyUptime.Input.Filters(tags=["fabric"]), + ), + ), + (VerifyReloadCause, {"filters": {"tags": ["leaf", "spine"]}}), + (VerifyCoredump, VerifyCoredump.Input()), + (VerifyAgentLogs, AntaTest.Input()), + (VerifyCPUUtilization, VerifyCPUUtilization.Input(filters=VerifyCPUUtilization.Input.Filters(tags=["leaf"]))), + (VerifyMemoryUtilization, VerifyMemoryUtilization.Input(filters=VerifyMemoryUtilization.Input.Filters(devices=["testdevice"]))), + (VerifyFileSystemUtilization, None), + (VerifyNTP, {}), + (VerifyMlagStatus, None), + (VerifyL3MTU, {"mtu": 1500, "filters": {"tags": ["demo"]}}), + ], + }, +] +INIT_CATALOG_FILENAME_FAIL_DATA: list[dict[str, Any]] = [ + { + "name": "undefined_tests", + "filename": "test_catalog_with_undefined_tests.yml", + "error": "FakeTest is not defined in Python module ", + }, + { + "name": "undefined_module", + "filename": "test_catalog_with_undefined_module.yml", + "error": "Module named anta.tests.undefined cannot be imported", + }, + { + "name": "undefined_module_nested", + "filename": "test_catalog_with_undefined_module_nested.yml", + "error": "Module named undefined from package anta.tests cannot be imported", + }, +] +INIT_CATALOG_TESTS_FAIL_DATA: list[dict[str, Any]] = [ + { + "name": "wrong_inputs", + "tests": [ + ( + FakeTestWithInput, + AntaTest.Input(), + ), + ], + "error": "Test input has type AntaTest.Input but expected type FakeTestWithInput.Input", + }, + { + "name": "no_test", + "tests": [(None, None)], + "error": "Input should be a subclass of AntaTest", + }, + { + "name": "no_test", + "tests": [(FakeTestWithInput, None)], + "error": "Field required", + }, +] + + +class Test_AntaCatalog: + """ + Test for anta.catalog.AntaCatalog + """ + + @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) + def test__init__filename(self, catalog_data: dict[str, Any]) -> None: + """ + Instantiate AntaCatalog from a file + """ + catalog = AntaCatalog(filename=DATA_DIR / catalog_data["filename"]) + catalog.check() + + assert len(catalog.tests) == len(catalog_data["tests"]) + for test_id, (test, inputs) in enumerate(catalog_data["tests"]): + assert catalog.tests[test_id].test == test + if inputs is not None: + if isinstance(inputs, dict): + inputs = test.Input(**inputs) + assert inputs == catalog.tests[test_id].inputs + + @pytest.mark.parametrize("catalog_data", INIT_CATALOG_FILENAME_FAIL_DATA, ids=generate_test_ids_list(INIT_CATALOG_FILENAME_FAIL_DATA)) + def test__init__filename_fail(self, catalog_data: dict[str, Any]) -> None: + """ + Errors when instantiating AntaCatalog from a file + """ + catalog = AntaCatalog(filename=DATA_DIR / catalog_data["filename"]) + with pytest.raises(ValidationError) as exec_info: + catalog.check() + assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + + @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) + def test__init__tests(self, catalog_data: dict[str, Any]) -> None: + """ + Instantiate AntaCatalog from a list of tuples + """ + catalog = AntaCatalog(tests=catalog_data["tests"]) + + assert len(catalog.tests) == len(catalog_data["tests"]) + for test_id, (test, inputs) in enumerate(catalog_data["tests"]): + assert catalog.tests[test_id].test == test + if inputs is not None: + if isinstance(inputs, dict): + inputs = test.Input(**inputs) + assert inputs == catalog.tests[test_id].inputs + + @pytest.mark.parametrize("catalog_data", INIT_CATALOG_TESTS_FAIL_DATA, ids=generate_test_ids_list(INIT_CATALOG_TESTS_FAIL_DATA)) + def test__init__tests_fail(self, catalog_data: dict[str, Any]) -> None: + """ + Errors when instantiating AntaCatalog from a list of tuples + """ + with pytest.raises(ValidationError) as exec_info: + AntaCatalog(tests=catalog_data["tests"]) + assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + + def test_get_tests_by_tags(self) -> None: + catalog = AntaCatalog(filename=DATA_DIR / "test_catalog_with_tags.yml") + catalog.check() + tests: list[AntaTestDefinition] = catalog.get_tests_by_tags(tags=["leaf"]) + assert len(tests) == 2 + + def test_get_tests_by_device(self, mocked_device: AntaDevice) -> None: + catalog = AntaCatalog(filename=DATA_DIR / "test_catalog_with_tags.yml") + catalog.check() + tests: list[AntaTestDefinition] = catalog.get_tests_by_device(device=mocked_device) + assert len(tests) == 1 diff --git a/tests/units/test_runner.py b/tests/units/test_runner.py index cccce1838..ed631878c 100644 --- a/tests/units/test_runner.py +++ b/tests/units/test_runner.py @@ -11,7 +11,7 @@ import pytest -from anta.catalog import AntaCatalog, AntaTestDefinition +from anta.catalog import AntaCatalog from anta.inventory import AntaInventory from anta.result_manager import ResultManager from anta.runner import main @@ -21,7 +21,7 @@ if TYPE_CHECKING: from pytest import LogCaptureFixture -FAKE_CATALOG = AntaCatalog(tests=[AntaTestDefinition(test=FakeTest, inputs=FakeTest.Input())]) +FAKE_CATALOG = AntaCatalog(tests=[(FakeTest, None)]) @pytest.mark.asyncio From e18ffd309d9febb5aec6fb3b3355679559345dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Mon, 30 Oct 2023 18:55:23 +0100 Subject: [PATCH 20/53] linting --- tests/units/test_catalog.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index 71c3971a1..516a77f5c 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -123,7 +123,7 @@ def test__init__filename(self, catalog_data: dict[str, Any]) -> None: """ Instantiate AntaCatalog from a file """ - catalog = AntaCatalog(filename=DATA_DIR / catalog_data["filename"]) + catalog = AntaCatalog(filename=str(DATA_DIR / catalog_data["filename"])) catalog.check() assert len(catalog.tests) == len(catalog_data["tests"]) @@ -139,7 +139,7 @@ def test__init__filename_fail(self, catalog_data: dict[str, Any]) -> None: """ Errors when instantiating AntaCatalog from a file """ - catalog = AntaCatalog(filename=DATA_DIR / catalog_data["filename"]) + catalog = AntaCatalog(filename=str(DATA_DIR / catalog_data["filename"])) with pytest.raises(ValidationError) as exec_info: catalog.check() assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] @@ -169,13 +169,19 @@ def test__init__tests_fail(self, catalog_data: dict[str, Any]) -> None: assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] def test_get_tests_by_tags(self) -> None: - catalog = AntaCatalog(filename=DATA_DIR / "test_catalog_with_tags.yml") + """ + Test AntaCatalog.test_get_tests_by_tags() + """ + catalog = AntaCatalog(filename=str(DATA_DIR / "test_catalog_with_tags.yml")) catalog.check() tests: list[AntaTestDefinition] = catalog.get_tests_by_tags(tags=["leaf"]) assert len(tests) == 2 def test_get_tests_by_device(self, mocked_device: AntaDevice) -> None: - catalog = AntaCatalog(filename=DATA_DIR / "test_catalog_with_tags.yml") + """ + Test AntaCatalog.test_get_tests_by_device() + """ + catalog = AntaCatalog(filename=str(DATA_DIR / "test_catalog_with_tags.yml")) catalog.check() tests: list[AntaTestDefinition] = catalog.get_tests_by_device(device=mocked_device) assert len(tests) == 1 From 79a5f03a0a56f820c28d347f98bf6350097e69a2 Mon Sep 17 00:00:00 2001 From: gmuloc Date: Tue, 31 Oct 2023 11:06:58 +0100 Subject: [PATCH 21/53] Refactor: Validate input using BeforeValidator --- anta/catalog.py | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index 9f4ae9d29..3f58efa1a 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -10,9 +10,10 @@ import logging from inspect import isclass from types import ModuleType -from typing import Any, Dict, List, Type +from typing import Annotated, Any, Dict, List, Type -from pydantic import BaseModel, RootModel, model_serializer, model_validator +from pydantic import BaseModel, FieldValidationInfo, RootModel, field_validator, model_serializer, model_validator +from pydantic.functional_validators import BeforeValidator from pydantic.types import ImportString from yaml import safe_load @@ -31,7 +32,18 @@ class AntaTestDefinition(BaseModel): """ test: Type[AntaTest] - inputs: AntaTest.Input + inputs: Annotated[AntaTest.Input, BeforeValidator(AntaTestDefinition.instantiate_inputs)] + + def __init__(self, **data: Any) -> None: + """ + Inject test in the context to allow to instantiate Input in the BeforeValidator + https://docs.pydantic.dev/2.0/usage/validators/#using-validation-context-with-basemodel-initialization + """ + self.__pydantic_validator__.validate_python( + data, + self_instance=self, + context={"test": data["test"]}, + ) @model_serializer def ser_model(self) -> Dict[str, AntaTest.Input]: @@ -40,24 +52,24 @@ def ser_model(self) -> Dict[str, AntaTest.Input]: """ return {self.test.__name__: self.inputs} - @model_validator(mode="before") @classmethod - def instantiate_inputs(cls, data: Any) -> Any: + def instantiate_inputs(cls, data: AntaTest.Input | dict[str, Any] | None, info: FieldValidationInfo) -> AntaTest.Input: """ If the test has no inputs, allow the user to omit providing the `inputs` field. If the test has inputs, allow the user to provide a valid dictionary of the input fields. This model validator will instantiate an Input class from the `test` class field. - - TODO: AntaTestDefinition typing suggest the user that `inputs` MUST be an AntaTest.Input instance - but this validator allow other types. Is there any way to change mypy behaviour without changing the - `inputs` typing that will change the pydantic validation logic ? """ - if isinstance(data, dict) and isclass(data["test"]) and issubclass(data["test"], AntaTest): - if data["inputs"] is None: - data["inputs"] = data["test"].Input() - if isinstance(data["inputs"], dict): - data["inputs"] = data["test"].Input(**data["inputs"]) - return data + if data is None: + return AntaTest.Input() + if isinstance(data, AntaTest.Input): + return data + if isinstance(data, dict): + if info.context is not None: + test_class = info.context["test"] + return test_class.Input(**data) + else: + raise ValueError("Coud not instantiate dict inputs as no test class could be identified") + raise ValueError("Coud not instantiate inputs") @model_validator(mode="after") def check_inputs(self) -> "AntaTestDefinition": @@ -203,8 +215,7 @@ def __init__(self, filename: str | None = None, tests: list[tuple[type[AntaTest] self.filename: str | None = filename self.tests: list[AntaTestDefinition] = [] if tests is not None: - for test, inputs in tests: - self.tests.append(AntaTestDefinition(test=test, inputs=inputs)) # type: ignore + self.tests.extend(AntaTestDefinition(test=test, inputs=inputs) for test, inputs in tests) self.file: AntaCatalogFile | None = None self._data = None if self.filename: From 0017ea142be32d291b2fc64f274713cab018d1d0 Mon Sep 17 00:00:00 2001 From: gmuloc Date: Tue, 31 Oct 2023 12:16:53 +0100 Subject: [PATCH 22/53] Fix: Adjust instantiate_inputs --- anta/catalog.py | 23 +++++++++++-------- tests/data/test_catalog_not_a_list.yml | 2 ++ .../test_catalog_with_undefined_module.yml | 2 +- ...t_catalog_with_undefined_module_nested.yml | 2 +- .../test_catalog_with_undefined_tests.yml | 2 +- tests/units/test_catalog.py | 19 +++++++++++---- 6 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 tests/data/test_catalog_not_a_list.yml diff --git a/anta/catalog.py b/anta/catalog.py index 3f58efa1a..a0be8c14f 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -12,7 +12,7 @@ from types import ModuleType from typing import Annotated, Any, Dict, List, Type -from pydantic import BaseModel, FieldValidationInfo, RootModel, field_validator, model_serializer, model_validator +from pydantic import BaseModel, RootModel, ValidationInfo, model_serializer, model_validator from pydantic.functional_validators import BeforeValidator from pydantic.types import ImportString from yaml import safe_load @@ -44,6 +44,7 @@ def __init__(self, **data: Any) -> None: self_instance=self, context={"test": data["test"]}, ) + super(BaseModel, self).__init__() @model_serializer def ser_model(self) -> Dict[str, AntaTest.Input]: @@ -53,23 +54,27 @@ def ser_model(self) -> Dict[str, AntaTest.Input]: return {self.test.__name__: self.inputs} @classmethod - def instantiate_inputs(cls, data: AntaTest.Input | dict[str, Any] | None, info: FieldValidationInfo) -> AntaTest.Input: + def instantiate_inputs(cls, data: AntaTest.Input | dict[str, Any] | None, info: ValidationInfo) -> AntaTest.Input: """ If the test has no inputs, allow the user to omit providing the `inputs` field. If the test has inputs, allow the user to provide a valid dictionary of the input fields. This model validator will instantiate an Input class from the `test` class field. """ + if info.context is None: + raise ValueError("Could not validate inputs as no test class could be identified") + # Pydantic guarantees at this stage that test_class is a subclass of AntaTest because of the ordering + # of fields in the class definition - so no need to check for this + test_class = info.context["test"] + if not (isclass(test_class) and issubclass(test_class, AntaTest)): + raise ValueError(f"Could not validate inputs as no test class {test_class} is not a subclass of AntaTest") + if data is None: - return AntaTest.Input() + return test_class.Input() if isinstance(data, AntaTest.Input): return data if isinstance(data, dict): - if info.context is not None: - test_class = info.context["test"] - return test_class.Input(**data) - else: - raise ValueError("Coud not instantiate dict inputs as no test class could be identified") - raise ValueError("Coud not instantiate inputs") + return test_class.Input(**data) + raise ValueError(f"Coud not instantiate inputs as type {type(data)} is not valid") @model_validator(mode="after") def check_inputs(self) -> "AntaTestDefinition": diff --git a/tests/data/test_catalog_not_a_list.yml b/tests/data/test_catalog_not_a_list.yml new file mode 100644 index 000000000..d8c42976d --- /dev/null +++ b/tests/data/test_catalog_not_a_list.yml @@ -0,0 +1,2 @@ +--- +anta.tests.configuration: true diff --git a/tests/data/test_catalog_with_undefined_module.yml b/tests/data/test_catalog_with_undefined_module.yml index bd847dc98..f2e116b6e 100644 --- a/tests/data/test_catalog_with_undefined_module.yml +++ b/tests/data/test_catalog_with_undefined_module.yml @@ -1,3 +1,3 @@ --- anta.tests.undefined: - - MyTest: \ No newline at end of file + - MyTest: diff --git a/tests/data/test_catalog_with_undefined_module_nested.yml b/tests/data/test_catalog_with_undefined_module_nested.yml index a3019c9a7..cf0f393ad 100644 --- a/tests/data/test_catalog_with_undefined_module_nested.yml +++ b/tests/data/test_catalog_with_undefined_module_nested.yml @@ -1,4 +1,4 @@ --- anta.tests: undefined: - - MyTest: \ No newline at end of file + - MyTest: diff --git a/tests/data/test_catalog_with_undefined_tests.yml b/tests/data/test_catalog_with_undefined_tests.yml index 88b477615..8282714f2 100644 --- a/tests/data/test_catalog_with_undefined_tests.yml +++ b/tests/data/test_catalog_with_undefined_tests.yml @@ -1,3 +1,3 @@ --- anta.tests.software: - - FakeTest: \ No newline at end of file + - FakeTest: diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index 516a77f5c..e62cc2ee7 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -4,7 +4,6 @@ """ test anta.device.py """ - from __future__ import annotations from pathlib import Path @@ -16,8 +15,6 @@ from anta.catalog import AntaCatalog, AntaTestDefinition from anta.device import AntaDevice from anta.models import AntaTest - -# Test classes used as expected values from anta.tests.configuration import VerifyZeroTouch from anta.tests.hardware import VerifyTemperature from anta.tests.interfaces import VerifyL3MTU @@ -36,6 +33,8 @@ from tests.lib.utils import generate_test_ids_list from tests.units.test_models import FakeTestWithInput +# Test classes used as expected values + DATA_DIR: Path = Path(__file__).parent.parent.resolve() / "data" INIT_CATALOG_DATA: list[dict[str, Any]] = [ @@ -76,7 +75,7 @@ { "name": "undefined_tests", "filename": "test_catalog_with_undefined_tests.yml", - "error": "FakeTest is not defined in Python module ", + "error": "FakeTest is not defined in Python module is not valid", + }, ] From 4e2a1ba8b50f41db9862752e9b7a5af5ba6ea800 Mon Sep 17 00:00:00 2001 From: gmuloc Date: Tue, 31 Oct 2023 12:53:47 +0100 Subject: [PATCH 23/53] Test: Increase catalog test coverage --- anta/catalog.py | 4 +- ...catalog_test_definition_multiple_dicts.yml | 9 ++++ ...est_catalog_test_definition_not_a_dict.yml | 3 ++ tests/units/test_catalog.py | 51 +++++++++++++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 tests/data/test_catalog_test_definition_multiple_dicts.yml create mode 100644 tests/data/test_catalog_test_definition_not_a_dict.yml diff --git a/anta/catalog.py b/anta/catalog.py index a0be8c14f..4903e3859 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -162,7 +162,7 @@ def flatten_modules(data: dict[str, Any], package: str | None = None) -> dict[Mo try: module: ModuleType = importlib.import_module(name=module_name, package=package) except ModuleNotFoundError as e: - module_str = module_name if not module_name.startswith(".") else module_name[1:] + module_str = module_name[1:] if module_name.startswith(".") else module_name if package: module_str += f" from package {package}" raise ValueError(f"Module named {module_str} cannot be imported") from e @@ -183,7 +183,7 @@ def flatten_modules(data: dict[str, Any], package: str | None = None) -> dict[Mo for test_definition in tests: if not isinstance(test_definition, dict): raise ValueError("AntaTestDefinition must be a dictionary") - if not len(test_definition) == 1: + if len(test_definition) != 1: raise ValueError("AntaTestDefinition must be a dictionary with a single entry") for test_name, test_inputs in test_definition.copy().items(): test: type[AntaTest] | None = getattr(module, test_name, None) diff --git a/tests/data/test_catalog_test_definition_multiple_dicts.yml b/tests/data/test_catalog_test_definition_multiple_dicts.yml new file mode 100644 index 000000000..9287ee6d3 --- /dev/null +++ b/tests/data/test_catalog_test_definition_multiple_dicts.yml @@ -0,0 +1,9 @@ +--- +anta.tests.software: + - VerifyEOSVersion: + versions: + - 4.25.4M + - 4.26.1F + VerifyTerminAttrVersion: + versions: + - 4.25.4M diff --git a/tests/data/test_catalog_test_definition_not_a_dict.yml b/tests/data/test_catalog_test_definition_not_a_dict.yml new file mode 100644 index 000000000..052ad267a --- /dev/null +++ b/tests/data/test_catalog_test_definition_not_a_dict.yml @@ -0,0 +1,3 @@ +--- +anta.tests.software: + - VerifyEOSVersion diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index e62cc2ee7..03e3b330d 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -92,6 +92,16 @@ "filename": "test_catalog_not_a_list.yml", "error": "Value error, True must be a list of AntaTestDefinition", }, + { + "name": "test_definition_not_a_dict", + "filename": "test_catalog_test_definition_not_a_dict.yml", + "error": "Value error, AntaTestDefinition must be a dictionary", + }, + { + "name": "test_definition_multiple_dicts", + "filename": "test_catalog_test_definition_multiple_dicts.yml", + "error": "Value error, AntaTestDefinition must be a dictionary with a single entry", + }, ] INIT_CATALOG_TESTS_FAIL_DATA: list[dict[str, Any]] = [ { @@ -121,12 +131,32 @@ }, ] +TESTS_SETTER_FAIL_DATA: list[dict[str, Any]] = [ + { + "name": "not_a_list", + "tests": "not_a_list", + "error": "The catalog must contain a list of tests", + }, + { + "name": "not_a_list_of_test_definitions", + "tests": [42, 43], + "error": "A test in the catalog must be an AntaTestDefinition instance", + }, +] + class Test_AntaCatalog: """ Test for anta.catalog.AntaCatalog """ + def test__init__filename_and_tests(self) -> None: + """ + Instantiate AntaCatalog from a file and give tests at the same time + """ + with pytest.raises(RuntimeError, match="'filename' and 'tests' arguments cannot be provided at the same time"): + AntaCatalog(filename=str(DATA_DIR / "test_catalog.yml"), tests=INIT_CATALOG_DATA[0]["tests"]) + @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) def test__init__filename(self, catalog_data: dict[str, Any]) -> None: """ @@ -153,6 +183,17 @@ def test__init__filename_fail(self, catalog_data: dict[str, Any]) -> None: catalog.check() assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + def test__init__filename_fail_parsing(self, caplog: pytest.LogCaptureFixture) -> None: + """ + Errors when instantiating AntaCatalog from a file + """ + with pytest.raises(Exception) as exec_info: + AntaCatalog(filename=str(DATA_DIR / "catalog_does_not_exist.yml")) + assert "No such file or directory" in str(exec_info) + assert len(caplog.record_tuples) == 1 + _, _, message = caplog.record_tuples[0] + assert "Something went wrong while parsing" in message + @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) def test__init__tests(self, catalog_data: dict[str, Any]) -> None: """ @@ -177,6 +218,16 @@ def test__init__tests_fail(self, catalog_data: dict[str, Any]) -> None: AntaCatalog(tests=catalog_data["tests"]) assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + @pytest.mark.parametrize("catalog_data", TESTS_SETTER_FAIL_DATA, ids=generate_test_ids_list(TESTS_SETTER_FAIL_DATA)) + def test__tests_setter_fail(self, catalog_data: dict[str, Any]) -> None: + """ + Errors when setting AntaCatalog.tests from a list of tuples + """ + catalog = AntaCatalog() + with pytest.raises(ValueError) as exec_info: + catalog.tests = catalog_data["tests"] + assert catalog_data["error"] in str(exec_info) + def test_get_tests_by_tags(self) -> None: """ Test AntaCatalog.test_get_tests_by_tags() From 840ec27a353234fb72c7b4c2caaff645a69b3516 Mon Sep 17 00:00:00 2001 From: gmuloc Date: Tue, 31 Oct 2023 14:01:26 +0100 Subject: [PATCH 24/53] Refactor: Make it great for python 3.8 --- anta/catalog.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index 4903e3859..2dc1e4144 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -10,10 +10,9 @@ import logging from inspect import isclass from types import ModuleType -from typing import Annotated, Any, Dict, List, Type +from typing import Any, Dict, List, Type -from pydantic import BaseModel, RootModel, ValidationInfo, model_serializer, model_validator -from pydantic.functional_validators import BeforeValidator +from pydantic import BaseModel, RootModel, ValidationInfo, field_validator, model_serializer, model_validator from pydantic.types import ImportString from yaml import safe_load @@ -32,7 +31,7 @@ class AntaTestDefinition(BaseModel): """ test: Type[AntaTest] - inputs: Annotated[AntaTest.Input, BeforeValidator(AntaTestDefinition.instantiate_inputs)] + inputs: AntaTest.Input def __init__(self, **data: Any) -> None: """ @@ -53,6 +52,7 @@ def ser_model(self) -> Dict[str, AntaTest.Input]: """ return {self.test.__name__: self.inputs} + @field_validator("inputs", mode="before") @classmethod def instantiate_inputs(cls, data: AntaTest.Input | dict[str, Any] | None, info: ValidationInfo) -> AntaTest.Input: """ From a4301d44e2ecaa4c6515efa91fcc414490efd37e Mon Sep 17 00:00:00 2001 From: gmuloc Date: Tue, 31 Oct 2023 15:39:00 +0100 Subject: [PATCH 25/53] Test: Fix result_manager tests to not tshoot the CI --- tests/data/json_data.py | 61 +----- tests/units/result_manager/test_models.py | 225 ++++------------------ 2 files changed, 47 insertions(+), 239 deletions(-) diff --git a/tests/data/json_data.py b/tests/data/json_data.py index 95760949c..9ff2a10d0 100644 --- a/tests/data/json_data.py +++ b/tests/data/json_data.py @@ -8,7 +8,6 @@ { "name": "validIPv6", "input": "fe80::cc62:a9ff:feef:932a", - "expected_result": "valid", }, ] @@ -16,18 +15,15 @@ { "name": "invalidIPv4_with_netmask", "input": "1.1.1.1/32", - "expected_result": "invalid", }, { "name": "invalidIPv6_with_netmask", "input": "fe80::cc62:a9ff:feef:932a/128", - "expected_result": "invalid", }, {"name": "invalidHost_format", "input": "@", "expected_result": "invalid"}, { "name": "invalidIPv6_format", "input": "fe80::cc62:a9ff:feef:", - "expected_result": "invalid", }, ] @@ -253,55 +249,10 @@ }, ] -TEST_RESULT_UNIT = [ - { - "name": "valid_with_host_ip_only", - "input": {"host": "1.1.1.1", "test": "pytest_unit_test"}, - "expected_result": "valid", - }, - { - "name": "valid_with_host_ip_and_success_result", - "input": {"host": "1.1.1.1", "test": "pytest_unit_test", "result": "success"}, - "expected_result": "valid", - }, - { - "name": "valid_with_host_ip_and_skipped_result", - "input": {"host": "1.1.1.1", "test": "pytest_unit_test", "result": "success"}, - "expected_result": "valid", - }, - { - "name": "valid_with_host_ip_and_failure_result", - "input": {"host": "1.1.1.1", "test": "pytest_unit_test", "result": "failure"}, - "expected_result": "valid", - }, - { - "name": "valid_with_host_ip_and_error_result", - "input": {"host": "1.1.1.1", "test": "pytest_unit_test", "result": "error"}, - "expected_result": "valid", - }, - { - "name": "valid_full", - "input": { - "host": "1.1.1.1", - "test": "pytest_unit_test", - "result": "success", - "messages": ["test"], - }, - "expected_result": "valid", - }, - { - "name": "invalid_by_host", - "input": {"host": "demo.arista.com", "test": "pytest_unit_test"}, - "expected_result": "invalid", - }, - { - "name": "invalid_by_test", - "input": {"host": "1.1.1.1"}, - "expected_result": "invalid", - }, - { - "name": "invelid_by_result", - "input": {"host": "1.1.1.1", "test": "pytest_unit_test", "result": "ok"}, - "expected_result": "invalid", - }, +TEST_RESULT_SET_STATUS = [ + {"name": "set_success", "target": "success", "message": "success"}, + {"name": "set_error", "target": "error", "message": "error"}, + {"name": "set_failure", "target": "failure", "message": "failure"}, + {"name": "set_skipped", "target": "skipped", "message": "skipped"}, + {"name": "set_unset", "target": "unset", "message": "unset"}, ] diff --git a/tests/units/result_manager/test_models.py b/tests/units/result_manager/test_models.py index 54e802745..20815b66f 100644 --- a/tests/units/result_manager/test_models.py +++ b/tests/units/result_manager/test_models.py @@ -4,197 +4,54 @@ """ANTA Result Manager models unit tests.""" from __future__ import annotations -import logging -from typing import Any +from typing import Any, Callable import pytest from anta.result_manager.models import TestResult -from tests.data.json_data import TEST_RESULT_UNIT +from tests.data.json_data import TEST_RESULT_SET_STATUS from tests.lib.utils import generate_test_ids_dict -pytest.skip(reason="Not yet ready for CI", allow_module_level=True) +# pytest.skip(reason="Not yet ready for CI", allow_module_level=True) -class Test_InventoryUnitModels: +class TestTestResultModels: """Test components of anta.result_manager.models.""" - @pytest.mark.parametrize("test_definition", TEST_RESULT_UNIT, ids=generate_test_ids_dict) - def test_anta_result_init_valid(self, test_definition: dict[str, Any]) -> None: - """Test Result model. - - Test structure: - --------------- - { - 'name': 'valid_full', - 'input': {"host": '1.1.1.1', 'test': 'pytest_unit_test', 'result': 'success', 'messages': ['test']}, - 'expected_result': 'valid', - } - - """ - if test_definition["expected_result"] == "invalid": - pytest.skip("Not concerned by the test") - try: - result = TestResult(**test_definition["input"]) - logging.info(f"TestResult is {result.dict()}") - # pylint: disable=W0703 - except Exception as e: - logging.error(f"Error loading data:\n{str(e)}") - assert False - - @pytest.mark.parametrize("test_definition", TEST_RESULT_UNIT, ids=generate_test_ids_dict) - def test_anta_result_init_invalid(self, test_definition: dict[str, Any]) -> None: - """Test Result model. - - Test structure: - --------------- - { - 'name': 'valid_full', - 'input': {"host": '1.1.1.1', 'test': 'pytest_unit_test', 'result': 'success', 'messages': ['test']}, - 'expected_result': 'valid', - } - - """ - if test_definition["expected_result"] == "valid": - pytest.skip("Not concerned by the test") - try: - TestResult(**test_definition["input"]) - except ValueError as e: - logging.warning(f"Error loading data:\n{str(e)}") - else: - logging.error("An exception is expected here") - assert False - - @pytest.mark.parametrize("test_definition", TEST_RESULT_UNIT, ids=generate_test_ids_dict) - def test_anta_result_set_status_success(self, test_definition: dict[str, Any]) -> None: - """Test Result model. - - Test structure: - --------------- - { - 'name': 'valid_full', - 'input': {"host": '1.1.1.1', 'test': 'pytest_unit_test', 'result': 'success', 'messages': ['test']}, - 'expected_result': 'valid', - } - - """ - if test_definition["expected_result"] == "invalid": - pytest.skip("Not concerned by the test") - - result = TestResult(**test_definition["input"]) - - result.is_success() - assert result.result == "success" - result_message_len = len(result.messages) - - if "messages" in test_definition["input"]: - assert result_message_len == len(test_definition["input"]["messages"]) - else: - assert result_message_len == 0 - - # Adding one message - result.is_success("it is a great success") - assert result.result == "success" - assert len(result.messages) == result_message_len + 1 - - @pytest.mark.parametrize("test_definition", TEST_RESULT_UNIT, ids=generate_test_ids_dict) - def test_anta_result_set_status_failure(self, test_definition: dict[str, Any]) -> None: - """Test Result model. - - Test structure: - --------------- - { - 'name': 'valid_full', - 'input': {"host": '1.1.1.1', 'test': 'pytest_unit_test', 'result': 'success', 'messages': ['test']}, - 'expected_result': 'valid', - } - - """ - if test_definition["expected_result"] == "invalid": - pytest.skip("Not concerned by the test") - - result = TestResult(**test_definition["input"]) - - result.is_failure() - assert result.result == "failure" - result_message_len = len(result.messages) - - if "messages" in test_definition["input"]: - assert result_message_len == len(test_definition["input"]["messages"]) - else: - assert result_message_len == 0 - - # Adding one message - result.is_failure("it is a great failure") - assert result.result == "failure" - assert len(result.messages) == result_message_len + 1 - - @pytest.mark.parametrize("test_definition", TEST_RESULT_UNIT, ids=generate_test_ids_dict) - def test_anta_result_set_status_error(self, test_definition: dict[str, Any]) -> None: - """Test Result model. - - Test structure: - --------------- - { - 'name': 'valid_full', - 'input': {"host": '1.1.1.1', 'test': 'pytest_unit_test', 'result': 'success', 'messages': ['test']}, - 'expected_result': 'valid', - } - - """ - if test_definition["expected_result"] == "invalid": - pytest.skip("Not concerned by the test") - - result = TestResult(**test_definition["input"]) - - result.is_error() - assert result.result == "error" - result_message_len = len(result.messages) - - if "messages" in test_definition["input"]: - assert result_message_len == len(test_definition["input"]["messages"]) - else: - assert result_message_len == 0 - - # Adding one message - result.is_error(message="it is a great error") - assert result.result == "error" - assert len(result.messages) == result_message_len + 1 - - # Adding one exception object - e = Exception() - result.is_error(exception=e) - assert result.result == "error" - assert result.error == e - - @pytest.mark.parametrize("test_definition", TEST_RESULT_UNIT, ids=generate_test_ids_dict) - def test_anta_result_set_status_skipped(self, test_definition: dict[str, Any]) -> None: - """Test Result model. - - Test structure: - --------------- - { - 'name': 'valid_full', - 'input': {"host": '1.1.1.1', 'test': 'pytest_unit_test', 'result': 'success', 'messages': ['test']}, - 'expected_result': 'valid', - } - - """ - if test_definition["expected_result"] == "invalid": - pytest.skip("Not concerned by the test") - - result = TestResult(**test_definition["input"]) - - result.is_skipped() - assert result.result == "skipped" - result_message_len = len(result.messages) - - if "messages" in test_definition["input"]: - assert result_message_len == len(test_definition["input"]["messages"]) - else: - assert result_message_len == 0 - - # Adding one message - result.is_skipped("it is a great skipped") - assert result.result == "skipped" - assert len(result.messages) == result_message_len + 1 + @pytest.mark.parametrize("data", TEST_RESULT_SET_STATUS, ids=generate_test_ids_dict) + def test__is_status_foo(self, test_result_factory: Callable[[int], TestResult], data: dict[str, Any]) -> None: + """Test TestResult.is_foo methods.""" + testresult = test_result_factory(1) + assert testresult.result == "unset" + assert len(testresult.messages) == 0 + if data["target"] == "success": + testresult.is_success(data["message"]) + assert testresult.result == data["target"] + assert data["message"] in testresult.messages + if data["target"] == "failure": + testresult.is_failure(data["message"]) + assert testresult.result == data["target"] + assert data["message"] in testresult.messages + if data["target"] == "error": + testresult.is_error(data["message"]) + assert testresult.result == data["target"] + assert data["message"] in testresult.messages + if data["target"] == "skipped": + testresult.is_skipped(data["message"]) + assert testresult.result == data["target"] + assert data["message"] in testresult.messages + # no helper for unset, testing _set_status + if data["target"] == "unset": + testresult._set_status("unset", data["message"]) # pylint: disable=W0212 + assert testresult.result == data["target"] + assert data["message"] in testresult.messages + + @pytest.mark.parametrize("data", TEST_RESULT_SET_STATUS, ids=generate_test_ids_dict) + def test____str__(self, test_result_factory: Callable[[int], TestResult], data: dict[str, Any]) -> None: + """Test TestResult.__str__.""" + testresult = test_result_factory(1) + assert testresult.result == "unset" + assert len(testresult.messages) == 0 + testresult._set_status(data["target"], data["message"]) # pylint: disable=W0212 + assert testresult.result == data["target"] + assert str(testresult) == f"Test VerifyTest1 on device testdevice has result {data['target']}" From 2c39acdc6f4c3014de23c2130e0ad23770fbe649 Mon Sep 17 00:00:00 2001 From: gmuloc Date: Fri, 3 Nov 2023 16:40:06 +0100 Subject: [PATCH 26/53] Refactor(anta.cli)!: One envvar to rule them all --- anta/cli/__init__.py | 1 + anta/cli/check/commands.py | 1 + docs/cli/nrfu.md | 2 +- docs/getting-started.md | 2 +- tests/lib/utils.py | 2 +- 5 files changed, 5 insertions(+), 3 deletions(-) diff --git a/anta/cli/__init__.py b/anta/cli/__init__.py index f2e259470..e6ad9e5d2 100644 --- a/anta/cli/__init__.py +++ b/anta/cli/__init__.py @@ -142,6 +142,7 @@ def anta( @click.option( "--catalog", "-c", + envvar="ANTA_CATALOG", show_envvar=True, help="Path to the tests catalog YAML file", type=click.Path(file_okay=True, dir_okay=False, exists=True, readable=True), diff --git a/anta/cli/check/commands.py b/anta/cli/check/commands.py index ca4339625..793b82c59 100644 --- a/anta/cli/check/commands.py +++ b/anta/cli/check/commands.py @@ -25,6 +25,7 @@ @click.option( "--catalog", "-c", + envvar="ANTA_CATALOG", show_envvar=True, help="Path to the tests catalog YAML file", type=click.Path(file_okay=True, dir_okay=False, exists=True, readable=True, resolve_path=True), diff --git a/docs/cli/nrfu.md b/docs/cli/nrfu.md index 6b05cb246..10a481c25 100644 --- a/docs/cli/nrfu.md +++ b/docs/cli/nrfu.md @@ -23,7 +23,7 @@ Usage: anta nrfu [OPTIONS] COMMAND [ARGS]... Options: -c, --catalog FILE Path to the tests catalog YAML file [env var: - ANTA_NRFU_CATALOG; required] + ANTA_CATALOG; required] --help Show this message and exit. Commands: diff --git a/docs/getting-started.md b/docs/getting-started.md index ab7c4337f..d5487a9a7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -140,7 +140,7 @@ Usage: anta nrfu [OPTIONS] COMMAND [ARGS]... Options: -c, --catalog FILE Path to the tests catalog YAML file [env var: - ANTA_NRFU_CATALOG; required] + ANTA_CATALOG; required] --help Show this message and exit. Commands: diff --git a/tests/lib/utils.py b/tests/lib/utils.py index 6f2f5143a..996e2e412 100644 --- a/tests/lib/utils.py +++ b/tests/lib/utils.py @@ -45,5 +45,5 @@ def default_anta_env() -> dict[str, str]: "ANTA_USERNAME": "anta", "ANTA_PASSWORD": "formica", "ANTA_INVENTORY": str(Path(__file__).parent.parent / "data" / "test_inventory.yml"), - "ANTA_NRFU_CATALOG": str(Path(__file__).parent.parent / "data" / "test_catalog.yml"), + "ANTA_CATALOG": str(Path(__file__).parent.parent / "data" / "test_catalog.yml"), } From c32027274976273fa82c1e2cd609c66d120a2f1d Mon Sep 17 00:00:00 2001 From: Carl Baillargeon Date: Fri, 3 Nov 2023 15:40:01 -0400 Subject: [PATCH 27/53] Fix tags in runner --- anta/runner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/anta/runner.py b/anta/runner.py index bd664ed37..3b58f7bc2 100644 --- a/anta/runner.py +++ b/anta/runner.py @@ -75,7 +75,7 @@ async def main( tests.append((test, device)) else: # If there is no CLI tags, execute all tests without filters - tests.extend([(t, device) for t in catalog.tests if t.inputs.filters is None or t.inputs.filters.tags is None]) + tests.extend([(t, device) for t in catalog.tests if t.inputs.filters is None]) # Also execute tests with filters conditionally if device.tags: # If the device has tags, execute tests with matching tags @@ -83,7 +83,9 @@ async def main( tests.append((t, device)) # Also execute tests with filters on this device name for t in catalog.get_tests_by_device(device): - tests.append((t, device)) + # Only add the test if it's not in the tests list already + if (t, device) not in tests: + tests.append((t, device)) for test_definition, device in tests: try: From d866a8ec6c85ef143e8a42f21dc9f91fe98da840 Mon Sep 17 00:00:00 2001 From: gmuloc Date: Mon, 6 Nov 2023 10:11:44 +0100 Subject: [PATCH 28/53] Refactor: Check file catalog in constructor --- anta/catalog.py | 28 +++++++++------------ anta/cli/__init__.py | 5 ++-- anta/cli/check/commands.py | 9 +++---- anta/cli/get/commands.py | 2 +- anta/cli/nrfu/utils.py | 2 +- anta/cli/utils.py | 49 ++++++++++++++++++++++++------------- anta/runner.py | 46 ++++++++++++---------------------- tests/units/test_catalog.py | 6 +---- 8 files changed, 67 insertions(+), 80 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index 2dc1e4144..699bd9a44 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -218,11 +218,10 @@ def __init__(self, filename: str | None = None, tests: list[tuple[type[AntaTest] if filename is not None and tests: raise RuntimeError("'filename' and 'tests' arguments cannot be provided at the same time") self.filename: str | None = filename - self.tests: list[AntaTestDefinition] = [] + self._tests: list[AntaTestDefinition] = [] if tests is not None: - self.tests.extend(AntaTestDefinition(test=test, inputs=inputs) for test, inputs in tests) + self._tests.extend(AntaTestDefinition(test=test, inputs=inputs) for test, inputs in tests) self.file: AntaCatalogFile | None = None - self._data = None if self.filename: self._parse_file() @@ -243,28 +242,23 @@ def tests(self, value: list[AntaTestDefinition]) -> None: def _parse_file(self) -> None: """ Parse the catalog YAML file + + TODO add a flag to prevent override ? """ if self.filename: try: with open(file=self.filename, mode="r", encoding="UTF-8") as file: - self._data = safe_load(file) + data = safe_load(file) # pylint: disable-next=broad-exception-caught except Exception: logger.critical(f"Something went wrong while parsing {self.filename}") raise - - def check(self: AntaCatalog) -> None: - """ - Check if the data in the catalog file is valid - and populate `tests` instance attribute. - """ - if self._data is not None: - self.file = AntaCatalogFile(**self._data) - if self._tests: - logger.warning(f"Overriding AntaCatalog data from file {self.filename}") - self._tests = [] - for tests in self.file.root.values(): - self._tests.extend(tests) + self.file = AntaCatalogFile(**data) + if self._tests: + logger.warning(f"Overriding AntaCatalog data from file {self.filename}") + self._tests = [] + for tests in self.file.root.values(): + self._tests.extend(tests) def get_tests_by_tags(self, tags: list[str], strict: bool = False) -> list[AntaTestDefinition]: """ diff --git a/anta/cli/__init__.py b/anta/cli/__init__.py index e6ad9e5d2..c3bb825df 100644 --- a/anta/cli/__init__.py +++ b/anta/cli/__init__.py @@ -21,7 +21,7 @@ from anta.cli.exec import commands as exec_commands from anta.cli.get import commands as get_commands from anta.cli.nrfu import commands as nrfu_commands -from anta.cli.utils import AliasedGroup, IgnoreRequiredWithHelp, check_catalog, parse_catalog, parse_inventory +from anta.cli.utils import AliasedGroup, IgnoreRequiredWithHelp, parse_catalog_cb, parse_inventory from anta.logger import setup_logging from anta.result_manager import ResultManager @@ -147,11 +147,10 @@ def anta( help="Path to the tests catalog YAML file", type=click.Path(file_okay=True, dir_okay=False, exists=True, readable=True), required=True, - callback=parse_catalog, + callback=parse_catalog_cb, ) def _nrfu(ctx: click.Context, catalog: AntaCatalog) -> None: """Run NRFU against inventory devices""" - check_catalog(ctx, catalog) ctx.obj["catalog"] = catalog ctx.obj["result_manager"] = ResultManager() diff --git a/anta/cli/check/commands.py b/anta/cli/check/commands.py index 793b82c59..6422e82c8 100644 --- a/anta/cli/check/commands.py +++ b/anta/cli/check/commands.py @@ -2,20 +2,20 @@ # Use of this source code is governed by the Apache License 2.0 # that can be found in the LICENSE file. # pylint: disable = redefined-outer-name - """ Commands for Anta CLI to run check commands. """ from __future__ import annotations import logging +from pathlib import Path import click from rich.pretty import pretty_repr from anta.catalog import AntaCatalog from anta.cli.console import console -from anta.cli.utils import check_catalog, parse_catalog +from anta.cli.utils import parse_catalog logger = logging.getLogger(__name__) @@ -30,12 +30,11 @@ help="Path to the tests catalog YAML file", type=click.Path(file_okay=True, dir_okay=False, exists=True, readable=True, resolve_path=True), required=True, - callback=parse_catalog, ) -def catalog(ctx: click.Context, catalog: AntaCatalog) -> None: +def catalog(ctx: click.Context, catalog: Path) -> None: """ Check that the catalog is valid """ logger.info(f"Checking syntax of catalog {ctx.obj['catalog_path']}") - check_catalog(ctx, catalog) + catalog: AntaCatalog = parse_catalog(str(catalog)) console.print(pretty_repr(catalog.file)) diff --git a/anta/cli/get/commands.py b/anta/cli/get/commands.py index 86ca1b354..5dddfbaaf 100644 --- a/anta/cli/get/commands.py +++ b/anta/cli/get/commands.py @@ -4,7 +4,7 @@ # pylint: disable = redefined-outer-name """ -Commands for Anta CLI to run check commands. +Commands for Anta CLI to get information / build inventories.. """ from __future__ import annotations diff --git a/anta/cli/nrfu/utils.py b/anta/cli/nrfu/utils.py index 493651d40..17aa4352d 100644 --- a/anta/cli/nrfu/utils.py +++ b/anta/cli/nrfu/utils.py @@ -2,7 +2,7 @@ # Use of this source code is governed by the Apache License 2.0 # that can be found in the LICENSE file. """ -Utils functions to use with anta.cli.check.commands module. +Utils functions to use with anta.cli.nrfu.commands module. """ from __future__ import annotations diff --git a/anta/cli/utils.py b/anta/cli/utils.py index 83e780895..660efe765 100644 --- a/anta/cli/utils.py +++ b/anta/cli/utils.py @@ -45,20 +45,6 @@ class ExitCode(enum.IntEnum): USAGE_ERROR = 4 -def check_catalog(ctx: click.Context, catalog: AntaCatalog) -> None: - """ - Helper function to check test catalog file and print - output using console. - """ - try: - catalog.check() - console.print(f"[bold][green]Catalog {catalog.filename} is valid") - except ValidationError as e: - console.print(f"[bold][red]Catalog {catalog.filename} is invalid") - anta_log_exception(e) - ctx.exit(1) - - def parse_inventory(ctx: click.Context, path: Path) -> AntaInventory: """ Helper function parse an ANTA inventory YAML file @@ -95,7 +81,7 @@ def parse_tags(ctx: click.Context, param: Option, value: str) -> list[str] | Non return None -def parse_catalog(ctx: click.Context, param: Option, value: str) -> AntaCatalog: +def parse_catalog_cb(ctx: click.Context, param: Option, value: str) -> AntaCatalog: # pylint: disable=unused-argument """ Click option callback to parse an ANTA tests catalog YAML file @@ -109,8 +95,11 @@ def parse_catalog(ctx: click.Context, param: Option, value: str) -> AntaCatalog: # Storing catalog path ctx.obj["catalog_path"] = value try: - catalog = AntaCatalog(filename=value) - # TODO catch proper exception + catalog = parse_catalog(value) + except ValidationError as e: + message = "Catalog {value} is invalid!" + anta_log_exception(e, message, logger) + ctx.fail(message) # pylint: disable-next=broad-exception-caught except Exception as e: message = f"Unable to parse ANTA Tests Catalog file '{value}'" @@ -120,6 +109,32 @@ def parse_catalog(ctx: click.Context, param: Option, value: str) -> AntaCatalog: return catalog +def parse_catalog(catalog_path: str) -> AntaCatalog: + # pylint: disable=unused-argument + """ + Args: + ---- + catalog_path (str): Path to the catalog YAML file. + + Returns: + ------- + the AntaCatalog parsed file + + Raises: + ----- + ValidationError: If the catalog file is invalid + Exception: If anything happens with opening the catalog file + """ + try: + catalog = AntaCatalog(filename=catalog_path) + console.print(f"[bold][green]Catalog {catalog_path} is valid") + except ValidationError: + console.print(f"[bold][red]Catalog {catalog_path} is invalid!") + raise + + return catalog + + def exit_with_code(ctx: click.Context) -> None: """ Exit the Click application with an exit code. diff --git a/anta/runner.py b/anta/runner.py index 3b58f7bc2..067202ab4 100644 --- a/anta/runner.py +++ b/anta/runner.py @@ -23,13 +23,7 @@ AntaTestRunner = Tuple[AntaTestDefinition, AntaDevice] -async def main( - manager: ResultManager, - inventory: AntaInventory, - catalog: AntaCatalog, - tags: list[str] | None = None, - established_only: bool = True, -) -> None: +async def main(manager: ResultManager, inventory: AntaInventory, catalog: AntaCatalog, tags: list[str] | None = None, established_only: bool = True) -> None: """ Main coroutine to run ANTA. Use this as an entrypoint to the test framwork in your script. @@ -44,58 +38,50 @@ async def main( Returns: any: ResultManager object gets updated with the test results. """ - - catalog.check() if not catalog.tests: logger.info("The list of tests is empty, exiting") return - if len(inventory) == 0: logger.info("The inventory is empty, exiting") return - await inventory.connect_inventory() - devices: list[AntaDevice] = list(inventory.get_inventory(established_only=established_only, tags=tags).values()) - if len(devices) == 0: + if not devices: logger.info( f"No device in the established state '{established_only}' " f"{f'matching the tags {tags} ' if tags else ''}was found. There is no device to run tests against, exiting" ) - return + return coros = [] - - tests: list[AntaTestRunner] = [] + # Using a set to avoid inserting duplicate tests + tests_set: set[AntaTestRunner] = set() for device in devices: if tags: # If there are CLI tags, only execute tests with matching tags - for test in catalog.get_tests_by_tags(tags): - tests.append((test, device)) + tests_set.update((test, device) for test in catalog.get_tests_by_tags(tags)) else: # If there is no CLI tags, execute all tests without filters - tests.extend([(t, device) for t in catalog.tests if t.inputs.filters is None]) - # Also execute tests with filters conditionally + tests_set.update((t, device) for t in catalog.tests if t.inputs.filters is None or t.inputs.filters.tags is None) + if device.tags: - # If the device has tags, execute tests with matching tags - for t in catalog.get_tests_by_tags(device.tags): - tests.append((t, device)) + # If the device has tags, add the tests with matching tags + tests_set.update((t, device) for t in catalog.get_tests_by_tags(device.tags)) + # Also execute tests with filters on this device name - for t in catalog.get_tests_by_device(device): - # Only add the test if it's not in the tests list already - if (t, device) not in tests: - tests.append((t, device)) + tests_set.update((t, device) for t in catalog.get_tests_by_device(device)) + + tests: list[AntaTestRunner] = list(tests_set) for test_definition, device in tests: try: - # Instantiate AntaTest object test_instance = test_definition.test(device=device, inputs=test_definition.inputs) + coros.append(test_instance.test()) except Exception as e: # pylint: disable=broad-exception-caught message = "Error when creating ANTA tests" anta_log_exception(e, message, logger) - if AntaTest.progress is not None: AntaTest.nrfu_task = AntaTest.progress.add_task("Running NRFU Tests...", total=len(coros)) @@ -108,8 +94,6 @@ async def main( anta_log_exception(r, message, logger) else: manager.add_test_result(r) - - # Get each device statistics for device in devices: if device.cache_statistics is not None: logger.info(f"Cache statistics for {device.name}: {device.cache_statistics}") diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index 03e3b330d..ec492d1e9 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -163,7 +163,6 @@ def test__init__filename(self, catalog_data: dict[str, Any]) -> None: Instantiate AntaCatalog from a file """ catalog = AntaCatalog(filename=str(DATA_DIR / catalog_data["filename"])) - catalog.check() assert len(catalog.tests) == len(catalog_data["tests"]) for test_id, (test, inputs) in enumerate(catalog_data["tests"]): @@ -178,9 +177,8 @@ def test__init__filename_fail(self, catalog_data: dict[str, Any]) -> None: """ Errors when instantiating AntaCatalog from a file """ - catalog = AntaCatalog(filename=str(DATA_DIR / catalog_data["filename"])) with pytest.raises(ValidationError) as exec_info: - catalog.check() + AntaCatalog(filename=str(DATA_DIR / catalog_data["filename"])) assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] def test__init__filename_fail_parsing(self, caplog: pytest.LogCaptureFixture) -> None: @@ -233,7 +231,6 @@ def test_get_tests_by_tags(self) -> None: Test AntaCatalog.test_get_tests_by_tags() """ catalog = AntaCatalog(filename=str(DATA_DIR / "test_catalog_with_tags.yml")) - catalog.check() tests: list[AntaTestDefinition] = catalog.get_tests_by_tags(tags=["leaf"]) assert len(tests) == 2 @@ -242,6 +239,5 @@ def test_get_tests_by_device(self, mocked_device: AntaDevice) -> None: Test AntaCatalog.test_get_tests_by_device() """ catalog = AntaCatalog(filename=str(DATA_DIR / "test_catalog_with_tags.yml")) - catalog.check() tests: list[AntaTestDefinition] = catalog.get_tests_by_device(device=mocked_device) assert len(tests) == 1 From 63f5b8d6733decd369656e108efd72dda068ba50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Mon, 6 Nov 2023 10:28:56 +0100 Subject: [PATCH 29/53] refactor: remove broad except --- anta/catalog.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index 699bd9a44..ff91224e3 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -14,7 +14,7 @@ from pydantic import BaseModel, RootModel, ValidationInfo, field_validator, model_serializer, model_validator from pydantic.types import ImportString -from yaml import safe_load +from yaml import safe_load, YAMLError from anta.device import AntaDevice from anta.models import AntaTest @@ -249,8 +249,7 @@ def _parse_file(self) -> None: try: with open(file=self.filename, mode="r", encoding="UTF-8") as file: data = safe_load(file) - # pylint: disable-next=broad-exception-caught - except Exception: + except YAMLError: logger.critical(f"Something went wrong while parsing {self.filename}") raise self.file = AntaCatalogFile(**data) From e9a82418905c5661a573c3c8764e41028bb62b0d Mon Sep 17 00:00:00 2001 From: gmuloc Date: Mon, 6 Nov 2023 10:34:47 +0100 Subject: [PATCH 30/53] Fix: Fix anta cli check variable --- anta/cli/check/commands.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/anta/cli/check/commands.py b/anta/cli/check/commands.py index 6422e82c8..492582620 100644 --- a/anta/cli/check/commands.py +++ b/anta/cli/check/commands.py @@ -21,7 +21,6 @@ @click.command(no_args_is_help=True) -@click.pass_context @click.option( "--catalog", "-c", @@ -31,10 +30,10 @@ type=click.Path(file_okay=True, dir_okay=False, exists=True, readable=True, resolve_path=True), required=True, ) -def catalog(ctx: click.Context, catalog: Path) -> None: +def catalog(catalog: Path) -> None: """ Check that the catalog is valid """ - logger.info(f"Checking syntax of catalog {ctx.obj['catalog_path']}") - catalog: AntaCatalog = parse_catalog(str(catalog)) - console.print(pretty_repr(catalog.file)) + logger.info(f"Checking syntax of catalog {catalog}") + catalog_obj: AntaCatalog = parse_catalog(str(catalog)) + console.print(pretty_repr(catalog_obj.file)) From 03bf560495151797196c94ad83a842c43a86ce52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Mon, 6 Nov 2023 11:23:23 +0100 Subject: [PATCH 31/53] fix: missing f-string --- anta/cli/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anta/cli/utils.py b/anta/cli/utils.py index 660efe765..1bafa31f6 100644 --- a/anta/cli/utils.py +++ b/anta/cli/utils.py @@ -97,7 +97,7 @@ def parse_catalog_cb(ctx: click.Context, param: Option, value: str) -> AntaCatal try: catalog = parse_catalog(value) except ValidationError as e: - message = "Catalog {value} is invalid!" + message = f"Catalog {value} is invalid!" anta_log_exception(e, message, logger) ctx.fail(message) # pylint: disable-next=broad-exception-caught From 869e3353922c19c145fd1a834883944b1e7575c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Mon, 6 Nov 2023 11:23:35 +0100 Subject: [PATCH 32/53] fix: implement hashing for AntaTestDefinition --- anta/catalog.py | 6 ++++-- anta/device.py | 33 +++++++++++++++++++++++---------- anta/models.py | 7 +++++++ 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index ff91224e3..0e897eb11 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -12,9 +12,9 @@ from types import ModuleType from typing import Any, Dict, List, Type -from pydantic import BaseModel, RootModel, ValidationInfo, field_validator, model_serializer, model_validator +from pydantic import BaseModel, ConfigDict, RootModel, ValidationInfo, field_validator, model_serializer, model_validator from pydantic.types import ImportString -from yaml import safe_load, YAMLError +from yaml import YAMLError, safe_load from anta.device import AntaDevice from anta.models import AntaTest @@ -30,6 +30,8 @@ class AntaTestDefinition(BaseModel): inputs: The associated AntaTest.Input subclass instance """ + model_config = ConfigDict(frozen=True) + test: Type[AntaTest] inputs: AntaTest.Input diff --git a/anta/device.py b/anta/device.py index a0e6fc45e..dccd3bc50 100644 --- a/anta/device.py +++ b/anta/device.py @@ -65,6 +65,27 @@ def __init__(self, name: str, tags: Optional[list[str]] = None, disable_cache: b if not disable_cache: self._init_cache() + @property + @abstractmethod + def _keys(self) -> tuple[Any]: + """ + Read-only property to implement hashing and equality for AntaDevice classes. + """ + + def __eq__(self, other: object) -> bool: + """ + Implement equality for AntaDevice objects. + """ + if not isinstance(other, AsyncEOSDevice): + return False + return self._keys == other._keys + + def __hash__(self) -> int: + """ + Implement hashing for AntaDevice objects. + """ + return hash(self._keys) + def _init_cache(self) -> None: """ Initialize cache for the device, can be overriden by subclasses to manipulate how it works @@ -96,12 +117,6 @@ def __rich_repr__(self) -> Iterator[tuple[str, Any]]: yield "established", self.established yield "disable_cache", self.cache is None - @abstractmethod - def __eq__(self, other: object) -> bool: - """ - AntaDevice equality depends on the class implementation. - """ - @abstractmethod async def _collect(self, command: AntaCommand) -> None: """ @@ -261,14 +276,12 @@ def __rich_repr__(self) -> Iterator[tuple[str, Any]]: yield "_session", vars(self._session) yield "_ssh_opts", _ssh_opts - def __eq__(self, other: object) -> bool: + def _keys(self) -> tuple[Any]: """ Two AsyncEOSDevice objects are equal if the hostname and the port are the same. This covers the use case of port forwarding when the host is localhost and the devices have different ports. """ - if not isinstance(other, AsyncEOSDevice): - return False - return self._session.host == other._session.host and self._session.port == other._session.port + return (self._session.host, self._session.port) async def _collect(self, command: AntaCommand) -> None: """ diff --git a/anta/models.py b/anta/models.py index 7cb98d0ee..a2ca08d88 100644 --- a/anta/models.py +++ b/anta/models.py @@ -290,6 +290,13 @@ class Input(BaseModel): result_overwrite: Optional[ResultOverwrite] = None filters: Optional[Filters] = None + def __hash__(self) -> int: + """ + Implement generic hashing for AntaTest.Input. + This will work in most cases but this does not consider 2 lists with different ordering as equal. + """ + return hash(self.model_dump_json()) + class ResultOverwrite(BaseModel): """Test inputs model to overwrite result fields From 527216cde6883cfd844f344e3442f6f304e1f595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Mon, 6 Nov 2023 11:26:20 +0100 Subject: [PATCH 33/53] fix linting --- anta/device.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/anta/device.py b/anta/device.py index dccd3bc50..2d8c145ef 100644 --- a/anta/device.py +++ b/anta/device.py @@ -67,7 +67,7 @@ def __init__(self, name: str, tags: Optional[list[str]] = None, disable_cache: b @property @abstractmethod - def _keys(self) -> tuple[Any]: + def _keys(self) -> tuple[Any, ...]: """ Read-only property to implement hashing and equality for AntaDevice classes. """ @@ -276,7 +276,8 @@ def __rich_repr__(self) -> Iterator[tuple[str, Any]]: yield "_session", vars(self._session) yield "_ssh_opts", _ssh_opts - def _keys(self) -> tuple[Any]: + @property + def _keys(self) -> tuple[Any, ...]: """ Two AsyncEOSDevice objects are equal if the hostname and the port are the same. This covers the use case of port forwarding when the host is localhost and the devices have different ports. From 12853590ec6ed1f6afe27763205ba7cf67334683 Mon Sep 17 00:00:00 2001 From: gmuloc Date: Mon, 6 Nov 2023 11:24:27 +0100 Subject: [PATCH 34/53] Test: Clean ANTA_ env variables before running tests --- tests/lib/fixture.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/lib/fixture.py b/tests/lib/fixture.py index ef15d0f41..6a333a8c4 100644 --- a/tests/lib/fixture.py +++ b/tests/lib/fixture.py @@ -4,6 +4,7 @@ """Fixture for Anta Testing""" from __future__ import annotations +from os import environ from typing import Callable from unittest.mock import MagicMock, create_autospec @@ -117,3 +118,14 @@ def click_runner() -> CliRunner: Convenience fixture to return a click.CliRunner for cli testing """ return CliRunner() + + +@pytest.fixture(autouse=True) +def clean_anta_env_variables() -> None: + """ + Autouse fixture that cleans the various ANTA_FOO env variables + that could come from the user environment and make some tests fail. + """ + for envvar in environ: + if envvar.startswith("ANTA_"): + environ.pop(envvar) From c789f634af14aaac6baa4632688575582224113e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Mon, 6 Nov 2023 11:32:38 +0100 Subject: [PATCH 35/53] fix: remove no_args_is_help for 'anta check catalog' --- anta/cli/check/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anta/cli/check/commands.py b/anta/cli/check/commands.py index 492582620..4025f4184 100644 --- a/anta/cli/check/commands.py +++ b/anta/cli/check/commands.py @@ -20,7 +20,7 @@ logger = logging.getLogger(__name__) -@click.command(no_args_is_help=True) +@click.command() @click.option( "--catalog", "-c", From ccaf006ce997d089a367397bedf88c01065b8042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Mon, 6 Nov 2023 11:39:05 +0100 Subject: [PATCH 36/53] fix: catch OSError in AntaCatalog._parse_file() --- anta/catalog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anta/catalog.py b/anta/catalog.py index 0e897eb11..e7bb90c5b 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -251,7 +251,7 @@ def _parse_file(self) -> None: try: with open(file=self.filename, mode="r", encoding="UTF-8") as file: data = safe_load(file) - except YAMLError: + except (YAMLError, OSError): logger.critical(f"Something went wrong while parsing {self.filename}") raise self.file = AntaCatalogFile(**data) From e25271da42fddb7fcb8da6ef4a72c938c81d758e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Mon, 6 Nov 2023 11:59:15 +0100 Subject: [PATCH 37/53] refactor: remove get_tests_by_device() --- anta/catalog.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index e7bb90c5b..06399eea0 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -273,14 +273,3 @@ def get_tests_by_tags(self, tags: list[str], strict: bool = False) -> list[AntaT if (strict and all(t in tags for t in f)) or (not strict and any(t in tags for t in f)): result.append(test) return result - - def get_tests_by_device(self, device: AntaDevice) -> list[AntaTestDefinition]: - """ - Return all the tests that have the provided device in their input filters. - """ - result: list[AntaTestDefinition] = [] - for test in self.tests: - if test.inputs.filters and (f := test.inputs.filters.devices): - if device.name in f: - result.append(test) - return result From aeedbd82a9805e93ef62f5f97ca774bd1e02a14f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Mon, 6 Nov 2023 12:00:46 +0100 Subject: [PATCH 38/53] refactor: remove get_tests_by_device() --- anta/catalog.py | 1 - anta/device.py | 2 ++ anta/models.py | 4 ++-- anta/runner.py | 8 ++------ tests/data/test_catalog_with_tags.yml | 2 +- tests/units/test_catalog.py | 11 +---------- 6 files changed, 8 insertions(+), 20 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index 06399eea0..4bcb7d4a3 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -16,7 +16,6 @@ from pydantic.types import ImportString from yaml import YAMLError, safe_load -from anta.device import AntaDevice from anta.models import AntaTest logger = logging.getLogger(__name__) diff --git a/anta/device.py b/anta/device.py index 2d8c145ef..74d744e57 100644 --- a/anta/device.py +++ b/anta/device.py @@ -56,6 +56,8 @@ def __init__(self, name: str, tags: Optional[list[str]] = None, disable_cache: b self.name: str = name self.hw_model: Optional[str] = None self.tags: list[str] = tags if tags is not None else [] + # A device always has its own name as tag + self.tags.append(self.name) self.is_online: bool = False self.established: bool = False self.cache: Optional[Cache] = None diff --git a/anta/models.py b/anta/models.py index a2ca08d88..59cfa144f 100644 --- a/anta/models.py +++ b/anta/models.py @@ -306,6 +306,7 @@ class ResultOverwrite(BaseModel): custom_field: a free string that will be included in the TestResult object """ + model_config = ConfigDict(extra="forbid") description: Optional[str] = None categories: Optional[List[str]] = None custom_field: Optional[str] = None @@ -314,11 +315,10 @@ class Filters(BaseModel): """Runtime filters to map tests with list of tags or devices Attributes: - devices: List of devices for the test. tags: List of device's tags for the test. """ - devices: Optional[List[str]] = None + model_config = ConfigDict(extra="forbid") tags: Optional[List[str]] = None def __init__( diff --git a/anta/runner.py b/anta/runner.py index 067202ab4..fe47b78c4 100644 --- a/anta/runner.py +++ b/anta/runner.py @@ -65,12 +65,8 @@ async def main(manager: ResultManager, inventory: AntaInventory, catalog: AntaCa # If there is no CLI tags, execute all tests without filters tests_set.update((t, device) for t in catalog.tests if t.inputs.filters is None or t.inputs.filters.tags is None) - if device.tags: - # If the device has tags, add the tests with matching tags - tests_set.update((t, device) for t in catalog.get_tests_by_tags(device.tags)) - - # Also execute tests with filters on this device name - tests_set.update((t, device) for t in catalog.get_tests_by_device(device)) + # Then add the tests with matching tags from device tags + tests_set.update((t, device) for t in catalog.get_tests_by_tags(device.tags)) tests: list[AntaTestRunner] = list(tests_set) diff --git a/tests/data/test_catalog_with_tags.yml b/tests/data/test_catalog_with_tags.yml index 30f90e20e..0c8f5f60c 100644 --- a/tests/data/test_catalog_with_tags.yml +++ b/tests/data/test_catalog_with_tags.yml @@ -14,7 +14,7 @@ anta.tests.system: tags: ['leaf'] - VerifyMemoryUtilization: filters: - devices: ['testdevice'] + tags: ['testdevice'] - VerifyFileSystemUtilization: - VerifyNTP: diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index ec492d1e9..a9576a547 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -13,7 +13,6 @@ from pydantic import ValidationError from anta.catalog import AntaCatalog, AntaTestDefinition -from anta.device import AntaDevice from anta.models import AntaTest from anta.tests.configuration import VerifyZeroTouch from anta.tests.hardware import VerifyTemperature @@ -63,7 +62,7 @@ (VerifyCoredump, VerifyCoredump.Input()), (VerifyAgentLogs, AntaTest.Input()), (VerifyCPUUtilization, VerifyCPUUtilization.Input(filters=VerifyCPUUtilization.Input.Filters(tags=["leaf"]))), - (VerifyMemoryUtilization, VerifyMemoryUtilization.Input(filters=VerifyMemoryUtilization.Input.Filters(devices=["testdevice"]))), + (VerifyMemoryUtilization, VerifyMemoryUtilization.Input(filters=VerifyMemoryUtilization.Input.Filters(tags=["testdevice"]))), (VerifyFileSystemUtilization, None), (VerifyNTP, {}), (VerifyMlagStatus, None), @@ -233,11 +232,3 @@ def test_get_tests_by_tags(self) -> None: catalog = AntaCatalog(filename=str(DATA_DIR / "test_catalog_with_tags.yml")) tests: list[AntaTestDefinition] = catalog.get_tests_by_tags(tags=["leaf"]) assert len(tests) == 2 - - def test_get_tests_by_device(self, mocked_device: AntaDevice) -> None: - """ - Test AntaCatalog.test_get_tests_by_device() - """ - catalog = AntaCatalog(filename=str(DATA_DIR / "test_catalog_with_tags.yml")) - tests: list[AntaTestDefinition] = catalog.get_tests_by_device(device=mocked_device) - assert len(tests) == 1 From 33995ca6dd2f98d201c80fd61973f04eb13146e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Mon, 6 Nov 2023 14:17:36 +0100 Subject: [PATCH 39/53] fix: log and exit if no tests can be run --- anta/runner.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/anta/runner.py b/anta/runner.py index fe47b78c4..60ab6592a 100644 --- a/anta/runner.py +++ b/anta/runner.py @@ -70,6 +70,13 @@ async def main(manager: ResultManager, inventory: AntaInventory, catalog: AntaCa tests: list[AntaTestRunner] = list(tests_set) + if not tests: + logger.info( + f"There is no tests{f' matching the tags {tags} ' if tags else ' '}to run on current inventory. " + "Exiting..." + ) + return + for test_definition, device in tests: try: test_instance = test_definition.test(device=device, inputs=test_definition.inputs) From 10a000094207ae972ec23769b84bb961039e0e0d Mon Sep 17 00:00:00 2001 From: gmuloc Date: Mon, 6 Nov 2023 14:20:29 +0100 Subject: [PATCH 40/53] Refactor: Apply sourcery --- anta/device.py | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/anta/device.py b/anta/device.py index 74d744e57..0a13e27fd 100644 --- a/anta/device.py +++ b/anta/device.py @@ -78,9 +78,7 @@ def __eq__(self, other: object) -> bool: """ Implement equality for AntaDevice objects. """ - if not isinstance(other, AsyncEOSDevice): - return False - return self._keys == other._keys + return self._keys == other._keys if isinstance(other, self.__class__) else False def __hash__(self) -> int: """ @@ -254,7 +252,7 @@ def __init__( # pylint: disable=R0913 self._session: Device = Device(host=host, port=port, username=username, password=password, proto=proto, timeout=timeout) ssh_params: dict[str, Any] = {} if insecure: - ssh_params.update({"known_hosts": None}) + ssh_params["known_hosts"] = None self._ssh_opts: SSHClientConnectionOptions = SSHClientConnectionOptions(host=host, port=ssh_port, username=username, password=password, **ssh_params) def __rich_repr__(self) -> Iterator[tuple[str, Any]]: @@ -262,21 +260,19 @@ def __rich_repr__(self) -> Iterator[tuple[str, Any]]: Implements Rich Repr Protocol https://rich.readthedocs.io/en/stable/pretty.html#rich-repr-protocol """ - - PASSWORD_VALUE = "" - yield from super().__rich_repr__() - yield "host", self._session.host - yield "eapi_port", self._session.port - yield "username", self._ssh_opts.username - yield "enable", self.enable - yield "insecure", self._ssh_opts.known_hosts is None + yield ("host", self._session.host) + yield ("eapi_port", self._session.port) + yield ("username", self._ssh_opts.username) + yield ("enable", self.enable) + yield ("insecure", self._ssh_opts.known_hosts is None) if __DEBUG__: _ssh_opts = vars(self._ssh_opts).copy() + PASSWORD_VALUE = "" _ssh_opts["password"] = PASSWORD_VALUE _ssh_opts["kwargs"]["password"] = PASSWORD_VALUE - yield "_session", vars(self._session) - yield "_ssh_opts", _ssh_opts + yield ("_session", vars(self._session)) + yield ("_ssh_opts", _ssh_opts) @property def _keys(self) -> tuple[Any, ...]: @@ -349,26 +345,28 @@ async def refresh(self) -> None: - established: When a command execution succeeds - hw_model: The hardware model of the device """ - # Refresh command - COMMAND: str = "show version" - # Hardware model definition in show version - HW_MODEL_KEY: str = "modelName" logger.debug(f"Refreshing device {self.name}") self.is_online = await self._session.check_connection() if self.is_online: + COMMAND: str = "show version" + HW_MODEL_KEY: str = "modelName" try: response = await self._session.cli(command=COMMAND) except EapiCommandError as e: logger.warning(f"Cannot get hardware information from device {self.name}: {e.errmsg}") + except (HTTPError, ConnectError) as e: logger.warning(f"Cannot get hardware information from device {self.name}: {exc_to_str(e)}") + else: if HW_MODEL_KEY in response: self.hw_model = response[HW_MODEL_KEY] else: logger.warning(f"Cannot get hardware information from device {self.name}: cannot parse '{COMMAND}'") + else: logger.warning(f"Could not connect to device {self.name}: cannot open eAPI port") + self.established = bool(self.is_online and self.hw_model) async def copy(self, sources: list[Path], destination: Path, direction: Literal["to", "from"] = "from") -> None: @@ -395,12 +393,15 @@ async def copy(self, sources: list[Path], destination: Path, direction: Literal[ dst = destination for file in sources: logger.info(f"Copying '{file}' from device {self.name} to '{destination}' locally") + elif direction == "to": src = sources - dst = (conn, destination) - for file in sources: + dst = conn, destination + for file in src: logger.info(f"Copying '{file}' to device {self.name} to '{destination}' remotely") + else: logger.critical(f"'direction' argument to copy() fonction is invalid: {direction}") + return await asyncssh.scp(src, dst) From 676107fd6c7d011eeb2f01b622063582576849ad Mon Sep 17 00:00:00 2001 From: gmuloc Date: Mon, 6 Nov 2023 15:30:41 +0100 Subject: [PATCH 41/53] Feat: Add support for raw input catalog --- anta/catalog.py | 33 ++++++++++++++++++++++++--------- anta/custom_types.py | 13 +++++++++++-- anta/runner.py | 5 +---- 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index 4bcb7d4a3..1c561dcd9 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -16,6 +16,7 @@ from pydantic.types import ImportString from yaml import YAMLError, safe_load +from anta.custom_types import ListAntaTestTuples, RawCatalogInput from anta.models import AntaTest logger = logging.getLogger(__name__) @@ -208,22 +209,29 @@ class AntaCatalog: file: The AntaCatalogFile model representing the catalog file. """ - def __init__(self, filename: str | None = None, tests: list[tuple[type[AntaTest], AntaTest.Input | dict[str, Any] | None]] | None = None) -> None: + def __init__(self, filename: str | None = None, raw_catalog_input: RawCatalogInput | None = None, tests: ListAntaTestTuples | None = None) -> None: """ - Constructor of AntaCatalog + Constructor of AntaCatalog. Args: + ---- filename: The path from which the catalog is loaded. Use this argument if you want to load the catalog from a file. + raw_catalog_input: The structure of a safe_loaded YAML file representing the catalog. tests: A list of tuple containing an AntaTest class and the associated input. Use this argument if you want to define the catalog programmatically. """ - if filename is not None and tests: - raise RuntimeError("'filename' and 'tests' arguments cannot be provided at the same time") - self.filename: str | None = filename + if len([var for var in [filename, raw_catalog_input, tests] if var is not None]) != 1: + raise RuntimeError("Exactly one of ['filename', 'raw_catalog_input','tests'] MUST be set at the same time.") self._tests: list[AntaTestDefinition] = [] + if tests is not None: self._tests.extend(AntaTestDefinition(test=test, inputs=inputs) for test, inputs in tests) - self.file: AntaCatalogFile | None = None - if self.filename: + + elif raw_catalog_input is not None: + # TODO fix type ignore + self._parse_anta_catalog_file(AntaCatalogFile(**raw_catalog_input)) # type: ignore[arg-type] + + elif filename is not None: + self.filename: str | None = filename self._parse_file() @property @@ -253,11 +261,18 @@ def _parse_file(self) -> None: except (YAMLError, OSError): logger.critical(f"Something went wrong while parsing {self.filename}") raise - self.file = AntaCatalogFile(**data) + + self._parse_anta_catalog_file(AntaCatalogFile(**data)) + + def _parse_anta_catalog_file(self, raw_catalog_input: AntaCatalogFile) -> None: + """ + Parse the catalog YAML file + + """ if self._tests: logger.warning(f"Overriding AntaCatalog data from file {self.filename}") self._tests = [] - for tests in self.file.root.values(): + for tests in raw_catalog_input.root.values(): self._tests.extend(tests) def get_tests_by_tags(self, tags: list[str], strict: bool = False) -> list[AntaTestDefinition]: diff --git a/anta/custom_types.py b/anta/custom_types.py index 79fd4e8f7..8a1db9d14 100644 --- a/anta/custom_types.py +++ b/anta/custom_types.py @@ -4,12 +4,14 @@ """ Module that provides predefined types for AntaTest.Input instances """ -from typing import Literal +from typing import Any, Literal from pydantic import Field from pydantic.functional_validators import AfterValidator from typing_extensions import Annotated +from anta.models import AntaTest + def aaa_group_prefix(v: str) -> str: """Prefix the AAA method with 'group' if it is known""" @@ -17,10 +19,17 @@ def aaa_group_prefix(v: str) -> str: return f"group {v}" if v not in built_in_methods and not v.startswith("group ") else v +# ANTA framework +TestStatus = Literal["unset", "success", "failure", "error", "skipped"] +# { : { : } } +RawCatalogInput = dict[str, list[dict[str, dict[str, Any] | None]]] +# [ ( , ), ... ] +ListAntaTestTuples = list[tuple[type[AntaTest], AntaTest.Input | dict[str, Any] | None]] + +# AntaTest.Input types AAAAuthMethod = Annotated[str, AfterValidator(aaa_group_prefix)] Vlan = Annotated[int, Field(ge=0, le=4094)] Vni = Annotated[int, Field(ge=1, le=16777215)] -TestStatus = Literal["unset", "success", "failure", "error", "skipped"] Interface = Annotated[str, Field(pattern=r"^(Ethernet|Fabric|Loopback|Management|Port-Channel|Tunnel|Vlan|Vxlan)[0-9]+(\/[0-9]+)*(\.[0-9]+)?$")] Afi = Literal["ipv4", "ipv6", "vpn-ipv4", "vpn-ipv6", "evpn", "rt-membership"] Safi = Literal["unicast", "multicast", "labeled-unicast"] diff --git a/anta/runner.py b/anta/runner.py index 60ab6592a..448bc1059 100644 --- a/anta/runner.py +++ b/anta/runner.py @@ -71,10 +71,7 @@ async def main(manager: ResultManager, inventory: AntaInventory, catalog: AntaCa tests: list[AntaTestRunner] = list(tests_set) if not tests: - logger.info( - f"There is no tests{f' matching the tags {tags} ' if tags else ' '}to run on current inventory. " - "Exiting..." - ) + logger.info(f"There is no tests{f' matching the tags {tags} ' if tags else ' '}to run on current inventory. " "Exiting...") return for test_definition, device in tests: From 5c8b17d77eeb4b0c69c1dd9b371a65d60d499de8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Mon, 6 Nov 2023 15:54:22 +0100 Subject: [PATCH 42/53] gm breaks everything --- anta/catalog.py | 19 +++++++++---------- anta/cli/check/commands.py | 2 +- anta/custom_types.py | 8 +------- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index 1c561dcd9..0f7a3dcc5 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -10,18 +10,24 @@ import logging from inspect import isclass from types import ModuleType -from typing import Any, Dict, List, Type +from typing import Any, Dict, List, Optional, Type, Union -from pydantic import BaseModel, ConfigDict, RootModel, ValidationInfo, field_validator, model_serializer, model_validator +from pydantic import BaseModel, ConfigDict, RootModel, ValidationInfo, field_validator, model_validator from pydantic.types import ImportString from yaml import YAMLError, safe_load -from anta.custom_types import ListAntaTestTuples, RawCatalogInput from anta.models import AntaTest logger = logging.getLogger(__name__) +# { : [ { : }, ... ] } +RawCatalogInput = dict[str, list[dict[str, Optional[dict[str, Any]]]]] + +# [ ( , ), ... ] +ListAntaTestTuples = list[tuple[type[AntaTest], Optional[Union[AntaTest.Input, dict[str, Any]]]]] + + class AntaTestDefinition(BaseModel): """ Define a test with its associated inputs. @@ -47,13 +53,6 @@ def __init__(self, **data: Any) -> None: ) super(BaseModel, self).__init__() - @model_serializer - def ser_model(self) -> Dict[str, AntaTest.Input]: - """ - Serialize an AntaTestDefinition as it is defined in a test catalog YAML file. - """ - return {self.test.__name__: self.inputs} - @field_validator("inputs", mode="before") @classmethod def instantiate_inputs(cls, data: AntaTest.Input | dict[str, Any] | None, info: ValidationInfo) -> AntaTest.Input: diff --git a/anta/cli/check/commands.py b/anta/cli/check/commands.py index 4025f4184..4a40478dd 100644 --- a/anta/cli/check/commands.py +++ b/anta/cli/check/commands.py @@ -36,4 +36,4 @@ def catalog(catalog: Path) -> None: """ logger.info(f"Checking syntax of catalog {catalog}") catalog_obj: AntaCatalog = parse_catalog(str(catalog)) - console.print(pretty_repr(catalog_obj.file)) + console.print(pretty_repr(catalog_obj.tests)) diff --git a/anta/custom_types.py b/anta/custom_types.py index 8a1db9d14..bb265a0bf 100644 --- a/anta/custom_types.py +++ b/anta/custom_types.py @@ -4,14 +4,12 @@ """ Module that provides predefined types for AntaTest.Input instances """ -from typing import Any, Literal +from typing import Literal from pydantic import Field from pydantic.functional_validators import AfterValidator from typing_extensions import Annotated -from anta.models import AntaTest - def aaa_group_prefix(v: str) -> str: """Prefix the AAA method with 'group' if it is known""" @@ -21,10 +19,6 @@ def aaa_group_prefix(v: str) -> str: # ANTA framework TestStatus = Literal["unset", "success", "failure", "error", "skipped"] -# { : { : } } -RawCatalogInput = dict[str, list[dict[str, dict[str, Any] | None]]] -# [ ( , ), ... ] -ListAntaTestTuples = list[tuple[type[AntaTest], AntaTest.Input | dict[str, Any] | None]] # AntaTest.Input types AAAAuthMethod = Annotated[str, AfterValidator(aaa_group_prefix)] From a564be227655574548a4087cccf9714b9595e666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Mon, 6 Nov 2023 16:02:34 +0100 Subject: [PATCH 43/53] fix unit tests --- anta/catalog.py | 4 ++-- tests/units/test_catalog.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index 0f7a3dcc5..aafe3dfab 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -218,8 +218,8 @@ def __init__(self, filename: str | None = None, raw_catalog_input: RawCatalogInp raw_catalog_input: The structure of a safe_loaded YAML file representing the catalog. tests: A list of tuple containing an AntaTest class and the associated input. Use this argument if you want to define the catalog programmatically. """ - if len([var for var in [filename, raw_catalog_input, tests] if var is not None]) != 1: - raise RuntimeError("Exactly one of ['filename', 'raw_catalog_input','tests'] MUST be set at the same time.") + if len([var for var in [filename, raw_catalog_input, tests] if var is not None]) > 1: + raise RuntimeError("Exactly one of filename, raw_catalog_input or tests MUST be set at the same time.") self._tests: list[AntaTestDefinition] = [] if tests is not None: diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index a9576a547..46fe692f3 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -153,7 +153,7 @@ def test__init__filename_and_tests(self) -> None: """ Instantiate AntaCatalog from a file and give tests at the same time """ - with pytest.raises(RuntimeError, match="'filename' and 'tests' arguments cannot be provided at the same time"): + with pytest.raises(RuntimeError, match="Exactly one of filename, raw_catalog_input or tests MUST be set at the same time."): AntaCatalog(filename=str(DATA_DIR / "test_catalog.yml"), tests=INIT_CATALOG_DATA[0]["tests"]) @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) From 671bcabed1b02b19b937f18587d07662a1b5db8f Mon Sep 17 00:00:00 2001 From: gmuloc Date: Mon, 6 Nov 2023 16:15:00 +0100 Subject: [PATCH 44/53] Test: Refactor pedantically --- anta/catalog.py | 2 +- tests/units/test_catalog.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index aafe3dfab..18dc212af 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -219,7 +219,7 @@ def __init__(self, filename: str | None = None, raw_catalog_input: RawCatalogInp tests: A list of tuple containing an AntaTest class and the associated input. Use this argument if you want to define the catalog programmatically. """ if len([var for var in [filename, raw_catalog_input, tests] if var is not None]) > 1: - raise RuntimeError("Exactly one of filename, raw_catalog_input or tests MUST be set at the same time.") + raise RuntimeError("Maximum one of filename, raw_catalog_input or tests MUST be set at the same time.") self._tests: list[AntaTestDefinition] = [] if tests is not None: diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index 46fe692f3..c9147a989 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -153,7 +153,7 @@ def test__init__filename_and_tests(self) -> None: """ Instantiate AntaCatalog from a file and give tests at the same time """ - with pytest.raises(RuntimeError, match="Exactly one of filename, raw_catalog_input or tests MUST be set at the same time."): + with pytest.raises(RuntimeError, match="Maximum one of filename, raw_catalog_input or tests MUST be set at the same time."): AntaCatalog(filename=str(DATA_DIR / "test_catalog.yml"), tests=INIT_CATALOG_DATA[0]["tests"]) @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) From 2dc6b699aefa6033a2bea3739782c9d678fe356c Mon Sep 17 00:00:00 2001 From: gmuloc Date: Mon, 6 Nov 2023 17:10:03 +0100 Subject: [PATCH 45/53] CI: Make 3.8 happy again --- anta/catalog.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index 18dc212af..a740c6b4e 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -10,7 +10,7 @@ import logging from inspect import isclass from types import ModuleType -from typing import Any, Dict, List, Optional, Type, Union +from typing import Any, Dict, List, Optional, Tuple, Type, Union from pydantic import BaseModel, ConfigDict, RootModel, ValidationInfo, field_validator, model_validator from pydantic.types import ImportString @@ -20,12 +20,11 @@ logger = logging.getLogger(__name__) - # { : [ { : }, ... ] } -RawCatalogInput = dict[str, list[dict[str, Optional[dict[str, Any]]]]] +RawCatalogInput = Dict[str, List[Dict[str, Optional[Dict[str, Any]]]]] # [ ( , ), ... ] -ListAntaTestTuples = list[tuple[type[AntaTest], Optional[Union[AntaTest.Input, dict[str, Any]]]]] +ListAntaTestTuples = List[Tuple[Type[AntaTest], Optional[Union[AntaTest.Input, Dict[str, Any]]]]] class AntaTestDefinition(BaseModel): From e54ff7e88d7e364754ad15f27873b5b4d1c2d54d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Wed, 8 Nov 2023 15:08:33 +0100 Subject: [PATCH 46/53] refactor: introduce AntaCatalog.parse(), AntaCatalog.from_list() and AntaCatalog.from_dict() --- anta/catalog.py | 94 +++++++++++++++++++++++-------------- tests/units/test_catalog.py | 77 +++++++++++++++++------------- 2 files changed, 104 insertions(+), 67 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index a740c6b4e..b2cb20430 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -8,6 +8,7 @@ import importlib import logging +from pathlib import Path from inspect import isclass from types import ModuleType from typing import Any, Dict, List, Optional, Tuple, Type, Union @@ -204,33 +205,31 @@ class AntaCatalog: Attributes: filename: The path from which the catalog is loaded. tests: A list of tuple containing an AntaTest class and the associated input. - file: The AntaCatalogFile model representing the catalog file. """ - def __init__(self, filename: str | None = None, raw_catalog_input: RawCatalogInput | None = None, tests: ListAntaTestTuples | None = None) -> None: + def __init__(self, tests: list[AntaTestDefinition] | None = None, filename: str | Path | None = None) -> None: """ Constructor of AntaCatalog. Args: ---- - filename: The path from which the catalog is loaded. Use this argument if you want to load the catalog from a file. - raw_catalog_input: The structure of a safe_loaded YAML file representing the catalog. tests: A list of tuple containing an AntaTest class and the associated input. Use this argument if you want to define the catalog programmatically. + filename: The path from which the catalog is loaded. This will be set as instance attribute. """ - if len([var for var in [filename, raw_catalog_input, tests] if var is not None]) > 1: - raise RuntimeError("Maximum one of filename, raw_catalog_input or tests MUST be set at the same time.") self._tests: list[AntaTestDefinition] = [] - if tests is not None: - self._tests.extend(AntaTestDefinition(test=test, inputs=inputs) for test, inputs in tests) - - elif raw_catalog_input is not None: - # TODO fix type ignore - self._parse_anta_catalog_file(AntaCatalogFile(**raw_catalog_input)) # type: ignore[arg-type] + self._tests = tests + self._filename: Path | None = None + if filename is not None: + if isinstance(filename, Path): + self._filename = filename + else: + self._filename = Path(filename) - elif filename is not None: - self.filename: str | None = filename - self._parse_file() + @property + def filename(self) -> Path | None: + """Path of the file used to create this AntaCatalog instance""" + return self._filename @property def tests(self) -> list[AntaTestDefinition]: @@ -246,32 +245,59 @@ def tests(self, value: list[AntaTestDefinition]) -> None: raise ValueError("A test in the catalog must be an AntaTestDefinition instance") self._tests = value - def _parse_file(self) -> None: + @staticmethod + def parse( + filename: str + ) -> AntaCatalog: """ - Parse the catalog YAML file + Create an AntaCatalog instance from a test catalog file. - TODO add a flag to prevent override ? + Args: + filename: Path to test catalog YAML file """ - if self.filename: - try: - with open(file=self.filename, mode="r", encoding="UTF-8") as file: - data = safe_load(file) - except (YAMLError, OSError): - logger.critical(f"Something went wrong while parsing {self.filename}") - raise - - self._parse_anta_catalog_file(AntaCatalogFile(**data)) + try: + with open(file=filename, mode="r", encoding="UTF-8") as file: + data = safe_load(file) + except (YAMLError, OSError): + logger.critical(f"Something went wrong while parsing {filename}") + raise + tests: list[AntaTestDefinition] = [] + for t in AntaCatalogFile(**data).root.values(): + tests.extend(t) + return AntaCatalog(tests, filename=filename) + + @staticmethod + def from_dict( + data: RawCatalogInput + ) -> AntaCatalog: + """ + Create an AntaCatalog instance from a dictionary data structure. + See RawCatalogInput type alias for details. + It is the data structure returned by `yaml.load()` function of a valid + YAML Test Catalog file. - def _parse_anta_catalog_file(self, raw_catalog_input: AntaCatalogFile) -> None: + Args: + data: Python dictionary used to instantiate the AntaCatalog instance """ - Parse the catalog YAML file + tests: list[AntaTestDefinition] = [] + for t in AntaCatalogFile(**data).root.values(): # type: ignore[arg-type] + tests.extend(t) + return AntaCatalog(tests) + + @staticmethod + def from_list( + data: ListAntaTestTuples + ) -> AntaCatalog: + """ + Create an AntaCatalog instance from a list data structure. + See ListAntaTestTuples type alias for details. + Args: + data: Python list used to instantiate the AntaCatalog instance """ - if self._tests: - logger.warning(f"Overriding AntaCatalog data from file {self.filename}") - self._tests = [] - for tests in raw_catalog_input.root.values(): - self._tests.extend(tests) + tests: list[AntaTestDefinition] = [] + tests.extend(AntaTestDefinition(test=test, inputs=inputs) for test, inputs in data) + return AntaCatalog(tests) def get_tests_by_tags(self, tags: list[str], strict: bool = False) -> list[AntaTestDefinition]: """ diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index c9147a989..a87dee5a8 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -11,6 +11,7 @@ import pytest from pydantic import ValidationError +from yaml import safe_load from anta.catalog import AntaCatalog, AntaTestDefinition from anta.models import AntaTest @@ -70,7 +71,7 @@ ], }, ] -INIT_CATALOG_FILENAME_FAIL_DATA: list[dict[str, Any]] = [ +CATALOG_PARSE_FAIL_DATA: list[dict[str, Any]] = [ { "name": "undefined_tests", "filename": "test_catalog_with_undefined_tests.yml", @@ -102,7 +103,7 @@ "error": "Value error, AntaTestDefinition must be a dictionary with a single entry", }, ] -INIT_CATALOG_TESTS_FAIL_DATA: list[dict[str, Any]] = [ +CATALOG_FROM_LIST_FAIL_DATA: list[dict[str, Any]] = [ { "name": "wrong_inputs", "tests": [ @@ -149,19 +150,27 @@ class Test_AntaCatalog: Test for anta.catalog.AntaCatalog """ - def test__init__filename_and_tests(self) -> None: + @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) + def test_parse(self, catalog_data: dict[str, Any]) -> None: """ - Instantiate AntaCatalog from a file and give tests at the same time + Instantiate AntaCatalog from a file """ - with pytest.raises(RuntimeError, match="Maximum one of filename, raw_catalog_input or tests MUST be set at the same time."): - AntaCatalog(filename=str(DATA_DIR / "test_catalog.yml"), tests=INIT_CATALOG_DATA[0]["tests"]) + catalog: AntaCatalog = AntaCatalog.parse(str(DATA_DIR / catalog_data["filename"])) + + assert len(catalog.tests) == len(catalog_data["tests"]) + for test_id, (test, inputs) in enumerate(catalog_data["tests"]): + assert catalog.tests[test_id].test == test + if inputs is not None: + if isinstance(inputs, dict): + inputs = test.Input(**inputs) + assert inputs == catalog.tests[test_id].inputs @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) - def test__init__filename(self, catalog_data: dict[str, Any]) -> None: + def test_from_list(self, catalog_data: dict[str, Any]) -> None: """ - Instantiate AntaCatalog from a file + Instantiate AntaCatalog from a list """ - catalog = AntaCatalog(filename=str(DATA_DIR / catalog_data["filename"])) + catalog: AntaCatalog = AntaCatalog.from_list(catalog_data["tests"]) assert len(catalog.tests) == len(catalog_data["tests"]) for test_id, (test, inputs) in enumerate(catalog_data["tests"]): @@ -171,48 +180,50 @@ def test__init__filename(self, catalog_data: dict[str, Any]) -> None: inputs = test.Input(**inputs) assert inputs == catalog.tests[test_id].inputs - @pytest.mark.parametrize("catalog_data", INIT_CATALOG_FILENAME_FAIL_DATA, ids=generate_test_ids_list(INIT_CATALOG_FILENAME_FAIL_DATA)) - def test__init__filename_fail(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) + def test_from_dict(self, catalog_data: dict[str, Any]) -> None: + """ + Instantiate AntaCatalog from a dict + """ + with open(file=str(DATA_DIR / catalog_data["filename"]), mode="r", encoding="UTF-8") as file: + data = safe_load(file) + catalog: AntaCatalog = AntaCatalog.from_dict(data) + + assert len(catalog.tests) == len(catalog_data["tests"]) + for test_id, (test, inputs) in enumerate(catalog_data["tests"]): + assert catalog.tests[test_id].test == test + if inputs is not None: + if isinstance(inputs, dict): + inputs = test.Input(**inputs) + assert inputs == catalog.tests[test_id].inputs + + @pytest.mark.parametrize("catalog_data", CATALOG_PARSE_FAIL_DATA, ids=generate_test_ids_list(CATALOG_PARSE_FAIL_DATA)) + def test_parse_fail(self, catalog_data: dict[str, Any]) -> None: """ Errors when instantiating AntaCatalog from a file """ with pytest.raises(ValidationError) as exec_info: - AntaCatalog(filename=str(DATA_DIR / catalog_data["filename"])) + AntaCatalog.parse(str(DATA_DIR / catalog_data["filename"])) assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] - def test__init__filename_fail_parsing(self, caplog: pytest.LogCaptureFixture) -> None: + def test_parse_fail_parsing(self, caplog: pytest.LogCaptureFixture) -> None: """ Errors when instantiating AntaCatalog from a file """ with pytest.raises(Exception) as exec_info: - AntaCatalog(filename=str(DATA_DIR / "catalog_does_not_exist.yml")) + AntaCatalog.parse(str(DATA_DIR / "catalog_does_not_exist.yml")) assert "No such file or directory" in str(exec_info) assert len(caplog.record_tuples) == 1 _, _, message = caplog.record_tuples[0] assert "Something went wrong while parsing" in message - @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) - def test__init__tests(self, catalog_data: dict[str, Any]) -> None: - """ - Instantiate AntaCatalog from a list of tuples - """ - catalog = AntaCatalog(tests=catalog_data["tests"]) - - assert len(catalog.tests) == len(catalog_data["tests"]) - for test_id, (test, inputs) in enumerate(catalog_data["tests"]): - assert catalog.tests[test_id].test == test - if inputs is not None: - if isinstance(inputs, dict): - inputs = test.Input(**inputs) - assert inputs == catalog.tests[test_id].inputs - - @pytest.mark.parametrize("catalog_data", INIT_CATALOG_TESTS_FAIL_DATA, ids=generate_test_ids_list(INIT_CATALOG_TESTS_FAIL_DATA)) - def test__init__tests_fail(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize("catalog_data", CATALOG_FROM_LIST_FAIL_DATA, ids=generate_test_ids_list(CATALOG_FROM_LIST_FAIL_DATA)) + def test_from_list_fail(self, catalog_data: dict[str, Any]) -> None: """ Errors when instantiating AntaCatalog from a list of tuples """ with pytest.raises(ValidationError) as exec_info: - AntaCatalog(tests=catalog_data["tests"]) + AntaCatalog.from_list(catalog_data["tests"]) assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] @pytest.mark.parametrize("catalog_data", TESTS_SETTER_FAIL_DATA, ids=generate_test_ids_list(TESTS_SETTER_FAIL_DATA)) @@ -229,6 +240,6 @@ def test_get_tests_by_tags(self) -> None: """ Test AntaCatalog.test_get_tests_by_tags() """ - catalog = AntaCatalog(filename=str(DATA_DIR / "test_catalog_with_tags.yml")) + catalog: AntaCatalog = AntaCatalog.parse(str(DATA_DIR / "test_catalog_with_tags.yml")) tests: list[AntaTestDefinition] = catalog.get_tests_by_tags(tags=["leaf"]) assert len(tests) == 2 From 4adcb36e5b40c2885f6a76781d7097f695d16267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Wed, 8 Nov 2023 15:42:04 +0100 Subject: [PATCH 47/53] Refactor AntaCatalog logging --- anta/catalog.py | 42 ++++++++++++++++++++------------- anta/cli/__init__.py | 4 ++-- anta/cli/check/commands.py | 9 ++++---- anta/cli/utils.py | 46 ++++--------------------------------- tests/units/test_catalog.py | 7 +++++- 5 files changed, 43 insertions(+), 65 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index b2cb20430..89757d99b 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -8,16 +8,17 @@ import importlib import logging -from pathlib import Path from inspect import isclass +from pathlib import Path from types import ModuleType from typing import Any, Dict, List, Optional, Tuple, Type, Union -from pydantic import BaseModel, ConfigDict, RootModel, ValidationInfo, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, RootModel, ValidationError, ValidationInfo, field_validator, model_validator from pydantic.types import ImportString from yaml import YAMLError, safe_load from anta.models import AntaTest +from anta.tools.misc import anta_log_exception logger = logging.getLogger(__name__) @@ -246,9 +247,7 @@ def tests(self, value: list[AntaTestDefinition]) -> None: self._tests = value @staticmethod - def parse( - filename: str - ) -> AntaCatalog: + def parse(filename: str | Path) -> AntaCatalog: """ Create an AntaCatalog instance from a test catalog file. @@ -258,18 +257,22 @@ def parse( try: with open(file=filename, mode="r", encoding="UTF-8") as file: data = safe_load(file) - except (YAMLError, OSError): - logger.critical(f"Something went wrong while parsing {filename}") + except (YAMLError, OSError) as e: + message = f"Unable to parse ANTA Tests Catalog file '{filename}'" + anta_log_exception(e, message, logger) + raise + try: + catalog_data = AntaCatalogFile(**data) + except ValidationError as e: + anta_log_exception(e, f"Test catalog '{filename}' is invalid!", logger) raise tests: list[AntaTestDefinition] = [] - for t in AntaCatalogFile(**data).root.values(): + for t in catalog_data.root.values(): tests.extend(t) return AntaCatalog(tests, filename=filename) @staticmethod - def from_dict( - data: RawCatalogInput - ) -> AntaCatalog: + def from_dict(data: RawCatalogInput) -> AntaCatalog: """ Create an AntaCatalog instance from a dictionary data structure. See RawCatalogInput type alias for details. @@ -280,14 +283,17 @@ def from_dict( data: Python dictionary used to instantiate the AntaCatalog instance """ tests: list[AntaTestDefinition] = [] - for t in AntaCatalogFile(**data).root.values(): # type: ignore[arg-type] + try: + catalog_data = AntaCatalogFile(**data) # type: ignore[arg-type] + except ValidationError as e: + anta_log_exception(e, "Test catalog is invalid!", logger) + raise + for t in catalog_data.root.values(): tests.extend(t) return AntaCatalog(tests) @staticmethod - def from_list( - data: ListAntaTestTuples - ) -> AntaCatalog: + def from_list(data: ListAntaTestTuples) -> AntaCatalog: """ Create an AntaCatalog instance from a list data structure. See ListAntaTestTuples type alias for details. @@ -296,7 +302,11 @@ def from_list( data: Python list used to instantiate the AntaCatalog instance """ tests: list[AntaTestDefinition] = [] - tests.extend(AntaTestDefinition(test=test, inputs=inputs) for test, inputs in data) + try: + tests.extend(AntaTestDefinition(test=test, inputs=inputs) for test, inputs in data) + except ValidationError as e: + anta_log_exception(e, "Test catalog is invalid!", logger) + raise return AntaCatalog(tests) def get_tests_by_tags(self, tags: list[str], strict: bool = False) -> list[AntaTestDefinition]: diff --git a/anta/cli/__init__.py b/anta/cli/__init__.py index c3bb825df..a0d7e4f1d 100644 --- a/anta/cli/__init__.py +++ b/anta/cli/__init__.py @@ -21,7 +21,7 @@ from anta.cli.exec import commands as exec_commands from anta.cli.get import commands as get_commands from anta.cli.nrfu import commands as nrfu_commands -from anta.cli.utils import AliasedGroup, IgnoreRequiredWithHelp, parse_catalog_cb, parse_inventory +from anta.cli.utils import AliasedGroup, IgnoreRequiredWithHelp, parse_catalog, parse_inventory from anta.logger import setup_logging from anta.result_manager import ResultManager @@ -147,7 +147,7 @@ def anta( help="Path to the tests catalog YAML file", type=click.Path(file_okay=True, dir_okay=False, exists=True, readable=True), required=True, - callback=parse_catalog_cb, + callback=parse_catalog, ) def _nrfu(ctx: click.Context, catalog: AntaCatalog) -> None: """Run NRFU against inventory devices""" diff --git a/anta/cli/check/commands.py b/anta/cli/check/commands.py index 4a40478dd..50af4fd3c 100644 --- a/anta/cli/check/commands.py +++ b/anta/cli/check/commands.py @@ -8,7 +8,6 @@ from __future__ import annotations import logging -from pathlib import Path import click from rich.pretty import pretty_repr @@ -29,11 +28,11 @@ help="Path to the tests catalog YAML file", type=click.Path(file_okay=True, dir_okay=False, exists=True, readable=True, resolve_path=True), required=True, + callback=parse_catalog, ) -def catalog(catalog: Path) -> None: +def catalog(catalog: AntaCatalog) -> None: """ Check that the catalog is valid """ - logger.info(f"Checking syntax of catalog {catalog}") - catalog_obj: AntaCatalog = parse_catalog(str(catalog)) - console.print(pretty_repr(catalog_obj.tests)) + console.print(f"[bold][green]Catalog {catalog} is valid") + console.print(pretty_repr(catalog.tests)) diff --git a/anta/cli/utils.py b/anta/cli/utils.py index 1bafa31f6..dcf5a0d66 100644 --- a/anta/cli/utils.py +++ b/anta/cli/utils.py @@ -15,9 +15,9 @@ import click from pydantic import ValidationError +from yaml import YAMLError from anta.catalog import AntaCatalog -from anta.cli.console import console from anta.inventory import AntaInventory from anta.tools.misc import anta_log_exception @@ -81,7 +81,7 @@ def parse_tags(ctx: click.Context, param: Option, value: str) -> list[str] | Non return None -def parse_catalog_cb(ctx: click.Context, param: Option, value: str) -> AntaCatalog: +def parse_catalog(ctx: click.Context, param: Option, value: Path) -> AntaCatalog: # pylint: disable=unused-argument """ Click option callback to parse an ANTA tests catalog YAML file @@ -92,46 +92,10 @@ def parse_catalog_cb(ctx: click.Context, param: Option, value: str) -> AntaCatal # Currently looking for help for a subcommand so no # need to parse the Catalog - return an empty catalog return AntaCatalog() - # Storing catalog path - ctx.obj["catalog_path"] = value try: - catalog = parse_catalog(value) - except ValidationError as e: - message = f"Catalog {value} is invalid!" - anta_log_exception(e, message, logger) - ctx.fail(message) - # pylint: disable-next=broad-exception-caught - except Exception as e: - message = f"Unable to parse ANTA Tests Catalog file '{value}'" - anta_log_exception(e, message, logger) - ctx.fail(message) - - return catalog - - -def parse_catalog(catalog_path: str) -> AntaCatalog: - # pylint: disable=unused-argument - """ - Args: - ---- - catalog_path (str): Path to the catalog YAML file. - - Returns: - ------- - the AntaCatalog parsed file - - Raises: - ----- - ValidationError: If the catalog file is invalid - Exception: If anything happens with opening the catalog file - """ - try: - catalog = AntaCatalog(filename=catalog_path) - console.print(f"[bold][green]Catalog {catalog_path} is valid") - except ValidationError: - console.print(f"[bold][red]Catalog {catalog_path} is invalid!") - raise - + catalog: AntaCatalog = AntaCatalog.parse(value) + except (ValidationError, YAMLError, OSError): + ctx.fail("Unable to load ANTA Tests Catalog") return catalog diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index a87dee5a8..bf8faa664 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -215,7 +215,12 @@ def test_parse_fail_parsing(self, caplog: pytest.LogCaptureFixture) -> None: assert "No such file or directory" in str(exec_info) assert len(caplog.record_tuples) == 1 _, _, message = caplog.record_tuples[0] - assert "Something went wrong while parsing" in message + assert ( + "Unable to parse ANTA Tests Catalog file " + "'/mnt/lab/projects/anta/tests/data/catalog_does_not_exist.yml': " + "FileNotFoundError ([Errno 2] No such file or directory: " + "'/mnt/lab/projects/anta/tests/data/catalog_does_not_exist.yml')" + ) in message @pytest.mark.parametrize("catalog_data", CATALOG_FROM_LIST_FAIL_DATA, ids=generate_test_ids_list(CATALOG_FROM_LIST_FAIL_DATA)) def test_from_list_fail(self, catalog_data: dict[str, Any]) -> None: From cf56b2e0a41d944f62f21393fb7fb74405c5ac48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Wed, 8 Nov 2023 15:42:56 +0100 Subject: [PATCH 48/53] fix test_runner.py --- tests/units/test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/units/test_runner.py b/tests/units/test_runner.py index ed631878c..26caac43d 100644 --- a/tests/units/test_runner.py +++ b/tests/units/test_runner.py @@ -21,7 +21,7 @@ if TYPE_CHECKING: from pytest import LogCaptureFixture -FAKE_CATALOG = AntaCatalog(tests=[(FakeTest, None)]) +FAKE_CATALOG: AntaCatalog = AntaCatalog.from_list([(FakeTest, None)]) @pytest.mark.asyncio From 5a53a67656bd7de470191e0c407696f37a4d27e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Wed, 8 Nov 2023 16:50:31 +0100 Subject: [PATCH 49/53] Update documentation for test catalog --- anta/catalog.py | 49 ++-------- anta/cli/__init__.py | 2 +- anta/cli/check/commands.py | 2 +- anta/cli/exec/commands.py | 2 +- anta/cli/utils.py | 4 +- docs/api/catalog.md | 13 +++ docs/cli/exec.md | 2 +- docs/cli/nrfu.md | 4 +- docs/cli/tag-management.md | 10 +-- docs/getting-started.md | 2 +- docs/usage-inventory-catalog.md | 153 +++++++++++++++++++++++--------- mkdocs.yml | 6 +- tests/units/test_catalog.py | 2 +- 13 files changed, 146 insertions(+), 105 deletions(-) create mode 100644 docs/api/catalog.md diff --git a/anta/catalog.py b/anta/catalog.py index 89757d99b..ad0670bb8 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -92,43 +92,10 @@ class AntaCatalogFile(RootModel[Dict[ImportString[Any], List[AntaTestDefinition] """ This model represents an ANTA Test Catalog File. - A valid test catalog file must follow the following structure: + A valid test catalog file must have the following structure: : - : - - Example: - ``` - anta.tests.connectivity: - - VerifyReachability: - hosts: - - dst: 8.8.8.8 - src: 172.16.0.1 - - dst: 1.1.1.1 - src: 172.16.0.1 - result_overwrite: - categories: - - "Overwritten category 1" - description: "Test with overwritten description" - custom_field: "Test run by John Doe" - ``` - - Also supports nesting for Python module definition: - ``` - anta.tests: - connectivity: - - VerifyReachability: - hosts: - - dst: 8.8.8.8 - src: 172.16.0.1 - - dst: 1.1.1.1 - src: 172.16.0.1 - result_overwrite: - categories: - - "Overwritten category 1" - description: "Test with overwritten description" - custom_field: "Test run by John Doe" - ``` """ root: Dict[ImportString[Any], List[AntaTestDefinition]] @@ -200,12 +167,7 @@ class AntaCatalog: """ Class representing an ANTA Catalog. - It can be defined programmatically by providing the `tests` argument to the constructor - or it can be loaded from a file using the `filename` argument. - - Attributes: - filename: The path from which the catalog is loaded. - tests: A list of tuple containing an AntaTest class and the associated input. + It can be instantiated using its contructor or one of the static methods: `parse()`, `from_list()` or `from_dict()` """ def __init__(self, tests: list[AntaTestDefinition] | None = None, filename: str | Path | None = None) -> None: @@ -213,9 +175,8 @@ def __init__(self, tests: list[AntaTestDefinition] | None = None, filename: str Constructor of AntaCatalog. Args: - ---- - tests: A list of tuple containing an AntaTest class and the associated input. Use this argument if you want to define the catalog programmatically. - filename: The path from which the catalog is loaded. This will be set as instance attribute. + tests: A list of AntaTestDefinition instances. + filename: The path from which the catalog is loaded. """ self._tests: list[AntaTestDefinition] = [] if tests is not None: @@ -258,7 +219,7 @@ def parse(filename: str | Path) -> AntaCatalog: with open(file=filename, mode="r", encoding="UTF-8") as file: data = safe_load(file) except (YAMLError, OSError) as e: - message = f"Unable to parse ANTA Tests Catalog file '{filename}'" + message = f"Unable to parse ANTA Test Catalog file '{filename}'" anta_log_exception(e, message, logger) raise try: diff --git a/anta/cli/__init__.py b/anta/cli/__init__.py index a0d7e4f1d..1349c15ff 100644 --- a/anta/cli/__init__.py +++ b/anta/cli/__init__.py @@ -144,7 +144,7 @@ def anta( "-c", envvar="ANTA_CATALOG", show_envvar=True, - help="Path to the tests catalog YAML file", + help="Path to the test catalog YAML file", type=click.Path(file_okay=True, dir_okay=False, exists=True, readable=True), required=True, callback=parse_catalog, diff --git a/anta/cli/check/commands.py b/anta/cli/check/commands.py index 50af4fd3c..1b16a6689 100644 --- a/anta/cli/check/commands.py +++ b/anta/cli/check/commands.py @@ -25,7 +25,7 @@ "-c", envvar="ANTA_CATALOG", show_envvar=True, - help="Path to the tests catalog YAML file", + help="Path to the test catalog YAML file", type=click.Path(file_okay=True, dir_okay=False, exists=True, readable=True, resolve_path=True), required=True, callback=parse_catalog, diff --git a/anta/cli/exec/commands.py b/anta/cli/exec/commands.py index ca61d7e34..765aa5906 100644 --- a/anta/cli/exec/commands.py +++ b/anta/cli/exec/commands.py @@ -66,7 +66,7 @@ def snapshot(ctx: click.Context, tags: list[str] | None, commands_list: Path, ou @click.command() @click.pass_context -@click.option("--output", "-o", default="./tech-support", show_default=True, help="Path for tests catalog", type=click.Path(path_type=Path), required=False) +@click.option("--output", "-o", default="./tech-support", show_default=True, help="Path for test catalog", type=click.Path(path_type=Path), required=False) @click.option("--latest", help="Number of scheduled show-tech to retrieve", type=int, required=False) @click.option( "--configure", diff --git a/anta/cli/utils.py b/anta/cli/utils.py index dcf5a0d66..43a8ba564 100644 --- a/anta/cli/utils.py +++ b/anta/cli/utils.py @@ -84,7 +84,7 @@ def parse_tags(ctx: click.Context, param: Option, value: str) -> list[str] | Non def parse_catalog(ctx: click.Context, param: Option, value: Path) -> AntaCatalog: # pylint: disable=unused-argument """ - Click option callback to parse an ANTA tests catalog YAML file + Click option callback to parse an ANTA test catalog YAML file Store the orignal value (catalog path) in the ctx.obj """ @@ -95,7 +95,7 @@ def parse_catalog(ctx: click.Context, param: Option, value: Path) -> AntaCatalog try: catalog: AntaCatalog = AntaCatalog.parse(value) except (ValidationError, YAMLError, OSError): - ctx.fail("Unable to load ANTA Tests Catalog") + ctx.fail("Unable to load ANTA Test Catalog") return catalog diff --git a/docs/api/catalog.md b/docs/api/catalog.md new file mode 100644 index 000000000..8b91fff9d --- /dev/null +++ b/docs/api/catalog.md @@ -0,0 +1,13 @@ + + +### ::: anta.catalog.AntaCatalog + options: + filters: ["!^_[^_]", "!__str__"] + +### ::: anta.catalog.AntaTestDefinition + +### ::: anta.catalog.AntaCatalogFile diff --git a/docs/cli/exec.md b/docs/cli/exec.md index be94b4bc5..2fe726193 100644 --- a/docs/cli/exec.md +++ b/docs/cli/exec.md @@ -161,7 +161,7 @@ Usage: anta exec collect-tech-support [OPTIONS] Collect scheduled tech-support from EOS devices Options: - -o, --output PATH Path for tests catalog [default: ./tech- + -o, --output PATH Path for test catalog [default: ./tech- support] --latest INTEGER Number of scheduled show-tech to retrieve --configure Ensure devices have 'aaa authorization exec default diff --git a/docs/cli/nrfu.md b/docs/cli/nrfu.md index 10a481c25..90617d086 100644 --- a/docs/cli/nrfu.md +++ b/docs/cli/nrfu.md @@ -22,7 +22,7 @@ Usage: anta nrfu [OPTIONS] COMMAND [ARGS]... Run NRFU against inventory devices Options: - -c, --catalog FILE Path to the tests catalog YAML file [env var: + -c, --catalog FILE Path to the test catalog YAML file [env var: ANTA_CATALOG; required] --help Show this message and exit. @@ -41,7 +41,7 @@ The `--tags` option can be used to target specific devices in your inventory and | Command | Description | | ------- | ----------- | -| `none` | Run all tests on all devices according `tag` definition in your inventory and tests catalog. And tests with no tag are executed on all devices| +| `none` | Run all tests on all devices according `tag` definition in your inventory and test catalog. And tests with no tag are executed on all devices| | `--tags leaf` | Run all tests marked with `leaf` tag on all devices configured with `leaf` tag.
All other tags are ignored | | `--tags leaf,spine` | Run all tests marked with `leaf` tag on all devices configured with `leaf` tag.
Run all tests marked with `spine` tag on all devices configured with `spine` tag.
All other tags are ignored | diff --git a/docs/cli/tag-management.md b/docs/cli/tag-management.md index 61a2c227a..3ec24f276 100644 --- a/docs/cli/tag-management.md +++ b/docs/cli/tag-management.md @@ -8,7 +8,7 @@ ## Overview -ANTA nrfu command comes with a `--tags` option. This allows users to specify a set of tests, marked with a given tag, to be run on devices marked with the same tag. For instance, you can run tests dedicated to leaf devices on your leaf devices only and not on other devices. +The `anta nrfu` command comes with a `--tags` option. This allows users to specify a set of tests, marked with a given tag, to be run on devices marked with the same tag. For instance, you can run tests dedicated to leaf devices on your leaf devices only and not on other devices. Tags are string defined by the user and can be anything considered as a string by Python. A [default one](#default-tags) is present for all tests and devices. @@ -16,13 +16,13 @@ The next table provides a short summary of the scope of tags using CLI | Command | Description | | ------- | ----------- | -| `none` | Run all tests on all devices according `tag` definition in your inventory and tests catalog. And tests with no tag are executed on all devices| +| `none` | Run all tests on all devices according `tag` definition in your inventory and test catalog. And tests with no tag are executed on all devices| | `--tags leaf` | Run all tests marked with `leaf` tag on all devices configured with `leaf` tag.
All other tags are ignored | | `--tags leaf,spine` | Run all tests marked with `leaf` tag on all devices configured with `leaf` tag.
Run all tests marked with `spine` tag on all devices configured with `spine` tag.
All other tags are ignored | ## Inventory and Catalog for tests -All commands in this page are based on the following inventory and tests catalog. +All commands in this page are based on the following inventory and test catalog. === "Inventory" @@ -50,7 +50,7 @@ All commands in this page are based on the following inventory and tests catalog tags: ['fabric', 'leaf' ``` -=== "Tests Catalog" +=== "Test Catalog" ```yaml anta.tests.system: @@ -135,7 +135,7 @@ leaf04 :: VerifyReloadCause :: SUCCESS leaf04 :: VerifyCPUUtilization :: SUCCESS ``` -In this case, only `leaf` devices defined in your [inventory](#inventory-and-catalog-for-tests) are used to run tests marked with `leaf` in your [tests catalog](#inventory-and-catalog-for-tests) +In this case, only `leaf` devices defined in your [inventory](#inventory-and-catalog-for-tests) are used to run tests marked with `leaf` in your [test catalog](#inventory-and-catalog-for-tests) ## Use multiple tags in CLI diff --git a/docs/getting-started.md b/docs/getting-started.md index d5487a9a7..63442034f 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -139,7 +139,7 @@ Usage: anta nrfu [OPTIONS] COMMAND [ARGS]... Run NRFU against inventory devices Options: - -c, --catalog FILE Path to the tests catalog YAML file [env var: + -c, --catalog FILE Path to the test catalog YAML file [env var: ANTA_CATALOG; required] --help Show this message and exit. diff --git a/docs/usage-inventory-catalog.md b/docs/usage-inventory-catalog.md index c1ebf9b23..a4d13b68d 100644 --- a/docs/usage-inventory-catalog.md +++ b/docs/usage-inventory-catalog.md @@ -4,13 +4,20 @@ ~ that can be found in the LICENSE file. --> -# Inventory and Catalog definition +# Inventory and Catalog -This page describes how to create an inventory and a tests catalog. +The ANTA framework needs 2 important inputs from the user to run: a **device inventory** and a **test catalog**. -## Create an inventory file +Both inputs can be defined in a file or programmatically. -`anta` cli needs an inventory file to list all devices to tests. This inventory is a YAML file with the folowing keys: +## Device Inventory + +A device inventory is an instance of the [AntaInventory](../api/inventory.md#anta.inventory.AntaInventory) class. + +### Device Inventory File + +The ANTA device inventory can easily be defined as a YAML file. +The file must comply with the following structure: ```yaml anta_inventory: @@ -31,12 +38,18 @@ anta_inventory: disable_cache: < Disable cache per range. Default is False. > ``` -Your inventory file can be based on any of these 3 keys and MUST start with `anta_inventory` key. A full description of the inventory model is available in [API documentation](api/inventory.models.input.md) +The inventory file must start with the `anta_inventory` key then define one or multiple methods: + +- `hosts`: define each device individually +- `networks`: scan a network for devices accesible via eAPI +- `ranges`: scan a range for devices accesible via eAPI + +A full description of the inventory model is available in [API documentation](api/inventory.models.input.md) !!! info Caching can be disabled per device, network or range by setting the `disable_cache` key to `True` in the inventory file. For more details about how caching is implemented in ANTA, please refer to [Caching in ANTA](advanced_usages/caching.md). -An inventory example: +### Example ```yaml --- @@ -59,19 +72,100 @@ anta_inventory: ## Test Catalog -In addition to your inventory file, you also have to define a catalog of tests to execute against all your devices. This catalog list all your tests and their parameters. -Its format is a YAML file and keys are tests functions inherited from the python path. +A test catalog is an instance of the [AntaCatalog](../api/catalog.md#anta.catalog.AntaCatalog) class. -### Default tests catalog +### Test Catalog File -All tests are located under `anta.tests` module and are categorised per family (one submodule). So to run test for software version, you can do: +In addition to the inventory file, you also have to define a catalog of tests to execute against your devices. This catalog list all your tests, their inputs and their tags. + +A valid test catalog file must have the following structure: +```yaml +--- +: + - : + +``` + +### Example + +```yaml +--- +anta.tests.connectivity: + - VerifyReachability: + hosts: + - source: Management0 + destination: 1.1.1.1 + vrf: MGMT + - source: Management0 + destination: 8.8.8.8 + vrf: MGMT + filters: + tags: ['leaf'] + result_overwrite: + categories: + - "Overwritten category 1" + description: "Test with overwritten description" + custom_field: "Test run by John Doe" +``` + +It is also possible to nest Python module definition: +```yaml +anta.tests: + connectivity: + - VerifyReachability: + hosts: + - source: Management0 + destination: 1.1.1.1 + vrf: MGMT + - source: Management0 + destination: 8.8.8.8 + vrf: MGMT + filters: + tags: ['leaf'] + result_overwrite: + categories: + - "Overwritten category 1" + description: "Test with overwritten description" + custom_field: "Test run by John Doe" +``` + +[This test catalog example](https://github.com/arista-netdevops-community/anta/blob/main/examples/tests.yaml) is maintained with all the tests defined in the `anta.tests` Python module. + +### Test tags + +All tests can be defined with a list of user defined tags. These tags will be mapped with device tags: when at least one tag is defined for a test, this test will only be executed on devices with the same tag. If a test is defined in the catalog without any tags, the test will be executed on all devices. + +```yaml +anta.tests.system: + - VerifyUptime: + minimum: 10 + filters: + tags: ['demo', 'leaf'] + - VerifyReloadCause: + - VerifyCoredump: + - VerifyAgentLogs: + - VerifyCPUUtilization: + filters: + tags: ['leaf'] +``` + +!!! info + When using the CLI, you can filter the NRFU execution using tags. Refer to [this section](cli/tag-management.md) of the CLI documentation. + +### Tests available in ANTA + +All tests available as part of the ANTA framework are defined under the `anta.tests` Python module and are categorised per family (Python submodule). +The complete list of the tests and their respective inputs is available at the [tests section](api/tests.md) of this website. + + +To run test to verify the EOS software version, you can do: ```yaml anta.tests.software: - VerifyEosVersion: ``` -It will load the test `VerifyEosVersion` located in `anta.tests.software`. But since this function has parameters, we will create a catalog with the following structure: +It will load the test `VerifyEosVersion` located in `anta.tests.software`. But since this test has mandatory inputs, we need to provide them as a dictionary in the YAML file: ```yaml anta.tests.software: @@ -82,9 +176,7 @@ anta.tests.software: - 4.26.1F ``` -To get a list of all available tests and their respective parameters, you can read the [tests section](./api/tests.md) of this website. - -The following example gives a very minimal tests catalog you can use in almost any situation +The following example is a very minimal test catalog: ```yaml --- @@ -110,35 +202,10 @@ anta.tests.configuration: - VerifyRunningConfigDiffs: ``` -All tests can be configured with a list of user defined tags. These tags will be mapped with device tags when `nrfu` cli is called with the `--tags` option. If a test is configured in the catalog without a list of tags, the test will be executed for all devices. When at least one tag is defined for a test, this test will be only executed on devices with the same tag. - -```yaml -anta.tests.system: - - VerifyUptime: - minimum: 10 - filters: - tags: ['demo', 'leaf'] - - VerifyReloadCause: - - VerifyCoredump: - - VerifyAgentLogs: - - VerifyCPUUtilization: - filters: - tags: ['leaf'] -``` - -!!! info - The `tag` field must be equal to values configured for `tags` under your device inventory. - -### Custom tests catalog - -In case you want to leverage your own tests collection, you can use the following syntax: - -```yaml -: - - : -``` +### Catalog with custom tests -So for instance, it could be: +In case you want to leverage your own tests collection, use your own Python package in the test catalog. +So for instance, if my custom tests are defined in the `titom73.tests.system` Python module, the test catalog will be: ```yaml titom73.tests.system: @@ -147,7 +214,7 @@ titom73.tests.system: ``` !!! tip "How to create custom tests" - To create your custom tests, you should refer to this [following documentation](advanced_usages/custom-tests.md) + To create your custom tests, you should refer to this [documentation](advanced_usages/custom-tests.md) ### Customize test description and categories diff --git a/mkdocs.yml b/mkdocs.yml index 9bffeb69c..3fd647743 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -170,7 +170,7 @@ nav: - Caching in ANTA: advanced_usages/caching.md - Developing ANTA tests: advanced_usages/custom-tests.md - ANTA as a Python Library: advanced_usages/as-python-lib.md - - Tests Catalog Documentation: + - Test Catalog Documentation: - Overview: api/tests.md - AAA: api/tests.aaa.md - Configuration: api/tests.configuration.md @@ -196,8 +196,8 @@ nav: - Inventory: - Inventory module: api/inventory.md - Inventory models: api/inventory.models.input.md - - Device: - - Device models: api/device.md + - Test Catalog: api/catalog.md + - Device: api/device.md - Test: - Test models: api/models.md - Input Types: api/types.md diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index bf8faa664..be92b5057 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -216,7 +216,7 @@ def test_parse_fail_parsing(self, caplog: pytest.LogCaptureFixture) -> None: assert len(caplog.record_tuples) == 1 _, _, message = caplog.record_tuples[0] assert ( - "Unable to parse ANTA Tests Catalog file " + "Unable to parse ANTA Test Catalog file " "'/mnt/lab/projects/anta/tests/data/catalog_does_not_exist.yml': " "FileNotFoundError ([Errno 2] No such file or directory: " "'/mnt/lab/projects/anta/tests/data/catalog_does_not_exist.yml')" From 3c2273305d4b04dade161dbd4bd8d68a9d25c21c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Wed, 8 Nov 2023 16:54:28 +0100 Subject: [PATCH 50/53] Update gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 119453dd5..4c2aee4e3 100644 --- a/.gitignore +++ b/.gitignore @@ -92,6 +92,7 @@ venv/ ENV/ env.bak/ venv.bak/ +.envrc # mkdocs documentation /site @@ -107,4 +108,4 @@ tech-support/* .*report.html clab-atd-anta/* -clab-atd-anta/ \ No newline at end of file +clab-atd-anta/ From fc25a476e4356e59076b0678f8bf238c87992772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthieu=20T=C3=A2che?= Date: Wed, 8 Nov 2023 17:04:41 +0100 Subject: [PATCH 51/53] fix unit tests --- tests/units/test_catalog.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index be92b5057..a84519eb4 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -215,12 +215,8 @@ def test_parse_fail_parsing(self, caplog: pytest.LogCaptureFixture) -> None: assert "No such file or directory" in str(exec_info) assert len(caplog.record_tuples) == 1 _, _, message = caplog.record_tuples[0] - assert ( - "Unable to parse ANTA Test Catalog file " - "'/mnt/lab/projects/anta/tests/data/catalog_does_not_exist.yml': " - "FileNotFoundError ([Errno 2] No such file or directory: " - "'/mnt/lab/projects/anta/tests/data/catalog_does_not_exist.yml')" - ) in message + assert "Unable to parse ANTA Test Catalog file" in message + assert "FileNotFoundError ([Errno 2] No such file or directory" in message @pytest.mark.parametrize("catalog_data", CATALOG_FROM_LIST_FAIL_DATA, ids=generate_test_ids_list(CATALOG_FROM_LIST_FAIL_DATA)) def test_from_list_fail(self, catalog_data: dict[str, Any]) -> None: From 06be61c3f292e802b8d215895d2a562eee95593e Mon Sep 17 00:00:00 2001 From: gmuloc Date: Wed, 8 Nov 2023 18:30:16 +0100 Subject: [PATCH 52/53] Test: Moare coverage --- tests/units/test_catalog.py | 42 +++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index a84519eb4..ba486a317 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -103,6 +103,13 @@ "error": "Value error, AntaTestDefinition must be a dictionary with a single entry", }, ] +CATALOG_FROM_DICT_FAIL_DATA: list[dict[str, Any]] = [ + { + "name": "undefined_tests", + "filename": "test_catalog_with_undefined_tests.yml", + "error": "FakeTest is not defined in Python module None: AntaCatalog.from_list(catalog_data["tests"]) assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + @pytest.mark.parametrize("catalog_data", CATALOG_FROM_DICT_FAIL_DATA, ids=generate_test_ids_list(CATALOG_FROM_DICT_FAIL_DATA)) + def test_from_dict_fail(self, catalog_data: dict[str, Any]) -> None: + """ + Errors when instantiating AntaCatalog from a list of tuples + """ + with open(file=str(DATA_DIR / catalog_data["filename"]), mode="r", encoding="UTF-8") as file: + data = safe_load(file) + with pytest.raises(ValidationError) as exec_info: + AntaCatalog.from_dict(data) + assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + + def test_filename(self) -> None: + """ + Test filename + """ + catalog = AntaCatalog(filename="test") + assert catalog.filename == Path("test") + catalog = AntaCatalog(filename=Path("test")) + assert catalog.filename == Path("test") + + @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) + def test__tests_setter_success(self, catalog_data: dict[str, Any]) -> None: + """ + Success when setting AntaCatalog.tests from a list of tuples + """ + catalog = AntaCatalog() + catalog.tests = [AntaTestDefinition(test=test, inputs=inputs) for test, inputs in catalog_data["tests"]] + assert len(catalog.tests) == len(catalog_data["tests"]) + for test_id, (test, inputs) in enumerate(catalog_data["tests"]): + assert catalog.tests[test_id].test == test + if inputs is not None: + if isinstance(inputs, dict): + inputs = test.Input(**inputs) + assert inputs == catalog.tests[test_id].inputs + @pytest.mark.parametrize("catalog_data", TESTS_SETTER_FAIL_DATA, ids=generate_test_ids_list(TESTS_SETTER_FAIL_DATA)) def test__tests_setter_fail(self, catalog_data: dict[str, Any]) -> None: """ From b7fe6550101551032eff60beb423140242d2c62d Mon Sep 17 00:00:00 2001 From: Guillaume Mulocher Date: Fri, 10 Nov 2023 14:14:59 +0100 Subject: [PATCH 53/53] Update anta/logger.py Co-authored-by: Carl Baillargeon --- anta/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anta/logger.py b/anta/logger.py index da8b75abd..f8cabdc79 100644 --- a/anta/logger.py +++ b/anta/logger.py @@ -2,7 +2,7 @@ # Use of this source code is governed by the Apache License 2.0 # that can be found in the LICENSE file. """ -Loader that parses a YAML test catalog and imports corresponding Python functions +Configure logging for ANTA """ from __future__ import annotations