diff --git a/.github/generate_release.py b/.github/generate_release.py index 97f139b7f..8cd4337fc 100644 --- a/.github/generate_release.py +++ b/.github/generate_release.py @@ -30,7 +30,7 @@ class SafeDumper(yaml.SafeDumper): https://github.com/yaml/pyyaml/issues/234#issuecomment-765894586. """ - # pylint: disable=R0901,W0613,W1113 + # pylint: disable=R0901 def increase_indent(self, flow=False, *args, **kwargs): return super().increase_indent(flow=flow, indentless=False) diff --git a/.gitignore b/.gitignore index a62de025c..29e00d517 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ __pycache__ *.pyc .pages -.coverage .pytest_cache +.mypy_cache +.ruff_cache +.cache build dist *.egg-info @@ -46,14 +48,13 @@ htmlcov/ .tox/ .nox/ .coverage +coverage_html_report .coverage.* -.cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ -.pytest_cache/ cover/ report.html @@ -97,17 +98,4 @@ venv.bak/ /site # VScode settings -.vscode -test.env -tech-support/ -tech-support/* -2* - -**/report.html -.*report.html - -# direnv file -.envrc - -clab-atd-anta/* -clab-atd-anta/ +.vscode \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 003c2e5c5..9944f5bfd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,6 +69,7 @@ repos: - types-pyOpenSSL - pylint_pydantic - pytest + - respx - repo: https://github.com/codespell-project/codespell rev: v2.3.0 diff --git a/.vscode/settings.json b/.vscode/settings.json index 60150c6d1..ac8ba0b02 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,28 +3,13 @@ "ruff.configuration": "pyproject.toml", "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, - "pylint.importStrategy": "fromEnvironment", - "pylint.severity": { - "refactor": "Warning" - }, - "pylint.args": [ - "--load-plugins", - "pylint_pydantic", - "--rcfile=pyproject.toml" - ], "python.testing.pytestArgs": [ "tests" ], - "autoDocstring.docstringFormat": "numpy", - "autoDocstring.includeName": false, - "autoDocstring.includeExtendedSummary": true, - "autoDocstring.startOnNewLine": true, - "autoDocstring.guessTypes": true, "python.languageServer": "Pylance", "githubIssues.issueBranchTitle": "issues/${issueNumber}-${issueTitle}", "editor.formatOnPaste": true, "files.trimTrailingWhitespace": true, - "mypy.configFile": "pyproject.toml", "workbench.remoteIndicator.showExtensionRecommendations": true, } \ No newline at end of file diff --git a/anta/catalog.py b/anta/catalog.py index 46b90d3e6..66520f9fa 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -173,7 +173,7 @@ def flatten_modules(data: dict[str, Any], package: str | None = None) -> dict[Mo module_name = f".{module_name}" # noqa: PLW2901 try: module: ModuleType = importlib.import_module(name=module_name, package=package) - except Exception as e: # pylint: disable=broad-exception-caught + except Exception as e: # A test module is potentially user-defined code. # We need to catch everything if we want to have meaningful logs module_str = f"{module_name[1:] if module_name.startswith('.') else module_name}{f' from package {package}' if package else ''}" diff --git a/anta/cli/_main.py b/anta/cli/_main.py index 1211a42d9..d70f1cf56 100644 --- a/anta/cli/_main.py +++ b/anta/cli/_main.py @@ -61,7 +61,7 @@ def cli() -> None: """Entrypoint for pyproject.toml.""" try: anta(obj={}, auto_envvar_prefix="ANTA") - except Exception as exc: # pylint: disable=broad-exception-caught + except Exception as exc: # noqa: BLE001 anta_log_exception( exc, f"Uncaught Exception when running ANTA CLI\n{GITHUB_SUGGESTION}", diff --git a/anta/cli/debug/commands.py b/anta/cli/debug/commands.py index 1304758a4..e6e456e67 100644 --- a/anta/cli/debug/commands.py +++ b/anta/cli/debug/commands.py @@ -35,7 +35,6 @@ def run_cmd( version: Literal["1", "latest"], revision: int, ) -> None: - # pylint: disable=too-many-arguments """Run arbitrary command to an ANTA device.""" console.print(f"Run command [green]{command}[/green] on [red]{device.name}[/red]") # I do not assume the following line, but click make me do it @@ -71,7 +70,6 @@ def run_template( version: Literal["1", "latest"], revision: int, ) -> None: - # pylint: disable=too-many-arguments # Using \b for click # ruff: noqa: D301 """Run arbitrary templated command to an ANTA device. diff --git a/anta/cli/debug/utils.py b/anta/cli/debug/utils.py index 4e20c5a74..454c3e640 100644 --- a/anta/cli/debug/utils.py +++ b/anta/cli/debug/utils.py @@ -48,7 +48,6 @@ def wrapper( **kwargs: Any, ) -> Any: # TODO: @gmuloc - tags come from context https://github.com/aristanetworks/anta/issues/584 - # pylint: disable=unused-argument # ruff: noqa: ARG001 if (d := inventory.get(device)) is None: logger.error("Device '%s' does not exist in Inventory", device) diff --git a/anta/cli/exec/__init__.py b/anta/cli/exec/__init__.py index 7f9b4c2b6..5fa6eb92a 100644 --- a/anta/cli/exec/__init__.py +++ b/anta/cli/exec/__init__.py @@ -9,7 +9,7 @@ @click.group("exec") -def _exec() -> None: # pylint: disable=redefined-builtin +def _exec() -> None: """Commands to execute various scripts on EOS devices.""" diff --git a/anta/cli/exec/utils.py b/anta/cli/exec/utils.py index a5f7da2b1..ce13622a7 100644 --- a/anta/cli/exec/utils.py +++ b/anta/cli/exec/utils.py @@ -10,16 +10,15 @@ import itertools import json import logging -import re from pathlib import Path from typing import TYPE_CHECKING, Literal from click.exceptions import UsageError from httpx import ConnectError, HTTPError -from anta.custom_types import REGEXP_PATH_MARKERS from anta.device import AntaDevice, AsyncEOSDevice from anta.models import AntaCommand +from anta.tools import safe_command from asynceapi import EapiCommandError if TYPE_CHECKING: @@ -52,7 +51,7 @@ async def clear(dev: AntaDevice) -> None: async def collect_commands( inv: AntaInventory, - commands: dict[str, str], + commands: dict[str, list[str]], root_dir: Path, tags: set[str] | None = None, ) -> None: @@ -61,17 +60,16 @@ async def collect_commands( async def collect(dev: AntaDevice, command: str, outformat: Literal["json", "text"]) -> None: outdir = Path() / root_dir / dev.name / outformat outdir.mkdir(parents=True, exist_ok=True) - safe_command = re.sub(rf"{REGEXP_PATH_MARKERS}", "_", command) c = AntaCommand(command=command, ofmt=outformat) await dev.collect(c) if not c.collected: logger.error("Could not collect commands on device %s: %s", dev.name, c.errors) return if c.ofmt == "json": - outfile = outdir / f"{safe_command}.json" + outfile = outdir / f"{safe_command(command)}.json" content = json.dumps(c.json_output, indent=2) elif c.ofmt == "text": - outfile = outdir / f"{safe_command}.log" + outfile = outdir / f"{safe_command(command)}.log" content = c.text_output else: logger.error("Command outformat is not in ['json', 'text'] for command '%s'", command) @@ -83,6 +81,9 @@ async def collect(dev: AntaDevice, command: str, outformat: Literal["json", "tex logger.info("Connecting to devices...") await inv.connect_inventory() devices = inv.get_inventory(established_only=True, tags=tags).devices + if not devices: + logger.info("No online device found. Exiting") + return logger.info("Collecting commands from remote devices") coros = [] if "json_format" in commands: @@ -134,8 +135,8 @@ async def collect(device: AntaDevice) -> None: if not isinstance(device, AsyncEOSDevice): msg = "anta exec collect-tech-support is only supported with AsyncEOSDevice for now." raise UsageError(msg) - if device.enable and device._enable_password is not None: # pylint: disable=protected-access - commands.append({"cmd": "enable", "input": device._enable_password}) # pylint: disable=protected-access + if device.enable and device._enable_password is not None: + commands.append({"cmd": "enable", "input": device._enable_password}) elif device.enable: commands.append({"cmd": "enable"}) commands.extend( @@ -146,7 +147,7 @@ async def collect(device: AntaDevice) -> None: ) logger.warning("Configuring 'aaa authorization exec default local' on device %s", device.name) command = AntaCommand(command="show running-config | include aaa authorization exec default local", ofmt="text") - await device._session.cli(commands=commands) # pylint: disable=protected-access + await device._session.cli(commands=commands) logger.info("Configured 'aaa authorization exec default local' on device %s", device.name) logger.debug("'aaa authorization exec default local' is already configured on device %s", device.name) diff --git a/anta/cli/get/commands.py b/anta/cli/get/commands.py index bfe94e618..ea1cc7561 100644 --- a/anta/cli/get/commands.py +++ b/anta/cli/get/commands.py @@ -45,7 +45,6 @@ default=False, ) def from_cvp(ctx: click.Context, output: Path, host: str, username: str, password: str, container: str | None, *, ignore_cert: bool) -> None: - # pylint: disable=too-many-arguments """Build ANTA inventory from CloudVision. NOTE: Only username/password authentication is supported for on-premises CloudVision instances. @@ -127,7 +126,6 @@ def inventory(inventory: AntaInventory, tags: set[str] | None, *, connected: boo @click.command @inventory_options def tags(inventory: AntaInventory, **kwargs: Any) -> None: - # pylint: disable=unused-argument """Get list of configured tags in user inventory.""" tags: set[str] = set() for device in inventory.values(): diff --git a/anta/cli/nrfu/__init__.py b/anta/cli/nrfu/__init__.py index d573b49c7..0272e0dba 100644 --- a/anta/cli/nrfu/__init__.py +++ b/anta/cli/nrfu/__init__.py @@ -103,7 +103,6 @@ def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]: is_flag=True, default=False, ) -# pylint: disable=too-many-arguments def nrfu( ctx: click.Context, inventory: AntaInventory, diff --git a/anta/cli/utils.py b/anta/cli/utils.py index 19ffb113f..a939c3220 100644 --- a/anta/cli/utils.py +++ b/anta/cli/utils.py @@ -40,7 +40,6 @@ class ExitCode(enum.IntEnum): def parse_tags(ctx: click.Context, param: Option, value: str | None) -> set[str] | None: - # pylint: disable=unused-argument # ruff: noqa: ARG001 """Click option callback to parse an ANTA inventory tags.""" if value is not None: @@ -207,7 +206,6 @@ def wrapper( disable_cache: bool, **kwargs: dict[str, Any], ) -> Any: - # pylint: disable=too-many-arguments # If help is invoke somewhere, do not parse inventory if ctx.obj.get("_anta_help"): return f(*args, inventory=None, **kwargs) @@ -272,7 +270,6 @@ def wrapper( tags: set[str] | None, **kwargs: dict[str, Any], ) -> Any: - # pylint: disable=too-many-arguments # If help is invoke somewhere, do not parse inventory if ctx.obj.get("_anta_help"): return f(*args, tags=tags, **kwargs) diff --git a/anta/device.py b/anta/device.py index 74b81d91e..35b7f4c04 100644 --- a/anta/device.py +++ b/anta/device.py @@ -106,7 +106,7 @@ def _init_cache(self) -> None: @property def cache_statistics(self) -> dict[str, Any] | None: - """Returns the device cache statistics for logging purposes.""" + """Return the device cache statistics for logging purposes.""" # Need to ignore pylint no-member as Cache is a proxy class and pylint is not smart enough # https://github.com/pylint-dev/pylint/issues/7258 if self.cache is not None: @@ -126,6 +126,17 @@ def __rich_repr__(self) -> Iterator[tuple[str, Any]]: yield "established", self.established yield "disable_cache", self.cache is None + def __repr__(self) -> str: + """Return a printable representation of an AntaDevice.""" + return ( + f"AntaDevice({self.name!r}, " + f"tags={self.tags!r}, " + f"hw_model={self.hw_model!r}, " + f"is_online={self.is_online!r}, " + f"established={self.established!r}, " + f"disable_cache={self.cache is None!r})" + ) + @abstractmethod async def _collect(self, command: AntaCommand, *, collection_id: str | None = None) -> None: """Collect device command output. @@ -244,7 +255,6 @@ class AsyncEOSDevice(AntaDevice): """ - # pylint: disable=R0913 def __init__( self, host: str, @@ -338,6 +348,22 @@ def __rich_repr__(self) -> Iterator[tuple[str, Any]]: yield ("_session", vars(self._session)) yield ("_ssh_opts", _ssh_opts) + def __repr__(self) -> str: + """Return a printable representation of an AsyncEOSDevice.""" + return ( + f"AsyncEOSDevice({self.name!r}, " + f"tags={self.tags!r}, " + f"hw_model={self.hw_model!r}, " + f"is_online={self.is_online!r}, " + f"established={self.established!r}, " + f"disable_cache={self.cache is None!r}, " + f"host={self._session.host!r}, " + f"eapi_port={self._session.port!r}, " + f"username={self._ssh_opts.username!r}, " + f"enable={self.enable!r}, " + f"insecure={self._ssh_opts.known_hosts is None!r})" + ) + @property def _keys(self) -> tuple[Any, ...]: """Two AsyncEOSDevice objects are equal if the hostname and the port are the same. @@ -346,7 +372,7 @@ def _keys(self) -> tuple[Any, ...]: """ return (self._session.host, self._session.port) - async def _collect(self, command: AntaCommand, *, collection_id: str | None = None) -> None: # noqa: C901 function is too complex - because of many required except blocks #pylint: disable=line-too-long + async def _collect(self, command: AntaCommand, *, collection_id: str | None = None) -> None: # noqa: C901 function is too complex - because of many required except blocks """Collect device command output from EOS using aio-eapi. Supports outformat `json` and `text` as output structure. diff --git a/anta/inventory/__init__.py b/anta/inventory/__init__.py index 29450be62..3046d7a66 100644 --- a/anta/inventory/__init__.py +++ b/anta/inventory/__init__.py @@ -171,7 +171,6 @@ def _parse_ranges( anta_log_exception(e, message, logger) raise InventoryIncorrectSchemaError(message) from e - # pylint: disable=too-many-arguments @staticmethod def parse( filename: str | Path, diff --git a/anta/models.py b/anta/models.py index 9a695bcd6..b103a9965 100644 --- a/anta/models.py +++ b/anta/models.py @@ -18,7 +18,7 @@ from anta import GITHUB_SUGGESTION from anta.custom_types import REGEXP_EOS_BLACKLIST_CMDS, Revision from anta.logger import anta_log_exception, exc_to_str -from anta.result_manager.models import TestResult +from anta.result_manager.models import AntaTestStatus, TestResult if TYPE_CHECKING: from collections.abc import Coroutine @@ -71,7 +71,6 @@ def __init__( *, use_cache: bool = True, ) -> None: - # pylint: disable=too-many-arguments self.template = template self.version = version self.revision = revision @@ -430,7 +429,7 @@ def __init__( description=self.description, ) self._init_inputs(inputs) - if self.result.result == "unset": + if self.result.result == AntaTestStatus.UNSET: self._init_commands(eos_data) def _init_inputs(self, inputs: dict[str, Any] | AntaTest.Input | None) -> None: @@ -481,7 +480,7 @@ def _init_commands(self, eos_data: list[dict[Any, Any] | str] | None) -> None: except NotImplementedError as e: self.result.is_error(message=e.args[0]) return - except Exception as e: # pylint: disable=broad-exception-caught + except Exception as e: # noqa: BLE001 # render() is user-defined code. # We need to catch everything if we want the AntaTest object # to live until the reporting @@ -559,7 +558,7 @@ async def collect(self) -> None: try: if self.blocked is False: await self.device.collect_commands(self.instance_commands, collection_id=self.name) - except Exception as e: # pylint: disable=broad-exception-caught + except Exception as e: # noqa: BLE001 # device._collect() is user-defined code. # We need to catch everything if we want the AntaTest object # to live until the reporting @@ -631,7 +630,7 @@ async def wrapper( try: function(self, **kwargs) - except Exception as e: # pylint: disable=broad-exception-caught + except Exception as e: # noqa: BLE001 # test() is user-defined code. # We need to catch everything if we want the AntaTest object # to live until the reporting diff --git a/anta/runner.py b/anta/runner.py index e07cba94f..c818d192d 100644 --- a/anta/runner.py +++ b/anta/runner.py @@ -147,14 +147,15 @@ def prepare_tests( # Create AntaTestRunner tuples from the tags for device in inventory.devices: if tags: - # If there are CLI tags, only execute tests with matching tags - device_to_tests[device].update(catalog.get_tests_by_tags(tags)) + if not any(tag in device.tags for tag in tags): + # The device does not have any selected tag, skipping + continue else: # If there is no CLI tags, execute all tests that do not have any tags device_to_tests[device].update(catalog.tag_to_tests[None]) - # Then add the tests with matching tags from device tags - device_to_tests[device].update(catalog.get_tests_by_tags(device.tags)) + # Add the tests with matching tags from device tags + device_to_tests[device].update(catalog.get_tests_by_tags(device.tags)) catalog.final_tests_count += len(device_to_tests[device]) @@ -187,12 +188,12 @@ def get_coroutines(selected_tests: defaultdict[AntaDevice, set[AntaTestDefinitio try: test_instance = test.test(device=device, inputs=test.inputs) coros.append(test_instance.test()) - except Exception as e: # noqa: PERF203, pylint: disable=broad-exception-caught + except Exception as e: # noqa: PERF203, BLE001 # An AntaTest instance is potentially user-defined code. # We need to catch everything and exit gracefully with an error message. message = "\n".join( [ - f"There is an error when creating test {test.test.module}.{test.test.__name__}.", + f"There is an error when creating test {test.test.__module__}.{test.test.__name__}.", f"If this is not a custom test implementation: {GITHUB_SUGGESTION}", ], ) @@ -212,7 +213,6 @@ async def main( # noqa: PLR0913 established_only: bool = True, dry_run: bool = False, ) -> None: - # pylint: disable=too-many-arguments """Run ANTA. Use this as an entrypoint to the test framework in your script. diff --git a/anta/tests/logging.py b/anta/tests/logging.py index c5202cce1..b32bc99fd 100644 --- a/anta/tests/logging.py +++ b/anta/tests/logging.py @@ -100,13 +100,13 @@ class VerifyLoggingSourceIntf(AntaTest): ``` """ - name = "VerifyLoggingSourceInt" + name = "VerifyLoggingSourceIntf" description = "Verifies logging source-interface for a specified VRF." categories: ClassVar[list[str]] = ["logging"] commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command="show logging", ofmt="text")] class Input(AntaTest.Input): - """Input model for the VerifyLoggingSourceInt test.""" + """Input model for the VerifyLoggingSourceIntf test.""" interface: str """Source-interface to use as source IP of log messages.""" @@ -115,7 +115,7 @@ class Input(AntaTest.Input): @AntaTest.anta_test def test(self) -> None: - """Main test function for VerifyLoggingSourceInt.""" + """Main test function for VerifyLoggingSourceIntf.""" output = self.instance_commands[0].text_output pattern = rf"Logging source-interface '{self.inputs.interface}'.*VRF {self.inputs.vrf}" if re.search(pattern, _get_logging_states(self.logger, output)): diff --git a/anta/tools.py b/anta/tools.py index 00aad5afe..dc4dc12ca 100644 --- a/anta/tools.py +++ b/anta/tools.py @@ -8,10 +8,12 @@ import cProfile import os import pstats +import re from functools import wraps from time import perf_counter from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast +from anta.custom_types import REGEXP_PATH_MARKERS from anta.logger import format_td if TYPE_CHECKING: @@ -82,7 +84,6 @@ def custom_division(numerator: float, denominator: float) -> int | float: return int(result) if result.is_integer() else result -# pylint: disable=too-many-arguments def get_dict_superset( list_of_dicts: list[dict[Any, Any]], input_dict: dict[Any, Any], @@ -142,7 +143,6 @@ def get_dict_superset( return default -# pylint: disable=too-many-arguments def get_value( dictionary: dict[Any, Any], key: str, @@ -199,7 +199,6 @@ def get_value( return value -# pylint: disable=too-many-arguments def get_item( list_of_dicts: list[dict[Any, Any]], key: Any, @@ -357,3 +356,19 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any: return cast(F, wrapper) return decorator + + +def safe_command(command: str) -> str: + """Return a sanitized command. + + Parameters + ---------- + command + The command to sanitize. + + Returns + ------- + str + The sanitized command. + """ + return re.sub(rf"{REGEXP_PATH_MARKERS}", "_", command) diff --git a/asynceapi/device.py b/asynceapi/device.py index 394abe40d..ca5a30c25 100644 --- a/asynceapi/device.py +++ b/asynceapi/device.py @@ -54,7 +54,7 @@ class Device(httpx.AsyncClient): EAPI_OFMT_OPTIONS = ("json", "text") EAPI_DEFAULT_OFMT = "json" - def __init__( # pylint: disable=too-many-arguments + def __init__( self, host: str | None = None, username: str | None = None, @@ -115,7 +115,7 @@ async def check_connection(self) -> bool: """ return await port_check_url(self.base_url) - async def cli( # noqa: PLR0913 # pylint: disable=too-many-arguments + async def cli( # noqa: PLR0913 self, command: str | dict[str, Any] | None = None, commands: Sequence[str | dict[str, Any]] | None = None, @@ -189,7 +189,7 @@ async def cli( # noqa: PLR0913 # pylint: disable=too-many-arguments return None raise - def _jsonrpc_command( # noqa: PLR0913 # pylint: disable=too-many-arguments + def _jsonrpc_command( # noqa: PLR0913 self, commands: Sequence[str | dict[str, Any]] | None = None, ofmt: str | None = None, diff --git a/asynceapi/errors.py b/asynceapi/errors.py index 020d3dc2f..e6794b7ef 100644 --- a/asynceapi/errors.py +++ b/asynceapi/errors.py @@ -24,7 +24,7 @@ class EapiCommandError(RuntimeError): not_exec: a list of commands that were not executed """ - def __init__(self, failed: str, errors: list[str], errmsg: str, passed: list[str | dict[str, Any]], not_exec: list[dict[str, Any]]) -> None: # pylint: disable=too-many-arguments + def __init__(self, failed: str, errors: list[str], errmsg: str, passed: list[str | dict[str, Any]], not_exec: list[dict[str, Any]]) -> None: """Initialize for the EapiCommandError exception.""" self.failed = failed self.errmsg = errmsg diff --git a/docs/advanced_usages/as-python-lib.md b/docs/advanced_usages/as-python-lib.md index 08bb818c1..d8790f3ef 100644 --- a/docs/advanced_usages/as-python-lib.md +++ b/docs/advanced_usages/as-python-lib.md @@ -40,8 +40,11 @@ The [AntaInventory](../api/inventory.md#anta.inventory.AntaInventory) class is a - The [connect_inventory()](../api/inventory.md#anta.inventory.AntaInventory.connect_inventory) coroutine will execute the [refresh()](../api/device.md#anta.device.AntaDevice.refresh) coroutines of all the devices in the inventory. - The [parse()](../api/inventory.md#anta.inventory.AntaInventory.parse) static method creates an [AntaInventory](../api/inventory.md#anta.inventory.AntaInventory) instance from a YAML file and returns it. The devices are [AsyncEOSDevice](../api/device.md#anta.device.AsyncEOSDevice) instances. +## Examples -To parse a YAML inventory file and print the devices connection status: +##### Parse an ANTA inventory file + +> This script parses an ANTA inventory file, connects to devices and print their status ```python """ @@ -81,7 +84,10 @@ if __name__ == "__main__": ??? note "How to create your inventory file" Please visit this [dedicated section](../usage-inventory-catalog.md) for how to use inventory and catalog files. -To run an EOS commands list on the reachable devices from the inventory: +##### Run EOS commands + +> This script runs a list of EOS commands on reachable devices + ```python """ Example @@ -138,176 +144,3 @@ if __name__ == "__main__": pprint(res) ``` - - -## Use tests from ANTA - -All the test classes inherit from the same abstract Base Class AntaTest. The Class definition indicates which commands are required for the test and the user should focus only on writing the `test` function with optional keywords argument. The instance of the class upon creation instantiates a TestResult object that can be accessed later on to check the status of the test ([unset, skipped, success, failure, error]). - -### Test structure - -All tests are built on a class named `AntaTest` which provides a complete toolset for a test: - -- Object creation -- Test definition -- TestResult definition -- Abstracted method to collect data - -This approach means each time you create a test it will be based on this `AntaTest` class. Besides that, you will have to provide some elements: - -- `name`: Name of the test -- `description`: A human readable description of your test -- `categories`: a list of categories to sort test. -- `commands`: a list of command to run. This list _must_ be a list of `AntaCommand` which is described in the next part of this document. - -Here is an example of a hardware test related to device temperature: - -```python -from __future__ import annotations - -import logging -from typing import Any, Dict, List, Optional, cast - -from anta.models import AntaTest, AntaCommand - - -class VerifyTemperature(AntaTest): - """ - Verifies device temparture is currently OK. - """ - - # The test name - name = "VerifyTemperature" - # A small description of the test, usually the first line of the class docstring - description = "Verifies device temparture is currently OK" - # The category of the test, usually the module name - categories = ["hardware"] - # The command(s) used for the test. Could be a template instead - commands = [AntaCommand(command="show system environment temperature", ofmt="json")] - - # Decorator - @AntaTest.anta_test - # abstract method that must be defined by the child Test class - def test(self) -> None: - """Run VerifyTemperature validation""" - command_output = cast(Dict[str, Dict[Any, Any]], self.instance_commands[0].output) - temperature_status = command_output["systemStatus"] if "systemStatus" in command_output.keys() else "" - if temperature_status == "temperatureOk": - self.result.is_success() - else: - self.result.is_failure(f"Device temperature is not OK, systemStatus: {temperature_status }") -``` - -When you run the test, object will automatically call its `anta.models.AntaTest.collect()` method to get device output for each command if no pre-collected data was given to the test. This method does a loop to call `anta.inventory.models.InventoryDevice.collect()` methods which is in charge of managing device connection and how to get data. - -??? info "run test offline" - You can also pass eos data directly to your test if you want to validate data collected in a different workflow. An example is provided below just for information: - - ```python - test = VerifyTemperature(device, eos_data=test_data["eos_data"]) - asyncio.run(test.test()) - ``` - -The `test` function is always the same and __must__ be defined with the `@AntaTest.anta_test` decorator. This function takes at least one argument which is a `anta.inventory.models.InventoryDevice` object. -In some cases a test would rely on some additional inputs from the user, for instance the number of expected peers or some expected numbers. All parameters __must__ come with a default value and the test function __should__ validate the parameters values (at this stage this is the only place where validation can be done but there are future plans to make this better). - -```python -class VerifyTemperature(AntaTest): - ... - @AntaTest.anta_test - def test(self) -> None: - pass - -class VerifyTransceiversManufacturers(AntaTest): - ... - @AntaTest.anta_test - def test(self, manufacturers: Optional[List[str]] = None) -> None: - # validate the manufactures parameter - pass -``` - -The test itself does not return any value, but the result is directly available from your AntaTest object and exposes a `anta.result_manager.models.TestResult` object with result, name of the test and optional messages: - - -- `name` (str): Device name where the test has run. -- `test` (str): Test name runs on the device. -- `categories` (List[str]): List of categories the TestResult belongs to, by default the AntaTest categories. -- `description` (str): TestResult description, by default the AntaTest description. -- `results` (str): Result of the test. Can be one of ["unset", "success", "failure", "error", "skipped"]. -- `message` (str, optional): Message to report after the test if any. -- `custom_field` (str, optional): Custom field to store a string for flexibility in integrating with ANTA - -```python -from anta.tests.hardware import VerifyTemperature - -test = VerifyTemperature(device, eos_data=test_data["eos_data"]) -asyncio.run(test.test()) -assert test.result.result == "success" -``` - -### Classes for commands - -To make it easier to get data, ANTA defines 2 different classes to manage commands to send to devices: - -#### [AntaCommand](../api/models.md#anta.models.AntaCommand) Class - -Represent a command with following information: - -- Command to run -- Output format expected -- eAPI version -- Output of the command - -Usage example: - -```python -from anta.models import AntaCommand - -cmd1 = AntaCommand(command="show zerotouch") -cmd2 = AntaCommand(command="show running-config diffs", ofmt="text") -``` - -!!! tip "Command revision and version" - * Most of EOS commands return a JSON structure according to a model (some commands may not be modeled hence the necessity to use `text` outformat sometimes. - * The model can change across time (adding feature, ... ) and when the model is changed in a non backward-compatible way, the __revision__ number is bumped. The initial model starts with __revision__ 1. - * A __revision__ applies to a particular CLI command whereas a __version__ is global to an eAPI call. The __version__ is internally translated to a specific __revision__ for each CLI command in the RPC call. The currently supported __version__ values are `1` and `latest`. - * A __revision takes precedence over a version__ (e.g. if a command is run with version="latest" and revision=1, the first revision of the model is returned) - * By default, eAPI returns the first revision of each model to ensure that when upgrading, integrations with existing tools are not broken. This is done by using by default `version=1` in eAPI calls. - - By default, ANTA uses `version="latest"` in AntaCommand, but when developing tests, the revision MUST be provided when the outformat of the command is `json`. As explained earlier, this is to ensure that the eAPI always returns the same output model and that the test remains always valid from the day it was created. For some commands, you may also want to run them with a different revision or version. - - For instance, the `VerifyBFDPeersHealth` test leverages the first revision of `show bfd peers`: - - ``` - # revision 1 as later revision introduce additional nesting for type - commands = [AntaCommand(command="show bfd peers", revision=1)] - ``` - -#### [AntaTemplate](../api/models.md#anta.models.AntaTemplate) Class - -Because some command can require more dynamic than just a command with no parameter provided by user, ANTA supports command template: you define a template in your test class and user provide parameters when creating test object. - -```python - -class RunArbitraryTemplateCommand(AntaTest): - """ - Run an EOS command and return result - Based on AntaTest to build relevant output for pytest - """ - - name = "Run aributrary EOS command" - description = "To be used only with anta debug commands" - template = AntaTemplate(template="show interfaces {ifd}") - categories = ["debug"] - - @AntaTest.anta_test - def test(self) -> None: - errdisabled_interfaces = [interface for interface, value in response["interfaceStatuses"].items() if value["linkStatus"] == "errdisabled"] - ... - - -params = [{"ifd": "Ethernet2"}, {"ifd": "Ethernet49/1"}] -run_command1 = RunArbitraryTemplateCommand(device_anta, params) -``` - -In this example, test waits for interfaces to check from user setup and will only check for interfaces in `params` diff --git a/docs/advanced_usages/custom-tests.md b/docs/advanced_usages/custom-tests.md index c6a2fa896..958a05539 100644 --- a/docs/advanced_usages/custom-tests.md +++ b/docs/advanced_usages/custom-tests.md @@ -199,6 +199,22 @@ class (AntaTest): ] ``` +!!! tip "Command revision and version" + * Most of EOS commands return a JSON structure according to a model (some commands may not be modeled hence the necessity to use `text` outformat sometimes. + * The model can change across time (adding feature, ... ) and when the model is changed in a non backward-compatible way, the __revision__ number is bumped. The initial model starts with __revision__ 1. + * A __revision__ applies to a particular CLI command whereas a __version__ is global to an eAPI call. The __version__ is internally translated to a specific __revision__ for each CLI command in the RPC call. The currently supported __version__ values are `1` and `latest`. + * A __revision takes precedence over a version__ (e.g. if a command is run with version="latest" and revision=1, the first revision of the model is returned) + * By default, eAPI returns the first revision of each model to ensure that when upgrading, integrations with existing tools are not broken. This is done by using by default `version=1` in eAPI calls. + + By default, ANTA uses `version="latest"` in AntaCommand, but when developing tests, the revision MUST be provided when the outformat of the command is `json`. As explained earlier, this is to ensure that the eAPI always returns the same output model and that the test remains always valid from the day it was created. For some commands, you may also want to run them with a different revision or version. + + For instance, the `VerifyBFDPeersHealth` test leverages the first revision of `show bfd peers`: + + ``` + # revision 1 as later revision introduce additional nesting for type + commands = [AntaCommand(command="show bfd peers", revision=1)] + ``` + ### Inputs definition If the user needs to provide inputs for your test, you need to define a [pydantic model](https://docs.pydantic.dev/latest/usage/models/) that defines the schema of the test inputs: diff --git a/docs/cli/tag-management.md b/docs/cli/tag-management.md index 8c043d712..0bba29e70 100644 --- a/docs/cli/tag-management.md +++ b/docs/cli/tag-management.md @@ -3,163 +3,198 @@ ~ Use of this source code is governed by the Apache License 2.0 ~ that can be found in the LICENSE file. --> +## Overview -# Tag management +ANTA commands can be used with a `--tags` option. This option **filters the inventory** with the specified tag(s) when running the command. -## Overview +Tags can also be used to **restrict a specific test** to a set of devices when using `anta nrfu`. + +## Defining tags + +### Device tags + +Device tags can be defined in the inventory: + +```yaml +anta_inventory: + hosts: + - name: leaf1 + host: leaf1.anta.arista.com + tags: ["leaf"] + - name: leaf2 + host: leaf2.anta.arista.com + tags: ["leaf"] + - name: spine1 + host: spine1.anta.arista.com + tags: ["spine"] +``` + +Each device also has its own name automatically added as a tag: + +```bash +anta get inventory +Current inventory content is: +{ + 'leaf1': AsyncEOSDevice( + name='leaf1', + tags={'leaf', 'leaf1'}, <-- + [...] + host='leaf1.anta.arista.com', + [...] + ), + 'leaf2': AsyncEOSDevice( + name='leaf2', + tags={'leaf', 'leaf2'}, <-- + [...] + host='leaf2.anta.arista.com', + [...] + ), + 'spine1': AsyncEOSDevice( + name='spine1', + tags={'spine1', 'spine'}, <-- + [...] + host='spine1.anta.arista.com', + [...] + ) +} +``` -Some of the ANTA commands like `anta nrfu` command come with a `--tags` option. +### Test tags + +Tags can be defined in the test catalog to restrict tests to tagged devices: + +```yaml +anta.tests.system: + - VerifyUptime: + minimum: 10 + filters: + tags: ['spine'] + - VerifyUptime: + minimum: 9 + filters: + tags: ['leaf'] + - VerifyReloadCause: + filters: + tags: ['spine', 'leaf'] + - VerifyCoredump: + - VerifyAgentLogs: + - VerifyCPUUtilization: + - VerifyMemoryUtilization: + - VerifyFileSystemUtilization: + - VerifyNTP: + +anta.tests.mlag: + - VerifyMlagStatus: + filters: + tags: ['leaf'] + +anta.tests.interfaces: + - VerifyL3MTU: + mtu: 1500 + filters: + tags: ['spine'] +``` -For `nrfu`, this allows users to specify a set of tests, marked with a given tag, to be run on devices marked with the same tag. For instance, you can run tests dedicated to leaf devices on your leaf devices only and not on other devices. +> A tag used to filter a test can also be a device name -Tags are string defined by the user and can be anything considered as a string by Python. A [default one](#default-tags) is present for all tests and devices. +!!! tip "Use different input values for a specific test" + Leverage tags to define different input values for a specific test. See the `VerifyUptime` example above. -The next table provides a short summary of the scope of tags using CLI +## Using tags | Command | Description | | ------- | ----------- | -| `none` | Run all tests on all devices according `tag` definition in your inventory and test catalog. And tests with no tag are executed on all devices| -| `--tags leaf` | Run all tests marked with `leaf` tag on all devices configured with `leaf` tag.
All other tags are ignored | -| `--tags leaf,spine` | Run all tests marked with `leaf` tag on all devices configured with `leaf` tag.
Run all tests marked with `spine` tag on all devices configured with `spine` tag.
All other tags are ignored | - -## Inventory and Catalog for tests - -All commands in this page are based on the following inventory and test catalog. - -=== "Inventory" - - ```yaml - --- - anta_inventory: - hosts: - - host: 192.168.0.10 - name: spine01 - tags: ['fabric', 'spine'] - - host: 192.168.0.11 - name: spine02 - tags: ['fabric', 'spine'] - - host: 192.168.0.12 - name: leaf01 - tags: ['fabric', 'leaf'] - - host: 192.168.0.13 - name: leaf02 - tags: ['fabric', 'leaf'] - - host: 192.168.0.14 - name: leaf03 - tags: ['fabric', 'leaf'] - - host: 192.168.0.15 - name: leaf04 - tags: ['fabric', 'leaf' - ``` - -=== "Test Catalog" - - ```yaml - anta.tests.system: - - VerifyUptime: - minimum: 10 - filters: - tags: ['fabric'] - - VerifyReloadCause: - tags: ['leaf', spine'] - - VerifyCoredump: - - VerifyAgentLogs: - - VerifyCPUUtilization: - filters: - tags: ['spine', 'leaf'] - - VerifyMemoryUtilization: - - VerifyFileSystemUtilization: - - VerifyNTP: - - anta.tests.mlag: - - VerifyMlagStatus: - - - anta.tests.interfaces: - - VerifyL3MTU: - mtu: 1500 - filters: - tags: ['demo'] - ``` - -## Default tags - -By default, ANTA uses a default tag for both devices and tests. This default tag is `all` and it can be explicit if you want to make it visible in your inventory and also implicit since the framework injects this tag if it is not defined. - -So this command will run all tests from your catalog on all devices. With a mapping for `tags` defined in your inventory and catalog. If no `tags` configured, then tests are executed against all devices. +| No `--tags` option | Run all tests on all devices according to the `tag` definitions in your inventory and test catalog.
Tests without tags are executed on all devices. | +| `--tags leaf` | Run all tests marked with the `leaf` tag on all devices configured with the `leaf` tag.
All other tests are ignored. | +| `--tags leaf,spine` | Run all tests marked with the `leaf` tag on all devices configured with the `leaf` tag.
Run all tests marked with the `spine` tag on all devices configured with the `spine` tag.
All other tests are ignored. | -```bash -$ anta nrfu -c .personal/catalog-class.yml table --group-by device +### Examples + +The following examples use the inventory and test catalog defined above. +##### No `--tags` option + +Tests without tags are run on all devices. +Tests with tags will only run on devices with matching tags. + +```bash +$ anta nrfu table --group-by device ╭────────────────────── Settings ──────────────────────╮ -│ Running ANTA tests: │ -│ - ANTA Inventory contains 6 devices (AsyncEOSDevice) │ -│ - Tests catalog contains 10 tests │ +│ - ANTA Inventory contains 3 devices (AsyncEOSDevice) │ +│ - Tests catalog contains 11 tests │ ╰──────────────────────────────────────────────────────╯ -┏━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Device ┃ # of success ┃ # of skipped ┃ # of failure ┃ # of errors ┃ List of failed or error test cases ┃ -┡━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ spine01 │ 5 │ 1 │ 1 │ 0 │ ['VerifyCPUUtilization'] │ -│ spine02 │ 5 │ 1 │ 1 │ 0 │ ['VerifyCPUUtilization'] │ -│ leaf01 │ 6 │ 0 │ 1 │ 0 │ ['VerifyCPUUtilization'] │ -│ leaf02 │ 6 │ 0 │ 1 │ 0 │ ['VerifyCPUUtilization'] │ -│ leaf03 │ 6 │ 0 │ 1 │ 0 │ ['VerifyCPUUtilization'] │ -│ leaf04 │ 6 │ 0 │ 1 │ 0 │ ['VerifyCPUUtilization'] │ -└─────────┴──────────────┴──────────────┴──────────────┴─────────────┴────────────────────────────────────┘ +--- ANTA NRFU Run Information --- +Number of devices: 3 (3 established) +Total number of selected tests: 27 +--------------------------------- + Summary per device +┏━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Device ┃ # of success ┃ # of skipped ┃ # of failure ┃ # of errors ┃ List of failed or error test cases ┃ +┡━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ leaf1 │ 9 │ 0 │ 0 │ 0 │ │ +├────────┼──────────────┼──────────────┼──────────────┼─────────────┼────────────────────────────────────┤ +│ leaf2 │ 7 │ 1 │ 1 │ 0 │ VerifyAgentLogs │ +├────────┼──────────────┼──────────────┼──────────────┼─────────────┼────────────────────────────────────┤ +│ spine1 │ 9 │ 0 │ 0 │ 0 │ │ +└────────┴──────────────┴──────────────┴──────────────┴─────────────┴────────────────────────────────────┘ ``` -## Use a single tag in CLI +##### Single tag -The most used approach is to use a single tag in your CLI to filter tests & devices configured with this one. - -In such scenario, ANTA will run tests marked with `$tag` only on devices marked with `$tag`. All other tests and devices will be ignored +With a tag specified, only tests matching this tag will be run on matching devices. ```bash -$ anta nrfu -c .personal/catalog-class.yml --tags leaf text +$ anta nrfu --tags leaf text ╭────────────────────── Settings ──────────────────────╮ -│ Running ANTA tests: │ -│ - ANTA Inventory contains 6 devices (AsyncEOSDevice) │ -│ - Tests catalog contains 10 tests │ +│ - ANTA Inventory contains 3 devices (AsyncEOSDevice) │ +│ - Tests catalog contains 11 tests │ ╰──────────────────────────────────────────────────────╯ -leaf01 :: VerifyUptime :: SUCCESS -leaf01 :: VerifyReloadCause :: SUCCESS -leaf01 :: VerifyCPUUtilization :: SUCCESS -leaf02 :: VerifyUptime :: SUCCESS -leaf02 :: VerifyReloadCause :: SUCCESS -leaf02 :: VerifyCPUUtilization :: SUCCESS -leaf03 :: VerifyUptime :: SUCCESS -leaf03 :: VerifyReloadCause :: SUCCESS -leaf03 :: VerifyCPUUtilization :: SUCCESS -leaf04 :: VerifyUptime :: SUCCESS -leaf04 :: VerifyReloadCause :: SUCCESS -leaf04 :: VerifyCPUUtilization :: SUCCESS +--- ANTA NRFU Run Information --- +Number of devices: 3 (2 established) +Total number of selected tests: 6 +--------------------------------- + +leaf1 :: VerifyReloadCause :: SUCCESS +leaf1 :: VerifyUptime :: SUCCESS +leaf1 :: VerifyMlagStatus :: SUCCESS +leaf2 :: VerifyReloadCause :: SUCCESS +leaf2 :: VerifyUptime :: SUCCESS +leaf2 :: VerifyMlagStatus :: SKIPPED (MLAG is disabled) ``` -In this case, only `leaf` devices defined in your [inventory](#inventory-and-catalog-for-tests) are used to run tests marked with `leaf` in your [test catalog](#inventory-and-catalog-for-tests) - -## Use multiple tags in CLI +In this case, only `leaf` devices defined in the inventory are used to run tests marked with the `leaf` in the test catalog. -A more advanced usage of the tag feature is to list multiple tags in your CLI using `--tags $tag1,$tag2` syntax. +##### Multiple tags -In such scenario, all devices marked with `$tag1` will be selected and ANTA will run tests with `$tag1`, then devices with `$tag2` will be selected and will be tested with tests marked with `$tag2` +It is possible to use multiple tags using the `--tags tag1,tag2` syntax. ```bash -anta nrfu -c .personal/catalog-class.yml --tags leaf,fabric text - -spine01 :: VerifyUptime :: SUCCESS -spine02 :: VerifyUptime :: SUCCESS -leaf01 :: VerifyUptime :: SUCCESS -leaf01 :: VerifyReloadCause :: SUCCESS -leaf01 :: VerifyCPUUtilization :: SUCCESS -leaf02 :: VerifyUptime :: SUCCESS -leaf02 :: VerifyReloadCause :: SUCCESS -leaf02 :: VerifyCPUUtilization :: SUCCESS -leaf03 :: VerifyUptime :: SUCCESS -leaf03 :: VerifyReloadCause :: SUCCESS -leaf03 :: VerifyCPUUtilization :: SUCCESS -leaf04 :: VerifyUptime :: SUCCESS -leaf04 :: VerifyReloadCause :: SUCCESS -leaf04 :: VerifyCPUUtilization :: SUCCESS +anta nrfu --tags leaf,spine text +╭────────────────────── Settings ──────────────────────╮ +│ - ANTA Inventory contains 3 devices (AsyncEOSDevice) │ +│ - Tests catalog contains 11 tests │ +╰──────────────────────────────────────────────────────╯ + +--- ANTA NRFU Run Information --- +Number of devices: 3 (3 established) +Total number of selected tests: 15 +--------------------------------- + +leaf1 :: VerifyReloadCause :: SUCCESS +leaf1 :: VerifyMlagStatus :: SUCCESS +leaf1 :: VerifyUptime :: SUCCESS +leaf1 :: VerifyL3MTU :: SUCCESS +leaf1 :: VerifyUptime :: SUCCESS +leaf2 :: VerifyReloadCause :: SUCCESS +leaf2 :: VerifyMlagStatus :: SKIPPED (MLAG is disabled) +leaf2 :: VerifyUptime :: SUCCESS +leaf2 :: VerifyL3MTU :: SUCCESS +leaf2 :: VerifyUptime :: SUCCESS +spine1 :: VerifyReloadCause :: SUCCESS +spine1 :: VerifyMlagStatus :: SUCCESS +spine1 :: VerifyUptime :: SUCCESS +spine1 :: VerifyL3MTU :: SUCCESS +spine1 :: VerifyUptime :: SUCCESS ``` diff --git a/docs/contribution.md b/docs/contribution.md index ac5d026f3..387e2f4c7 100644 --- a/docs/contribution.md +++ b/docs/contribution.md @@ -39,10 +39,10 @@ default environments: clean -> Erase previous coverage reports lint -> Check the code style type -> Check typing -py38 -> Run pytest with py38 py39 -> Run pytest with py39 py310 -> Run pytest with py310 py311 -> Run pytest with py311 +py312 -> Run pytest with py312 report -> Generate coverage report ``` @@ -51,21 +51,22 @@ report -> Generate coverage report ```bash tox -e lint [...] -lint: commands[0]> black --check --diff --color . -All done! ✨ 🍰 ✨ -104 files would be left unchanged. -lint: commands[1]> isort --check --diff --color . -Skipped 7 files -lint: commands[2]> flake8 --max-line-length=165 --config=/dev/null anta -lint: commands[3]> flake8 --max-line-length=165 --config=/dev/null tests -lint: commands[4]> pylint anta +lint: commands[0]> ruff check . +All checks passed! +lint: commands[1]> ruff format . --check +158 files already formatted +lint: commands[2]> pylint anta -------------------------------------------------------------------- Your code has been rated at 10.00/10 (previous run: 10.00/10, +0.00) -.pkg: _exit> python /Users/guillaumemulocher/.pyenv/versions/3.8.13/envs/anta/lib/python3.8/site-packages/pyproject_api/_backend.py True setuptools.build_meta - lint: OK (19.26=setup[5.83]+cmd[1.50,0.76,1.19,1.20,8.77] seconds) - congratulations :) (19.56 seconds) +lint: commands[3]> pylint tests + +-------------------------------------------------------------------- +Your code has been rated at 10.00/10 (previous run: 10.00/10, +0.00) + + lint: OK (22.69=setup[2.19]+cmd[0.02,0.02,9.71,10.75] seconds) + congratulations :) (22.72 seconds) ``` ### Code Typing @@ -75,10 +76,11 @@ tox -e type [...] type: commands[0]> mypy --config-file=pyproject.toml anta -Success: no issues found in 52 source files -.pkg: _exit> python /Users/guillaumemulocher/.pyenv/versions/3.8.13/envs/anta/lib/python3.8/site-packages/pyproject_api/_backend.py True setuptools.build_meta - type: OK (46.66=setup[24.20]+cmd[22.46] seconds) - congratulations :) (47.01 seconds) +Success: no issues found in 68 source files +type: commands[1]> mypy --config-file=pyproject.toml tests +Success: no issues found in 82 source files + type: OK (31.15=setup[14.62]+cmd[6.05,10.48] seconds) + congratulations :) (31.18 seconds) ``` > NOTE: Typing is configured quite strictly, do not hesitate to reach out if you have any questions, struggles, nightmares. @@ -92,7 +94,7 @@ All submodule should have its own pytest section under `tests/units/anta_tests/< ### How to write a unit test for an AntaTest subclass The Python modules in the `tests/units/anta_tests` folder define test parameters for AntaTest subclasses unit tests. -A generic test function is written for all unit tests in `tests.lib.anta` module. +A generic test function is written for all unit tests in `tests.units.anta_tests` module. The `pytest_generate_tests` function definition in `conftest.py` is called during test collection. @@ -116,7 +118,7 @@ Test example for `anta.tests.system.VerifyUptime` AntaTest. ``` python # Import the generic test function -from tests.lib.anta import test # noqa: F401 +from tests.units.anta_tests import test # Import your AntaTest from anta.tests.system import VerifyUptime @@ -157,19 +159,20 @@ pre-commit install When running a commit or a pre-commit check: ``` bash -❯ echo "import foobaz" > test.py && git add test.py ❯ pre-commit -pylint...................................................................Failed -- hook id: pylint -- exit code: 22 - -************* Module test -test.py:1:0: C0114: Missing module docstring (missing-module-docstring) -test.py:1:0: E0401: Unable to import 'foobaz' (import-error) -test.py:1:0: W0611: Unused import foobaz (unused-import) +trim trailing whitespace.................................................Passed +fix end of files.........................................................Passed +check for added large files..............................................Passed +check for merge conflicts................................................Passed +Check and insert license on Python files.................................Passed +Check and insert license on Markdown files...............................Passed +Run Ruff linter..........................................................Passed +Run Ruff formatter.......................................................Passed +Check code style with pylint.............................................Passed +Checks for common misspellings in text files.............................Passed +Check typing with mypy...................................................Passed ``` -> NOTE: It could happen that pre-commit and tox disagree on something, in that case please open an issue on Github so we can take a look.. It is most probably wrong configuration on our side. ## Configure MYPYPATH diff --git a/docs/scripts/generate_svg.py b/docs/scripts/generate_svg.py index e6bf87abe..f017b243d 100644 --- a/docs/scripts/generate_svg.py +++ b/docs/scripts/generate_svg.py @@ -24,6 +24,7 @@ from rich.console import Console from rich.logging import RichHandler +from rich.progress import Progress from anta.cli.console import console from anta.cli.nrfu.utils import anta_progress_bar @@ -37,7 +38,7 @@ OUTPUT_DIR = pathlib.Path(__file__).parent.parent / "imgs" -def custom_progress_bar() -> None: +def custom_progress_bar() -> Progress: """Set the console of progress_bar to main anta console. Caveat: this capture all steps of the progress bar.. diff --git a/pyproject.toml b/pyproject.toml index cfcbdb6d1..c07dd0e96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,7 @@ dev = [ "pytest-httpx>=0.30.0", "pytest-metadata>=3.0.0", "pytest>=7.4.0", + "respx>=0.21.1", "ruff>=0.5.4,<0.7.0", "tox>=4.10.0,<5.0.0", "types-PyYAML", @@ -168,8 +169,9 @@ addopts = "-ra -q -vv --cov --cov-report term:skip-covered --color yes" log_level = "WARNING" render_collapsed = true testpaths = ["tests"] +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" filterwarnings = [ - "error", # cvprac is raising the next warning "default:pkg_resources is deprecated:DeprecationWarning", # Need to investigate the following - only occuring when running the full pytest suite @@ -337,7 +339,6 @@ select = ["ALL", # "D417", ] ignore = [ - "ANN101", # Missing type annotation for `self` in method - we know what self is.. "COM812", # Ignoring conflicting rules that may cause conflicts when used with the formatter "ISC001", # Ignoring conflicting rules that may cause conflicts when used with the formatter "TD002", # We don't have require authors in TODO @@ -382,9 +383,9 @@ runtime-evaluated-base-classes = ["pydantic.BaseModel", "anta.models.AntaTest.In "SLF001", # Lots of private member accessed for test purposes ] "tests/units/*" = [ - "BLE001", # Do not catch blind exception: `Exception` - already disabling this in pylint + "ARG002", # Sometimes we need to declare unused arguments when a parameter is not used but declared in @pytest.mark.parametrize "FBT001", # Boolean-typed positional argument in function definition - "PLR0913", # Too many arguments to function call (8 > 5) + "PLR0913", # Too many arguments to function call "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable "S105", # Passwords are indeed hardcoded in tests "S106", # Passwords are indeed hardcoded in tests @@ -393,8 +394,10 @@ runtime-evaluated-base-classes = ["pydantic.BaseModel", "anta.models.AntaTest.In "tests/units/anta_tests/test_interfaces.py" = [ "S104", # False positive for 0.0.0.0 bindings in test inputs ] +"tests/units/anta_tests/*" = [ + "F401", # In this module, we import tests.units.anta_tests.test without using it to auto-generate tests +] "anta/*" = [ - "BLE001", # Do not catch blind exception: `Exception` - caught by other linter "TRY400", # Use `logging.exception` instead of `logging.error` - we know what we are doing ] "anta/cli/exec/utils.py" = [ @@ -438,34 +441,33 @@ runtime-evaluated-base-classes = ["pydantic.BaseModel", "anta.models.AntaTest.In ################################ # Pylint ################################ -[tool.pylint.'MESSAGES CONTROL'] -disable = [ +[tool.pylint] +disable = [ # Any rule listed here can be disabled: https://github.com/astral-sh/ruff/issues/970 "invalid-name", - "fixme" + "fixme", + "unused-import", + "unused-argument", + "keyword-arg-before-vararg", + "protected-access", + "too-many-arguments", + "wrong-import-position", + "pointless-statement", + "broad-exception-caught", + "line-too-long", + "unused-variable", + "redefined-builtin", + "abstract-class-instantiated", # Overlap with https://mypy.readthedocs.io/en/stable/error_code_list.html#check-instantiation-of-abstract-classes-abstract + "unexpected-keyword-arg", # Overlap with https://mypy.readthedocs.io/en/stable/error_code_list.html#check-arguments-in-calls-call-arg and other rules + "no-value-for-parameter" # Overlap with https://mypy.readthedocs.io/en/stable/error_code_list.html#check-arguments-in-calls-call-arg ] - -[tool.pylint.DESIGN] max-statements=61 max-returns=8 max-locals=23 - -[tool.pylint.FORMAT] max-line-length=165 max-module-lines=1700 - -[tool.pylint.SIMILARITIES] # making similarity lines limit a bit higher than default 4 min-similarity-lines=10 - -[tool.pylint.TYPECHECK] # https://stackoverflow.com/questions/49680191/click-and-pylint signature-mutators="click.decorators.option" - -[tool.pylint.MAIN] load-plugins="pylint_pydantic" extension-pkg-whitelist="pydantic" -ignore-paths = [ - "^tests/units/anta_tests/.*/data.py$", - "^tests/units/anta_tests/routing/.*/data.py$", - "^docs/scripts/anta_runner.py", -] diff --git a/tests/conftest.py b/tests/conftest.py index e31533840..7347d4430 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,53 +1,56 @@ # Copyright (c) 2023-2024 Arista Networks, Inc. # Use of this source code is governed by the Apache License 2.0 # that can be found in the LICENSE file. -"""conftest.py - used to store anta specific fixtures used for tests.""" +"""See https://docs.pytest.org/en/stable/reference/fixtures.html#conftest-py-sharing-fixtures-across-multiple-files.""" -from __future__ import annotations - -import logging -from typing import Any +import asyncio +from collections.abc import Iterator +from pathlib import Path +from unittest.mock import AsyncMock, Mock, patch import pytest - -# Load fixtures from dedicated file tests/lib/fixture.py -# As well as pytest_asyncio plugin to test co-routines -pytest_plugins = [ - "tests.lib.fixture", - "pytest_asyncio", -] - -# Enable nice assert messages -# https://docs.pytest.org/en/7.1.x/how-to/writing_plugins.html#assertion-rewriting -pytest.register_assert_rewrite("tests.lib.anta") - -# Placeholder to disable logging of some external libs -for _ in ("asyncio", "httpx"): - logging.getLogger(_).setLevel(logging.CRITICAL) - - -def build_test_id(val: dict[str, Any]) -> str: - """Build id for a unit test of an AntaTest subclass. - - { - "name": "meaniful test name", - "test": , - ... - } - """ - return f"{val['test'].module}.{val['test'].__name__}-{val['name']}" - - -def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: - """Generate ANTA testts unit tests dynamically during test collection. - - It will parametrize test cases based on the `DATA` data structure defined in `tests.units.anta_tests` modules. - See `tests/units/anta_tests/README.md` for more information on how to use it. - Test IDs are generated using the `build_test_id` function above. - - Checking that only the function "test" is parametrized with data to allow for writing tests for helper functions - in each module. - """ - if "tests.units.anta_tests" in metafunc.module.__package__ and metafunc.function.__name__ == "test": - # This is a unit test for an AntaTest subclass - metafunc.parametrize("data", metafunc.module.DATA, ids=build_test_id) +import respx + +from anta.device import AsyncEOSDevice +from anta.inventory import AntaInventory + +DATA_DIR: Path = Path(__file__).parent.resolve() / "data" + + +@pytest.fixture(params=[{"count": 1}]) +def inventory(request: pytest.FixtureRequest) -> Iterator[AntaInventory]: + """Generate an ANTA inventory.""" + user = "admin" + password = "password" # noqa: S105 + disable_cache = request.param.get("disable_cache", True) + reachable = request.param.get("reachable", True) + if "filename" in request.param: + inv = AntaInventory.parse(DATA_DIR / request.param["filename"], username=user, password=password, disable_cache=disable_cache) + else: + inv = AntaInventory() + for i in range(request.param["count"]): + inv.add_device( + AsyncEOSDevice( + host=f"device-{i}.anta.arista.com", + username=user, + password=password, + name=f"device-{i}", + disable_cache=disable_cache, + ) + ) + if reachable: + # This context manager makes all devices reachable + with patch("asyncio.open_connection", AsyncMock(spec=asyncio.open_connection, return_value=(Mock(), Mock()))), respx.mock: + respx.post(path="/command-api", headers={"Content-Type": "application/json-rpc"}, json__params__cmds__0__cmd="show version").respond( + json={ + "result": [ + { + "modelName": "pytest", + } + ], + } + ) + yield inv + else: + with patch("asyncio.open_connection", AsyncMock(spec=asyncio.open_connection, side_effect=TimeoutError)): + yield inv diff --git a/tests/data/json_data.py b/tests/data/json_data.py deleted file mode 100644 index 563084065..000000000 --- a/tests/data/json_data.py +++ /dev/null @@ -1,259 +0,0 @@ -# Copyright (c) 2023-2024 Arista Networks, Inc. -# Use of this source code is governed by the Apache License 2.0 -# that can be found in the LICENSE file. -# pylint: skip-file -"""JSON Data for unit tests.""" - -INVENTORY_MODEL_HOST_VALID = [ - {"name": "validIPv4", "input": "1.1.1.1", "expected_result": "valid"}, - { - "name": "validIPv6", - "input": "fe80::cc62:a9ff:feef:932a", - }, -] - -INVENTORY_MODEL_HOST_INVALID = [ - { - "name": "invalidIPv4_with_netmask", - "input": "1.1.1.1/32", - }, - { - "name": "invalidIPv6_with_netmask", - "input": "fe80::cc62:a9ff:feef:932a/128", - }, - {"name": "invalidHost_format", "input": "@", "expected_result": "invalid"}, - { - "name": "invalidIPv6_format", - "input": "fe80::cc62:a9ff:feef:", - }, -] - -INVENTORY_MODEL_HOST_CACHE = [ - {"name": "Host cache default", "input": {"host": "1.1.1.1"}, "expected_result": False}, - {"name": "Host cache enabled", "input": {"host": "1.1.1.1", "disable_cache": False}, "expected_result": False}, - {"name": "Host cache disabled", "input": {"host": "1.1.1.1", "disable_cache": True}, "expected_result": True}, -] - -INVENTORY_MODEL_NETWORK_VALID = [ - {"name": "ValidIPv4_Subnet", "input": "1.1.1.0/24", "expected_result": "valid"}, - {"name": "ValidIPv6_Subnet", "input": "2001:db8::/32", "expected_result": "valid"}, -] - -INVENTORY_MODEL_NETWORK_INVALID = [ - {"name": "ValidIPv4_Subnet", "input": "1.1.1.0/17", "expected_result": "invalid"}, - { - "name": "InvalidIPv6_Subnet", - "input": "2001:db8::/16", - "expected_result": "invalid", - }, -] - -INVENTORY_MODEL_NETWORK_CACHE = [ - {"name": "Network cache default", "input": {"network": "1.1.1.0/24"}, "expected_result": False}, - {"name": "Network cache enabled", "input": {"network": "1.1.1.0/24", "disable_cache": False}, "expected_result": False}, - {"name": "Network cache disabled", "input": {"network": "1.1.1.0/24", "disable_cache": True}, "expected_result": True}, -] - -INVENTORY_MODEL_RANGE_VALID = [ - { - "name": "ValidIPv4_Range", - "input": {"start": "10.1.0.1", "end": "10.1.0.10"}, - "expected_result": "valid", - }, -] - -INVENTORY_MODEL_RANGE_INVALID = [ - { - "name": "InvalidIPv4_Range_name", - "input": {"start": "toto", "end": "10.1.0.1"}, - "expected_result": "invalid", - }, -] - -INVENTORY_MODEL_RANGE_CACHE = [ - {"name": "Range cache default", "input": {"start": "1.1.1.1", "end": "1.1.1.10"}, "expected_result": False}, - {"name": "Range cache enabled", "input": {"start": "1.1.1.1", "end": "1.1.1.10", "disable_cache": False}, "expected_result": False}, - {"name": "Range cache disabled", "input": {"start": "1.1.1.1", "end": "1.1.1.10", "disable_cache": True}, "expected_result": True}, -] - -INVENTORY_MODEL_VALID = [ - { - "name": "Valid_Host_Only", - "input": {"hosts": [{"host": "192.168.0.17"}, {"host": "192.168.0.2"}]}, - "expected_result": "valid", - }, - { - "name": "Valid_Networks_Only", - "input": {"networks": [{"network": "192.168.0.0/16"}, {"network": "192.168.1.0/24"}]}, - "expected_result": "valid", - }, - { - "name": "Valid_Ranges_Only", - "input": { - "ranges": [ - {"start": "10.1.0.1", "end": "10.1.0.10"}, - {"start": "10.2.0.1", "end": "10.2.1.10"}, - ], - }, - "expected_result": "valid", - }, -] - -INVENTORY_MODEL_INVALID = [ - { - "name": "Host_with_Invalid_entry", - "input": {"hosts": [{"host": "192.168.0.17"}, {"host": "192.168.0.2/32"}]}, - "expected_result": "invalid", - }, -] - -INVENTORY_DEVICE_MODEL_VALID = [ - { - "name": "Valid_Inventory", - "input": [{"host": "1.1.1.1", "username": "arista", "password": "arista123!"}, {"host": "1.1.1.2", "username": "arista", "password": "arista123!"}], - "expected_result": "valid", - }, -] - -INVENTORY_DEVICE_MODEL_INVALID = [ - { - "name": "Invalid_Inventory", - "input": [{"host": "1.1.1.1", "password": "arista123!"}, {"host": "1.1.1.1", "username": "arista"}], - "expected_result": "invalid", - }, -] - -ANTA_INVENTORY_TESTS_VALID = [ - { - "name": "ValidInventory_with_host_only", - "input": {"anta_inventory": {"hosts": [{"host": "192.168.0.17"}, {"host": "192.168.0.2"}, {"host": "my.awesome.host.com"}]}}, - "expected_result": "valid", - "parameters": { - "ipaddress_in_scope": "192.168.0.17", - "ipaddress_out_of_scope": "192.168.1.1", - "nb_hosts": 2, - }, - }, - { - "name": "ValidInventory_with_networks_only", - "input": {"anta_inventory": {"networks": [{"network": "192.168.0.0/24"}]}}, - "expected_result": "valid", - "parameters": { - "ipaddress_in_scope": "192.168.0.1", - "ipaddress_out_of_scope": "192.168.1.1", - "nb_hosts": 256, - }, - }, - { - "name": "ValidInventory_with_ranges_only", - "input": { - "anta_inventory": { - "ranges": [ - {"start": "10.0.0.1", "end": "10.0.0.11"}, - {"start": "10.0.0.101", "end": "10.0.0.111"}, - ], - }, - }, - "expected_result": "valid", - "parameters": { - "ipaddress_in_scope": "10.0.0.10", - "ipaddress_out_of_scope": "192.168.1.1", - "nb_hosts": 22, - }, - }, - { - "name": "ValidInventory_with_host_port", - "input": {"anta_inventory": {"hosts": [{"host": "192.168.0.17", "port": 443}, {"host": "192.168.0.2", "port": 80}]}}, - "expected_result": "valid", - "parameters": { - "ipaddress_in_scope": "192.168.0.17", - "ipaddress_out_of_scope": "192.168.1.1", - "nb_hosts": 2, - }, - }, - { - "name": "ValidInventory_with_host_tags", - "input": {"anta_inventory": {"hosts": [{"host": "192.168.0.17", "tags": ["leaf"]}, {"host": "192.168.0.2", "tags": ["spine"]}]}}, - "expected_result": "valid", - "parameters": { - "ipaddress_in_scope": "192.168.0.17", - "ipaddress_out_of_scope": "192.168.1.1", - "nb_hosts": 2, - }, - }, - { - "name": "ValidInventory_with_networks_tags", - "input": {"anta_inventory": {"networks": [{"network": "192.168.0.0/24", "tags": ["leaf"]}]}}, - "expected_result": "valid", - "parameters": { - "ipaddress_in_scope": "192.168.0.1", - "ipaddress_out_of_scope": "192.168.1.1", - "nb_hosts": 256, - }, - }, - { - "name": "ValidInventory_with_ranges_tags", - "input": { - "anta_inventory": { - "ranges": [ - {"start": "10.0.0.1", "end": "10.0.0.11", "tags": ["leaf"]}, - {"start": "10.0.0.101", "end": "10.0.0.111", "tags": ["spine"]}, - ], - }, - }, - "expected_result": "valid", - "parameters": { - "ipaddress_in_scope": "10.0.0.10", - "ipaddress_out_of_scope": "192.168.1.1", - "nb_hosts": 22, - }, - }, -] - -ANTA_INVENTORY_TESTS_INVALID = [ - { - "name": "InvalidInventory_with_host_only", - "input": {"anta_inventory": {"hosts": [{"host": "192.168.0.17/32"}, {"host": "192.168.0.2"}]}}, - "expected_result": "invalid", - }, - { - "name": "InvalidInventory_wrong_network_bits", - "input": {"anta_inventory": {"networks": [{"network": "192.168.42.0/8"}]}}, - "expected_result": "invalid", - }, - { - "name": "InvalidInventory_wrong_network", - "input": {"anta_inventory": {"networks": [{"network": "toto"}]}}, - "expected_result": "invalid", - }, - { - "name": "InvalidInventory_wrong_range", - "input": {"anta_inventory": {"ranges": [{"start": "toto", "end": "192.168.42.42"}]}}, - "expected_result": "invalid", - }, - { - "name": "InvalidInventory_wrong_range_type_mismatch", - "input": {"anta_inventory": {"ranges": [{"start": "fe80::cafe", "end": "192.168.42.42"}]}}, - "expected_result": "invalid", - }, - { - "name": "Invalid_Root_Key", - "input": { - "inventory": { - "ranges": [ - {"start": "10.0.0.1", "end": "10.0.0.11"}, - {"start": "10.0.0.100", "end": "10.0.0.111"}, - ], - }, - }, - "expected_result": "invalid", - }, -] - -TEST_RESULT_SET_STATUS = [ - {"name": "set_success", "target": "success", "message": "success"}, - {"name": "set_error", "target": "error", "message": "error"}, - {"name": "set_failure", "target": "failure", "message": "failure"}, - {"name": "set_skipped", "target": "skipped", "message": "skipped"}, - {"name": "set_unset", "target": "unset", "message": "unset"}, -] diff --git a/tests/data/test_catalog_with_tags.yml b/tests/data/test_catalog_with_tags.yml index 109781eaa..cf2bdffe7 100644 --- a/tests/data/test_catalog_with_tags.yml +++ b/tests/data/test_catalog_with_tags.yml @@ -3,30 +3,28 @@ anta.tests.system: - VerifyUptime: minimum: 10 filters: - tags: ['fabric'] + tags: ['spine'] - VerifyUptime: minimum: 9 filters: tags: ['leaf'] - VerifyReloadCause: filters: - tags: ['leaf', 'spine'] + tags: ['spine', 'leaf'] - VerifyCoredump: - VerifyAgentLogs: - VerifyCPUUtilization: - filters: - tags: ['leaf'] - VerifyMemoryUtilization: - filters: - tags: ['testdevice'] - VerifyFileSystemUtilization: - VerifyNTP: anta.tests.mlag: - VerifyMlagStatus: + filters: + tags: ['leaf'] anta.tests.interfaces: - VerifyL3MTU: mtu: 1500 filters: - tags: ['demo'] + tags: ['spine'] diff --git a/tests/data/test_inventory.yml b/tests/data/test_inventory.yml deleted file mode 100644 index d0ca45719..000000000 --- a/tests/data/test_inventory.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -anta_inventory: - hosts: - - name: dummy - host: dummy.anta.ninja - tags: ["leaf"] - - name: dummy2 - host: dummy2.anta.ninja - tags: ["leaf"] - - name: dummy3 - host: dummy3.anta.ninja - tags: ["spine"] diff --git a/tests/data/test_inventory_with_tags.yml b/tests/data/test_inventory_with_tags.yml new file mode 100644 index 000000000..cbbcd75e6 --- /dev/null +++ b/tests/data/test_inventory_with_tags.yml @@ -0,0 +1,12 @@ +--- +anta_inventory: + hosts: + - name: leaf1 + host: leaf1.anta.arista.com + tags: ["leaf"] + - name: leaf2 + host: leaf2.anta.arista.com + tags: ["leaf"] + - name: spine1 + host: spine1.anta.arista.com + tags: ["spine"] diff --git a/tests/data/toto.yml b/tests/data/toto.yml deleted file mode 100644 index c0f92cb81..000000000 --- a/tests/data/toto.yml +++ /dev/null @@ -1,16 +0,0 @@ -anta_inventory: - hosts: - - host: 10.73.1.238 - name: cv_atd1 - - host: 192.168.0.10 - name: spine1 - - host: 192.168.0.11 - name: spine2 - - host: 192.168.0.12 - name: leaf1 - - host: 192.168.0.13 - name: leaf2 - - host: 192.168.0.14 - name: leaf3 - - host: 192.168.0.15 - name: leaf4 diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py deleted file mode 100644 index cd54f3aac..000000000 --- a/tests/lib/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2023-2024 Arista Networks, Inc. -# Use of this source code is governed by the Apache License 2.0 -# that can be found in the LICENSE file. -"""Library for ANTA unit tests.""" diff --git a/tests/lib/anta.py b/tests/lib/anta.py deleted file mode 100644 index cabb27bc2..000000000 --- a/tests/lib/anta.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2023-2024 Arista Networks, Inc. -# Use of this source code is governed by the Apache License 2.0 -# that can be found in the LICENSE file. -"""generic test function used to generate unit tests for each AntaTest.""" - -from __future__ import annotations - -import asyncio -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from anta.device import AntaDevice - - -def test(device: AntaDevice, data: dict[str, Any]) -> None: - """Generic test function for AntaTest subclass. - - See `tests/units/anta_tests/README.md` for more information on how to use it. - """ - # Instantiate the AntaTest subclass - test_instance = data["test"](device, inputs=data["inputs"], eos_data=data["eos_data"]) - # Run the test() method - asyncio.run(test_instance.test()) - # Assert expected result - assert test_instance.result.result == data["expected"]["result"], test_instance.result.messages - if "messages" in data["expected"]: - # We expect messages in test result - assert len(test_instance.result.messages) == len(data["expected"]["messages"]) - # Test will pass if the expected message is included in the test result message - for message, expected in zip(test_instance.result.messages, data["expected"]["messages"]): # NOTE: zip(strict=True) has been added in Python 3.10 - assert expected in message - else: - # Test result should not have messages - assert test_instance.result.messages == [] diff --git a/tests/lib/fixture.py b/tests/lib/fixture.py deleted file mode 100644 index 92210acfa..000000000 --- a/tests/lib/fixture.py +++ /dev/null @@ -1,274 +0,0 @@ -# Copyright (c) 2023-2024 Arista Networks, Inc. -# Use of this source code is governed by the Apache License 2.0 -# that can be found in the LICENSE file. -"""Fixture for Anta Testing.""" - -from __future__ import annotations - -import json -import logging -import shutil -from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable -from unittest.mock import patch - -import pytest -from click.testing import CliRunner, Result - -import asynceapi -from anta.cli.console import console -from anta.device import AntaDevice, AsyncEOSDevice -from anta.inventory import AntaInventory -from anta.result_manager import ResultManager -from anta.result_manager.models import TestResult -from tests.lib.utils import default_anta_env - -if TYPE_CHECKING: - from collections.abc import Iterator - - from anta.models import AntaCommand - -logger = logging.getLogger(__name__) - -DATA_DIR: Path = Path(__file__).parent.parent.resolve() / "data" - -JSON_RESULTS = "test_md_report_results.json" - -DEVICE_HW_MODEL = "pytest" -DEVICE_NAME = "pytest" -COMMAND_OUTPUT = "retrieved" - -MOCK_CLI_JSON: dict[str, asynceapi.EapiCommandError | dict[str, Any]] = { - "show version": { - "modelName": "DCS-7280CR3-32P4-F", - "version": "4.31.1F", - }, - "enable": {}, - "clear counters": {}, - "clear hardware counter drop": {}, - "undefined": asynceapi.EapiCommandError( - passed=[], - failed="show version", - errors=["Authorization denied for command 'show version'"], - errmsg="Invalid command", - not_exec=[], - ), -} - -MOCK_CLI_TEXT: dict[str, asynceapi.EapiCommandError | str] = { - "show version": "Arista cEOSLab", - "bash timeout 10 ls -1t /mnt/flash/schedule/tech-support": "dummy_tech-support_2023-12-01.1115.log.gz\ndummy_tech-support_2023-12-01.1015.log.gz", - "bash timeout 10 ls -1t /mnt/flash/schedule/tech-support | head -1": "dummy_tech-support_2023-12-01.1115.log.gz", - "show running-config | include aaa authorization exec default": "aaa authorization exec default local", -} - - -@pytest.fixture -def device(request: pytest.FixtureRequest) -> Iterator[AntaDevice]: - """Return an AntaDevice instance with mocked abstract method.""" - - def _collect(command: AntaCommand, *args: Any, **kwargs: Any) -> None: # noqa: ARG001, ANN401 #pylint: disable=unused-argument - command.output = COMMAND_OUTPUT - - kwargs = {"name": DEVICE_NAME, "hw_model": DEVICE_HW_MODEL} - - if hasattr(request, "param"): - # Fixture is parametrized indirectly - kwargs.update(request.param) - with patch.object(AntaDevice, "__abstractmethods__", set()), patch("anta.device.AntaDevice._collect", side_effect=_collect): - # AntaDevice constructor does not have hw_model argument - hw_model = kwargs.pop("hw_model") - dev = AntaDevice(**kwargs) # type: ignore[abstract, arg-type] # pylint: disable=abstract-class-instantiated, unexpected-keyword-arg - dev.hw_model = hw_model - yield dev - - -@pytest.fixture -def test_inventory() -> AntaInventory: - """Return the test_inventory.""" - env = default_anta_env() - assert env["ANTA_INVENTORY"] - assert env["ANTA_USERNAME"] - assert env["ANTA_PASSWORD"] is not None - return AntaInventory.parse( - filename=env["ANTA_INVENTORY"], - username=env["ANTA_USERNAME"], - password=env["ANTA_PASSWORD"], - ) - - -# tests.unit.test_device.py fixture -@pytest.fixture -def async_device(request: pytest.FixtureRequest) -> AsyncEOSDevice: - """Return an AsyncEOSDevice instance.""" - kwargs = { - "name": DEVICE_NAME, - "host": "42.42.42.42", - "username": "anta", - "password": "anta", - } - - if hasattr(request, "param"): - # Fixture is parametrized indirectly - kwargs.update(request.param) - return AsyncEOSDevice(**kwargs) # type: ignore[arg-type] - - -# tests.units.result_manager fixtures -@pytest.fixture -def test_result_factory(device: AntaDevice) -> Callable[[int], TestResult]: - """Return a anta.result_manager.models.TestResult object.""" - # pylint: disable=redefined-outer-name - - def _create(index: int = 0) -> TestResult: - """Actual Factory.""" - return TestResult( - name=device.name, - test=f"VerifyTest{index}", - categories=["test"], - description=f"Verifies Test {index}", - custom_field=None, - ) - - return _create - - -@pytest.fixture -def list_result_factory(test_result_factory: Callable[[int], TestResult]) -> Callable[[int], list[TestResult]]: - """Return a list[TestResult] with 'size' TestResult instantiated using the test_result_factory fixture.""" - # pylint: disable=redefined-outer-name - - def _factory(size: int = 0) -> list[TestResult]: - """Create a factory for list[TestResult] entry of size entries.""" - return [test_result_factory(i) for i in range(size)] - - return _factory - - -@pytest.fixture -def result_manager_factory(list_result_factory: Callable[[int], list[TestResult]]) -> Callable[[int], ResultManager]: - """Return a ResultManager factory that takes as input a number of tests.""" - # pylint: disable=redefined-outer-name - - def _factory(number: int = 0) -> ResultManager: - """Create a factory for list[TestResult] entry of size entries.""" - result_manager = ResultManager() - result_manager.results = list_result_factory(number) - return result_manager - - return _factory - - -@pytest.fixture -def result_manager() -> ResultManager: - """Return a ResultManager with 30 random tests loaded from a JSON file. - - Devices: DC1-SPINE1, DC1-LEAF1A - - - Total tests: 30 - - Success: 7 - - Skipped: 2 - - Failure: 19 - - Error: 2 - - See `tests/data/test_md_report_results.json` and `tests/data/test_md_report_all_tests.md` for details. - """ - manager = ResultManager() - - with (DATA_DIR / JSON_RESULTS).open("r", encoding="utf-8") as f: - results = json.load(f) - - for result in results: - manager.add(TestResult(**result)) - - return manager - - -# tests.units.cli fixtures -@pytest.fixture -def temp_env(tmp_path: Path) -> dict[str, str | None]: - """Fixture that create a temporary ANTA inventory. - - The inventory can be overridden and returns the corresponding environment variables. - """ - env = default_anta_env() - anta_inventory = str(env["ANTA_INVENTORY"]) - temp_inventory = tmp_path / "test_inventory.yml" - shutil.copy(anta_inventory, temp_inventory) - env["ANTA_INVENTORY"] = str(temp_inventory) - return env - - -@pytest.fixture -# Disabling C901 - too complex as we like our runner like this -def click_runner(capsys: pytest.CaptureFixture[str]) -> Iterator[CliRunner]: # noqa: C901 - """Return a click.CliRunner for cli testing.""" - - class AntaCliRunner(CliRunner): - """Override CliRunner to inject specific variables for ANTA.""" - - def invoke( - self, - *args: Any, # noqa: ANN401 - **kwargs: Any, # noqa: ANN401 - ) -> Result: - # Inject default env if not provided - kwargs["env"] = kwargs["env"] if "env" in kwargs else default_anta_env() - # Deterministic terminal width - kwargs["env"]["COLUMNS"] = "165" - - kwargs["auto_envvar_prefix"] = "ANTA" - # Way to fix https://github.com/pallets/click/issues/824 - with capsys.disabled(): - result = super().invoke(*args, **kwargs) - # disabling T201 as we want to print here - print("--- CLI Output ---") # noqa: T201 - print(result.output) # noqa: T201 - return result - - def cli( - command: str | None = None, - commands: list[dict[str, Any]] | None = None, - ofmt: str = "json", - _version: int | str | None = "latest", - **_kwargs: Any, # noqa: ANN401 - ) -> dict[str, Any] | list[dict[str, Any]]: - def get_output(command: str | dict[str, Any]) -> dict[str, Any]: - if isinstance(command, dict): - command = command["cmd"] - mock_cli: dict[str, Any] - if ofmt == "json": - mock_cli = MOCK_CLI_JSON - elif ofmt == "text": - mock_cli = MOCK_CLI_TEXT - for mock_cmd, output in mock_cli.items(): - if command == mock_cmd: - logger.info("Mocking command %s", mock_cmd) - if isinstance(output, asynceapi.EapiCommandError): - raise output - return output - message = f"Command '{command}' is not mocked" - logger.critical(message) - raise NotImplementedError(message) - - res: dict[str, Any] | list[dict[str, Any]] - if command is not None: - logger.debug("Mock input %s", command) - res = get_output(command) - if commands is not None: - logger.debug("Mock input %s", commands) - res = list(map(get_output, commands)) - logger.debug("Mock output %s", res) - return res - - # Patch asynceapi methods used by AsyncEOSDevice. See tests/units/test_device.py - with ( - patch("asynceapi.device.Device.check_connection", return_value=True), - patch("asynceapi.device.Device.cli", side_effect=cli), - patch("asyncssh.connect"), - patch( - "asyncssh.scp", - ), - ): - console._color_system = None # pylint: disable=protected-access - yield AntaCliRunner() diff --git a/tests/lib/utils.py b/tests/lib/utils.py deleted file mode 100644 index ba669c287..000000000 --- a/tests/lib/utils.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2023-2024 Arista Networks, Inc. -# Use of this source code is governed by the Apache License 2.0 -# that can be found in the LICENSE file. -"""tests.lib.utils.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - - -def generate_test_ids_dict(val: dict[str, Any], key: str = "name") -> str: - """generate_test_ids Helper to generate test ID for parametrize.""" - return val.get(key, "unamed_test") - - -def generate_test_ids_list(val: list[dict[str, Any]], key: str = "name") -> list[str]: - """generate_test_ids Helper to generate test ID for parametrize.""" - return [entry.get(key, "unamed_test") for entry in val] - - -def generate_test_ids(data: list[dict[str, Any]]) -> list[str]: - """Build id for a unit test of an AntaTest subclass. - - { - "name": "meaniful test name", - "test": , - ... - } - """ - return [f"{val['test'].module}.{val['test'].__name__}-{val['name']}" for val in data] - - -def default_anta_env() -> dict[str, str | None]: - """Return a default_anta_environement which can be passed to a cliRunner.invoke method.""" - return { - "ANTA_USERNAME": "anta", - "ANTA_PASSWORD": "formica", - "ANTA_INVENTORY": str(Path(__file__).parent.parent / "data" / "test_inventory.yml"), - "ANTA_CATALOG": str(Path(__file__).parent.parent / "data" / "test_catalog.yml"), - } diff --git a/tests/mock_data/show_ntp_status_text_synchronised.out b/tests/mock_data/show_ntp_status_text_synchronised.out deleted file mode 100644 index 081a8a834..000000000 --- a/tests/mock_data/show_ntp_status_text_synchronised.out +++ /dev/null @@ -1 +0,0 @@ -[{'output': 'synchronised to NTP server (51.254.83.231) at stratum 3\n time correct to within 82 ms\n polling server every 1024 s\n\n'}] diff --git a/tests/mock_data/show_uptime_json_1000000.out b/tests/mock_data/show_uptime_json_1000000.out deleted file mode 100644 index 754025a53..000000000 --- a/tests/mock_data/show_uptime_json_1000000.out +++ /dev/null @@ -1 +0,0 @@ -[{'upTime': 1000000.68, 'loadAvg': [0.17, 0.21, 0.18], 'users': 1, 'currentTime': 1643761588.030645}] diff --git a/tests/mock_data/show_version_json_4.27.1.1F.out b/tests/mock_data/show_version_json_4.27.1.1F.out deleted file mode 100644 index fc720d41b..000000000 --- a/tests/mock_data/show_version_json_4.27.1.1F.out +++ /dev/null @@ -1 +0,0 @@ -[{'imageFormatVersion': '2.0', 'uptime': 2697.76, 'modelName': 'DCS-7280TRA-48C6-F', 'internalVersion': '4.27.1.1F-25536724.42711F', 'memTotal': 8098984, 'mfgName': 'Arista', 'serialNumber': 'SSJ16376415', 'systemMacAddress': '44:4c:a8:c7:1f:6b', 'bootupTimestamp': 1643715179.0, 'memFree': 6131068, 'version': '4.27.1.1F', 'configMacAddress': '00:00:00:00:00:00', 'isIntlVersion': False, 'internalBuildId': '38c43eab-c660-477a-915b-5a7b28da781d', 'hardwareRevision': '21.02', 'hwMacAddress': '44:4c:a8:c7:1f:6b', 'architecture': 'i686'}] diff --git a/tests/units/__init__.py b/tests/units/__init__.py index 6f96a0d1c..6b2d4ace6 100644 --- a/tests/units/__init__.py +++ b/tests/units/__init__.py @@ -1,4 +1,10 @@ # Copyright (c) 2023-2024 Arista Networks, Inc. # Use of this source code is governed by the Apache License 2.0 # that can be found in the LICENSE file. -"""Unit tests for anta.""" +"""Unit tests for ANTA.""" + +import pytest + +# Enable nice assert messages for tests.units.anta_tests unit tests +# https://docs.pytest.org/en/stable/how-to/writing_plugins.html#assertion-rewriting +pytest.register_assert_rewrite("tests.units.anta_tests") diff --git a/tests/units/anta_tests/__init__.py b/tests/units/anta_tests/__init__.py index 8ca0e8c7c..bfebc6d22 100644 --- a/tests/units/anta_tests/__init__.py +++ b/tests/units/anta_tests/__init__.py @@ -1,4 +1,33 @@ # Copyright (c) 2023-2024 Arista Networks, Inc. # Use of this source code is governed by the Apache License 2.0 # that can be found in the LICENSE file. -"""Test for anta.tests submodule.""" +"""Tests for anta.tests module.""" + +import asyncio +from typing import Any + +from anta.device import AntaDevice + + +def test(device: AntaDevice, data: dict[str, Any]) -> None: + """Generic test function for AntaTest subclass. + + Generate unit tests for each AntaTest subclass. + + See `tests/units/anta_tests/README.md` for more information on how to use it. + """ + # Instantiate the AntaTest subclass + test_instance = data["test"](device, inputs=data["inputs"], eos_data=data["eos_data"]) + # Run the test() method + asyncio.run(test_instance.test()) + # Assert expected result + assert test_instance.result.result == data["expected"]["result"], f"Expected '{data['expected']['result']}' result, got '{test_instance.result.result}'" + if "messages" in data["expected"]: + # We expect messages in test result + assert len(test_instance.result.messages) == len(data["expected"]["messages"]) + # Test will pass if the expected message is included in the test result message + for message, expected in zip(test_instance.result.messages, data["expected"]["messages"]): # NOTE: zip(strict=True) has been added in Python 3.10 + assert expected in message + else: + # Test result should not have messages + assert test_instance.result.messages == [] diff --git a/tests/units/anta_tests/conftest.py b/tests/units/anta_tests/conftest.py new file mode 100644 index 000000000..5da7606cc --- /dev/null +++ b/tests/units/anta_tests/conftest.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023-2024 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +"""See https://docs.pytest.org/en/stable/reference/fixtures.html#conftest-py-sharing-fixtures-across-multiple-files.""" + +from typing import Any + +import pytest + + +def build_test_id(val: dict[str, Any]) -> str: + """Build id for a unit test of an AntaTest subclass. + + { + "name": "meaniful test name", + "test": , + ... + } + """ + return f"{val['test'].__module__}.{val['test'].__name__}-{val['name']}" + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + """Generate ANTA testts unit tests dynamically during test collection. + + It will parametrize test cases based on the `DATA` data structure defined in `tests.units.anta_tests` modules. + See `tests/units/anta_tests/README.md` for more information on how to use it. + Test IDs are generated using the `build_test_id` function above. + + Checking that only the function "test" is parametrized with data to allow for writing tests for helper functions + in each module. + """ + if "tests.units.anta_tests" in metafunc.module.__package__ and metafunc.function.__name__ == "test": + # This is a unit test for an AntaTest subclass + metafunc.parametrize("data", metafunc.module.DATA, ids=build_test_id) diff --git a/tests/units/anta_tests/routing/test_bgp.py b/tests/units/anta_tests/routing/test_bgp.py index ae306cdff..e256b04dd 100644 --- a/tests/units/anta_tests/routing/test_bgp.py +++ b/tests/units/anta_tests/routing/test_bgp.py @@ -8,8 +8,6 @@ from typing import Any -# pylint: disable=C0413 -# because of the patch above from anta.tests.routing.bgp import ( VerifyBGPAdvCommunities, VerifyBGPExchangedRoutes, @@ -27,7 +25,7 @@ VerifyBGPTimers, VerifyEVPNType2Route, ) -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { @@ -4313,31 +4311,6 @@ ], }, }, - { - "name": "failure-not-found", - "test": VerifyBGPPeerUpdateErrors, - "eos_data": [ - { - "vrfs": {}, - }, - { - "vrfs": {}, - }, - ], - "inputs": { - "bgp_peers": [ - {"peer_address": "10.100.0.8", "vrf": "default", "update_errors": ["inUpdErrWithdraw", "inUpdErrIgnore", "disabledAfiSafi"]}, - {"peer_address": "10.100.0.9", "vrf": "MGMT", "update_errors": ["inUpdErrWithdraw", "inUpdErrIgnore", "disabledAfiSafi"]}, - ] - }, - "expected": { - "result": "failure", - "messages": [ - "The following BGP peers are not configured or have non-zero update error counters:\n" - "{'10.100.0.8': {'default': 'Not configured'}, '10.100.0.9': {'MGMT': 'Not configured'}}" - ], - }, - }, { "name": "success-all-error-counters", "test": VerifyBGPPeerUpdateErrors, diff --git a/tests/units/anta_tests/routing/test_generic.py b/tests/units/anta_tests/routing/test_generic.py index 621cf22ad..0ac43f3c5 100644 --- a/tests/units/anta_tests/routing/test_generic.py +++ b/tests/units/anta_tests/routing/test_generic.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.routing.generic import VerifyRoutingProtocolModel, VerifyRoutingTableEntry, VerifyRoutingTableSize -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/routing/test_isis.py b/tests/units/anta_tests/routing/test_isis.py index 2167ea434..84f5bdcf7 100644 --- a/tests/units/anta_tests/routing/test_isis.py +++ b/tests/units/anta_tests/routing/test_isis.py @@ -20,7 +20,7 @@ VerifyISISSegmentRoutingTunnels, _get_interface_data, ) -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/routing/test_ospf.py b/tests/units/anta_tests/routing/test_ospf.py index 81d8010ae..1555af6e6 100644 --- a/tests/units/anta_tests/routing/test_ospf.py +++ b/tests/units/anta_tests/routing/test_ospf.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.routing.ospf import VerifyOSPFMaxLSA, VerifyOSPFNeighborCount, VerifyOSPFNeighborState -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_aaa.py b/tests/units/anta_tests/test_aaa.py index 40bf82e09..119e20696 100644 --- a/tests/units/anta_tests/test_aaa.py +++ b/tests/units/anta_tests/test_aaa.py @@ -16,7 +16,7 @@ VerifyTacacsServers, VerifyTacacsSourceIntf, ) -from tests.lib.anta import test # noqa: F401; pylint: disable=unused-import +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_avt.py b/tests/units/anta_tests/test_avt.py index 7ef6be323..80fbce036 100644 --- a/tests/units/anta_tests/test_avt.py +++ b/tests/units/anta_tests/test_avt.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.avt import VerifyAVTPathHealth, VerifyAVTRole, VerifyAVTSpecificPath -from tests.lib.anta import test # noqa: F401; pylint: disable=unused-import +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_bfd.py b/tests/units/anta_tests/test_bfd.py index 3b1b8b86a..9bd64656c 100644 --- a/tests/units/anta_tests/test_bfd.py +++ b/tests/units/anta_tests/test_bfd.py @@ -8,10 +8,8 @@ from typing import Any -# pylint: disable=C0413 -# because of the patch above from anta.tests.bfd import VerifyBFDPeersHealth, VerifyBFDPeersIntervals, VerifyBFDPeersRegProtocols, VerifyBFDSpecificPeers -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_configuration.py b/tests/units/anta_tests/test_configuration.py index 7f198a33c..dbe22d365 100644 --- a/tests/units/anta_tests/test_configuration.py +++ b/tests/units/anta_tests/test_configuration.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.configuration import VerifyRunningConfigDiffs, VerifyRunningConfigLines, VerifyZeroTouch -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_connectivity.py b/tests/units/anta_tests/test_connectivity.py index 4cc57676c..beeaae65c 100644 --- a/tests/units/anta_tests/test_connectivity.py +++ b/tests/units/anta_tests/test_connectivity.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.connectivity import VerifyLLDPNeighbors, VerifyReachability -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_field_notices.py b/tests/units/anta_tests/test_field_notices.py index 3cb7286fd..a30604b8b 100644 --- a/tests/units/anta_tests/test_field_notices.py +++ b/tests/units/anta_tests/test_field_notices.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.field_notices import VerifyFieldNotice44Resolution, VerifyFieldNotice72Resolution -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_flow_tracking.py b/tests/units/anta_tests/test_flow_tracking.py index 21b47222a..f50a76b5d 100644 --- a/tests/units/anta_tests/test_flow_tracking.py +++ b/tests/units/anta_tests/test_flow_tracking.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.flow_tracking import VerifyHardwareFlowTrackerStatus -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_greent.py b/tests/units/anta_tests/test_greent.py index 2c483012d..16f36165e 100644 --- a/tests/units/anta_tests/test_greent.py +++ b/tests/units/anta_tests/test_greent.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.greent import VerifyGreenT, VerifyGreenTCounters -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_hardware.py b/tests/units/anta_tests/test_hardware.py index e601c681a..646ca5829 100644 --- a/tests/units/anta_tests/test_hardware.py +++ b/tests/units/anta_tests/test_hardware.py @@ -16,7 +16,7 @@ VerifyTransceiversManufacturers, VerifyTransceiversTemperature, ) -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_interfaces.py b/tests/units/anta_tests/test_interfaces.py index c38ac89f2..73ef6c6aa 100644 --- a/tests/units/anta_tests/test_interfaces.py +++ b/tests/units/anta_tests/test_interfaces.py @@ -27,7 +27,7 @@ VerifyStormControlDrops, VerifySVI, ) -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_lanz.py b/tests/units/anta_tests/test_lanz.py index bfbf6ae48..03694d4e4 100644 --- a/tests/units/anta_tests/test_lanz.py +++ b/tests/units/anta_tests/test_lanz.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.lanz import VerifyLANZ -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_logging.py b/tests/units/anta_tests/test_logging.py index d46c86581..cfc034c87 100644 --- a/tests/units/anta_tests/test_logging.py +++ b/tests/units/anta_tests/test_logging.py @@ -17,7 +17,7 @@ VerifyLoggingSourceIntf, VerifyLoggingTimestamp, ) -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_mlag.py b/tests/units/anta_tests/test_mlag.py index ae8ff7cf6..1ef547259 100644 --- a/tests/units/anta_tests/test_mlag.py +++ b/tests/units/anta_tests/test_mlag.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.mlag import VerifyMlagConfigSanity, VerifyMlagDualPrimary, VerifyMlagInterfaces, VerifyMlagPrimaryPriority, VerifyMlagReloadDelay, VerifyMlagStatus -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_multicast.py b/tests/units/anta_tests/test_multicast.py index a52a1d2ae..1fdcadd23 100644 --- a/tests/units/anta_tests/test_multicast.py +++ b/tests/units/anta_tests/test_multicast.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.multicast import VerifyIGMPSnoopingGlobal, VerifyIGMPSnoopingVlans -from tests.lib.anta import test # noqa: F401; pylint: disable=unused-import +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_path_selection.py b/tests/units/anta_tests/test_path_selection.py index c5fb07933..d1882d04b 100644 --- a/tests/units/anta_tests/test_path_selection.py +++ b/tests/units/anta_tests/test_path_selection.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.path_selection import VerifyPathsHealth, VerifySpecificPath -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_profiles.py b/tests/units/anta_tests/test_profiles.py index d58e987c2..f822d09d3 100644 --- a/tests/units/anta_tests/test_profiles.py +++ b/tests/units/anta_tests/test_profiles.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.profiles import VerifyTcamProfile, VerifyUnifiedForwardingTableMode -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_ptp.py b/tests/units/anta_tests/test_ptp.py index 8f4c77ff9..fc94480da 100644 --- a/tests/units/anta_tests/test_ptp.py +++ b/tests/units/anta_tests/test_ptp.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.ptp import VerifyPtpGMStatus, VerifyPtpLockStatus, VerifyPtpModeStatus, VerifyPtpOffset, VerifyPtpPortModeStatus -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { @@ -295,14 +295,14 @@ "expected": {"result": "success"}, }, { - "name": "failure", + "name": "failure-no-interfaces", "test": VerifyPtpPortModeStatus, "eos_data": [{"ptpIntfSummaries": {}}], "inputs": None, "expected": {"result": "failure", "messages": ["No interfaces are PTP enabled"]}, }, { - "name": "failure", + "name": "failure-invalid-state", "test": VerifyPtpPortModeStatus, "eos_data": [ { diff --git a/tests/units/anta_tests/test_security.py b/tests/units/anta_tests/test_security.py index eabc40bd8..792b06595 100644 --- a/tests/units/anta_tests/test_security.py +++ b/tests/units/anta_tests/test_security.py @@ -24,7 +24,7 @@ VerifySSHStatus, VerifyTelnetStatus, ) -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_services.py b/tests/units/anta_tests/test_services.py index 61c44d0d6..3f13dfc0b 100644 --- a/tests/units/anta_tests/test_services.py +++ b/tests/units/anta_tests/test_services.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.services import VerifyDNSLookup, VerifyDNSServers, VerifyErrdisableRecovery, VerifyHostname -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_snmp.py b/tests/units/anta_tests/test_snmp.py index 64c44382e..f6d964f83 100644 --- a/tests/units/anta_tests/test_snmp.py +++ b/tests/units/anta_tests/test_snmp.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.snmp import VerifySnmpContact, VerifySnmpIPv4Acl, VerifySnmpIPv6Acl, VerifySnmpLocation, VerifySnmpStatus -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_software.py b/tests/units/anta_tests/test_software.py index e46f52659..d2172bb6f 100644 --- a/tests/units/anta_tests/test_software.py +++ b/tests/units/anta_tests/test_software.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.software import VerifyEOSExtensions, VerifyEOSVersion, VerifyTerminAttrVersion -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_stp.py b/tests/units/anta_tests/test_stp.py index 64a116804..a6855aa88 100644 --- a/tests/units/anta_tests/test_stp.py +++ b/tests/units/anta_tests/test_stp.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.stp import VerifySTPBlockedPorts, VerifySTPCounters, VerifySTPForwardingPorts, VerifySTPMode, VerifySTPRootPriority -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_stun.py b/tests/units/anta_tests/test_stun.py index 0c5fdc143..005ae35f8 100644 --- a/tests/units/anta_tests/test_stun.py +++ b/tests/units/anta_tests/test_stun.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.stun import VerifyStunClient, VerifyStunServer -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_system.py b/tests/units/anta_tests/test_system.py index 54849b734..22b9787b2 100644 --- a/tests/units/anta_tests/test_system.py +++ b/tests/units/anta_tests/test_system.py @@ -18,7 +18,7 @@ VerifyReloadCause, VerifyUptime, ) -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_vlan.py b/tests/units/anta_tests/test_vlan.py index 53bf92f94..6bbfac496 100644 --- a/tests/units/anta_tests/test_vlan.py +++ b/tests/units/anta_tests/test_vlan.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.vlan import VerifyVlanInternalPolicy -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { diff --git a/tests/units/anta_tests/test_vxlan.py b/tests/units/anta_tests/test_vxlan.py index f450897a6..4278a5945 100644 --- a/tests/units/anta_tests/test_vxlan.py +++ b/tests/units/anta_tests/test_vxlan.py @@ -8,7 +8,7 @@ from typing import Any from anta.tests.vxlan import VerifyVxlan1ConnSettings, VerifyVxlan1Interface, VerifyVxlanConfigSanity, VerifyVxlanVniBinding, VerifyVxlanVtep -from tests.lib.anta import test # noqa: F401; pylint: disable=W0611 +from tests.units.anta_tests import test DATA: list[dict[str, Any]] = [ { @@ -26,21 +26,21 @@ "expected": {"result": "skipped", "messages": ["Vxlan1 interface is not configured"]}, }, { - "name": "failure", + "name": "failure-down-up", "test": VerifyVxlan1Interface, "eos_data": [{"interfaceDescriptions": {"Vxlan1": {"lineProtocolStatus": "down", "interfaceStatus": "up"}}}], "inputs": None, "expected": {"result": "failure", "messages": ["Vxlan1 interface is down/up"]}, }, { - "name": "failure", + "name": "failure-up-down", "test": VerifyVxlan1Interface, "eos_data": [{"interfaceDescriptions": {"Vxlan1": {"lineProtocolStatus": "up", "interfaceStatus": "down"}}}], "inputs": None, "expected": {"result": "failure", "messages": ["Vxlan1 interface is up/down"]}, }, { - "name": "failure", + "name": "failure-down-down", "test": VerifyVxlan1Interface, "eos_data": [{"interfaceDescriptions": {"Vxlan1": {"lineProtocolStatus": "down", "interfaceStatus": "down"}}}], "inputs": None, diff --git a/tests/units/asynceapi/test_device.py b/tests/units/asynceapi/test_device.py index 8a140ee3b..2c6375a85 100644 --- a/tests/units/asynceapi/test_device.py +++ b/tests/units/asynceapi/test_device.py @@ -18,7 +18,6 @@ from pytest_httpx import HTTPXMock -@pytest.mark.asyncio @pytest.mark.parametrize( "cmds", [ @@ -44,7 +43,6 @@ async def test_jsonrpc_exec_success( assert result == SUCCESS_EAPI_RESPONSE["result"] -@pytest.mark.asyncio @pytest.mark.parametrize( "cmds", [ @@ -76,7 +74,6 @@ async def test_jsonrpc_exec_eapi_command_error( assert exc_info.value.not_exec == [jsonrpc_request["params"]["cmds"][2]] -@pytest.mark.asyncio async def test_jsonrpc_exec_http_status_error(asynceapi_device: Device, httpx_mock: HTTPXMock) -> None: """Test the Device.jsonrpc_exec method with an HTTPStatusError.""" jsonrpc_request: dict[str, Any] = JSONRPC_REQUEST_TEMPLATE.copy() diff --git a/tests/units/cli/conftest.py b/tests/units/cli/conftest.py new file mode 100644 index 000000000..e63e60eb2 --- /dev/null +++ b/tests/units/cli/conftest.py @@ -0,0 +1,133 @@ +# Copyright (c) 2023-2024 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +"""See https://docs.pytest.org/en/stable/reference/fixtures.html#conftest-py-sharing-fixtures-across-multiple-files.""" + +from __future__ import annotations + +import logging +import shutil +from typing import TYPE_CHECKING, Any +from unittest.mock import patch + +import pytest +from click.testing import CliRunner, Result + +import asynceapi +from anta.cli.console import console + +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + +logger = logging.getLogger(__name__) + + +MOCK_CLI_JSON: dict[str, asynceapi.EapiCommandError | dict[str, Any]] = { + "show version": { + "modelName": "DCS-7280CR3-32P4-F", + "version": "4.31.1F", + }, + "enable": {}, + "clear counters": {}, + "clear hardware counter drop": {}, + "undefined": asynceapi.EapiCommandError( + passed=[], + failed="show version", + errors=["Authorization denied for command 'show version'"], + errmsg="Invalid command", + not_exec=[], + ), +} + +MOCK_CLI_TEXT: dict[str, asynceapi.EapiCommandError | str] = { + "show version": "Arista cEOSLab", + "bash timeout 10 ls -1t /mnt/flash/schedule/tech-support": "dummy_tech-support_2023-12-01.1115.log.gz\ndummy_tech-support_2023-12-01.1015.log.gz", + "bash timeout 10 ls -1t /mnt/flash/schedule/tech-support | head -1": "dummy_tech-support_2023-12-01.1115.log.gz", + "show running-config | include aaa authorization exec default": "aaa authorization exec default local", +} + + +@pytest.fixture +def temp_env(anta_env: dict[str, str], tmp_path: Path) -> dict[str, str]: + """Fixture that create a temporary ANTA inventory. + + The inventory can be overridden and returns the corresponding environment variables. + """ + anta_inventory = str(anta_env["ANTA_INVENTORY"]) + temp_inventory = tmp_path / "test_inventory.yml" + shutil.copy(anta_inventory, temp_inventory) + anta_env["ANTA_INVENTORY"] = str(temp_inventory) + return anta_env + + +@pytest.fixture +# Disabling C901 - too complex as we like our runner like this +def click_runner(capsys: pytest.CaptureFixture[str], anta_env: dict[str, str]) -> Iterator[CliRunner]: # noqa: C901 + """Return a click.CliRunner for cli testing.""" + + class AntaCliRunner(CliRunner): + """Override CliRunner to inject specific variables for ANTA.""" + + def invoke(self, *args: Any, **kwargs: Any) -> Result: # noqa: ANN401 + # Inject default env vars if not provided + kwargs["env"] = anta_env | kwargs.get("env", {}) + # Deterministic terminal width + kwargs["env"]["COLUMNS"] = "165" + + kwargs["auto_envvar_prefix"] = "ANTA" + # Way to fix https://github.com/pallets/click/issues/824 + with capsys.disabled(): + result = super().invoke(*args, **kwargs) + # disabling T201 as we want to print here + print("--- CLI Output ---") # noqa: T201 + print(result.output) # noqa: T201 + return result + + def cli( + command: str | None = None, + commands: list[dict[str, Any]] | None = None, + ofmt: str = "json", + _version: int | str | None = "latest", + **_kwargs: Any, # noqa: ANN401 + ) -> dict[str, Any] | list[dict[str, Any]]: + def get_output(command: str | dict[str, Any]) -> dict[str, Any]: + if isinstance(command, dict): + command = command["cmd"] + mock_cli: dict[str, Any] + if ofmt == "json": + mock_cli = MOCK_CLI_JSON + elif ofmt == "text": + mock_cli = MOCK_CLI_TEXT + for mock_cmd, output in mock_cli.items(): + if command == mock_cmd: + logger.info("Mocking command %s", mock_cmd) + if isinstance(output, asynceapi.EapiCommandError): + raise output + return output + message = f"Command '{command}' is not mocked" + logger.critical(message) + raise NotImplementedError(message) + + res: dict[str, Any] | list[dict[str, Any]] + if command is not None: + logger.debug("Mock input %s", command) + res = get_output(command) + if commands is not None: + logger.debug("Mock input %s", commands) + res = list(map(get_output, commands)) + logger.debug("Mock output %s", res) + return res + + # Patch asynceapi methods used by AsyncEOSDevice. See tests/units/test_device.py + with ( + patch("asynceapi.device.Device.check_connection", return_value=True), + patch("asynceapi.device.Device.cli", side_effect=cli), + patch("asyncssh.connect"), + patch( + "asyncssh.scp", + ), + ): + console._color_system = None + yield AntaCliRunner() diff --git a/tests/units/cli/debug/test_commands.py b/tests/units/cli/debug/test_commands.py index 039e09eb0..c802b0de8 100644 --- a/tests/units/cli/debug/test_commands.py +++ b/tests/units/cli/debug/test_commands.py @@ -19,12 +19,12 @@ @pytest.mark.parametrize( ("command", "ofmt", "version", "revision", "device", "failed"), [ - pytest.param("show version", "json", None, None, "dummy", False, id="json command"), - pytest.param("show version", "text", None, None, "dummy", False, id="text command"), - pytest.param("show version", None, "latest", None, "dummy", False, id="version-latest"), - pytest.param("show version", None, "1", None, "dummy", False, id="version"), - pytest.param("show version", None, None, 3, "dummy", False, id="revision"), - pytest.param("undefined", None, None, None, "dummy", True, id="command fails"), + pytest.param("show version", "json", None, None, "leaf1", False, id="json command"), + pytest.param("show version", "text", None, None, "leaf1", False, id="text command"), + pytest.param("show version", None, "latest", None, "leaf1", False, id="version-latest"), + pytest.param("show version", None, "1", None, "leaf1", False, id="version"), + pytest.param("show version", None, None, 3, "leaf1", False, id="revision"), + pytest.param("undefined", None, None, None, "leaf1", True, id="command fails"), pytest.param("undefined", None, None, None, "doesnotexist", True, id="Device does not exist"), ], ) @@ -38,7 +38,6 @@ def test_run_cmd( failed: bool, ) -> None: """Test `anta debug run-cmd`.""" - # pylint: disable=too-many-arguments cli_args = ["-l", "debug", "debug", "run-cmd", "--command", command, "--device", device] # ofmt diff --git a/tests/units/cli/exec/test_utils.py b/tests/units/cli/exec/test_utils.py index f4c0cc5fd..503327ab7 100644 --- a/tests/units/cli/exec/test_utils.py +++ b/tests/units/cli/exec/test_utils.py @@ -5,17 +5,19 @@ from __future__ import annotations +import logging +from pathlib import Path from typing import TYPE_CHECKING, Any from unittest.mock import call, patch import pytest +import respx -from anta.cli.exec.utils import ( - clear_counters, -) +from anta.cli.exec.utils import clear_counters, collect_commands from anta.models import AntaCommand +from anta.tools import safe_command -# , collect_commands, collect_scheduled_show_tech +# collect_scheduled_show_tech if TYPE_CHECKING: from anta.device import AntaDevice @@ -23,55 +25,59 @@ # TODO: complete test cases -@pytest.mark.asyncio @pytest.mark.parametrize( - ("inventory_state", "per_device_command_output", "tags"), + ("inventory", "inventory_state", "per_device_command_output", "tags"), [ pytest.param( + {"count": 3}, { - "dummy": {"is_online": False}, - "dummy2": {"is_online": False}, - "dummy3": {"is_online": False}, + "device-0": {"is_online": False}, + "device-1": {"is_online": False}, + "device-2": {"is_online": False}, }, {}, None, id="no_connected_device", ), pytest.param( + {"count": 3}, { - "dummy": {"is_online": True, "hw_model": "cEOSLab"}, - "dummy2": {"is_online": True, "hw_model": "vEOS-lab"}, - "dummy3": {"is_online": False}, + "device-0": {"is_online": True, "hw_model": "cEOSLab"}, + "device-1": {"is_online": True, "hw_model": "vEOS-lab"}, + "device-2": {"is_online": False}, }, {}, None, id="cEOSLab and vEOS-lab devices", ), pytest.param( + {"count": 3}, { - "dummy": {"is_online": True}, - "dummy2": {"is_online": True}, - "dummy3": {"is_online": False}, + "device-0": {"is_online": True}, + "device-1": {"is_online": True}, + "device-2": {"is_online": False}, }, - {"dummy": None}, # None means the command failed to collect + {"device-0": None}, # None means the command failed to collect None, id="device with error", ), pytest.param( + {"count": 3}, { - "dummy": {"is_online": True}, - "dummy2": {"is_online": True}, - "dummy3": {"is_online": True}, + "device-0": {"is_online": True}, + "device-1": {"is_online": True}, + "device-2": {"is_online": True}, }, {}, ["spine"], id="tags", ), ], + indirect=["inventory"], ) async def test_clear_counters( caplog: pytest.LogCaptureFixture, - test_inventory: AntaInventory, + inventory: AntaInventory, inventory_state: dict[str, Any], per_device_command_output: dict[str, Any], tags: set[str] | None, @@ -80,12 +86,12 @@ async def test_clear_counters( async def mock_connect_inventory() -> None: """Mock connect_inventory coroutine.""" - for name, device in test_inventory.items(): + for name, device in inventory.items(): device.is_online = inventory_state[name].get("is_online", True) device.established = inventory_state[name].get("established", device.is_online) device.hw_model = inventory_state[name].get("hw_model", "dummy") - async def collect(self: AntaDevice, command: AntaCommand, *args: Any, **kwargs: Any) -> None: # noqa: ARG001, ANN401 #pylint: disable=unused-argument + async def collect(self: AntaDevice, command: AntaCommand, *args: Any, **kwargs: Any) -> None: # noqa: ARG001, ANN401 """Mock collect coroutine.""" command.output = per_device_command_output.get(self.name, "") @@ -97,10 +103,10 @@ async def collect(self: AntaDevice, command: AntaCommand, *args: Any, **kwargs: side_effect=mock_connect_inventory, ) as mocked_connect_inventory, ): - await clear_counters(test_inventory, tags=tags) + await clear_counters(inventory, tags=tags) mocked_connect_inventory.assert_awaited_once() - devices_established = test_inventory.get_inventory(established_only=True, tags=tags).devices + devices_established = inventory.get_inventory(established_only=True, tags=tags).devices if devices_established: # Building the list of calls calls = [] @@ -142,3 +148,172 @@ async def collect(self: AntaDevice, command: AntaCommand, *args: Any, **kwargs: assert f"Could not clear counters on device {key}: []" in caplog.text else: mocked_collect.assert_not_awaited() + + +# TODO: test with changing root_dir, test with failing to write (OSError) +@pytest.mark.parametrize( + ("inventory", "inventory_state", "commands", "tags"), + [ + pytest.param( + {"count": 1}, + { + "device-0": {"is_online": False}, + }, + {"json_format": ["show version"]}, + None, + id="no_connected_device", + ), + pytest.param( + {"count": 3}, + { + "device-0": {"is_online": True}, + "device-1": {"is_online": True}, + "device-2": {"is_online": False}, + }, + {"json_format": ["show version", "show ip interface brief"]}, + None, + id="JSON commands", + ), + pytest.param( + {"count": 3}, + { + "device-0": {"is_online": True}, + "device-1": {"is_online": True}, + "device-2": {"is_online": False}, + }, + {"json_format": ["show version"], "text_format": ["show running-config", "show ip interface"]}, + None, + id="Text commands", + ), + pytest.param( + {"count": 2}, + { + "device-0": {"is_online": True, "tags": {"spine"}}, + "device-1": {"is_online": True}, + }, + {"json_format": ["show version"]}, + {"spine"}, + id="tags", + ), + pytest.param( # TODO: This test should not be there we should catch the wrong user input with pydantic. + {"count": 1}, + { + "device-0": {"is_online": True}, + }, + {"blah_format": ["42"]}, + None, + id="bad-input", + ), + pytest.param( + {"count": 1}, + { + "device-0": {"is_online": True}, + }, + {"json_format": ["undefined command", "show version"]}, + None, + id="command-failed-to-be-collected", + ), + pytest.param( + {"count": 1}, + { + "device-0": {"is_online": True}, + }, + {"json_format": ["uncaught exception"]}, + None, + id="uncaught-exception", + ), + ], + indirect=["inventory"], +) +async def test_collect_commands( + caplog: pytest.LogCaptureFixture, + tmp_path: Path, + inventory: AntaInventory, + inventory_state: dict[str, Any], + commands: dict[str, list[str]], + tags: set[str] | None, +) -> None: + """Test anta.cli.exec.utils.collect_commands.""" + caplog.set_level(logging.INFO) + root_dir = tmp_path + + async def mock_connect_inventory() -> None: + """Mock connect_inventory coroutine.""" + for name, device in inventory.items(): + device.is_online = inventory_state[name].get("is_online", True) + device.established = inventory_state[name].get("established", device.is_online) + device.hw_model = inventory_state[name].get("hw_model", "dummy") + device.tags = inventory_state[name].get("tags", set()) + + # Need to patch the child device class + # ruff: noqa: C901 + with ( + respx.mock, + patch( + "anta.inventory.AntaInventory.connect_inventory", + side_effect=mock_connect_inventory, + ) as mocked_connect_inventory, + ): + # Mocking responses from devices + respx.post(path="/command-api", headers={"Content-Type": "application/json-rpc"}, json__params__cmds__0__cmd="show version").respond( + json={"result": [{"toto": 42}]} + ) + respx.post(path="/command-api", headers={"Content-Type": "application/json-rpc"}, json__params__cmds__0__cmd="show ip interface brief").respond( + json={"result": [{"toto": 42}]} + ) + respx.post(path="/command-api", headers={"Content-Type": "application/json-rpc"}, json__params__cmds__0__cmd="show running-config").respond( + json={"result": [{"output": "blah"}]} + ) + respx.post(path="/command-api", headers={"Content-Type": "application/json-rpc"}, json__params__cmds__0__cmd="show ip interface").respond( + json={"result": [{"output": "blah"}]} + ) + respx.post(path="/command-api", headers={"Content-Type": "application/json-rpc"}, json__params__cmds__0__cmd="undefined command").respond( + json={ + "error": { + "code": 1002, + "message": "CLI command 1 of 1 'undefined command' failed: invalid command", + "data": [{"errors": ["Invalid input (at token 0: 'undefined')"]}], + } + } + ) + await collect_commands(inventory, commands, root_dir, tags=tags) + + mocked_connect_inventory.assert_awaited_once() + devices_established = inventory.get_inventory(established_only=True, tags=tags or None).devices + if not devices_established: + assert "INFO" in caplog.text + assert "No online device found. Exiting" in caplog.text + return + + for device in devices_established: + # Verify tags selection + assert device.tags.intersection(tags) != {} if tags else True + json_path = root_dir / device.name / "json" + text_path = root_dir / device.name / "text" + if "json_format" in commands: + # Handle undefined command + if "undefined command" in commands["json_format"]: + assert "ERROR" in caplog.text + assert "Command 'undefined command' failed on device-0: Invalid input (at token 0: 'undefined')" in caplog.text + # Verify we don't claim it was collected + assert f"Collected command 'undefined command' from device {device.name}" not in caplog.text + commands["json_format"].remove("undefined command") + # Handle uncaught exception + elif "uncaught exception" in commands["json_format"]: + assert "ERROR" in caplog.text + assert "Error when collecting commands: " in caplog.text + # Verify we don't claim it was collected + assert f"Collected command 'uncaught exception' from device {device.name}" not in caplog.text + commands["json_format"].remove("uncaught exception") + + assert json_path.is_dir() + assert len(list(Path.iterdir(json_path))) == len(commands["json_format"]) + for command in commands["json_format"]: + assert Path.is_file(json_path / f"{safe_command(command)}.json") + assert f"Collected command '{command}' from device {device.name}" in caplog.text + if "text_format" in commands: + assert text_path.is_dir() + assert len(list(text_path.iterdir())) == len(commands["text_format"]) + for command in commands["text_format"]: + assert Path.is_file(text_path / f"{safe_command(command)}.log") + assert f"Collected command '{command}' from device {device.name}" in caplog.text diff --git a/tests/units/cli/get/test_commands.py b/tests/units/cli/get/test_commands.py index 1e8c6e95d..ff3d922b2 100644 --- a/tests/units/cli/get/test_commands.py +++ b/tests/units/cli/get/test_commands.py @@ -42,7 +42,6 @@ def test_from_cvp( cv_token_failure: bool, cvp_connect_failure: bool, ) -> None: - # pylint: disable=too-many-arguments # ruff: noqa: C901 """Test `anta get from-cvp`. @@ -144,7 +143,6 @@ def test_from_ansible( expected_exit: int, expected_log: str | None, ) -> None: - # pylint: disable=too-many-arguments """Test `anta get from-ansible`. This test verifies: @@ -230,7 +228,6 @@ def test_from_ansible_overwrite( expected_exit: int, expected_log: str | None, ) -> None: - # pylint: disable=too-many-arguments """Test `anta get from-ansible` overwrite mechanism. The test uses a static ansible-inventory and output as these are tested in other functions diff --git a/tests/units/cli/get/test_utils.py b/tests/units/cli/get/test_utils.py index e105f9476..46ce14f93 100644 --- a/tests/units/cli/get/test_utils.py +++ b/tests/units/cli/get/test_utils.py @@ -144,7 +144,6 @@ def test_create_inventory_from_ansible( expected_inv_length: int, ) -> None: """Test anta.get.utils.create_inventory_from_ansible.""" - # pylint: disable=R0913 target_file = tmp_path / "inventory.yml" inventory_file_path = DATA_DIR / inventory_filename diff --git a/tests/units/cli/nrfu/test__init__.py b/tests/units/cli/nrfu/test__init__.py index 7227a699f..d08499c6a 100644 --- a/tests/units/cli/nrfu/test__init__.py +++ b/tests/units/cli/nrfu/test__init__.py @@ -9,7 +9,6 @@ from anta.cli import anta from anta.cli.utils import ExitCode -from tests.lib.utils import default_anta_env if TYPE_CHECKING: from click.testing import CliRunner @@ -58,8 +57,7 @@ def test_anta_nrfu_wrong_catalog_format(click_runner: CliRunner) -> None: def test_anta_password_required(click_runner: CliRunner) -> None: """Test that password is provided.""" - env = default_anta_env() - env["ANTA_PASSWORD"] = None + env = {"ANTA_PASSWORD": None} result = click_runner.invoke(anta, ["nrfu"], env=env) assert result.exit_code == ExitCode.USAGE_ERROR @@ -68,8 +66,7 @@ def test_anta_password_required(click_runner: CliRunner) -> None: def test_anta_password(click_runner: CliRunner) -> None: """Test that password can be provided either via --password or --prompt.""" - env = default_anta_env() - env["ANTA_PASSWORD"] = None + env = {"ANTA_PASSWORD": None} result = click_runner.invoke(anta, ["nrfu", "--password", "secret"], env=env) assert result.exit_code == ExitCode.OK result = click_runner.invoke(anta, ["nrfu", "--prompt"], input="password\npassword\n", env=env) diff --git a/tests/units/cli/nrfu/test_commands.py b/tests/units/cli/nrfu/test_commands.py index 27f01a78c..6a2624c1e 100644 --- a/tests/units/cli/nrfu/test_commands.py +++ b/tests/units/cli/nrfu/test_commands.py @@ -52,7 +52,7 @@ def test_anta_nrfu_table(click_runner: CliRunner) -> None: """Test anta nrfu, catalog is given via env.""" result = click_runner.invoke(anta, ["nrfu", "table"]) assert result.exit_code == ExitCode.OK - assert "dummy │ VerifyEOSVersion │ success" in result.output + assert "leaf1 │ VerifyEOSVersion │ success" in result.output def test_anta_nrfu_table_group_by_device(click_runner: CliRunner) -> None: @@ -73,7 +73,7 @@ def test_anta_nrfu_text(click_runner: CliRunner) -> None: """Test anta nrfu, catalog is given via env.""" result = click_runner.invoke(anta, ["nrfu", "text"]) assert result.exit_code == ExitCode.OK - assert "dummy :: VerifyEOSVersion :: SUCCESS" in result.output + assert "leaf1 :: VerifyEOSVersion :: SUCCESS" in result.output def test_anta_nrfu_json(click_runner: CliRunner) -> None: @@ -85,7 +85,7 @@ def test_anta_nrfu_json(click_runner: CliRunner) -> None: assert match is not None result_list = json.loads(match.group()) for res in result_list: - if res["name"] == "dummy": + if res["name"] == "leaf1": assert res["test"] == "VerifyEOSVersion" assert res["result"] == "success" @@ -131,7 +131,7 @@ def test_anta_nrfu_template(click_runner: CliRunner) -> None: """Test anta nrfu, catalog is given via env.""" result = click_runner.invoke(anta, ["nrfu", "tpl-report", "--template", str(DATA_DIR / "template.j2")]) assert result.exit_code == ExitCode.OK - assert "* VerifyEOSVersion is SUCCESS for dummy" in result.output + assert "* VerifyEOSVersion is SUCCESS for leaf1" in result.output def test_anta_nrfu_csv(click_runner: CliRunner, tmp_path: Path) -> None: diff --git a/tests/units/conftest.py b/tests/units/conftest.py new file mode 100644 index 000000000..665075c6f --- /dev/null +++ b/tests/units/conftest.py @@ -0,0 +1,85 @@ +# Copyright (c) 2023-2024 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +"""See https://docs.pytest.org/en/stable/reference/fixtures.html#conftest-py-sharing-fixtures-across-multiple-files.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, Any +from unittest.mock import patch + +import pytest +import yaml + +from anta.device import AntaDevice, AsyncEOSDevice + +if TYPE_CHECKING: + from collections.abc import Iterator + + from anta.models import AntaCommand + +DEVICE_HW_MODEL = "pytest" +DEVICE_NAME = "pytest" +COMMAND_OUTPUT = "retrieved" + + +@pytest.fixture(name="anta_env") +def anta_env_fixture() -> dict[str, str]: + """Return an ANTA environment for testing.""" + return { + "ANTA_USERNAME": "anta", + "ANTA_PASSWORD": "formica", + "ANTA_INVENTORY": str(Path(__file__).parent.parent / "data" / "test_inventory_with_tags.yml"), + "ANTA_CATALOG": str(Path(__file__).parent.parent / "data" / "test_catalog.yml"), + } + + +@pytest.fixture +def device(request: pytest.FixtureRequest) -> Iterator[AntaDevice]: + """Return an AntaDevice instance with mocked abstract method.""" + + def _collect(command: AntaCommand, *args: Any, **kwargs: Any) -> None: # noqa: ARG001, ANN401 + command.output = COMMAND_OUTPUT + + kwargs = {"name": DEVICE_NAME, "hw_model": DEVICE_HW_MODEL} + + if hasattr(request, "param"): + # Fixture is parametrized indirectly + kwargs.update(request.param) + with patch.object(AntaDevice, "__abstractmethods__", set()), patch("anta.device.AntaDevice._collect", side_effect=_collect): + # AntaDevice constructor does not have hw_model argument + hw_model = kwargs.pop("hw_model") + dev = AntaDevice(**kwargs) # type: ignore[abstract, arg-type] + dev.hw_model = hw_model + yield dev + + +@pytest.fixture +def async_device(request: pytest.FixtureRequest) -> AsyncEOSDevice: + """Return an AsyncEOSDevice instance.""" + kwargs = { + "name": DEVICE_NAME, + "host": "42.42.42.42", + "username": "anta", + "password": "anta", + } + + if hasattr(request, "param"): + # Fixture is parametrized indirectly + kwargs.update(request.param) + return AsyncEOSDevice(**kwargs) # type: ignore[arg-type] + + +@pytest.fixture +def yaml_file(request: pytest.FixtureRequest, tmp_path: Path) -> Path: + """Fixture to create a temporary YAML file and return the path. + + Fixture is indirectly parametrized with the YAML file content. + """ + assert hasattr(request, "param") + file = tmp_path / "test_file.yaml" + assert isinstance(request.param, dict) + content: dict[str, Any] = request.param + file.write_text(yaml.dump(content, allow_unicode=True)) + return file diff --git a/tests/units/inventory/test__init__.py b/tests/units/inventory/test__init__.py new file mode 100644 index 000000000..20a794a76 --- /dev/null +++ b/tests/units/inventory/test__init__.py @@ -0,0 +1,78 @@ +# Copyright (c) 2023-2024 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +"""ANTA Inventory unit tests.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +from pydantic import ValidationError + +from anta.inventory import AntaInventory +from anta.inventory.exceptions import InventoryIncorrectSchemaError, InventoryRootKeyError + +if TYPE_CHECKING: + from _pytest.mark.structures import ParameterSet + +FILE_DIR: Path = Path(__file__).parent.parent.resolve() / "data" / "inventory" + + +INIT_VALID_PARAMS: list[ParameterSet] = [ + pytest.param( + {"anta_inventory": {"hosts": [{"host": "192.168.0.17"}, {"host": "192.168.0.2"}, {"host": "my.awesome.host.com"}]}}, + id="Inventory_with_host_only", + ), + pytest.param({"anta_inventory": {"networks": [{"network": "192.168.0.0/24"}]}}, id="ValidInventory_with_networks_only"), + pytest.param( + {"anta_inventory": {"ranges": [{"start": "10.0.0.1", "end": "10.0.0.11"}, {"start": "10.0.0.101", "end": "10.0.0.111"}]}}, + id="Inventory_with_ranges_only", + ), + pytest.param( + {"anta_inventory": {"hosts": [{"host": "192.168.0.17", "port": 443}, {"host": "192.168.0.2", "port": 80}]}}, + id="Inventory_with_host_port", + ), + pytest.param( + {"anta_inventory": {"hosts": [{"host": "192.168.0.17", "tags": ["leaf"]}, {"host": "192.168.0.2", "tags": ["spine"]}]}}, + id="Inventory_with_host_tags", + ), + pytest.param({"anta_inventory": {"networks": [{"network": "192.168.0.0/24", "tags": ["leaf"]}]}}, id="ValidInventory_with_networks_tags"), + pytest.param( + { + "anta_inventory": { + "ranges": [{"start": "10.0.0.1", "end": "10.0.0.11", "tags": ["leaf"]}, {"start": "10.0.0.101", "end": "10.0.0.111", "tags": ["spine"]}] + } + }, + id="Inventory_with_ranges_tags", + ), +] + + +INIT_INVALID_PARAMS = [ + pytest.param({"anta_inventory": {"hosts": [{"host": "192.168.0.17/32"}, {"host": "192.168.0.2"}]}}, id="Inventory_with_host_only"), + pytest.param({"anta_inventory": {"networks": [{"network": "192.168.42.0/8"}]}}, id="Inventory_wrong_network_bits"), + pytest.param({"anta_inventory": {"networks": [{"network": "toto"}]}}, id="Inventory_wrong_network"), + pytest.param({"anta_inventory": {"ranges": [{"start": "toto", "end": "192.168.42.42"}]}}, id="Inventory_wrong_range"), + pytest.param({"anta_inventory": {"ranges": [{"start": "fe80::cafe", "end": "192.168.42.42"}]}}, id="Inventory_wrong_range_type_mismatch"), + pytest.param( + {"inventory": {"ranges": [{"start": "10.0.0.1", "end": "10.0.0.11"}, {"start": "10.0.0.100", "end": "10.0.0.111"}]}}, + id="Invalid_Root_Key", + ), +] + + +class TestAntaInventory: + """Tests for anta.inventory.AntaInventory.""" + + @pytest.mark.parametrize("yaml_file", INIT_VALID_PARAMS, indirect=["yaml_file"]) + def test_parse_valid(self, yaml_file: Path) -> None: + """Parse valid YAML file to create ANTA inventory.""" + AntaInventory.parse(filename=yaml_file, username="arista", password="arista123") + + @pytest.mark.parametrize("yaml_file", INIT_INVALID_PARAMS, indirect=["yaml_file"]) + def test_parse_invalid(self, yaml_file: Path) -> None: + """Parse invalid YAML file to create ANTA inventory.""" + with pytest.raises((InventoryIncorrectSchemaError, InventoryRootKeyError, ValidationError)): + AntaInventory.parse(filename=yaml_file, username="arista", password="arista123") diff --git a/tests/units/inventory/test_inventory.py b/tests/units/inventory/test_inventory.py deleted file mode 100644 index 430ca21cd..000000000 --- a/tests/units/inventory/test_inventory.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2023-2024 Arista Networks, Inc. -# Use of this source code is governed by the Apache License 2.0 -# that can be found in the LICENSE file. -"""ANTA Inventory unit tests.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -import pytest -import yaml -from pydantic import ValidationError - -from anta.inventory import AntaInventory -from anta.inventory.exceptions import InventoryIncorrectSchemaError, InventoryRootKeyError -from tests.data.json_data import ANTA_INVENTORY_TESTS_INVALID, ANTA_INVENTORY_TESTS_VALID -from tests.lib.utils import generate_test_ids_dict - -if TYPE_CHECKING: - from pathlib import Path - - -class TestAntaInventory: - """Test AntaInventory class.""" - - def create_inventory(self, content: str, tmp_path: Path) -> str: - """Create fakefs inventory file.""" - tmp_inventory = tmp_path / "mydir/myfile" - tmp_inventory.parent.mkdir() - tmp_inventory.touch() - tmp_inventory.write_text(yaml.dump(content, allow_unicode=True)) - return str(tmp_inventory) - - def check_parameter(self, parameter: str, test_definition: dict[Any, Any]) -> bool: - """Check if parameter is configured in testbed.""" - return "parameters" in test_definition and parameter in test_definition["parameters"] - - @pytest.mark.parametrize("test_definition", ANTA_INVENTORY_TESTS_VALID, ids=generate_test_ids_dict) - def test_init_valid(self, test_definition: dict[str, Any], tmp_path: Path) -> None: - """Test class constructor with valid data. - - Test structure: - --------------- - - { - 'name': 'ValidInventory_with_host_only', - 'input': {"anta_inventory":{"hosts":[{"host":"192.168.0.17"},{"host":"192.168.0.2"}]}}, - 'expected_result': 'valid', - 'parameters': { - 'ipaddress_in_scope': '192.168.0.17', - 'ipaddress_out_of_scope': '192.168.1.1', - } - } - - """ - inventory_file = self.create_inventory(content=test_definition["input"], tmp_path=tmp_path) - try: - AntaInventory.parse(filename=inventory_file, username="arista", password="arista123") - except ValidationError as exc: - raise AssertionError from exc - - @pytest.mark.parametrize("test_definition", ANTA_INVENTORY_TESTS_INVALID, ids=generate_test_ids_dict) - def test_init_invalid(self, test_definition: dict[str, Any], tmp_path: Path) -> None: - """Test class constructor with invalid data. - - Test structure: - --------------- - - { - 'name': 'ValidInventory_with_host_only', - 'input': {"anta_inventory":{"hosts":[{"host":"192.168.0.17"},{"host":"192.168.0.2"}]}}, - 'expected_result': 'invalid', - 'parameters': { - 'ipaddress_in_scope': '192.168.0.17', - 'ipaddress_out_of_scope': '192.168.1.1', - } - } - - """ - inventory_file = self.create_inventory(content=test_definition["input"], tmp_path=tmp_path) - with pytest.raises((InventoryIncorrectSchemaError, InventoryRootKeyError, ValidationError)): - AntaInventory.parse(filename=inventory_file, username="arista", password="arista123") diff --git a/tests/units/inventory/test_models.py b/tests/units/inventory/test_models.py index 0dccfb830..dfe9722d5 100644 --- a/tests/units/inventory/test_models.py +++ b/tests/units/inventory/test_models.py @@ -5,387 +5,162 @@ from __future__ import annotations -import logging -from typing import Any +from typing import TYPE_CHECKING, Any import pytest from pydantic import ValidationError -from anta.device import AsyncEOSDevice -from anta.inventory.models import AntaInventoryHost, AntaInventoryInput, AntaInventoryNetwork, AntaInventoryRange -from tests.data.json_data import ( - INVENTORY_DEVICE_MODEL_INVALID, - INVENTORY_DEVICE_MODEL_VALID, - INVENTORY_MODEL_HOST_CACHE, - INVENTORY_MODEL_HOST_INVALID, - INVENTORY_MODEL_HOST_VALID, - INVENTORY_MODEL_INVALID, - INVENTORY_MODEL_NETWORK_CACHE, - INVENTORY_MODEL_NETWORK_INVALID, - INVENTORY_MODEL_NETWORK_VALID, - INVENTORY_MODEL_RANGE_CACHE, - INVENTORY_MODEL_RANGE_INVALID, - INVENTORY_MODEL_RANGE_VALID, - INVENTORY_MODEL_VALID, -) -from tests.lib.utils import generate_test_ids_dict - - -class TestInventoryUnitModels: - """Test components of AntaInventoryInput model.""" - - @pytest.mark.parametrize("test_definition", INVENTORY_MODEL_HOST_VALID, ids=generate_test_ids_dict) - def test_anta_inventory_host_valid(self, test_definition: dict[str, Any]) -> None: - """Test host input model. - - Test structure: - --------------- - - { - 'name': 'ValidIPv4_Host', - 'input': '1.1.1.1', - 'expected_result': 'valid' - } - - """ - try: - host_inventory = AntaInventoryHost(host=test_definition["input"]) - except ValidationError as exc: - logging.warning("Error: %s", str(exc)) - raise AssertionError from exc - assert test_definition["input"] == str(host_inventory.host) - - @pytest.mark.parametrize("test_definition", INVENTORY_MODEL_HOST_INVALID, ids=generate_test_ids_dict) - def test_anta_inventory_host_invalid(self, test_definition: dict[str, Any]) -> None: - """Test host input model. - - Test structure: - --------------- - - { - 'name': 'ValidIPv4_Host', - 'input': '1.1.1.1/32', - 'expected_result': 'invalid' - } - - """ - with pytest.raises(ValidationError): - AntaInventoryHost(host=test_definition["input"]) - - @pytest.mark.parametrize("test_definition", INVENTORY_MODEL_HOST_CACHE, ids=generate_test_ids_dict) - def test_anta_inventory_host_cache(self, test_definition: dict[str, Any]) -> None: - """Test host disable_cache. - - Test structure: - --------------- - - { - 'name': 'Cache', - 'input': {"host": '1.1.1.1', "disable_cache": True}, - 'expected_result': True - } - - """ - if "disable_cache" in test_definition["input"]: - host_inventory = AntaInventoryHost(host=test_definition["input"]["host"], disable_cache=test_definition["input"]["disable_cache"]) - else: - host_inventory = AntaInventoryHost(host=test_definition["input"]["host"]) - assert test_definition["expected_result"] == host_inventory.disable_cache - - @pytest.mark.parametrize("test_definition", INVENTORY_MODEL_NETWORK_VALID, ids=generate_test_ids_dict) - def test_anta_inventory_network_valid(self, test_definition: dict[str, Any]) -> None: - """Test Network input model with valid data. - - Test structure: - --------------- - - { - 'name': 'ValidIPv4_Subnet', - 'input': '1.1.1.0/24', - 'expected_result': 'valid' - } - - """ - try: - network_inventory = AntaInventoryNetwork(network=test_definition["input"]) - except ValidationError as exc: - logging.warning("Error: %s", str(exc)) - raise AssertionError from exc - assert test_definition["input"] == str(network_inventory.network) - - @pytest.mark.parametrize("test_definition", INVENTORY_MODEL_NETWORK_INVALID, ids=generate_test_ids_dict) - def test_anta_inventory_network_invalid(self, test_definition: dict[str, Any]) -> None: - """Test Network input model with invalid data. - - Test structure: - --------------- - - { - 'name': 'ValidIPv4_Subnet', - 'input': '1.1.1.0/16', - 'expected_result': 'invalid' - } - - """ - try: - AntaInventoryNetwork(network=test_definition["input"]) - except ValidationError as exc: - logging.warning("Error: %s", str(exc)) - else: - raise AssertionError - - @pytest.mark.parametrize("test_definition", INVENTORY_MODEL_NETWORK_CACHE, ids=generate_test_ids_dict) - def test_anta_inventory_network_cache(self, test_definition: dict[str, Any]) -> None: - """Test network disable_cache. - - Test structure: - --------------- - - { - 'name': 'Cache', - 'input': {"network": '1.1.1.1/24', "disable_cache": True}, - 'expected_result': True - } - - """ - if "disable_cache" in test_definition["input"]: - network_inventory = AntaInventoryNetwork(network=test_definition["input"]["network"], disable_cache=test_definition["input"]["disable_cache"]) - else: - network_inventory = AntaInventoryNetwork(network=test_definition["input"]["network"]) - assert test_definition["expected_result"] == network_inventory.disable_cache - - @pytest.mark.parametrize("test_definition", INVENTORY_MODEL_RANGE_VALID, ids=generate_test_ids_dict) - def test_anta_inventory_range_valid(self, test_definition: dict[str, Any]) -> None: - """Test range input model. - - Test structure: - --------------- - - { - 'name': 'ValidIPv4_Range', - 'input': {'start':'10.1.0.1', 'end':'10.1.0.10'}, - 'expected_result': 'valid' - } - - """ - try: - range_inventory = AntaInventoryRange( - start=test_definition["input"]["start"], - end=test_definition["input"]["end"], - ) - except ValidationError as exc: - logging.warning("Error: %s", str(exc)) - raise AssertionError from exc - assert test_definition["input"]["start"] == str(range_inventory.start) - assert test_definition["input"]["end"] == str(range_inventory.end) - - @pytest.mark.parametrize("test_definition", INVENTORY_MODEL_RANGE_INVALID, ids=generate_test_ids_dict) - def test_anta_inventory_range_invalid(self, test_definition: dict[str, Any]) -> None: - """Test range input model. - - Test structure: - --------------- - - { - 'name': 'ValidIPv4_Range', - 'input': {'start':'10.1.0.1', 'end':'10.1.0.10/32'}, - 'expected_result': 'invalid' - } - - """ - try: - AntaInventoryRange( - start=test_definition["input"]["start"], - end=test_definition["input"]["end"], - ) - except ValidationError as exc: - logging.warning("Error: %s", str(exc)) +from anta.inventory.models import AntaInventoryHost, AntaInventoryNetwork, AntaInventoryRange + +if TYPE_CHECKING: + from _pytest.mark.structures import ParameterSet + +INVENTORY_HOST_VALID_PARAMS: list[ParameterSet] = [ + pytest.param(None, "1.1.1.1", None, None, None, id="IPv4"), + pytest.param(None, "fe80::cc62:a9ff:feef:932a", None, None, None, id="IPv6"), + pytest.param(None, "1.1.1.1", 666, None, None, id="IPv4_with_port"), + pytest.param(None, "1.1.1.1", None, None, True, id="cache_enabled"), + pytest.param(None, "1.1.1.1", None, None, False, id="cache_disabled"), +] + +INVENTORY_HOST_INVALID_PARAMS: list[ParameterSet] = [ + pytest.param(None, "1.1.1.1/32", None, None, False, id="IPv4_with_netmask"), + pytest.param(None, "1.1.1.1", 66666, None, False, id="IPv4_with_wrong_port"), + pytest.param(None, "fe80::cc62:a9ff:feef:932a/128", None, None, False, id="IPv6_with_netmask"), + pytest.param(None, "fe80::cc62:a9ff:feef:", None, None, False, id="invalid_IPv6"), + pytest.param(None, "@", None, None, False, id="special_char"), + pytest.param(None, "1.1.1.1", None, None, None, id="cache_is_None"), +] + +INVENTORY_NETWORK_VALID_PARAMS: list[ParameterSet] = [ + pytest.param("1.1.1.0/24", None, None, id="IPv4_subnet"), + pytest.param("2001:db8::/32", None, None, id="IPv6_subnet"), + pytest.param("1.1.1.0/24", None, False, id="cache_enabled"), + pytest.param("1.1.1.0/24", None, True, id="cache_disabled"), +] + +INVENTORY_NETWORK_INVALID_PARAMS: list[ParameterSet] = [ + pytest.param("1.1.1.0/17", None, False, id="IPv4_subnet"), + pytest.param("2001:db8::/16", None, False, id="IPv6_subnet"), + pytest.param("1.1.1.0/24", None, None, id="cache_is_None"), +] + +INVENTORY_RANGE_VALID_PARAMS: list[ParameterSet] = [ + pytest.param("10.1.0.1", "10.1.0.10", None, None, id="IPv4_range"), + pytest.param("10.1.0.1", "10.1.0.10", None, True, id="cache_enabled"), + pytest.param("10.1.0.1", "10.1.0.10", None, False, id="cache_disabled"), +] + +INVENTORY_RANGE_INVALID_PARAMS: list[ParameterSet] = [ + pytest.param("toto", "10.1.0.10", None, False, id="IPv4_range"), + pytest.param("10.1.0.1", "10.1.0.10", None, None, id="cache_is_None"), +] + +INVENTORY_MODEL_VALID = [ + { + "name": "Valid_Host_Only", + "input": {"hosts": [{"host": "192.168.0.17"}, {"host": "192.168.0.2"}]}, + "expected_result": "valid", + }, + { + "name": "Valid_Networks_Only", + "input": {"networks": [{"network": "192.168.0.0/16"}, {"network": "192.168.1.0/24"}]}, + "expected_result": "valid", + }, + { + "name": "Valid_Ranges_Only", + "input": { + "ranges": [ + {"start": "10.1.0.1", "end": "10.1.0.10"}, + {"start": "10.2.0.1", "end": "10.2.1.10"}, + ], + }, + "expected_result": "valid", + }, +] + +INVENTORY_MODEL_INVALID = [ + { + "name": "Host_with_Invalid_entry", + "input": {"hosts": [{"host": "192.168.0.17"}, {"host": "192.168.0.2/32"}]}, + "expected_result": "invalid", + }, +] + + +class TestAntaInventoryHost: + """Test anta.inventory.models.AntaInventoryHost.""" + + @pytest.mark.parametrize(("name", "host", "port", "tags", "disable_cache"), INVENTORY_HOST_VALID_PARAMS) + def test_valid(self, name: str, host: str, port: int, tags: set[str], disable_cache: bool | None) -> None: + """Valid model parameters.""" + params: dict[str, Any] = {"name": name, "host": host, "port": port, "tags": tags} + if disable_cache is not None: + params = params | {"disable_cache": disable_cache} + inventory_host = AntaInventoryHost.model_validate(params) + assert host == str(inventory_host.host) + assert port == inventory_host.port + assert name == inventory_host.name + assert tags == inventory_host.tags + if disable_cache is None: + # Check cache default value + assert inventory_host.disable_cache is False else: - raise AssertionError + assert inventory_host.disable_cache == disable_cache - @pytest.mark.parametrize("test_definition", INVENTORY_MODEL_RANGE_CACHE, ids=generate_test_ids_dict) - def test_anta_inventory_range_cache(self, test_definition: dict[str, Any]) -> None: - """Test range disable_cache. - - Test structure: - --------------- - - { - 'name': 'Cache', - 'input': {"start": '1.1.1.1', "end": "1.1.1.10", "disable_cache": True}, - 'expected_result': True - } - - """ - if "disable_cache" in test_definition["input"]: - range_inventory = AntaInventoryRange( - start=test_definition["input"]["start"], - end=test_definition["input"]["end"], - disable_cache=test_definition["input"]["disable_cache"], - ) + @pytest.mark.parametrize(("name", "host", "port", "tags", "disable_cache"), INVENTORY_HOST_INVALID_PARAMS) + def test_invalid(self, name: str, host: str, port: int, tags: set[str], disable_cache: bool | None) -> None: + """Invalid model parameters.""" + with pytest.raises(ValidationError): + AntaInventoryHost.model_validate({"name": name, "host": host, "port": port, "tags": tags, "disable_cache": disable_cache}) + + +class TestAntaInventoryNetwork: + """Test anta.inventory.models.AntaInventoryNetwork.""" + + @pytest.mark.parametrize(("network", "tags", "disable_cache"), INVENTORY_NETWORK_VALID_PARAMS) + def test_valid(self, network: str, tags: set[str], disable_cache: bool | None) -> None: + """Valid model parameters.""" + params: dict[str, Any] = {"network": network, "tags": tags} + if disable_cache is not None: + params = params | {"disable_cache": disable_cache} + inventory_network = AntaInventoryNetwork.model_validate(params) + assert network == str(inventory_network.network) + assert tags == inventory_network.tags + if disable_cache is None: + # Check cache default value + assert inventory_network.disable_cache is False else: - range_inventory = AntaInventoryRange(start=test_definition["input"]["start"], end=test_definition["input"]["end"]) - assert test_definition["expected_result"] == range_inventory.disable_cache - - -class TestAntaInventoryInputModel: - """Unit test of AntaInventoryInput model.""" - - def test_inventory_input_structure(self) -> None: - """Test inventory keys are those expected.""" - inventory = AntaInventoryInput() - logging.info("Inventory keys are: %s", str(inventory.model_dump().keys())) - assert all(elem in inventory.model_dump() for elem in ["hosts", "networks", "ranges"]) - - @pytest.mark.parametrize("inventory_def", INVENTORY_MODEL_VALID, ids=generate_test_ids_dict) - def test_anta_inventory_intput_valid(self, inventory_def: dict[str, Any]) -> None: - """Test loading valid data to inventory class. - - Test structure: - --------------- - - { - "name": "Valid_Host_Only", - "input": { - "hosts": [ - { - "host": "192.168.0.17" - }, - { - "host": "192.168.0.2" - } - ] - }, - "expected_result": "valid" - } - - """ - try: - inventory = AntaInventoryInput(**inventory_def["input"]) - except ValidationError as exc: - logging.warning("Error: %s", str(exc)) - raise AssertionError from exc - logging.info("Checking if all root keys are correctly lodaded") - assert all(elem in inventory.model_dump() for elem in inventory_def["input"]) - - @pytest.mark.parametrize("inventory_def", INVENTORY_MODEL_INVALID, ids=generate_test_ids_dict) - def test_anta_inventory_intput_invalid(self, inventory_def: dict[str, Any]) -> None: - """Test loading invalid data to inventory class. + assert inventory_network.disable_cache == disable_cache - Test structure: - --------------- - - { - "name": "Valid_Host_Only", - "input": { - "hosts": [ - { - "host": "192.168.0.17" - }, - { - "host": "192.168.0.2/32" - } - ] - }, - "expected_result": "invalid" - } - - """ - try: - if "hosts" in inventory_def["input"]: - logging.info( - "Loading %s into AntaInventoryInput hosts section", - str(inventory_def["input"]["hosts"]), - ) - AntaInventoryInput(hosts=inventory_def["input"]["hosts"]) - if "networks" in inventory_def["input"]: - logging.info( - "Loading %s into AntaInventoryInput networks section", - str(inventory_def["input"]["networks"]), - ) - AntaInventoryInput(networks=inventory_def["input"]["networks"]) - if "ranges" in inventory_def["input"]: - logging.info( - "Loading %s into AntaInventoryInput ranges section", - str(inventory_def["input"]["ranges"]), - ) - AntaInventoryInput(ranges=inventory_def["input"]["ranges"]) - except ValidationError as exc: - logging.warning("Error: %s", str(exc)) + @pytest.mark.parametrize(("network", "tags", "disable_cache"), INVENTORY_NETWORK_INVALID_PARAMS) + def test_invalid(self, network: str, tags: set[str], disable_cache: bool | None) -> None: + """Invalid model parameters.""" + with pytest.raises(ValidationError): + AntaInventoryNetwork.model_validate({"network": network, "tags": tags, "disable_cache": disable_cache}) + + +class TestAntaInventoryRange: + """Test anta.inventory.models.AntaInventoryRange.""" + + @pytest.mark.parametrize(("start", "end", "tags", "disable_cache"), INVENTORY_RANGE_VALID_PARAMS) + def test_valid(self, start: str, end: str, tags: set[str], disable_cache: bool | None) -> None: + """Valid model parameters.""" + params: dict[str, Any] = {"start": start, "end": end, "tags": tags} + if disable_cache is not None: + params = params | {"disable_cache": disable_cache} + inventory_range = AntaInventoryRange.model_validate(params) + assert start == str(inventory_range.start) + assert end == str(inventory_range.end) + assert tags == inventory_range.tags + if disable_cache is None: + # Check cache default value + assert inventory_range.disable_cache is False else: - raise AssertionError - - -class TestInventoryDeviceModel: - """Unit test of InventoryDevice model.""" - - @pytest.mark.parametrize("test_definition", INVENTORY_DEVICE_MODEL_VALID, ids=generate_test_ids_dict) - def test_inventory_device_valid(self, test_definition: dict[str, Any]) -> None: - """Test loading valid data to InventoryDevice class. - - Test structure: - --------------- - - { - "name": "Valid_Inventory", - "input": [ - { - 'host': '1.1.1.1', - 'username': 'arista', - 'password': 'arista123!' - }, - { - 'host': '1.1.1.1', - 'username': 'arista', - 'password': 'arista123!' - } - ], - "expected_result": "valid" - } + assert inventory_range.disable_cache == disable_cache - """ - if test_definition["expected_result"] == "invalid": - pytest.skip("Not concerned by the test") - - try: - for entity in test_definition["input"]: - AsyncEOSDevice(**entity) - except TypeError as exc: - logging.warning("Error: %s", str(exc)) - raise AssertionError from exc - - @pytest.mark.parametrize("test_definition", INVENTORY_DEVICE_MODEL_INVALID, ids=generate_test_ids_dict) - def test_inventory_device_invalid(self, test_definition: dict[str, Any]) -> None: - """Test loading invalid data to InventoryDevice class. - - Test structure: - --------------- - - { - "name": "Valid_Inventory", - "input": [ - { - 'host': '1.1.1.1', - 'username': 'arista', - 'password': 'arista123!' - }, - { - 'host': '1.1.1.1', - 'username': 'arista', - 'password': 'arista123!' - } - ], - "expected_result": "valid" - } - - """ - if test_definition["expected_result"] == "valid": - pytest.skip("Not concerned by the test") - - try: - for entity in test_definition["input"]: - AsyncEOSDevice(**entity) - except TypeError as exc: - logging.info("Error: %s", str(exc)) - else: - raise AssertionError + @pytest.mark.parametrize(("start", "end", "tags", "disable_cache"), INVENTORY_RANGE_INVALID_PARAMS) + def test_invalid(self, start: str, end: str, tags: set[str], disable_cache: bool | None) -> None: + """Invalid model parameters.""" + with pytest.raises(ValidationError): + AntaInventoryRange.model_validate({"start": start, "end": end, "tags": tags, "disable_cache": disable_cache}) diff --git a/tests/units/reporter/conftest.py b/tests/units/reporter/conftest.py new file mode 100644 index 000000000..ae7d3dfea --- /dev/null +++ b/tests/units/reporter/conftest.py @@ -0,0 +1,8 @@ +# Copyright (c) 2023-2024 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +"""See https://docs.pytest.org/en/stable/reference/fixtures.html#conftest-py-sharing-fixtures-across-multiple-files.""" + +from tests.units.result_manager.conftest import list_result_factory, result_manager, result_manager_factory, test_result_factory + +__all__ = ["result_manager", "result_manager_factory", "list_result_factory", "test_result_factory"] diff --git a/tests/units/reporter/test__init__.py b/tests/units/reporter/test__init__.py index f0e44b41a..af26b54cb 100644 --- a/tests/units/reporter/test__init__.py +++ b/tests/units/reporter/test__init__.py @@ -47,7 +47,6 @@ class TestReportTable: ) def test__split_list_to_txt_list(self, usr_list: list[str], delimiter: str | None, expected_output: str) -> None: """Test _split_list_to_txt_list.""" - # pylint: disable=protected-access report = ReportTable() assert report._split_list_to_txt_list(usr_list, delimiter) == expected_output @@ -61,7 +60,6 @@ def test__split_list_to_txt_list(self, usr_list: list[str], delimiter: str | Non ) def test__build_headers(self, headers: list[str]) -> None: """Test _build_headers.""" - # pylint: disable=protected-access report = ReportTable() table = Table() table_column_before = len(table.columns) @@ -82,7 +80,6 @@ def test__build_headers(self, headers: list[str]) -> None: ) def test__color_result(self, status: AntaTestStatus, expected_status: str) -> None: """Test _build_headers.""" - # pylint: disable=protected-access report = ReportTable() assert report._color_result(status) == expected_status @@ -103,7 +100,6 @@ def test_report_all( expected_length: int, ) -> None: """Test report_all.""" - # pylint: disable=too-many-arguments manager = result_manager_factory(number_of_tests) report = ReportTable() @@ -132,7 +128,6 @@ def test_report_summary_tests( expected_length: int, ) -> None: """Test report_summary_tests.""" - # pylint: disable=too-many-arguments # TODO: refactor this later... this is injecting double test results by modyfing the device name # should be a fixture manager = result_manager_factory(number_of_tests) @@ -167,7 +162,6 @@ def test_report_summary_devices( expected_length: int, ) -> None: """Test report_summary_devices.""" - # pylint: disable=too-many-arguments # TODO: refactor this later... this is injecting double test results by modyfing the device name # should be a fixture manager = result_manager_factory(number_of_tests) diff --git a/tests/units/result_manager/conftest.py b/tests/units/result_manager/conftest.py new file mode 100644 index 000000000..2c5dc8a69 --- /dev/null +++ b/tests/units/result_manager/conftest.py @@ -0,0 +1,85 @@ +# Copyright (c) 2023-2024 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +"""See https://docs.pytest.org/en/stable/reference/fixtures.html#conftest-py-sharing-fixtures-across-multiple-files.""" + +import json +from pathlib import Path +from typing import Callable + +import pytest + +from anta.device import AntaDevice +from anta.result_manager import ResultManager +from anta.result_manager.models import TestResult + +TEST_RESULTS: Path = Path(__file__).parent.resolve() / "test_files" / "test_md_report_results.json" + + +@pytest.fixture +def result_manager_factory(list_result_factory: Callable[[int], list[TestResult]]) -> Callable[[int], ResultManager]: + """Return a ResultManager factory that takes as input a number of tests.""" + # pylint: disable=redefined-outer-name + + def _factory(number: int = 0) -> ResultManager: + """Create a factory for list[TestResult] entry of size entries.""" + result_manager = ResultManager() + result_manager.results = list_result_factory(number) + return result_manager + + return _factory + + +@pytest.fixture +def result_manager() -> ResultManager: + """Return a ResultManager with 30 random tests loaded from a JSON file. + + Devices: DC1-SPINE1, DC1-LEAF1A + + - Total tests: 30 + - Success: 7 + - Skipped: 2 + - Failure: 19 + - Error: 2 + + See `tests/units/result_manager/test_md_report_results.json` for details. + """ + manager = ResultManager() + + with TEST_RESULTS.open("r", encoding="utf-8") as f: + results = json.load(f) + + for result in results: + manager.add(TestResult(**result)) + + return manager + + +@pytest.fixture +def test_result_factory(device: AntaDevice) -> Callable[[int], TestResult]: + """Return a anta.result_manager.models.TestResult object.""" + # pylint: disable=redefined-outer-name + + def _create(index: int = 0) -> TestResult: + """Actual Factory.""" + return TestResult( + name=device.name, + test=f"VerifyTest{index}", + categories=["test"], + description=f"Verifies Test {index}", + custom_field=None, + ) + + return _create + + +@pytest.fixture +def list_result_factory(test_result_factory: Callable[[int], TestResult]) -> Callable[[int], list[TestResult]]: + """Return a list[TestResult] with 'size' TestResult instantiated using the test_result_factory fixture.""" + # pylint: disable=redefined-outer-name + + def _factory(size: int = 0) -> list[TestResult]: + """Create a factory for list[TestResult] entry of size entries.""" + return [test_result_factory(i) for i in range(size)] + + return _factory diff --git a/tests/units/result_manager/test__init__.py b/tests/units/result_manager/test__init__.py index 802d4a4e3..e1087536f 100644 --- a/tests/units/result_manager/test__init__.py +++ b/tests/units/result_manager/test__init__.py @@ -152,7 +152,6 @@ def test_add( expected_status: str, expected_raise: AbstractContextManager[Exception], ) -> None: - # pylint: disable=too-many-arguments """Test ResultManager_update_status.""" result_manager = ResultManager() result_manager.status = AntaTestStatus(starting_status) diff --git a/tests/data/test_md_report_results.json b/tests/units/result_manager/test_files/test_md_report_results.json similarity index 100% rename from tests/data/test_md_report_results.json rename to tests/units/result_manager/test_files/test_md_report_results.json diff --git a/tests/units/result_manager/test_models.py b/tests/units/result_manager/test_models.py index bc44ccfd8..0561dffd2 100644 --- a/tests/units/result_manager/test_models.py +++ b/tests/units/result_manager/test_models.py @@ -5,58 +5,65 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Callable +from typing import TYPE_CHECKING, Callable import pytest from anta.result_manager.models import AntaTestStatus - -# Import as Result to avoid pytest collection -from tests.data.json_data import TEST_RESULT_SET_STATUS -from tests.lib.fixture import DEVICE_NAME -from tests.lib.utils import generate_test_ids_dict +from tests.units.conftest import DEVICE_NAME if TYPE_CHECKING: + from _pytest.mark.structures import ParameterSet + + # Import as Result to avoid pytest collection from anta.result_manager.models import TestResult as Result +TEST_RESULT_SET_STATUS: list[ParameterSet] = [ + pytest.param(AntaTestStatus.SUCCESS, "test success message", id="set_success"), + pytest.param(AntaTestStatus.ERROR, "test error message", id="set_error"), + pytest.param(AntaTestStatus.FAILURE, "test failure message", id="set_failure"), + pytest.param(AntaTestStatus.SKIPPED, "test skipped message", id="set_skipped"), + pytest.param(AntaTestStatus.UNSET, "test unset message", id="set_unset"), +] + class TestTestResultModels: """Test components of anta.result_manager.models.""" - @pytest.mark.parametrize("data", TEST_RESULT_SET_STATUS, ids=generate_test_ids_dict) - def test__is_status_foo(self, test_result_factory: Callable[[int], Result], data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("target", "message"), TEST_RESULT_SET_STATUS) + def test__is_status_foo(self, test_result_factory: Callable[[int], Result], target: AntaTestStatus, message: str) -> None: """Test TestResult.is_foo methods.""" testresult = test_result_factory(1) - assert testresult.result == "unset" + assert testresult.result == AntaTestStatus.UNSET assert len(testresult.messages) == 0 - if data["target"] == "success": - testresult.is_success(data["message"]) - assert testresult.result == data["target"] - assert data["message"] in testresult.messages - if data["target"] == "failure": - testresult.is_failure(data["message"]) - assert testresult.result == data["target"] - assert data["message"] in testresult.messages - if data["target"] == "error": - testresult.is_error(data["message"]) - assert testresult.result == data["target"] - assert data["message"] in testresult.messages - if data["target"] == "skipped": - testresult.is_skipped(data["message"]) - assert testresult.result == data["target"] - assert data["message"] in testresult.messages - # no helper for unset, testing _set_status - if data["target"] == "unset": - testresult._set_status(AntaTestStatus.UNSET, data["message"]) # pylint: disable=W0212 - assert testresult.result == data["target"] - assert data["message"] in testresult.messages - - @pytest.mark.parametrize("data", TEST_RESULT_SET_STATUS, ids=generate_test_ids_dict) - def test____str__(self, test_result_factory: Callable[[int], Result], data: dict[str, Any]) -> None: + if target == AntaTestStatus.SUCCESS: + testresult.is_success(message) + assert testresult.result == "success" + assert message in testresult.messages + if target == AntaTestStatus.FAILURE: + testresult.is_failure(message) + assert testresult.result == "failure" + assert message in testresult.messages + if target == AntaTestStatus.ERROR: + testresult.is_error(message) + assert testresult.result == "error" + assert message in testresult.messages + if target == AntaTestStatus.SKIPPED: + testresult.is_skipped(message) + assert testresult.result == "skipped" + assert message in testresult.messages + if target == AntaTestStatus.UNSET: + # no helper for unset, testing _set_status + testresult._set_status(AntaTestStatus.UNSET, message) + assert testresult.result == "unset" + assert message in testresult.messages + + @pytest.mark.parametrize(("target", "message"), TEST_RESULT_SET_STATUS) + def test____str__(self, test_result_factory: Callable[[int], Result], target: AntaTestStatus, message: str) -> None: """Test TestResult.__str__.""" testresult = test_result_factory(1) - assert testresult.result == "unset" + assert testresult.result == AntaTestStatus.UNSET assert len(testresult.messages) == 0 - testresult._set_status(data["target"], data["message"]) # pylint: disable=W0212 - assert testresult.result == data["target"] - assert str(testresult) == f"Test 'VerifyTest1' (on '{DEVICE_NAME}'): Result '{data['target']}'\nMessages: {[data['message']]}" + testresult._set_status(target, message) + assert testresult.result == target + assert str(testresult) == f"Test 'VerifyTest1' (on '{DEVICE_NAME}'): Result '{target}'\nMessages: {[message]}" diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index 13046f294..c2bb57c93 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -7,7 +7,7 @@ from json import load as json_load from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any, Literal import pytest from pydantic import ValidationError @@ -28,38 +28,25 @@ VerifyReloadCause, VerifyUptime, ) -from tests.lib.utils import generate_test_ids_list from tests.units.test_models import FakeTestWithInput -# Test classes used as expected values +if TYPE_CHECKING: + from _pytest.mark.structures import ParameterSet DATA_DIR: Path = Path(__file__).parent.parent.resolve() / "data" -INIT_CATALOG_DATA: list[dict[str, Any]] = [ - { - "name": "test_catalog", - "filename": "test_catalog.yml", - "tests": [ - (VerifyEOSVersion, VerifyEOSVersion.Input(versions=["4.31.1F"])), - ], - }, - { - "name": "test_catalog", - "filename": "test_catalog.json", - "file_format": "json", - "tests": [ - (VerifyEOSVersion, VerifyEOSVersion.Input(versions=["4.31.1F"])), - ], - }, - { - "name": "test_catalog_with_tags", - "filename": "test_catalog_with_tags.yml", - "tests": [ +INIT_CATALOG_PARAMS: list[ParameterSet] = [ + pytest.param("test_catalog.yml", "yaml", [(VerifyEOSVersion, VerifyEOSVersion.Input(versions=["4.31.1F"]))], id="test_catalog_yaml"), + pytest.param("test_catalog.json", "json", [(VerifyEOSVersion, VerifyEOSVersion.Input(versions=["4.31.1F"]))], id="test_catalog_json"), + pytest.param( + "test_catalog_with_tags.yml", + "yaml", + [ ( VerifyUptime, VerifyUptime.Input( minimum=10, - filters=VerifyUptime.Input.Filters(tags={"fabric"}), + filters=VerifyUptime.Input.Filters(tags={"spine"}), ), ), ( @@ -69,189 +56,143 @@ filters=VerifyUptime.Input.Filters(tags={"leaf"}), ), ), - (VerifyReloadCause, {"filters": {"tags": ["leaf", "spine"]}}), + (VerifyReloadCause, {"filters": {"tags": ["spine", "leaf"]}}), (VerifyCoredump, VerifyCoredump.Input()), (VerifyAgentLogs, AntaTest.Input()), - (VerifyCPUUtilization, VerifyCPUUtilization.Input(filters=VerifyCPUUtilization.Input.Filters(tags={"leaf"}))), - (VerifyMemoryUtilization, VerifyMemoryUtilization.Input(filters=VerifyMemoryUtilization.Input.Filters(tags={"testdevice"}))), + (VerifyCPUUtilization, None), + (VerifyMemoryUtilization, None), (VerifyFileSystemUtilization, None), (VerifyNTP, {}), - (VerifyMlagStatus, None), - (VerifyL3MTU, {"mtu": 1500, "filters": {"tags": ["demo"]}}), + (VerifyMlagStatus, {"filters": {"tags": ["leaf"]}}), + (VerifyL3MTU, {"mtu": 1500, "filters": {"tags": ["spine"]}}), ], - }, - { - "name": "test_empty_catalog", - "filename": "test_empty_catalog.yml", - "tests": [], - }, - { - "name": "test_empty_dict_catalog", - "filename": "test_empty_dict_catalog.yml", - "tests": [], - }, + id="test_catalog_with_tags", + ), + pytest.param("test_empty_catalog.yml", "yaml", [], id="test_empty_catalog"), + pytest.param("test_empty_dict_catalog.yml", "yaml", [], id="test_empty_dict_catalog"), ] -CATALOG_PARSE_FAIL_DATA: list[dict[str, Any]] = [ - { - "name": "undefined_tests", - "filename": "test_catalog_wrong_format.toto", - "file_format": "toto", - "error": "'toto' is not a valid format for an AntaCatalog file. Only 'yaml' and 'json' are supported.", - }, - { - "name": "invalid_json", - "filename": "test_catalog_invalid_json.json", - "file_format": "json", - "error": "JSONDecodeError", - }, - { - "name": "undefined_tests", - "filename": "test_catalog_with_undefined_tests.yml", - "error": "FakeTest is not defined in Python module anta.tests.software", - }, - { - "name": "undefined_module", - "filename": "test_catalog_with_undefined_module.yml", - "error": "Module named anta.tests.undefined cannot be imported", - }, - { - "name": "undefined_module", - "filename": "test_catalog_with_undefined_module.yml", - "error": "Module named anta.tests.undefined cannot be imported", - }, - { - "name": "syntax_error", - "filename": "test_catalog_with_syntax_error_module.yml", - "error": "Value error, Module named tests.data.syntax_error cannot be imported. Verify that the module exists and there is no Python syntax issues.", - }, - { - "name": "undefined_module_nested", - "filename": "test_catalog_with_undefined_module_nested.yml", - "error": "Module named undefined from package anta.tests cannot be imported", - }, - { - "name": "not_a_list", - "filename": "test_catalog_not_a_list.yml", - "error": "Value error, Syntax error when parsing: True\nIt must be a list of ANTA tests. Check the test catalog.", - }, - { - "name": "test_definition_not_a_dict", - "filename": "test_catalog_test_definition_not_a_dict.yml", - "error": "Value error, Syntax error when parsing: VerifyEOSVersion\nIt must be a dictionary. Check the test catalog.", - }, - { - "name": "test_definition_multiple_dicts", - "filename": "test_catalog_test_definition_multiple_dicts.yml", - "error": "Value error, Syntax error when parsing: {'VerifyEOSVersion': {'versions': ['4.25.4M', '4.26.1F']}, " - "'VerifyTerminAttrVersion': {'versions': ['4.25.4M']}}\nIt must be a dictionary with a single entry. Check the indentation in the test catalog.", - }, - {"name": "wrong_type_after_parsing", "filename": "test_catalog_wrong_type.yml", "error": "must be a dict, got str"}, +CATALOG_PARSE_FAIL_PARAMS: list[ParameterSet] = [ + pytest.param( + "test_catalog_wrong_format.toto", + "toto", + "'toto' is not a valid format for an AntaCatalog file. Only 'yaml' and 'json' are supported.", + id="undefined_tests", + ), + pytest.param("test_catalog_invalid_json.json", "json", "JSONDecodeError", id="invalid_json"), + pytest.param("test_catalog_with_undefined_tests.yml", "yaml", "FakeTest is not defined in Python module anta.tests.software", id="undefined_tests"), + pytest.param("test_catalog_with_undefined_module.yml", "yaml", "Module named anta.tests.undefined cannot be imported", id="undefined_module"), + pytest.param( + "test_catalog_with_syntax_error_module.yml", + "yaml", + "Value error, Module named tests.data.syntax_error cannot be imported. Verify that the module exists and there is no Python syntax issues.", + id="syntax_error", + ), + pytest.param( + "test_catalog_with_undefined_module_nested.yml", + "yaml", + "Module named undefined from package anta.tests cannot be imported", + id="undefined_module_nested", + ), + pytest.param( + "test_catalog_not_a_list.yml", + "yaml", + "Value error, Syntax error when parsing: True\nIt must be a list of ANTA tests. Check the test catalog.", + id="not_a_list", + ), + pytest.param( + "test_catalog_test_definition_not_a_dict.yml", + "yaml", + "Value error, Syntax error when parsing: VerifyEOSVersion\nIt must be a dictionary. Check the test catalog.", + id="test_definition_not_a_dict", + ), + pytest.param( + "test_catalog_test_definition_multiple_dicts.yml", + "yaml", + "Value error, Syntax error when parsing: {'VerifyEOSVersion': {'versions': ['4.25.4M', '4.26.1F']}, 'VerifyTerminAttrVersion': {'versions': ['4.25.4M']}}\n" + "It must be a dictionary with a single entry. Check the indentation in the test catalog.", + id="test_definition_multiple_dicts", + ), + pytest.param("test_catalog_wrong_type.yml", "yaml", "must be a dict, got str", id="wrong_type_after_parsing"), ] -CATALOG_FROM_DICT_FAIL_DATA: list[dict[str, Any]] = [ - { - "name": "undefined_tests", - "filename": "test_catalog_with_undefined_tests.yml", - "error": "FakeTest is not defined in Python module anta.tests.software", - }, - { - "name": "wrong_type", - "filename": "test_catalog_wrong_type.yml", - "error": "Wrong input type for catalog data, must be a dict, got str", - }, +CATALOG_FROM_DICT_FAIL_PARAMS: list[ParameterSet] = [ + pytest.param("test_catalog_with_undefined_tests.yml", "FakeTest is not defined in Python module anta.tests.software", id="undefined_tests"), + pytest.param("test_catalog_wrong_type.yml", "Wrong input type for catalog data, must be a dict, got str", id="wrong_type"), ] -CATALOG_FROM_LIST_FAIL_DATA: list[dict[str, Any]] = [ - { - "name": "wrong_inputs", - "tests": [ - ( - FakeTestWithInput, - AntaTest.Input(), - ), - ], - "error": "Test input has type AntaTest.Input but expected type FakeTestWithInput.Input", - }, - { - "name": "no_test", - "tests": [(None, None)], - "error": "Input should be a subclass of AntaTest", - }, - { - "name": "no_input_when_required", - "tests": [(FakeTestWithInput, None)], - "error": "FakeTestWithInput test inputs are not valid: 1 validation error for Input\n\tstring\n\t Field required", - }, - { - "name": "wrong_input_type", - "tests": [(FakeTestWithInput, {"string": True})], - "error": "FakeTestWithInput test inputs are not valid: 1 validation error for Input\n\tstring\n\t Input should be a valid string", - }, +CATALOG_FROM_LIST_FAIL_PARAMS: list[ParameterSet] = [ + pytest.param([(FakeTestWithInput, AntaTest.Input())], "Test input has type AntaTest.Input but expected type FakeTestWithInput.Input", id="wrong_inputs"), + pytest.param([(None, None)], "Input should be a subclass of AntaTest", id="no_test"), + pytest.param( + [(FakeTestWithInput, None)], + "FakeTestWithInput test inputs are not valid: 1 validation error for Input\n\tstring\n\t Field required", + id="no_input_when_required", + ), + pytest.param( + [(FakeTestWithInput, {"string": True})], + "FakeTestWithInput test inputs are not valid: 1 validation error for Input\n\tstring\n\t Input should be a valid string", + id="wrong_input_type", + ), ] -TESTS_SETTER_FAIL_DATA: list[dict[str, Any]] = [ - { - "name": "not_a_list", - "tests": "not_a_list", - "error": "The catalog must contain a list of tests", - }, - { - "name": "not_a_list_of_test_definitions", - "tests": [42, 43], - "error": "A test in the catalog must be an AntaTestDefinition instance", - }, +TESTS_SETTER_FAIL_PARAMS: list[ParameterSet] = [ + pytest.param("not_a_list", "The catalog must contain a list of tests", id="not_a_list"), + pytest.param([42, 43], "A test in the catalog must be an AntaTestDefinition instance", id="not_a_list_of_test_definitions"), ] class TestAntaCatalog: - """Test for anta.catalog.AntaCatalog.""" + """Tests for anta.catalog.AntaCatalog.""" - @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) - def test_parse(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("filename", "file_format", "tests"), INIT_CATALOG_PARAMS) + def test_parse(self, filename: str, file_format: Literal["yaml", "json"], tests: list[tuple[type[AntaTest], AntaTest.Input | dict[str, Any] | None]]) -> None: """Instantiate AntaCatalog from a file.""" - catalog: AntaCatalog = AntaCatalog.parse(DATA_DIR / catalog_data["filename"], file_format=catalog_data.get("file_format", "yaml")) + catalog: AntaCatalog = AntaCatalog.parse(DATA_DIR / filename, file_format=file_format) - assert len(catalog.tests) == len(catalog_data["tests"]) - for test_id, (test, inputs_data) in enumerate(catalog_data["tests"]): + assert len(catalog.tests) == len(tests) + for test_id, (test, inputs_data) in enumerate(tests): assert catalog.tests[test_id].test == test if inputs_data is not None: inputs = test.Input(**inputs_data) if isinstance(inputs_data, dict) else inputs_data assert inputs == catalog.tests[test_id].inputs - @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) - def test_from_list(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("filename", "file_format", "tests"), INIT_CATALOG_PARAMS) + def test_from_list( + self, filename: str, file_format: Literal["yaml", "json"], tests: list[tuple[type[AntaTest], AntaTest.Input | dict[str, Any] | None]] + ) -> None: """Instantiate AntaCatalog from a list.""" - catalog: AntaCatalog = AntaCatalog.from_list(catalog_data["tests"]) + catalog: AntaCatalog = AntaCatalog.from_list(tests) - assert len(catalog.tests) == len(catalog_data["tests"]) - for test_id, (test, inputs_data) in enumerate(catalog_data["tests"]): + assert len(catalog.tests) == len(tests) + for test_id, (test, inputs_data) in enumerate(tests): assert catalog.tests[test_id].test == test if inputs_data is not None: inputs = test.Input(**inputs_data) if isinstance(inputs_data, dict) else inputs_data assert inputs == catalog.tests[test_id].inputs - @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) - def test_from_dict(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("filename", "file_format", "tests"), INIT_CATALOG_PARAMS) + def test_from_dict( + self, filename: str, file_format: Literal["yaml", "json"], tests: list[tuple[type[AntaTest], AntaTest.Input | dict[str, Any] | None]] + ) -> None: """Instantiate AntaCatalog from a dict.""" - file = DATA_DIR / catalog_data["filename"] - with file.open(encoding="UTF-8") as file: - file_format = catalog_data.get("file_format", "yaml") - data = safe_load(file) if file_format == "yaml" else json_load(file) + file = DATA_DIR / filename + with file.open(encoding="UTF-8") as f: + data = safe_load(f) if file_format == "yaml" else json_load(f) catalog: AntaCatalog = AntaCatalog.from_dict(data) - assert len(catalog.tests) == len(catalog_data["tests"]) - for test_id, (test, inputs_data) in enumerate(catalog_data["tests"]): + assert len(catalog.tests) == len(tests) + for test_id, (test, inputs_data) in enumerate(tests): assert catalog.tests[test_id].test == test if inputs_data is not None: inputs = test.Input(**inputs_data) if isinstance(inputs_data, dict) else inputs_data assert inputs == catalog.tests[test_id].inputs - @pytest.mark.parametrize("catalog_data", CATALOG_PARSE_FAIL_DATA, ids=generate_test_ids_list(CATALOG_PARSE_FAIL_DATA)) - def test_parse_fail(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("filename", "file_format", "error"), CATALOG_PARSE_FAIL_PARAMS) + def test_parse_fail(self, filename: str, file_format: Literal["yaml", "json"], error: str) -> None: """Errors when instantiating AntaCatalog from a file.""" with pytest.raises((ValidationError, TypeError, ValueError, OSError)) as exec_info: - AntaCatalog.parse(DATA_DIR / catalog_data["filename"], file_format=catalog_data.get("file_format", "yaml")) + AntaCatalog.parse(DATA_DIR / filename, file_format=file_format) if isinstance(exec_info.value, ValidationError): - assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + assert error in exec_info.value.errors()[0]["msg"] else: - assert catalog_data["error"] in str(exec_info) + assert error in str(exec_info) def test_parse_fail_parsing(self, caplog: pytest.LogCaptureFixture) -> None: """Errors when instantiating AntaCatalog from a file.""" @@ -263,25 +204,25 @@ def test_parse_fail_parsing(self, caplog: pytest.LogCaptureFixture) -> None: assert "Unable to parse ANTA Test Catalog file" in message assert "FileNotFoundError: [Errno 2] No such file or directory" in message - @pytest.mark.parametrize("catalog_data", CATALOG_FROM_LIST_FAIL_DATA, ids=generate_test_ids_list(CATALOG_FROM_LIST_FAIL_DATA)) - def test_from_list_fail(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("tests", "error"), CATALOG_FROM_LIST_FAIL_PARAMS) + def test_from_list_fail(self, tests: list[tuple[type[AntaTest], AntaTest.Input | dict[str, Any] | None]], error: str) -> None: """Errors when instantiating AntaCatalog from a list of tuples.""" with pytest.raises(ValidationError) as exec_info: - AntaCatalog.from_list(catalog_data["tests"]) - assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + AntaCatalog.from_list(tests) + assert error in exec_info.value.errors()[0]["msg"] - @pytest.mark.parametrize("catalog_data", CATALOG_FROM_DICT_FAIL_DATA, ids=generate_test_ids_list(CATALOG_FROM_DICT_FAIL_DATA)) - def test_from_dict_fail(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("filename", "error"), CATALOG_FROM_DICT_FAIL_PARAMS) + def test_from_dict_fail(self, filename: str, error: str) -> None: """Errors when instantiating AntaCatalog from a list of tuples.""" - file = DATA_DIR / catalog_data["filename"] - with file.open(encoding="UTF-8") as file: - data = safe_load(file) + file = DATA_DIR / filename + with file.open(encoding="UTF-8") as f: + data = safe_load(f) with pytest.raises((ValidationError, TypeError)) as exec_info: AntaCatalog.from_dict(data) if isinstance(exec_info.value, ValidationError): - assert catalog_data["error"] in exec_info.value.errors()[0]["msg"] + assert error in exec_info.value.errors()[0]["msg"] else: - assert catalog_data["error"] in str(exec_info) + assert error in str(exec_info) def test_filename(self) -> None: """Test filename.""" @@ -290,31 +231,36 @@ def test_filename(self) -> None: catalog = AntaCatalog(filename=Path("test")) assert catalog.filename == Path("test") - @pytest.mark.parametrize("catalog_data", INIT_CATALOG_DATA, ids=generate_test_ids_list(INIT_CATALOG_DATA)) - def test__tests_setter_success(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("filename", "file_format", "tests"), INIT_CATALOG_PARAMS) + def test__tests_setter_success( + self, + filename: str, + file_format: Literal["yaml", "json"], + tests: list[tuple[type[AntaTest], AntaTest.Input | dict[str, Any] | None]], + ) -> None: """Success when setting AntaCatalog.tests from a list of tuples.""" catalog = AntaCatalog() - catalog.tests = [AntaTestDefinition(test=test, inputs=inputs) for test, inputs in catalog_data["tests"]] - assert len(catalog.tests) == len(catalog_data["tests"]) - for test_id, (test, inputs_data) in enumerate(catalog_data["tests"]): + catalog.tests = [AntaTestDefinition(test=test, inputs=inputs) for test, inputs in tests] + assert len(catalog.tests) == len(tests) + for test_id, (test, inputs_data) in enumerate(tests): assert catalog.tests[test_id].test == test if inputs_data is not None: inputs = test.Input(**inputs_data) if isinstance(inputs_data, dict) else inputs_data assert inputs == catalog.tests[test_id].inputs - @pytest.mark.parametrize("catalog_data", TESTS_SETTER_FAIL_DATA, ids=generate_test_ids_list(TESTS_SETTER_FAIL_DATA)) - def test__tests_setter_fail(self, catalog_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("tests", "error"), TESTS_SETTER_FAIL_PARAMS) + def test__tests_setter_fail(self, tests: list[Any], error: str) -> None: """Errors when setting AntaCatalog.tests from a list of tuples.""" catalog = AntaCatalog() with pytest.raises(TypeError) as exec_info: - catalog.tests = catalog_data["tests"] - assert catalog_data["error"] in str(exec_info) + catalog.tests = tests + assert error in str(exec_info) def test_build_indexes_all(self) -> None: """Test AntaCatalog.build_indexes().""" catalog: AntaCatalog = AntaCatalog.parse(DATA_DIR / "test_catalog_with_tags.yml") catalog.build_indexes() - assert len(catalog.tests_without_tags) == 5 + assert len(catalog.tests_without_tags) == 6 assert "leaf" in catalog.tag_to_tests assert len(catalog.tag_to_tests["leaf"]) == 3 all_unique_tests = catalog.tests_without_tags diff --git a/tests/units/test_device.py b/tests/units/test_device.py index d3c50cc8e..62c16c9ef 100644 --- a/tests/units/test_device.py +++ b/tests/units/test_device.py @@ -10,129 +10,51 @@ from typing import TYPE_CHECKING, Any from unittest.mock import patch -import httpx import pytest from asyncssh import SSHClientConnection, SSHClientConnectionOptions +from httpx import ConnectError, HTTPError from rich import print as rprint -import asynceapi from anta.device import AntaDevice, AsyncEOSDevice from anta.models import AntaCommand -from tests.lib.fixture import COMMAND_OUTPUT -from tests.lib.utils import generate_test_ids_list +from asynceapi import EapiCommandError +from tests.units.conftest import COMMAND_OUTPUT if TYPE_CHECKING: from _pytest.mark.structures import ParameterSet -INIT_DATA: list[dict[str, Any]] = [ - { - "name": "no name, no port", - "device": { - "host": "42.42.42.42", - "username": "anta", - "password": "anta", - }, - "expected": {"name": "42.42.42.42"}, - }, - { - "name": "no name, port", - "device": { - "host": "42.42.42.42", - "username": "anta", - "password": "anta", - "port": 666, - }, - "expected": {"name": "42.42.42.42:666"}, - }, - { - "name": "name", - "device": { - "host": "42.42.42.42", - "username": "anta", - "password": "anta", - "name": "test.anta.ninja", - "disable_cache": True, - }, - "expected": {"name": "test.anta.ninja"}, - }, - { - "name": "insecure", - "device": { - "host": "42.42.42.42", - "username": "anta", - "password": "anta", - "name": "test.anta.ninja", - "insecure": True, - }, - "expected": {"name": "test.anta.ninja"}, - }, +INIT_PARAMS: list[ParameterSet] = [ + pytest.param({"host": "42.42.42.42", "username": "anta", "password": "anta"}, {"name": "42.42.42.42"}, id="no name, no port"), + pytest.param({"host": "42.42.42.42", "username": "anta", "password": "anta", "port": 666}, {"name": "42.42.42.42:666"}, id="no name, port"), + pytest.param( + {"host": "42.42.42.42", "username": "anta", "password": "anta", "name": "test.anta.ninja", "disable_cache": True}, {"name": "test.anta.ninja"}, id="name" + ), + pytest.param( + {"host": "42.42.42.42", "username": "anta", "password": "anta", "name": "test.anta.ninja", "insecure": True}, {"name": "test.anta.ninja"}, id="insecure" + ), ] -EQUALITY_DATA: list[dict[str, Any]] = [ - { - "name": "equal", - "device1": { - "host": "42.42.42.42", - "username": "anta", - "password": "anta", - }, - "device2": { - "host": "42.42.42.42", - "username": "anta", - "password": "blah", - }, - "expected": True, - }, - { - "name": "equals-name", - "device1": { - "host": "42.42.42.42", - "username": "anta", - "password": "anta", - "name": "device1", - }, - "device2": { - "host": "42.42.42.42", - "username": "plop", - "password": "anta", - "name": "device2", - }, - "expected": True, - }, - { - "name": "not-equal-port", - "device1": { - "host": "42.42.42.42", - "username": "anta", - "password": "anta", - }, - "device2": { - "host": "42.42.42.42", - "username": "anta", - "password": "anta", - "port": 666, - }, - "expected": False, - }, - { - "name": "not-equal-host", - "device1": { - "host": "42.42.42.41", - "username": "anta", - "password": "anta", - }, - "device2": { - "host": "42.42.42.42", - "username": "anta", - "password": "anta", - }, - "expected": False, - }, +EQUALITY_PARAMS: list[ParameterSet] = [ + pytest.param({"host": "42.42.42.42", "username": "anta", "password": "anta"}, {"host": "42.42.42.42", "username": "anta", "password": "blah"}, True, id="equal"), + pytest.param( + {"host": "42.42.42.42", "username": "anta", "password": "anta", "name": "device1"}, + {"host": "42.42.42.42", "username": "plop", "password": "anta", "name": "device2"}, + True, + id="equals-name", + ), + pytest.param( + {"host": "42.42.42.42", "username": "anta", "password": "anta"}, + {"host": "42.42.42.42", "username": "anta", "password": "anta", "port": 666}, + False, + id="not-equal-port", + ), + pytest.param( + {"host": "42.42.42.41", "username": "anta", "password": "anta"}, {"host": "42.42.42.42", "username": "anta", "password": "anta"}, False, id="not-equal-host" + ), ] -ASYNCEAPI_COLLECT_DATA: list[dict[str, Any]] = [ - { - "name": "command", - "device": {}, - "command": { +ASYNCEAPI_COLLECT_PARAMS: list[ParameterSet] = [ + pytest.param( + {}, + { "command": "show version", "patch_kwargs": { "return_value": [ @@ -155,11 +77,11 @@ "memTotal": 8099732, "memFree": 4989568, "isIntlVersion": False, - }, - ], + } + ] }, }, - "expected": { + { "output": { "mfgName": "Arista", "modelName": "DCS-7280CR3-32P4-F", @@ -182,11 +104,11 @@ }, "errors": [], }, - }, - { - "name": "enable", - "device": {"enable": True}, - "command": { + id="command", + ), + pytest.param( + {"enable": True}, + { "command": "show version", "patch_kwargs": { "return_value": [ @@ -211,10 +133,10 @@ "memFree": 4989568, "isIntlVersion": False, }, - ], + ] }, }, - "expected": { + { "output": { "mfgName": "Arista", "modelName": "DCS-7280CR3-32P4-F", @@ -237,11 +159,11 @@ }, "errors": [], }, - }, - { - "name": "enable password", - "device": {"enable": True, "enable_password": "anta"}, - "command": { + id="enable", + ), + pytest.param( + {"enable": True, "enable_password": "anta"}, + { "command": "show version", "patch_kwargs": { "return_value": [ @@ -266,10 +188,10 @@ "memFree": 4989568, "isIntlVersion": False, }, - ], + ] }, }, - "expected": { + { "output": { "mfgName": "Arista", "modelName": "DCS-7280CR3-32P4-F", @@ -292,11 +214,11 @@ }, "errors": [], }, - }, - { - "name": "revision", - "device": {}, - "command": { + id="enable password", + ), + pytest.param( + {}, + { "command": "show version", "revision": 3, "patch_kwargs": { @@ -322,10 +244,10 @@ "memFree": 4989568, "isIntlVersion": False, }, - ], + ] }, }, - "expected": { + { "output": { "mfgName": "Arista", "modelName": "DCS-7280CR3-32P4-F", @@ -348,77 +270,47 @@ }, "errors": [], }, - }, - { - "name": "asynceapi.EapiCommandError", - "device": {}, - "command": { + id="revision", + ), + pytest.param( + {}, + { "command": "show version", "patch_kwargs": { - "side_effect": asynceapi.EapiCommandError( + "side_effect": EapiCommandError( passed=[], failed="show version", errors=["Authorization denied for command 'show version'"], errmsg="Invalid command", not_exec=[], - ), + ) }, }, - "expected": {"output": None, "errors": ["Authorization denied for command 'show version'"]}, - }, - { - "name": "httpx.HTTPError", - "device": {}, - "command": { - "command": "show version", - "patch_kwargs": {"side_effect": httpx.HTTPError(message="404")}, - }, - "expected": {"output": None, "errors": ["HTTPError: 404"]}, - }, - { - "name": "httpx.ConnectError", - "device": {}, - "command": { - "command": "show version", - "patch_kwargs": {"side_effect": httpx.ConnectError(message="Cannot open port")}, - }, - "expected": {"output": None, "errors": ["ConnectError: Cannot open port"]}, - }, + {"output": None, "errors": ["Authorization denied for command 'show version'"]}, + id="asynceapi.EapiCommandError", + ), + pytest.param( + {}, + {"command": "show version", "patch_kwargs": {"side_effect": HTTPError("404")}}, + {"output": None, "errors": ["HTTPError: 404"]}, + id="httpx.HTTPError", + ), + pytest.param( + {}, + {"command": "show version", "patch_kwargs": {"side_effect": ConnectError("Cannot open port")}}, + {"output": None, "errors": ["ConnectError: Cannot open port"]}, + id="httpx.ConnectError", + ), ] -ASYNCEAPI_COPY_DATA: list[dict[str, Any]] = [ - { - "name": "from", - "device": {}, - "copy": { - "sources": [Path("/mnt/flash"), Path("/var/log/agents")], - "destination": Path(), - "direction": "from", - }, - }, - { - "name": "to", - "device": {}, - "copy": { - "sources": [Path("/mnt/flash"), Path("/var/log/agents")], - "destination": Path(), - "direction": "to", - }, - }, - { - "name": "wrong", - "device": {}, - "copy": { - "sources": [Path("/mnt/flash"), Path("/var/log/agents")], - "destination": Path(), - "direction": "wrong", - }, - }, +ASYNCEAPI_COPY_PARAMS: list[ParameterSet] = [ + pytest.param({}, {"sources": [Path("/mnt/flash"), Path("/var/log/agents")], "destination": Path(), "direction": "from"}, id="from"), + pytest.param({}, {"sources": [Path("/mnt/flash"), Path("/var/log/agents")], "destination": Path(), "direction": "to"}, id="to"), + pytest.param({}, {"sources": [Path("/mnt/flash"), Path("/var/log/agents")], "destination": Path(), "direction": "wrong"}, id="wrong"), ] -REFRESH_DATA: list[dict[str, Any]] = [ - { - "name": "established", - "device": {}, - "patch_kwargs": ( +REFRESH_PARAMS: list[ParameterSet] = [ + pytest.param( + {}, + ( {"return_value": True}, { "return_value": [ @@ -442,15 +334,15 @@ "memFree": 4989568, "isIntlVersion": False, } - ], + ] }, ), - "expected": {"is_online": True, "established": True, "hw_model": "DCS-7280CR3-32P4-F"}, - }, - { - "name": "is not online", - "device": {}, - "patch_kwargs": ( + {"is_online": True, "established": True, "hw_model": "DCS-7280CR3-32P4-F"}, + id="established", + ), + pytest.param( + {}, + ( {"return_value": False}, { "return_value": { @@ -472,15 +364,15 @@ "memTotal": 8099732, "memFree": 4989568, "isIntlVersion": False, - }, + } }, ), - "expected": {"is_online": False, "established": False, "hw_model": None}, - }, - { - "name": "cannot parse command", - "device": {}, - "patch_kwargs": ( + {"is_online": False, "established": False, "hw_model": None}, + id="is not online", + ), + pytest.param( + {}, + ( {"return_value": True}, { "return_value": [ @@ -503,108 +395,71 @@ "memFree": 4989568, "isIntlVersion": False, } - ], + ] }, ), - "expected": {"is_online": True, "established": False, "hw_model": None}, - }, - { - "name": "asynceapi.EapiCommandError", - "device": {}, - "patch_kwargs": ( + {"is_online": True, "established": False, "hw_model": None}, + id="cannot parse command", + ), + pytest.param( + {}, + ( {"return_value": True}, { - "side_effect": asynceapi.EapiCommandError( + "side_effect": EapiCommandError( passed=[], failed="show version", errors=["Authorization denied for command 'show version'"], errmsg="Invalid command", not_exec=[], - ), + ) }, ), - "expected": {"is_online": True, "established": False, "hw_model": None}, - }, - { - "name": "httpx.HTTPError", - "device": {}, - "patch_kwargs": ( - {"return_value": True}, - {"side_effect": httpx.HTTPError(message="404")}, - ), - "expected": {"is_online": True, "established": False, "hw_model": None}, - }, - { - "name": "httpx.ConnectError", - "device": {}, - "patch_kwargs": ( - {"return_value": True}, - {"side_effect": httpx.ConnectError(message="Cannot open port")}, - ), - "expected": {"is_online": True, "established": False, "hw_model": None}, - }, + {"is_online": True, "established": False, "hw_model": None}, + id="asynceapi.EapiCommandError", + ), + pytest.param( + {}, + ({"return_value": True}, {"side_effect": HTTPError("404")}), + {"is_online": True, "established": False, "hw_model": None}, + id="httpx.HTTPError", + ), + pytest.param( + {}, + ({"return_value": True}, {"side_effect": ConnectError("Cannot open port")}), + {"is_online": True, "established": False, "hw_model": None}, + id="httpx.ConnectError", + ), ] -COLLECT_DATA: list[dict[str, Any]] = [ - { - "name": "device cache enabled, command cache enabled, no cache hit", - "device": {"disable_cache": False}, - "command": { - "command": "show version", - "use_cache": True, - }, - "expected": {"cache_hit": False}, - }, - { - "name": "device cache enabled, command cache enabled, cache hit", - "device": {"disable_cache": False}, - "command": { - "command": "show version", - "use_cache": True, - }, - "expected": {"cache_hit": True}, - }, - { - "name": "device cache disabled, command cache enabled", - "device": {"disable_cache": True}, - "command": { - "command": "show version", - "use_cache": True, - }, - "expected": {}, - }, - { - "name": "device cache enabled, command cache disabled, cache has command", - "device": {"disable_cache": False}, - "command": { - "command": "show version", - "use_cache": False, - }, - "expected": {"cache_hit": True}, - }, - { - "name": "device cache enabled, command cache disabled, cache does not have data", - "device": { - "disable_cache": False, - }, - "command": { - "command": "show version", - "use_cache": False, - }, - "expected": {"cache_hit": False}, - }, - { - "name": "device cache disabled, command cache disabled", - "device": { - "disable_cache": True, - }, - "command": { - "command": "show version", - "use_cache": False, - }, - "expected": {}, - }, +COLLECT_PARAMS: list[ParameterSet] = [ + pytest.param( + {"disable_cache": False}, + {"command": "show version", "use_cache": True}, + {"cache_hit": False}, + id="device cache enabled, command cache enabled, no cache hit", + ), + pytest.param( + {"disable_cache": False}, + {"command": "show version", "use_cache": True}, + {"cache_hit": True}, + id="device cache enabled, command cache enabled, cache hit", + ), + pytest.param({"disable_cache": True}, {"command": "show version", "use_cache": True}, {}, id="device cache disabled, command cache enabled"), + pytest.param( + {"disable_cache": False}, + {"command": "show version", "use_cache": False}, + {"cache_hit": True}, + id="device cache enabled, command cache disabled, cache has command", + ), + pytest.param( + {"disable_cache": False}, + {"command": "show version", "use_cache": False}, + {"cache_hit": False}, + id="device cache enabled, command cache disabled, cache does not have data", + ), + pytest.param({"disable_cache": True}, {"command": "show version", "use_cache": False}, {}, id="device cache disabled, command cache disabled"), ] -CACHE_STATS_DATA: list[ParameterSet] = [ +CACHE_STATS_PARAMS: list[ParameterSet] = [ pytest.param({"disable_cache": False}, {"total_commands_sent": 0, "cache_hits": 0, "cache_hit_ratio": "0.00%"}, id="with_cache"), pytest.param({"disable_cache": True}, None, id="without_cache"), ] @@ -613,48 +468,42 @@ class TestAntaDevice: """Test for anta.device.AntaDevice Abstract class.""" - @pytest.mark.asyncio - @pytest.mark.parametrize( - ("device", "command_data", "expected_data"), - ((d["device"], d["command"], d["expected"]) for d in COLLECT_DATA), - indirect=["device"], - ids=generate_test_ids_list(COLLECT_DATA), - ) - async def test_collect(self, device: AntaDevice, command_data: dict[str, Any], expected_data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("device", "command", "expected"), COLLECT_PARAMS, indirect=["device"]) + async def test_collect(self, device: AntaDevice, command: dict[str, Any], expected: dict[str, Any]) -> None: """Test AntaDevice.collect behavior.""" - command = AntaCommand(command=command_data["command"], use_cache=command_data["use_cache"]) + cmd = AntaCommand(command=command["command"], use_cache=command["use_cache"]) # Dummy output for cache hit cached_output = "cached_value" - if device.cache is not None and expected_data["cache_hit"] is True: - await device.cache.set(command.uid, cached_output) + if device.cache is not None and expected["cache_hit"] is True: + await device.cache.set(cmd.uid, cached_output) - await device.collect(command) + await device.collect(cmd) if device.cache is not None: # device_cache is enabled - current_cached_data = await device.cache.get(command.uid) - if command.use_cache is True: # command is allowed to use cache - if expected_data["cache_hit"] is True: - assert command.output == cached_output + current_cached_data = await device.cache.get(cmd.uid) + if cmd.use_cache is True: # command is allowed to use cache + if expected["cache_hit"] is True: + assert cmd.output == cached_output assert current_cached_data == cached_output assert device.cache.hit_miss_ratio["hits"] == 2 else: - assert command.output == COMMAND_OUTPUT + assert cmd.output == COMMAND_OUTPUT assert current_cached_data == COMMAND_OUTPUT assert device.cache.hit_miss_ratio["hits"] == 1 else: # command is not allowed to use cache - device._collect.assert_called_once_with(command=command, collection_id=None) # type: ignore[attr-defined] # pylint: disable=protected-access - assert command.output == COMMAND_OUTPUT - if expected_data["cache_hit"] is True: + device._collect.assert_called_once_with(command=cmd, collection_id=None) # type: ignore[attr-defined] + assert cmd.output == COMMAND_OUTPUT + if expected["cache_hit"] is True: assert current_cached_data == cached_output else: assert current_cached_data is None else: # device is disabled assert device.cache is None - device._collect.assert_called_once_with(command=command, collection_id=None) # type: ignore[attr-defined] # pylint: disable=protected-access + device._collect.assert_called_once_with(command=cmd, collection_id=None) # type: ignore[attr-defined] - @pytest.mark.parametrize(("device", "expected"), CACHE_STATS_DATA, indirect=["device"]) + @pytest.mark.parametrize(("device", "expected"), CACHE_STATS_PARAMS, indirect=["device"]) def test_cache_statistics(self, device: AntaDevice, expected: dict[str, Any] | None) -> None: """Verify that when cache statistics attribute does not exist. @@ -666,42 +515,39 @@ def test_cache_statistics(self, device: AntaDevice, expected: dict[str, Any] | N class TestAsyncEOSDevice: """Test for anta.device.AsyncEOSDevice.""" - @pytest.mark.parametrize("data", INIT_DATA, ids=generate_test_ids_list(INIT_DATA)) - def test__init__(self, data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("device", "expected"), INIT_PARAMS) + def test__init__(self, device: dict[str, Any], expected: dict[str, Any]) -> None: """Test the AsyncEOSDevice constructor.""" - device = AsyncEOSDevice(**data["device"]) + dev = AsyncEOSDevice(**device) - assert device.name == data["expected"]["name"] - if data["device"].get("disable_cache") is True: - assert device.cache is None - assert device.cache_locks is None + assert dev.name == expected["name"] + if device.get("disable_cache") is True: + assert dev.cache is None + assert dev.cache_locks is None else: # False or None - assert device.cache is not None - assert device.cache_locks is not None - hash(device) + assert dev.cache is not None + assert dev.cache_locks is not None + hash(dev) with patch("anta.device.__DEBUG__", new=True): - rprint(device) + rprint(dev) - @pytest.mark.parametrize("data", EQUALITY_DATA, ids=generate_test_ids_list(EQUALITY_DATA)) - def test__eq(self, data: dict[str, Any]) -> None: + @pytest.mark.parametrize(("device1", "device2", "expected"), EQUALITY_PARAMS) + def test__eq(self, device1: dict[str, Any], device2: dict[str, Any], expected: bool) -> None: """Test the AsyncEOSDevice equality.""" - device1 = AsyncEOSDevice(**data["device1"]) - device2 = AsyncEOSDevice(**data["device2"]) - if data["expected"]: - assert device1 == device2 + dev1 = AsyncEOSDevice(**device1) + dev2 = AsyncEOSDevice(**device2) + if expected: + assert dev1 == dev2 else: - assert device1 != device2 + assert dev1 != dev2 - @pytest.mark.asyncio @pytest.mark.parametrize( ("async_device", "patch_kwargs", "expected"), - ((d["device"], d["patch_kwargs"], d["expected"]) for d in REFRESH_DATA), - ids=generate_test_ids_list(REFRESH_DATA), + REFRESH_PARAMS, indirect=["async_device"], ) async def test_refresh(self, async_device: AsyncEOSDevice, patch_kwargs: list[dict[str, Any]], expected: dict[str, Any]) -> None: - # pylint: disable=protected-access """Test AsyncEOSDevice.refresh().""" with patch.object(async_device._session, "check_connection", **patch_kwargs[0]), patch.object(async_device._session, "cli", **patch_kwargs[1]): await async_device.refresh() @@ -712,15 +558,12 @@ async def test_refresh(self, async_device: AsyncEOSDevice, patch_kwargs: list[di assert async_device.established == expected["established"] assert async_device.hw_model == expected["hw_model"] - @pytest.mark.asyncio @pytest.mark.parametrize( ("async_device", "command", "expected"), - ((d["device"], d["command"], d["expected"]) for d in ASYNCEAPI_COLLECT_DATA), - ids=generate_test_ids_list(ASYNCEAPI_COLLECT_DATA), + ASYNCEAPI_COLLECT_PARAMS, indirect=["async_device"], ) async def test__collect(self, async_device: AsyncEOSDevice, command: dict[str, Any], expected: dict[str, Any]) -> None: - # pylint: disable=protected-access """Test AsyncEOSDevice._collect().""" cmd = AntaCommand(command=command["command"], revision=command["revision"]) if "revision" in command else AntaCommand(command=command["command"]) with patch.object(async_device._session, "cli", **command["patch_kwargs"]): @@ -741,15 +584,13 @@ async def test__collect(self, async_device: AsyncEOSDevice, command: dict[str, A commands.append({"cmd": cmd.command, "revision": cmd.revision}) else: commands.append({"cmd": cmd.command}) - async_device._session.cli.assert_called_once_with(commands=commands, ofmt=cmd.ofmt, version=cmd.version, req_id=f"ANTA-{collection_id}-{id(cmd)}") # type: ignore[attr-defined] # asynceapi.Device.cli is patched # pylint: disable=line-too-long + async_device._session.cli.assert_called_once_with(commands=commands, ofmt=cmd.ofmt, version=cmd.version, req_id=f"ANTA-{collection_id}-{id(cmd)}") # type: ignore[attr-defined] # asynceapi.Device.cli is patched assert cmd.output == expected["output"] assert cmd.errors == expected["errors"] - @pytest.mark.asyncio @pytest.mark.parametrize( ("async_device", "copy"), - ((d["device"], d["copy"]) for d in ASYNCEAPI_COPY_DATA), - ids=generate_test_ids_list(ASYNCEAPI_COPY_DATA), + ASYNCEAPI_COPY_PARAMS, indirect=["async_device"], ) async def test_copy(self, async_device: AsyncEOSDevice, copy: dict[str, Any]) -> None: diff --git a/tests/units/test_logger.py b/tests/units/test_logger.py index d9b7c7672..706591fac 100644 --- a/tests/units/test_logger.py +++ b/tests/units/test_logger.py @@ -58,7 +58,6 @@ def test_anta_log_exception( debug_value: bool, expected_message: str, ) -> None: - # pylint: disable=too-many-arguments """Test anta_log_exception.""" if calling_logger is not None: # https://github.com/pytest-dev/pytest/issues/3697 diff --git a/tests/units/test_models.py b/tests/units/test_models.py index 180f6bfe5..d604b4835 100644 --- a/tests/units/test_models.py +++ b/tests/units/test_models.py @@ -8,14 +8,16 @@ from __future__ import annotations import asyncio +import sys from typing import TYPE_CHECKING, Any, ClassVar import pytest from anta.decorators import deprecated_test, skip_on_platforms from anta.models import AntaCommand, AntaTemplate, AntaTest -from tests.lib.fixture import DEVICE_HW_MODEL -from tests.lib.utils import generate_test_ids +from anta.result_manager.models import AntaTestStatus +from tests.units.anta_tests.conftest import build_test_id +from tests.units.conftest import DEVICE_HW_MODEL if TYPE_CHECKING: from anta.device import AntaDevice @@ -302,6 +304,15 @@ def test(self) -> None: self.result.is_success() +class FakeTestWithMissingTest(AntaTest): + """ANTA test with missing test() method implementation.""" + + name = "FakeTestWithMissingTest" + description = "ANTA test with missing test() method implementation" + categories: ClassVar[list[str]] = [] + commands: ClassVar[list[AntaCommand | AntaTemplate]] = [] + + ANTATEST_DATA: list[dict[str, Any]] = [ { "name": "no input", @@ -507,17 +518,17 @@ def test(self) -> None: }, ] +BLACKLIST_COMMANDS_PARAMS = ["reload", "reload --force", "write", "wr mem"] + class TestAntaTest: """Test for anta.models.AntaTest.""" - def test__init_subclass__name(self) -> None: + def test__init_subclass__(self) -> None: """Test __init_subclass__.""" - # Pylint detects all the classes in here as unused which is on purpose - # pylint: disable=unused-variable with pytest.raises(NotImplementedError) as exec_info: - class WrongTestNoName(AntaTest): + class _WrongTestNoName(AntaTest): """ANTA test that is missing a name.""" description = "ANTA test that is missing a name" @@ -528,11 +539,11 @@ class WrongTestNoName(AntaTest): def test(self) -> None: self.result.is_success() - assert exec_info.value.args[0] == "Class tests.units.test_models.WrongTestNoName is missing required class attribute name" + assert exec_info.value.args[0] == "Class tests.units.test_models._WrongTestNoName is missing required class attribute name" with pytest.raises(NotImplementedError) as exec_info: - class WrongTestNoDescription(AntaTest): + class _WrongTestNoDescription(AntaTest): """ANTA test that is missing a description.""" name = "WrongTestNoDescription" @@ -543,11 +554,11 @@ class WrongTestNoDescription(AntaTest): def test(self) -> None: self.result.is_success() - assert exec_info.value.args[0] == "Class tests.units.test_models.WrongTestNoDescription is missing required class attribute description" + assert exec_info.value.args[0] == "Class tests.units.test_models._WrongTestNoDescription is missing required class attribute description" with pytest.raises(NotImplementedError) as exec_info: - class WrongTestNoCategories(AntaTest): + class _WrongTestNoCategories(AntaTest): """ANTA test that is missing categories.""" name = "WrongTestNoCategories" @@ -558,11 +569,11 @@ class WrongTestNoCategories(AntaTest): def test(self) -> None: self.result.is_success() - assert exec_info.value.args[0] == "Class tests.units.test_models.WrongTestNoCategories is missing required class attribute categories" + assert exec_info.value.args[0] == "Class tests.units.test_models._WrongTestNoCategories is missing required class attribute categories" with pytest.raises(NotImplementedError) as exec_info: - class WrongTestNoCommands(AntaTest): + class _WrongTestNoCommands(AntaTest): """ANTA test that is missing commands.""" name = "WrongTestNoCommands" @@ -573,22 +584,34 @@ class WrongTestNoCommands(AntaTest): def test(self) -> None: self.result.is_success() - assert exec_info.value.args[0] == "Class tests.units.test_models.WrongTestNoCommands is missing required class attribute commands" + assert exec_info.value.args[0] == "Class tests.units.test_models._WrongTestNoCommands is missing required class attribute commands" + + def test_abc(self) -> None: + """Test that an error is raised if AntaTest is not implemented.""" + with pytest.raises(TypeError) as exec_info: + FakeTestWithMissingTest() # type: ignore[abstract,call-arg] + msg = ( + "Can't instantiate abstract class FakeTestWithMissingTest without an implementation for abstract method 'test'" + if sys.version_info >= (3, 12) + else "Can't instantiate abstract class FakeTestWithMissingTest with abstract method test" + ) + assert exec_info.value.args[0] == msg def _assert_test(self, test: AntaTest, expected: dict[str, Any]) -> None: assert test.result.result == expected["result"] if "messages" in expected: + assert len(test.result.messages) == len(expected["messages"]) for result_msg, expected_msg in zip(test.result.messages, expected["messages"]): # NOTE: zip(strict=True) has been added in Python 3.10 assert expected_msg in result_msg - @pytest.mark.parametrize("data", ANTATEST_DATA, ids=generate_test_ids(ANTATEST_DATA)) + @pytest.mark.parametrize("data", ANTATEST_DATA, ids=build_test_id) def test__init__(self, device: AntaDevice, data: dict[str, Any]) -> None: """Test the AntaTest constructor.""" expected = data["expected"]["__init__"] test = data["test"](device, inputs=data["inputs"]) self._assert_test(test, expected) - @pytest.mark.parametrize("data", ANTATEST_DATA, ids=generate_test_ids(ANTATEST_DATA)) + @pytest.mark.parametrize("data", ANTATEST_DATA, ids=build_test_id) def test_test(self, device: AntaDevice, data: dict[str, Any]) -> None: """Test the AntaTest.test method.""" expected = data["expected"]["test"] @@ -596,38 +619,42 @@ def test_test(self, device: AntaDevice, data: dict[str, Any]) -> None: asyncio.run(test.test()) self._assert_test(test, expected) + @pytest.mark.parametrize("command", BLACKLIST_COMMANDS_PARAMS) + def test_blacklist(self, device: AntaDevice, command: str) -> None: + """Test that blacklisted commands are not collected.""" -ANTATEST_BLACKLIST_DATA = ["reload", "reload --force", "write", "wr mem"] - - -@pytest.mark.parametrize("data", ANTATEST_BLACKLIST_DATA) -def test_blacklist(device: AntaDevice, data: str) -> None: - """Test for blacklisting function.""" + class FakeTestWithBlacklist(AntaTest): + """Fake Test for blacklist.""" - class FakeTestWithBlacklist(AntaTest): - """Fake Test for blacklist.""" + name = "FakeTestWithBlacklist" + description = "ANTA test that has blacklisted command" + categories: ClassVar[list[str]] = [] + commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command=command)] - name = "FakeTestWithBlacklist" - description = "ANTA test that has blacklisted command" - categories: ClassVar[list[str]] = [] - commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command=data)] + @AntaTest.anta_test + def test(self) -> None: + self.result.is_success() - @AntaTest.anta_test - def test(self) -> None: - self.result.is_success() - - test_instance = FakeTestWithBlacklist(device) + test = FakeTestWithBlacklist(device) + asyncio.run(test.test()) + assert test.result.result == AntaTestStatus.ERROR + assert f"<{command}> is blocked for security reason" in test.result.messages + assert test.instance_commands[0].collected is False - # Run the test() method - asyncio.run(test_instance.test()) - assert test_instance.result.result == "error" + def test_result_overwrite(self, device: AntaDevice) -> None: + """Test the AntaTest.Input.ResultOverwrite model.""" + test = FakeTest(device, inputs={"result_overwrite": {"categories": ["hardware"], "description": "a description", "custom_field": "a custom field"}}) + asyncio.run(test.test()) + assert test.result.result == AntaTestStatus.SUCCESS + assert "hardware" in test.result.categories + assert test.result.description == "a description" + assert test.result.custom_field == "a custom field" class TestAntaComamnd: """Test for anta.models.AntaCommand.""" # ruff: noqa: B018 - # pylint: disable=pointless-statement def test_empty_output_access(self) -> None: """Test for both json and text ofmt.""" @@ -656,16 +683,20 @@ def test_wrong_format_output_access(self) -> None: text_cmd_2.json_output def test_supported(self) -> None: - """Test if the supported property.""" + """Test the supported property.""" command = AntaCommand(command="show hardware counter drop", errors=["Unavailable command (not supported on this hardware platform) (at token 2: 'counter')"]) assert command.supported is False command = AntaCommand( command="show hardware counter drop", output={"totalAdverseDrops": 0, "totalCongestionDrops": 0, "totalPacketProcessorDrops": 0, "dropEvents": {}} ) assert command.supported is True + command = AntaCommand(command="show hardware counter drop") + with pytest.raises(RuntimeError) as exec_info: + command.supported + assert exec_info.value.args[0] == "Command 'show hardware counter drop' has not been collected and has not returned an error. Call AntaDevice.collect()." def test_requires_privileges(self) -> None: - """Test if the requires_privileges property.""" + """Test the requires_privileges property.""" command = AntaCommand(command="show aaa methods accounting", errors=["Invalid input (privileged mode required)"]) assert command.requires_privileges is True command = AntaCommand( @@ -678,3 +709,7 @@ def test_requires_privileges(self) -> None: }, ) assert command.requires_privileges is False + command = AntaCommand(command="show aaa methods accounting") + with pytest.raises(RuntimeError) as exec_info: + command.requires_privileges + assert exec_info.value.args[0] == "Command 'show aaa methods accounting' has not been collected and has not returned an error. Call AntaDevice.collect()." diff --git a/tests/units/test_runner.py b/tests/units/test_runner.py index 53d0bf758..b80259cc3 100644 --- a/tests/units/test_runner.py +++ b/tests/units/test_runner.py @@ -7,73 +7,62 @@ import logging import resource +import sys from pathlib import Path from unittest.mock import patch import pytest -from anta import logger from anta.catalog import AntaCatalog from anta.inventory import AntaInventory from anta.result_manager import ResultManager from anta.runner import adjust_rlimit_nofile, main, prepare_tests -from .test_models import FakeTest +from .test_models import FakeTest, FakeTestWithMissingTest DATA_DIR: Path = Path(__file__).parent.parent.resolve() / "data" FAKE_CATALOG: AntaCatalog = AntaCatalog.from_list([(FakeTest, None)]) -@pytest.mark.asyncio -async def test_runner_empty_tests(caplog: pytest.LogCaptureFixture, test_inventory: AntaInventory) -> None: - """Test that when the list of tests is empty, a log is raised. - - caplog is the pytest fixture to capture logs - test_inventory is a fixture that gives a default inventory for tests - """ - logger.setup_logging(logger.Log.INFO) +async def test_empty_tests(caplog: pytest.LogCaptureFixture, inventory: AntaInventory) -> None: + """Test that when the list of tests is empty, a log is raised.""" caplog.set_level(logging.INFO) manager = ResultManager() - await main(manager, test_inventory, AntaCatalog()) + await main(manager, inventory, AntaCatalog()) assert len(caplog.record_tuples) == 1 assert "The list of tests is empty, exiting" in caplog.records[0].message -@pytest.mark.asyncio -async def test_runner_empty_inventory(caplog: pytest.LogCaptureFixture) -> None: - """Test that when the Inventory is empty, a log is raised. - - caplog is the pytest fixture to capture logs - """ - logger.setup_logging(logger.Log.INFO) +async def test_empty_inventory(caplog: pytest.LogCaptureFixture) -> None: + """Test that when the Inventory is empty, a log is raised.""" caplog.set_level(logging.INFO) manager = ResultManager() - inventory = AntaInventory() - await main(manager, inventory, FAKE_CATALOG) + await main(manager, AntaInventory(), FAKE_CATALOG) assert len(caplog.record_tuples) == 3 assert "The inventory is empty, exiting" in caplog.records[1].message -@pytest.mark.asyncio -async def test_runner_no_selected_device(caplog: pytest.LogCaptureFixture, test_inventory: AntaInventory) -> None: - """Test that when the list of established device. - - caplog is the pytest fixture to capture logs - test_inventory is a fixture that gives a default inventory for tests - """ - logger.setup_logging(logger.Log.INFO) - caplog.set_level(logging.INFO) +@pytest.mark.parametrize( + ("inventory", "tags", "devices"), + [ + pytest.param({"count": 1, "reachable": False}, None, None, id="not-reachable"), + pytest.param({"filename": "test_inventory_with_tags.yml", "reachable": False}, {"leaf"}, None, id="not-reachable-with-tag"), + pytest.param({"count": 1, "reachable": True}, {"invalid-tag"}, None, id="reachable-with-invalid-tag"), + pytest.param({"filename": "test_inventory_with_tags.yml", "reachable": True}, None, {"invalid-device"}, id="reachable-with-invalid-device"), + pytest.param({"filename": "test_inventory_with_tags.yml", "reachable": False}, None, {"leaf1"}, id="not-reachable-with-device"), + pytest.param({"filename": "test_inventory_with_tags.yml", "reachable": False}, {"leaf"}, {"leaf1"}, id="not-reachable-with-device-and-tag"), + pytest.param({"filename": "test_inventory_with_tags.yml", "reachable": False}, {"invalid"}, {"invalid-device"}, id="reachable-with-invalid-tag-and-device"), + ], + indirect=["inventory"], +) +async def test_no_selected_device(caplog: pytest.LogCaptureFixture, inventory: AntaInventory, tags: set[str], devices: set[str]) -> None: + """Test that when the list of established devices is empty a log is raised.""" + caplog.set_level(logging.WARNING) manager = ResultManager() - await main(manager, test_inventory, FAKE_CATALOG) - - assert "No reachable device was found." in [record.message for record in caplog.records] - - # Reset logs and run with tags - caplog.clear() - await main(manager, test_inventory, FAKE_CATALOG, tags={"toto"}) - - assert "No reachable device matching the tags {'toto'} was found." in [record.message for record in caplog.records] + await main(manager, inventory, FAKE_CATALOG, tags=tags, devices=devices) + msg = f'No reachable device {f"matching the tags {tags} " if tags else ""}was found.{f" Selected devices: {devices} " if devices is not None else ""}' + assert msg in caplog.messages def test_adjust_rlimit_nofile_valid_env(caplog: pytest.LogCaptureFixture) -> None: @@ -140,67 +129,55 @@ def side_effect_setrlimit(resource_id: int, limits: tuple[int, int]) -> None: setrlimit_mock.assert_called_once_with(resource.RLIMIT_NOFILE, (16384, 1048576)) -@pytest.mark.asyncio @pytest.mark.parametrize( - ("tags", "expected_tests_count", "expected_devices_count"), + ("inventory", "tags", "tests", "devices_count", "tests_count"), [ - (None, 22, 3), - ({"leaf"}, 9, 3), - ({"invalid_tag"}, 0, 0), + pytest.param({"filename": "test_inventory_with_tags.yml"}, None, None, 3, 27, id="all-tests"), + pytest.param({"filename": "test_inventory_with_tags.yml"}, {"leaf"}, None, 2, 6, id="1-tag"), + pytest.param({"filename": "test_inventory_with_tags.yml"}, {"leaf", "spine"}, None, 3, 9, id="2-tags"), + pytest.param({"filename": "test_inventory_with_tags.yml"}, None, {"VerifyMlagStatus", "VerifyUptime"}, 3, 5, id="filtered-tests"), + pytest.param({"filename": "test_inventory_with_tags.yml"}, {"leaf"}, {"VerifyMlagStatus", "VerifyUptime"}, 2, 4, id="1-tag-filtered-tests"), + pytest.param({"filename": "test_inventory_with_tags.yml"}, {"invalid"}, None, 0, 0, id="invalid-tag"), ], - ids=["no_tags", "leaf_tag", "invalid_tag"], + indirect=["inventory"], ) async def test_prepare_tests( - caplog: pytest.LogCaptureFixture, - test_inventory: AntaInventory, - tags: set[str] | None, - expected_tests_count: int, - expected_devices_count: int, + caplog: pytest.LogCaptureFixture, inventory: AntaInventory, tags: set[str], tests: set[str], devices_count: int, tests_count: int ) -> None: - """Test the runner prepare_tests function.""" - logger.setup_logging(logger.Log.INFO) - caplog.set_level(logging.INFO) - - catalog: AntaCatalog = AntaCatalog.parse(str(DATA_DIR / "test_catalog_with_tags.yml")) - selected_tests = prepare_tests(inventory=test_inventory, catalog=catalog, tags=tags, tests=None) - - if selected_tests is None: - assert expected_tests_count == 0 - expected_log = f"There are no tests matching the tags {tags} to run in the current test catalog and device inventory, please verify your inputs." - assert expected_log in caplog.text - else: - assert len(selected_tests) == expected_devices_count - assert sum(len(tests) for tests in selected_tests.values()) == expected_tests_count - - -@pytest.mark.asyncio -async def test_prepare_tests_with_specific_tests(caplog: pytest.LogCaptureFixture, test_inventory: AntaInventory) -> None: """Test the runner prepare_tests function with specific tests.""" - logger.setup_logging(logger.Log.INFO) - caplog.set_level(logging.INFO) - + caplog.set_level(logging.WARNING) catalog: AntaCatalog = AntaCatalog.parse(str(DATA_DIR / "test_catalog_with_tags.yml")) - selected_tests = prepare_tests(inventory=test_inventory, catalog=catalog, tags=None, tests={"VerifyMlagStatus", "VerifyUptime"}) - + selected_tests = prepare_tests(inventory=inventory, catalog=catalog, tags=tags, tests=tests) + if selected_tests is None: + msg = f"There are no tests matching the tags {tags} to run in the current test catalog and device inventory, please verify your inputs." + assert msg in caplog.messages + return assert selected_tests is not None - assert len(selected_tests) == 3 - assert sum(len(tests) for tests in selected_tests.values()) == 5 - + assert len(selected_tests) == devices_count + assert sum(len(tests) for tests in selected_tests.values()) == tests_count -@pytest.mark.asyncio -async def test_runner_dry_run(caplog: pytest.LogCaptureFixture, test_inventory: AntaInventory) -> None: - """Test that when dry_run is True, no tests are run. - caplog is the pytest fixture to capture logs - test_inventory is a fixture that gives a default inventory for tests - """ - logger.setup_logging(logger.Log.INFO) +async def test_dry_run(caplog: pytest.LogCaptureFixture, inventory: AntaInventory) -> None: + """Test that when dry_run is True, no tests are run.""" caplog.set_level(logging.INFO) manager = ResultManager() - catalog_path = Path(__file__).parent.parent / "data" / "test_catalog.yml" - catalog = AntaCatalog.parse(catalog_path) + await main(manager, inventory, FAKE_CATALOG, dry_run=True) + assert "Dry-run mode, exiting before running the tests." in caplog.records[-1].message - await main(manager, test_inventory, catalog, dry_run=True) - # Check that the last log contains Dry-run - assert "Dry-run" in caplog.records[-1].message +async def test_cannot_create_test(caplog: pytest.LogCaptureFixture, inventory: AntaInventory) -> None: + """Test that when an Exception is raised during test instantiation, it is caught and a log is raised.""" + caplog.set_level(logging.CRITICAL) + manager = ResultManager() + catalog = AntaCatalog.from_list([(FakeTestWithMissingTest, None)]) # type: ignore[type-abstract] + await main(manager, inventory, catalog) + msg = ( + "There is an error when creating test tests.units.test_models.FakeTestWithMissingTest.\nIf this is not a custom test implementation: " + "Please reach out to the maintainer team or open an issue on Github: https://github.com/aristanetworks/anta.\nTypeError: " + ) + msg += ( + "Can't instantiate abstract class FakeTestWithMissingTest without an implementation for abstract method 'test'" + if sys.version_info >= (3, 12) + else "Can't instantiate abstract class FakeTestWithMissingTest with abstract method test" + ) + assert msg in caplog.messages diff --git a/tests/units/test_tools.py b/tests/units/test_tools.py index c3a57e5af..29abac5e6 100644 --- a/tests/units/test_tools.py +++ b/tests/units/test_tools.py @@ -313,7 +313,6 @@ def test_get_dict_superset( expected_raise: AbstractContextManager[Exception], ) -> None: """Test get_dict_superset.""" - # pylint: disable=too-many-arguments with expected_raise: assert get_dict_superset(list_of_dicts, input_dict, default, var_name, custom_error_msg, required=required) == expected_result @@ -421,7 +420,6 @@ def test_get_value( expected_raise: AbstractContextManager[Exception], ) -> None: """Test get_value.""" - # pylint: disable=too-many-arguments kwargs = { "default": default, "required": required, @@ -485,7 +483,6 @@ def test_get_item( expected_raise: AbstractContextManager[Exception], ) -> None: """Test get_item.""" - # pylint: disable=too-many-arguments with expected_raise: assert get_item(list_of_dicts, key, value, default, var_name, custom_error_msg, required=required, case_sensitive=case_sensitive) == expected_result