From 79d0483112480570664381b8ffa3c3642b8bc363 Mon Sep 17 00:00:00 2001 From: Alastair Lyall Date: Fri, 28 Jun 2024 17:35:46 +0200 Subject: [PATCH] feat(cli): add test command (#724) Closes #719 --- AUTHORS.md | 1 + docs/cmd_list.txt | 3 + reana_client/cli/__init__.py | 11 +- reana_client/cli/files.py | 8 +- reana_client/cli/test.py | 169 +++++++++++++++ tests/test_cli_files.py | 29 +++ tests/test_cli_test.py | 391 +++++++++++++++++++++++++++++++++++ 7 files changed, 607 insertions(+), 5 deletions(-) create mode 100644 reana_client/cli/test.py create mode 100644 tests/test_cli_test.py diff --git a/AUTHORS.md b/AUTHORS.md index c8a30603..29c3f245 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -3,6 +3,7 @@ The list of contributors in alphabetical order: - [Adelina Lintuluoto](https://orcid.org/0000-0002-0726-1452) +- [Alastair Lyall](https://orcid.org/0009-0000-4955-8935) - [Anton Khodak](https://orcid.org/0000-0003-3263-4553) - [Audrius Mecionis](https://orcid.org/0000-0002-3759-1663) - [Camila Diaz](https://orcid.org/0000-0001-5543-797X) diff --git a/docs/cmd_list.txt b/docs/cmd_list.txt index 0b9a7d36..2131cd7e 100644 --- a/docs/cmd_list.txt +++ b/docs/cmd_list.txt @@ -55,3 +55,6 @@ Secret management commands: secrets-add Add secrets from literal string or from file. secrets-delete Delete user secrets by name. secrets-list List user secrets. + +Workflow run test commands: + test Test workflow execution, based on a given Gherkin file. diff --git a/reana_client/cli/__init__.py b/reana_client/cli/__init__.py index 2b46cf0f..6a940e77 100644 --- a/reana_client/cli/__init__.py +++ b/reana_client/cli/__init__.py @@ -11,7 +11,15 @@ import sys import click -from reana_client.cli import files, ping, quotas, retention_rules, secrets, workflow +from reana_client.cli import ( + files, + ping, + quotas, + retention_rules, + secrets, + test, + workflow, +) from reana_client.utils import get_api_url from urllib3 import disable_warnings @@ -45,6 +53,7 @@ class ReanaCLI(click.Group): files.files_group, retention_rules.retention_rules_group, secrets.secrets_group, + test.test_group, ] def __init__(self, name=None, commands=None, **attrs): diff --git a/reana_client/cli/files.py b/reana_client/cli/files.py index 50ff6e57..3acc0a85 100644 --- a/reana_client/cli/files.py +++ b/reana_client/cli/files.py @@ -326,10 +326,11 @@ def upload_files( # noqa: C901 msg_type="error", ) sys.exit(1) - + filenames = [] + if reana_spec.get("tests"): + for f in reana_spec["tests"].get("files") or []: + filenames.append(os.path.join(os.getcwd(), f)) if reana_spec.get("inputs"): - filenames = [] - # collect all files in input.files for f in reana_spec["inputs"].get("files") or []: # check for directories in files @@ -340,7 +341,6 @@ def upload_files( # noqa: C901 ) sys.exit(1) filenames.append(os.path.join(os.getcwd(), f)) - # collect all files in input.directories files_from_directories = [] directories = reana_spec["inputs"].get("directories") or [] diff --git a/reana_client/cli/test.py b/reana_client/cli/test.py new file mode 100644 index 00000000..e6676a09 --- /dev/null +++ b/reana_client/cli/test.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- +# +# This file is part of REANA. +# Copyright (C) 2024 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. +"""REANA client test commands.""" + +import sys +import click +import logging +import traceback +import time +from reana_client.cli.utils import add_access_token_options, check_connection +from reana_commons.gherkin_parser.parser import ( + parse_and_run_tests, + AnalysisTestStatus, +) +from reana_commons.gherkin_parser.data_fetcher import DataFetcherBase +from reana_commons.gherkin_parser.errors import FeatureFileError +from reana_client.printer import display_message +from reana_client.api.client import ( + list_files, + get_workflow_disk_usage, + get_workflow_logs, + get_workflow_status, + get_workflow_specification, + download_file, +) +from reana_client.cli.utils import add_workflow_option + + +class DataFetcherClient(DataFetcherBase): + """Implementation of the DataFetcherBase using reana_client.api.client methods.""" + + def __init__(self, access_token): + """Initialize DataFetcherClient with access token.""" + self.access_token = access_token + + def list_files(self, workflow, file_name=None, page=None, size=None, search=None): + """Return the list of files for a given workflow workspace.""" + return list_files(workflow, self.access_token, file_name, page, size, search) + + def get_workflow_disk_usage(self, workflow, parameters): + """Display disk usage workflow.""" + return get_workflow_disk_usage(workflow, parameters, self.access_token) + + def get_workflow_logs(self, workflow, steps=None, page=None, size=None): + """Get logs from a workflow engine, use existing API function.""" + return get_workflow_logs(workflow, self.access_token, steps, page, size) + + def get_workflow_status(self, workflow): + """Get of a previously created workflow.""" + return get_workflow_status(workflow, self.access_token) + + def get_workflow_specification(self, workflow): + """Get specification of previously created workflow.""" + return get_workflow_specification(workflow, self.access_token) + + def download_file(self, workflow, file_path): + """Download the requested file if it exists.""" + return download_file(workflow, file_path, self.access_token) + + +@click.group(help="Workflow run test commands") +def test_group(): + """Workflow run test commands.""" + + +@test_group.command("test") +@click.option( + "-n", + "--test-files", + multiple=True, + default=None, + help="Gherkin file for testing properties of a workflow execution. Overrides files in reana.yaml if provided.", +) +@click.pass_context +@add_access_token_options +@check_connection +@add_workflow_option +def test(ctx, workflow, test_files, access_token): + r""" + Test workflow execution, based on a given Gherkin file. + + Gherkin files can be specified in the reana specification file (reana.yaml), + or by using the ``-n`` option. + + The ``test`` command allows for testing of a workflow execution, + by assessing whether it meets certain properties specified in a + chosen gherkin file. + + Example: + $ reana-client test -w myanalysis -n test_analysis.feature + $ reana-client test -w myanalysis + $ reana-client test -w myanalysis -n test1.feature -n test2.feature + """ + start_time = time.time() + try: + workflow_status = get_workflow_status( + workflow=workflow, access_token=access_token + ) + status = workflow_status["status"] + workflow_name = workflow_status["name"] + except Exception as e: + logging.debug(traceback.format_exc()) + logging.debug(str(e)) + display_message(f"Could not find workflow ``{workflow}``.", msg_type="error") + sys.exit(1) + + if status != "finished": + display_message( + f"``{workflow}`` is {status}. It must be finished to run tests.", + msg_type="error", + ) + sys.exit(1) + + if not test_files: + reana_specification = get_workflow_specification(workflow, access_token) + try: + test_files = reana_specification["specification"]["tests"]["files"] + except KeyError as e: + logging.debug(traceback.format_exc()) + logging.debug(str(e)) + display_message( + "No test files specified in reana.yaml and no -n option provided.", + msg_type="error", + ) + sys.exit(1) + + passed = 0 + failed = 0 + data_fetcher = DataFetcherClient(access_token) + for test_file in test_files: + click.echo("\n", nl=False) + display_message(f'Testing file "{test_file}"...', msg_type="info") + try: + results = parse_and_run_tests(test_file, workflow_name, data_fetcher) + except FileNotFoundError as e: + logging.debug(traceback.format_exc()) + logging.debug(str(e)) + display_message(f"Test file {test_file} not found.", msg_type="error") + sys.exit(1) + except FeatureFileError as e: + logging.debug(traceback.format_exc()) + logging.debug(str(e)) + display_message( + f"Error parsing feature file {test_file}: {e}", msg_type="error" + ) + sys.exit(1) + + for scenario in results[1]: + if scenario.result == AnalysisTestStatus.failed: + display_message( + f'Scenario "{scenario.scenario}"', msg_type="error", indented=True + ) + failed += 1 + else: + display_message( + f'Scenario "{scenario.scenario}"', msg_type="success", indented=True + ) + passed += 1 + + end_time = time.time() + duration = round(end_time - start_time) + click.echo(f"\n{passed} passed, {failed} failed in {duration}s") + if failed > 0: + sys.exit(1) diff --git a/tests/test_cli_files.py b/tests/test_cli_files.py index 8e22e8ab..fbb182a7 100644 --- a/tests/test_cli_files.py +++ b/tests/test_cli_files.py @@ -240,6 +240,35 @@ def test_upload_file(create_yaml_workflow_schema): assert message in result.output +def test_upload_file_with_test_files_from_spec( + get_workflow_specification_with_directory, +): + """Test upload file with test files from the specification, not from the command line.""" + reana_token = "000000" + file = "upload-this-test.feature" + env = {"REANA_SERVER_URL": "http://localhost"} + runner = CliRunner(env=env) + + with patch( + "reana_client.api.client.get_workflow_specification" + ) as mock_specification, patch("reana_client.api.client.requests.post"): + with runner.isolated_filesystem(): + with open(file, "w") as f: + f.write("Scenario: Test scenario") + + get_workflow_specification_with_directory["specification"]["tests"] = { + "files": [file] + } + mock_specification.return_value = get_workflow_specification_with_directory + result = runner.invoke( + cli, ["upload", "-t", reana_token, "--workflow", "test-workflow.1"] + ) + assert result.exit_code == 0 + assert ( + "upload-this-test.feature was successfully uploaded." in result.output + ) + + def test_upload_file_respect_gitignore( get_workflow_specification_with_directory, ): diff --git a/tests/test_cli_test.py b/tests/test_cli_test.py new file mode 100644 index 00000000..cd623511 --- /dev/null +++ b/tests/test_cli_test.py @@ -0,0 +1,391 @@ +# -*- coding: utf-8 -*- +# +# This file is part of REANA. +# Copyright (C) 2024 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""REANA client test tests.""" + +from click.testing import CliRunner +from reana_client.cli import cli +from unittest.mock import patch + +from reana_commons.gherkin_parser.parser import AnalysisTestStatus, TestResult +from reana_commons.gherkin_parser.errors import FeatureFileError +from dataclasses import replace + +passed_test = TestResult( + scenario="scenario1", + error_log=None, + result=AnalysisTestStatus.passed, + failed_testcase=None, + feature="Run Duration", + checked_at="2024-01-01T00:00:00.000000", +) + +failed_test = TestResult( + scenario="scenario2", + error_log="Test designed to fail", + result=AnalysisTestStatus.failed, + failed_testcase="Scenario to fail", + feature="Run Duration", + checked_at="2024-01-01T00:00:00.000000", +) + + +def test_test_workflow_not_found(): + """Test test command when workflow is not found.""" + env = {"REANA_SERVER_URL": "localhost"} + runner = CliRunner(env=env) + with runner.isolated_filesystem(): + result = runner.invoke( + cli, + [ + "test", + "-w", + "myanalysis", + "-n", + "test_analysis.feature", + "-t", + "000000", + ], + ) + assert "Could not find workflow ``myanalysis``." in result.output + assert result.exit_code == 1 + + +@patch( + "reana_client.cli.test.get_workflow_status", + return_value={"status": "running", "name": "myanalysis"}, +) +def test_test_workflow_not_finished(mock_get_workflow_status): + """Test test command when workflow is not finished.""" + env = {"REANA_SERVER_URL": "localhost"} + runner = CliRunner(env=env) + with runner.isolation(): + result = runner.invoke( + cli, + [ + "test", + "-w", + "myanalysis", + "-n", + "test_analysis.feature", + "-t", + "000000", + ], + ) + assert ( + "``myanalysis`` is running. It must be finished to run tests." + in result.output + ) + assert result.exit_code == 1 + + +@patch( + "reana_client.cli.test.get_workflow_status", + return_value={"status": "deleted", "name": "myanalysis"}, +) +def test_test_workflow_deleted(mock_get_workflow_status): + """Test test command when workflow is deleted.""" + env = {"REANA_SERVER_URL": "localhost"} + runner = CliRunner(env=env) + with runner.isolation(): + result = runner.invoke( + cli, + ["test", "-w", "myanalysis", "-n", "test_analysis.feature", "-t", "000000"], + ) + assert ( + "``myanalysis`` is deleted. It must be finished to run tests." + in result.output + ) + assert result.exit_code == 1 + + +@patch( + "reana_client.cli.test.get_workflow_status", + return_value={"status": "finished", "name": "myanalysis"}, +) +@patch( + "reana_client.cli.test.get_workflow_specification", + return_value={"specification": {"inputs": {"directories": ["data"]}}}, +) +def test_test_no_test_files(mock_get_workflow_status, mock_get_workflow_specification): + """Test test command when no test files are specified.""" + env = {"REANA_SERVER_URL": "localhost"} + runner = CliRunner(env=env) + with runner.isolation(): + result = runner.invoke(cli, ["test", "-w", "myanalysis", "-t", "000000"]) + assert ( + "No test files specified in reana.yaml and no -n option provided." + in result.output + ) + assert result.exit_code == 1 + + +@patch( + "reana_client.cli.test.get_workflow_status", + return_value={"status": "finished", "name": "myanalysis"}, +) +@patch( + "reana_client.cli.test.get_workflow_specification", + return_value={"specification": {"inputs": {"directories": ["data"]}}}, +) +def test_test_no_test_files_with_test_file_option( + mock_get_workflow_status, mock_get_workflow_specification +): + """Test test command when no test files are specified in reana.yml and when the test file option is provided.""" + env = {"REANA_SERVER_URL": "localhost"} + runner = CliRunner(env=env) + with runner.isolation(): + result = runner.invoke( + cli, + [ + "test", + "-w", + "myanalysis", + "-t", + "000000", + "-n", + "test_analysis.feature", + ], + ) + assert 'Testing file "test_analysis.feature"' in result.output + + +@patch( + "reana_client.cli.test.get_workflow_status", + return_value={"status": "finished", "name": "myanalysis"}, +) +@patch( + "reana_client.cli.test.get_workflow_specification", + return_value={ + "specification": { + "tests": {"files": ["test_analysis.feature", "test_analysis2.feature"]} + } + }, +) +def test_test_multiple_test_files_with_test_file_option( + mock_get_workflow_status, mock_get_workflow_specification +): + """Test test command when multiple test files are specified in reana.yml and test file option is provided. + In this case, the test-file option should be used instead of the test files specified in reana.yml. + """ + env = {"REANA_SERVER_URL": "localhost"} + runner = CliRunner(env=env) + with runner.isolation(): + result = runner.invoke( + cli, + [ + "test", + "-w", + "myanalysis", + "-t", + "000000", + "-n", + "use_this.feature", + ], + ) + assert 'Testing file "use_this.feature"' in result.output + assert 'Testing file "test_analysis.feature"' not in result.output + assert 'Testing file "test_analysis2.feature"' not in result.output + + +@patch( + "reana_client.cli.test.get_workflow_status", + return_value={"status": "finished", "name": "myanalysis"}, +) +@patch( + "reana_client.cli.test.get_workflow_specification", + return_value={ + "specification": {"tests": {"files": ["use-me.feature", "me-too.feature"]}} + }, +) +@patch( + "reana_client.cli.test.parse_and_run_tests", + return_value=( + "myanalysis", + [passed_test], + ), +) +def test_test_files_from_spec( + mock_get_workflow_status, get_workflow_specification, mock_parse_and_run_tests +): + """Test test command when test files are specified in reana.yml.""" + env = {"REANA_SERVER_URL": "localhost"} + runner = CliRunner(env=env) + with runner.isolation(): + result = runner.invoke( + cli, + ["test", "-w", "myanalysis", "-t", "000000"], + ) + assert 'Testing file "use-me.feature"' in result.output + assert 'Testing file "me-too.feature"' in result.output + + +@patch( + "reana_client.cli.test.get_workflow_status", + return_value={"status": "finished", "name": "myanalysis"}, +) +@patch( + "reana_client.cli.test.parse_and_run_tests", + side_effect=FeatureFileError, +) +def test_test_parser_error(mock_get_workflow_status, mock_parse_and_run_tests): + """Test test command when parser error occurs.""" + env = {"REANA_SERVER_URL": "localhost"} + runner = CliRunner(env=env) + with runner.isolation(): + result = runner.invoke( + cli, + [ + "test", + "-w", + "myanalysis", + "-t", + "000000", + "-n", + "test_analysis.feature", + ], + ) + assert "Error parsing feature file test_analysis.feature" in result.output + assert result.exit_code == 1 + + +@patch( + "reana_client.cli.test.get_workflow_status", + return_value={"status": "finished", "name": "myanalysis"}, +) +@patch( + "reana_client.cli.test.parse_and_run_tests", + side_effect=FileNotFoundError, +) +def test_test_feature_file_not_found( + mock_get_workflow_status, mock_parse_and_run_tests +): + """Test test command when a feature file is not found.""" + env = {"REANA_SERVER_URL": "localhost"} + runner = CliRunner(env=env) + with runner.isolation(): + result = runner.invoke( + cli, + [ + "test", + "-w", + "myanalysis", + "-t", + "000000", + "-n", + "test_analysis.feature", + ], + ) + assert "Test file test_analysis.feature not found." in result.output + assert result.exit_code == 1 + + +@patch( + "reana_client.cli.test.get_workflow_status", + return_value={"status": "finished", "name": "myanalysis"}, +) +@patch( + "reana_client.cli.test.parse_and_run_tests", + return_value=( + "myanalysis", + [passed_test], + ), +) +def test_test_multiple_test_files(mock_workflow_status, mock_parse_and_run_tests): + """Test test command when multiple test files are specified.""" + env = {"REANA_SERVER_URL": "localhost"} + runner = CliRunner(env=env) + with runner.isolation(): + result = runner.invoke( + cli, + [ + "test", + "-w", + "myanalysis", + "-t", + "000000", + "-n", + "test_analysis.feature", + "-n", + "test_analysis2.feature", + ], + ) + assert 'Testing file "test_analysis.feature"' in result.output + assert 'Testing file "test_analysis2.feature"' in result.output + + +@patch( + "reana_client.cli.test.get_workflow_status", + return_value={"status": "finished", "name": "myanalysis"}, +) +@patch( + "reana_client.cli.test.parse_and_run_tests", + return_value=( + "myanalysis", + [passed_test, replace(passed_test, scenario="scenario2")], + ), +) +def test_test_all_scenarios_pass(mock_workflow_status, mock_parse_and_run_tests): + """Test test command when tests pass.""" + env = {"REANA_SERVER_URL": "localhost"} + runner = CliRunner(env=env) + with runner.isolated_filesystem(): + result = runner.invoke( + cli, + ["test", "-w", "myanalysis", "-n", "test_analysis.feature", "-t", "000000"], + ) + assert "SUCCESS" in result.output and "ERROR" not in result.output + assert result.exit_code == 0 + + +@patch( + "reana_client.cli.test.get_workflow_status", + return_value={"status": "finished", "name": "myanalysis"}, +) +@patch( + "reana_client.cli.test.parse_and_run_tests", + return_value=( + "myanalysis", + [replace(failed_test, scenario="scenario1"), failed_test], + ), +) +def test_test_all_scenarios_fail(mock_workflow_status, mock_parse_and_run_tests): + """Test test command when tests fail.""" + env = {"REANA_SERVER_URL": "localhost"} + runner = CliRunner(env=env) + with runner.isolation(): + result = runner.invoke( + cli, + ["test", "-w", "myanalysis", "-n", "test_analysis.feature", "-t", "000000"], + ) + assert "ERROR" in result.output and "SUCCESS" not in result.output + assert result.exit_code == 1 + + +@patch( + "reana_client.cli.test.get_workflow_status", + return_value={"status": "finished", "name": "myanalysis"}, +) +@patch( + "reana_client.cli.test.parse_and_run_tests", + return_value=( + "myanalysis", + [passed_test, failed_test], + ), +) +def test_test_some_scenarios_pass(mock_workflow_status, mock_parse_and_run_tests): + """Test test command when some tests pass and some fail.""" + env = {"REANA_SERVER_URL": "localhost"} + runner = CliRunner(env=env) + with runner.isolated_filesystem(): + result = runner.invoke( + cli, + ["test", "-w", "myanalysis", "-n", "test_analysis.feature", "-t", "000000"], + ) + assert "SUCCESS" in result.output and "ERROR" in result.output + assert result.exit_code == 1