diff --git a/tests/system_tests/test_data/__init__.py b/tests/system_tests/__init__.py similarity index 100% rename from tests/system_tests/test_data/__init__.py rename to tests/system_tests/__init__.py diff --git a/tests/system_tests/common.py b/tests/system_tests/common.py new file mode 100644 index 000000000..089907090 --- /dev/null +++ b/tests/system_tests/common.py @@ -0,0 +1,81 @@ +import os + +import pytest + +from blueapi.client.client import BlueapiClient +from blueapi.client.event_bus import AnyEvent +from blueapi.core.bluesky_types import DataEvent +from blueapi.worker.event import TaskStatus, WorkerEvent, WorkerState + +BEAMLINE = os.environ.get("BEAMLINE", "") + +DISABLE_SIDE_EFFECTS = bool(os.environ.get("DISABLE_SIDE_EFFECTS", 0)) +DISABLE_SIDE_EFFECTS_MESSAGE = """ + This test would cause side effects on the beamline, it has been disabled + so as not to interfere with operation. To run tests that may interfere with + the beamline export DISABLE_SIDE_EFFECTS=0 + """ +disable_side_effects = pytest.mark.skipif( + DISABLE_SIDE_EFFECTS, reason=DISABLE_SIDE_EFFECTS_MESSAGE +) + +REQUIRES_AUTH = bool(os.environ.get("REQUIRES_AUTH", 0)) +REQUIRES_AUTH_MESSAGE = """ +Authentication credentials are required to run this test. +The test has been skipped because authentication is currently disabled. +For more details, see: https://github.com/DiamondLightSource/blueapi/issues/676. +To enable and execute these tests, set `REQUIRES_AUTH=1` and provide valid credentials. +""" +requires_auth = pytest.mark.skipif(not REQUIRES_AUTH, reason=REQUIRES_AUTH_MESSAGE) + +# Mark for beamline-specific tests +BEAMLINE_SPECIFIC_MESSAGE = """ + This test is beamline-specific but no beamline has been set. + Set the BEAMLINE environment variable to enable this test. + """ +beamline_specific_test = pytest.mark.skipif( + not BEAMLINE, reason=BEAMLINE_SPECIFIC_MESSAGE +) + + +def clean_existing_tasks(client: BlueapiClient) -> None: + for task in client.get_all_tasks().tasks: + client.clear_task(task.task_id) + + +def check_all_events(all_events: list[AnyEvent]): + assert isinstance(all_events[0], WorkerEvent) and all_events[0].task_status + task_id = all_events[0].task_status.task_id + # First event is WorkerEvent + assert all_events[0] == WorkerEvent( + state=WorkerState.RUNNING, + task_status=TaskStatus( + task_id=task_id, + task_complete=False, + task_failed=False, + ), + ) + + assert all( + isinstance(event, DataEvent) for event in all_events[1:-2] + ), "Middle elements must be DataEvents." + + # Last 2 events are WorkerEvent + assert all_events[-2:] == [ + WorkerEvent( + state=WorkerState.IDLE, + task_status=TaskStatus( + task_id=task_id, + task_complete=False, + task_failed=False, + ), + ), + WorkerEvent( + state=WorkerState.IDLE, + task_status=TaskStatus( + task_id=task_id, + task_complete=True, + task_failed=False, + ), + ), + ] diff --git a/tests/system_tests/conftest.py b/tests/system_tests/conftest.py new file mode 100644 index 000000000..d7b5e14da --- /dev/null +++ b/tests/system_tests/conftest.py @@ -0,0 +1,140 @@ +import inspect +from pathlib import Path + +import pytest +from bluesky_stomp.models import BasicAuthentication +from pydantic import TypeAdapter + +from blueapi.client.client import BlueapiClient +from blueapi.config import ApplicationConfig, RestConfig, StompConfig +from blueapi.service.model import DeviceResponse, PlanResponse +from blueapi.worker.task import Task +from tests.system_tests.utils import BEAMLINE + +# Step 1: Ensure a message bus that supports stomp is running and available: +# src/script/start_rabbitmq.sh +# +# Step 2: Start the BlueAPI server with valid configuration: +# blueapi -c tests/unit_tests/example_yaml/valid_stomp_config.yaml serve +# +# Step 3: Run the system tests using tox: +# tox -e system-test + +_DATA_PATH = Path(__file__).parent / "expected_data" + + +@pytest.fixture +def expected_plans() -> PlanResponse: + file_name = "plans.json" if not BEAMLINE else f"plans_{BEAMLINE}.json" + return TypeAdapter(PlanResponse).validate_json((_DATA_PATH / file_name).read_text()) + + +@pytest.fixture +def expected_devices() -> DeviceResponse: + file_name = "devices.json" if not BEAMLINE else f"devices_{BEAMLINE}.json" + return TypeAdapter(DeviceResponse).validate_json( + (_DATA_PATH / file_name).read_text() + ) + + +@pytest.fixture +def blueapi_client_get_methods() -> list[str]: + # Get a list of methods that take only one argument (self) + # This will currently return + # ['get_plans', 'get_devices', 'get_state', 'get_all_tasks', + # 'get_active_task','get_environment','resume', 'stop','get_oidc_config'] + return [ + method + for method in BlueapiClient.__dict__ + if callable(getattr(BlueapiClient, method)) + and not method.startswith("__") + and len(inspect.signature(getattr(BlueapiClient, method)).parameters) == 1 + and "self" in inspect.signature(getattr(BlueapiClient, method)).parameters + ] + + +@pytest.fixture +def task_definition() -> dict[str, Task]: + return { + "simple_plan": Task(name="sleep", params={"time": 0.0}), + "long_plan": Task(name="sleep", params={"time": 1.0}), + "spec_scan": Task( + name="spec_scan", + params={ + "detectors": ["det"], + "spec": { + "axis": "sample_stage.x", + "start": 1.0, + "stop": 10.0, + "num": 10, + "type": "Line", + }, + }, + ), + } + + +@pytest.fixture +def config() -> ApplicationConfig: + if BEAMLINE == "p46": + return ApplicationConfig( + api=RestConfig( + host="p46-blueapi.diamond.ac.uk", port=443, protocol="https" + ), + ) + else: + return ApplicationConfig() + + +@pytest.fixture +def config_without_auth(tmp_path: Path) -> ApplicationConfig: + if BEAMLINE == "p46": + return ApplicationConfig( + stomp=StompConfig( + host="172.23.168.198", + auth=BasicAuthentication(username="guest", password="guest"), # type: ignore + ), + api=RestConfig( + host="p46-blueapi.diamond.ac.uk", port=443, protocol="https" + ), + auth_token_path=tmp_path, + ) + else: + return ApplicationConfig(auth_token_path=tmp_path) + + +@pytest.fixture +def config_with_stomp() -> ApplicationConfig: + if BEAMLINE == "p46": + return ApplicationConfig( + stomp=StompConfig( + host="172.23.168.198", + auth=BasicAuthentication(username="p46", password="64p"), # type: ignore + ), + api=RestConfig( + host="p46-blueapi.diamond.ac.uk", port=443, protocol="https" + ), + ) + else: + return ApplicationConfig( + stomp=StompConfig( + host="localhost", + auth=BasicAuthentication(username="guest", password="guest"), # type: ignore + ) + ) + + +# This client will have auth enabled if it finds cached valid token +@pytest.fixture +def client(config) -> BlueapiClient: + return BlueapiClient.from_config(config=config) + + +@pytest.fixture +def client_without_auth(config_without_auth) -> BlueapiClient: + return BlueapiClient.from_config(config=config_without_auth) + + +@pytest.fixture +def client_with_stomp(config_with_stomp) -> BlueapiClient: + return BlueapiClient.from_config(config=config_with_stomp) diff --git a/tests/system_tests/devices.json b/tests/system_tests/expected_data/devices.json similarity index 100% rename from tests/system_tests/devices.json rename to tests/system_tests/expected_data/devices.json diff --git a/tests/system_tests/plans.json b/tests/system_tests/expected_data/plans.json similarity index 100% rename from tests/system_tests/plans.json rename to tests/system_tests/expected_data/plans.json diff --git a/tests/system_tests/test_blueapi_system.py b/tests/system_tests/test_blueapi_system.py index 477a637c3..21fe5a16c 100644 --- a/tests/system_tests/test_blueapi_system.py +++ b/tests/system_tests/test_blueapi_system.py @@ -1,9 +1,6 @@ -import inspect import time -from pathlib import Path import pytest -from bluesky_stomp.models import BasicAuthentication from pydantic import TypeAdapter from blueapi.client.client import ( @@ -12,9 +9,7 @@ ) from blueapi.client.event_bus import AnyEvent from blueapi.config import ( - ApplicationConfig, OIDCConfig, - StompConfig, ) from blueapi.service.model import ( DeviceResponse, @@ -26,81 +21,14 @@ from blueapi.worker.event import TaskStatus, WorkerEvent, WorkerState from blueapi.worker.task import Task from blueapi.worker.task_worker import TrackableTask - -_SIMPLE_TASK = Task(name="sleep", params={"time": 0.0}) -_LONG_TASK = Task(name="sleep", params={"time": 1.0}) - -_DATA_PATH = Path(__file__).parent - -# Step 1: Ensure a message bus that supports stomp is running and available: -# src/script/start_rabbitmq.sh -# -# Step 2: Start the BlueAPI server with valid configuration: -# blueapi -c tests/unit_tests/example_yaml/valid_stomp_config.yaml serve -# -# Step 3: Run the system tests using tox: -# tox -e system-test - - -@pytest.fixture -def client_without_auth(tmp_path) -> BlueapiClient: - return BlueapiClient.from_config(config=ApplicationConfig(auth_token_path=tmp_path)) - - -@pytest.fixture -def client_with_stomp() -> BlueapiClient: - return BlueapiClient.from_config( - config=ApplicationConfig( - stomp=StompConfig( - auth=BasicAuthentication(username="guest", password="guest") # type: ignore - ) - ) - ) - - -# This client will have auth enabled if it finds cached valid token -@pytest.fixture -def client() -> BlueapiClient: - return BlueapiClient.from_config(config=ApplicationConfig()) - - -@pytest.fixture -def expected_plans() -> PlanResponse: - return TypeAdapter(PlanResponse).validate_json( - (_DATA_PATH / "plans.json").read_text() - ) - - -@pytest.fixture -def expected_devices() -> DeviceResponse: - return TypeAdapter(DeviceResponse).validate_json( - (_DATA_PATH / "devices.json").read_text() - ) - - -@pytest.fixture -def blueapi_client_get_methods() -> list[str]: - # Get a list of methods that take only one argument (self) - # This will currently return - # ['get_plans', 'get_devices', 'get_state', 'get_all_tasks', - # 'get_active_task','get_environment','resume', 'stop','get_oidc_config'] - return [ - method - for method in BlueapiClient.__dict__ - if callable(getattr(BlueapiClient, method)) - and not method.startswith("__") - and len(inspect.signature(getattr(BlueapiClient, method)).parameters) == 1 - and "self" in inspect.signature(getattr(BlueapiClient, method)).parameters - ] - - -@pytest.fixture(autouse=True) -def clean_existing_tasks(client: BlueapiClient): - for task in client.get_all_tasks().tasks: - client.clear_task(task.task_id) - yield +from tests.system_tests.common import ( + clean_existing_tasks, + disable_side_effects, + requires_auth, +) +@requires_auth def test_cannot_access_endpoints( client_without_auth: BlueapiClient, blueapi_client_get_methods: list[str] ): @@ -112,6 +40,7 @@ def test_cannot_access_endpoints( getattr(client_without_auth, get_method)() +@requires_auth def test_can_get_oidc_config_without_auth(client_without_auth: BlueapiClient): assert client_without_auth.get_oidc_config() == OIDCConfig( well_known_url="https://example.com/realms/master/.well-known/openid-configuration", @@ -155,8 +84,11 @@ def test_get_non_existent_device(client: BlueapiClient): client.get_device("Not exists") -def test_create_task_and_delete_task_by_id(client: BlueapiClient): - create_task = client.create_task(_SIMPLE_TASK) +@disable_side_effects +def test_create_task_and_delete_task_by_id( + client: BlueapiClient, task_definition: dict[str, Task] +): + create_task = client.create_task(task_definition["simple_plan"]) client.clear_task(create_task.task_id) @@ -165,9 +97,11 @@ def test_create_task_validation_error(client: BlueapiClient): client.create_task(Task(name="Not-exists", params={"Not-exists": 0.0})) -def test_get_all_tasks(client: BlueapiClient): +@disable_side_effects +def test_get_all_tasks(client: BlueapiClient, task_definition: dict[str, Task]): + clean_existing_tasks(client) created_tasks: list[TaskResponse] = [] - for task in [_SIMPLE_TASK, _LONG_TASK]: + for task in [task_definition["simple_plan"], task_definition["long_plan"]]: created_task = client.create_task(task) created_tasks.append(created_task) task_ids = [task.task_id for task in created_tasks] @@ -181,8 +115,9 @@ def test_get_all_tasks(client: BlueapiClient): client.clear_task(task_id) -def test_get_task_by_id(client: BlueapiClient): - created_task = client.create_task(_SIMPLE_TASK) +@disable_side_effects +def test_get_task_by_id(client: BlueapiClient, task_definition: dict[str, Task]): + created_task = client.create_task(task_definition["simple_plan"]) get_task = client.get_task(created_task.task_id) assert ( @@ -205,17 +140,21 @@ def test_delete_non_existent_task(client: BlueapiClient): client.clear_task("Not-exists") -def test_put_worker_task(client: BlueapiClient): - created_task = client.create_task(_SIMPLE_TASK) +@disable_side_effects +def test_put_worker_task(client: BlueapiClient, task_definition: dict[str, Task]): + created_task = client.create_task(task_definition["simple_plan"]) client.start_task(WorkerTask(task_id=created_task.task_id)) active_task = client.get_active_task() assert active_task.task_id == created_task.task_id client.clear_task(created_task.task_id) -def test_put_worker_task_fails_if_not_idle(client: BlueapiClient): - small_task = client.create_task(_SIMPLE_TASK) - long_task = client.create_task(_LONG_TASK) +@disable_side_effects +def test_put_worker_task_fails_if_not_idle( + client: BlueapiClient, task_definition: dict[str, Task] +): + small_task = client.create_task(task_definition["simple_plan"]) + long_task = client.create_task(task_definition["long_plan"]) client.start_task(WorkerTask(task_id=long_task.task_id)) active_task = client.get_active_task() @@ -233,6 +172,7 @@ def test_get_worker_state(client: BlueapiClient): assert client.get_state() == WorkerState.IDLE +@disable_side_effects def test_set_state_transition_error(client: BlueapiClient): with pytest.raises(BlueskyRemoteControlError) as exception: client.resume() @@ -242,9 +182,11 @@ def test_set_state_transition_error(client: BlueapiClient): assert "" in str(exception) -def test_get_task_by_status(client: BlueapiClient): - task_1 = client.create_task(_SIMPLE_TASK) - task_2 = client.create_task(_SIMPLE_TASK) +@disable_side_effects +def test_get_task_by_status(client: BlueapiClient, task_definition: dict[str, Task]): + clean_existing_tasks(client) + task_1 = client.create_task(task_definition["simple_plan"]) + task_2 = client.create_task(task_definition["simple_plan"]) task_by_pending = client.get_all_tasks() # https://github.com/DiamondLightSource/blueapi/issues/680 # task_by_pending = client.get_tasks_by_status(TaskStatusEnum.PENDING) @@ -273,13 +215,16 @@ def test_get_task_by_status(client: BlueapiClient): client.clear_task(task_id=task_2.task_id) -def test_progress_with_stomp(client_with_stomp: BlueapiClient): +@disable_side_effects +def test_progress_with_stomp( + client_with_stomp: BlueapiClient, task_definition: dict[str, Task] +): all_events: list[AnyEvent] = [] def on_event(event: AnyEvent): all_events.append(event) - client_with_stomp.run_task(_SIMPLE_TASK, on_event=on_event) + client_with_stomp.run_task(task_definition["simple_plan"], on_event=on_event) assert isinstance(all_events[0], WorkerEvent) and all_events[0].task_status task_id = all_events[0].task_status.task_id assert all_events == [ @@ -314,6 +259,7 @@ def test_get_current_state_of_environment(client: BlueapiClient): assert client.get_environment() == EnvironmentResponse(initialized=True) +@disable_side_effects def test_delete_current_environment(client: BlueapiClient): client.reload_environment() assert client.get_environment() == EnvironmentResponse(initialized=True) diff --git a/tests/system_tests/test_data/data_profiles.py b/tests/system_tests/test_data/data_profiles.py deleted file mode 100644 index 56a4879b2..000000000 --- a/tests/system_tests/test_data/data_profiles.py +++ /dev/null @@ -1,62 +0,0 @@ -from collections.abc import Mapping -from dataclasses import dataclass -from pprint import pprint -from typing import Any - -import h5py as h5 -import numpy as np - - -@dataclass -class ExpectedDataset: - shape: tuple[int, ...] - value: np.ndarray | None = None - - -def validate_data( - actual_data: h5.HLObject, - expected: ExpectedDataset | Mapping[str, Any], - total: bool = False, -) -> None: - print(f"Validating {actual_data} against the following tree:") - pprint(expected) - - if isinstance(expected, dict): - if total: - assert set(expected.keys()) == set(actual_data.keys()) # type: ignore - else: - assert set(expected.keys()).issubset(set(actual_data.keys())) # type: ignore - for name, dataset_or_group in actual_data.items(): # type: ignore - child = expected.get(name) - if name is not None: - validate_data(dataset_or_group, child) # type: ignore - elif total: - raise AssertionError( - f"{actual_data} has a child called {name} that is not expected" - ) - elif isinstance(expected, ExpectedDataset): - name = actual_data.name - _assert_is_dataset(actual_data, expected) - assert ( - actual_data.shape == expected.shape # type: ignore - ), f"{name}: {actual_data.shape} should be {expected.shape}" # type: ignore - if expected.value is not None: - arr = np.array(actual_data) - assert np.equal( - arr, expected.value - ), f"{name}: {arr} should be {expected.value}" - - -def _assert_is_dataset( - maybe_dataset: h5.HLObject, expected_dataset: ExpectedDataset -) -> None: - valid_types = [h5.Dataset] - name = maybe_dataset.name - for t in valid_types: - if isinstance(maybe_dataset, t): - return - raise AssertionError( - f"{maybe_dataset} of type {type(maybe_dataset)} is" - f" not one of the valid dataset types: {valid_types}." - f" The following was expected: {name}: {expected_dataset}" - ) diff --git a/tests/system_tests/test_data/eventual_data.py b/tests/system_tests/test_data/eventual_data.py deleted file mode 100644 index a22fe04f2..000000000 --- a/tests/system_tests/test_data/eventual_data.py +++ /dev/null @@ -1,42 +0,0 @@ -import time -from contextlib import contextmanager -from pathlib import Path - -import h5py as h5 - -# Currently the nexus writer has no way of letting us know when it is done with a file. -# This repeated try will have to do. - - -@contextmanager -def hdf5_file_with_backoff( - path: Path, - max_attempts: int = 5, - interval: float = 0.5, -): - f = _open_file_with_backoff( - path, - max_attempts, - interval, - ) - try: - yield f - finally: - f.close() - - -def _open_file_with_backoff( - path: Path, - max_attempts: int = 5, - interval: float = 0.5, -) -> h5.File: - while max_attempts > 0: - try: - return h5.File(str(path)) - except BlockingIOError as ex: - if max_attempts > 1: - max_attempts -= 1 - time.sleep(0.5) - else: - raise ex - raise Exception("Failed to open file") diff --git a/tests/system_tests/test_plans.py b/tests/system_tests/test_plans.py index ecd40c015..26547986d 100644 --- a/tests/system_tests/test_plans.py +++ b/tests/system_tests/test_plans.py @@ -1,122 +1,33 @@ -import os -from pathlib import Path - import pytest -from bluesky_stomp.models import BasicAuthentication from blueapi.client.client import BlueapiClient from blueapi.client.event_bus import AnyEvent -from blueapi.config import ApplicationConfig, RestConfig, StompConfig -from blueapi.worker.event import TaskStatus, WorkerEvent, WorkerState +from blueapi.worker.event import WorkerState from blueapi.worker.task import Task - -BEAMLINE = os.environ.get("BEAMLINE", "p46") - -DISABLE_SIDE_EFFECTS = bool(os.environ.get("DISABLE_SIDE_EFFECTS", 0)) -DISABLE_SIDE_EFFECTS_MESSAGE = """ - This test would cause side effects on the beamline, it has been disabled - so as not to interfere with operation. To run tests that may interfere with - the beamline export DISABLE_SIDE_EFFECTS=0 - """ -disable_side_effects = pytest.mark.skipif( - DISABLE_SIDE_EFFECTS, reason=DISABLE_SIDE_EFFECTS_MESSAGE -) - -VISIT_DIRECTORY: Path = { - "p46": Path("/exports/mybeamline/p46/data/2024/cm11111-1/"), - "p47": Path("/exports/mybeamline/p47/data/2024/cm11111-1/"), -}[BEAMLINE] - -VISIT_NOT_MOUNTED = not (VISIT_DIRECTORY.exists() and VISIT_DIRECTORY.is_dir()) - -VISIT_NOT_MOUNTED_MESSAGE = f""" - This test inspects data so it has to run on a machine that mounts - {VISIT_DIRECTORY} -""" - - -@pytest.fixture -def training_rig_config() -> ApplicationConfig: - return ApplicationConfig( - stomp=StompConfig( - host="172.23.168.198", - auth=BasicAuthentication(username="p46", password="64p"), # type: ignore - ), - api=RestConfig(host="p46-blueapi.diamond.ac.uk", port=443, protocol="https"), - ) - - -@pytest.fixture -def client(training_rig_config) -> BlueapiClient: - return BlueapiClient.from_config(config=training_rig_config) - - -STEP_SCAN = Task( - name="plan_step_scan", - params={ - "detectors": ["det"], - "motor": "sample_stage", - }, -) -SPEC_SCAN = Task( - name="spec_scan", - params={ - "detectors": ["det"], - "spec": { - "axis": "sample_stage.x", - "start": 1.0, - "stop": 10.0, - "num": 10, - "type": "Line", - }, - }, -) -CONNECT_DEVICES = Task( - name="connect_devices", - params={"devices": ["det", "sample_stage"]}, +from tests.system_tests.common import ( + beamline_specific_test, + check_all_events, + disable_side_effects, ) @disable_side_effects -def test_spec_scan_task(client: BlueapiClient, plan: str = "spec_scan"): - # assert client.get_plan("connect_devices") - # assert client.create_and_start_task(CONNECT_DEVICES) - - assert client.get_plan(plan), f"In {plan} is available" +@beamline_specific_test +@pytest.mark.parametrize("plan", ["spec_scan"]) +def test_spec_scan_task( + client_with_stomp: BlueapiClient, + task_definition: dict[str, Task], + plan: str = "spec_scan", +): + assert client_with_stomp.get_plan(plan), f"In {plan} is available" all_events: list[AnyEvent] = [] def on_event(event: AnyEvent): all_events.append(event) - client.run_task(SPEC_SCAN, on_event=on_event) - assert isinstance(all_events[0], WorkerEvent) and all_events[0].task_status - task_id = all_events[0].task_status.task_id - assert all_events == [ - WorkerEvent( - state=WorkerState.RUNNING, - task_status=TaskStatus( - task_id=task_id, - task_complete=False, - task_failed=False, - ), - ), - WorkerEvent( - state=WorkerState.IDLE, - task_status=TaskStatus( - task_id=task_id, - task_complete=False, - task_failed=False, - ), - ), - WorkerEvent( - state=WorkerState.IDLE, - task_status=TaskStatus( - task_id=task_id, - task_complete=True, - task_failed=False, - ), - ), - ] + client_with_stomp.run_task(task_definition[plan], on_event=on_event) + + check_all_events(all_events) - assert client.get_state() is WorkerState.IDLE + assert client_with_stomp.get_state() is WorkerState.IDLE