Merge branch '3616-bb-tests-download-plugins' into develop

Issue #3616
PR #3627

build_scripts/appimage/server_config.json.standard

@@ -1,9 +1,6 @@
 {
   "data_dir": "~/.monkey_island",
   "log_level": "DEBUG",
-  "environment": {
-    "server_config": "password"
-  },
   "mongodb": {
         "start_mongodb": true
   }

envs/monkey_zoo/blackbox/island_client/monkey_island_client.py

@@ -2,9 +2,11 @@
 import logging
 import time
 from http import HTTPStatus
-from typing import List, Mapping, Optional, Sequence
+from threading import Thread
+from typing import Any, Dict, List, Mapping, Optional, Sequence
 
 from common import OperatingSystem
+from common.agent_plugins import AgentPluginRepositoryIndex, AgentPluginType
 from common.credentials import Credentials
 from common.types import AgentID, MachineID
 from envs.monkey_zoo.blackbox.island_client.i_monkey_island_requests import IMonkeyIslandRequests
@@ -19,6 +21,7 @@
 GET_AGENT_EVENTS_ENDPOINT = "api/agent-events"
 LOGOUT_ENDPOINT = "api/logout"
 GET_AGENT_OTP_ENDPOINT = "/api/agent-otp"
+INSTALL_PLUGIN_URL = "api/install-agent-plugin"
 
 logger = logging.getLogger(__name__)
 
@@ -35,6 +38,86 @@ def __init__(self, requests: IMonkeyIslandRequests):
     def get_api_status(self):
         return self.requests.get("api")
 
+    def install_agent_plugins(self):
+        available_plugins_index_url = "api/agent-plugins/available/index"
+        installed_plugins_manifests_url = "api/agent-plugins/installed/manifests"
+
+        response = self.requests.get(available_plugins_index_url)
+        plugin_repository_index = AgentPluginRepositoryIndex(**response.json())
+
+        response = self.requests.get(installed_plugins_manifests_url)
+        installed_plugins = response.json()
+
+        install_threads: List[Thread] = []
+
+        # all of the responses from the API endpoints are serialized
+        # so we don't need to worry about type conversion
+        for plugin_type in plugin_repository_index.plugins:
+            install_threads.extend(
+                self._install_all_agent_plugins_of_type(
+                    plugin_type, plugin_repository_index, installed_plugins
+                )
+            )
+
+        for t in install_threads:
+            t.join()
+
+    def _install_all_agent_plugins_of_type(
+        self,
+        plugin_type: AgentPluginType,
+        plugin_repository_index: AgentPluginRepositoryIndex,
+        installed_plugins: Dict[str, Any],
+    ) -> Sequence[Thread]:
+        logger.info(f"Installing {plugin_type} plugins")
+        install_threads: List[Thread] = []
+        for plugin_name in plugin_repository_index.plugins[plugin_type]:
+            plugin_versions = plugin_repository_index.plugins[plugin_type][plugin_name]
+            latest_version = str(plugin_versions[-1].version)
+
+            if self._latest_version_already_installed(
+                installed_plugins, plugin_type, plugin_name, latest_version
+            ):
+                logger.info(f"{plugin_type}-{plugin_name}-v{latest_version} is already installed")
+                continue
+
+            t = Thread(
+                target=self._install_single_agent_plugin,
+                args=(plugin_name, plugin_type, latest_version),
+                daemon=True,
+            )
+            t.start()
+            install_threads.append(t)
+
+        return install_threads
+
+    def _latest_version_already_installed(
+        self,
+        installed_plugins: Dict[str, Any],
+        plugin_type: AgentPluginType,
+        plugin_name: str,
+        latest_version: str,
+    ) -> bool:
+        installed_plugin = installed_plugins.get(plugin_type, {}).get(plugin_name, {})
+        return installed_plugin and installed_plugin.get("version", "") == latest_version
+
+    def _install_single_agent_plugin(
+        self,
+        plugin_name: str,
+        plugin_type: AgentPluginType,
+        latest_version: str,
+    ):
+        install_plugin_request = {
+            "plugin_type": plugin_type,
+            "name": plugin_name,
+            "version": latest_version,
+        }
+        if self.requests.put_json(url=INSTALL_PLUGIN_URL, json=install_plugin_request).ok:
+            logger.info(f"Installed {plugin_name} {plugin_type} v{latest_version} to Island")
+        else:
+            logger.error(
+                f"Could not install {plugin_name} {plugin_type} " f"v{latest_version} to Island"
+            )
+
     @avoid_race_condition
     def set_masque(self, masque):
         masque = b"" if masque is None else masque

envs/monkey_zoo/blackbox/test_blackbox.py

@@ -118,10 +118,13 @@ def island_client(monkey_island_requests):
 
 
 @pytest.fixture(autouse=True, scope="session")
-def register(island_client):
+def setup_island(island_client):
     logging.info("Registering a new user")
     island_client.register()
 
+    logging.info("Installing all available plugins")
+    island_client.install_agent_plugins()
+
 
 @pytest.mark.parametrize(
     "authenticated_endpoint",
@@ -599,6 +602,7 @@ def test_credentials_reuse_ssh_key(self, island_client):
 
     def test_depth_2_a(self, island_client):
         test_name = "Depth2A test suite"
+
         communication_analyzer = CommunicationAnalyzer(
             island_client,
             get_target_ips(depth_2_a_test_configuration),
@@ -659,6 +663,7 @@ def test_depth_1_a(self, island_client):
 
     def test_depth_3_a(self, island_client):
         test_name = "Depth3A test suite"
+
         communication_analyzer = CommunicationAnalyzer(
             island_client,
             get_target_ips(depth_3_a_test_configuration),

monkey/common/agent_plugins/agent_plugin_manifest.py

@@ -1,5 +1,5 @@
 import re
-from typing import Callable, Mapping, Optional, Tuple, Type
+from typing import Callable, Mapping, Optional, Self, Tuple, Type
 
 from pydantic import ConstrainedStr, HttpUrl
 from semver import VersionInfo
@@ -24,7 +24,7 @@ class PluginVersion(VersionInfo):
     @classmethod
     def __get_validators__(cls):
         """Return a list of validator methods for pydantic models."""
-        yield cls.parse
+        yield cls.from_str
 
     @classmethod
     def __modify_schema__(cls, field_schema):
@@ -37,6 +37,11 @@ def __modify_schema__(cls, field_schema):
             ]
         )
 
+    @classmethod
+    def from_str(cls, version: str) -> Self:
+        """Convert a string to a PluginVersion."""
+        return cls.parse(version)
+
 
 class AgentPluginManifest(InfectionMonkeyBaseModel):
     """

monkey/common/agent_plugins/agent_plugin_repository_index.py

@@ -63,3 +63,7 @@ def _infection_monkey_version_parser(
             return VersionInfo.parse(value)
 
         raise TypeError(f'Expected "{DEVELOPMENT}" or a valid semantic version, got {type(value)}')
+
+    @validator("plugins")
+    def _convert_str_type_to_enum(cls, plugins):
+        return {AgentPluginType(t): plugins for t, plugins in plugins.items()}

monkey/common/agent_plugins/agent_plugin_type.py

@@ -1,7 +1,7 @@
-from enum import Enum
+from enum import StrEnum
 
 
-class AgentPluginType(Enum):
+class AgentPluginType(StrEnum):
     CREDENTIALS_COLLECTOR = "Credentials_Collector"
     EXPLOITER = "Exploiter"
     FINGERPRINTER = "Fingerprinter"

monkey/monkey_island/cc/server_setup.py

@@ -37,7 +37,6 @@
 from monkey_island.cc.services.initialize import initialize_services  # noqa: E402
 from monkey_island.cc.setup import (  # noqa: E402
     PyWSGILoggingFilter,
-    install_plugins,
     island_config_options_validator,
     setup_agent_event_handlers,
     setup_island_event_handlers,
@@ -68,7 +67,6 @@ def run_monkey_island():
     container = _initialize_di_container(ip_addresses, version, config_options.data_dir)
     setup_island_event_handlers(container)
     setup_agent_event_handlers(container)
-    install_plugins(container, config_options.data_dir)
 
     _start_island_server(ip_addresses, island_args.setup_only, config_options, container)
 
@@ -104,7 +102,11 @@ def _configure_logging(config_options):
 
 def _collect_system_info() -> Tuple[Sequence[IPv4Address], Deployment, Version]:
     deployment = _get_deployment()
+    logger.info(f"Monkey Island deployment: {deployment}")
+
     version = Version(get_version(), deployment)
+    logger.info(f"Monkey Island version: {version.version_number}")
+
     return (get_my_ip_addresses(), deployment, version)
 
 

monkey/monkey_island/cc/server_utils/consts.py

@@ -36,8 +36,6 @@ def _get_monkey_island_abs_path() -> str:
 
 DEFAULT_LOG_LEVEL = "INFO"
 
-PLUGIN_DIR_NAME = "plugins"
-
 DEFAULT_START_MONGO_DB = True
 
 DEFAULT_CRT_PATH = str(Path(MONKEY_ISLAND_ABS_PATH, "cc", "server.crt"))

monkey/monkey_island/cc/services/agent_configuration_service/agent_configuration_schema_compiler.py

@@ -41,13 +41,12 @@ def _add_plugins(self, schema: Dict[str, Any]) -> Dict[str, Any]:
         schema = self._add_hard_coded_plugins(schema)
 
         config_schemas = deepcopy(self._agent_plugin_service.get_all_plugin_configuration_schemas())
+        all_plugin_manifests = self._agent_plugin_service.get_all_plugin_manifests()
 
         for plugin_type in config_schemas.keys():
             for plugin_name in config_schemas[plugin_type].keys():
                 config_schema = config_schemas[plugin_type][plugin_name]
-                plugin_manifest = self._agent_plugin_service.get_all_plugin_manifests()[
-                    plugin_type
-                ][plugin_name]
+                plugin_manifest = all_plugin_manifests[plugin_type][plugin_name]
                 config_schema.update(plugin_manifest.dict(simplify=True))
                 schema = self._add_plugin_to_schema(schema, plugin_type, plugin_name, config_schema)
         return schema

monkey/monkey_island/cc/services/agent_plugin_service/flask_resources/install_agent_plugin.py

@@ -66,7 +66,7 @@ def _get_plugin_information_from_request(
             raise ValueError(message)
 
         try:
-            plugin_version = PluginVersion(**plugin_version_arg)
+            plugin_version = PluginVersion.from_str(plugin_version_arg)
         except ValueError as err:
             message = f"Invalid plugin version argument: {plugin_version_arg}: {err}."
             raise ValueError(message)

monkey/monkey_island/cc/services/initialize.py

@@ -46,7 +46,7 @@
     NetworkModelUpdateFacade,
     initialize_machine_repository,
 )
-from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH, PLUGIN_DIR_NAME
+from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH
 from monkey_island.cc.server_utils.encryption import ILockableEncryptor, RepositoryEncryptor
 from monkey_island.cc.services import (
     AgentSignalsService,
@@ -133,13 +133,6 @@ def _register_repositories(container: DIContainer, data_dir: Path):
         IFileRepository,
         _decorate_file_repository(LocalStorageFileRepository(data_dir / "runtime_data")),
     )
-    container.register_convention(
-        IFileRepository,
-        "plugin_file_repository",
-        FileRepositoryLockingDecorator(
-            FileRepositoryLoggingDecorator(LocalStorageFileRepository(data_dir / PLUGIN_DIR_NAME))
-        ),
-    )
 
     container.register_instance(ISimulationRepository, container.resolve(FileSimulationRepository))
     container.register_instance(

monkey/monkey_island/cc/setup/__init__.py

@@ -1,4 +1,3 @@
 from .pywsgi_logging_filter import PyWSGILoggingFilter
 from .island_event_handlers import setup_island_event_handlers
 from .agent_event_handlers import setup_agent_event_handlers
-from .plugin_installation import install_plugins

monkey/monkey_island/cc/setup/data_dir.py

@@ -5,7 +5,6 @@
 
 from common.utils.file_utils import create_secure_directory
 from common.version import get_version
-from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH, PLUGIN_DIR_NAME
 from monkey_island.cc.setup.env_utils import is_running_on_docker
 from monkey_island.cc.setup.version_file_setup import get_version_from_dir, write_version
 
@@ -23,7 +22,6 @@ def setup_data_dir(data_dir_path: Path):
         _handle_old_data_directory(data_dir_path)
     create_secure_directory(data_dir_path)
     write_version(data_dir_path)
-    _copy_plugins_into_data_dir(data_dir_path)
     logger.info(f"Data directory set up in {data_dir_path}.")
 
 
@@ -92,42 +90,3 @@ def _data_dir_version_mismatch_exists(data_dir_path: Path) -> bool:
     island_version = get_version()
 
     return island_version != data_dir_version
-
-
-def _copy_plugins_into_data_dir(data_dir_path: Path):
-    plugin_source_dir = Path(MONKEY_ISLAND_ABS_PATH) / PLUGIN_DIR_NAME
-    try:
-        plugins_dir = _create_plugins_dir(data_dir_path)
-        plugin_tar_files = list(plugin_source_dir.glob("*.tar"))
-    except Exception:
-        logger.exception(
-            f"An error occured while creating plugins data directory: {plugin_source_dir}"
-        )
-        return
-
-    for plugin_tar_file in plugin_tar_files:
-        plugin_dest_path = plugins_dir / plugin_tar_file.name
-        if plugin_dest_path.exists():
-            logger.info(
-                "Skipping plugin tar file copy: "
-                f"destination file {plugin_dest_path} already exists."
-            )
-            continue
-
-        try:
-            logger.info(f"Copying plugin tar file: {plugin_tar_file} -> {plugin_dest_path}")
-            shutil.copy2(plugin_tar_file, plugin_dest_path)
-        except FileNotFoundError:
-            logger.exception(
-                f"An error occured while copying plugin {plugin_tar_file} "
-                f"to the data directory: {data_dir_path}"
-            )
-
-
-def _create_plugins_dir(plugins_dir_parent_dir: Path) -> Path:
-    plugins_dir = plugins_dir_parent_dir / PLUGIN_DIR_NAME
-    logger.info(f"Plugins directory: {plugins_dir}")
-
-    if not plugins_dir.exists():
-        create_secure_directory(plugins_dir)
-    return plugins_dir