From 76d82cecea1aecb74076b28a6592f4b85f519dc3 Mon Sep 17 00:00:00 2001 From: Shreya Date: Wed, 12 May 2021 19:41:49 +0530 Subject: [PATCH 01/37] Create `data_dir` if no `--server-config` is passed during Monkey Island initialisation --- monkey/monkey_island.py | 7 +++++-- monkey/monkey_island/cc/arg_parser.py | 7 +------ monkey/monkey_island/cc/server_utils/consts.py | 7 ++++--- monkey/monkey_island/config_loader.py | 11 ++++++++++- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 650cfe95d81..958a1f529e7 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -16,7 +16,10 @@ # This is here in order to catch EVERYTHING, some functions are being called on # imports, so the log init needs to be first. try: - server_config_path = os.path.expanduser(island_args.server_config) + if island_args.server_config: + server_config_path = os.path.expanduser(island_args.server_config) + else: + server_config_path = config_loader.create_default_server_config_path() config = config_loader.load_server_config_from_file(server_config_path) @@ -32,4 +35,4 @@ from monkey_island.cc.main import main # noqa: E402 - main(config["data_dir"], island_args.setup_only, island_args.server_config) + main(config["data_dir"], island_args.setup_only, server_config_path) diff --git a/monkey/monkey_island/cc/arg_parser.py b/monkey/monkey_island/cc/arg_parser.py index 6e12ef38f36..98277623076 100644 --- a/monkey/monkey_island/cc/arg_parser.py +++ b/monkey/monkey_island/cc/arg_parser.py @@ -1,7 +1,5 @@ from dataclasses import dataclass -from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH - @dataclass class IslandArgs: @@ -25,10 +23,7 @@ def parse_cli_args() -> IslandArgs: "compiling/packaging Islands.", ) parser.add_argument( - "--server-config", - action="store", - help="The path to the server configuration file.", - default=DEFAULT_SERVER_CONFIG_PATH, + "--server-config", action="store", help="The path to the server configuration file." ) args = parser.parse_args() diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index f0dba26dc6b..1c0e53d8476 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -3,12 +3,13 @@ __author__ = "itay.mizeretz" MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island") + +DEFAULT_DATA_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc") + DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5 -DEFAULT_SERVER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json") +DEFAULT_SERVER_CONFIG_PATH = os.path.join(DEFAULT_DATA_DIR, "server_config.json") DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join( MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json.develop" ) - -DEFAULT_DATA_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc") diff --git a/monkey/monkey_island/config_loader.py b/monkey/monkey_island/config_loader.py index aaa9185d700..036be2c69c4 100644 --- a/monkey/monkey_island/config_loader.py +++ b/monkey/monkey_island/config_loader.py @@ -1,11 +1,20 @@ import json import os -from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR +import monkey_island.cc.environment.server_config_generator as server_config_generator +from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH DEFAULT_LOG_LEVEL = "INFO" +def create_default_server_config_path(): + if not os.path.isfile(DEFAULT_SERVER_CONFIG_PATH): + if not os.path.isdir(DEFAULT_DATA_DIR): + os.mkdir(DEFAULT_DATA_DIR, mode=0o700) + server_config_generator.create_default_config_file(DEFAULT_SERVER_CONFIG_PATH) + return DEFAULT_SERVER_CONFIG_PATH + + def load_server_config_from_file(server_config_path): with open(server_config_path, "r") as f: config_content = f.read() From 82463416f85fad02212eb6d863d85f53b6a3db13 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 13 May 2021 15:07:02 +0530 Subject: [PATCH 02/37] Create data dir if `--server-config` is passed, "data_dir" field exists, but the dir doesn't exist --- monkey/monkey_island/cc/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index cf56144ed06..497323f8b5c 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -41,6 +41,8 @@ def main( ): logger.info("Starting bootloader server") + create_data_dir(data_dir) + env_singleton.initialize_from_file(server_config_filename) initialize_encryptor(data_dir) initialize_services(data_dir) @@ -55,6 +57,11 @@ def main( bootloader_server_thread.join() +def create_data_dir(data_dir): + if not os.path.isdir(data_dir): + os.makedirs(data_dir, mode=0o700) + + def start_island_server(should_setup_only): mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url()) wait_for_mongo_db_server(mongo_url) From 808e86df1a851ad37006db7ae5b4aaa2aecbb8b8 Mon Sep 17 00:00:00 2001 From: Shreya Date: Fri, 14 May 2021 17:08:55 +0530 Subject: [PATCH 03/37] Create data_dir before logger is set up --- monkey/monkey_island.py | 8 ++++++++ monkey/monkey_island/cc/main.py | 7 ------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 958a1f529e7..31fa26382d7 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -10,6 +10,12 @@ from monkey_island import config_loader # noqa: E402 from monkey_island.cc.server_utils.island_logger import setup_logging # noqa: E402 + +def create_data_dir(data_dir): + if not os.path.isdir(data_dir): + os.makedirs(data_dir, mode=0o700) + + if "__main__" == __name__: island_args = parse_cli_args() @@ -23,6 +29,8 @@ config = config_loader.load_server_config_from_file(server_config_path) + create_data_dir(config["data_dir"]) + setup_logging(config["data_dir"], config["log_level"]) except OSError as ex: diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index 497323f8b5c..cf56144ed06 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -41,8 +41,6 @@ def main( ): logger.info("Starting bootloader server") - create_data_dir(data_dir) - env_singleton.initialize_from_file(server_config_filename) initialize_encryptor(data_dir) initialize_services(data_dir) @@ -57,11 +55,6 @@ def main( bootloader_server_thread.join() -def create_data_dir(data_dir): - if not os.path.isdir(data_dir): - os.makedirs(data_dir, mode=0o700) - - def start_island_server(should_setup_only): mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url()) wait_for_mongo_db_server(mongo_url) From 9eedac4eb252276fc68333e006a66337963e5fcf Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 17 May 2021 16:46:39 +0530 Subject: [PATCH 04/37] Move `create_data_dir()` from monkey_island.py to monkey_island/cc/setup.py --- monkey/monkey_island.py | 7 +------ monkey/monkey_island/cc/setup.py | 6 ++++++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 31fa26382d7..4f28bc606a6 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -9,12 +9,7 @@ from monkey_island import config_loader # noqa: E402 from monkey_island.cc.server_utils.island_logger import setup_logging # noqa: E402 - - -def create_data_dir(data_dir): - if not os.path.isdir(data_dir): - os.makedirs(data_dir, mode=0o700) - +from monkey_island.cc.setup import create_data_dir # noqa: E402 if "__main__" == __name__: island_args = parse_cli_args() diff --git a/monkey/monkey_island/cc/setup.py b/monkey/monkey_island/cc/setup.py index a03c554be52..fbe709dec73 100644 --- a/monkey/monkey_island/cc/setup.py +++ b/monkey/monkey_island/cc/setup.py @@ -1,4 +1,5 @@ import logging +import os from pymongo import errors @@ -49,3 +50,8 @@ def store_mitigations_on_mongo(): ) for key, mongo_object in mongo_mitigations.items(): mongo_object.save() + + +def create_data_dir(data_dir): + if not os.path.isdir(data_dir): + os.makedirs(data_dir, mode=0o700) From 805e5e6c22243d98077b08899bdd61627476b304 Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 17 May 2021 17:42:25 +0530 Subject: [PATCH 05/37] Restructure code for creating default data directory and server config --- monkey/monkey_island.py | 5 +++-- .../cc/environment/data_dir_generator.py | 13 +++++++++++++ .../cc/environment/server_config_generator.py | 16 ++++++++++++++-- monkey/monkey_island/cc/setup.py | 6 ------ monkey/monkey_island/config_loader.py | 11 +---------- 5 files changed, 31 insertions(+), 20 deletions(-) create mode 100644 monkey/monkey_island/cc/environment/data_dir_generator.py diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 4f28bc606a6..55a6d047547 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -7,9 +7,10 @@ import json # noqa: E402 import os # noqa: E402 +import monkey_island.cc.environment.server_config_generator as server_config_generator # noqa: E402 from monkey_island import config_loader # noqa: E402 +from monkey_island.cc.environment.data_dir_generator import create_data_dir # noqa: E402 from monkey_island.cc.server_utils.island_logger import setup_logging # noqa: E402 -from monkey_island.cc.setup import create_data_dir # noqa: E402 if "__main__" == __name__: island_args = parse_cli_args() @@ -20,7 +21,7 @@ if island_args.server_config: server_config_path = os.path.expanduser(island_args.server_config) else: - server_config_path = config_loader.create_default_server_config_path() + server_config_path = server_config_generator.create_default_server_config_file() config = config_loader.load_server_config_from_file(server_config_path) diff --git a/monkey/monkey_island/cc/environment/data_dir_generator.py b/monkey/monkey_island/cc/environment/data_dir_generator.py new file mode 100644 index 00000000000..efbec857f83 --- /dev/null +++ b/monkey/monkey_island/cc/environment/data_dir_generator.py @@ -0,0 +1,13 @@ +import os + +from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR + + +def create_data_dir(data_dir: str) -> None: + if not os.path.isdir(data_dir): + os.makedirs(data_dir, mode=0o700) + + +def create_default_data_dir() -> None: + if not os.path.isdir(DEFAULT_DATA_DIR): + os.mkdir(DEFAULT_DATA_DIR, mode=0o700) diff --git a/monkey/monkey_island/cc/environment/server_config_generator.py b/monkey/monkey_island/cc/environment/server_config_generator.py index 211b745c5c2..45dde6f781d 100644 --- a/monkey/monkey_island/cc/environment/server_config_generator.py +++ b/monkey/monkey_island/cc/environment/server_config_generator.py @@ -1,8 +1,20 @@ +import os from pathlib import Path -from monkey_island.cc.server_utils.consts import DEFAULT_DEVELOP_SERVER_CONFIG_PATH +from monkey_island.cc.environment.data_dir_generator import create_default_data_dir +from monkey_island.cc.server_utils.consts import ( + DEFAULT_DEVELOP_SERVER_CONFIG_PATH, + DEFAULT_SERVER_CONFIG_PATH, +) -def create_default_config_file(path): +def create_default_server_config_file() -> str: + if not os.path.isfile(DEFAULT_SERVER_CONFIG_PATH): + create_default_data_dir() + write_default_server_config_to_file(DEFAULT_SERVER_CONFIG_PATH) + return DEFAULT_SERVER_CONFIG_PATH + + +def write_default_server_config_to_file(path: str) -> None: default_config = Path(DEFAULT_DEVELOP_SERVER_CONFIG_PATH).read_text() Path(path).write_text(default_config) diff --git a/monkey/monkey_island/cc/setup.py b/monkey/monkey_island/cc/setup.py index fbe709dec73..a03c554be52 100644 --- a/monkey/monkey_island/cc/setup.py +++ b/monkey/monkey_island/cc/setup.py @@ -1,5 +1,4 @@ import logging -import os from pymongo import errors @@ -50,8 +49,3 @@ def store_mitigations_on_mongo(): ) for key, mongo_object in mongo_mitigations.items(): mongo_object.save() - - -def create_data_dir(data_dir): - if not os.path.isdir(data_dir): - os.makedirs(data_dir, mode=0o700) diff --git a/monkey/monkey_island/config_loader.py b/monkey/monkey_island/config_loader.py index 036be2c69c4..aaa9185d700 100644 --- a/monkey/monkey_island/config_loader.py +++ b/monkey/monkey_island/config_loader.py @@ -1,20 +1,11 @@ import json import os -import monkey_island.cc.environment.server_config_generator as server_config_generator -from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH +from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR DEFAULT_LOG_LEVEL = "INFO" -def create_default_server_config_path(): - if not os.path.isfile(DEFAULT_SERVER_CONFIG_PATH): - if not os.path.isdir(DEFAULT_DATA_DIR): - os.mkdir(DEFAULT_DATA_DIR, mode=0o700) - server_config_generator.create_default_config_file(DEFAULT_SERVER_CONFIG_PATH) - return DEFAULT_SERVER_CONFIG_PATH - - def load_server_config_from_file(server_config_path): with open(server_config_path, "r") as f: config_content = f.read() From d8927a5c5568c06341e077332f47313875c54e60 Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 17 May 2021 17:45:55 +0530 Subject: [PATCH 06/37] Create constant SERVER_CONFIG_FILENAME --- monkey/monkey_island/cc/server_utils/consts.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index 1c0e53d8476..eaf7ec4161c 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -2,14 +2,17 @@ __author__ = "itay.mizeretz" + +SERVER_CONFIG_FILENAME = "server_config.json" + MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island") DEFAULT_DATA_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc") DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5 -DEFAULT_SERVER_CONFIG_PATH = os.path.join(DEFAULT_DATA_DIR, "server_config.json") +DEFAULT_SERVER_CONFIG_PATH = os.path.join(DEFAULT_DATA_DIR, SERVER_CONFIG_FILENAME) DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join( - MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json.develop" + MONKEY_ISLAND_ABS_PATH, "cc", f"{SERVER_CONFIG_FILENAME}.develop" ) From af42c01aa9fc5d9e4f8000425ff5d0d75248b8dd Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 17 May 2021 17:49:12 +0530 Subject: [PATCH 07/37] Replace missed out function name --- monkey/monkey_island/cc/environment/environment_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/environment/environment_config.py b/monkey/monkey_island/cc/environment/environment_config.py index 6f4626c9ed8..5befa202b92 100644 --- a/monkey/monkey_island/cc/environment/environment_config.py +++ b/monkey/monkey_island/cc/environment/environment_config.py @@ -25,7 +25,7 @@ def _load_from_file(self, file_path): file_path = os.path.expanduser(file_path) if not Path(file_path).is_file(): - server_config_generator.create_default_config_file(file_path) + server_config_generator.write_default_server_config_to_file(file_path) with open(file_path, "r") as f: config_content = f.read() From ff1e6bdb2c8614374369ac3c3852537c9a306c1c Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 17 May 2021 17:53:22 +0530 Subject: [PATCH 08/37] Remove logic for creating default server config in appimage script --- appimage/run_appimage.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/appimage/run_appimage.sh b/appimage/run_appimage.sh index 837ef5d3a5d..d31b41843b0 100644 --- a/appimage/run_appimage.sh +++ b/appimage/run_appimage.sh @@ -3,20 +3,12 @@ PYTHON_CMD="$APPDIR"/opt/python3.7/bin/python3.7 DOT_MONKEY="$HOME"/.monkey_island/ -configure_default_server() { - if [ ! -f "$DOT_MONKEY"/server_config.json ]; then - cp "$APPDIR"/usr/src/monkey_island/cc/server_config.json.standard "$DOT_MONKEY"/server_config.json - fi -} - # shellcheck disable=SC2174 mkdir --mode=0700 --parents "$DOT_MONKEY" DB_DIR="$DOT_MONKEY"/db mkdir --parents "$DB_DIR" -configure_default_server - cd "$APPDIR"/usr/src || exit 1 ./monkey_island/bin/mongodb/bin/mongod --dbpath "$DB_DIR" & ${PYTHON_CMD} ./monkey_island.py --server-config "$DOT_MONKEY"/server_config.json From a1beee95f31bac8f8555898bbe31861f2b026973 Mon Sep 17 00:00:00 2001 From: shreyamalviya Date: Tue, 18 May 2021 23:32:44 +0530 Subject: [PATCH 09/37] Change data_dir permissions on Windows --- .../cc/environment/data_dir_generator.py | 33 +++++++++++++++++++ monkey/monkey_island/cc/environment/os.py | 5 +++ 2 files changed, 38 insertions(+) create mode 100644 monkey/monkey_island/cc/environment/os.py diff --git a/monkey/monkey_island/cc/environment/data_dir_generator.py b/monkey/monkey_island/cc/environment/data_dir_generator.py index efbec857f83..877faba3cb3 100644 --- a/monkey/monkey_island/cc/environment/data_dir_generator.py +++ b/monkey/monkey_island/cc/environment/data_dir_generator.py @@ -1,13 +1,46 @@ import os +import ntsecuritycon +import win32api +import win32con +import win32security + +from monkey_island.cc.environment.os import is_windows_os from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR def create_data_dir(data_dir: str) -> None: if not os.path.isdir(data_dir): os.makedirs(data_dir, mode=0o700) + if is_windows_os(): # `mode=0o700` doesn't work on Windows + set_data_dir_security_to_read_by_owner(data_dir_path=data_dir) def create_default_data_dir() -> None: if not os.path.isdir(DEFAULT_DATA_DIR): os.mkdir(DEFAULT_DATA_DIR, mode=0o700) + if is_windows_os(): # `mode=0o700` doesn't work on Windows + set_data_dir_security_to_read_by_owner(data_dir_path=DEFAULT_DATA_DIR) + + +def set_data_dir_security_to_read_by_owner(data_dir_path: str) -> None: + user = get_user_pySID_object() # current user is newly created data dir's owner + + security_descriptor = win32security.GetFileSecurity( + data_dir_path, win32security.DACL_SECURITY_INFORMATION + ) + dacl = win32security.ACL() + dacl.AddAccessAllowedAce(win32security.ACL_REVISION, ntsecuritycon.FILE_GENERIC_READ, user) + security_descriptor.SetSecurityDescriptorDacl(1, dacl, 0) + win32security.SetFileSecurity( + data_dir_path, win32security.DACL_SECURITY_INFORMATION, security_descriptor + ) + + +def get_user_pySID_object(): + # get current user's name + username = win32api.GetUserNameEx(win32con.NameSamCompatible) + # pySID object for the current user + user, _, _ = win32security.LookupAccountName("", username) + + return user diff --git a/monkey/monkey_island/cc/environment/os.py b/monkey/monkey_island/cc/environment/os.py new file mode 100644 index 00000000000..195e54fd355 --- /dev/null +++ b/monkey/monkey_island/cc/environment/os.py @@ -0,0 +1,5 @@ +import sys + + +def is_windows_os(): + return sys.platform.startswith("win") From 320167224864227f6adac60bd25ddad534bc1a90 Mon Sep 17 00:00:00 2001 From: shreyamalviya Date: Wed, 19 May 2021 15:59:38 +0530 Subject: [PATCH 10/37] Move `is_windows_os` to data_dir_generator.py and add user write permissions to newly created data directory --- .../cc/environment/data_dir_generator.py | 18 ++++++++++++------ monkey/monkey_island/cc/environment/os.py | 5 ----- 2 files changed, 12 insertions(+), 11 deletions(-) delete mode 100644 monkey/monkey_island/cc/environment/os.py diff --git a/monkey/monkey_island/cc/environment/data_dir_generator.py b/monkey/monkey_island/cc/environment/data_dir_generator.py index 877faba3cb3..a809f43600d 100644 --- a/monkey/monkey_island/cc/environment/data_dir_generator.py +++ b/monkey/monkey_island/cc/environment/data_dir_generator.py @@ -1,26 +1,28 @@ import os +import sys import ntsecuritycon import win32api import win32con import win32security -from monkey_island.cc.environment.os import is_windows_os from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR +is_windows_os = sys.platform.startswith("win") + def create_data_dir(data_dir: str) -> None: if not os.path.isdir(data_dir): os.makedirs(data_dir, mode=0o700) - if is_windows_os(): # `mode=0o700` doesn't work on Windows - set_data_dir_security_to_read_by_owner(data_dir_path=data_dir) + if is_windows_os: # `mode=0o700` doesn't work on Windows + set_data_dir_security_to_read_by_owner(data_dir_path=data_dir) def create_default_data_dir() -> None: if not os.path.isdir(DEFAULT_DATA_DIR): os.mkdir(DEFAULT_DATA_DIR, mode=0o700) - if is_windows_os(): # `mode=0o700` doesn't work on Windows - set_data_dir_security_to_read_by_owner(data_dir_path=DEFAULT_DATA_DIR) + if is_windows_os: # `mode=0o700` doesn't work on Windows + set_data_dir_security_to_read_by_owner(data_dir_path=DEFAULT_DATA_DIR) def set_data_dir_security_to_read_by_owner(data_dir_path: str) -> None: @@ -30,7 +32,11 @@ def set_data_dir_security_to_read_by_owner(data_dir_path: str) -> None: data_dir_path, win32security.DACL_SECURITY_INFORMATION ) dacl = win32security.ACL() - dacl.AddAccessAllowedAce(win32security.ACL_REVISION, ntsecuritycon.FILE_GENERIC_READ, user) + dacl.AddAccessAllowedAce( + win32security.ACL_REVISION, + ntsecuritycon.FILE_GENERIC_READ | ntsecuritycon.FILE_GENERIC_WRITE, + user, + ) security_descriptor.SetSecurityDescriptorDacl(1, dacl, 0) win32security.SetFileSecurity( data_dir_path, win32security.DACL_SECURITY_INFORMATION, security_descriptor diff --git a/monkey/monkey_island/cc/environment/os.py b/monkey/monkey_island/cc/environment/os.py deleted file mode 100644 index 195e54fd355..00000000000 --- a/monkey/monkey_island/cc/environment/os.py +++ /dev/null @@ -1,5 +0,0 @@ -import sys - - -def is_windows_os(): - return sys.platform.startswith("win") From e7a26aa2d1fc9570925637a7d1ea9015c02d0b5e Mon Sep 17 00:00:00 2001 From: shreyamalviya Date: Wed, 19 May 2021 16:01:35 +0530 Subject: [PATCH 11/37] Rename `set_data_dir_security_to_read_by_owner()` to `set_data_dir_security_to_read_and_write_by_owner` --- monkey/monkey_island/cc/environment/data_dir_generator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/environment/data_dir_generator.py b/monkey/monkey_island/cc/environment/data_dir_generator.py index a809f43600d..5883cf25b6b 100644 --- a/monkey/monkey_island/cc/environment/data_dir_generator.py +++ b/monkey/monkey_island/cc/environment/data_dir_generator.py @@ -15,17 +15,17 @@ def create_data_dir(data_dir: str) -> None: if not os.path.isdir(data_dir): os.makedirs(data_dir, mode=0o700) if is_windows_os: # `mode=0o700` doesn't work on Windows - set_data_dir_security_to_read_by_owner(data_dir_path=data_dir) + set_data_dir_security_to_read_and_write_by_owner(data_dir_path=data_dir) def create_default_data_dir() -> None: if not os.path.isdir(DEFAULT_DATA_DIR): os.mkdir(DEFAULT_DATA_DIR, mode=0o700) if is_windows_os: # `mode=0o700` doesn't work on Windows - set_data_dir_security_to_read_by_owner(data_dir_path=DEFAULT_DATA_DIR) + set_data_dir_security_to_read_and_write_by_owner(data_dir_path=DEFAULT_DATA_DIR) -def set_data_dir_security_to_read_by_owner(data_dir_path: str) -> None: +def set_data_dir_security_to_read_and_write_by_owner(data_dir_path: str) -> None: user = get_user_pySID_object() # current user is newly created data dir's owner security_descriptor = win32security.GetFileSecurity( From 8c575b9d358415f2c3c80da8904503b9dcdb585a Mon Sep 17 00:00:00 2001 From: shreyamalviya Date: Wed, 19 May 2021 17:00:58 +0530 Subject: [PATCH 12/37] Import Windows specific modules only on Windows --- .../cc/environment/data_dir_generator.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/monkey/monkey_island/cc/environment/data_dir_generator.py b/monkey/monkey_island/cc/environment/data_dir_generator.py index 5883cf25b6b..9d235e30711 100644 --- a/monkey/monkey_island/cc/environment/data_dir_generator.py +++ b/monkey/monkey_island/cc/environment/data_dir_generator.py @@ -1,15 +1,16 @@ import os import sys -import ntsecuritycon -import win32api -import win32con -import win32security - from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR is_windows_os = sys.platform.startswith("win") +if is_windows_os: + import ntsecuritycon + import win32api + import win32con + import win32security + def create_data_dir(data_dir: str) -> None: if not os.path.isdir(data_dir): From 8ce506ac6fee3345f45adfff96ded37e3c6ac25d Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 19 May 2021 17:00:57 +0300 Subject: [PATCH 13/37] Refactored windows permission handling into a separate file --- .../cc/environment/data_dir_generator.py | 51 +++---------------- .../cc/environment/windows_permissions.py | 31 +++++++++++ 2 files changed, 38 insertions(+), 44 deletions(-) create mode 100644 monkey/monkey_island/cc/environment/windows_permissions.py diff --git a/monkey/monkey_island/cc/environment/data_dir_generator.py b/monkey/monkey_island/cc/environment/data_dir_generator.py index 9d235e30711..399ade277c8 100644 --- a/monkey/monkey_island/cc/environment/data_dir_generator.py +++ b/monkey/monkey_island/cc/environment/data_dir_generator.py @@ -1,53 +1,16 @@ import os import sys -from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR +from monkey_island.cc.environment.windows_permissions import set_full_folder_access is_windows_os = sys.platform.startswith("win") -if is_windows_os: - import ntsecuritycon - import win32api - import win32con - import win32security - -def create_data_dir(data_dir: str) -> None: +def create_data_dir(data_dir: str, create_parent_dirs: bool) -> None: if not os.path.isdir(data_dir): - os.makedirs(data_dir, mode=0o700) + if create_parent_dirs: + os.makedirs(data_dir, mode=0o700) + else: + os.mkdir(data_dir, mode=0o700) if is_windows_os: # `mode=0o700` doesn't work on Windows - set_data_dir_security_to_read_and_write_by_owner(data_dir_path=data_dir) - - -def create_default_data_dir() -> None: - if not os.path.isdir(DEFAULT_DATA_DIR): - os.mkdir(DEFAULT_DATA_DIR, mode=0o700) - if is_windows_os: # `mode=0o700` doesn't work on Windows - set_data_dir_security_to_read_and_write_by_owner(data_dir_path=DEFAULT_DATA_DIR) - - -def set_data_dir_security_to_read_and_write_by_owner(data_dir_path: str) -> None: - user = get_user_pySID_object() # current user is newly created data dir's owner - - security_descriptor = win32security.GetFileSecurity( - data_dir_path, win32security.DACL_SECURITY_INFORMATION - ) - dacl = win32security.ACL() - dacl.AddAccessAllowedAce( - win32security.ACL_REVISION, - ntsecuritycon.FILE_GENERIC_READ | ntsecuritycon.FILE_GENERIC_WRITE, - user, - ) - security_descriptor.SetSecurityDescriptorDacl(1, dacl, 0) - win32security.SetFileSecurity( - data_dir_path, win32security.DACL_SECURITY_INFORMATION, security_descriptor - ) - - -def get_user_pySID_object(): - # get current user's name - username = win32api.GetUserNameEx(win32con.NameSamCompatible) - # pySID object for the current user - user, _, _ = win32security.LookupAccountName("", username) - - return user + set_full_folder_access(folder_path=data_dir) diff --git a/monkey/monkey_island/cc/environment/windows_permissions.py b/monkey/monkey_island/cc/environment/windows_permissions.py new file mode 100644 index 00000000000..5d4913151fb --- /dev/null +++ b/monkey/monkey_island/cc/environment/windows_permissions.py @@ -0,0 +1,31 @@ +import ntsecuritycon +import win32api +import win32con +import win32security + + +def set_full_folder_access(folder_path: str) -> None: + user = get_user_pySID_object() + + security_descriptor = win32security.GetFileSecurity( + folder_path, win32security.DACL_SECURITY_INFORMATION + ) + dacl = win32security.ACL() + dacl.AddAccessAllowedAce( + win32security.ACL_REVISION, + ntsecuritycon.FILE_ALL_ACCESS, + user, + ) + security_descriptor.SetSecurityDescriptorDacl(1, dacl, 0) + win32security.SetFileSecurity( + folder_path, win32security.DACL_SECURITY_INFORMATION, security_descriptor + ) + + +def get_user_pySID_object(): + # get current user's name + username = win32api.GetUserNameEx(win32con.NameSamCompatible) + # pySID object for the current user + user, _, _ = win32security.LookupAccountName("", username) + + return user From 409a3c5234ffaa80b1efdfe3051c313997fcaea1 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 19 May 2021 17:01:50 +0300 Subject: [PATCH 14/37] Refactored code duplication by adding a parameter for create_data_dir --- monkey/monkey_island.py | 2 +- .../monkey_island/cc/environment/server_config_generator.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 55a6d047547..76399b6ff00 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -25,7 +25,7 @@ config = config_loader.load_server_config_from_file(server_config_path) - create_data_dir(config["data_dir"]) + create_data_dir(config["data_dir"], True) setup_logging(config["data_dir"], config["log_level"]) diff --git a/monkey/monkey_island/cc/environment/server_config_generator.py b/monkey/monkey_island/cc/environment/server_config_generator.py index 45dde6f781d..fd47889ea39 100644 --- a/monkey/monkey_island/cc/environment/server_config_generator.py +++ b/monkey/monkey_island/cc/environment/server_config_generator.py @@ -1,7 +1,7 @@ import os from pathlib import Path -from monkey_island.cc.environment.data_dir_generator import create_default_data_dir +from monkey_island.cc.environment.data_dir_generator import create_data_dir from monkey_island.cc.server_utils.consts import ( DEFAULT_DEVELOP_SERVER_CONFIG_PATH, DEFAULT_SERVER_CONFIG_PATH, @@ -10,7 +10,7 @@ def create_default_server_config_file() -> str: if not os.path.isfile(DEFAULT_SERVER_CONFIG_PATH): - create_default_data_dir() + create_data_dir(DEFAULT_SERVER_CONFIG_PATH, False) write_default_server_config_to_file(DEFAULT_SERVER_CONFIG_PATH) return DEFAULT_SERVER_CONFIG_PATH From 2fb77fcb390adee87ad038c9335d920eb8d9f55f Mon Sep 17 00:00:00 2001 From: shreyamalviya Date: Thu, 20 May 2021 16:27:11 +0530 Subject: [PATCH 15/37] Get default directory depending on OS --- .../cc/environment/server_config_generator.py | 3 ++- monkey/monkey_island/cc/server_utils/consts.py | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/environment/server_config_generator.py b/monkey/monkey_island/cc/environment/server_config_generator.py index fd47889ea39..17ee4a50c09 100644 --- a/monkey/monkey_island/cc/environment/server_config_generator.py +++ b/monkey/monkey_island/cc/environment/server_config_generator.py @@ -3,6 +3,7 @@ from monkey_island.cc.environment.data_dir_generator import create_data_dir from monkey_island.cc.server_utils.consts import ( + DEFAULT_DATA_DIR, DEFAULT_DEVELOP_SERVER_CONFIG_PATH, DEFAULT_SERVER_CONFIG_PATH, ) @@ -10,7 +11,7 @@ def create_default_server_config_file() -> str: if not os.path.isfile(DEFAULT_SERVER_CONFIG_PATH): - create_data_dir(DEFAULT_SERVER_CONFIG_PATH, False) + create_data_dir(DEFAULT_DATA_DIR, False) write_default_server_config_to_file(DEFAULT_SERVER_CONFIG_PATH) return DEFAULT_SERVER_CONFIG_PATH diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index eaf7ec4161c..a3005ad3128 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -1,17 +1,28 @@ import os +import sys __author__ = "itay.mizeretz" +def get_default_data_dir() -> str: + is_windows_os = sys.platform.startswith("win") + if is_windows_os: + return r"%AppData%\monkey_island" + else: + return r"$HOME/.monkey_island" + + SERVER_CONFIG_FILENAME = "server_config.json" MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island") -DEFAULT_DATA_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc") +DEFAULT_DATA_DIR = os.path.expandvars(get_default_data_dir()) DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5 -DEFAULT_SERVER_CONFIG_PATH = os.path.join(DEFAULT_DATA_DIR, SERVER_CONFIG_FILENAME) +DEFAULT_SERVER_CONFIG_PATH = os.path.expandvars( + os.path.join(DEFAULT_DATA_DIR, SERVER_CONFIG_FILENAME) +) DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join( MONKEY_ISLAND_ABS_PATH, "cc", f"{SERVER_CONFIG_FILENAME}.develop" From 37b7815e00173ab6698dc74ff86d3d13367dc8d9 Mon Sep 17 00:00:00 2001 From: shreyamalviya Date: Thu, 20 May 2021 16:35:19 +0530 Subject: [PATCH 16/37] Move `is_windows_os()` to separate file --- monkey/monkey_island/cc/environment/data_dir_generator.py | 6 ++---- monkey/monkey_island/cc/environment/utils.py | 5 +++++ monkey/monkey_island/cc/server_utils/consts.py | 6 +++--- 3 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 monkey/monkey_island/cc/environment/utils.py diff --git a/monkey/monkey_island/cc/environment/data_dir_generator.py b/monkey/monkey_island/cc/environment/data_dir_generator.py index 399ade277c8..98907d63948 100644 --- a/monkey/monkey_island/cc/environment/data_dir_generator.py +++ b/monkey/monkey_island/cc/environment/data_dir_generator.py @@ -1,10 +1,8 @@ import os -import sys +from monkey_island.cc.environment.utils import is_windows_os from monkey_island.cc.environment.windows_permissions import set_full_folder_access -is_windows_os = sys.platform.startswith("win") - def create_data_dir(data_dir: str, create_parent_dirs: bool) -> None: if not os.path.isdir(data_dir): @@ -12,5 +10,5 @@ def create_data_dir(data_dir: str, create_parent_dirs: bool) -> None: os.makedirs(data_dir, mode=0o700) else: os.mkdir(data_dir, mode=0o700) - if is_windows_os: # `mode=0o700` doesn't work on Windows + if is_windows_os(): # `mode=0o700` doesn't work on Windows set_full_folder_access(folder_path=data_dir) diff --git a/monkey/monkey_island/cc/environment/utils.py b/monkey/monkey_island/cc/environment/utils.py new file mode 100644 index 00000000000..cbb8a1d6fa9 --- /dev/null +++ b/monkey/monkey_island/cc/environment/utils.py @@ -0,0 +1,5 @@ +import sys + + +def is_windows_os() -> bool: + return sys.platform.startswith("win") diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index a3005ad3128..8b88b000b30 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -1,12 +1,12 @@ import os -import sys + +from monkey_island.cc.environment.utils import is_windows_os __author__ = "itay.mizeretz" def get_default_data_dir() -> str: - is_windows_os = sys.platform.startswith("win") - if is_windows_os: + if is_windows_os(): return r"%AppData%\monkey_island" else: return r"$HOME/.monkey_island" From dc129c017b32bec3ebe56d374492f77d9f38e58a Mon Sep 17 00:00:00 2001 From: shreyamalviya Date: Thu, 20 May 2021 16:43:36 +0530 Subject: [PATCH 17/37] Fix unit test in test_consts.py --- monkey/tests/unit_tests/monkey_island/cc/test_consts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/test_consts.py b/monkey/tests/unit_tests/monkey_island/cc/test_consts.py index eebb7414f34..993ddaa6447 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/test_consts.py +++ b/monkey/tests/unit_tests/monkey_island/cc/test_consts.py @@ -5,8 +5,8 @@ def test_default_server_config_file_path(): if platform.system() == "Windows": - server_file_path = consts.MONKEY_ISLAND_ABS_PATH + r"\cc\server_config.json" + server_file_path = f"{consts.DEFAULT_DATA_DIR}\\{consts.SERVER_CONFIG_FILENAME}" else: - server_file_path = consts.MONKEY_ISLAND_ABS_PATH + "/cc/server_config.json" + server_file_path = f"{consts.DEFAULT_DATA_DIR}/{consts.SERVER_CONFIG_FILENAME}" assert consts.DEFAULT_SERVER_CONFIG_PATH == server_file_path From 4640a760ff46cbe21bd30ea98a16253fba7dfb0e Mon Sep 17 00:00:00 2001 From: shreyamalviya Date: Thu, 20 May 2021 16:53:31 +0530 Subject: [PATCH 18/37] Update CHANGELOG (create data dir on island init) --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3021263d47c..4cd76b3ccb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Improved the structure of unit tests by scoping fixtures only to relevant modules instead of having a one huge fixture file, improved and renamed the directory structure of unit tests and unit test infrastructure. #1178 +- Create/check data directory on Island init. #1170 ### Removed - Relevant dead code as reported by Vulture. #1149 From b4708fc2cbc4f954f89a00f5e1eea7f1e3acaa91 Mon Sep 17 00:00:00 2001 From: shreyamalviya Date: Thu, 20 May 2021 16:55:27 +0530 Subject: [PATCH 19/37] Import Windows specific modules only on Windows systems --- .../cc/environment/windows_permissions.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/environment/windows_permissions.py b/monkey/monkey_island/cc/environment/windows_permissions.py index 5d4913151fb..d17947a2e43 100644 --- a/monkey/monkey_island/cc/environment/windows_permissions.py +++ b/monkey/monkey_island/cc/environment/windows_permissions.py @@ -1,7 +1,10 @@ -import ntsecuritycon -import win32api -import win32con -import win32security +from monkey_island.cc.environment.utils import is_windows_os + +if is_windows_os(): + import ntsecuritycon + import win32api + import win32con + import win32security def set_full_folder_access(folder_path: str) -> None: From 9705c73978226e02d5759d1087914d58ff8e5b24 Mon Sep 17 00:00:00 2001 From: Shreya Date: Fri, 21 May 2021 18:57:40 +0530 Subject: [PATCH 20/37] Remove code in environment_config.py which was calling `write_default_server_config_to_file()` This was causing an error in the unit tests, and is handled in monkey_island.py (commit 3de620e3). --- monkey/monkey_island/cc/environment/environment_config.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/monkey/monkey_island/cc/environment/environment_config.py b/monkey/monkey_island/cc/environment/environment_config.py index 5befa202b92..b013bdcf302 100644 --- a/monkey/monkey_island/cc/environment/environment_config.py +++ b/monkey/monkey_island/cc/environment/environment_config.py @@ -2,10 +2,8 @@ import json import os -from pathlib import Path from typing import Dict, List -import monkey_island.cc.environment.server_config_generator as server_config_generator from monkey_island.cc.environment.user_creds import UserCreds from monkey_island.cc.resources.auth.auth_user import User from monkey_island.cc.resources.auth.user_store import UserStore @@ -24,8 +22,6 @@ def __init__(self, file_path): def _load_from_file(self, file_path): file_path = os.path.expanduser(file_path) - if not Path(file_path).is_file(): - server_config_generator.write_default_server_config_to_file(file_path) with open(file_path, "r") as f: config_content = f.read() From 23b04920b26b7a54b501d82f282496c331e06f50 Mon Sep 17 00:00:00 2001 From: Shreya Date: Fri, 21 May 2021 19:18:08 +0530 Subject: [PATCH 21/37] Remove unit test related to code removed in the previous commit --- .../cc/environment/test_environment_config.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/environment/test_environment_config.py b/monkey/tests/unit_tests/monkey_island/cc/environment/test_environment_config.py index 481e871ae77..0e3efda04b7 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/environment/test_environment_config.py +++ b/monkey/tests/unit_tests/monkey_island/cc/environment/test_environment_config.py @@ -93,15 +93,3 @@ def test_get_users(standard_with_credentials): assert users[0].id == 1 assert users[0].username == "test" assert users[0].secret == "abcdef" - - -def test_generate_default_file(config_file): - environment_config = EnvironmentConfig(config_file) - - assert os.path.isfile(config_file) - - assert environment_config.server_config == "password" - assert environment_config.deployment == "develop" - assert environment_config.user_creds.username == "" - assert environment_config.user_creds.password_hash == "" - assert environment_config.aws is None From 43d919a3ebde5b036811936670719bc71ddd86ae Mon Sep 17 00:00:00 2001 From: Shreya Date: Fri, 21 May 2021 20:09:17 +0530 Subject: [PATCH 22/37] Create default server_config.json when running unit tests --- monkey/monkey_island/cc/models/__init__.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py index 602d815c416..a46e5bd7697 100644 --- a/monkey/monkey_island/cc/models/__init__.py +++ b/monkey/monkey_island/cc/models/__init__.py @@ -1,16 +1,24 @@ from mongoengine import connect -import monkey_island.cc.environment.environment_singleton as env_singleton +# Needed so that a server_config.json file exists at the default path, +# otherwise, unit tests will error while importing `env_singleton` below. +from monkey_island.cc.environment.server_config_generator import ( # noqa: E402 + create_default_server_config_file, +) + +create_default_server_config_file() + +import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 -from .command_control_channel import CommandControlChannel # noqa: F401 +from .command_control_channel import CommandControlChannel # noqa: F401, E402 # Order of importing matters here, for registering the embedded and referenced documents before # using them. -from .config import Config # noqa: F401 -from .creds import Creds # noqa: F401 -from .monkey import Monkey # noqa: F401 -from .monkey_ttl import MonkeyTtl # noqa: F401 -from .pba_results import PbaResults # noqa: F401 +from .config import Config # noqa: F401, E402 +from .creds import Creds # noqa: F401, E402 +from .monkey import Monkey # noqa: F401, E402 +from .monkey_ttl import MonkeyTtl # noqa: F401, E402 +from .pba_results import PbaResults # noqa: F401, E402 connect( db=env_singleton.env.mongo_db_name, From 70b9a9f6b73a0314ad34ef333b804410246a18dd Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 11:05:15 +0300 Subject: [PATCH 23/37] Refactored config_loader.py and server_config_handler.py into a single file, with server config write/read operations --- .../cc/environment/server_config_generator.py | 21 ---------- .../cc/environment/server_config_handler.py | 39 +++++++++++++++++++ .../monkey_island/cc/server_utils/consts.py | 2 + monkey/monkey_island/config_loader.py | 25 ------------ .../monkey_island/cc/environment/__init__.py | 0 .../test_server_config_handler.py} | 8 ++-- 6 files changed, 45 insertions(+), 50 deletions(-) delete mode 100644 monkey/monkey_island/cc/environment/server_config_generator.py create mode 100644 monkey/monkey_island/cc/environment/server_config_handler.py delete mode 100644 monkey/monkey_island/config_loader.py create mode 100644 monkey/tests/unit_tests/monkey_island/cc/environment/__init__.py rename monkey/tests/unit_tests/monkey_island/{test_config_loader.py => cc/environment/test_server_config_handler.py} (64%) diff --git a/monkey/monkey_island/cc/environment/server_config_generator.py b/monkey/monkey_island/cc/environment/server_config_generator.py deleted file mode 100644 index 17ee4a50c09..00000000000 --- a/monkey/monkey_island/cc/environment/server_config_generator.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -from pathlib import Path - -from monkey_island.cc.environment.data_dir_generator import create_data_dir -from monkey_island.cc.server_utils.consts import ( - DEFAULT_DATA_DIR, - DEFAULT_DEVELOP_SERVER_CONFIG_PATH, - DEFAULT_SERVER_CONFIG_PATH, -) - - -def create_default_server_config_file() -> str: - if not os.path.isfile(DEFAULT_SERVER_CONFIG_PATH): - create_data_dir(DEFAULT_DATA_DIR, False) - write_default_server_config_to_file(DEFAULT_SERVER_CONFIG_PATH) - return DEFAULT_SERVER_CONFIG_PATH - - -def write_default_server_config_to_file(path: str) -> None: - default_config = Path(DEFAULT_DEVELOP_SERVER_CONFIG_PATH).read_text() - Path(path).write_text(default_config) diff --git a/monkey/monkey_island/cc/environment/server_config_handler.py b/monkey/monkey_island/cc/environment/server_config_handler.py new file mode 100644 index 00000000000..35cc91f3e84 --- /dev/null +++ b/monkey/monkey_island/cc/environment/server_config_handler.py @@ -0,0 +1,39 @@ +import json +import os +from pathlib import Path + +from monkey_island.cc.server_utils.consts import ( + DEFAULT_DATA_DIR, + DEFAULT_DEVELOP_SERVER_CONFIG_PATH, + DEFAULT_LOG_LEVEL, + DEFAULT_SERVER_CONFIG_PATH, +) + + +def create_default_server_config_file() -> None: + if not os.path.isfile(DEFAULT_SERVER_CONFIG_PATH): + write_default_server_config_to_file(DEFAULT_SERVER_CONFIG_PATH) + + +def write_default_server_config_to_file(path: str) -> None: + default_config = Path(DEFAULT_DEVELOP_SERVER_CONFIG_PATH).read_text() + Path(path).write_text(default_config) + + +def load_server_config_from_file(server_config_path): + with open(server_config_path, "r") as f: + config_content = f.read() + config = json.loads(config_content) + add_default_values_to_config(config) + + return config + + +def add_default_values_to_config(config): + config["data_dir"] = os.path.abspath( + os.path.expanduser(os.path.expandvars(config.get("data_dir", DEFAULT_DATA_DIR))) + ) + + config.setdefault("log_level", DEFAULT_LOG_LEVEL) + + return config diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index 8b88b000b30..a62bb98abb7 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -14,6 +14,8 @@ def get_default_data_dir() -> str: SERVER_CONFIG_FILENAME = "server_config.json" +DEFAULT_LOG_LEVEL = "INFO" + MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island") DEFAULT_DATA_DIR = os.path.expandvars(get_default_data_dir()) diff --git a/monkey/monkey_island/config_loader.py b/monkey/monkey_island/config_loader.py deleted file mode 100644 index aaa9185d700..00000000000 --- a/monkey/monkey_island/config_loader.py +++ /dev/null @@ -1,25 +0,0 @@ -import json -import os - -from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR - -DEFAULT_LOG_LEVEL = "INFO" - - -def load_server_config_from_file(server_config_path): - with open(server_config_path, "r") as f: - config_content = f.read() - config = json.loads(config_content) - add_default_values_to_config(config) - - return config - - -def add_default_values_to_config(config): - config["data_dir"] = os.path.abspath( - os.path.expanduser(os.path.expandvars(config.get("data_dir", DEFAULT_DATA_DIR))) - ) - - config.setdefault("log_level", DEFAULT_LOG_LEVEL) - - return config diff --git a/monkey/tests/unit_tests/monkey_island/cc/environment/__init__.py b/monkey/tests/unit_tests/monkey_island/cc/environment/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/monkey/tests/unit_tests/monkey_island/test_config_loader.py b/monkey/tests/unit_tests/monkey_island/cc/environment/test_server_config_handler.py similarity index 64% rename from monkey/tests/unit_tests/monkey_island/test_config_loader.py rename to monkey/tests/unit_tests/monkey_island/cc/environment/test_server_config_handler.py index 20c330f6a91..acd89d84fb8 100644 --- a/monkey/tests/unit_tests/monkey_island/test_config_loader.py +++ b/monkey/tests/unit_tests/monkey_island/cc/environment/test_server_config_handler.py @@ -1,11 +1,11 @@ import os -from monkey_island import config_loader +from monkey_island.cc.environment import server_config_handler from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR def test_load_server_config_from_file(test_server_config, mock_home_env): - config = config_loader.load_server_config_from_file(test_server_config) + config = server_config_handler.load_server_config_from_file(test_server_config) assert config["data_dir"] == os.path.join(mock_home_env, ".monkey_island") assert config["log_level"] == "NOTICE" @@ -13,7 +13,7 @@ def test_load_server_config_from_file(test_server_config, mock_home_env): def test_default_log_level(): test_config = {} - config = config_loader.add_default_values_to_config(test_config) + config = server_config_handler.add_default_values_to_config(test_config) assert "log_level" in config assert config["log_level"] == "INFO" @@ -21,7 +21,7 @@ def test_default_log_level(): def test_default_data_dir(mock_home_env): test_config = {} - config = config_loader.add_default_values_to_config(test_config) + config = server_config_handler.add_default_values_to_config(test_config) assert "data_dir" in config assert config["data_dir"] == DEFAULT_DATA_DIR From 8afe93747f47d73141330d4fe00c6cc6bdf573ca Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 10:27:16 +0300 Subject: [PATCH 24/37] Improved monkey_island.py setup readability by extracting workflows of server_config setup by cmd_arguments and setup of server_config using defaults --- monkey/monkey_island.py | 17 +++++------------ monkey/monkey_island/setup/config_setup.py | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 monkey/monkey_island/setup/config_setup.py diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 76399b6ff00..662ebe89202 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -1,32 +1,25 @@ from gevent import monkey as gevent_monkey from monkey_island.cc.arg_parser import parse_cli_args +from monkey_island.setup.config_setup import setup_config_by_cmd_arg, setup_default_config gevent_monkey.patch_all() import json # noqa: E402 -import os # noqa: E402 -import monkey_island.cc.environment.server_config_generator as server_config_generator # noqa: E402 -from monkey_island import config_loader # noqa: E402 -from monkey_island.cc.environment.data_dir_generator import create_data_dir # noqa: E402 from monkey_island.cc.server_utils.island_logger import setup_logging # noqa: E402 if "__main__" == __name__: island_args = parse_cli_args() - # This is here in order to catch EVERYTHING, some functions are being called on - # imports, so the log init needs to be first. try: if island_args.server_config: - server_config_path = os.path.expanduser(island_args.server_config) + config, server_config_path = setup_config_by_cmd_arg(island_args.server_config) else: - server_config_path = server_config_generator.create_default_server_config_file() - - config = config_loader.load_server_config_from_file(server_config_path) - - create_data_dir(config["data_dir"], True) + config, server_config_path = setup_default_config() + # This is here in order to catch EVERYTHING, some functions are being called on + # imports, so the log init needs to be first. setup_logging(config["data_dir"], config["log_level"]) except OSError as ex: diff --git a/monkey/monkey_island/setup/config_setup.py b/monkey/monkey_island/setup/config_setup.py new file mode 100644 index 00000000000..9980c66cd15 --- /dev/null +++ b/monkey/monkey_island/setup/config_setup.py @@ -0,0 +1,21 @@ +import os +from typing import Tuple + +from monkey_island.cc.environment import server_config_handler +from monkey_island.cc.environment.data_dir_generator import create_data_dir # noqa: E402 +from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH + + +def setup_config_by_cmd_arg(server_config_path) -> Tuple[dict, str]: + server_config_path = os.path.expanduser(server_config_path) + config = server_config_handler.load_server_config_from_file(server_config_path) + create_data_dir(config["data_dir"], create_parent_dirs=True) + return config, server_config_path + + +def setup_default_config() -> Tuple[dict, str]: + server_config_path = DEFAULT_SERVER_CONFIG_PATH + create_data_dir(DEFAULT_DATA_DIR, create_parent_dirs=False) + server_config_handler.create_default_server_config_file() + config = server_config_handler.load_server_config_from_file(server_config_path) + return config, server_config_path From 5f88f6f04be429be9f356dbafdee578c35a875e2 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 12:13:33 +0300 Subject: [PATCH 25/37] Moved a comment to a proper place --- monkey/monkey_island.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 662ebe89202..e3d8348d1dc 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -12,14 +12,14 @@ if "__main__" == __name__: island_args = parse_cli_args() + # This is here in order to catch EVERYTHING, some functions are being called on + # imports, so the log init needs to be first. try: if island_args.server_config: config, server_config_path = setup_config_by_cmd_arg(island_args.server_config) else: config, server_config_path = setup_default_config() - # This is here in order to catch EVERYTHING, some functions are being called on - # imports, so the log init needs to be first. setup_logging(config["data_dir"], config["log_level"]) except OSError as ex: From 3098ac1459f841aaead2d587beda14c44bdef8b5 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 12:25:10 +0300 Subject: [PATCH 26/37] Fixed a failing environment setup in unit tests --- monkey/monkey_island/cc/models/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py index a46e5bd7697..9b244a97435 100644 --- a/monkey/monkey_island/cc/models/__init__.py +++ b/monkey/monkey_island/cc/models/__init__.py @@ -2,11 +2,12 @@ # Needed so that a server_config.json file exists at the default path, # otherwise, unit tests will error while importing `env_singleton` below. -from monkey_island.cc.environment.server_config_generator import ( # noqa: E402 - create_default_server_config_file, -) +from monkey_island.cc.environment import data_dir_generator, server_config_handler # noqa: E402 + +from ..server_utils.consts import DEFAULT_DATA_DIR -create_default_server_config_file() +data_dir_generator.create_data_dir(DEFAULT_DATA_DIR, False) +server_config_handler.create_default_server_config_file() import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 From d33c5d68d861640f63fcef01defeab3bd24ab3d3 Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 24 May 2021 15:17:22 +0530 Subject: [PATCH 27/37] Use `os.path.expandvars()` on server config argument --- monkey/monkey_island/setup/config_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/setup/config_setup.py b/monkey/monkey_island/setup/config_setup.py index 9980c66cd15..05f410fa88a 100644 --- a/monkey/monkey_island/setup/config_setup.py +++ b/monkey/monkey_island/setup/config_setup.py @@ -7,7 +7,7 @@ def setup_config_by_cmd_arg(server_config_path) -> Tuple[dict, str]: - server_config_path = os.path.expanduser(server_config_path) + server_config_path = os.path.expandvars(os.path.expanduser(server_config_path)) config = server_config_handler.load_server_config_from_file(server_config_path) create_data_dir(config["data_dir"], create_parent_dirs=True) return config, server_config_path From 0a7cf1d5ee838915d60d593e4bd19bce02e88a0c Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 16:42:30 +0300 Subject: [PATCH 28/37] Improved readability in the arg_parser.py by extracting defaults to the argparser flag section --- monkey/monkey_island/cc/arg_parser.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/monkey/monkey_island/cc/arg_parser.py b/monkey/monkey_island/cc/arg_parser.py index 457ffbac254..8de146513e6 100644 --- a/monkey/monkey_island/cc/arg_parser.py +++ b/monkey/monkey_island/cc/arg_parser.py @@ -1,18 +1,15 @@ +from dataclasses import dataclass + from monkey_island.cc.server_utils.consts import ( DEFAULT_SERVER_CONFIG_PATH, DEFAULT_SHOULD_SETUP_ONLY, ) +@dataclass class IslandCmdArgs: - setup_only: bool = DEFAULT_SHOULD_SETUP_ONLY - server_config_path: str = DEFAULT_SERVER_CONFIG_PATH - - def __init__(self, setup_only: None, server_config_path: None): - if setup_only: - self.setup_only = setup_only - if server_config_path: - self.server_config_path = server_config_path + setup_only: bool + server_config_path: str def parse_cli_args() -> IslandCmdArgs: @@ -29,9 +26,13 @@ def parse_cli_args() -> IslandCmdArgs: help="Pass this flag to cause the Island to setup and exit without actually starting. " "This is useful for preparing Island to boot faster later-on, so for " "compiling/packaging Islands.", + default=DEFAULT_SHOULD_SETUP_ONLY, ) parser.add_argument( - "--server-config", action="store", help="The path to the server configuration file." + "--server-config", + action="store", + help="The path to the server configuration file.", + default=DEFAULT_SERVER_CONFIG_PATH, ) args = parser.parse_args() From 2b257f00128d61505828b501ee70e55c69d50178 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 16:44:43 +0300 Subject: [PATCH 29/37] Fixed bugs merge bugs where structures are being accessed in an outdated ways --- monkey/monkey_island.py | 4 ++-- monkey/monkey_island/setup/config_setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 200dd9b1fb2..3ea35eed1c5 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -15,8 +15,8 @@ # This is here in order to catch EVERYTHING, some functions are being called on # imports, so the log init needs to be first. try: - if island_args.server_config: - config, server_config_path = setup_config_by_cmd_arg(island_args.server_config) + if island_args.server_config_path: + config, server_config_path = setup_config_by_cmd_arg(island_args.server_config_path) else: config, server_config_path = setup_default_config() diff --git a/monkey/monkey_island/setup/config_setup.py b/monkey/monkey_island/setup/config_setup.py index 12f073f642f..5c9625ac476 100644 --- a/monkey/monkey_island/setup/config_setup.py +++ b/monkey/monkey_island/setup/config_setup.py @@ -11,7 +11,7 @@ def setup_config_by_cmd_arg(server_config_path) -> Tuple[IslandConfigOptions, st server_config_path = os.path.expandvars(os.path.expanduser(server_config_path)) config = server_config_handler.load_server_config_from_file(server_config_path) - create_data_dir(config["data_dir"], create_parent_dirs=True) + create_data_dir(config.data_dir, create_parent_dirs=True) return config, server_config_path From d273e8585849f9db3d1c22b0ea99ea85ba146498 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 17:09:14 +0300 Subject: [PATCH 30/37] Fixed bugs in argument parser passing default server config path even though server path is not specified. Island thinks that server config path was specified. --- monkey/monkey_island.py | 2 +- monkey/monkey_island/cc/arg_parser.py | 11 +---------- monkey/monkey_island/cc/server_utils/consts.py | 1 - 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 3ea35eed1c5..9367cc04a50 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -32,4 +32,4 @@ from monkey_island.cc.main import main # noqa: E402 - main(island_args.setup_only, island_args.server_config_path, config) + main(island_args.setup_only, server_config_path, config) diff --git a/monkey/monkey_island/cc/arg_parser.py b/monkey/monkey_island/cc/arg_parser.py index 8de146513e6..6176580801e 100644 --- a/monkey/monkey_island/cc/arg_parser.py +++ b/monkey/monkey_island/cc/arg_parser.py @@ -1,10 +1,5 @@ from dataclasses import dataclass -from monkey_island.cc.server_utils.consts import ( - DEFAULT_SERVER_CONFIG_PATH, - DEFAULT_SHOULD_SETUP_ONLY, -) - @dataclass class IslandCmdArgs: @@ -26,13 +21,9 @@ def parse_cli_args() -> IslandCmdArgs: help="Pass this flag to cause the Island to setup and exit without actually starting. " "This is useful for preparing Island to boot faster later-on, so for " "compiling/packaging Islands.", - default=DEFAULT_SHOULD_SETUP_ONLY, ) parser.add_argument( - "--server-config", - action="store", - help="The path to the server configuration file.", - default=DEFAULT_SERVER_CONFIG_PATH, + "--server-config", action="store", help="The path to the server configuration file." ) args = parser.parse_args() diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index 4178928e2d4..bc99b43940c 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -30,4 +30,3 @@ def get_default_data_dir() -> str: DEFAULT_LOG_LEVEL = "INFO" DEFAULT_START_MONGO_DB = True -DEFAULT_SHOULD_SETUP_ONLY = False From 3a800d9a442c59b1cf098c4ccb3ef746badedede Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 18:06:53 +0300 Subject: [PATCH 31/37] Fixed a bug where models were getting imported without mongodb connection and failing --- monkey/monkey_island.py | 8 +++++++- .../monkey_island/cc/environment/environment_singleton.py | 6 ------ monkey/monkey_island/cc/main.py | 4 +--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 9367cc04a50..3b681bf89e6 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -1,5 +1,6 @@ from gevent import monkey as gevent_monkey +import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 from monkey_island.cc.arg_parser import parse_cli_args from monkey_island.setup.config_setup import setup_config_by_cmd_arg, setup_default_config @@ -30,6 +31,11 @@ print(f"Error loading server config: {ex}") exit(1) + # We need to initialize environment singleton before importing main, + # because main imports modules from monkey_island/cc/models and models need a connection to the + # mongodb. Mongodb connection parameters are initialized in environment singleton. + env_singleton.initialize_from_file(server_config_path) + from monkey_island.cc.main import main # noqa: E402 - main(island_args.setup_only, server_config_path, config) + main(island_args.setup_only, config) diff --git a/monkey/monkey_island/cc/environment/environment_singleton.py b/monkey/monkey_island/cc/environment/environment_singleton.py index e7e316ac54d..f1a6a2a39a3 100644 --- a/monkey/monkey_island/cc/environment/environment_singleton.py +++ b/monkey/monkey_island/cc/environment/environment_singleton.py @@ -2,7 +2,6 @@ import monkey_island.cc.resources.auth.user_store as user_store from monkey_island.cc.environment import EnvironmentConfig, aws, password, standard -from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH __author__ = "itay.mizeretz" @@ -48,8 +47,3 @@ def initialize_from_file(file_path): except Exception: logger.error("Failed initializing environment", exc_info=True) raise - - -# TODO: This is only needed so that unit tests pass. After PR #848 is merged, we may be -# able to remove this line. -initialize_from_file(DEFAULT_SERVER_CONFIG_PATH) diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index 21b1184471d..293dd871bef 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -35,9 +35,7 @@ MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" -def main(setup_only: bool, server_config_path: str, config_options: IslandConfigOptions): - - env_singleton.initialize_from_file(server_config_path) +def main(setup_only: bool, config_options: IslandConfigOptions): initialize_encryptor(config_options.data_dir) initialize_services(config_options.data_dir) From 9337c68ae0d90da5a1917e3c262991d3e6633424 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 18:08:26 +0300 Subject: [PATCH 32/37] Removed the infrastructure from starting main.py - it can no longer be the entrypoint to the application --- monkey/monkey_island/cc/main.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index 293dd871bef..df015863ba4 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -120,7 +120,3 @@ def assert_mongo_db_version(mongo_url): sys.exit(-1) else: logger.info("Mongo DB version OK. Got {0}".format(str(server_version))) - - -if __name__ == "__main__": - main() From 057a579d62c44fa5370f68a25f23fdf19c1144fc Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 25 May 2021 09:49:34 +0300 Subject: [PATCH 33/37] Rolled back the changes that made default server config on model packages import --- monkey/monkey_island/cc/models/__init__.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py index 9b244a97435..3b23fa89f7c 100644 --- a/monkey/monkey_island/cc/models/__init__.py +++ b/monkey/monkey_island/cc/models/__init__.py @@ -1,14 +1,5 @@ from mongoengine import connect -# Needed so that a server_config.json file exists at the default path, -# otherwise, unit tests will error while importing `env_singleton` below. -from monkey_island.cc.environment import data_dir_generator, server_config_handler # noqa: E402 - -from ..server_utils.consts import DEFAULT_DATA_DIR - -data_dir_generator.create_data_dir(DEFAULT_DATA_DIR, False) -server_config_handler.create_default_server_config_file() - import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 from .command_control_channel import CommandControlChannel # noqa: F401, E402 From 5b7329b3d151c78f3b3d75270429bcbb4378a7bc Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 25 May 2021 10:21:27 +0300 Subject: [PATCH 34/37] Fixed stack-overflow that has been happening due to gevent unpatched imports --- monkey/monkey_island.py | 12 +++++------- monkey/monkey_island/setup/gevent_setup.py | 6 ++++++ 2 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 monkey/monkey_island/setup/gevent_setup.py diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 3b681bf89e6..554e5a442b2 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -1,14 +1,12 @@ -from gevent import monkey as gevent_monkey - -import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 -from monkey_island.cc.arg_parser import parse_cli_args -from monkey_island.setup.config_setup import setup_config_by_cmd_arg, setup_default_config - -gevent_monkey.patch_all() +# This import patches other imports and needs to be first +import monkey_island.setup.gevent_setup # noqa: E402 F401 isort:skip import json # noqa: E402 +import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 +from monkey_island.cc.arg_parser import parse_cli_args from monkey_island.cc.server_utils.island_logger import setup_logging # noqa: E402 +from monkey_island.setup.config_setup import setup_config_by_cmd_arg, setup_default_config if "__main__" == __name__: island_args = parse_cli_args() diff --git a/monkey/monkey_island/setup/gevent_setup.py b/monkey/monkey_island/setup/gevent_setup.py new file mode 100644 index 00000000000..9fa2b47f9d4 --- /dev/null +++ b/monkey/monkey_island/setup/gevent_setup.py @@ -0,0 +1,6 @@ +from gevent import monkey as gevent_monkey + +# We need to monkeypatch before any other imports to +# make standard libraries compatible with gevent. +# http://www.gevent.org/api/gevent.monkey.html +gevent_monkey.patch_all() From 17e994c8d8260c39b576506e4017357e9bf88f3b Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 25 May 2021 11:12:58 +0300 Subject: [PATCH 35/37] Fixed UT's for models by mocking an environment singleton with needed mongodb parameters --- monkey/monkey_island/cc/models/__init__.py | 1 + monkey/tests/unit_tests/monkey_island/cc/conftest.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py index 3b23fa89f7c..d787a59ef7e 100644 --- a/monkey/monkey_island/cc/models/__init__.py +++ b/monkey/monkey_island/cc/models/__init__.py @@ -12,6 +12,7 @@ from .monkey_ttl import MonkeyTtl # noqa: F401, E402 from .pba_results import PbaResults # noqa: F401, E402 +# TODO refactor into explicit call when implementing mongodb startup connect( db=env_singleton.env.mongo_db_name, host=env_singleton.env.mongo_db_host, diff --git a/monkey/tests/unit_tests/monkey_island/cc/conftest.py b/monkey/tests/unit_tests/monkey_island/cc/conftest.py index af35c9b25df..4bccfd862b7 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/conftest.py +++ b/monkey/tests/unit_tests/monkey_island/cc/conftest.py @@ -1,3 +1,11 @@ +import monkey_island.cc.environment.environment_singleton as env_singleton +from monkey_island.cc.environment.testing import TestingEnvironment + +# Mock environment singleton because it contains mongodb parameters +# needed for model tests. See monkey/monkey_island/cc/models/__init__.py +env_config = {} +env_singleton.env = TestingEnvironment(env_config) + # Without these imports pytests can't use fixtures, # because they are not found -from tests.unit_tests.monkey_island.cc.mongomock_fixtures import * # noqa: F401,F403 +from tests.unit_tests.monkey_island.cc.mongomock_fixtures import * # noqa: F401,F403,E402 From a89a62cb5c73eceb1da3cea381003f281fa4969c Mon Sep 17 00:00:00 2001 From: Shreya Date: Wed, 26 May 2021 12:55:23 +0530 Subject: [PATCH 36/37] Remove unneeded "noqa" statements --- monkey/monkey_island.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 554e5a442b2..82b930a3e14 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -1,11 +1,11 @@ # This import patches other imports and needs to be first -import monkey_island.setup.gevent_setup # noqa: E402 F401 isort:skip +import monkey_island.setup.gevent_setup # noqa: F401 isort:skip -import json # noqa: E402 +import json -import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 +import monkey_island.cc.environment.environment_singleton as env_singleton from monkey_island.cc.arg_parser import parse_cli_args -from monkey_island.cc.server_utils.island_logger import setup_logging # noqa: E402 +from monkey_island.cc.server_utils.island_logger import setup_logging from monkey_island.setup.config_setup import setup_config_by_cmd_arg, setup_default_config if "__main__" == __name__: From baee74b761b2d343688ba898f161c52215b47c10 Mon Sep 17 00:00:00 2001 From: Shreya Date: Wed, 26 May 2021 15:22:52 +0530 Subject: [PATCH 37/37] Add exception messages during data directory creation --- .../cc/environment/data_dir_generator.py | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/monkey/monkey_island/cc/environment/data_dir_generator.py b/monkey/monkey_island/cc/environment/data_dir_generator.py index 98907d63948..58e16d4b797 100644 --- a/monkey/monkey_island/cc/environment/data_dir_generator.py +++ b/monkey/monkey_island/cc/environment/data_dir_generator.py @@ -1,14 +1,30 @@ +import logging import os from monkey_island.cc.environment.utils import is_windows_os from monkey_island.cc.environment.windows_permissions import set_full_folder_access +LOG = logging.getLogger(__name__) + def create_data_dir(data_dir: str, create_parent_dirs: bool) -> None: if not os.path.isdir(data_dir): - if create_parent_dirs: - os.makedirs(data_dir, mode=0o700) - else: - os.mkdir(data_dir, mode=0o700) + try: + if create_parent_dirs: + os.makedirs(data_dir, mode=0o700) + else: + os.mkdir(data_dir, mode=0o700) + except Exception as ex: + LOG.error( + f'Could not create data directory at "{data_dir}" (maybe `$HOME` could not be ' + f"resolved?): {str(ex)}" + ) + if is_windows_os(): # `mode=0o700` doesn't work on Windows - set_full_folder_access(folder_path=data_dir) + try: + set_full_folder_access(folder_path=data_dir) + except Exception as ex: + LOG.error( + f'Data directory was created at "{data_dir}" but permissions could not be ' + f"set successfully: {str(ex)}" + )