From a2b65e0c174fe3a75d3ea704c467abeea3c9a163 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Fri, 1 Mar 2024 12:03:27 -0300 Subject: [PATCH 1/2] Case sensitivity fix - Remove case sensitivity for modules configuration. Adding unit tests - Minor logging improvements - Always set the job organization id for lean config --- lean/commands/cloud/live/deploy.py | 4 +- lean/commands/live/deploy.py | 6 +-- lean/components/config/lean_config_manager.py | 3 ++ lean/components/util/json_modules_handler.py | 31 ++++++++---- lean/components/util/organization_manager.py | 17 +++++-- lean/models/json_module.py | 13 +++-- .../cloud/live/test_cloud_live_commands.py | 4 +- tests/commands/test_init.py | 1 + tests/commands/test_live.py | 2 +- .../util/test_json_modules_handler.py | 49 +++++++++++++++++++ .../util/test_organization_manager.py | 4 +- 11 files changed, 108 insertions(+), 26 deletions(-) create mode 100644 tests/components/util/test_json_modules_handler.py diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index b038bdf0..59cff148 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -277,12 +277,12 @@ def deploy(project: str, if cash_balance_option != LiveInitialStateInput.NotSupported: live_cash_balance = configure_initial_cash_balance(logger, cash_balance_option, live_cash_balance, last_cash) elif live_cash_balance is not None and live_cash_balance != "": - raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_instance.get_name()}") + raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_instance}") if holdings_option != LiveInitialStateInput.NotSupported: live_holdings = configure_initial_holdings(logger, holdings_option, live_holdings, last_holdings) elif live_holdings is not None and live_holdings != "": - raise RuntimeError(f"Custom portfolio holdings setting is not available for {brokerage_instance.get_name()}") + raise RuntimeError(f"Custom portfolio holdings setting is not available for {brokerage_instance}") else: # let the user choose the brokerage diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 1882084f..ffc33473 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -224,7 +224,7 @@ def deploy(project: Path, if environment_name in lean_config["environments"]: lean_environment = lean_config["environments"][environment_name] - for key in ["live-mode-brokerage", "data-queue-handler"]: + for key in ["live-mode-brokerage", "data-queue-handler", "history-provider"]: if key not in lean_environment: raise MoreInfoError(f"The '{environment_name}' environment does not specify a {rename_internal_config_to_user_friendly_format(key)}", "https://www.lean.io/docs/v2/lean-cli/live-trading/algorithm-control") @@ -319,12 +319,12 @@ def deploy(project: Path, if cash_balance_option != LiveInitialStateInput.NotSupported: live_cash_balance = configure_initial_cash_balance(logger, cash_balance_option, live_cash_balance, last_cash) elif live_cash_balance is not None and live_cash_balance != "": - raise RuntimeError(f"Custom cash balance setting is not available for {brokerage}") + raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_instance}") if holdings_option != LiveInitialStateInput.NotSupported: live_holdings = configure_initial_holdings(logger, holdings_option, live_holdings, last_holdings) elif live_holdings is not None and live_holdings != "": - raise RuntimeError(f"Custom portfolio holdings setting is not available for {brokerage}") + raise RuntimeError(f"Custom portfolio holdings setting is not available for {brokerage_instance}") if live_cash_balance: lean_config["live-cash-balance"] = live_cash_balance diff --git a/lean/components/config/lean_config_manager.py b/lean/components/config/lean_config_manager.py index 4dbdb823..a33e7bc7 100644 --- a/lean/components/config/lean_config_manager.py +++ b/lean/components/config/lean_config_manager.py @@ -235,11 +235,14 @@ def get_complete_lean_config(self, config["debugging"] = False config["debugging-method"] = "LocalCmdline" + from lean.components.util.organization_manager import get_organization + # The following key -> value pairs are added to the config unless they are already set by the user config_defaults = { "job-user-id": self._cli_config_manager.user_id.get_value(default="0"), "api-access-token": self._cli_config_manager.api_token.get_value(default=""), "job-project-id": self._project_config_manager.get_local_id(algorithm_file.parent), + "job-organization-id": get_organization(config), "ib-host": "127.0.0.1", "ib-port": "4002", diff --git a/lean/components/util/json_modules_handler.py b/lean/components/util/json_modules_handler.py index bcbc4796..6657db2c 100644 --- a/lean/components/util/json_modules_handler.py +++ b/lean/components/util/json_modules_handler.py @@ -44,30 +44,41 @@ def non_interactive_config_build_for_name(lean_config: Dict[str, Any], target_mo environment_name=environment_name) -def config_build_for_name(lean_config: Dict[str, Any], target_module_name: str, module_list: List[JsonModule], - properties: Dict[str, Any], logger: Logger, interactive: bool, - environment_name: str = None) -> JsonModule: +def find_module(target_module_name: str, module_list: List[JsonModule], logger: Logger) -> JsonModule: target_module: JsonModule = None + # because we compare str we normalize everything to lower case + target_module_name = target_module_name.lower() + module_class_name = target_module_name.rfind('.') for module in module_list: - if module.get_id() == target_module_name or module.get_name() == target_module_name: + # we search in the modules name and id + module_id = module.get_id().lower() + module_name = module.get_name().lower() + + if module_id == target_module_name or module_name == target_module_name: target_module = module break else: - index = target_module_name.rfind('.') - if (index != -1 and module.get_id() == target_module_name[index + 1:] - or module.get_name() == target_module_name[index + 1:]): + if (module_class_name != -1 and module_id == target_module_name[module_class_name + 1:] + or module_name == target_module_name[module_class_name + 1:]): target_module = module break if not target_module: for module in module_list: - if module.get_config_value_from_value(target_module_name): + # we search in the modules configuration values, this is for when the user provides an environment + if (module.is_value_in_config(target_module_name) + or module_class_name != -1 and module.is_value_in_config(target_module_name[module_class_name + 1:])): target_module = module if not target_module: raise RuntimeError(f"""Failed to resolve module for name: '{target_module_name}'""") - else: - logger.debug(f'Found module \'{target_module_name}\' from given name') + logger.debug(f'Found module \'{target_module_name}\' from given name') + return target_module + +def config_build_for_name(lean_config: Dict[str, Any], target_module_name: str, module_list: List[JsonModule], + properties: Dict[str, Any], logger: Logger, interactive: bool, + environment_name: str = None) -> JsonModule: + target_module = find_module(target_module_name, module_list, logger) target_module.config_build(lean_config, logger, interactive=interactive, properties=properties, environment_name=environment_name) _update_settings(logger, environment_name, target_module, lean_config) diff --git a/lean/components/util/organization_manager.py b/lean/components/util/organization_manager.py index 11f9ab54..2de76969 100644 --- a/lean/components/util/organization_manager.py +++ b/lean/components/util/organization_manager.py @@ -11,12 +11,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional +from typing import Optional, Dict, Any from lean.components.config.lean_config_manager import LeanConfigManager from lean.components.util.logger import Logger +def get_organization(config: Dict[str, Any]): + organization_id = None + if "job-organization-id" in config: + organization_id = config["job-organization-id"] + elif "organization-id" in config: + # for backwards compatibility with local platform and old lean cli setups + organization_id = config["organization-id"] + return organization_id + + class OrganizationManager: """The OrganizationManager class provides utilities to handle the working organization.""" @@ -24,7 +34,6 @@ def __init__(self, logger: Logger, lean_config_manager: LeanConfigManager) -> No """Creates a new OrganizationManager instance. :param logger: the logger to use to log messages with - :param api_client: the API client to use to fetch organizations info :param lean_config_manager: the LeanConfigManager to use to manipulate the lean configuration file """ self._logger = logger @@ -39,7 +48,7 @@ def get_working_organization_id(self) -> Optional[str]: """ if self._working_organization_id is None: lean_config = self._lean_config_manager.get_lean_config() - self._working_organization_id = lean_config.get("organization-id") + self._working_organization_id = get_organization(lean_config) return self._working_organization_id @@ -64,5 +73,7 @@ def configure_working_organization_id(self, organization_id: str) -> None: :param organization_id: the working organization di """ + self._lean_config_manager.set_properties({"job-organization-id": organization_id}) + # for backwards compatibility with local platform self._lean_config_manager.set_properties({"organization-id": organization_id}) self._working_organization_id = organization_id diff --git a/lean/models/json_module.py b/lean/models/json_module.py index cd1478be..071daac6 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -92,9 +92,16 @@ def get_config_value_from_name(self, target_name: str) -> str: if self._lean_configs[i]._id == target_name] return self._lean_configs[idx]._value - def get_config_value_from_value(self, value: str) -> bool: + def is_value_in_config(self, searched_value: str) -> bool: + searched_value = searched_value.lower() for i in range(len(self._lean_configs)): - if value in self._lean_configs[i]._value: + value = self._lean_configs[i]._value + if isinstance(value, str): + value = value.lower() + if isinstance(value, list): + value = [x.lower() for x in value] + + if searched_value in value: return True return False @@ -223,7 +230,7 @@ def config_build(self, def ensure_module_installed(self, organization_id: str) -> None: if not self._is_module_installed and self._installs: - container.logger.debug(f"JsonModule.ensure_module_installed(): installing module for module {self._id}: {self._product_id}") + container.logger.debug(f"JsonModule.ensure_module_installed(): installing module {self}: {self._product_id}") container.module_manager.install_module( self._product_id, organization_id) self._is_module_installed = True diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index f525ac2e..46b8390b 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -333,7 +333,7 @@ def test_cloud_live_deploy_with_live_holdings(brokerage: str, holdings: str) -> "--node", "live", "--auto-restart", "yes", "--notify-order-events", "no", "--notify-insights", "no", *options, "--data-provider-live", "QuantConnect"]) - if (brokerage != "Paper Trading" and holdings != "")\ + if (brokerage != "Paper Trading" and brokerage != "Binance" and holdings != "")\ or brokerage == "Terminal Link": # non-cloud brokerage assert result.exit_code != 0 api_client.live.start.assert_not_called() @@ -348,7 +348,7 @@ def test_cloud_live_deploy_with_live_holdings(brokerage: str, holdings: str) -> {"symbol": "AA", "symbolId": "AA 2T", "quantity": 2, "averagePrice": 20.35}] elif len(holding) == 1: holding_list = [{"symbol": "A", "symbolId": "A 2T", "quantity": 1, "averagePrice": 145.1}] - elif brokerage == "Paper Trading": + elif brokerage == "Paper Trading" or brokerage == "Binance": holding_list = [] api_client.live.start.assert_called_once_with(mock.ANY, diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index 331dff26..f39c8ce0 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -144,6 +144,7 @@ def test_init_creates_clean_config_file_from_repo() -> None: assert config_path.read_text(encoding="utf-8") == """ {{ "data-folder": "data", + "job-organization-id": "{0}", "organization-id": "{0}" }} """.format(_get_test_organization().id).strip() diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index 9690c7b1..e9c78c9f 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -1004,7 +1004,7 @@ def test_live_passes_live_holdings_to_lean_runner_when_given_as_option(brokerage result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", brokerage, "--live-holdings", holdings, "--data-provider-live", "Custom data only", *options]) - if brokerage not in ["Paper Trading", "Terminal Link"] and holdings != "": + if brokerage not in ["Paper Trading", "Terminal Link", "Binance"] and holdings != "": assert result.exit_code != 0 lean_runner.run_lean.start.assert_not_called() return diff --git a/tests/components/util/test_json_modules_handler.py b/tests/components/util/test_json_modules_handler.py new file mode 100644 index 00000000..0fd95922 --- /dev/null +++ b/tests/components/util/test_json_modules_handler.py @@ -0,0 +1,49 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from unittest.mock import MagicMock + +import pytest + +from lean.components.util.json_modules_handler import find_module +from lean.constants import MODULE_CLI_PLATFORM, MODULE_BROKERAGE +from lean.models.json_module import JsonModule +from tests.test_helpers import create_fake_lean_cli_directory + + +@pytest.mark.parametrize("id,display,search_name", [("ads", "binAnce", "BiNAnce"), + ("binAnce", "a", "BiNAnce"), + ("ads", "binAnce", "QC.Brokerage.Binance.BiNAnce"), + ("binAnce", "a", "QC.Brokerage.Binance.BiNAnce")]) +def test_finds_module_case_insensitive_name(id: str, display: str, search_name: str) -> None: + create_fake_lean_cli_directory() + + module = JsonModule({"id": id, "configurations": [], "display-id": display}, + MODULE_BROKERAGE, MODULE_CLI_PLATFORM) + result = find_module(search_name, [module], MagicMock()) + assert result == module + + +@pytest.mark.parametrize("searching,expected", [("ads", False), + ("BinanceFuturesBrokerage", True)]) +def test_is_value_in_config(searching: str, expected: bool) -> None: + module = JsonModule({"id": "asd", "configurations": [ + { + "id": "live-mode-brokerage", + "type": "info", + "value": "BinanceFuturesBrokerage" + } + ], "display-id": "OUS"}, MODULE_BROKERAGE, MODULE_CLI_PLATFORM) + + result = module.is_value_in_config(searching) + + assert expected == result diff --git a/tests/components/util/test_organization_manager.py b/tests/components/util/test_organization_manager.py index 0d6e9467..92c817a1 100644 --- a/tests/components/util/test_organization_manager.py +++ b/tests/components/util/test_organization_manager.py @@ -48,7 +48,6 @@ def test_try_get_id_aborts_if_organization_id_is_not_in_the_lean_config() -> Non def test_organization_manager_sets_working_organization_id_in_lean_config(): organization_id = "abc123" - lean_config_updates = {"organization-id": organization_id} lean_config_manager = mock.Mock() lean_config_manager.set_properties = mock.Mock() @@ -57,4 +56,5 @@ def test_organization_manager_sets_working_organization_id_in_lean_config(): organization_manager.configure_working_organization_id(organization_id) - lean_config_manager.set_properties.assert_called_once_with(lean_config_updates) + lean_config_manager.set_properties.assert_called_with({"job-organization-id": organization_id}) + lean_config_manager.set_properties.assert_called_with({"organization-id": organization_id}) From f2a17c0f6ef4c43fd9e8cbf5ac80d05642103dfd Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Fri, 1 Mar 2024 14:50:40 -0300 Subject: [PATCH 2/2] Minor unit test fix --- tests/components/util/test_organization_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/components/util/test_organization_manager.py b/tests/components/util/test_organization_manager.py index 92c817a1..c91bd2e2 100644 --- a/tests/components/util/test_organization_manager.py +++ b/tests/components/util/test_organization_manager.py @@ -56,5 +56,5 @@ def test_organization_manager_sets_working_organization_id_in_lean_config(): organization_manager.configure_working_organization_id(organization_id) - lean_config_manager.set_properties.assert_called_with({"job-organization-id": organization_id}) - lean_config_manager.set_properties.assert_called_with({"organization-id": organization_id}) + lean_config_manager.set_properties.assert_has_calls(calls=[mock.call({"job-organization-id": organization_id}), + mock.call({"organization-id": organization_id})])