From 757dbacb8550393178d5d01665adb1d542f8e7bd Mon Sep 17 00:00:00 2001 From: Ronit Jain Date: Tue, 23 Jan 2024 00:13:30 +0530 Subject: [PATCH] Fix Local Live Deploy Using Environment Option (#399) * update essential properties when loading from envrionment * review * handle null case * auto fill essential properties * add test --- lean/commands/live/deploy.py | 63 ++++++++++++++++++-- lean/components/util/json_modules_handler.py | 9 +++ lean/models/json_module.py | 4 +- tests/commands/test_live.py | 42 +++++++++++++ 4 files changed, 112 insertions(+), 6 deletions(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 4a76cca7..baf0d2da 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -21,14 +21,14 @@ from lean.models.errors import MoreInfoError from lean.models.lean_config_configurer import LeanConfigConfigurer from lean.models.logger import Option -from lean.models.configuration import InternalInputUserInput +from lean.models.configuration import ConfigurationsEnvConfiguration, InternalInputUserInput from lean.models.click_options import options_from_json, get_configs_for_options from lean.models.json_module import LiveInitialStateInput from lean.commands.live.live import live from lean.components.util.live_utils import get_last_portfolio_cash_holdings, configure_initial_cash_balance, configure_initial_holdings,\ _configure_initial_cash_interactively, _configure_initial_holdings_interactively from lean.models.data_providers import all_data_providers -from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module +from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module, update_essential_properties_available _environment_skeleton = { "live-mode": True, @@ -54,11 +54,20 @@ def _get_configurable_modules_from_environment(lean_config: Dict[str, Any], envi brokerage = environment["live-mode-brokerage"] data_queue_handlers = environment["data-queue-handler"] - [brokerage_configurer] = [local_brokerage for local_brokerage in all_local_brokerages if local_brokerage.get_live_name(environment_name) == brokerage] - data_feed_configurers = [local_data_feed for local_data_feed in all_local_data_feeds if local_data_feed.get_live_name(environment_name) in data_queue_handlers] + [brokerage_configurer] = [local_brokerage for local_brokerage in all_local_brokerages if _get_brokerage_base_name(local_brokerage.get_live_name(environment_name)) == _get_brokerage_base_name(brokerage)] + data_queue_handlers_base_names = [_get_brokerage_base_name(data_queue_handler) for data_queue_handler in data_queue_handlers] + data_feed_configurers = [local_data_feed for local_data_feed in all_local_data_feeds if _get_brokerage_base_name(local_data_feed.get_live_name(environment_name)) in data_queue_handlers_base_names] return brokerage_configurer, data_feed_configurers +def _get_brokerage_base_name(brokerage: str) -> str: + """Returns the base name of the brokerage. + + :param brokerage: the name of the brokerage + :return: the base name of the brokerage + """ + return brokerage.split('.')[-1] + def _install_modules(modules: List[LeanConfigConfigurer], user_kwargs: Dict[str, Any]) -> None: """Raises an error if any of the given modules are not installed. @@ -326,6 +335,52 @@ def deploy(project: Path, if environment is not None: environment_name = environment lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) + + lean_environment = lean_config["environments"][environment_name] + for key in ["live-mode-brokerage", "data-queue-handler"]: + if key not in lean_environment: + raise MoreInfoError(f"The '{environment_name}' environment does not specify a {key}", + "https://www.lean.io/docs/v2/lean-cli/live-trading/algorithm-control") + + brokerage = lean_environment["live-mode-brokerage"] + data_queue_handlers = lean_environment["data-queue-handler"] + data_queue_handlers_base_names = [_get_brokerage_base_name(data_queue_handler) for data_queue_handler in data_queue_handlers] + data_feed_configurers = [] + + for local_brokerage in all_local_brokerages: + configuration_environments: List[ConfigurationsEnvConfiguration] = [config for config in local_brokerage._lean_configs if config._is_type_configurations_env] + for configuration_environment in configuration_environments: + configuration_environment_values = list(configuration_environment._env_and_values.values())[0] + if any(True for x in configuration_environment_values if x["name"] == "live-mode-brokerage" and _get_brokerage_base_name(x["value"]) == _get_brokerage_base_name(brokerage)): + brokerage_configurer = local_brokerage + # fill essential properties + for condition in configuration_environment._filter._conditions: + if condition._type != "exact-match": + continue + property_name_to_fill = local_brokerage.convert_lean_key_to_variable(condition._dependent_config_id) + property_value_to_fill = condition._pattern + kwargs[property_name_to_fill] = property_value_to_fill + lean_config[condition._dependent_config_id] = property_value_to_fill + break + + for local_data_feed in all_local_data_feeds: + configuration_environments: List[ConfigurationsEnvConfiguration] = [config for config in local_data_feed._lean_configs if config._is_type_configurations_env] + for configuration_environment in configuration_environments: + configuration_environment_values = list(configuration_environment._env_and_values.values())[0] + if any(True for x in configuration_environment_values if x["name"] == "data-queue-handler" and _get_brokerage_base_name(x["value"]) in data_queue_handlers_base_names): + data_feed_configurers.append(local_data_feed) + # fill essential properties + for condition in configuration_environment._filter._conditions: + if condition._type != "exact-match": + continue + property_name_to_fill = local_data_feed.convert_lean_key_to_variable(condition._dependent_config_id) + property_value_to_fill = condition._pattern + kwargs[property_name_to_fill] = property_value_to_fill + lean_config[condition._dependent_config_id] = property_value_to_fill + + [update_essential_properties_available([brokerage_configurer], kwargs)] + [update_essential_properties_available(data_feed_configurers, kwargs)] + elif brokerage is not None or len(data_feed) > 0: ensure_options(["brokerage", "data_feed"]) diff --git a/lean/components/util/json_modules_handler.py b/lean/components/util/json_modules_handler.py index a8d53e06..85713a55 100644 --- a/lean/components/util/json_modules_handler.py +++ b/lean/components/util/json_modules_handler.py @@ -56,3 +56,12 @@ def get_and_build_module(target_module_name: str, module_list: List[JsonModule], target_module.update_configs(required_properties_value) logger.debug(f"json_module_handler.get_and_build_module(): non-interactive: required_properties_value with module {target_module_name}: {required_properties_value}") return target_module + + +def update_essential_properties_available(module_list: List[JsonModule], properties: Dict[str, Any]) -> JsonModule: + for target_module in module_list: + # update essential properties from brokerage to datafeed + # needs to be updated before fetching required properties + essential_properties = [target_module.convert_lean_key_to_variable(prop) for prop in target_module.get_essential_properties()] + essential_properties_value = {target_module.convert_variable_to_lean_key(prop) : properties[prop] for prop in essential_properties if properties[prop] is not None} + target_module.update_configs(essential_properties_value) diff --git a/lean/models/json_module.py b/lean/models/json_module.py index 06b37f12..c27ce36d 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -91,8 +91,8 @@ def get_configurations_env_values_from_name(self, target_env: str) -> List[Dict[ config._is_type_configurations_env and self.check_if_config_passes_filters( config) ] or [None] - if env_config is not None and target_env in env_config._env_and_values.keys(): - env_config_values = env_config._env_and_values[target_env] + if env_config is not None: + env_config_values = list(env_config._env_and_values.values())[0] return env_config_values def get_config_from_type(self, config_type: Configuration) -> str: diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index d8d0bc67..b8ec5fd5 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -65,6 +65,34 @@ def create_fake_environment(name: str, live_mode: bool) -> None: path.write_text(config, encoding="utf-8") +def create_fake_binance_environment(name: str, live_mode: bool) -> None: + path = Path.cwd() / "lean.json" + config = path.read_text(encoding="utf-8") + config = config.replace("{", f""" +{{ + "binance-use-testnet": "live", + "binance-exchange-name": "binance", + "binance-api-secret": "abc", + "binance-api-key": "abc", + "organization-id": "abc", + + "environments": {{ + "{name}": {{ + "live-mode": {str(live_mode).lower()}, + + "live-mode-brokerage": "QuantConnect.BinanceBrokerage.BinanceCoinFuturesBrokerage", + "data-queue-handler": [ "QuantConnect.BinanceBrokerage.BinanceCoinFuturesBrokerage" ], + "setup-handler": "QuantConnect.Lean.Engine.Setup.BrokerageSetupHandler", + "result-handler": "QuantConnect.Lean.Engine.Results.LiveTradingResultHandler", + "data-feed-handler": "QuantConnect.Lean.Engine.DataFeeds.LiveTradingDataFeed", + "real-time-handler": "QuantConnect.Lean.Engine.RealTime.LiveTradingRealTimeHandler", + "transaction-handler": "QuantConnect.Lean.Engine.TransactionHandlers.BrokerageTransactionHandler", + "history-provider": [ "BrokerageHistoryProvider", "SubscriptionDataReaderHistoryProvider" ] + }} + }}, + """) + + path.write_text(config, encoding="utf-8") def test_live_calls_lean_runner_with_correct_algorithm_file() -> None: # TODO: currently it is not using the live-paper envrionment @@ -258,6 +286,20 @@ def test_live_aborts_when_lean_config_is_missing_properties(target: str, replace lean_runner.run_lean.assert_not_called() +def test_live_sets_dependent_configurations_from_modules_json_based_on_environment() -> None: + create_fake_lean_cli_directory() + create_fake_binance_environment("live-binance", True) + lean_runner = container.lean_runner + + config_path = Path.cwd() / "lean.json" + config = config_path.read_text(encoding="utf-8") + config_path.write_text(config.replace("binance-exchange-name", "different-config"), encoding="utf-8") + + result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-binance"]) + + assert result.exit_code == 0 + + lean_runner.run_lean.assert_called() terminal_link_required_options = { "terminal-link-connection-type": "SAPI",