From 74de12761abf35609214646b1b0365320140e860 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 24 Jan 2024 09:28:05 -0400 Subject: [PATCH 1/2] Fix backtest data provider configuration. An exception was being raised when configuring the data provider from the json modules --- lean/commands/live/deploy.py | 12 ++++++++---- lean/models/brokerages/local/data_feed.py | 8 +++----- lean/models/brokerages/local/local_brokerage.py | 8 +++----- lean/models/json_module.py | 3 ++- lean/models/lean_config_configurer.py | 4 +--- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index baf0d2da..a8f35864 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -54,9 +54,13 @@ def _get_configurable_modules_from_environment(lean_config: Dict[str, Any], envi brokerage = environment["live-mode-brokerage"] data_queue_handlers = environment["data-queue-handler"] - [brokerage_configurer] = [local_brokerage for local_brokerage in all_local_brokerages if _get_brokerage_base_name(local_brokerage.get_live_name(environment_name)) == _get_brokerage_base_name(brokerage)] + [brokerage_configurer] = [local_brokerage + for local_brokerage in all_local_brokerages + if _get_brokerage_base_name(local_brokerage.get_live_name()) == _get_brokerage_base_name(brokerage)] data_queue_handlers_base_names = [_get_brokerage_base_name(data_queue_handler) for data_queue_handler in data_queue_handlers] - data_feed_configurers = [local_data_feed for local_data_feed in all_local_data_feeds if _get_brokerage_base_name(local_data_feed.get_live_name(environment_name)) in data_queue_handlers_base_names] + data_feed_configurers = [local_data_feed + for local_data_feed in all_local_data_feeds + if _get_brokerage_base_name(local_data_feed.get_live_name()) in data_queue_handlers_base_names] return brokerage_configurer, data_feed_configurers @@ -335,7 +339,7 @@ def deploy(project: Path, if environment is not None: environment_name = environment lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) - + lean_environment = lean_config["environments"][environment_name] for key in ["live-mode-brokerage", "data-queue-handler"]: if key not in lean_environment: @@ -362,7 +366,7 @@ def deploy(project: Path, kwargs[property_name_to_fill] = property_value_to_fill lean_config[condition._dependent_config_id] = property_value_to_fill break - + for local_data_feed in all_local_data_feeds: configuration_environments: List[ConfigurationsEnvConfiguration] = [config for config in local_data_feed._lean_configs if config._is_type_configurations_env] for configuration_environment in configuration_environments: diff --git a/lean/models/brokerages/local/data_feed.py b/lean/models/brokerages/local/data_feed.py index ec5feac7..34be975e 100644 --- a/lean/models/brokerages/local/data_feed.py +++ b/lean/models/brokerages/local/data_feed.py @@ -21,11 +21,9 @@ class DataFeed(LeanConfigConfigurer): def __init__(self, json_datafeed_data: Dict[str, Any]) -> None: super().__init__(json_datafeed_data) - def get_live_name(self, environment_name: str) -> str: + def get_live_name(self) -> str: live_name = self._id - environment_obj = self.get_configurations_env_values_from_name( - environment_name) + environment_obj = self.get_configurations_env_values() if environment_obj: - [live_name] = [x["value"] - for x in environment_obj if x["name"] == "data-queue-handler"] + [live_name] = [x["value"] for x in environment_obj if x["name"] == "data-queue-handler"] return live_name diff --git a/lean/models/brokerages/local/local_brokerage.py b/lean/models/brokerages/local/local_brokerage.py index d1743ba0..c22f7fe7 100644 --- a/lean/models/brokerages/local/local_brokerage.py +++ b/lean/models/brokerages/local/local_brokerage.py @@ -21,11 +21,9 @@ class LocalBrokerage(LeanConfigConfigurer): def __init__(self, json_brokerage_data: Dict[str, Any]) -> None: super().__init__(json_brokerage_data) - def get_live_name(self, environment_name: str) -> str: + def get_live_name(self) -> str: live_name = self._id - environment_obj = self.get_configurations_env_values_from_name( - environment_name) + environment_obj = self.get_configurations_env_values() if environment_obj: - [live_name] = [x["value"] - for x in environment_obj if x["name"] == "live-mode-brokerage"] + [live_name] = [x["value"] for x in environment_obj if x["name"] == "live-mode-brokerage"] return live_name diff --git a/lean/models/json_module.py b/lean/models/json_module.py index c27ce36d..41ee9967 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -85,13 +85,14 @@ def update_configs(self, key_and_values: Dict[str, str]): for key, value in key_and_values.items(): self.update_value_for_given_config(key, value) - def get_configurations_env_values_from_name(self, target_env: str) -> List[Dict[str, str]]: + def get_configurations_env_values(self) -> List[Dict[str, str]]: env_config_values = [] [env_config] = [config for config in self._lean_configs if config._is_type_configurations_env and self.check_if_config_passes_filters( config) ] or [None] if env_config is not None: + # Always get the first one, since we only expect one env config in the json modules file env_config_values = list(env_config._env_and_values.values())[0] return env_config_values diff --git a/lean/models/lean_config_configurer.py b/lean/models/lean_config_configurer.py index c8a560e7..7ff8f8f6 100644 --- a/lean/models/lean_config_configurer.py +++ b/lean/models/lean_config_configurer.py @@ -39,7 +39,7 @@ def _configure_environment(self, lean_config: Dict[str, Any], environment_name: :param lean_config: the Lean configuration dict to write to :param environment_name: the name of the environment to update """ - for environment_config in self.get_configurations_env_values_from_name(environment_name): + for environment_config in self.get_configurations_env_values(): environment_config_name = environment_config["name"] if self.__class__.__name__ == 'DataFeed': if environment_config_name == "data-queue-handler": @@ -51,8 +51,6 @@ def _configure_environment(self, lean_config: Dict[str, Any], environment_name: elif self.__class__.__name__ == 'LocalBrokerage': if environment_config_name != "data-queue-handler": lean_config["environments"][environment_name][environment_config_name] = environment_config["value"] - else: - raise ValueError(f'{self.__class__.__name__} not valid for _configure_environment()') def configure_credentials(self, lean_config: Dict[str, Any]) -> None: """Configures the credentials in the Lean config for this brokerage and saves them persistently to disk. From 1559a20470b9e97b28eeac8ddf8a6c9d6567159c Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 24 Jan 2024 11:18:26 -0400 Subject: [PATCH 2/2] Add unit tests --- tests/models/test_lean_config_configurer.py | 123 ++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 tests/models/test_lean_config_configurer.py diff --git a/tests/models/test_lean_config_configurer.py b/tests/models/test_lean_config_configurer.py new file mode 100644 index 00000000..dccfd93d --- /dev/null +++ b/tests/models/test_lean_config_configurer.py @@ -0,0 +1,123 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from typing import Dict, Any +from unittest import mock + +from lean.models.brokerages.local import DataFeed +from lean.models.data_providers import DataProvider +from lean.models.lean_config_configurer import LeanConfigConfigurer + +JSON_MODULE = json.loads(""" +{ + "type": [ + "data-queue-handler", + "data-provider" + ], + "product-id": "305", + "id": "PolygonDataFeed", + "display-id": "Polygon", + "installs": true, + "configurations": [ + { + "id": "polygon-api-key", + "cloud-id": "apiKey", + "type": "input", + "value": "", + "input-method": "prompt", + "prompt-info": "Your Polygon.io API Key", + "help": "Your Polygon.io API Key" + }, + { + "id": "environments", + "type": "configurations-env", + "value": [ + { + "name": "lean-cli", + "value": [ + { + "name": "data-queue-handler", + "value": "QuantConnect.Polygon.PolygonDataQueueHandler" + }, + { + "name": "history-provider", + "value": [ + "QuantConnect.Polygon.PolygonDataQueueHandler", + "SubscriptionDataReaderHistoryProvider" + ] + } + ] + } + ] + }, + { + "id": "data-provider", + "type": "info", + "value": "QuantConnect.Lean.Engine.DataFeeds.DownloaderDataProvider" + }, + { + "id": "data-downloader", + "type": "info", + "value": "QuantConnect.Polygon.PolygonDataDownloader" + } + ] +} +""") + + +def test_gets_environment_from_configuration() -> None: + module = LeanConfigConfigurer(JSON_MODULE) + environment_values = module.get_configurations_env_values() + + assert environment_values == JSON_MODULE["configurations"][1]["value"][0]["value"] + + +def get_lean_config() -> Dict[str, Any]: + return { + "environments": { + "live-ib-polygon": { + "live-mode": True, + "live-mode-brokerage": "InteractiveBrokersBrokerage", + "setup-handler": "QuantConnect.Lean.Engine.Setup.BrokerageSetupHandler", + "result-handler": "QuantConnect.Lean.Engine.Results.LiveTradingResultHandler", + "data-feed-handler": "QuantConnect.Lean.Engine.DataFeeds.LiveTradingDataFeed", + "data-queue-handler": ["QuantConnect.Brokerages.InteractiveBrokers.InteractiveBrokersBrokerage"], + "real-time-handler": "QuantConnect.Lean.Engine.RealTime.LiveTradingRealTimeHandler", + "transaction-handler": "QuantConnect.Lean.Engine.TransactionHandlers.BrokerageTransactionHandler", + "history-provider": [ + "BrokerageHistoryProvider", + "SubscriptionDataReaderHistoryProvider" + ] + } + } + } + + +def test_configures_environment_with_module() -> None: + with mock.patch.object(DataFeed, "configure_credentials"): + lean_config = get_lean_config() + module = DataFeed(JSON_MODULE) + module.configure(lean_config, "live-ib-polygon") + + assert lean_config != get_lean_config() + assert "QuantConnect.Polygon.PolygonDataQueueHandler" in lean_config["environments"]["live-ib-polygon"]["data-queue-handler"] + + +def test_invalid_environment_configuration_is_ignored() -> None: + with mock.patch.object(DataProvider, "configure_credentials"): + lean_config = get_lean_config() + module = DataProvider(JSON_MODULE) + module.configure(lean_config, "live-ib-polygon") + + assert lean_config == get_lean_config()