From a031b02c2b03325cd2f53eb500a5f0a283646529 Mon Sep 17 00:00:00 2001 From: Roman Yavnikov <45608740+Romazes@users.noreply.github.com> Date: Tue, 20 Feb 2024 19:01:24 +0200 Subject: [PATCH] Optional Downloader Data Provider (#424) * feat: optional download data provider * feat: test to download data provider with different params * feat: installed data provider list * feat: general tests * refactor: separate data-provider from data-feed * fix: validation of amount of request on api * feat: remove default value for data-provider-historical * feat: test case with local historical provider * remove: repeat test case refactor: assertion in test with different version of py * fix: missed rename data_feed to data_provider * feat: additional test case * revert: data_provider -> data_feed commit: e52b3a4 * revert: repeat code of finding data-provider * rename: helper method more fit name * rename: missed PR: #416 --- lean/commands/live/deploy.py | 25 ++- lean/models/data_providers/__init__.py | 2 +- tests/commands/test_live.py | 209 ++++++++++++++++++++++++- tests/test_helpers.py | 10 ++ 4 files changed, 234 insertions(+), 12 deletions(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 0e7d61c0..4f406dba 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -79,7 +79,7 @@ def _install_modules(modules: List[LeanConfigConfigurer], user_kwargs: Dict[str, :param modules: the modules to check """ for module in modules: - if not module._installs: + if module is None or not module._installs: continue organization_id = container.organization_manager.try_get_working_organization_id() module.ensure_module_installed(organization_id) @@ -192,6 +192,15 @@ def _configure_lean_config_interactively(lean_config: Dict[str, Any], _cached_lean_config = None +def _try_get_data_historical_name(data_provider_historical_name: str, data_provider_live_name: str) -> str: + """ Get name for historical data provider based on data provider live (if exist) + + :param data_provider_historical_name: the current (default) data provider historical + :param data_provider_live_name: the current data provider live name + """ + return next((live_data_historical.get_name() for live_data_historical in all_data_providers + if live_data_historical.get_name() in data_provider_live_name), data_provider_historical_name) + # being used by lean.models.click_options.get_the_correct_type_default_value() def _get_default_value(key: str) -> Optional[Any]: @@ -235,7 +244,6 @@ def _get_default_value(key: str) -> Optional[Any]: help="The live data provider to use") @option("--data-provider-historical", type=Choice([dp.get_name() for dp in all_data_providers if dp._id != "TerminalLinkBrokerage"], case_sensitive=False), - default="Local", help="Update the Lean configuration file to retrieve data from the given historical provider") @options_from_json(get_configs_for_options("live-local")) @option("--release", @@ -387,7 +395,7 @@ def deploy(project: Path, [update_essential_properties_available(data_feed_configurers, kwargs)] elif brokerage is not None or len(data_provider_live) > 0: - ensure_options(["brokerage", "data_feed"]) + ensure_options(["brokerage", "data_provider_live"]) environment_name = "lean-cli" lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) @@ -408,9 +416,10 @@ def deploy(project: Path, lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) _configure_lean_config_interactively(lean_config, environment_name, kwargs, show_secrets=show_secrets) - if data_provider_historical is not None: - [data_provider_configurer] = [get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger)] - data_provider_configurer.configure(lean_config, environment_name) + if data_provider_historical is None: + data_provider_historical = _try_get_data_historical_name("Local", data_provider_live) + [data_provider_configurer] = [get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger)] + data_provider_configurer.configure(lean_config, environment_name) if "environments" not in lean_config or environment_name not in lean_config["environments"]: lean_config_path = lean_config_manager.get_lean_config_path() @@ -422,7 +431,7 @@ def deploy(project: Path, "https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading") env_brokerage, env_data_queue_handlers = _get_configurable_modules_from_environment(lean_config, environment_name) - _install_modules([env_brokerage] + env_data_queue_handlers, kwargs) + _install_modules([env_brokerage] + env_data_queue_handlers + [data_provider_configurer], kwargs) _raise_for_missing_properties(lean_config, environment_name, lean_config_manager.get_lean_config_path()) @@ -497,4 +506,4 @@ def deploy(project: Path, raise RuntimeError(f"InteractiveBrokers is currently not supported for ARM hosts") lean_runner = container.lean_runner - lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config)) + lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config)) \ No newline at end of file diff --git a/lean/models/data_providers/__init__.py b/lean/models/data_providers/__init__.py index 268e9e34..fb0c47e2 100644 --- a/lean/models/data_providers/__init__.py +++ b/lean/models/data_providers/__init__.py @@ -23,4 +23,4 @@ # QuantConnect DataProvider [QuantConnectDataProvider] = [ - data_provider for data_provider in all_data_providers if data_provider._id == "QuantConnect"] + data_provider for data_provider in all_data_providers if data_provider._id == "QuantConnect"] \ No newline at end of file diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index c914c0d3..915bdf80 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -25,8 +25,9 @@ from lean.constants import DEFAULT_ENGINE_IMAGE from lean.container import container from lean.models.docker import DockerImage -from tests.test_helpers import create_fake_lean_cli_directory +from tests.test_helpers import create_fake_lean_cli_directory, reset_state_installed_modules from tests.conftest import initialize_container +from click.testing import Result ENGINE_IMAGE = DockerImage.parse(DEFAULT_ENGINE_IMAGE) @@ -413,7 +414,21 @@ def test_live_sets_dependent_configurations_from_modules_json_based_on_environme "Terminal Link": terminal_link_required_options, "Kraken": brokerage_required_options["Kraken"], "TDAmeritrade": brokerage_required_options["TDAmeritrade"], - "Bybit": brokerage_required_options["Bybit"] + "Bybit": brokerage_required_options["Bybit"], +} + +data_provider_required_options = { + "IEX": { + "iex-cloud-api-key": "123", + "iex-price-plan": "Launch", + }, + "Polygon": { + "polygon-api-key": "123", + }, + "AlphaVantage": { + "alpha-vantage-api-key": "111", + "alpha-vantage-price-plan": "Free" + } } @@ -530,7 +545,7 @@ def test_live_non_interactive_raise_error_when_missing_data_provider_live_option error_msg = str(result.exc_info[1]).split() - assert "data-provider-live" in error_msg + assert "--data-provider-live" in error_msg assert "data-queue-handler" not in error_msg assert result.exit_code != 0 @@ -1026,3 +1041,191 @@ def test_live_passes_live_holdings_to_lean_runner_when_given_as_option(brokerage return assert args[0]["live-holdings"] == holding_list + +def test_live_non_interactive_deploy_with_live_and_historical_provider_missed_historical_not_optional_config() -> None: + create_fake_lean_cli_directory() + create_fake_environment("live-paper", True) + + container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client = mock.MagicMock()) + + provider_live_option = ["--data-provider-live", "IEX", + "--iex-cloud-api-key", "123", + "--iex-price-plan", "Launch"] + + provider_history_option = ["--data-provider-historical", "Polygon"] + # "--polygon-api-key", "123"] + + result = CliRunner().invoke(lean, ["live", "deploy" , "--brokerage", "Paper Trading", + *provider_live_option, + *provider_history_option, + "Python Project", + ]) + error_msg = str(result.exc_info[1]).split() + + assert "--polygon-api-key" in error_msg + assert "--iex-cloud-api-key" not in error_msg + + assert result.exit_code == 1 + +def test_live_non_interactive_deploy_with_live_and_historical_provider_missed_live_not_optional_config() -> None: + create_fake_lean_cli_directory() + create_fake_environment("live-paper", True) + + container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client = mock.MagicMock()) + + provider_live_option = ["--data-provider-live", "IEX", + "--iex-cloud-api-key", "123"] + #"--iex-price-plan", "Launch"] + + provider_history_option = ["--data-provider-historical", "Polygon", "--polygon-api-key", "123"] + + result = CliRunner().invoke(lean, ["live", "deploy" , "--brokerage", "Paper Trading", + *provider_live_option, + *provider_history_option, + "Python Project", + ]) + + error_msg = str(result.exc_info[1]).split() + + assert "--iex-price-plan" in error_msg + assert "--polygon-api-key" not in error_msg + + assert result.exit_code == 1 + +def test_live_non_interactive_deploy_with_real_brokerage_without_credentials() -> None: + create_fake_lean_cli_directory() + create_fake_environment("live-paper", True) + + container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client = mock.MagicMock()) + + # create fake environment has IB configs already + brokerage = ["--brokerage", "OANDA"] + + provider_live_option = ["--data-provider-live", "IEX", + "--iex-cloud-api-key", "123", + "--iex-price-plan", "Launch"] + + result = CliRunner().invoke(lean, ["live", "deploy" , + *brokerage, + *provider_live_option, + "Python Project", + ]) + assert result.exit_code == 1 + + error_msg = str(result.exc_info[1]).split() + + assert "--oanda-account-id" in error_msg + assert "--oanda-access-token" in error_msg + assert "--oanda-environment" in error_msg + assert "--iex-price-plan" not in error_msg + +def create_lean_option(brokerage_name: str, data_provider_live_name: str, data_provider_historical_name: str, api_client: any) -> Result: + reset_state_installed_modules() + create_fake_lean_cli_directory() + create_fake_environment("live-paper", True) + + initialize_container(api_client_to_use=api_client) + + option = ["--brokerage", brokerage_name] + for key, value in brokerage_required_options[brokerage_name].items(): + option.extend([f"--{key}", value]) + + data_feed_required_options.update(data_provider_required_options) + + option.extend(["--data-provider-live", data_provider_live_name]) + for key, value in data_feed_required_options[data_provider_live_name].items(): + if f"--{key}" not in option: + option.extend([f"--{key}", value]) + + if data_provider_historical_name is not None: + option.extend(["--data-provider-historical", data_provider_historical_name]) + if data_provider_historical_name is not "Local": + for key, value in data_feed_required_options[data_provider_historical_name].items(): + if f"--{key}" not in option: + option.extend([f"--{key}", value]) + + result = CliRunner().invoke(lean, ["live", "deploy", + *option, + "Python Project", + ]) + assert result.exit_code == 0 + return result + +@pytest.mark.parametrize("brokerage_name,data_provider_live_name,data_provider_historical_name,brokerage_product_id,data_provider_live_product_id,data_provider_historical_id", + [("Interactive Brokers", "IEX", "Polygon", "181", "333", "306"), + ("Paper Trading", "IEX", "Polygon", None, "333", "306"), + ("Tradier", "IEX", "AlphaVantage", "185", "333", "334"), + ("Paper Trading", "IEX", "Local", None, "333", "222")]) +def test_live_deploy_with_different_brokerage_and_different_live_data_provider_and_historical_data_provider(brokerage_name: str, data_provider_live_name: str, data_provider_historical_name: str, brokerage_product_id: str, data_provider_live_product_id: str, data_provider_historical_id: str) -> None: + api_client = mock.MagicMock() + create_lean_option(brokerage_name, data_provider_live_name, data_provider_historical_name, api_client) + + is_exists = [] + if brokerage_product_id is None and data_provider_historical_name != "Local": + assert len(api_client.method_calls) == 3 + for m_c, id in zip(api_client.method_calls, [data_provider_live_product_id, data_provider_historical_id]): + if id in m_c[1]: + is_exists.append(True) + assert is_exists + assert len(is_exists) == 2 + elif brokerage_product_id is None and data_provider_historical_name == "Local": + assert len(api_client.method_calls) == 2 + if data_provider_live_product_id in api_client.method_calls[0][1]: + is_exists.append(True) + assert is_exists + assert len(is_exists) == 1 + else: + assert len(api_client.method_calls) == 3 + for m_c, id in zip(api_client.method_calls, [brokerage_product_id, data_provider_live_product_id, data_provider_historical_id]): + if id in f"{m_c[1]}": + is_exists.append(True) + assert is_exists + assert len(is_exists) == 3 + +@pytest.mark.parametrize("brokerage_name,data_provider_live_name,brokerage_product_id,data_provider_live_product_id", + [("Interactive Brokers", "IEX", "181", "333"), + ("Tradier", "IEX", "185", "333")]) +def test_live_non_interactive_deploy_with_different_brokerage_and_different_live_data_provider(brokerage_name: str, data_provider_live_name: str, brokerage_product_id: str, data_provider_live_product_id: str) -> None: + api_client = mock.MagicMock() + create_lean_option(brokerage_name, data_provider_live_name, None, api_client) + + assert len(api_client.method_calls) == 2 + is_exists = [] + for m_c, id in zip(api_client.method_calls, [brokerage_product_id, data_provider_live_product_id]): + if id in m_c[1]: + is_exists.append(True) + + assert is_exists + assert len(is_exists) == 2 + +@pytest.mark.parametrize("brokerage_name,data_provider_live_name,brokerage_product_id", + [("Bybit", "Bybit", "305"), + ("Coinbase Advanced Trade", "Coinbase Advanced Trade", "183"), + ("Interactive Brokers", "Interactive Brokers", "181"), + ("Tradier", "Tradier", "185")]) +def test_live_non_interactive_deploy_with_different_brokerage_with_the_same_live_data_provider(brokerage_name: str, data_provider_live_name: str, brokerage_product_id: str) -> None: + api_client = mock.MagicMock() + create_lean_option(brokerage_name, data_provider_live_name, None, api_client) + + print(api_client.call_args_list) + print(api_client.call_args) + + for m_c in api_client.method_calls: + if brokerage_product_id in m_c[1]: + is_exist = True + + assert is_exist + +@pytest.mark.parametrize("brokerage_name,data_provider_live_name,data_provider_live_product_id", + [("Paper Trading", "IEX", "333"), + ("Paper Trading", "Polygon", "306")]) +def test_live_non_interactive_deploy_paper_brokerage_different_live_data_provider(brokerage_name: str, data_provider_live_name: str, data_provider_live_product_id: str) -> None: + api_client = mock.MagicMock() + create_lean_option(brokerage_name, data_provider_live_name, None, api_client) + + assert len(api_client.method_calls) == 2 + for m_c in api_client.method_calls: + if data_provider_live_product_id in m_c[1]: + is_exist = True + + assert is_exist \ No newline at end of file diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 7c979d98..746f1229 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -15,6 +15,8 @@ from datetime import datetime from pathlib import Path from typing import List +from lean.models.data_providers import all_data_providers +from lean.models.brokerages.local import all_local_brokerages, all_local_data_feeds from lean.commands.create_project import (DEFAULT_CSHARP_MAIN, DEFAULT_CSHARP_NOTEBOOK, DEFAULT_PYTHON_MAIN, DEFAULT_PYTHON_NOTEBOOK, LIBRARY_PYTHON_MAIN, LIBRARY_CSHARP_MAIN) @@ -224,3 +226,11 @@ def create_lean_environments() -> List[QCLeanEnvironment]: description="", public=True) ] + +def reset_state_installed_modules() -> None: + for data_provider in all_data_providers: + data_provider.__setattr__("_is_module_installed", False) + for local_brokerage in all_local_brokerages: + local_brokerage.__setattr__("_is_module_installed", False) + for local_data_feed in all_local_data_feeds: + local_data_feed.__setattr__("_is_module_installed", False)