Skip to content

Commit

Permalink
Optional Downloader Data Provider (#424)
Browse files Browse the repository at this point in the history
* feat: optional download data provider

* feat: test to download data provider with different params

* feat: installed data provider list

* feat: general tests

* refactor: separate data-provider from data-feed

* fix: validation of amount of request on api

* feat: remove default value for data-provider-historical

* feat: test case with local historical provider

* remove: repeat test case
refactor: assertion in test with different version of py

* fix: missed rename data_feed to data_provider

* feat: additional test case

* revert: data_provider -> data_feed
commit: e52b3a4

* revert: repeat code of finding data-provider

* rename: helper method more fit name

* rename: missed
PR: #416
  • Loading branch information
Romazes authored Feb 20, 2024
1 parent fe5c546 commit a031b02
Show file tree
Hide file tree
Showing 4 changed files with 234 additions and 12 deletions.
25 changes: 17 additions & 8 deletions lean/commands/live/deploy.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ def _install_modules(modules: List[LeanConfigConfigurer], user_kwargs: Dict[str,
:param modules: the modules to check
"""
for module in modules:
if not module._installs:
if module is None or not module._installs:
continue
organization_id = container.organization_manager.try_get_working_organization_id()
module.ensure_module_installed(organization_id)
Expand Down Expand Up @@ -192,6 +192,15 @@ def _configure_lean_config_interactively(lean_config: Dict[str, Any],

_cached_lean_config = None

def _try_get_data_historical_name(data_provider_historical_name: str, data_provider_live_name: str) -> str:
""" Get name for historical data provider based on data provider live (if exist)
:param data_provider_historical_name: the current (default) data provider historical
:param data_provider_live_name: the current data provider live name
"""
return next((live_data_historical.get_name() for live_data_historical in all_data_providers
if live_data_historical.get_name() in data_provider_live_name), data_provider_historical_name)


# being used by lean.models.click_options.get_the_correct_type_default_value()
def _get_default_value(key: str) -> Optional[Any]:
Expand Down Expand Up @@ -235,7 +244,6 @@ def _get_default_value(key: str) -> Optional[Any]:
help="The live data provider to use")
@option("--data-provider-historical",
type=Choice([dp.get_name() for dp in all_data_providers if dp._id != "TerminalLinkBrokerage"], case_sensitive=False),
default="Local",
help="Update the Lean configuration file to retrieve data from the given historical provider")
@options_from_json(get_configs_for_options("live-local"))
@option("--release",
Expand Down Expand Up @@ -387,7 +395,7 @@ def deploy(project: Path,
[update_essential_properties_available(data_feed_configurers, kwargs)]

elif brokerage is not None or len(data_provider_live) > 0:
ensure_options(["brokerage", "data_feed"])
ensure_options(["brokerage", "data_provider_live"])

environment_name = "lean-cli"
lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None)
Expand All @@ -408,9 +416,10 @@ def deploy(project: Path,
lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None)
_configure_lean_config_interactively(lean_config, environment_name, kwargs, show_secrets=show_secrets)

if data_provider_historical is not None:
[data_provider_configurer] = [get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger)]
data_provider_configurer.configure(lean_config, environment_name)
if data_provider_historical is None:
data_provider_historical = _try_get_data_historical_name("Local", data_provider_live)
[data_provider_configurer] = [get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger)]
data_provider_configurer.configure(lean_config, environment_name)

if "environments" not in lean_config or environment_name not in lean_config["environments"]:
lean_config_path = lean_config_manager.get_lean_config_path()
Expand All @@ -422,7 +431,7 @@ def deploy(project: Path,
"https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading")

env_brokerage, env_data_queue_handlers = _get_configurable_modules_from_environment(lean_config, environment_name)
_install_modules([env_brokerage] + env_data_queue_handlers, kwargs)
_install_modules([env_brokerage] + env_data_queue_handlers + [data_provider_configurer], kwargs)

_raise_for_missing_properties(lean_config, environment_name, lean_config_manager.get_lean_config_path())

Expand Down Expand Up @@ -497,4 +506,4 @@ def deploy(project: Path,
raise RuntimeError(f"InteractiveBrokers is currently not supported for ARM hosts")

lean_runner = container.lean_runner
lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config))
lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config))
2 changes: 1 addition & 1 deletion lean/models/data_providers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,4 @@

# QuantConnect DataProvider
[QuantConnectDataProvider] = [
data_provider for data_provider in all_data_providers if data_provider._id == "QuantConnect"]
data_provider for data_provider in all_data_providers if data_provider._id == "QuantConnect"]
209 changes: 206 additions & 3 deletions tests/commands/test_live.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@
from lean.constants import DEFAULT_ENGINE_IMAGE
from lean.container import container
from lean.models.docker import DockerImage
from tests.test_helpers import create_fake_lean_cli_directory
from tests.test_helpers import create_fake_lean_cli_directory, reset_state_installed_modules
from tests.conftest import initialize_container
from click.testing import Result

ENGINE_IMAGE = DockerImage.parse(DEFAULT_ENGINE_IMAGE)

Expand Down Expand Up @@ -413,7 +414,21 @@ def test_live_sets_dependent_configurations_from_modules_json_based_on_environme
"Terminal Link": terminal_link_required_options,
"Kraken": brokerage_required_options["Kraken"],
"TDAmeritrade": brokerage_required_options["TDAmeritrade"],
"Bybit": brokerage_required_options["Bybit"]
"Bybit": brokerage_required_options["Bybit"],
}

data_provider_required_options = {
"IEX": {
"iex-cloud-api-key": "123",
"iex-price-plan": "Launch",
},
"Polygon": {
"polygon-api-key": "123",
},
"AlphaVantage": {
"alpha-vantage-api-key": "111",
"alpha-vantage-price-plan": "Free"
}
}


Expand Down Expand Up @@ -530,7 +545,7 @@ def test_live_non_interactive_raise_error_when_missing_data_provider_live_option

error_msg = str(result.exc_info[1]).split()

assert "data-provider-live" in error_msg
assert "--data-provider-live" in error_msg
assert "data-queue-handler" not in error_msg

assert result.exit_code != 0
Expand Down Expand Up @@ -1026,3 +1041,191 @@ def test_live_passes_live_holdings_to_lean_runner_when_given_as_option(brokerage
return

assert args[0]["live-holdings"] == holding_list

def test_live_non_interactive_deploy_with_live_and_historical_provider_missed_historical_not_optional_config() -> None:
create_fake_lean_cli_directory()
create_fake_environment("live-paper", True)

container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client = mock.MagicMock())

provider_live_option = ["--data-provider-live", "IEX",
"--iex-cloud-api-key", "123",
"--iex-price-plan", "Launch"]

provider_history_option = ["--data-provider-historical", "Polygon"]
# "--polygon-api-key", "123"]

result = CliRunner().invoke(lean, ["live", "deploy" , "--brokerage", "Paper Trading",
*provider_live_option,
*provider_history_option,
"Python Project",
])
error_msg = str(result.exc_info[1]).split()

assert "--polygon-api-key" in error_msg
assert "--iex-cloud-api-key" not in error_msg

assert result.exit_code == 1

def test_live_non_interactive_deploy_with_live_and_historical_provider_missed_live_not_optional_config() -> None:
create_fake_lean_cli_directory()
create_fake_environment("live-paper", True)

container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client = mock.MagicMock())

provider_live_option = ["--data-provider-live", "IEX",
"--iex-cloud-api-key", "123"]
#"--iex-price-plan", "Launch"]

provider_history_option = ["--data-provider-historical", "Polygon", "--polygon-api-key", "123"]

result = CliRunner().invoke(lean, ["live", "deploy" , "--brokerage", "Paper Trading",
*provider_live_option,
*provider_history_option,
"Python Project",
])

error_msg = str(result.exc_info[1]).split()

assert "--iex-price-plan" in error_msg
assert "--polygon-api-key" not in error_msg

assert result.exit_code == 1

def test_live_non_interactive_deploy_with_real_brokerage_without_credentials() -> None:
create_fake_lean_cli_directory()
create_fake_environment("live-paper", True)

container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client = mock.MagicMock())

# create fake environment has IB configs already
brokerage = ["--brokerage", "OANDA"]

provider_live_option = ["--data-provider-live", "IEX",
"--iex-cloud-api-key", "123",
"--iex-price-plan", "Launch"]

result = CliRunner().invoke(lean, ["live", "deploy" ,
*brokerage,
*provider_live_option,
"Python Project",
])
assert result.exit_code == 1

error_msg = str(result.exc_info[1]).split()

assert "--oanda-account-id" in error_msg
assert "--oanda-access-token" in error_msg
assert "--oanda-environment" in error_msg
assert "--iex-price-plan" not in error_msg

def create_lean_option(brokerage_name: str, data_provider_live_name: str, data_provider_historical_name: str, api_client: any) -> Result:
reset_state_installed_modules()
create_fake_lean_cli_directory()
create_fake_environment("live-paper", True)

initialize_container(api_client_to_use=api_client)

option = ["--brokerage", brokerage_name]
for key, value in brokerage_required_options[brokerage_name].items():
option.extend([f"--{key}", value])

data_feed_required_options.update(data_provider_required_options)

option.extend(["--data-provider-live", data_provider_live_name])
for key, value in data_feed_required_options[data_provider_live_name].items():
if f"--{key}" not in option:
option.extend([f"--{key}", value])

if data_provider_historical_name is not None:
option.extend(["--data-provider-historical", data_provider_historical_name])
if data_provider_historical_name is not "Local":
for key, value in data_feed_required_options[data_provider_historical_name].items():
if f"--{key}" not in option:
option.extend([f"--{key}", value])

result = CliRunner().invoke(lean, ["live", "deploy",
*option,
"Python Project",
])
assert result.exit_code == 0
return result

@pytest.mark.parametrize("brokerage_name,data_provider_live_name,data_provider_historical_name,brokerage_product_id,data_provider_live_product_id,data_provider_historical_id",
[("Interactive Brokers", "IEX", "Polygon", "181", "333", "306"),
("Paper Trading", "IEX", "Polygon", None, "333", "306"),
("Tradier", "IEX", "AlphaVantage", "185", "333", "334"),
("Paper Trading", "IEX", "Local", None, "333", "222")])
def test_live_deploy_with_different_brokerage_and_different_live_data_provider_and_historical_data_provider(brokerage_name: str, data_provider_live_name: str, data_provider_historical_name: str, brokerage_product_id: str, data_provider_live_product_id: str, data_provider_historical_id: str) -> None:
api_client = mock.MagicMock()
create_lean_option(brokerage_name, data_provider_live_name, data_provider_historical_name, api_client)

is_exists = []
if brokerage_product_id is None and data_provider_historical_name != "Local":
assert len(api_client.method_calls) == 3
for m_c, id in zip(api_client.method_calls, [data_provider_live_product_id, data_provider_historical_id]):
if id in m_c[1]:
is_exists.append(True)
assert is_exists
assert len(is_exists) == 2
elif brokerage_product_id is None and data_provider_historical_name == "Local":
assert len(api_client.method_calls) == 2
if data_provider_live_product_id in api_client.method_calls[0][1]:
is_exists.append(True)
assert is_exists
assert len(is_exists) == 1
else:
assert len(api_client.method_calls) == 3
for m_c, id in zip(api_client.method_calls, [brokerage_product_id, data_provider_live_product_id, data_provider_historical_id]):
if id in f"{m_c[1]}":
is_exists.append(True)
assert is_exists
assert len(is_exists) == 3

@pytest.mark.parametrize("brokerage_name,data_provider_live_name,brokerage_product_id,data_provider_live_product_id",
[("Interactive Brokers", "IEX", "181", "333"),
("Tradier", "IEX", "185", "333")])
def test_live_non_interactive_deploy_with_different_brokerage_and_different_live_data_provider(brokerage_name: str, data_provider_live_name: str, brokerage_product_id: str, data_provider_live_product_id: str) -> None:
api_client = mock.MagicMock()
create_lean_option(brokerage_name, data_provider_live_name, None, api_client)

assert len(api_client.method_calls) == 2
is_exists = []
for m_c, id in zip(api_client.method_calls, [brokerage_product_id, data_provider_live_product_id]):
if id in m_c[1]:
is_exists.append(True)

assert is_exists
assert len(is_exists) == 2

@pytest.mark.parametrize("brokerage_name,data_provider_live_name,brokerage_product_id",
[("Bybit", "Bybit", "305"),
("Coinbase Advanced Trade", "Coinbase Advanced Trade", "183"),
("Interactive Brokers", "Interactive Brokers", "181"),
("Tradier", "Tradier", "185")])
def test_live_non_interactive_deploy_with_different_brokerage_with_the_same_live_data_provider(brokerage_name: str, data_provider_live_name: str, brokerage_product_id: str) -> None:
api_client = mock.MagicMock()
create_lean_option(brokerage_name, data_provider_live_name, None, api_client)

print(api_client.call_args_list)
print(api_client.call_args)

for m_c in api_client.method_calls:
if brokerage_product_id in m_c[1]:
is_exist = True

assert is_exist

@pytest.mark.parametrize("brokerage_name,data_provider_live_name,data_provider_live_product_id",
[("Paper Trading", "IEX", "333"),
("Paper Trading", "Polygon", "306")])
def test_live_non_interactive_deploy_paper_brokerage_different_live_data_provider(brokerage_name: str, data_provider_live_name: str, data_provider_live_product_id: str) -> None:
api_client = mock.MagicMock()
create_lean_option(brokerage_name, data_provider_live_name, None, api_client)

assert len(api_client.method_calls) == 2
for m_c in api_client.method_calls:
if data_provider_live_product_id in m_c[1]:
is_exist = True

assert is_exist
10 changes: 10 additions & 0 deletions tests/test_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
from datetime import datetime
from pathlib import Path
from typing import List
from lean.models.data_providers import all_data_providers
from lean.models.brokerages.local import all_local_brokerages, all_local_data_feeds

from lean.commands.create_project import (DEFAULT_CSHARP_MAIN, DEFAULT_CSHARP_NOTEBOOK, DEFAULT_PYTHON_MAIN,
DEFAULT_PYTHON_NOTEBOOK, LIBRARY_PYTHON_MAIN, LIBRARY_CSHARP_MAIN)
Expand Down Expand Up @@ -224,3 +226,11 @@ def create_lean_environments() -> List[QCLeanEnvironment]:
description="",
public=True)
]

def reset_state_installed_modules() -> None:
for data_provider in all_data_providers:
data_provider.__setattr__("_is_module_installed", False)
for local_brokerage in all_local_brokerages:
local_brokerage.__setattr__("_is_module_installed", False)
for local_data_feed in all_local_data_feeds:
local_data_feed.__setattr__("_is_module_installed", False)

0 comments on commit a031b02

Please sign in to comment.