Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Minor fix for QC data download #454

Merged
merged 2 commits into from
May 2, 2024
Merged

Minor fix for QC data download #454

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8ed68ae
Minor fix for QC data download
Martin-Molinero May 2, 2024
ad28f25
More fixes and improvements
Martin-Molinero May 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -880,16 +880,16 @@ Options:
--dataset TEXT The name of the dataset to download non-interactively
--overwrite Overwrite existing local data
-y, --yes Automatically confirm payment confirmation prompts
--data-type [Trade|Quote|OpenInterest]
--data-type [Trade|Quote|Bulk|OpenInterest]
Specify the type of historical data
--resolution [Tick|Second|Minute|Hour|Daily]
Specify the resolution of the historical data
--security-type [Equity|Index|Forex|Cfd|Future|Crypto|CryptoFuture|Option|IndexOption|Commodity|FutureOption]
Specify the security type of the historical data
--market TEXT Specify the market name for tickers (e.g., 'USA', 'NYMEX', 'Binance')
--tickers TEXT Specify comma separated list of tickers to use for historical data request.
--start-date TEXT Specify the start date for the historical data request in the format yyyyMMdd.
--end-date TEXT Specify the end date for the historical data request in the format yyyyMMdd. (defaults
--ticker TEXT Specify comma separated list of tickers to use for historical data request.
--start TEXT Specify the start date for the historical data request in the format yyyyMMdd.
--end TEXT Specify the end date for the historical data request in the format yyyyMMdd. (defaults
to today)
--image TEXT The LEAN engine image to use (defaults to quantconnect/lean:latest)
--update Pull the LEAN engine image before running the Downloader Data Provider
Expand Down
46 changes: 27 additions & 19 deletions lean/commands/data/download.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,8 @@ def _select_products_non_interactive(organization: QCFullOrganization,
if option.condition is not None and not option.condition.check(option_results):
continue

user_input = ctx.params.get(option.id, None)
# if the option id has a '-' in its name, and it's a click option, in the click context it's available with '_'
user_input = ctx.params.get(option.id.replace('-', '_'), ctx.params.get(option.id, None))

if user_input is None:
missing_options.append(f"--{option.id} <{option.get_placeholder()}>: {option.description}")
Expand Down Expand Up @@ -512,13 +513,13 @@ def _configure_date_option(date_value: str, option_id: str, option_label: str) -
help="Specify the security type of the historical data")
@option("--market", type=str,
help="Specify the market name for tickers (e.g., 'USA', 'NYMEX', 'Binance')")
@option("--tickers",
@option("--ticker",
type=str,
help="Specify comma separated list of tickers to use for historical data request.")
@option("--start-date",
@option("--start",
type=str,
help="Specify the start date for the historical data request in the format yyyyMMdd.")
@option("--end-date",
@option("--end",
type=str,
help="Specify the end date for the historical data request in the format yyyyMMdd. (defaults to today)")
@option("--image",
Expand All @@ -543,9 +544,9 @@ def download(ctx: Context,
resolution: Optional[str],
security_type: Optional[str],
market: Optional[str],
tickers: Optional[str],
start_date: Optional[str],
end_date: Optional[str],
ticker: Optional[str],
start: Optional[str],
end: Optional[str],
image: Optional[str],
update: bool,
no_update: bool,
Expand Down Expand Up @@ -575,6 +576,9 @@ def download(ctx: Context,
"""
organization = _get_organization()

if dataset:
data_provider_historical = 'QuantConnect'

if data_provider_historical is None:
data_provider_historical = _get_historical_data_provider()

Expand Down Expand Up @@ -626,17 +630,21 @@ def download(ctx: Context,
market = _get_user_input_or_prompt(market, data_provider_support_markets,
data_provider_historical, "Select a Market")

if not tickers:
tickers = ','.join(DatasetTextOption(id="id",
label="Enter comma separated list of tickers to use for historical data request.",
description="description",
transform=DatasetTextOptionTransform.Lowercase,
multiple=True).configure_interactive().value)
if not ticker:
ticker = ','.join(DatasetTextOption(id="id",
label="Enter comma separated list of tickers to use for historical data request.",
description="description",
transform=DatasetTextOptionTransform.Uppercase,
multiple=True).configure_interactive().value)
else:
split_tickers = ticker.split(',')
# don't trust user provider tickers without spaces in between
ticker = ','.join([a_ticker.strip().upper() for a_ticker in split_tickers])

start_date = _configure_date_option(start_date, "start", "Please enter a Start Date in the format")
end_date = _configure_date_option(end_date, "end", "Please enter a End Date in the format")
start = _configure_date_option(start, "start", "Please enter a Start Date in the format")
end = _configure_date_option(end, "end", "Please enter a End Date in the format")

if start_date.value >= end_date.value:
if start.value >= end.value:
raise ValueError("Historical start date cannot be greater than or equal to historical end date.")

logger = container.logger
Expand Down Expand Up @@ -674,12 +682,12 @@ def download(ctx: Context,

dll_arguments = ["dotnet", "QuantConnect.DownloaderDataProvider.Launcher.dll",
"--data-type", data_type,
"--start-date", start_date.value.strftime("%Y%m%d"),
"--end-date", end_date.value.strftime("%Y%m%d"),
"--start-date", start.value.strftime("%Y%m%d"),
"--end-date", end.value.strftime("%Y%m%d"),
"--security-type", security_type,
"--market", market,
"--resolution", resolution,
"--tickers", tickers]
"--tickers", ticker]

run_options["commands"].append(' '.join(dll_arguments))

Expand Down
1 change: 1 addition & 0 deletions lean/models/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,6 +410,7 @@ class QCMinimalOrganization(WrappedBaseModel):
class QCDataType(str, Enum):
Trade = "Trade"
Quote = "Quote"
Bulk = "Bulk"
OpenInterest = "OpenInterest"

@classmethod
Expand Down
20 changes: 10 additions & 10 deletions tests/commands/data/test_download.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,9 @@ def _create_lean_data_download(data_provider_name: str,
"--data-type", data_type,
"--resolution", resolution,
"--security-type", security_type,
"--tickers", ','.join(tickers),
"--start-date", start_date,
"--end-date", end_date,
"--ticker", ','.join(tickers),
"--start", start_date,
"--end", end_date,
]
if market:
run_parameters.extend(["--market", market])
Expand All @@ -135,7 +135,7 @@ def _create_lean_data_download(data_provider_name: str,
return CliRunner().invoke(lean, run_parameters)


@pytest.mark.parametrize("data_provider,market,is_crypto,security_type,tickers,data_provider_parameters",
@pytest.mark.parametrize("data_provider,market,is_crypto,security_type,ticker,data_provider_parameters",
[("Polygon", "NYSE", False, "Equity", ["AAPL"], ["--polygon-api-key", "123"]),
("Binance", "Binance", True, "CryptoFuture", ["BTCUSDT"],
["--binance-exchange-name", "BinanceUS", "--binanceus-api-key", "123",
Expand All @@ -145,9 +145,9 @@ def _create_lean_data_download(data_provider_name: str,
("Interactive Brokers", "USA", False, "Index", ["INTL", "NVDA"],
["--ib-user-name", "123", "--ib-account", "Individual", "--ib-password", "123"])])
def test_download_data_non_interactive(data_provider: str, market: str, is_crypto: bool, security_type: str,
tickers: List[str], data_provider_parameters: List[str]):
ticker: List[str], data_provider_parameters: List[str]):
run_data_download = _create_lean_data_download(
data_provider, "Trade", "Minute", security_type, tickers, "20240101", "20240202",
data_provider, "Trade", "Minute", security_type, ticker, "20240101", "20240202",
_get_data_provider_config(is_crypto), market, data_provider_parameters)
assert run_data_download.exit_code == 0

Expand All @@ -174,11 +174,11 @@ def test_download_data_non_interactive_wrong_security_type(data_provider: str, w
assert wrong_security_type in error_msg


@pytest.mark.parametrize("data_provider,start_date,end_date",
@pytest.mark.parametrize("data_provider,start,end",
[("Polygon", "20240101", "20230202"), ("Polygon", "2024-01-01", "2023-02-02")])
def test_download_data_non_interactive_wrong_start_end_date(data_provider: str, start_date: str, end_date: str):
run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", "Equity", ["AAPL"], start_date,
end_date, _get_data_provider_config(), "USA",
def test_download_data_non_interactive_wrong_start_end_date(data_provider: str, start: str, end: str):
run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", "Equity", ["AAPL"], start,
end, _get_data_provider_config(), "USA",
extra_run_command=["--polygon-api-key", "123"])
assert run_data_download.exit_code == 1

Expand Down
Loading