Skip to content

Commit

Permalink
Use new Global Configs (#10)
Browse files Browse the repository at this point in the history 
* feat: Globals instead of Config in ValidateSubscription

* refactor: get rid apiKey to Initialize()
remove: validation of ApiKey in GetHistory()

* feat: return null from GetHistory and Get Downloader
refactor: test with support null

* refactor: invalid index history/download tests

* remove: extra ienumerable()
rename: several methods
fix: return empty result if data is wrong

* refactor: getting history for Option Symbols

* refactor: remove extra class members from downloader

* feat: missed flags to prevent user from spam
  • Loading branch information
@Romazes
Romazes authored Feb 27, 2024
1 parent 00b0f00 commit e0a3180
Show file tree
Hide file tree
Showing 5 changed files with 201 additions and 74 deletions.
47 changes: 38 additions & 9 deletions QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/

using NUnit.Framework;
using QuantConnect.Data;
using QuantConnect.Logging;
using QuantConnect.Util;
using System;
Expand Down Expand Up @@ -60,17 +61,13 @@ public void DownloadsHistoricalData(Symbol symbol, Resolution resolution, TimeSp

[TestCaseSource(nameof(IndexHistoricalDataTestCases))]
[Explicit("This tests require a Polygon.io api key, requires internet and are long.")]
public void DownloadsIndexHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpy)
public void DownloadsIndexHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty)
{
var symbol = Symbol.Create("SPX", SecurityType.Index, Market.USA);
var request = PolygonHistoryTests.CreateHistoryRequest(symbol, resolution, tickType, period);

var parameters = new DataDownloaderGetParameters(symbol, resolution, request.StartTimeUtc, request.EndTimeUtc, tickType);
var data = _downloader.Get(parameters).ToList();
var data = DownloadIndexHistoryData(resolution, period, tickType);

Log.Trace("Data points retrieved: " + data.Count);

if (shouldBeEmpy)
if (shouldBeEmpty)
{
Assert.That(data, Is.Empty);
}
Expand All @@ -80,15 +77,26 @@ public void DownloadsIndexHistoricalData(Resolution resolution, TimeSpan period,
}
}

private static TestCaseData[] IndexHistoricalInvalidDataTestCases => PolygonHistoryTests.IndexHistoricalInvalidDataTestCases;

[TestCaseSource(nameof(IndexHistoricalInvalidDataTestCases))]
[Explicit("This tests require a Polygon.io api key, requires internet and are long.")]
public void DownloadsIndexInvalidHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty)
{
var data = DownloadIndexHistoryData(resolution, period, tickType);

Assert.IsNull(data);
}

[Test]
[Explicit("This tests require a Polygon.io api key, requires internet and are long.")]
public void DownloadsDataFromCanonicalOptionSymbol()
{
var symbol = Symbol.CreateCanonicalOption(Symbol.Create("SPY", SecurityType.Equity, Market.USA));
var parameters = new DataDownloaderGetParameters(symbol, Resolution.Hour,
new DateTime(2024, 01, 03), new DateTime(2024, 01, 04), TickType.Trade);
new DateTime(2024, 02, 22), new DateTime(2024, 02, 23), TickType.Trade);
using var downloader = new TestablePolygonDataDownloader();
var data = downloader.Get(parameters).ToList();
var data = downloader.Get(parameters)?.ToList();

Log.Trace("Data points retrieved: " + data.Count);

Expand All @@ -99,6 +107,27 @@ public void DownloadsDataFromCanonicalOptionSymbol()
Assert.That(distinctSymbols, Has.Count.GreaterThan(1).And.All.Matches<Symbol>(x => x.Canonical == symbol));
}

/// <summary>
/// Downloads historical data of an hardcoded index [SPX] based on specified parameters.
/// </summary>
/// <param name="resolution">The resolution of the historical data to download.</param>
/// <param name="period">The time period for which historical data is requested.</param>
/// <param name="tickType">The type of ticks for the historical data.</param>
/// <returns>A list of <see cref="BaseData"/> containing downloaded historical data of the index.</returns>
/// <remarks>
/// The <paramref name="resolution"/> parameter determines the granularity of the historical data,
/// while the <paramref name="period"/> parameter specifies the duration of the historical data to be downloaded.
/// The <paramref name="tickType"/> parameter specifies the type of ticks to be included in the historical data.
/// </remarks>
private List<BaseData> DownloadIndexHistoryData(Resolution resolution, TimeSpan period, TickType tickType)
{
var symbol = Symbol.Create("SPX", SecurityType.Index, Market.USA);
var request = PolygonHistoryTests.CreateHistoryRequest(symbol, resolution, tickType, period);

var parameters = new DataDownloaderGetParameters(symbol, resolution, request.StartTimeUtc, request.EndTimeUtc, tickType);
return _downloader.Get(parameters)?.ToList();
}

private class TestablePolygonDataDownloader : PolygonDataDownloader
{
protected override IEnumerable<Symbol> GetOptions(Symbol symbol, DateTime startUtc, DateTime endUtc)
Expand Down
56 changes: 46 additions & 10 deletions QuantConnect.Polygon.Tests/PolygonHistoryTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,10 +134,7 @@ internal static TestCaseData[] IndexHistoricalDataTestCases

// Quotes: quote data is not available for indexes
new TestCaseData(Resolution.Tick, TimeSpan.FromMinutes(5), TickType.Quote, true),
new TestCaseData(Resolution.Second, TimeSpan.FromMinutes(5), TickType.Quote, true),
new TestCaseData(Resolution.Minute, TimeSpan.FromMinutes(5), TickType.Quote, true),
new TestCaseData(Resolution.Hour, TimeSpan.FromMinutes(5), TickType.Quote, true),
new TestCaseData(Resolution.Daily, TimeSpan.FromMinutes(5), TickType.Quote, true),
new TestCaseData(Resolution.Second, TimeSpan.FromMinutes(5), TickType.Quote, true)
};
}
}
Expand All @@ -146,9 +143,7 @@ internal static TestCaseData[] IndexHistoricalDataTestCases
[Explicit("This tests require a Polygon.io api key, requires internet and are long.")]
public void GetsIndexHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty)
{
var symbol = Symbol.Create("SPX", SecurityType.Index, Market.USA);
var requests = new List<HistoryRequest> { CreateHistoryRequest(symbol, resolution, tickType, period) };
var history = _historyProvider.GetHistory(requests, TimeZones.Utc).ToList();
var history = GetIndexHistory(resolution, period, tickType);

Log.Trace("Data points retrieved: " + history.Count);

Expand All @@ -162,6 +157,28 @@ public void GetsIndexHistoricalData(Resolution resolution, TimeSpan period, Tick
}
}

internal static TestCaseData[] IndexHistoricalInvalidDataTestCases
{
get
{
return new[]
{
new TestCaseData(Resolution.Daily, TimeSpan.FromMinutes(5), TickType.Quote, true),
new TestCaseData(Resolution.Hour, TimeSpan.FromMinutes(5), TickType.Quote, true),
new TestCaseData(Resolution.Minute, TimeSpan.FromMinutes(5), TickType.Quote, true),
};
}
}

[TestCaseSource(nameof(IndexHistoricalInvalidDataTestCases))]
[Explicit("This tests require a Polygon.io api key, requires internet and are long.")]
public void GetsIndexInvalidHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty)
{
var history = GetIndexHistory(resolution, period, tickType);

Assert.IsNull(history);
}

[Test]
[Explicit("This tests require a Polygon.io api key, requires internet and are long.")]
public void GetsSameBarCountForDifferentResponseLimits()
Expand Down Expand Up @@ -219,13 +236,13 @@ public void GetsSameBarCountForDifferentResponseLimits()
};

[TestCaseSource(nameof(UssuportedSecurityTypesResolutionsAndTickTypesTestCases))]
public void ReturnsEmptyForUnsupportedSecurityTypeResolutionOrTickType(Symbol symbol, Resolution resolution, TickType tickType)
public void ReturnsNullForUnsupportedSecurityTypeResolutionOrTickType(Symbol symbol, Resolution resolution, TickType tickType)
{
using var historyProvider = new TestPolygonHistoryProvider();
var request = CreateHistoryRequest(symbol, resolution, tickType, TimeSpan.FromDays(100));
var history = historyProvider.GetHistory(request).ToList();
var history = historyProvider.GetHistory(request)?.ToList();

Assert.That(history, Is.Empty);
Assert.IsNull(history);
Assert.That(historyProvider.TestRestApiClient.ApiCallsCount, Is.EqualTo(0));
}

Expand Down Expand Up @@ -273,6 +290,25 @@ public void RateLimitsHistoryApiCalls(int historyRequestsCount)
Assert.That(delay, Is.LessThanOrEqualTo(upperBound), $"The rate gate was late: {delay - upperBound}");
}

/// <summary>
/// Retrieves the historical data of an hardcoded index [SPX] based on specified parameters.
/// </summary>
/// <param name="resolution">The resolution of the historical data to retrieve.</param>
/// <param name="period">The time period for which historical data is requested.</param>
/// <param name="tickType">The type of ticks for the historical data.</param>
/// <returns>A list of <see cref="Slice"/> containing historical data of the index.</returns>
/// <remarks>
/// The <paramref name="resolution"/> parameter determines the granularity of the historical data,
/// while the <paramref name="period"/> parameter specifies the duration of the historical data to be retrieved.
/// The <paramref name="tickType"/> parameter specifies the type of ticks to be included in the historical data.
/// </remarks>
internal List<Slice> GetIndexHistory(Resolution resolution, TimeSpan period, TickType tickType)
{
var symbol = Symbol.Create("SPX", SecurityType.Index, Market.USA);
var requests = new List<HistoryRequest> { CreateHistoryRequest(symbol, resolution, tickType, period) };
return _historyProvider.GetHistory(requests, TimeZones.Utc)?.ToList();
}

internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period)
{
var end = new DateTime(2023, 12, 15, 16, 0, 0);
Expand Down
83 changes: 54 additions & 29 deletions QuantConnect.Polygon/PolygonDataDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,11 @@
* limitations under the License.
*/

using QuantConnect.Configuration;
using NodaTime;
using QuantConnect.Data;
using QuantConnect.Securities;
using QuantConnect.Util;
using QuantConnect.Securities;
using QuantConnect.Configuration;
using System.Collections.Concurrent;

namespace QuantConnect.Lean.DataSource.Polygon
Expand All @@ -26,8 +27,10 @@ namespace QuantConnect.Lean.DataSource.Polygon
/// </summary>
public class PolygonDataDownloader : IDataDownloader, IDisposable
{
/// <inheritdoc cref="PolygonDataProvider"/>
private readonly PolygonDataProvider _historyProvider;

/// <inheritdoc cref="MarketHoursDatabase" />
private readonly MarketHoursDatabase _marketHoursDatabase;

/// <summary>
Expand Down Expand Up @@ -55,55 +58,77 @@ public PolygonDataDownloader()
/// </summary>
/// <param name="parameters">Parameters for the historical data request</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable<BaseData> Get(DataDownloaderGetParameters parameters)
public IEnumerable<BaseData>? Get(DataDownloaderGetParameters parameters)
{
var symbol = parameters.Symbol;
var resolution = parameters.Resolution;
var startUtc = parameters.StartUtc;
var endUtc = parameters.EndUtc;
var tickType = parameters.TickType;

if (endUtc < startUtc)
{
yield break;
}

var dataType = LeanData.GetDataType(resolution, tickType);
var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);

if (symbol.IsCanonical())
{
using var dataQueue = new BlockingCollection<BaseData>();
var symbols = GetOptions(symbol, startUtc, endUtc);
return GetCanonicalOptionHistory(symbol, startUtc, endUtc, dataType, resolution, exchangeHours, dataTimeZone, tickType);
}
else
{
var historyRequest = new HistoryRequest(startUtc, endUtc, dataType, symbol, resolution, exchangeHours, dataTimeZone, resolution,
true, false, DataNormalizationMode.Raw, tickType);

Task.Run(() => Parallel.ForEach(symbols, targetSymbol =>
{
var historyRequest = new HistoryRequest(startUtc, endUtc, dataType, targetSymbol, resolution, exchangeHours, dataTimeZone,
resolution, true, false, DataNormalizationMode.Raw, tickType);
foreach (var data in _historyProvider.GetHistory(historyRequest))
{
dataQueue.Add(data);
}
})).ContinueWith(_ =>
{
dataQueue.CompleteAdding();
});
var historyData = _historyProvider.GetHistory(historyRequest);

foreach (var data in dataQueue.GetConsumingEnumerable())
if (historyData == null)
{
yield return data;
return null;
}

return historyData;
}
else
}

private IEnumerable<BaseData>? GetCanonicalOptionHistory(Symbol symbol, DateTime startUtc, DateTime endUtc, Type dataType,
Resolution resolution, SecurityExchangeHours exchangeHours, DateTimeZone dataTimeZone, TickType tickType)
{
var blockingOptionCollection = new BlockingCollection<BaseData>();
var symbols = GetOptions(symbol, startUtc, endUtc);

// Symbol can have a lot of Option parameters
Task.Run(() => Parallel.ForEach(symbols, targetSymbol =>
{
var historyRequest = new HistoryRequest(startUtc, endUtc, dataType, symbol, resolution, exchangeHours, dataTimeZone, resolution,
true, false, DataNormalizationMode.Raw, tickType);
foreach (var data in _historyProvider.GetHistory(historyRequest))
var historyRequest = new HistoryRequest(startUtc, endUtc, dataType, targetSymbol, resolution, exchangeHours, dataTimeZone,
resolution, true, false, DataNormalizationMode.Raw, tickType);

var history = _historyProvider.GetHistory(historyRequest);

// If history is null, it indicates an incorrect or missing request for historical data,
// so we skip processing for this symbol and move to the next one.
if (history == null)
{
yield return data;
return;
}

foreach (var data in history)
{
blockingOptionCollection.Add(data);
}
})).ContinueWith(_ =>
{
blockingOptionCollection.CompleteAdding();
});

var options = blockingOptionCollection.GetConsumingEnumerable();

// Validate if the collection contains at least one successful response from history.
if (!options.Any())
{
return null;
}

return options;
}

protected virtual IEnumerable<Symbol> GetOptions(Symbol symbol, DateTime startUtc, DateTime endUtc)
Expand Down
25 changes: 13 additions & 12 deletions QuantConnect.Polygon/PolygonDataProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,16 +121,20 @@ public PolygonDataProvider(string apiKey, int maxSubscriptionsPerWebSocket, bool
return;
}

_apiKey = apiKey;

Initialize(maxSubscriptionsPerWebSocket, streamingEnabled);
Initialize(apiKey, maxSubscriptionsPerWebSocket, streamingEnabled);
}

/// <summary>
/// Initializes the data queue handler and validates the product subscription
/// </summary>
private void Initialize(int maxSubscriptionsPerWebSocket, bool streamingEnabled = true)
private void Initialize(string apiKey, int maxSubscriptionsPerWebSocket, bool streamingEnabled = true)
{
if (string.IsNullOrWhiteSpace(apiKey))
{
throw new PolygonAuthenticationException("History calls for Polygon.io require an API key.");
}
_apiKey = apiKey;

_initialized = true;
_dataAggregator = new PolygonAggregationManager();
RestApiClient = new PolygonRestApiClient(_apiKey);
Expand Down Expand Up @@ -175,16 +179,13 @@ public void SetJob(LiveNodePacket job)
throw new ArgumentException("The Polygon.io API key is missing from the brokerage data.");
}

_apiKey = apiKey;

var maxSubscriptionsPerWebSocket = 0;
if (!job.BrokerageData.TryGetValue("polygon-max-subscriptions-per-websocket", out var maxSubscriptionsPerWebSocketStr) ||
!int.TryParse(maxSubscriptionsPerWebSocketStr, out maxSubscriptionsPerWebSocket))
!int.TryParse(maxSubscriptionsPerWebSocketStr, out var maxSubscriptionsPerWebSocket))
{
maxSubscriptionsPerWebSocket = -1;
}

Initialize(maxSubscriptionsPerWebSocket);
Initialize(apiKey, maxSubscriptionsPerWebSocket);
}

/// <summary>
Expand Down Expand Up @@ -488,9 +489,9 @@ private static void ValidateSubscription()
try
{
const int productId = 306;
var userId = Config.GetInt("job-user-id");
var token = Config.Get("api-access-token");
var organizationId = Config.Get("job-organization-id", null);
var userId = Globals.UserId;
var token = Globals.UserToken;
var organizationId = Globals.OrganizationID;
// Verify we can authenticate with this user and token
var api = new ApiConnection(userId, token);
if (!api.Connected)
Expand Down
Loading

0 comments on commit e0a3180

Please sign in to comment.