From 34272ee3a2f7908e26661078c3a8c51f332276d9 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 22 Feb 2024 00:32:56 +0200 Subject: [PATCH 1/8] feat: Globals instead of Config in ValidateSubscription --- QuantConnect.Polygon/PolygonDataProvider.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/QuantConnect.Polygon/PolygonDataProvider.cs b/QuantConnect.Polygon/PolygonDataProvider.cs index fdaf608..260cb31 100644 --- a/QuantConnect.Polygon/PolygonDataProvider.cs +++ b/QuantConnect.Polygon/PolygonDataProvider.cs @@ -488,9 +488,9 @@ private static void ValidateSubscription() try { const int productId = 306; - var userId = Config.GetInt("job-user-id"); - var token = Config.Get("api-access-token"); - var organizationId = Config.Get("job-organization-id", null); + var userId = Globals.UserId; + var token = Globals.UserToken; + var organizationId = Globals.OrganizationID; // Verify we can authenticate with this user and token var api = new ApiConnection(userId, token); if (!api.Connected) From 6a99c6e1d9014775834c8c9d3ded3e5eddfadb59 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 22 Feb 2024 00:41:09 +0200 Subject: [PATCH 2/8] refactor: get rid apiKey to Initialize() remove: validation of ApiKey in GetHistory() --- QuantConnect.Polygon/PolygonDataProvider.cs | 19 ++++++++++--------- .../PolygonHistoryProvider.cs | 6 +----- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/QuantConnect.Polygon/PolygonDataProvider.cs b/QuantConnect.Polygon/PolygonDataProvider.cs index 260cb31..ab55d64 100644 --- a/QuantConnect.Polygon/PolygonDataProvider.cs +++ b/QuantConnect.Polygon/PolygonDataProvider.cs @@ -121,16 +121,20 @@ public PolygonDataProvider(string apiKey, int maxSubscriptionsPerWebSocket, bool return; } - _apiKey = apiKey; - - Initialize(maxSubscriptionsPerWebSocket, streamingEnabled); + Initialize(apiKey, maxSubscriptionsPerWebSocket, streamingEnabled); } /// /// Initializes the data queue handler and validates the product subscription /// - private void Initialize(int maxSubscriptionsPerWebSocket, bool streamingEnabled = true) + private void Initialize(string apiKey, int maxSubscriptionsPerWebSocket, bool streamingEnabled = true) { + if (string.IsNullOrWhiteSpace(apiKey)) + { + throw new PolygonAuthenticationException("History calls for Polygon.io require an API key."); + } + _apiKey = apiKey; + _initialized = true; _dataAggregator = new PolygonAggregationManager(); RestApiClient = new PolygonRestApiClient(_apiKey); @@ -175,16 +179,13 @@ public void SetJob(LiveNodePacket job) throw new ArgumentException("The Polygon.io API key is missing from the brokerage data."); } - _apiKey = apiKey; - - var maxSubscriptionsPerWebSocket = 0; if (!job.BrokerageData.TryGetValue("polygon-max-subscriptions-per-websocket", out var maxSubscriptionsPerWebSocketStr) || - !int.TryParse(maxSubscriptionsPerWebSocketStr, out maxSubscriptionsPerWebSocket)) + !int.TryParse(maxSubscriptionsPerWebSocketStr, out var maxSubscriptionsPerWebSocket)) { maxSubscriptionsPerWebSocket = -1; } - Initialize(maxSubscriptionsPerWebSocket); + Initialize(apiKey, maxSubscriptionsPerWebSocket); } /// diff --git a/QuantConnect.Polygon/PolygonHistoryProvider.cs b/QuantConnect.Polygon/PolygonHistoryProvider.cs index 1ebb3a8..e5fbe58 100644 --- a/QuantConnect.Polygon/PolygonHistoryProvider.cs +++ b/QuantConnect.Polygon/PolygonHistoryProvider.cs @@ -68,14 +68,10 @@ public override IEnumerable GetHistory(IEnumerable reques /// An enumerable of BaseData points public IEnumerable GetHistory(HistoryRequest request) { - if (string.IsNullOrWhiteSpace(_apiKey)) - { - throw new PolygonAuthenticationException("History calls for Polygon.io require an API key."); - } - if (request.Symbol.IsCanonical() || !IsSupported(request.Symbol.SecurityType, request.DataType, request.TickType, request.Resolution)) { + // It is Logged in IsSupported(...) yield break; } From 0a4b3787c8ae190760f7b77586479c6efac3632a Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 23 Feb 2024 16:07:38 +0200 Subject: [PATCH 3/8] feat: return null from GetHistory and Get Downloader refactor: test with support null --- .../PolygonDataDownloaderTests.cs | 15 ++-- .../PolygonHistoryTests.cs | 11 ++- QuantConnect.Polygon/PolygonDataDownloader.cs | 80 +++++++++++++------ .../PolygonHistoryProvider.cs | 31 +++++-- 4 files changed, 96 insertions(+), 41 deletions(-) diff --git a/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs b/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs index 0610175..9bc846d 100644 --- a/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs +++ b/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs @@ -60,17 +60,22 @@ public void DownloadsHistoricalData(Symbol symbol, Resolution resolution, TimeSp [TestCaseSource(nameof(IndexHistoricalDataTestCases))] [Explicit("This tests require a Polygon.io api key, requires internet and are long.")] - public void DownloadsIndexHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpy) + public void DownloadsIndexHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty) { var symbol = Symbol.Create("SPX", SecurityType.Index, Market.USA); var request = PolygonHistoryTests.CreateHistoryRequest(symbol, resolution, tickType, period); var parameters = new DataDownloaderGetParameters(symbol, resolution, request.StartTimeUtc, request.EndTimeUtc, tickType); - var data = _downloader.Get(parameters).ToList(); + var data = _downloader.Get(parameters)?.ToList(); + + if (data == null) + { + Assert.Pass("History returns null result"); + } Log.Trace("Data points retrieved: " + data.Count); - if (shouldBeEmpy) + if (shouldBeEmpty) { Assert.That(data, Is.Empty); } @@ -86,9 +91,9 @@ public void DownloadsDataFromCanonicalOptionSymbol() { var symbol = Symbol.CreateCanonicalOption(Symbol.Create("SPY", SecurityType.Equity, Market.USA)); var parameters = new DataDownloaderGetParameters(symbol, Resolution.Hour, - new DateTime(2024, 01, 03), new DateTime(2024, 01, 04), TickType.Trade); + new DateTime(2024, 02, 22), new DateTime(2024, 02, 23), TickType.Trade); using var downloader = new TestablePolygonDataDownloader(); - var data = downloader.Get(parameters).ToList(); + var data = downloader.Get(parameters)?.ToList(); Log.Trace("Data points retrieved: " + data.Count); diff --git a/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs b/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs index 6ce4e9b..3cd1c9e 100644 --- a/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs +++ b/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs @@ -148,7 +148,12 @@ public void GetsIndexHistoricalData(Resolution resolution, TimeSpan period, Tick { var symbol = Symbol.Create("SPX", SecurityType.Index, Market.USA); var requests = new List { CreateHistoryRequest(symbol, resolution, tickType, period) }; - var history = _historyProvider.GetHistory(requests, TimeZones.Utc).ToList(); + var history = _historyProvider.GetHistory(requests, TimeZones.Utc)?.ToList(); + + if (history == null) + { + Assert.Pass("History returns null result"); + } Log.Trace("Data points retrieved: " + history.Count); @@ -223,9 +228,9 @@ public void ReturnsEmptyForUnsupportedSecurityTypeResolutionOrTickType(Symbol sy { using var historyProvider = new TestPolygonHistoryProvider(); var request = CreateHistoryRequest(symbol, resolution, tickType, TimeSpan.FromDays(100)); - var history = historyProvider.GetHistory(request).ToList(); + var history = historyProvider.GetHistory(request)?.ToList(); - Assert.That(history, Is.Empty); + Assert.IsNull(history); Assert.That(historyProvider.TestRestApiClient.ApiCallsCount, Is.EqualTo(0)); } diff --git a/QuantConnect.Polygon/PolygonDataDownloader.cs b/QuantConnect.Polygon/PolygonDataDownloader.cs index 60089d2..479fbfb 100644 --- a/QuantConnect.Polygon/PolygonDataDownloader.cs +++ b/QuantConnect.Polygon/PolygonDataDownloader.cs @@ -13,10 +13,11 @@ * limitations under the License. */ -using QuantConnect.Configuration; +using NodaTime; using QuantConnect.Data; -using QuantConnect.Securities; using QuantConnect.Util; +using QuantConnect.Securities; +using QuantConnect.Configuration; using System.Collections.Concurrent; namespace QuantConnect.Lean.DataSource.Polygon @@ -55,7 +56,7 @@ public PolygonDataDownloader() /// /// Parameters for the historical data request /// Enumerable of base data for this symbol - public IEnumerable Get(DataDownloaderGetParameters parameters) + public IEnumerable? Get(DataDownloaderGetParameters parameters) { var symbol = parameters.Symbol; var resolution = parameters.Resolution; @@ -65,7 +66,7 @@ public IEnumerable Get(DataDownloaderGetParameters parameters) if (endUtc < startUtc) { - yield break; + return null; } var dataType = LeanData.GetDataType(resolution, tickType); @@ -74,35 +75,64 @@ public IEnumerable Get(DataDownloaderGetParameters parameters) if (symbol.IsCanonical()) { - using var dataQueue = new BlockingCollection(); - var symbols = GetOptions(symbol, startUtc, endUtc); + return GetBaseData(symbol, startUtc, endUtc, dataType, resolution, exchangeHours, dataTimeZone, tickType); + } + else + { + var historyRequest = new HistoryRequest(startUtc, endUtc, dataType, symbol, resolution, exchangeHours, dataTimeZone, resolution, + true, false, DataNormalizationMode.Raw, tickType); - Task.Run(() => Parallel.ForEach(symbols, targetSymbol => - { - var historyRequest = new HistoryRequest(startUtc, endUtc, dataType, targetSymbol, resolution, exchangeHours, dataTimeZone, - resolution, true, false, DataNormalizationMode.Raw, tickType); - foreach (var data in _historyProvider.GetHistory(historyRequest)) - { - dataQueue.Add(data); - } - })).ContinueWith(_ => - { - dataQueue.CompleteAdding(); - }); + var historyData = _historyProvider.GetHistory(historyRequest); - foreach (var data in dataQueue.GetConsumingEnumerable()) + if (historyData == null) { - yield return data; + return null; } + + return GetBaseData(historyData); } - else + } + + private IEnumerable? GetBaseData(Symbol symbol, DateTime startUtc, DateTime endUtc, Type dataType, Resolution resolution, SecurityExchangeHours exchangeHours, DateTimeZone dataTimeZone, TickType tickType) + { + var dataQueue = new BlockingCollection(); + var symbols = GetOptions(symbol, startUtc, endUtc); + + Task.Run(() => Parallel.ForEach(symbols, targetSymbol => { - var historyRequest = new HistoryRequest(startUtc, endUtc, dataType, symbol, resolution, exchangeHours, dataTimeZone, resolution, - true, false, DataNormalizationMode.Raw, tickType); - foreach (var data in _historyProvider.GetHistory(historyRequest)) + var historyRequest = new HistoryRequest(startUtc, endUtc, dataType, targetSymbol, resolution, exchangeHours, dataTimeZone, + resolution, true, false, DataNormalizationMode.Raw, tickType); + + var history = _historyProvider.GetHistory(historyRequest); + + if (history == null) { - yield return data; + return; } + + foreach (var data in history) + { + dataQueue.Add(data); + } + })).ContinueWith(_ => + { + dataQueue.CompleteAdding(); + }); + + // Validate: data is not null, we have gotten anything at least + if (!dataQueue.TryTake(out _, TimeSpan.FromSeconds(10))) + { + return null; + } + + return GetBaseData(dataQueue.GetConsumingEnumerable()); + } + + private IEnumerable GetBaseData(IEnumerable baseData) + { + foreach (var data in baseData) + { + yield return data; } } diff --git a/QuantConnect.Polygon/PolygonHistoryProvider.cs b/QuantConnect.Polygon/PolygonHistoryProvider.cs index e5fbe58..fa0c59a 100644 --- a/QuantConnect.Polygon/PolygonHistoryProvider.cs +++ b/QuantConnect.Polygon/PolygonHistoryProvider.cs @@ -48,16 +48,24 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) /// The historical data requests /// The time zone used when time stamping the slice instances /// An enumerable of the slices of data covering the span specified in each request - public override IEnumerable GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) + public override IEnumerable? GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) { var subscriptions = new List(); foreach (var request in requests) { var history = GetHistory(request); + if (history == null) + { + continue; + } var subscription = CreateSubscription(request, history); subscriptions.Add(subscription); } + if (subscriptions.Count == 0) + { + return null; + } return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); } @@ -66,13 +74,13 @@ public override IEnumerable GetHistory(IEnumerable reques /// /// The historical data request /// An enumerable of BaseData points - public IEnumerable GetHistory(HistoryRequest request) + public IEnumerable? GetHistory(HistoryRequest request) { if (request.Symbol.IsCanonical() || !IsSupported(request.Symbol.SecurityType, request.DataType, request.TickType, request.Resolution)) { // It is Logged in IsSupported(...) - yield break; + return null; } // Quote data can only be fetched from Polygon from their Quote Tick endpoint, @@ -80,21 +88,27 @@ public IEnumerable GetHistory(HistoryRequest request) if (request.TickType == TickType.Quote && request.Resolution > Resolution.Second) { Log.Error("PolygonDataProvider.GetHistory(): Quote data above second resolution is not supported."); - yield break; + return null; } // Use the trade aggregates API for resolutions above tick for fastest results if (request.TickType == TickType.Trade && request.Resolution > Resolution.Tick) { - foreach (var data in GetAggregates(request)) + var data = GetAggregates(request); + + if (data == null) { - Interlocked.Increment(ref _dataPointCount); - yield return data; + return null; } - yield break; + return data; } + return GetHistoryThroughDataConsolidator(request); + } + + private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request) + { IDataConsolidator consolidator; IEnumerable history; @@ -156,6 +170,7 @@ private IEnumerable GetAggregates(HistoryRequest request) var utcTime = Time.UnixMillisecondTimeStampToDateTime(bar.Timestamp); var time = GetTickTime(request.Symbol, utcTime); + Interlocked.Increment(ref _dataPointCount); yield return new TradeBar(time, request.Symbol, bar.Open, bar.High, bar.Low, bar.Close, bar.Volume, resolutionTimeSpan); } From f45475cf388a896ec78b291f51acc27f4b0f6eaa Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 23 Feb 2024 18:31:26 +0200 Subject: [PATCH 4/8] refactor: invalid index history/download tests --- .../PolygonDataDownloaderTests.cs | 44 +++++++++++---- .../PolygonHistoryTests.cs | 55 +++++++++++++++---- 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs b/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs index 9bc846d..6664e69 100644 --- a/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs +++ b/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs @@ -15,6 +15,7 @@ */ using NUnit.Framework; +using QuantConnect.Data; using QuantConnect.Logging; using QuantConnect.Util; using System; @@ -62,16 +63,7 @@ public void DownloadsHistoricalData(Symbol symbol, Resolution resolution, TimeSp [Explicit("This tests require a Polygon.io api key, requires internet and are long.")] public void DownloadsIndexHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty) { - var symbol = Symbol.Create("SPX", SecurityType.Index, Market.USA); - var request = PolygonHistoryTests.CreateHistoryRequest(symbol, resolution, tickType, period); - - var parameters = new DataDownloaderGetParameters(symbol, resolution, request.StartTimeUtc, request.EndTimeUtc, tickType); - var data = _downloader.Get(parameters)?.ToList(); - - if (data == null) - { - Assert.Pass("History returns null result"); - } + var data = DownloadIndexHistoryData(resolution, period, tickType); Log.Trace("Data points retrieved: " + data.Count); @@ -85,6 +77,17 @@ public void DownloadsIndexHistoricalData(Resolution resolution, TimeSpan period, } } + private static TestCaseData[] IndexHistoricalInvalidDataTestCases => PolygonHistoryTests.IndexHistoricalInvalidDataTestCases; + + [TestCaseSource(nameof(IndexHistoricalInvalidDataTestCases))] + [Explicit("This tests require a Polygon.io api key, requires internet and are long.")] + public void DownloadsIndexInvalidHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty) + { + var data = DownloadIndexHistoryData(resolution, period, tickType); + + Assert.IsNull(data); + } + [Test] [Explicit("This tests require a Polygon.io api key, requires internet and are long.")] public void DownloadsDataFromCanonicalOptionSymbol() @@ -104,6 +107,27 @@ public void DownloadsDataFromCanonicalOptionSymbol() Assert.That(distinctSymbols, Has.Count.GreaterThan(1).And.All.Matches(x => x.Canonical == symbol)); } + /// + /// Downloads historical data of an hardcoded index [SPX] based on specified parameters. + /// + /// The resolution of the historical data to download. + /// The time period for which historical data is requested. + /// The type of ticks for the historical data. + /// A list of containing downloaded historical data of the index. + /// + /// The parameter determines the granularity of the historical data, + /// while the parameter specifies the duration of the historical data to be downloaded. + /// The parameter specifies the type of ticks to be included in the historical data. + /// + private List DownloadIndexHistoryData(Resolution resolution, TimeSpan period, TickType tickType) + { + var symbol = Symbol.Create("SPX", SecurityType.Index, Market.USA); + var request = PolygonHistoryTests.CreateHistoryRequest(symbol, resolution, tickType, period); + + var parameters = new DataDownloaderGetParameters(symbol, resolution, request.StartTimeUtc, request.EndTimeUtc, tickType); + return _downloader.Get(parameters)?.ToList(); + } + private class TestablePolygonDataDownloader : PolygonDataDownloader { protected override IEnumerable GetOptions(Symbol symbol, DateTime startUtc, DateTime endUtc) diff --git a/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs b/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs index 3cd1c9e..af832be 100644 --- a/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs +++ b/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs @@ -134,10 +134,7 @@ internal static TestCaseData[] IndexHistoricalDataTestCases // Quotes: quote data is not available for indexes new TestCaseData(Resolution.Tick, TimeSpan.FromMinutes(5), TickType.Quote, true), - new TestCaseData(Resolution.Second, TimeSpan.FromMinutes(5), TickType.Quote, true), - new TestCaseData(Resolution.Minute, TimeSpan.FromMinutes(5), TickType.Quote, true), - new TestCaseData(Resolution.Hour, TimeSpan.FromMinutes(5), TickType.Quote, true), - new TestCaseData(Resolution.Daily, TimeSpan.FromMinutes(5), TickType.Quote, true), + new TestCaseData(Resolution.Second, TimeSpan.FromMinutes(5), TickType.Quote, true) }; } } @@ -146,14 +143,7 @@ internal static TestCaseData[] IndexHistoricalDataTestCases [Explicit("This tests require a Polygon.io api key, requires internet and are long.")] public void GetsIndexHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty) { - var symbol = Symbol.Create("SPX", SecurityType.Index, Market.USA); - var requests = new List { CreateHistoryRequest(symbol, resolution, tickType, period) }; - var history = _historyProvider.GetHistory(requests, TimeZones.Utc)?.ToList(); - - if (history == null) - { - Assert.Pass("History returns null result"); - } + var history = GetIndexHistory(resolution, period, tickType); Log.Trace("Data points retrieved: " + history.Count); @@ -167,6 +157,28 @@ public void GetsIndexHistoricalData(Resolution resolution, TimeSpan period, Tick } } + internal static TestCaseData[] IndexHistoricalInvalidDataTestCases + { + get + { + return new[] + { + new TestCaseData(Resolution.Daily, TimeSpan.FromMinutes(5), TickType.Quote, true), + new TestCaseData(Resolution.Hour, TimeSpan.FromMinutes(5), TickType.Quote, true), + new TestCaseData(Resolution.Minute, TimeSpan.FromMinutes(5), TickType.Quote, true), + }; + } + } + + [TestCaseSource(nameof(IndexHistoricalInvalidDataTestCases))] + [Explicit("This tests require a Polygon.io api key, requires internet and are long.")] + public void GetsIndexInvalidHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty) + { + var history = GetIndexHistory(resolution, period, tickType); + + Assert.IsNull(history); + } + [Test] [Explicit("This tests require a Polygon.io api key, requires internet and are long.")] public void GetsSameBarCountForDifferentResponseLimits() @@ -278,6 +290,25 @@ public void RateLimitsHistoryApiCalls(int historyRequestsCount) Assert.That(delay, Is.LessThanOrEqualTo(upperBound), $"The rate gate was late: {delay - upperBound}"); } + /// + /// Retrieves the historical data of an hardcoded index [SPX] based on specified parameters. + /// + /// The resolution of the historical data to retrieve. + /// The time period for which historical data is requested. + /// The type of ticks for the historical data. + /// A list of containing historical data of the index. + /// + /// The parameter determines the granularity of the historical data, + /// while the parameter specifies the duration of the historical data to be retrieved. + /// The parameter specifies the type of ticks to be included in the historical data. + /// + internal List GetIndexHistory(Resolution resolution, TimeSpan period, TickType tickType) + { + var symbol = Symbol.Create("SPX", SecurityType.Index, Market.USA); + var requests = new List { CreateHistoryRequest(symbol, resolution, tickType, period) }; + return _historyProvider.GetHistory(requests, TimeZones.Utc)?.ToList(); + } + internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period) { var end = new DateTime(2023, 12, 15, 16, 0, 0); From 1db7907c6bb9f915eda7e32988f3b850c2836b73 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 23 Feb 2024 18:47:12 +0200 Subject: [PATCH 5/8] remove: extra ienumerable() rename: several methods fix: return empty result if data is wrong --- .../PolygonHistoryTests.cs | 2 +- QuantConnect.Polygon/PolygonDataDownloader.cs | 18 +++++------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs b/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs index af832be..c16c4d4 100644 --- a/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs +++ b/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs @@ -236,7 +236,7 @@ public void GetsSameBarCountForDifferentResponseLimits() }; [TestCaseSource(nameof(UssuportedSecurityTypesResolutionsAndTickTypesTestCases))] - public void ReturnsEmptyForUnsupportedSecurityTypeResolutionOrTickType(Symbol symbol, Resolution resolution, TickType tickType) + public void ReturnsNullForUnsupportedSecurityTypeResolutionOrTickType(Symbol symbol, Resolution resolution, TickType tickType) { using var historyProvider = new TestPolygonHistoryProvider(); var request = CreateHistoryRequest(symbol, resolution, tickType, TimeSpan.FromDays(100)); diff --git a/QuantConnect.Polygon/PolygonDataDownloader.cs b/QuantConnect.Polygon/PolygonDataDownloader.cs index 479fbfb..bdf8078 100644 --- a/QuantConnect.Polygon/PolygonDataDownloader.cs +++ b/QuantConnect.Polygon/PolygonDataDownloader.cs @@ -66,7 +66,7 @@ public PolygonDataDownloader() if (endUtc < startUtc) { - return null; + return Enumerable.Empty(); } var dataType = LeanData.GetDataType(resolution, tickType); @@ -75,7 +75,7 @@ public PolygonDataDownloader() if (symbol.IsCanonical()) { - return GetBaseData(symbol, startUtc, endUtc, dataType, resolution, exchangeHours, dataTimeZone, tickType); + return GetCanonicalOptionHistory(symbol, startUtc, endUtc, dataType, resolution, exchangeHours, dataTimeZone, tickType); } else { @@ -89,11 +89,11 @@ public PolygonDataDownloader() return null; } - return GetBaseData(historyData); + return historyData; } } - private IEnumerable? GetBaseData(Symbol symbol, DateTime startUtc, DateTime endUtc, Type dataType, Resolution resolution, SecurityExchangeHours exchangeHours, DateTimeZone dataTimeZone, TickType tickType) + private IEnumerable? GetCanonicalOptionHistory(Symbol symbol, DateTime startUtc, DateTime endUtc, Type dataType, Resolution resolution, SecurityExchangeHours exchangeHours, DateTimeZone dataTimeZone, TickType tickType) { var dataQueue = new BlockingCollection(); var symbols = GetOptions(symbol, startUtc, endUtc); @@ -125,15 +125,7 @@ public PolygonDataDownloader() return null; } - return GetBaseData(dataQueue.GetConsumingEnumerable()); - } - - private IEnumerable GetBaseData(IEnumerable baseData) - { - foreach (var data in baseData) - { - yield return data; - } + return dataQueue.GetConsumingEnumerable(); } protected virtual IEnumerable GetOptions(Symbol symbol, DateTime startUtc, DateTime endUtc) From 72a1d4192e00fc27e5c495691ebe03657d9f234b Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 23 Feb 2024 20:57:50 +0200 Subject: [PATCH 6/8] refactor: getting history for Option Symbols --- QuantConnect.Polygon/PolygonDataDownloader.cs | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/QuantConnect.Polygon/PolygonDataDownloader.cs b/QuantConnect.Polygon/PolygonDataDownloader.cs index bdf8078..5596c65 100644 --- a/QuantConnect.Polygon/PolygonDataDownloader.cs +++ b/QuantConnect.Polygon/PolygonDataDownloader.cs @@ -27,10 +27,20 @@ namespace QuantConnect.Lean.DataSource.Polygon /// public class PolygonDataDownloader : IDataDownloader, IDisposable { + /// private readonly PolygonDataProvider _historyProvider; + /// private readonly MarketHoursDatabase _marketHoursDatabase; + /// + private readonly CancellationTokenSource _cancellationTokenSource = new(); + + /// + /// Collection to get history for in enumerable way + /// + private BlockingCollection? _blockingOptionCollection; + /// /// Initializes a new instance of the /// @@ -66,7 +76,7 @@ public PolygonDataDownloader() if (endUtc < startUtc) { - return Enumerable.Empty(); + return null; } var dataType = LeanData.GetDataType(resolution, tickType); @@ -93,11 +103,13 @@ public PolygonDataDownloader() } } - private IEnumerable? GetCanonicalOptionHistory(Symbol symbol, DateTime startUtc, DateTime endUtc, Type dataType, Resolution resolution, SecurityExchangeHours exchangeHours, DateTimeZone dataTimeZone, TickType tickType) + private IEnumerable? GetCanonicalOptionHistory(Symbol symbol, DateTime startUtc, DateTime endUtc, Type dataType, + Resolution resolution, SecurityExchangeHours exchangeHours, DateTimeZone dataTimeZone, TickType tickType) { - var dataQueue = new BlockingCollection(); + _blockingOptionCollection = new BlockingCollection(); var symbols = GetOptions(symbol, startUtc, endUtc); + // Symbol can have a lot of Option parameters Task.Run(() => Parallel.ForEach(symbols, targetSymbol => { var historyRequest = new HistoryRequest(startUtc, endUtc, dataType, targetSymbol, resolution, exchangeHours, dataTimeZone, @@ -105,6 +117,8 @@ public PolygonDataDownloader() var history = _historyProvider.GetHistory(historyRequest); + // If history is null, it indicates an incorrect or missing request for historical data, + // so we skip processing for this symbol and move to the next one. if (history == null) { return; @@ -112,20 +126,22 @@ public PolygonDataDownloader() foreach (var data in history) { - dataQueue.Add(data); + _blockingOptionCollection.Add(data); } - })).ContinueWith(_ => + }), _cancellationTokenSource.Token).ContinueWith(_ => { - dataQueue.CompleteAdding(); - }); + _blockingOptionCollection.CompleteAdding(); + }, _cancellationTokenSource.Token); + + var options = _blockingOptionCollection.GetConsumingEnumerable(); - // Validate: data is not null, we have gotten anything at least - if (!dataQueue.TryTake(out _, TimeSpan.FromSeconds(10))) + // Validate if the collection contains at least one successful response from history. + if (!options.Any()) { return null; } - return dataQueue.GetConsumingEnumerable(); + return options; } protected virtual IEnumerable GetOptions(Symbol symbol, DateTime startUtc, DateTime endUtc) @@ -145,6 +161,8 @@ protected virtual IEnumerable GetOptions(Symbol symbol, DateTime startUt public void Dispose() { _historyProvider.DisposeSafely(); + _blockingOptionCollection.DisposeSafely(); + _cancellationTokenSource.DisposeSafely(); } } } From 208c8bb752e046fcbae0c7027e71d72a8e1b2f45 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 26 Feb 2024 16:03:16 +0200 Subject: [PATCH 7/8] refactor: remove extra class members from downloader --- QuantConnect.Polygon/PolygonDataDownloader.cs | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/QuantConnect.Polygon/PolygonDataDownloader.cs b/QuantConnect.Polygon/PolygonDataDownloader.cs index 5596c65..93083d9 100644 --- a/QuantConnect.Polygon/PolygonDataDownloader.cs +++ b/QuantConnect.Polygon/PolygonDataDownloader.cs @@ -33,14 +33,6 @@ public class PolygonDataDownloader : IDataDownloader, IDisposable /// private readonly MarketHoursDatabase _marketHoursDatabase; - /// - private readonly CancellationTokenSource _cancellationTokenSource = new(); - - /// - /// Collection to get history for in enumerable way - /// - private BlockingCollection? _blockingOptionCollection; - /// /// Initializes a new instance of the /// @@ -106,7 +98,7 @@ public PolygonDataDownloader() private IEnumerable? GetCanonicalOptionHistory(Symbol symbol, DateTime startUtc, DateTime endUtc, Type dataType, Resolution resolution, SecurityExchangeHours exchangeHours, DateTimeZone dataTimeZone, TickType tickType) { - _blockingOptionCollection = new BlockingCollection(); + var blockingOptionCollection = new BlockingCollection(); var symbols = GetOptions(symbol, startUtc, endUtc); // Symbol can have a lot of Option parameters @@ -126,14 +118,14 @@ public PolygonDataDownloader() foreach (var data in history) { - _blockingOptionCollection.Add(data); + blockingOptionCollection.Add(data); } - }), _cancellationTokenSource.Token).ContinueWith(_ => + })).ContinueWith(_ => { - _blockingOptionCollection.CompleteAdding(); - }, _cancellationTokenSource.Token); + blockingOptionCollection.CompleteAdding(); + }); - var options = _blockingOptionCollection.GetConsumingEnumerable(); + var options = blockingOptionCollection.GetConsumingEnumerable(); // Validate if the collection contains at least one successful response from history. if (!options.Any()) @@ -161,8 +153,6 @@ protected virtual IEnumerable GetOptions(Symbol symbol, DateTime startUt public void Dispose() { _historyProvider.DisposeSafely(); - _blockingOptionCollection.DisposeSafely(); - _cancellationTokenSource.DisposeSafely(); } } } From 1e627eea64cf02eec97db921cbb66004be80d625 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 27 Feb 2024 01:05:51 +0200 Subject: [PATCH 8/8] feat: missed flags to prevent user from spam --- QuantConnect.Polygon/PolygonDataDownloader.cs | 5 ---- .../PolygonHistoryProvider.cs | 27 ++++++++++++++++++- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/QuantConnect.Polygon/PolygonDataDownloader.cs b/QuantConnect.Polygon/PolygonDataDownloader.cs index 93083d9..19da7ba 100644 --- a/QuantConnect.Polygon/PolygonDataDownloader.cs +++ b/QuantConnect.Polygon/PolygonDataDownloader.cs @@ -66,11 +66,6 @@ public PolygonDataDownloader() var endUtc = parameters.EndUtc; var tickType = parameters.TickType; - if (endUtc < startUtc) - { - return null; - } - var dataType = LeanData.GetDataType(resolution, tickType); var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); diff --git a/QuantConnect.Polygon/PolygonHistoryProvider.cs b/QuantConnect.Polygon/PolygonHistoryProvider.cs index fa0c59a..1e0eadc 100644 --- a/QuantConnect.Polygon/PolygonHistoryProvider.cs +++ b/QuantConnect.Polygon/PolygonHistoryProvider.cs @@ -29,6 +29,16 @@ public partial class PolygonDataProvider : SynchronizingHistoryProvider { private int _dataPointCount; + /// + /// Indicates whether a error for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC. + /// + private bool _invalidStartTimeErrorFired; + + /// + /// Indicates whether an error has been fired due to invalid conditions if the TickType is and the is greater than one second. + /// + private bool _invalidTickTypeAndResolutionErrorFired; + /// /// Gets the total number of data points emitted by this history provider /// @@ -87,10 +97,25 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) // which would be too slow for anything above second resolution or long time spans. if (request.TickType == TickType.Quote && request.Resolution > Resolution.Second) { - Log.Error("PolygonDataProvider.GetHistory(): Quote data above second resolution is not supported."); + if (!_invalidTickTypeAndResolutionErrorFired) + { + Log.Error("PolygonDataProvider.GetHistory(): Quote data above second resolution is not supported."); + _invalidTickTypeAndResolutionErrorFired = true; + } return null; } + if (request.EndTimeUtc < request.StartTimeUtc) + { + if (!_invalidStartTimeErrorFired) + { + Log.Error($"{nameof(PolygonDataProvider)}.{nameof(GetHistory)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); + _invalidStartTimeErrorFired = true; + } + return null; + } + + // Use the trade aggregates API for resolutions above tick for fastest results if (request.TickType == TickType.Trade && request.Resolution > Resolution.Tick) {