diff --git a/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs b/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs index d6d258e..55f21cc 100644 --- a/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs +++ b/QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs @@ -107,6 +107,30 @@ public void DownloadsDataFromCanonicalOptionSymbol() Assert.That(distinctSymbols, Has.Count.GreaterThan(1).And.All.Matches(x => x.Canonical == symbol)); } + [Test] + public void OptionTradeHistoryIsSortedByTimeTest() + { + var under = Symbol.Create("OXY", SecurityType.Equity, Market.USA); + var symbol = Symbol.CreateOption(under, Market.USA, OptionStyle.American, OptionRight.Call, 55m, new DateTime(2024, 7, 19)); + + var startDateTime = new DateTime(2024, 7, 18); + var endDateTime = new DateTime(2024, 7, 19); + + var parameters = new DataDownloaderGetParameters(symbol, Resolution.Minute, startDateTime, endDateTime, TickType.Trade); + + using var downloader = new TestablePolygonDataDownloader(); + var history = downloader.Get(parameters)?.ToList(); + + Assert.IsNotNull(history); + Assert.IsNotEmpty(history); + + for (int i = 1; i < history.Count; i++) + { + if (history[i].Time < history[i - 1].Time) + Assert.Fail(); + } + } + /// /// Downloads historical data of an hardcoded index [SPX] based on specified parameters. /// diff --git a/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs b/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs index c1a671e..d4695c1 100644 --- a/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs +++ b/QuantConnect.Polygon.Tests/PolygonHistoryTests.cs @@ -42,7 +42,7 @@ public class PolygonHistoryTests public void SetUp() { _historyProvider = new PolygonDataProvider(_apiKey, streamingEnabled: false); - _historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null, false, null, null)); + _historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null, false, null, null, null)); } diff --git a/QuantConnect.Polygon/PolygonDataDownloader.cs b/QuantConnect.Polygon/PolygonDataDownloader.cs index 19da7ba..6ecc09b 100644 --- a/QuantConnect.Polygon/PolygonDataDownloader.cs +++ b/QuantConnect.Polygon/PolygonDataDownloader.cs @@ -16,6 +16,7 @@ using NodaTime; using QuantConnect.Data; using QuantConnect.Util; +using QuantConnect.Logging; using QuantConnect.Securities; using QuantConnect.Configuration; using System.Collections.Concurrent; @@ -115,9 +116,15 @@ public PolygonDataDownloader() { blockingOptionCollection.Add(data); } - })).ContinueWith(_ => + })).ContinueWith(task => { blockingOptionCollection.CompleteAdding(); + if (task.IsFaulted && task.Exception != null) + { + var aggregateException = task.Exception.Flatten(); + var errorMessages = string.Join("; ", aggregateException.InnerExceptions.Select(e => e.Message)); + Log.Error($"{nameof(PolygonDataDownloader)}.{nameof(GetCanonicalOptionHistory)}: Task failed with error(s): {errorMessages}"); + } }); var options = blockingOptionCollection.GetConsumingEnumerable(); @@ -133,11 +140,15 @@ public PolygonDataDownloader() protected virtual IEnumerable GetOptions(Symbol symbol, DateTime startUtc, DateTime endUtc) { + HashSet seenOptions = new(); foreach (var date in Time.EachDay(startUtc.Date, endUtc.Date)) { foreach (var option in _historyProvider.GetOptionChain(symbol, date)) { - yield return option; + if (seenOptions.Add(option)) + { + yield return option; + } } } } diff --git a/QuantConnect.Polygon/PolygonRestApiClient.cs b/QuantConnect.Polygon/PolygonRestApiClient.cs index aac9a0b..f727528 100644 --- a/QuantConnect.Polygon/PolygonRestApiClient.cs +++ b/QuantConnect.Polygon/PolygonRestApiClient.cs @@ -13,11 +13,12 @@ * limitations under the License. */ -using Newtonsoft.Json.Linq; using RestSharp; +using Newtonsoft.Json; +using QuantConnect.Util; +using Newtonsoft.Json.Linq; using QuantConnect.Logging; using QuantConnect.Configuration; -using QuantConnect.Util; namespace QuantConnect.Lean.DataSource.Polygon { @@ -30,6 +31,11 @@ public class PolygonRestApiClient : IDisposable private readonly RestClient _restClient; + /// + /// The maximum number of retry attempts for downloading data or executing a request. + /// + private const int MaxRetries = 10; + private readonly string _apiKey; // Made virtual for testing purposes @@ -45,7 +51,7 @@ public class PolygonRestApiClient : IDisposable public PolygonRestApiClient(string apiKey) { _apiKey = apiKey; - _restClient = new RestClient(RestApiBaseUrl); + _restClient = new RestClient(RestApiBaseUrl) { Timeout = 300000 }; // 5 minutes in milliseconds } /// @@ -63,6 +69,29 @@ public virtual IEnumerable DownloadAndParseData(RestRequest? request) { Log.Debug($"PolygonRestApi.DownloadAndParseData(): Downloading {request.Resource}"); + var responseContent = DownloadWithRetries(request); + if (string.IsNullOrEmpty(responseContent)) + { + throw new Exception($"{nameof(PolygonRestApiClient)}.{nameof(DownloadAndParseData)}: Failed to download data for {request.Resource} after {MaxRetries} attempts."); + } + + var result = ParseResponse(responseContent); + + if (result == null) + { + yield break; + } + + yield return result; + + request = result.NextUrl != null ? new RestRequest(result.NextUrl, Method.GET) : null; + } + } + + private string? DownloadWithRetries(RestRequest request) + { + for (int attempt = 0; attempt < MaxRetries; attempt++) + { if (RateLimiter != null) { if (RateLimiter.IsRateLimited) @@ -72,35 +101,38 @@ public virtual IEnumerable DownloadAndParseData(RestRequest? request) RateLimiter.WaitToProceed(); } - request.AddHeader("Authorization", $"Bearer {_apiKey}"); + request.AddOrUpdateHeader("Authorization", $"Bearer {_apiKey}"); var response = _restClient.Execute(request); - if (response == null) - { - Log.Debug($"PolygonRestApi.DownloadAndParseData(): No response for {request.Resource}"); - yield break; - } - // If the data download was not successful, log the reason - var resultJson = JObject.Parse(response.Content); - if (resultJson["status"]?.ToString().ToUpperInvariant() != "OK") + var baseResponse = JsonConvert.DeserializeObject(response.Content); + + if (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests) { - Log.Debug($"PolygonRestApi.DownloadAndParseData(): No data for {request.Resource}. Reason: {response.Content}"); - yield break; + Log.Debug($"PolygonRestApi.DownloadAndParseData(): Attempt {attempt + 1} failed. Error: {baseResponse?.Error ?? "Unknown error"}"); + continue; } - var result = resultJson.ToObject(); - if (result == null) + if (response != null && response.Content.Length > 0 && response.StatusCode == System.Net.HttpStatusCode.OK) { - Log.Debug($"PolygonRestApi.DownloadAndParseData(): Unable to parse response for {request.Resource}. " + - $"Response: {response.Content}"); - yield break; + return response.Content; } + } - yield return result; + Log.Debug($"PolygonRestApi.DownloadAndParseData(): Failed after {MaxRetries} attempts for {request.Resource}"); + return null; + } - request = result.NextUrl != null ? new RestRequest(result.NextUrl, Method.GET) : null; + private T? ParseResponse(string responseContent) where T : BaseResponse + { + var result = JObject.Parse(responseContent).ToObject(); + + if (result == null) + { + throw new ArgumentException($"{nameof(PolygonRestApiClient)}.{nameof(ParseResponse)}: Unable to parse response. Response: {responseContent}"); } + + return result; } public void Dispose() diff --git a/QuantConnect.Polygon/Rest/BaseResponse.cs b/QuantConnect.Polygon/Rest/BaseResponse.cs index 4385ac1..700a2c3 100644 --- a/QuantConnect.Polygon/Rest/BaseResponse.cs +++ b/QuantConnect.Polygon/Rest/BaseResponse.cs @@ -28,6 +28,12 @@ public class BaseResponse [JsonProperty("status")] public string Status { get; set; } + /// + /// The error message of response + /// + [JsonProperty("error")] + public string? Error { get; set; } + /// /// The URL to the next page of results. This is null if there are no more results. ///