Fix 17 data wrong sorted queue

QuantConnect.Polygon.Tests/PolygonAdditionalTests.cs

@@ -0,0 +1,34 @@
+using System;
+using RestSharp;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Configuration;
+
+namespace QuantConnect.Lean.DataSource.Polygon.Tests
+{
+    [TestFixture]
+    public class PolygonAdditionalTests
+    {
+        private PolygonRestApiClient _restApiClient = new PolygonRestApiClient(Config.Get("polygon-api-key"));
+
+        [TestCase(5)]
+        public void GetOptionChainRequestTimeout(int amountRequest)
+        {
+            var referenceDate = new DateTime(2024, 08, 16);
+            var underlying = Symbol.Create("AAPL", SecurityType.Index, Market.USA);
+
+            var request = new RestRequest("/v3/reference/options/contracts", Method.GET);
+            request.AddQueryParameter("underlying_ticker", underlying.Value);
+            request.AddQueryParameter("as_of", referenceDate.ToStringInvariant("yyyy-MM-dd"));
+            request.AddQueryParameter("limit", "1000");
+
+            do
+            {
+                foreach (var contracts in _restApiClient.DownloadAndParseData<OptionChainResponse>(request))
+                {
+                    Assert.IsTrue(contracts.Results.Any());
+                } 
+            } while (--amountRequest > 0);
+        }
+    }
+}

QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs

@@ -107,6 +107,30 @@ public void DownloadsDataFromCanonicalOptionSymbol()
             Assert.That(distinctSymbols, Has.Count.GreaterThan(1).And.All.Matches<Symbol>(x => x.Canonical == symbol));
         }
 
+        [Test]
+        public void OptionTradeHistoryIsSortedByTimeTest()
+        {
+            var under = Symbol.Create("OXY", SecurityType.Equity, Market.USA);
+            var symbol = Symbol.CreateOption(under, Market.USA, OptionStyle.American, OptionRight.Call, 55m, new DateTime(2024, 7, 19));
+
+            var startDateTime = new DateTime(2024, 7, 18);
+            var endDateTime = new DateTime(2024, 7, 19);
+
+            var parameters = new DataDownloaderGetParameters(symbol, Resolution.Minute, startDateTime, endDateTime, TickType.Trade);
+
+            using var downloader = new TestablePolygonDataDownloader();
+            var history = downloader.Get(parameters)?.ToList();
+
+            Assert.IsNotNull(history);
+            Assert.IsNotEmpty(history);
+
+            for (int i = 1; i < history.Count; i++)
+            {
+                if (history[i].Time < history[i - 1].Time)
+                    Assert.Fail();
+            }
+        }
+
         /// <summary>
         /// Downloads historical data of an hardcoded index [SPX] based on specified parameters.
         /// </summary>

QuantConnect.Polygon.Tests/PolygonHistoryTests.cs

@@ -42,7 +42,7 @@ public class PolygonHistoryTests
         public void SetUp()
         {
             _historyProvider = new PolygonDataProvider(_apiKey, streamingEnabled: false);
-            _historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null, false, null, null));
+            _historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null, false, null, null, null));
 
         }
 

QuantConnect.Polygon/PolygonDataDownloader.cs

@@ -16,6 +16,7 @@
 using NodaTime;
 using QuantConnect.Data;
 using QuantConnect.Util;
+using QuantConnect.Logging;
 using QuantConnect.Securities;
 using QuantConnect.Configuration;
 using System.Collections.Concurrent;
@@ -115,9 +116,15 @@ public PolygonDataDownloader()
                 {
                     blockingOptionCollection.Add(data);
                 }
-            })).ContinueWith(_ =>
+            })).ContinueWith(task =>
             {
                 blockingOptionCollection.CompleteAdding();
+                if (task.IsFaulted && task.Exception != null)
+                {
+                    var aggregateException = task.Exception.Flatten();
+                    var errorMessages = string.Join("; ", aggregateException.InnerExceptions.Select(e => e.Message));
+                    Log.Error($"{nameof(PolygonDataDownloader)}.{nameof(GetCanonicalOptionHistory)}: Task failed with error(s): {errorMessages}");
+                }
             });
 
             var options = blockingOptionCollection.GetConsumingEnumerable();
@@ -133,11 +140,15 @@ public PolygonDataDownloader()
 
         protected virtual IEnumerable<Symbol> GetOptions(Symbol symbol, DateTime startUtc, DateTime endUtc)
         {
+            HashSet<Symbol> seenOptions = new();
             foreach (var date in Time.EachDay(startUtc.Date, endUtc.Date))
             {
                 foreach (var option in _historyProvider.GetOptionChain(symbol, date))
                 {
-                    yield return option;
+                    if (seenOptions.Add(option))
+                    {
+                        yield return option;
+                    }
                 }
             }
         }

QuantConnect.Polygon/PolygonRestApiClient.cs

@@ -13,11 +13,12 @@
  * limitations under the License.
 */
 
-using Newtonsoft.Json.Linq;
 using RestSharp;
+using Newtonsoft.Json;
+using QuantConnect.Util;
+using Newtonsoft.Json.Linq;
 using QuantConnect.Logging;
 using QuantConnect.Configuration;
-using QuantConnect.Util;
 
 namespace QuantConnect.Lean.DataSource.Polygon
 {
@@ -30,6 +31,11 @@ public class PolygonRestApiClient : IDisposable
 
         private readonly RestClient _restClient;
 
+        /// <summary>
+        /// The maximum number of retry attempts for downloading data or executing a request.
+        /// </summary>
+        private const int MaxRetries = 10;
+
         private readonly string _apiKey;
 
         // Made virtual for testing purposes
@@ -45,7 +51,7 @@ public class PolygonRestApiClient : IDisposable
         public PolygonRestApiClient(string apiKey)
         {
             _apiKey = apiKey;
-            _restClient = new RestClient(RestApiBaseUrl);
+            _restClient = new RestClient(RestApiBaseUrl) { Timeout = 300000 }; // 5 minutes in milliseconds
         }
 
         /// <summary>
@@ -63,6 +69,29 @@ public virtual IEnumerable<T> DownloadAndParseData<T>(RestRequest? request)
             {
                 Log.Debug($"PolygonRestApi.DownloadAndParseData(): Downloading {request.Resource}");
 
+                var responseContent = DownloadWithRetries(request);
+                if (string.IsNullOrEmpty(responseContent))
+                {
+                    throw new Exception($"{nameof(PolygonRestApiClient)}.{nameof(DownloadAndParseData)}: Failed to download data for {request.Resource} after {MaxRetries} attempts.");
+                }
+
+                var result = ParseResponse<T>(responseContent);
+
+                if (result == null)
+                {
+                    yield break;
+                }
+
+                yield return result;
+
+                request = result.NextUrl != null ? new RestRequest(result.NextUrl, Method.GET) : null;
+            }
+        }
+
+        private string? DownloadWithRetries(RestRequest request)
+        {
+            for (int attempt = 0; attempt < MaxRetries; attempt++)
+            {
                 if (RateLimiter != null)
                 {
                     if (RateLimiter.IsRateLimited)
@@ -72,35 +101,38 @@ public virtual IEnumerable<T> DownloadAndParseData<T>(RestRequest? request)
                     RateLimiter.WaitToProceed();
                 }
 
-                request.AddHeader("Authorization", $"Bearer {_apiKey}");
+                request.AddOrUpdateHeader("Authorization", $"Bearer {_apiKey}");
 
                 var response = _restClient.Execute(request);
-                if (response == null)
-                {
-                    Log.Debug($"PolygonRestApi.DownloadAndParseData(): No response for {request.Resource}");
-                    yield break;
-                }
 
-                // If the data download was not successful, log the reason
-                var resultJson = JObject.Parse(response.Content);
-                if (resultJson["status"]?.ToString().ToUpperInvariant() != "OK")
+                var baseResponse = JsonConvert.DeserializeObject<BaseResponse>(response.Content);
+
+                if (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
                 {
-                    Log.Debug($"PolygonRestApi.DownloadAndParseData(): No data for {request.Resource}. Reason: {response.Content}");
-                    yield break;
+                    Log.Debug($"PolygonRestApi.DownloadAndParseData(): Attempt {attempt + 1} failed. Error: {baseResponse?.Error ?? "Unknown error"}");
+                    continue;
                 }
 
-                var result = resultJson.ToObject<T>();
-                if (result == null)
+                if (response != null && response.Content.Length > 0 && response.StatusCode == System.Net.HttpStatusCode.OK)
                 {
-                    Log.Debug($"PolygonRestApi.DownloadAndParseData(): Unable to parse response for {request.Resource}. " +
-                        $"Response: {response.Content}");
-                    yield break;
+                    return response.Content;
                 }
+            }
 
-                yield return result;
+            Log.Debug($"PolygonRestApi.DownloadAndParseData(): Failed after {MaxRetries} attempts for {request.Resource}");
+            return null;
+        }
 
-                request = result.NextUrl != null ? new RestRequest(result.NextUrl, Method.GET) : null;
+        private T? ParseResponse<T>(string responseContent) where T : BaseResponse
+        {
+            var result = JObject.Parse(responseContent).ToObject<T>();
+
+            if (result == null)
+            {
+                throw new ArgumentException($"{nameof(PolygonRestApiClient)}.{nameof(ParseResponse)}: Unable to parse response. Response: {responseContent}");
             }
+
+            return result;
         }
 
         public void Dispose()

QuantConnect.Polygon/Rest/BaseResponse.cs

@@ -28,6 +28,12 @@ public class BaseResponse
         [JsonProperty("status")]
         public string Status { get; set; }
 
+        /// <summary>
+        /// The error message of response
+        /// </summary>
+        [JsonProperty("error")]
+        public string? Error { get; set; }
+
         /// <summary>
         /// The URL to the next page of results. This is null if there are no more results.
         /// </summary>