Skip to content

Commit

Permalink
Fix 17 data wrong sorted queue (#18)
Browse files Browse the repository at this point in the history 
* fix: response return timeout can not be null but content.lenght == 0

* refactor: Rest Execute process (add retry process)
feat: update BaseResponse entity new property
feat: add log error if task is failed

* fix: historyProvider initizlizer

* test:feat: get option contracts in loop

* feat: increase timeout on 5 min of restClient

* test:feat: validate sort of time of history requests

* fix: use HashSet when get OptionChain in downloader

* Cleanup unit tests

---------

Co-authored-by: Jhonathan Abreu <[email protected]>
  • Loading branch information
@Romazes @jhonabreul
Romazes and jhonabreul authored Aug 12, 2024
1 parent 254cf22 commit 426e686
Show file tree
Hide file tree
Showing 5 changed files with 97 additions and 24 deletions.
24 changes: 24 additions & 0 deletions QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,30 @@ public void DownloadsDataFromCanonicalOptionSymbol()
Assert.That(distinctSymbols, Has.Count.GreaterThan(1).And.All.Matches<Symbol>(x => x.Canonical == symbol));
}

[Test]
public void OptionTradeHistoryIsSortedByTimeTest()
{
var under = Symbol.Create("OXY", SecurityType.Equity, Market.USA);
var symbol = Symbol.CreateOption(under, Market.USA, OptionStyle.American, OptionRight.Call, 55m, new DateTime(2024, 7, 19));

var startDateTime = new DateTime(2024, 7, 18);
var endDateTime = new DateTime(2024, 7, 19);

var parameters = new DataDownloaderGetParameters(symbol, Resolution.Minute, startDateTime, endDateTime, TickType.Trade);

using var downloader = new TestablePolygonDataDownloader();
var history = downloader.Get(parameters)?.ToList();

Assert.IsNotNull(history);
Assert.IsNotEmpty(history);

for (int i = 1; i < history.Count; i++)
{
if (history[i].Time < history[i - 1].Time)
Assert.Fail();
}
}

/// <summary>
/// Downloads historical data of an hardcoded index [SPX] based on specified parameters.
/// </summary>
Expand Down
2 changes: 1 addition & 1 deletion QuantConnect.Polygon.Tests/PolygonHistoryTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public class PolygonHistoryTests
public void SetUp()
{
_historyProvider = new PolygonDataProvider(_apiKey, streamingEnabled: false);
_historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null, false, null, null));
_historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null, false, null, null, null));

}

Expand Down
15 changes: 13 additions & 2 deletions QuantConnect.Polygon/PolygonDataDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
using NodaTime;
using QuantConnect.Data;
using QuantConnect.Util;
using QuantConnect.Logging;
using QuantConnect.Securities;
using QuantConnect.Configuration;
using System.Collections.Concurrent;
Expand Down Expand Up @@ -115,9 +116,15 @@ public PolygonDataDownloader()
{
blockingOptionCollection.Add(data);
}
})).ContinueWith(_ =>
})).ContinueWith(task =>
{
blockingOptionCollection.CompleteAdding();
if (task.IsFaulted && task.Exception != null)
{
var aggregateException = task.Exception.Flatten();
var errorMessages = string.Join("; ", aggregateException.InnerExceptions.Select(e => e.Message));
Log.Error($"{nameof(PolygonDataDownloader)}.{nameof(GetCanonicalOptionHistory)}: Task failed with error(s): {errorMessages}");
}
});

var options = blockingOptionCollection.GetConsumingEnumerable();
Expand All @@ -133,11 +140,15 @@ public PolygonDataDownloader()

protected virtual IEnumerable<Symbol> GetOptions(Symbol symbol, DateTime startUtc, DateTime endUtc)
{
HashSet<Symbol> seenOptions = new();
foreach (var date in Time.EachDay(startUtc.Date, endUtc.Date))
{
foreach (var option in _historyProvider.GetOptionChain(symbol, date))
{
yield return option;
if (seenOptions.Add(option))
{
yield return option;
}
}
}
}
Expand Down
74 changes: 53 additions & 21 deletions QuantConnect.Polygon/PolygonRestApiClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,12 @@
* limitations under the License.
*/

using Newtonsoft.Json.Linq;
using RestSharp;
using Newtonsoft.Json;
using QuantConnect.Util;
using Newtonsoft.Json.Linq;
using QuantConnect.Logging;
using QuantConnect.Configuration;
using QuantConnect.Util;

namespace QuantConnect.Lean.DataSource.Polygon
{
Expand All @@ -30,6 +31,11 @@ public class PolygonRestApiClient : IDisposable

private readonly RestClient _restClient;

/// <summary>
/// The maximum number of retry attempts for downloading data or executing a request.
/// </summary>
private const int MaxRetries = 10;

private readonly string _apiKey;

// Made virtual for testing purposes
Expand All @@ -45,7 +51,7 @@ public class PolygonRestApiClient : IDisposable
public PolygonRestApiClient(string apiKey)
{
_apiKey = apiKey;
_restClient = new RestClient(RestApiBaseUrl);
_restClient = new RestClient(RestApiBaseUrl) { Timeout = 300000 }; // 5 minutes in milliseconds
}

/// <summary>
Expand All @@ -63,6 +69,29 @@ public virtual IEnumerable<T> DownloadAndParseData<T>(RestRequest? request)
{
Log.Debug($"PolygonRestApi.DownloadAndParseData(): Downloading {request.Resource}");

var responseContent = DownloadWithRetries(request);
if (string.IsNullOrEmpty(responseContent))
{
throw new Exception($"{nameof(PolygonRestApiClient)}.{nameof(DownloadAndParseData)}: Failed to download data for {request.Resource} after {MaxRetries} attempts.");
}

var result = ParseResponse<T>(responseContent);

if (result == null)
{
yield break;
}

yield return result;

request = result.NextUrl != null ? new RestRequest(result.NextUrl, Method.GET) : null;
}
}

private string? DownloadWithRetries(RestRequest request)
{
for (int attempt = 0; attempt < MaxRetries; attempt++)
{
if (RateLimiter != null)
{
if (RateLimiter.IsRateLimited)
Expand All @@ -72,35 +101,38 @@ public virtual IEnumerable<T> DownloadAndParseData<T>(RestRequest? request)
RateLimiter.WaitToProceed();
}

request.AddHeader("Authorization", $"Bearer {_apiKey}");
request.AddOrUpdateHeader("Authorization", $"Bearer {_apiKey}");

var response = _restClient.Execute(request);
if (response == null)
{
Log.Debug($"PolygonRestApi.DownloadAndParseData(): No response for {request.Resource}");
yield break;
}

// If the data download was not successful, log the reason
var resultJson = JObject.Parse(response.Content);
if (resultJson["status"]?.ToString().ToUpperInvariant() != "OK")
var baseResponse = JsonConvert.DeserializeObject<BaseResponse>(response.Content);

if (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
{
Log.Debug($"PolygonRestApi.DownloadAndParseData(): No data for {request.Resource}. Reason: {response.Content}");
yield break;
Log.Debug($"PolygonRestApi.DownloadAndParseData(): Attempt {attempt + 1} failed. Error: {baseResponse?.Error ?? "Unknown error"}");
continue;
}

var result = resultJson.ToObject<T>();
if (result == null)
if (response != null && response.Content.Length > 0 && response.StatusCode == System.Net.HttpStatusCode.OK)
{
Log.Debug($"PolygonRestApi.DownloadAndParseData(): Unable to parse response for {request.Resource}. " +
$"Response: {response.Content}");
yield break;
return response.Content;
}
}

yield return result;
Log.Debug($"PolygonRestApi.DownloadAndParseData(): Failed after {MaxRetries} attempts for {request.Resource}");
return null;
}

request = result.NextUrl != null ? new RestRequest(result.NextUrl, Method.GET) : null;
private T? ParseResponse<T>(string responseContent) where T : BaseResponse
{
var result = JObject.Parse(responseContent).ToObject<T>();

if (result == null)
{
throw new ArgumentException($"{nameof(PolygonRestApiClient)}.{nameof(ParseResponse)}: Unable to parse response. Response: {responseContent}");
}

return result;
}

public void Dispose()
Expand Down
6 changes: 6 additions & 0 deletions QuantConnect.Polygon/Rest/BaseResponse.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,12 @@ public class BaseResponse
[JsonProperty("status")]
public string Status { get; set; }

/// <summary>
/// The error message of response
/// </summary>
[JsonProperty("error")]
public string? Error { get; set; }

/// <summary>
/// The URL to the next page of results. This is null if there are no more results.
/// </summary>
Expand Down

0 comments on commit 426e686

Please sign in to comment.