From aa846f428c098c3ac4449fc5e16252ba2183b587 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 21 Feb 2024 17:09:18 +0200 Subject: [PATCH 01/14] rename: namespaces --- .github/workflows/build.yml | 12 ++++++------ DataProcessing/CoinApiDataConverter.cs | 2 +- DataProcessing/DataProcessing.csproj | 2 +- Lean.DataSource.CoinAPI.sln | 4 ++-- .../CoinAPIDataDownloaderTests.cs | 2 +- .../CoinAPIHistoryProviderTests.cs | 2 +- .../CoinAPISymbolMapperTests.cs | 2 +- QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs | 2 +- .../CoinApiDataQueueHandlerTest.cs | 2 +- QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs | 2 +- ... => QuantConnect.DataSource.CoinAPI.Tests.csproj} | 10 +++++----- QuantConnect.CoinAPI.Tests/TestSetup.cs | 2 +- QuantConnect.CoinAPI/CoinAPIDataDownloader.cs | 2 +- QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs | 6 +++--- QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs | 2 +- QuantConnect.CoinAPI/CoinApiProduct.cs | 2 +- QuantConnect.CoinAPI/CoinApiSymbol.cs | 2 +- QuantConnect.CoinAPI/CoinApiSymbolMapper.cs | 4 ++-- QuantConnect.CoinAPI/Messages/BaseMessage.cs | 2 +- QuantConnect.CoinAPI/Messages/ErrorMessage.cs | 2 +- QuantConnect.CoinAPI/Messages/HelloMessage.cs | 2 +- .../Messages/HistoricalDataMessage.cs | 2 +- QuantConnect.CoinAPI/Messages/QuoteMessage.cs | 2 +- QuantConnect.CoinAPI/Messages/TradeMessage.cs | 2 +- QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs | 2 +- ...csproj => QuantConnect.DataSource.CoinAPI.csproj} | 8 ++++---- 26 files changed, 42 insertions(+), 42 deletions(-) rename QuantConnect.CoinAPI.Tests/{QuantConnect.CoinAPI.Tests.csproj => QuantConnect.DataSource.CoinAPI.Tests.csproj} (81%) rename QuantConnect.CoinAPI/{QuantConnect.CoinAPI.csproj => QuantConnect.DataSource.CoinAPI.csproj} (84%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1a93820..bee2592 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,14 +46,14 @@ jobs: image: quantconnect/lean:foundation options: -v /home/runner/work:/__w --workdir /__w/Lean.DataSource.CoinAPI/Lean.DataSource.CoinAPI -e QC_JOB_USER_ID=${{ secrets.QC_JOB_USER_ID }} -e QC_API_ACCESS_TOKEN=${{ secrets.QC_API_ACCESS_TOKEN }} -e QC_JOB_ORGANIZATION_ID=${{ secrets.QC_JOB_ORGANIZATION_ID }} -e QC_COINAPI_API_KEY=${{ secrets.QC_COINAPI_API_KEY }} - - name: Build QuantConnect.CoinAPI - run: dotnet build ./QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + - name: Build QuantConnect.DataSource.CoinAPI + run: dotnet build ./QuantConnect.CoinAPI/QuantConnect.DataSource.CoinAPI.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - name: Build DataProcessing run: dotnet build ./DataProcessing/DataProcessing.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - - name: Build QuantConnect.CoinAPI.Tests - run: dotnet build ./QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + - name: Build QuantConnect.DataSource.CoinAPI.Tests + run: dotnet build ./QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - - name: Run QuantConnect.CoinAPI.Tests - run: dotnet test ./QuantConnect.CoinAPI.Tests/bin/Release/QuantConnect.CoinAPI.Tests.dll \ No newline at end of file + - name: Run QuantConnect.DataSource.CoinAPI.Tests + run: dotnet test ./QuantConnect.CoinAPI.Tests/bin/Release/QuantConnect.DataSource.CoinAPI.Tests.dll \ No newline at end of file diff --git a/DataProcessing/CoinApiDataConverter.cs b/DataProcessing/CoinApiDataConverter.cs index ccd00e9..ef1698c 100644 --- a/DataProcessing/CoinApiDataConverter.cs +++ b/DataProcessing/CoinApiDataConverter.cs @@ -19,7 +19,7 @@ using System.Diagnostics; using QuantConnect.Logging; using QuantConnect.ToolBox; -using QuantConnect.CoinAPI; +using QuantConnect.DataSource.CoinAPI; namespace QuantConnect.DataProcessing { diff --git a/DataProcessing/DataProcessing.csproj b/DataProcessing/DataProcessing.csproj index a5bc5cb..d27e9b0 100644 --- a/DataProcessing/DataProcessing.csproj +++ b/DataProcessing/DataProcessing.csproj @@ -29,7 +29,7 @@ - + diff --git a/Lean.DataSource.CoinAPI.sln b/Lean.DataSource.CoinAPI.sln index 1c6eb39..c53e966 100644 --- a/Lean.DataSource.CoinAPI.sln +++ b/Lean.DataSource.CoinAPI.sln @@ -3,9 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.5.002.0 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.CoinAPI", "QuantConnect.CoinAPI\QuantConnect.CoinAPI.csproj", "{2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataSource.CoinAPI", "QuantConnect.CoinAPI\QuantConnect.DataSource.CoinAPI.csproj", "{2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.CoinAPI.Tests", "QuantConnect.CoinAPI.Tests\QuantConnect.CoinAPI.Tests.csproj", "{337CEE6E-639A-448D-95ED-2C1628E26AF2}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataSource.CoinAPI.Tests", "QuantConnect.CoinAPI.Tests\QuantConnect.DataSource.CoinAPI.Tests.csproj", "{337CEE6E-639A-448D-95ED-2C1628E26AF2}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataProcessing", "DataProcessing\DataProcessing.csproj", "{881514B4-641E-4EDC-8020-6BEA0CC8F48C}" EndProject diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs index 669d48a..f70140c 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs @@ -17,7 +17,7 @@ using QuantConnect.Util; using QuantConnect.Logging; -namespace QuantConnect.CoinAPI.Tests +namespace QuantConnect.DataSource.CoinAPI.Tests { [TestFixture] public class CoinAPIDataDownloaderTests diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs index a7c93fd..c65b6a5 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs @@ -20,7 +20,7 @@ using QuantConnect.Data.Market; using QuantConnect.Securities; -namespace QuantConnect.CoinAPI.Tests +namespace QuantConnect.DataSource.CoinAPI.Tests { [TestFixture] public class CoinAPIHistoryProviderTests diff --git a/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs index 7303fc1..ccb1135 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs @@ -16,7 +16,7 @@ using NUnit.Framework; -namespace QuantConnect.CoinAPI.Tests +namespace QuantConnect.DataSource.CoinAPI.Tests { [TestFixture] public class CoinAPISymbolMapperTests diff --git a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs index 97ee813..c25bc37 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs @@ -16,7 +16,7 @@ using NUnit.Framework; using QuantConnect.Configuration; -namespace QuantConnect.CoinAPI.Tests +namespace QuantConnect.DataSource.CoinAPI.Tests { [TestFixture] public class CoinApiAdditionalTests diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs index c63d45a..193c281 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -21,7 +21,7 @@ using QuantConnect.Data.Market; using System.Collections.Concurrent; -namespace QuantConnect.CoinAPI.Tests +namespace QuantConnect.DataSource.CoinAPI.Tests { [TestFixture] public class CoinApiDataQueueHandlerTest diff --git a/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs index 849a1c8..3ed1ab5 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs @@ -19,7 +19,7 @@ using QuantConnect.Logging; using QuantConnect.Data.Market; -namespace QuantConnect.CoinAPI.Tests +namespace QuantConnect.DataSource.CoinAPI.Tests { public static class CoinApiTestHelper { diff --git a/QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj b/QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj similarity index 81% rename from QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj rename to QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj index d73485b..e689f25 100644 --- a/QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj +++ b/QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj @@ -5,10 +5,10 @@ AnyCPU net6.0 bin\$(Configuration)\ - QuantConnect.CoinAPI.Tests - QuantConnect.CoinAPI.Tests - QuantConnect.CoinAPI.Tests - QuantConnect.CoinAPI.Tests + QuantConnect.DataSource.CoinAPI.Tests + QuantConnect.DataSource.CoinAPI.Tests + QuantConnect.DataSource.CoinAPI.Tests + QuantConnect.DataSource.CoinAPI.Tests false enable enable @@ -30,7 +30,7 @@ - + diff --git a/QuantConnect.CoinAPI.Tests/TestSetup.cs b/QuantConnect.CoinAPI.Tests/TestSetup.cs index 05bf2fb..a0ebc0e 100644 --- a/QuantConnect.CoinAPI.Tests/TestSetup.cs +++ b/QuantConnect.CoinAPI.Tests/TestSetup.cs @@ -19,7 +19,7 @@ using QuantConnect.Logging; using QuantConnect.Configuration; -namespace QuantConnect.CoinAPI.Tests +namespace QuantConnect.DataSource.CoinAPI.Tests { [SetUpFixture] public static class TestSetup diff --git a/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs b/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs index 5f9e336..bd6a08c 100644 --- a/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs +++ b/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs @@ -19,7 +19,7 @@ using QuantConnect.Securities; using QuantConnect.Data.Market; -namespace QuantConnect.CoinAPI +namespace QuantConnect.DataSource.CoinAPI { public class CoinAPIDataDownloader : IDataDownloader, IDisposable { diff --git a/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs b/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs index 6f067ff..72ab3f9 100644 --- a/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs +++ b/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs @@ -19,12 +19,12 @@ using QuantConnect.Data; using QuantConnect.Logging; using QuantConnect.Data.Market; -using QuantConnect.CoinAPI.Messages; +using QuantConnect.DataSource.CoinAPI.Messages; using QuantConnect.Lean.Engine.DataFeeds; using HistoryRequest = QuantConnect.Data.HistoryRequest; -using QuantConnect.CoinAPI.Models; +using QuantConnect.DataSource.CoinAPI.Models; -namespace QuantConnect.CoinAPI +namespace QuantConnect.DataSource.CoinAPI { public partial class CoinApiDataQueueHandler { diff --git a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs index a2e612b..3dd9174 100644 --- a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs +++ b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs @@ -33,7 +33,7 @@ using CoinAPI.WebSocket.V1.DataModels; using QuantConnect.Lean.Engine.HistoricalData; -namespace QuantConnect.CoinAPI +namespace QuantConnect.DataSource.CoinAPI { /// /// An implementation of for CoinAPI diff --git a/QuantConnect.CoinAPI/CoinApiProduct.cs b/QuantConnect.CoinAPI/CoinApiProduct.cs index 1055342..a47bd18 100644 --- a/QuantConnect.CoinAPI/CoinApiProduct.cs +++ b/QuantConnect.CoinAPI/CoinApiProduct.cs @@ -13,7 +13,7 @@ * limitations under the License. */ -namespace QuantConnect.CoinAPI +namespace QuantConnect.DataSource.CoinAPI { /// /// Coin API's available tariff plans (or products). diff --git a/QuantConnect.CoinAPI/CoinApiSymbol.cs b/QuantConnect.CoinAPI/CoinApiSymbol.cs index 0f56083..3a08a43 100644 --- a/QuantConnect.CoinAPI/CoinApiSymbol.cs +++ b/QuantConnect.CoinAPI/CoinApiSymbol.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.CoinAPI +namespace QuantConnect.DataSource.CoinAPI { public class CoinApiSymbol { diff --git a/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs b/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs index 8a3d94e..2644116 100644 --- a/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs +++ b/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs @@ -18,9 +18,9 @@ using QuantConnect.Securities; using QuantConnect.Brokerages; using QuantConnect.Configuration; -using QuantConnect.CoinAPI.Models; +using QuantConnect.DataSource.CoinAPI.Models; -namespace QuantConnect.CoinAPI +namespace QuantConnect.DataSource.CoinAPI { /// /// Provides the mapping between Lean symbols and CoinAPI symbols. diff --git a/QuantConnect.CoinAPI/Messages/BaseMessage.cs b/QuantConnect.CoinAPI/Messages/BaseMessage.cs index a168252..0b47cf3 100644 --- a/QuantConnect.CoinAPI/Messages/BaseMessage.cs +++ b/QuantConnect.CoinAPI/Messages/BaseMessage.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.CoinAPI.Messages +namespace QuantConnect.DataSource.CoinAPI.Messages { public class BaseMessage { diff --git a/QuantConnect.CoinAPI/Messages/ErrorMessage.cs b/QuantConnect.CoinAPI/Messages/ErrorMessage.cs index 924c972..b518f8c 100644 --- a/QuantConnect.CoinAPI/Messages/ErrorMessage.cs +++ b/QuantConnect.CoinAPI/Messages/ErrorMessage.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.CoinAPI.Messages +namespace QuantConnect.DataSource.CoinAPI.Messages { public class ErrorMessage : BaseMessage { diff --git a/QuantConnect.CoinAPI/Messages/HelloMessage.cs b/QuantConnect.CoinAPI/Messages/HelloMessage.cs index cd79290..6029a72 100644 --- a/QuantConnect.CoinAPI/Messages/HelloMessage.cs +++ b/QuantConnect.CoinAPI/Messages/HelloMessage.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.CoinAPI.Messages +namespace QuantConnect.DataSource.CoinAPI.Messages { public class HelloMessage { diff --git a/QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs b/QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs index ce5a83a..1196203 100644 --- a/QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs +++ b/QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.CoinAPI.Messages +namespace QuantConnect.DataSource.CoinAPI.Messages { public class HistoricalDataMessage { diff --git a/QuantConnect.CoinAPI/Messages/QuoteMessage.cs b/QuantConnect.CoinAPI/Messages/QuoteMessage.cs index 59c273c..7607387 100644 --- a/QuantConnect.CoinAPI/Messages/QuoteMessage.cs +++ b/QuantConnect.CoinAPI/Messages/QuoteMessage.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.CoinAPI.Messages +namespace QuantConnect.DataSource.CoinAPI.Messages { public class QuoteMessage : BaseMessage { diff --git a/QuantConnect.CoinAPI/Messages/TradeMessage.cs b/QuantConnect.CoinAPI/Messages/TradeMessage.cs index cfa6b31..c810b78 100644 --- a/QuantConnect.CoinAPI/Messages/TradeMessage.cs +++ b/QuantConnect.CoinAPI/Messages/TradeMessage.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.CoinAPI.Messages +namespace QuantConnect.DataSource.CoinAPI.Messages { public class TradeMessage : BaseMessage { diff --git a/QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs b/QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs index 56133b1..72a542d 100644 --- a/QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs +++ b/QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.CoinAPI.Models +namespace QuantConnect.DataSource.CoinAPI.Models { public readonly struct CoinApiErrorResponse { diff --git a/QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj b/QuantConnect.CoinAPI/QuantConnect.DataSource.CoinAPI.csproj similarity index 84% rename from QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj rename to QuantConnect.CoinAPI/QuantConnect.DataSource.CoinAPI.csproj index 3096705..4916280 100644 --- a/QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj +++ b/QuantConnect.CoinAPI/QuantConnect.DataSource.CoinAPI.csproj @@ -4,10 +4,10 @@ Release AnyCPU net6.0 - QuantConnect.CoinAPI - QuantConnect.CoinAPI - QuantConnect.CoinAPI - QuantConnect.CoinAPI + QuantConnect.DataSource.CoinAPI + QuantConnect.DataSource.CoinAPI + QuantConnect.DataSource.CoinAPI + QuantConnect.DataSource.CoinAPI Library bin\$(Configuration) false From 8628fef311c988fa082c8bb2d51fa6e82094d38d Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 21 Feb 2024 17:21:17 +0200 Subject: [PATCH 02/14] rename: DataQueueHandler -> DataProvider --- .../CoinAPIHistoryProviderTests.cs | 2 +- .../CoinApiAdditionalTests.cs | 2 +- .../CoinApiDataQueueHandlerTest.cs | 2 +- QuantConnect.CoinAPI/CoinAPIDataDownloader.cs | 4 ++-- ...ryProvider.cs => CoinApiDataHistoryProvider.cs} | 12 ++++++------ ...iDataQueueHandler.cs => CoinApiDataProvider.cs} | 14 +++++++------- 6 files changed, 18 insertions(+), 18 deletions(-) rename QuantConnect.CoinAPI/{CoinApi.HistoryProvider.cs => CoinApiDataHistoryProvider.cs} (90%) rename QuantConnect.CoinAPI/{CoinApiDataQueueHandler.cs => CoinApiDataProvider.cs} (96%) diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs index c65b6a5..c7813dd 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs @@ -114,7 +114,7 @@ public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, i } } - public class CoinApiDataQueueHandlerMock : CoinApiDataQueueHandler + public class CoinApiDataQueueHandlerMock : CoinApiDataProvider { public new void SetUpHistDataLimit(int limit) { diff --git a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs index c25bc37..f8d1dda 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs @@ -28,7 +28,7 @@ public void ThrowsOnFailedAuthentication() Assert.Throws(() => { - using var _coinApiDataQueueHandler = new CoinApiDataQueueHandler(); + using var _coinApiDataQueueHandler = new CoinApiDataProvider(); }); // reset api key diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs index 193c281..2b41be5 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -26,7 +26,7 @@ namespace QuantConnect.DataSource.CoinAPI.Tests [TestFixture] public class CoinApiDataQueueHandlerTest { - private CoinApiDataQueueHandler _coinApiDataQueueHandler; + private CoinApiDataProvider _coinApiDataQueueHandler; private CancellationTokenSource _cancellationTokenSource; [SetUp] diff --git a/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs b/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs index bd6a08c..a32b141 100644 --- a/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs +++ b/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs @@ -23,13 +23,13 @@ namespace QuantConnect.DataSource.CoinAPI { public class CoinAPIDataDownloader : IDataDownloader, IDisposable { - private readonly CoinApiDataQueueHandler _historyProvider; + private readonly CoinApiDataProvider _historyProvider; private readonly MarketHoursDatabase _marketHoursDatabase; public CoinAPIDataDownloader() { - _historyProvider = new CoinApiDataQueueHandler(); + _historyProvider = new CoinApiDataProvider(); _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); } diff --git a/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs b/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs similarity index 90% rename from QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs rename to QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs index 72ab3f9..c15acd6 100644 --- a/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs +++ b/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs @@ -26,7 +26,7 @@ namespace QuantConnect.DataSource.CoinAPI { - public partial class CoinApiDataQueueHandler + public partial class CoinApiDataProvider { private readonly RestClient restClient = new RestClient(); @@ -58,13 +58,13 @@ public IEnumerable GetHistory(HistoryRequest historyRequest) { if (historyRequest.Symbol.SecurityType != SecurityType.Crypto && historyRequest.Symbol.SecurityType != SecurityType.CryptoFuture) { - Log.Error($"CoinApiDataQueueHandler.GetHistory(): Invalid security type {historyRequest.Symbol.SecurityType}"); + Log.Error($"CoinApiDataProvider.GetHistory(): Invalid security type {historyRequest.Symbol.SecurityType}"); yield break; } if (historyRequest.Resolution == Resolution.Tick) { - Log.Error($"CoinApiDataQueueHandler.GetHistory(): No historical ticks, only OHLCV timeseries"); + Log.Error($"CoinApiDataProvider.GetHistory(): No historical ticks, only OHLCV timeseries"); yield break; } @@ -72,7 +72,7 @@ public IEnumerable GetHistory(HistoryRequest historyRequest) { if (!_invalidHistoryDataTypeWarningFired) { - Log.Error("CoinApiDataQueueHandler.GetHistory(): No historical QuoteBars , only TradeBars"); + Log.Error("CoinApiDataProvider.GetHistory(): No historical QuoteBars , only TradeBars"); _invalidHistoryDataTypeWarningFired = true; } yield break; @@ -128,7 +128,7 @@ public IEnumerable GetHistory(HistoryRequest historyRequest) // Can be no historical data for a short period interval if (!coinApiHistoryBars.Any()) { - Log.Error($"CoinApiDataQueueHandler.GetHistory(): API returned no data for the requested period [{coinApiStartTime} - {coinApiEndTime}] for symbol [{historyRequest.Symbol}]"); + Log.Error($"CoinApiDataProvider.GetHistory(): API returned no data for the requested period [{coinApiStartTime} - {coinApiEndTime}] for symbol [{historyRequest.Symbol}]"); continue; } @@ -150,7 +150,7 @@ private void TraceRestUsage(IRestResponse response) var used = GetHttpHeaderValue(response, "x-ratelimit-used"); var remaining = GetHttpHeaderValue(response, "x-ratelimit-remaining"); - Log.Trace($"CoinApiDataQueueHandler.TraceRestUsage(): Used {used}, Remaining {remaining}, Total {total}"); + Log.Trace($"CoinApiDataProvider.TraceRestUsage(): Used {used}, Remaining {remaining}, Total {total}"); } private string GetHttpHeaderValue(IRestResponse response, string propertyName) diff --git a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs b/QuantConnect.CoinAPI/CoinApiDataProvider.cs similarity index 96% rename from QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs rename to QuantConnect.CoinAPI/CoinApiDataProvider.cs index 3dd9174..4e0988c 100644 --- a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs +++ b/QuantConnect.CoinAPI/CoinApiDataProvider.cs @@ -38,7 +38,7 @@ namespace QuantConnect.DataSource.CoinAPI /// /// An implementation of for CoinAPI /// - public partial class CoinApiDataQueueHandler : SynchronizingHistoryProvider, IDataQueueHandler + public partial class CoinApiDataProvider : SynchronizingHistoryProvider, IDataQueueHandler { protected int HistoricalDataPerRequestLimit = 10000; private static readonly Dictionary _ResolutionToCoinApiPeriodMappings = new Dictionary @@ -69,9 +69,9 @@ public partial class CoinApiDataQueueHandler : SynchronizingHistoryProvider, IDa private readonly ConcurrentDictionary _previousQuotes = new ConcurrentDictionary(); /// - /// Initializes a new instance of the class + /// Initializes a new instance of the class /// - public CoinApiDataQueueHandler() + public CoinApiDataProvider() { _dataAggregator = Composer.Instance.GetPart(); if (_dataAggregator == null) @@ -84,7 +84,7 @@ public CoinApiDataQueueHandler() ? new[] { "trade" } : new[] { "trade", "quote" }; - Log.Trace($"{nameof(CoinApiDataQueueHandler)}: using plan '{product}'. Available data types: '{string.Join(",", _streamingDataType)}'"); + Log.Trace($"{nameof(CoinApiDataProvider)}: using plan '{product}'. Available data types: '{string.Join(",", _streamingDataType)}'"); ValidateSubscription(); @@ -179,7 +179,7 @@ public void Dispose() /// List of LEAN markets (exchanges) to subscribe public void SubscribeMarkets(List markets) { - Log.Trace($"CoinApiDataQueueHandler.SubscribeMarkets(): {string.Join(",", markets)}"); + Log.Trace($"CoinApiDataProvider.SubscribeMarkets(): {string.Join(",", markets)}"); // we add '_' to be more precise, for example requesting 'BINANCE' doesn't match 'BINANCEUS' SendHelloMessage(markets.Select(x => string.Concat(_symbolMapper.GetExchangeId(x.ToLowerInvariant()), "_"))); @@ -263,7 +263,7 @@ private static bool CanSubscribe(Symbol symbol) /// The list of symbols to subscribe private void SubscribeSymbols(List symbolsToSubscribe) { - Log.Trace($"CoinApiDataQueueHandler.SubscribeSymbols(): {string.Join(",", symbolsToSubscribe)}"); + Log.Trace($"CoinApiDataProvider.SubscribeSymbols(): {string.Join(",", symbolsToSubscribe)}"); // subscribe to symbols using exact match SendHelloMessage(symbolsToSubscribe.Select(x => @@ -518,7 +518,7 @@ private static void ValidateSubscription() } catch (Exception e) { - Log.Error($"{nameof(CoinApiDataQueueHandler)}.{nameof(ValidateSubscription)}: Failed during validation, shutting down. Error : {e.Message}"); + Log.Error($"{nameof(CoinApiDataProvider)}.{nameof(ValidateSubscription)}: Failed during validation, shutting down. Error : {e.Message}"); throw; } } From ef48d8c25640bb7314372ddadf85401cddfc1a20 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 21 Feb 2024 17:22:45 +0200 Subject: [PATCH 03/14] feat: Config -> Globals in ValidateSubscription --- QuantConnect.CoinAPI/CoinApiDataProvider.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/QuantConnect.CoinAPI/CoinApiDataProvider.cs b/QuantConnect.CoinAPI/CoinApiDataProvider.cs index 4e0988c..7d408e0 100644 --- a/QuantConnect.CoinAPI/CoinApiDataProvider.cs +++ b/QuantConnect.CoinAPI/CoinApiDataProvider.cs @@ -401,9 +401,9 @@ private static void ValidateSubscription() try { const int productId = 335; - var userId = Config.GetInt("job-user-id"); - var token = Config.Get("api-access-token"); - var organizationId = Config.Get("job-organization-id", null); + var userId = Globals.UserId; + var token = Globals.UserToken; + var organizationId = Globals.OrganizationID; // Verify we can authenticate with this user and token var api = new ApiConnection(userId, token); if (!api.Connected) From 74ae990dad581ca7a7588e422a739ad763a0b71e Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 21 Feb 2024 20:33:00 +0200 Subject: [PATCH 04/14] feat: SetJob fix: IsConnected Property feat: handle of null warnings --- .../CoinApiAdditionalTests.cs | 27 ++++ QuantConnect.CoinAPI/CoinApiDataProvider.cs | 135 +++++++++++++----- 2 files changed, 130 insertions(+), 32 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs index f8d1dda..bcf426f 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs @@ -14,6 +14,7 @@ */ using NUnit.Framework; +using QuantConnect.Packets; using QuantConnect.Configuration; namespace QuantConnect.DataSource.CoinAPI.Tests @@ -34,5 +35,31 @@ public void ThrowsOnFailedAuthentication() // reset api key TestSetup.GlobalSetup(); } + + [Test] + public void CanInitializeUsingJobPacket() + { + var apiKey = Config.Get("coinapi-api-key"); + Config.Set("coinapi-api-key", ""); + + var job = new LiveNodePacket + { + BrokerageData = new Dictionary() { + { "coinapi-api-key", "InvalidApiKeyThatWontBeUsed" }, + { "coinapi-product", "Startup" } + } + }; + + using var iexDataProvider = new CoinApiDataProvider(); + + // Throw because CoinApiSymbolMapper makes request to API (we have invalid api key in LiveNodePacket) + Assert.Throws(() => + { + iexDataProvider.SetJob(job); + }); + + // revert Config of ApiKey for another tests + Config.Set("coinapi-api-key", apiKey); + } } } diff --git a/QuantConnect.CoinAPI/CoinApiDataProvider.cs b/QuantConnect.CoinAPI/CoinApiDataProvider.cs index 7d408e0..eb59c37 100644 --- a/QuantConnect.CoinAPI/CoinApiDataProvider.cs +++ b/QuantConnect.CoinAPI/CoinApiDataProvider.cs @@ -49,14 +49,14 @@ public partial class CoinApiDataProvider : SynchronizingHistoryProvider, IDataQu { Resolution.Daily, "1DAY" }, }; - private readonly string _apiKey = Config.Get("coinapi-api-key"); - private readonly string[] _streamingDataType; - private readonly CoinApiWsClient _client; + private string? _apiKey; + private string[]? _streamingDataType; + private CoinApiWsClient? _client; private readonly object _locker = new object(); private ConcurrentDictionary _symbolCache = new ConcurrentDictionary(); - private readonly CoinApiSymbolMapper _symbolMapper = new CoinApiSymbolMapper(); - private readonly IDataAggregator _dataAggregator; - private readonly EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager; + private CoinApiSymbolMapper? _symbolMapper; + private IDataAggregator? _dataAggregator; + private EventBasedDataQueueHandlerSubscriptionManager? _subscriptionManager; private readonly TimeSpan _subscribeDelay = TimeSpan.FromMilliseconds(250); private readonly object _lockerSubscriptions = new object(); @@ -68,33 +68,29 @@ public partial class CoinApiDataProvider : SynchronizingHistoryProvider, IDataQu private readonly ConcurrentDictionary _previousQuotes = new ConcurrentDictionary(); + /// + /// + /// + private bool _initialized; + /// /// Initializes a new instance of the class /// public CoinApiDataProvider() { - _dataAggregator = Composer.Instance.GetPart(); - if (_dataAggregator == null) + if (!Config.TryGetValue("coinapi-api-key", out var configApiKey) || string.IsNullOrEmpty(configApiKey)) { - _dataAggregator = - Composer.Instance.GetExportedValueByTypeName(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); + // If the API key is not provided, we can't do anything. + // The handler might going to be initialized using a node packet job. + return; } - var product = Config.GetValue("coinapi-product"); - _streamingDataType = product < CoinApiProduct.Streamer - ? new[] { "trade" } - : new[] { "trade", "quote" }; - Log.Trace($"{nameof(CoinApiDataProvider)}: using plan '{product}'. Available data types: '{string.Join(",", _streamingDataType)}'"); - - ValidateSubscription(); + if (!Config.TryGetValue("coinapi-product", out var product) || string.IsNullOrEmpty(product)) + { + product = "Free"; + } - _client = new CoinApiWsClient(); - _client.TradeEvent += OnTrade; - _client.QuoteEvent += OnQuote; - _client.Error += OnError; - _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager(); - _subscriptionManager.SubscribeImpl += (s, t) => Subscribe(s); - _subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s); + Initialize(configApiKey, product); } /// @@ -110,6 +106,11 @@ public IEnumerator Subscribe(SubscriptionDataConfig dataConfig, EventH return null; } + if (_dataAggregator == null || _subscriptionManager == null) + { + throw new InvalidOperationException($"{nameof(CoinApiDataProvider)}.{nameof(Subscribe)}: {nameof(_dataAggregator)} or {nameof(_subscriptionManager)} is not initialized."); + } + var enumerator = _dataAggregator.Add(dataConfig, newDataAvailableHandler); _subscriptionManager.Subscribe(dataConfig); @@ -122,6 +123,63 @@ public IEnumerator Subscribe(SubscriptionDataConfig dataConfig, EventH /// Job we're subscribing for public void SetJob(LiveNodePacket job) { + if (_initialized) + { + return; + } + + if (!job.BrokerageData.TryGetValue("coinapi-api-key", out var apiKey) || string.IsNullOrEmpty(apiKey)) + { + throw new ArgumentException("Invalid or missing Coin API key. Please ensure that the API key is set and not empty."); + } + + if (!job.BrokerageData.TryGetValue("coinapi-product", out var product) || string.IsNullOrEmpty(product)) + { + product = "Free"; + } + + Initialize(apiKey, product); + } + + /// + /// + /// + /// + /// + /// + private void Initialize(string apiKey, string product) + { + // ValidateSubscription(); + + _apiKey = apiKey; + + if (!Enum.TryParse(product, true, out var parsedProduct) || !Enum.IsDefined(typeof(CoinApiProduct), parsedProduct)) + { + throw new ArgumentException($"An error occurred while parsing the price plan '{product}'. Please ensure that the provided price plan is valid and supported by the system."); + } + + _dataAggregator = Composer.Instance.GetPart(); + if (_dataAggregator == null) + { + _dataAggregator = + Composer.Instance.GetExportedValueByTypeName(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); + } + + _streamingDataType = parsedProduct < CoinApiProduct.Streamer + ? new[] { "trade" } + : new[] { "trade", "quote" }; + + Log.Trace($"{nameof(CoinApiDataProvider)}: using plan '{product}'. Available data types: '{string.Join(",", _streamingDataType)}'"); + + _symbolMapper = new CoinApiSymbolMapper(); + _client = new CoinApiWsClient(); + _client.TradeEvent += OnTrade; + _client.QuoteEvent += OnQuote; + _client.Error += OnError; + _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager(); + _subscriptionManager.SubscribeImpl += (s, t) => Subscribe(s); + _subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s); + _initialized = true; } /// @@ -140,8 +198,8 @@ private bool Subscribe(IEnumerable symbols) /// Subscription config to be removed public void Unsubscribe(SubscriptionDataConfig dataConfig) { - _subscriptionManager.Unsubscribe(dataConfig); - _dataAggregator.Remove(dataConfig); + _subscriptionManager?.Unsubscribe(dataConfig); + _dataAggregator?.Remove(dataConfig); } @@ -159,17 +217,20 @@ private bool Unsubscribe(IEnumerable symbols) /// Returns whether the data provider is connected /// /// true if the data provider is connected - public bool IsConnected => true; + public bool IsConnected { get; private set; } /// /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. /// public void Dispose() { - _client.TradeEvent -= OnTrade; - _client.QuoteEvent -= OnQuote; - _client.Error -= OnError; - _client.Dispose(); + if (_client != null) + { + _client.TradeEvent -= OnTrade; + _client.QuoteEvent -= OnQuote; + _client.Error -= OnError; + _client.Dispose(); + } _dataAggregator.DisposeSafely(); } @@ -292,7 +353,7 @@ private void SendHelloMessage(IEnumerable subscribeFilter) list.Add("$no_symbol_requested$"); } - _client.SendHelloMessage(new Hello + _client?.SendHelloMessage(new Hello { apikey = Guid.Parse(_apiKey), heartbeat = true, @@ -300,6 +361,15 @@ private void SendHelloMessage(IEnumerable subscribeFilter) subscribe_filter_symbol_id = list.ToArray() }); + if (!IsConnected && !_client.ConnectedEvent.WaitOne(TimeSpan.FromSeconds(30))) + { + throw new Exception("Not connected..."); + } + else + { + IsConnected = true; + } + _nextHelloMessageUtcTime = DateTime.UtcNow.Add(_minimumTimeBetweenHelloMessages); } @@ -381,6 +451,7 @@ private Symbol GetSymbolUsingCache(string ticker) private void OnError(object? sender, Exception e) { + IsConnected = false; Log.Error(e); } From 98b04b5405bf4251635bf08c80a3ec4afb5b5936 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 21 Feb 2024 20:33:54 +0200 Subject: [PATCH 05/14] feat: reference on Lean\Tests + init Globals --- QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs | 2 ++ .../QuantConnect.DataSource.CoinAPI.Tests.csproj | 1 + 2 files changed, 3 insertions(+) diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs index f70140c..8b27425 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs @@ -15,6 +15,7 @@ using NUnit.Framework; using QuantConnect.Util; +using QuantConnect.Tests; using QuantConnect.Logging; namespace QuantConnect.DataSource.CoinAPI.Tests @@ -92,6 +93,7 @@ private static IEnumerable HistoricalInvalidDataThrowExceptionTest { get { + TestGlobals.Initialize(); yield return new TestCaseData(Symbol.Create("BTCBTC", SecurityType.Crypto, Market.Binance)) .SetDescription($"Wrong Symbol - 'BTCBTC'"); yield return new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Equity, Market.Binance)) diff --git a/QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj b/QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj index e689f25..944f39d 100644 --- a/QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj +++ b/QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj @@ -31,6 +31,7 @@ + From 8eb41d4d7ffcf88171439f7a5f00cad63b67fcc7 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 21 Feb 2024 22:39:54 +0200 Subject: [PATCH 06/14] refactor: use CanSubscribe for Symbol in GetHistory() --- QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs b/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs index c15acd6..123623c 100644 --- a/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs +++ b/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs @@ -56,7 +56,7 @@ public override IEnumerable GetHistory(IEnumerable reques public IEnumerable GetHistory(HistoryRequest historyRequest) { - if (historyRequest.Symbol.SecurityType != SecurityType.Crypto && historyRequest.Symbol.SecurityType != SecurityType.CryptoFuture) + if (!CanSubscribe(historyRequest.Symbol)) { Log.Error($"CoinApiDataProvider.GetHistory(): Invalid security type {historyRequest.Symbol.SecurityType}"); yield break; From bf94ae2528a1e1cb7507192ca324dc0d762501c6 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 21 Feb 2024 23:52:30 +0200 Subject: [PATCH 07/14] refactor: missed Lean in namespaces --- DataProcessing/CoinApiDataConverter.cs | 2 +- .../CoinAPIDataDownloaderTests.cs | 2 +- .../CoinAPIHistoryProviderTests.cs | 2 +- .../CoinAPISymbolMapperTests.cs | 2 +- .../CoinApiAdditionalTests.cs | 2 +- .../CoinApiDataQueueHandlerTest.cs | 13 +++++++------ QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs | 2 +- .../QuantConnect.DataSource.CoinAPI.Tests.csproj | 8 ++++---- QuantConnect.CoinAPI.Tests/TestSetup.cs | 2 +- QuantConnect.CoinAPI/CoinAPIDataDownloader.cs | 2 +- QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs | 6 +++--- QuantConnect.CoinAPI/CoinApiDataProvider.cs | 2 +- QuantConnect.CoinAPI/CoinApiProduct.cs | 2 +- QuantConnect.CoinAPI/CoinApiSymbol.cs | 2 +- QuantConnect.CoinAPI/CoinApiSymbolMapper.cs | 4 ++-- QuantConnect.CoinAPI/Messages/BaseMessage.cs | 2 +- QuantConnect.CoinAPI/Messages/ErrorMessage.cs | 2 +- QuantConnect.CoinAPI/Messages/HelloMessage.cs | 2 +- .../Messages/HistoricalDataMessage.cs | 2 +- QuantConnect.CoinAPI/Messages/QuoteMessage.cs | 2 +- QuantConnect.CoinAPI/Messages/TradeMessage.cs | 2 +- QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs | 2 +- .../QuantConnect.DataSource.CoinAPI.csproj | 8 ++++---- 23 files changed, 38 insertions(+), 37 deletions(-) diff --git a/DataProcessing/CoinApiDataConverter.cs b/DataProcessing/CoinApiDataConverter.cs index ef1698c..12b61ff 100644 --- a/DataProcessing/CoinApiDataConverter.cs +++ b/DataProcessing/CoinApiDataConverter.cs @@ -19,7 +19,7 @@ using System.Diagnostics; using QuantConnect.Logging; using QuantConnect.ToolBox; -using QuantConnect.DataSource.CoinAPI; +using QuantConnect.Lean.DataSource.CoinAPI; namespace QuantConnect.DataProcessing { diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs index 8b27425..6252410 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs @@ -18,7 +18,7 @@ using QuantConnect.Tests; using QuantConnect.Logging; -namespace QuantConnect.DataSource.CoinAPI.Tests +namespace QuantConnect.Lean.DataSource.CoinAPI.Tests { [TestFixture] public class CoinAPIDataDownloaderTests diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs index c7813dd..2f9c08a 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs @@ -20,7 +20,7 @@ using QuantConnect.Data.Market; using QuantConnect.Securities; -namespace QuantConnect.DataSource.CoinAPI.Tests +namespace QuantConnect.Lean.DataSource.CoinAPI.Tests { [TestFixture] public class CoinAPIHistoryProviderTests diff --git a/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs index ccb1135..1f2e9cd 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs @@ -16,7 +16,7 @@ using NUnit.Framework; -namespace QuantConnect.DataSource.CoinAPI.Tests +namespace QuantConnect.Lean.DataSource.CoinAPI.Tests { [TestFixture] public class CoinAPISymbolMapperTests diff --git a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs index bcf426f..23b76a1 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs @@ -17,7 +17,7 @@ using QuantConnect.Packets; using QuantConnect.Configuration; -namespace QuantConnect.DataSource.CoinAPI.Tests +namespace QuantConnect.Lean.DataSource.CoinAPI.Tests { [TestFixture] public class CoinApiAdditionalTests diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs index 2b41be5..b12855d 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -21,7 +21,7 @@ using QuantConnect.Data.Market; using System.Collections.Concurrent; -namespace QuantConnect.DataSource.CoinAPI.Tests +namespace QuantConnect.Lean.DataSource.CoinAPI.Tests { [TestFixture] public class CoinApiDataQueueHandlerTest @@ -64,14 +64,15 @@ public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest)}: {tick}"); tradeBars.Add(tick); - if (tradeBars.Count > 5) - { - resetEvent.Set(); - } + //if (tradeBars.Count > 5) + //{ + // resetEvent.Set(); + //} }, () => _cancellationTokenSource.Cancel()); - Assert.IsTrue(resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token)); + resetEvent.WaitOne(-1, _cancellationTokenSource.Token); + // Assert.IsTrue(resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token)); _coinApiDataQueueHandler.Unsubscribe(dataConfig); diff --git a/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs index 3ed1ab5..4d0ad22 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs @@ -19,7 +19,7 @@ using QuantConnect.Logging; using QuantConnect.Data.Market; -namespace QuantConnect.DataSource.CoinAPI.Tests +namespace QuantConnect.Lean.DataSource.CoinAPI.Tests { public static class CoinApiTestHelper { diff --git a/QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj b/QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj index 944f39d..06c29fd 100644 --- a/QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj +++ b/QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj @@ -5,10 +5,10 @@ AnyCPU net6.0 bin\$(Configuration)\ - QuantConnect.DataSource.CoinAPI.Tests - QuantConnect.DataSource.CoinAPI.Tests - QuantConnect.DataSource.CoinAPI.Tests - QuantConnect.DataSource.CoinAPI.Tests + QuantConnect.Lean.DataSource.CoinAPI.Tests + QuantConnect.Lean.DataSource.CoinAPI.Tests + QuantConnect.Lean.DataSource.CoinAPI.Tests + QuantConnect.Lean.DataSource.CoinAPI.Tests false enable enable diff --git a/QuantConnect.CoinAPI.Tests/TestSetup.cs b/QuantConnect.CoinAPI.Tests/TestSetup.cs index a0ebc0e..b227107 100644 --- a/QuantConnect.CoinAPI.Tests/TestSetup.cs +++ b/QuantConnect.CoinAPI.Tests/TestSetup.cs @@ -19,7 +19,7 @@ using QuantConnect.Logging; using QuantConnect.Configuration; -namespace QuantConnect.DataSource.CoinAPI.Tests +namespace QuantConnect.Lean.DataSource.CoinAPI.Tests { [SetUpFixture] public static class TestSetup diff --git a/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs b/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs index a32b141..dd759ff 100644 --- a/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs +++ b/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs @@ -19,7 +19,7 @@ using QuantConnect.Securities; using QuantConnect.Data.Market; -namespace QuantConnect.DataSource.CoinAPI +namespace QuantConnect.Lean.DataSource.CoinAPI { public class CoinAPIDataDownloader : IDataDownloader, IDisposable { diff --git a/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs b/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs index 123623c..f2a14e1 100644 --- a/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs +++ b/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs @@ -19,12 +19,12 @@ using QuantConnect.Data; using QuantConnect.Logging; using QuantConnect.Data.Market; -using QuantConnect.DataSource.CoinAPI.Messages; +using QuantConnect.Lean.DataSource.CoinAPI.Messages; using QuantConnect.Lean.Engine.DataFeeds; using HistoryRequest = QuantConnect.Data.HistoryRequest; -using QuantConnect.DataSource.CoinAPI.Models; +using QuantConnect.Lean.DataSource.CoinAPI.Models; -namespace QuantConnect.DataSource.CoinAPI +namespace QuantConnect.Lean.DataSource.CoinAPI { public partial class CoinApiDataProvider { diff --git a/QuantConnect.CoinAPI/CoinApiDataProvider.cs b/QuantConnect.CoinAPI/CoinApiDataProvider.cs index eb59c37..f631cec 100644 --- a/QuantConnect.CoinAPI/CoinApiDataProvider.cs +++ b/QuantConnect.CoinAPI/CoinApiDataProvider.cs @@ -33,7 +33,7 @@ using CoinAPI.WebSocket.V1.DataModels; using QuantConnect.Lean.Engine.HistoricalData; -namespace QuantConnect.DataSource.CoinAPI +namespace QuantConnect.Lean.DataSource.CoinAPI { /// /// An implementation of for CoinAPI diff --git a/QuantConnect.CoinAPI/CoinApiProduct.cs b/QuantConnect.CoinAPI/CoinApiProduct.cs index a47bd18..4316f74 100644 --- a/QuantConnect.CoinAPI/CoinApiProduct.cs +++ b/QuantConnect.CoinAPI/CoinApiProduct.cs @@ -13,7 +13,7 @@ * limitations under the License. */ -namespace QuantConnect.DataSource.CoinAPI +namespace QuantConnect.Lean.DataSource.CoinAPI { /// /// Coin API's available tariff plans (or products). diff --git a/QuantConnect.CoinAPI/CoinApiSymbol.cs b/QuantConnect.CoinAPI/CoinApiSymbol.cs index 3a08a43..02b8017 100644 --- a/QuantConnect.CoinAPI/CoinApiSymbol.cs +++ b/QuantConnect.CoinAPI/CoinApiSymbol.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.DataSource.CoinAPI +namespace QuantConnect.Lean.DataSource.CoinAPI { public class CoinApiSymbol { diff --git a/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs b/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs index 2644116..40b60e1 100644 --- a/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs +++ b/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs @@ -18,9 +18,9 @@ using QuantConnect.Securities; using QuantConnect.Brokerages; using QuantConnect.Configuration; -using QuantConnect.DataSource.CoinAPI.Models; +using QuantConnect.Lean.DataSource.CoinAPI.Models; -namespace QuantConnect.DataSource.CoinAPI +namespace QuantConnect.Lean.DataSource.CoinAPI { /// /// Provides the mapping between Lean symbols and CoinAPI symbols. diff --git a/QuantConnect.CoinAPI/Messages/BaseMessage.cs b/QuantConnect.CoinAPI/Messages/BaseMessage.cs index 0b47cf3..bcbd138 100644 --- a/QuantConnect.CoinAPI/Messages/BaseMessage.cs +++ b/QuantConnect.CoinAPI/Messages/BaseMessage.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.DataSource.CoinAPI.Messages +namespace QuantConnect.Lean.DataSource.CoinAPI.Messages { public class BaseMessage { diff --git a/QuantConnect.CoinAPI/Messages/ErrorMessage.cs b/QuantConnect.CoinAPI/Messages/ErrorMessage.cs index b518f8c..a62dcfc 100644 --- a/QuantConnect.CoinAPI/Messages/ErrorMessage.cs +++ b/QuantConnect.CoinAPI/Messages/ErrorMessage.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.DataSource.CoinAPI.Messages +namespace QuantConnect.Lean.DataSource.CoinAPI.Messages { public class ErrorMessage : BaseMessage { diff --git a/QuantConnect.CoinAPI/Messages/HelloMessage.cs b/QuantConnect.CoinAPI/Messages/HelloMessage.cs index 6029a72..c69d554 100644 --- a/QuantConnect.CoinAPI/Messages/HelloMessage.cs +++ b/QuantConnect.CoinAPI/Messages/HelloMessage.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.DataSource.CoinAPI.Messages +namespace QuantConnect.Lean.DataSource.CoinAPI.Messages { public class HelloMessage { diff --git a/QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs b/QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs index 1196203..f612705 100644 --- a/QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs +++ b/QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.DataSource.CoinAPI.Messages +namespace QuantConnect.Lean.DataSource.CoinAPI.Messages { public class HistoricalDataMessage { diff --git a/QuantConnect.CoinAPI/Messages/QuoteMessage.cs b/QuantConnect.CoinAPI/Messages/QuoteMessage.cs index 7607387..7350fe4 100644 --- a/QuantConnect.CoinAPI/Messages/QuoteMessage.cs +++ b/QuantConnect.CoinAPI/Messages/QuoteMessage.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.DataSource.CoinAPI.Messages +namespace QuantConnect.Lean.DataSource.CoinAPI.Messages { public class QuoteMessage : BaseMessage { diff --git a/QuantConnect.CoinAPI/Messages/TradeMessage.cs b/QuantConnect.CoinAPI/Messages/TradeMessage.cs index c810b78..2ef1ae3 100644 --- a/QuantConnect.CoinAPI/Messages/TradeMessage.cs +++ b/QuantConnect.CoinAPI/Messages/TradeMessage.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.DataSource.CoinAPI.Messages +namespace QuantConnect.Lean.DataSource.CoinAPI.Messages { public class TradeMessage : BaseMessage { diff --git a/QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs b/QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs index 72a542d..3e7553b 100644 --- a/QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs +++ b/QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs @@ -15,7 +15,7 @@ using Newtonsoft.Json; -namespace QuantConnect.DataSource.CoinAPI.Models +namespace QuantConnect.Lean.DataSource.CoinAPI.Models { public readonly struct CoinApiErrorResponse { diff --git a/QuantConnect.CoinAPI/QuantConnect.DataSource.CoinAPI.csproj b/QuantConnect.CoinAPI/QuantConnect.DataSource.CoinAPI.csproj index 4916280..410707d 100644 --- a/QuantConnect.CoinAPI/QuantConnect.DataSource.CoinAPI.csproj +++ b/QuantConnect.CoinAPI/QuantConnect.DataSource.CoinAPI.csproj @@ -4,10 +4,10 @@ Release AnyCPU net6.0 - QuantConnect.DataSource.CoinAPI - QuantConnect.DataSource.CoinAPI - QuantConnect.DataSource.CoinAPI - QuantConnect.DataSource.CoinAPI + QuantConnect.Lean.DataSource.CoinAPI + QuantConnect.Lean.DataSource.CoinAPI + QuantConnect.Lean.DataSource.CoinAPI + QuantConnect.Lean.DataSource.CoinAPI Library bin\$(Configuration) false From 5f3eaed359f9deb62302f0b1c0faf55b3f8790d7 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 22 Feb 2024 00:03:48 +0200 Subject: [PATCH 08/14] rename: QueueHandler -> DataProvider Tests revert: commented code --- ...lerTest.cs => CoinApiDataProviderTests.cs} | 21 +++++++++---------- .../CoinApiTestHelper.cs | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) rename QuantConnect.CoinAPI.Tests/{CoinApiDataQueueHandlerTest.cs => CoinApiDataProviderTests.cs} (88%) diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataProviderTests.cs similarity index 88% rename from QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs rename to QuantConnect.CoinAPI.Tests/CoinApiDataProviderTests.cs index b12855d..692520e 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataProviderTests.cs @@ -24,7 +24,7 @@ namespace QuantConnect.Lean.DataSource.CoinAPI.Tests { [TestFixture] - public class CoinApiDataQueueHandlerTest + public class CoinApiDataProviderTests { private CoinApiDataProvider _coinApiDataQueueHandler; private CancellationTokenSource _cancellationTokenSource; @@ -61,18 +61,17 @@ public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() _cancellationTokenSource.Token, tick => { - Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest)}: {tick}"); + Log.Debug($"{nameof(CoinApiDataProviderTests)}.{nameof(SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest)}: {tick}"); tradeBars.Add(tick); - //if (tradeBars.Count > 5) - //{ - // resetEvent.Set(); - //} + if (tradeBars.Count > 5) + { + resetEvent.Set(); + } }, () => _cancellationTokenSource.Cancel()); - resetEvent.WaitOne(-1, _cancellationTokenSource.Token); - // Assert.IsTrue(resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token)); + Assert.IsTrue(resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token)); _coinApiDataQueueHandler.Unsubscribe(dataConfig); @@ -112,7 +111,7 @@ public void SubscribeToBTCUSDSecondOnDifferentMarkets() _cancellationTokenSource.Token, tick => { - Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDSecondOnDifferentMarkets)}: {tick}"); + Log.Debug($"{nameof(CoinApiDataProviderTests)}.{nameof(SubscribeToBTCUSDSecondOnDifferentMarkets)}: {tick}"); symbolBaseData[tick.Symbol].Add(tick); }, () => @@ -166,7 +165,7 @@ public void SubscribeToBTCUSDTFutureSecondBinance() _cancellationTokenSource.Token, tick => { - Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)}: {tick}"); + Log.Debug($"{nameof(CoinApiDataProviderTests)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)}: {tick}"); tickData.Add(tick); if (tickData.Count > 5) @@ -191,7 +190,7 @@ public void SubscribeToBTCUSDTFutureSecondBinance() if (tickData.Count == 0) { - Assert.Fail($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)} is nothing returned. {symbol}|{resolution}|tickData = {tickData.Count}"); + Assert.Fail($"{nameof(CoinApiDataProviderTests)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)} is nothing returned. {symbol}|{resolution}|tickData = {tickData.Count}"); } CoinApiTestHelper.AssertSymbol(tickData.First().Symbol, symbol); diff --git a/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs index 4d0ad22..5d77a43 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs @@ -68,7 +68,7 @@ public static void AssertBaseData(List tradeBars, Resolution expectedR Assert.IsTrue(trade.Period.ToHigherResolutionEquivalent(true) == expectedResolution); break; default: - Assert.Fail($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(AssertBaseData)}: The tick type doesn't support"); + Assert.Fail($"{nameof(CoinApiDataProviderTests)}.{nameof(AssertBaseData)}: The tick type doesn't support"); break; } } From 980a9e6e873b697e8f0d605d470bb7ff9f96afeb Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 22 Feb 2024 15:18:10 +0200 Subject: [PATCH 09/14] fix: workflow dotnet test path --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bee2592..f6a6d06 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,4 +56,4 @@ jobs: run: dotnet build ./QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - name: Run QuantConnect.DataSource.CoinAPI.Tests - run: dotnet test ./QuantConnect.CoinAPI.Tests/bin/Release/QuantConnect.DataSource.CoinAPI.Tests.dll \ No newline at end of file + run: dotnet test ./QuantConnect.CoinAPI.Tests/bin/Release/QuantConnect.Lean.DataSource.CoinAPI.Tests.dll \ No newline at end of file From 0de9587f5434fcb6e6cd347d68bf10d9cf19b25d Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 22 Feb 2024 17:49:25 +0200 Subject: [PATCH 10/14] fix: testcase param forex instead of equity --- QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs index 6252410..34ec67e 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs @@ -96,8 +96,8 @@ private static IEnumerable HistoricalInvalidDataThrowExceptionTest TestGlobals.Initialize(); yield return new TestCaseData(Symbol.Create("BTCBTC", SecurityType.Crypto, Market.Binance)) .SetDescription($"Wrong Symbol - 'BTCBTC'"); - yield return new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Equity, Market.Binance)) - .SetDescription($"Wrong SecurityType - {SecurityType.Equity}"); + yield return new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Forex, Market.Binance)) + .SetDescription($"Wrong SecurityType - {SecurityType.Forex}"); } } From ff7e0f463659d6f423e50fa0acda4bdea65702cb Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 22 Feb 2024 21:47:04 +0200 Subject: [PATCH 11/14] refactor: return null in wrong request of GetHistory() --- .../CoinAPIDataDownloaderTests.cs | 9 ++-- .../CoinAPIHistoryProviderTests.cs | 22 +++++----- QuantConnect.CoinAPI/CoinAPIDataDownloader.cs | 21 ++++++++-- .../CoinApiDataHistoryProvider.cs | 42 ++++++++++++++----- 4 files changed, 64 insertions(+), 30 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs index 34ec67e..e028d34 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs @@ -53,8 +53,9 @@ public void DownloadsHistoricalDataWithValidDataTestParameters(Symbol symbol, Re { var parameters = new DataDownloaderGetParameters(symbol, resolution, startDateTimeUtc, endDateTimeUtc, TickType.Trade); - var downloadResponse = _downloader.Get(parameters).ToList(); + var downloadResponse = _downloader.Get(parameters)?.ToList(); + Assert.IsNotNull(downloadResponse); Assert.IsNotEmpty(downloadResponse); Log.Trace($"{symbol}.{resolution}.[{startDateTimeUtc} - {endDateTimeUtc}]: Amount = {downloadResponse.Count}"); @@ -84,9 +85,9 @@ public void DownloadsHistoricalDataWithInvalidDataTestParameters(Symbol symbol, { var parameters = new DataDownloaderGetParameters(symbol, resolution, startDateTimeUtc, endDateTimeUtc, tickType); - var downloadResponse = _downloader.Get(parameters).ToList(); + var downloadResponse = _downloader.Get(parameters)?.ToList(); - Assert.IsEmpty(downloadResponse); + Assert.IsNull(downloadResponse); } private static IEnumerable HistoricalInvalidDataThrowExceptionTestCases @@ -106,7 +107,7 @@ public void DownloadsHistoricalDataWithInvalidDataTestParametersThrowException(S { var parameters = new DataDownloaderGetParameters(symbol, Resolution.Minute, new DateTime(2024, 1, 1), new DateTime(2024, 2, 1), TickType.Trade); - Assert.That(() => _downloader.Get(parameters).ToList(), Throws.Exception); + Assert.That(() => _downloader.Get(parameters)?.ToList(), Throws.Exception); } } } diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs index 2f9c08a..4408f77 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs @@ -52,7 +52,7 @@ public void OneTimeSetUp() [Test] [TestCaseSource(nameof(TestData))] - public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, int period, bool isNonEmptyResult) + public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, int period, bool isNotNullResult) { _coinApiDataQueueHandler.SetUpHistDataLimit(100); @@ -67,14 +67,15 @@ public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, i resolution, true, false, DataNormalizationMode.Raw, TickType.Trade) }; - var slices = _coinApiDataQueueHandler.GetHistory(historyRequests, TimeZones.Utc).ToArray(); + var slices = _coinApiDataQueueHandler.GetHistory(historyRequests, TimeZones.Utc)?.ToArray(); - if (isNonEmptyResult) + if (isNotNullResult) { + Assert.IsNotNull(slices); // For resolution larger than second do more tests if (resolution > Resolution.Second) { - Assert.AreEqual(period, slices.Length); + Assert.That(slices.Length, Is.EqualTo(period)); var firstSliceTradeBars = slices.First().Bars.Values; @@ -83,8 +84,8 @@ public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, i firstSliceTradeBars.DoForEach(tb => { var resTimeSpan = resolution.ToTimeSpan(); - Assert.AreEqual(resTimeSpan, tb.Period); - Assert.AreEqual(startTimeUtc.RoundUp(resTimeSpan), tb.Time); + Assert.That(tb.Period, Is.EqualTo(resTimeSpan)); + Assert.That(tb.Time, Is.EqualTo(startTimeUtc.RoundUp(resTimeSpan))); }); var lastSliceTradeBars = slices.Last().Bars.Values; @@ -92,8 +93,8 @@ public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, i lastSliceTradeBars.DoForEach(tb => { var resTimeSpan = resolution.ToTimeSpan(); - Assert.AreEqual(resTimeSpan, tb.Period); - Assert.AreEqual(nowUtc.RoundDown(resTimeSpan), tb.Time); + Assert.That(tb.Period, Is.EqualTo(resTimeSpan)); + Assert.That(tb.Time, Is.EqualTo(nowUtc.RoundDown(resTimeSpan))); }); } // For res. second data counts, start/end dates may slightly vary from historical request's @@ -101,7 +102,7 @@ public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, i else { Assert.IsTrue(slices.Length > 0); - Assert.AreEqual(resolution.ToTimeSpan(), slices.First().Bars.Values.FirstOrDefault()?.Period); + Assert.That(slices.First().Bars.Values.FirstOrDefault()?.Period, Is.EqualTo(resolution.ToTimeSpan())); } // Slices are ordered by time @@ -109,8 +110,7 @@ public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, i } else { - // Empty - Assert.IsEmpty(slices); + Assert.IsNull(slices); } } diff --git a/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs b/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs index dd759ff..08b5bf4 100644 --- a/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs +++ b/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs @@ -33,19 +33,19 @@ public CoinAPIDataDownloader() _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); } - public IEnumerable Get(DataDownloaderGetParameters dataDownloaderGetParameters) + public IEnumerable? Get(DataDownloaderGetParameters dataDownloaderGetParameters) { if (dataDownloaderGetParameters.TickType != TickType.Trade) { Log.Error($"{nameof(CoinAPIDataDownloader)}.{nameof(Get)}: Not supported data type - {dataDownloaderGetParameters.TickType}. " + $"Currently available support only for historical of type - {nameof(TickType.Trade)}"); - yield break; + return null; } if (dataDownloaderGetParameters.EndUtc < dataDownloaderGetParameters.StartUtc) { Log.Error($"{nameof(CoinAPIDataDownloader)}.{nameof(Get)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); - yield break; + return null; } var symbol = dataDownloaderGetParameters.Symbol; @@ -64,7 +64,20 @@ public IEnumerable Get(DataDownloaderGetParameters dataDownloaderGetPa dataNormalizationMode: DataNormalizationMode.Raw, tickType: TickType.Trade); - foreach (var slice in _historyProvider.GetHistory(historyRequests)) + var history = _historyProvider.GetHistory(historyRequests); + + // historyRequest contains wrong data request + if (history == null) + { + return null; + } + + return GetHistoryInSlice(history); + } + + private IEnumerable GetHistoryInSlice(IEnumerable history) + { + foreach (var slice in history) { yield return slice; } diff --git a/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs b/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs index f2a14e1..3c96491 100644 --- a/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs +++ b/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs @@ -48,9 +48,20 @@ public override IEnumerable GetHistory(IEnumerable reques foreach (var request in requests) { var history = GetHistory(request); + + if (history == null) + { + continue; + } + var subscription = CreateSubscription(request, history); subscriptions.Add(subscription); } + + if (subscriptions.Count == 0) + { + return null; + } return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); } @@ -59,13 +70,13 @@ public IEnumerable GetHistory(HistoryRequest historyRequest) if (!CanSubscribe(historyRequest.Symbol)) { Log.Error($"CoinApiDataProvider.GetHistory(): Invalid security type {historyRequest.Symbol.SecurityType}"); - yield break; + return null; } if (historyRequest.Resolution == Resolution.Tick) { Log.Error($"CoinApiDataProvider.GetHistory(): No historical ticks, only OHLCV timeseries"); - yield break; + return null; } if (historyRequest.DataType == typeof(QuoteBar)) @@ -75,12 +86,21 @@ public IEnumerable GetHistory(HistoryRequest historyRequest) Log.Error("CoinApiDataProvider.GetHistory(): No historical QuoteBars , only TradeBars"); _invalidHistoryDataTypeWarningFired = true; } - yield break; + return null; } - var resolutionTimeSpan = historyRequest.Resolution.ToTimeSpan(); - var lastRequestedBarStartTime = historyRequest.EndTimeUtc.RoundDown(resolutionTimeSpan); - var currentStartTime = historyRequest.StartTimeUtc.RoundUp(resolutionTimeSpan); + return GetHistory(historyRequest.Symbol, + historyRequest.Resolution, + historyRequest.StartTimeUtc, + historyRequest.EndTimeUtc + ); + } + + private IEnumerable GetHistory(Symbol symbol, Resolution resolution, DateTime startDateTimeUtc, DateTime endDateTimeUtc) + { + var resolutionTimeSpan = resolution.ToTimeSpan(); + var lastRequestedBarStartTime = endDateTimeUtc.RoundDown(resolutionTimeSpan); + var currentStartTime = startDateTimeUtc.RoundUp(resolutionTimeSpan); var currentEndTime = lastRequestedBarStartTime; // Perform a check of the number of bars requested, this must not exceed a static limit @@ -95,8 +115,8 @@ public IEnumerable GetHistory(HistoryRequest historyRequest) while (currentStartTime < lastRequestedBarStartTime) { - var coinApiSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); - var coinApiPeriod = _ResolutionToCoinApiPeriodMappings[historyRequest.Resolution]; + var coinApiSymbol = _symbolMapper.GetBrokerageSymbol(symbol); + var coinApiPeriod = _ResolutionToCoinApiPeriodMappings[resolution]; // Time must be in ISO 8601 format var coinApiStartTime = currentStartTime.ToStringInvariant("s"); @@ -128,15 +148,15 @@ public IEnumerable GetHistory(HistoryRequest historyRequest) // Can be no historical data for a short period interval if (!coinApiHistoryBars.Any()) { - Log.Error($"CoinApiDataProvider.GetHistory(): API returned no data for the requested period [{coinApiStartTime} - {coinApiEndTime}] for symbol [{historyRequest.Symbol}]"); + Log.Error($"CoinApiDataProvider.GetHistory(): API returned no data for the requested period [{coinApiStartTime} - {coinApiEndTime}] for symbol [{symbol}]"); continue; } foreach (var ohlcv in coinApiHistoryBars) { yield return - new TradeBar(ohlcv.TimePeriodStart, historyRequest.Symbol, ohlcv.PriceOpen, ohlcv.PriceHigh, - ohlcv.PriceLow, ohlcv.PriceClose, ohlcv.VolumeTraded, historyRequest.Resolution.ToTimeSpan()); + new TradeBar(ohlcv.TimePeriodStart, symbol, ohlcv.PriceOpen, ohlcv.PriceHigh, + ohlcv.PriceLow, ohlcv.PriceClose, ohlcv.VolumeTraded, resolutionTimeSpan); } currentStartTime = currentEndTime; From 4c2f7fab01210f6d87523c87d833ca63f2db3d1e Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 23 Feb 2024 13:18:11 +0200 Subject: [PATCH 12/14] feat: add stop spamming flags feat: null sign refactor: remove extra else block --- .../CoinApiDataHistoryProvider.cs | 27 ++++++++++++++++--- QuantConnect.CoinAPI/CoinApiDataProvider.cs | 8 +++--- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs b/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs index 3c96491..8e5509d 100644 --- a/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs +++ b/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs @@ -37,12 +37,23 @@ public partial class CoinApiDataProvider /// private bool _invalidHistoryDataTypeWarningFired; + /// + /// Indicates whether the warning for invalid has been fired. + /// + private bool _invalidSecurityTypeWarningFired; + + /// + /// Indicates whether the warning for invalid has been fired. + /// + private bool _invalidResolutionTypeWarningFired; + + public override void Initialize(HistoryProviderInitializeParameters parameters) { // NOP } - public override IEnumerable GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) + public override IEnumerable? GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) { var subscriptions = new List(); foreach (var request in requests) @@ -65,17 +76,25 @@ public override IEnumerable GetHistory(IEnumerable reques return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); } - public IEnumerable GetHistory(HistoryRequest historyRequest) + public IEnumerable? GetHistory(HistoryRequest historyRequest) { if (!CanSubscribe(historyRequest.Symbol)) { - Log.Error($"CoinApiDataProvider.GetHistory(): Invalid security type {historyRequest.Symbol.SecurityType}"); + if (!_invalidSecurityTypeWarningFired) + { + Log.Error($"CoinApiDataProvider.GetHistory(): Invalid security type {historyRequest.Symbol.SecurityType}"); + _invalidSecurityTypeWarningFired = true; + } return null; } if (historyRequest.Resolution == Resolution.Tick) { - Log.Error($"CoinApiDataProvider.GetHistory(): No historical ticks, only OHLCV timeseries"); + if (!_invalidResolutionTypeWarningFired) + { + Log.Error($"CoinApiDataProvider.GetHistory(): No historical ticks, only OHLCV timeseries"); + _invalidResolutionTypeWarningFired = true; + } return null; } diff --git a/QuantConnect.CoinAPI/CoinApiDataProvider.cs b/QuantConnect.CoinAPI/CoinApiDataProvider.cs index f631cec..2a87e11 100644 --- a/QuantConnect.CoinAPI/CoinApiDataProvider.cs +++ b/QuantConnect.CoinAPI/CoinApiDataProvider.cs @@ -149,7 +149,7 @@ public void SetJob(LiveNodePacket job) /// private void Initialize(string apiKey, string product) { - // ValidateSubscription(); + ValidateSubscription(); _apiKey = apiKey; @@ -365,10 +365,8 @@ private void SendHelloMessage(IEnumerable subscribeFilter) { throw new Exception("Not connected..."); } - else - { - IsConnected = true; - } + + IsConnected = true; _nextHelloMessageUtcTime = DateTime.UtcNow.Add(_minimumTimeBetweenHelloMessages); } From 69e8e20b62669d2e9e224813f986eb62600038c5 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 27 Feb 2024 00:36:02 +0200 Subject: [PATCH 13/14] remove: duplicate validation from downloader --- QuantConnect.CoinAPI/CoinAPIDataDownloader.cs | 31 +++---------------- .../CoinApiDataHistoryProvider.cs | 14 +++++++++ 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs b/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs index 08b5bf4..de55552 100644 --- a/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs +++ b/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs @@ -35,22 +35,10 @@ public CoinAPIDataDownloader() public IEnumerable? Get(DataDownloaderGetParameters dataDownloaderGetParameters) { - if (dataDownloaderGetParameters.TickType != TickType.Trade) - { - Log.Error($"{nameof(CoinAPIDataDownloader)}.{nameof(Get)}: Not supported data type - {dataDownloaderGetParameters.TickType}. " + - $"Currently available support only for historical of type - {nameof(TickType.Trade)}"); - return null; - } - - if (dataDownloaderGetParameters.EndUtc < dataDownloaderGetParameters.StartUtc) - { - Log.Error($"{nameof(CoinAPIDataDownloader)}.{nameof(Get)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); - return null; - } - var symbol = dataDownloaderGetParameters.Symbol; - var historyRequests = new HistoryRequest( + var history = _historyProvider.GetHistory( + new HistoryRequest( startTimeUtc: dataDownloaderGetParameters.StartUtc, endTimeUtc: dataDownloaderGetParameters.EndUtc, dataType: typeof(TradeBar), @@ -62,9 +50,8 @@ public CoinAPIDataDownloader() includeExtendedMarketHours: true, isCustomData: false, dataNormalizationMode: DataNormalizationMode.Raw, - tickType: TickType.Trade); - - var history = _historyProvider.GetHistory(historyRequests); + tickType: TickType.Trade) + ); // historyRequest contains wrong data request if (history == null) @@ -72,15 +59,7 @@ public CoinAPIDataDownloader() return null; } - return GetHistoryInSlice(history); - } - - private IEnumerable GetHistoryInSlice(IEnumerable history) - { - foreach (var slice in history) - { - yield return slice; - } + return history.Select(slice => slice); } public void Dispose() diff --git a/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs b/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs index 8e5509d..55c379b 100644 --- a/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs +++ b/QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs @@ -47,6 +47,10 @@ public partial class CoinApiDataProvider /// private bool _invalidResolutionTypeWarningFired; + /// + /// Indicates whether a warning for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC. + /// + private bool _invalidStartTimeWarningFired; public override void Initialize(HistoryProviderInitializeParameters parameters) { @@ -108,6 +112,16 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) return null; } + if (historyRequest.EndTimeUtc < historyRequest.StartTimeUtc) + { + if (!_invalidStartTimeWarningFired) + { + Log.Error($"{nameof(CoinAPIDataDownloader)}.{nameof(GetHistory)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); + _invalidStartTimeWarningFired = true; + } + return null; + } + return GetHistory(historyRequest.Symbol, historyRequest.Resolution, historyRequest.StartTimeUtc, From 0af9d747131d58020941b669a3876557b802855c Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 27 Feb 2024 15:21:47 +0200 Subject: [PATCH 14/14] revert: IsConnected handle connection --- QuantConnect.CoinAPI/CoinApiDataProvider.cs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/QuantConnect.CoinAPI/CoinApiDataProvider.cs b/QuantConnect.CoinAPI/CoinApiDataProvider.cs index 2a87e11..2791b13 100644 --- a/QuantConnect.CoinAPI/CoinApiDataProvider.cs +++ b/QuantConnect.CoinAPI/CoinApiDataProvider.cs @@ -217,7 +217,7 @@ private bool Unsubscribe(IEnumerable symbols) /// Returns whether the data provider is connected /// /// true if the data provider is connected - public bool IsConnected { get; private set; } + public bool IsConnected => true; /// /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. @@ -361,13 +361,6 @@ private void SendHelloMessage(IEnumerable subscribeFilter) subscribe_filter_symbol_id = list.ToArray() }); - if (!IsConnected && !_client.ConnectedEvent.WaitOne(TimeSpan.FromSeconds(30))) - { - throw new Exception("Not connected..."); - } - - IsConnected = true; - _nextHelloMessageUtcTime = DateTime.UtcNow.Add(_minimumTimeBetweenHelloMessages); } @@ -449,7 +442,6 @@ private Symbol GetSymbolUsingCache(string ticker) private void OnError(object? sender, Exception e) { - IsConnected = false; Log.Error(e); }