Initialization via Job Packet and Rename DataQueueHandler to DataProv…

…ider (#2)

* rename: namespaces

* rename: DataQueueHandler -> DataProvider

* feat: Config -> Globals in ValidateSubscription

* feat: SetJob
fix: IsConnected Property
feat: handle of null warnings

* feat: reference on Lean\Tests + init Globals

* refactor: use CanSubscribe for Symbol in GetHistory()

* refactor: missed Lean in namespaces

* rename: QueueHandler -> DataProvider Tests
revert: commented code

* fix: workflow dotnet test path

* fix: testcase param forex instead of equity

* refactor: return null in wrong request of GetHistory()

* feat: add stop spamming flags
feat: null sign
refactor: remove extra else block

* remove: duplicate validation from downloader

* revert: IsConnected handle connection

.github/workflows/build.yml

@@ -46,14 +46,14 @@ jobs:
           image: quantconnect/lean:foundation
           options: -v /home/runner/work:/__w --workdir /__w/Lean.DataSource.CoinAPI/Lean.DataSource.CoinAPI -e QC_JOB_USER_ID=${{ secrets.QC_JOB_USER_ID }} -e QC_API_ACCESS_TOKEN=${{ secrets.QC_API_ACCESS_TOKEN }} -e QC_JOB_ORGANIZATION_ID=${{ secrets.QC_JOB_ORGANIZATION_ID }} -e QC_COINAPI_API_KEY=${{ secrets.QC_COINAPI_API_KEY }}
 
-      - name: Build QuantConnect.CoinAPI
-        run: dotnet build ./QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
+      - name: Build QuantConnect.DataSource.CoinAPI
+        run: dotnet build ./QuantConnect.CoinAPI/QuantConnect.DataSource.CoinAPI.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
 
       - name: Build DataProcessing
         run: dotnet build ./DataProcessing/DataProcessing.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
 
-      - name: Build QuantConnect.CoinAPI.Tests
-        run: dotnet build ./QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
+      - name: Build QuantConnect.DataSource.CoinAPI.Tests
+        run: dotnet build ./QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
 
-      - name: Run QuantConnect.CoinAPI.Tests
-        run: dotnet test ./QuantConnect.CoinAPI.Tests/bin/Release/QuantConnect.CoinAPI.Tests.dll
+      - name: Run QuantConnect.DataSource.CoinAPI.Tests
+        run: dotnet test ./QuantConnect.CoinAPI.Tests/bin/Release/QuantConnect.Lean.DataSource.CoinAPI.Tests.dll

DataProcessing/CoinApiDataConverter.cs

@@ -19,7 +19,7 @@
 using System.Diagnostics;
 using QuantConnect.Logging;
 using QuantConnect.ToolBox;
-using QuantConnect.CoinAPI;
+using QuantConnect.Lean.DataSource.CoinAPI;
 
 namespace QuantConnect.DataProcessing
 {

DataProcessing/DataProcessing.csproj

@@ -29,7 +29,7 @@
 	</ItemGroup>
 
 	<ItemGroup>
-		<ProjectReference Include="..\QuantConnect.CoinAPI\QuantConnect.CoinAPI.csproj" />
+		<ProjectReference Include="..\QuantConnect.CoinAPI\QuantConnect.DataSource.CoinAPI.csproj" />
 	</ItemGroup>
 
 	<ItemGroup>

Lean.DataSource.CoinAPI.sln

@@ -3,9 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.5.002.0
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.CoinAPI", "QuantConnect.CoinAPI\QuantConnect.CoinAPI.csproj", "{2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataSource.CoinAPI", "QuantConnect.CoinAPI\QuantConnect.DataSource.CoinAPI.csproj", "{2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.CoinAPI.Tests", "QuantConnect.CoinAPI.Tests\QuantConnect.CoinAPI.Tests.csproj", "{337CEE6E-639A-448D-95ED-2C1628E26AF2}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataSource.CoinAPI.Tests", "QuantConnect.CoinAPI.Tests\QuantConnect.DataSource.CoinAPI.Tests.csproj", "{337CEE6E-639A-448D-95ED-2C1628E26AF2}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataProcessing", "DataProcessing\DataProcessing.csproj", "{881514B4-641E-4EDC-8020-6BEA0CC8F48C}"
 EndProject

QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs

@@ -15,9 +15,10 @@
 
 using NUnit.Framework;
 using QuantConnect.Util;
+using QuantConnect.Tests;
 using QuantConnect.Logging;
 
-namespace QuantConnect.CoinAPI.Tests
+namespace QuantConnect.Lean.DataSource.CoinAPI.Tests
 {
     [TestFixture]
     public class CoinAPIDataDownloaderTests
@@ -52,8 +53,9 @@ public void DownloadsHistoricalDataWithValidDataTestParameters(Symbol symbol, Re
         {
             var parameters = new DataDownloaderGetParameters(symbol, resolution, startDateTimeUtc, endDateTimeUtc, TickType.Trade);
 
-            var downloadResponse = _downloader.Get(parameters).ToList();
+            var downloadResponse = _downloader.Get(parameters)?.ToList();
 
+            Assert.IsNotNull(downloadResponse);
             Assert.IsNotEmpty(downloadResponse);
 
             Log.Trace($"{symbol}.{resolution}.[{startDateTimeUtc} - {endDateTimeUtc}]: Amount = {downloadResponse.Count}");
@@ -83,19 +85,20 @@ public void DownloadsHistoricalDataWithInvalidDataTestParameters(Symbol symbol,
         {
             var parameters = new DataDownloaderGetParameters(symbol, resolution, startDateTimeUtc, endDateTimeUtc, tickType);
 
-            var downloadResponse = _downloader.Get(parameters).ToList();
+            var downloadResponse = _downloader.Get(parameters)?.ToList();
 
-            Assert.IsEmpty(downloadResponse);
+            Assert.IsNull(downloadResponse);
         }
 
         private static IEnumerable<TestCaseData> HistoricalInvalidDataThrowExceptionTestCases
         {
             get
             {
+                TestGlobals.Initialize();
                 yield return new TestCaseData(Symbol.Create("BTCBTC", SecurityType.Crypto, Market.Binance))
                     .SetDescription($"Wrong Symbol - 'BTCBTC'");
-                yield return new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Equity, Market.Binance))
-                    .SetDescription($"Wrong SecurityType - {SecurityType.Equity}");
+                yield return new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Forex, Market.Binance))
+                    .SetDescription($"Wrong SecurityType - {SecurityType.Forex}");
             }
         }
 
@@ -104,7 +107,7 @@ public void DownloadsHistoricalDataWithInvalidDataTestParametersThrowException(S
         {
             var parameters = new DataDownloaderGetParameters(symbol, Resolution.Minute, new DateTime(2024, 1, 1), new DateTime(2024, 2, 1), TickType.Trade);
 
-            Assert.That(() => _downloader.Get(parameters).ToList(), Throws.Exception);
+            Assert.That(() => _downloader.Get(parameters)?.ToList(), Throws.Exception);
         }
     }
 }

QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs

@@ -20,7 +20,7 @@
 using QuantConnect.Data.Market;
 using QuantConnect.Securities;
 
-namespace QuantConnect.CoinAPI.Tests
+namespace QuantConnect.Lean.DataSource.CoinAPI.Tests
 {
     [TestFixture]
     public class CoinAPIHistoryProviderTests
@@ -52,7 +52,7 @@ public void OneTimeSetUp()
 
         [Test]
         [TestCaseSource(nameof(TestData))]
-        public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, int period, bool isNonEmptyResult)
+        public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, int period, bool isNotNullResult)
         {
             _coinApiDataQueueHandler.SetUpHistDataLimit(100);
 
@@ -67,14 +67,15 @@ public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, i
                     resolution, true, false, DataNormalizationMode.Raw, TickType.Trade)
             };
 
-            var slices = _coinApiDataQueueHandler.GetHistory(historyRequests, TimeZones.Utc).ToArray();
+            var slices = _coinApiDataQueueHandler.GetHistory(historyRequests, TimeZones.Utc)?.ToArray();
 
-            if (isNonEmptyResult)
+            if (isNotNullResult)
             {
+                Assert.IsNotNull(slices);
                 // For resolution larger than second do more tests
                 if (resolution > Resolution.Second)
                 {
-                    Assert.AreEqual(period, slices.Length);
+                    Assert.That(slices.Length, Is.EqualTo(period));
 
                     var firstSliceTradeBars = slices.First().Bars.Values;
 
@@ -83,38 +84,37 @@ public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, i
                     firstSliceTradeBars.DoForEach(tb =>
                     {
                         var resTimeSpan = resolution.ToTimeSpan();
-                        Assert.AreEqual(resTimeSpan, tb.Period);
-                        Assert.AreEqual(startTimeUtc.RoundUp(resTimeSpan), tb.Time);
+                        Assert.That(tb.Period, Is.EqualTo(resTimeSpan));
+                        Assert.That(tb.Time, Is.EqualTo(startTimeUtc.RoundUp(resTimeSpan)));
                     });
 
                     var lastSliceTradeBars = slices.Last().Bars.Values;
 
                     lastSliceTradeBars.DoForEach(tb =>
                     {
                         var resTimeSpan = resolution.ToTimeSpan();
-                        Assert.AreEqual(resTimeSpan, tb.Period);
-                        Assert.AreEqual(nowUtc.RoundDown(resTimeSpan), tb.Time);
+                        Assert.That(tb.Period, Is.EqualTo(resTimeSpan));
+                        Assert.That(tb.Time, Is.EqualTo(nowUtc.RoundDown(resTimeSpan)));
                     });
                 }
                 // For res. second data counts, start/end dates may slightly vary from historical request's 
                 // Make sure just that resolution is correct and amount is positive numb.
                 else
                 {
                     Assert.IsTrue(slices.Length > 0);
-                    Assert.AreEqual(resolution.ToTimeSpan(), slices.First().Bars.Values.FirstOrDefault()?.Period);
+                    Assert.That(slices.First().Bars.Values.FirstOrDefault()?.Period, Is.EqualTo(resolution.ToTimeSpan()));
                 }
 
                 // Slices are ordered by time
                 Assert.That(slices, Is.Ordered.By("Time"));
             }
             else
             {
-                // Empty
-                Assert.IsEmpty(slices);
+                Assert.IsNull(slices);
             }
         }
 
-        public class CoinApiDataQueueHandlerMock : CoinApiDataQueueHandler
+        public class CoinApiDataQueueHandlerMock : CoinApiDataProvider
         {
             public new void SetUpHistDataLimit(int limit)
             {

QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs

@@ -16,7 +16,7 @@
 
 using NUnit.Framework;
 
-namespace QuantConnect.CoinAPI.Tests
+namespace QuantConnect.Lean.DataSource.CoinAPI.Tests
 {
     [TestFixture]
     public class CoinAPISymbolMapperTests

QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs

@@ -14,9 +14,10 @@
 */
 
 using NUnit.Framework;
+using QuantConnect.Packets;
 using QuantConnect.Configuration;
 
-namespace QuantConnect.CoinAPI.Tests
+namespace QuantConnect.Lean.DataSource.CoinAPI.Tests
 {
     [TestFixture]
     public class CoinApiAdditionalTests
@@ -28,11 +29,37 @@ public void ThrowsOnFailedAuthentication()
 
             Assert.Throws<Exception>(() =>
             {
-                using var _coinApiDataQueueHandler = new CoinApiDataQueueHandler();
+                using var _coinApiDataQueueHandler = new CoinApiDataProvider();
             });
 
             // reset api key
             TestSetup.GlobalSetup();
         }
+
+        [Test]
+        public void CanInitializeUsingJobPacket()
+        {
+            var apiKey = Config.Get("coinapi-api-key");
+            Config.Set("coinapi-api-key", "");
+
+            var job = new LiveNodePacket
+            {
+                BrokerageData = new Dictionary<string, string>() {
+                    { "coinapi-api-key", "InvalidApiKeyThatWontBeUsed" },
+                    { "coinapi-product", "Startup" }
+                }
+            };
+
+            using var iexDataProvider = new CoinApiDataProvider();
+
+            // Throw because CoinApiSymbolMapper makes request to API (we have invalid api key in LiveNodePacket)
+            Assert.Throws<Exception>(() =>
+            {
+                iexDataProvider.SetJob(job);
+            });
+
+            // revert Config of ApiKey for another tests
+            Config.Set("coinapi-api-key", apiKey);
+        }
     }
 }

QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs → QuantConnect.CoinAPI.Tests/CoinApiDataProviderTests.cs

@@ -21,12 +21,12 @@
 using QuantConnect.Data.Market;
 using System.Collections.Concurrent;
 
-namespace QuantConnect.CoinAPI.Tests
+namespace QuantConnect.Lean.DataSource.CoinAPI.Tests
 {
     [TestFixture]
-    public class CoinApiDataQueueHandlerTest
+    public class CoinApiDataProviderTests
     {
-        private CoinApiDataQueueHandler _coinApiDataQueueHandler;
+        private CoinApiDataProvider _coinApiDataQueueHandler;
         private CancellationTokenSource _cancellationTokenSource;
 
         [SetUp]
@@ -61,12 +61,12 @@ public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest()
                 _cancellationTokenSource.Token,
                 tick =>
                 {
-                    Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest)}: {tick}");
+                    Log.Debug($"{nameof(CoinApiDataProviderTests)}.{nameof(SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest)}: {tick}");
                     tradeBars.Add(tick);
 
                     if (tradeBars.Count > 5)
                     {
-                        resetEvent.Set();
+                       resetEvent.Set();
                     }
                 },
             () => _cancellationTokenSource.Cancel());
@@ -111,7 +111,7 @@ public void SubscribeToBTCUSDSecondOnDifferentMarkets()
                     _cancellationTokenSource.Token,
                     tick =>
                     {
-                        Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDSecondOnDifferentMarkets)}: {tick}");
+                        Log.Debug($"{nameof(CoinApiDataProviderTests)}.{nameof(SubscribeToBTCUSDSecondOnDifferentMarkets)}: {tick}");
                         symbolBaseData[tick.Symbol].Add(tick);
                     },
                 () =>
@@ -165,7 +165,7 @@ public void SubscribeToBTCUSDTFutureSecondBinance()
                 _cancellationTokenSource.Token,
                 tick =>
                 {
-                    Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)}: {tick}");
+                    Log.Debug($"{nameof(CoinApiDataProviderTests)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)}: {tick}");
                     tickData.Add(tick);
 
                     if (tickData.Count > 5)
@@ -190,7 +190,7 @@ public void SubscribeToBTCUSDTFutureSecondBinance()
 
             if (tickData.Count == 0)
             {
-                Assert.Fail($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)} is nothing returned. {symbol}|{resolution}|tickData = {tickData.Count}");
+                Assert.Fail($"{nameof(CoinApiDataProviderTests)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)} is nothing returned. {symbol}|{resolution}|tickData = {tickData.Count}");
             }
 
             CoinApiTestHelper.AssertSymbol(tickData.First().Symbol, symbol);

QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs

@@ -19,7 +19,7 @@
 using QuantConnect.Logging;
 using QuantConnect.Data.Market;
 
-namespace QuantConnect.CoinAPI.Tests
+namespace QuantConnect.Lean.DataSource.CoinAPI.Tests
 {
     public static class CoinApiTestHelper
     {
@@ -68,7 +68,7 @@ public static void AssertBaseData(List<BaseData> tradeBars, Resolution expectedR
                         Assert.IsTrue(trade.Period.ToHigherResolutionEquivalent(true) == expectedResolution);
                         break;
                     default:
-                        Assert.Fail($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(AssertBaseData)}: The tick type doesn't support");
+                        Assert.Fail($"{nameof(CoinApiDataProviderTests)}.{nameof(AssertBaseData)}: The tick type doesn't support");
                         break;
                 }
             }

QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj → QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj

@@ -5,10 +5,10 @@
 		<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
 		<TargetFramework>net6.0</TargetFramework>
 		<OutputPath>bin\$(Configuration)\</OutputPath>
-		<Product>QuantConnect.CoinAPI.Tests</Product>
-		<AssemblyName>QuantConnect.CoinAPI.Tests</AssemblyName>
-		<RootNamespace>QuantConnect.CoinAPI.Tests</RootNamespace>
-		<AssemblyTitle>QuantConnect.CoinAPI.Tests</AssemblyTitle>
+		<Product>QuantConnect.Lean.DataSource.CoinAPI.Tests</Product>
+		<AssemblyName>QuantConnect.Lean.DataSource.CoinAPI.Tests</AssemblyName>
+		<RootNamespace>QuantConnect.Lean.DataSource.CoinAPI.Tests</RootNamespace>
+		<AssemblyTitle>QuantConnect.Lean.DataSource.CoinAPI.Tests</AssemblyTitle>
 		<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
 		<ImplicitUsings>enable</ImplicitUsings>
 		<Nullable>enable</Nullable>
@@ -30,7 +30,8 @@
 	</ItemGroup>
 
 	<ItemGroup>
-		<ProjectReference Include="..\QuantConnect.CoinAPI\QuantConnect.CoinAPI.csproj" />
+		<ProjectReference Include="..\QuantConnect.CoinAPI\QuantConnect.DataSource.CoinAPI.csproj" />
+		<ProjectReference Include="..\..\Lean\Tests\QuantConnect.Tests.csproj" />
 	</ItemGroup>
 
 	<ItemGroup>

QuantConnect.CoinAPI.Tests/TestSetup.cs

@@ -19,7 +19,7 @@
 using QuantConnect.Logging;
 using QuantConnect.Configuration;
 
-namespace QuantConnect.CoinAPI.Tests
+namespace QuantConnect.Lean.DataSource.CoinAPI.Tests
 {
     [SetUpFixture]
     public static class TestSetup