Standarize implementation (#8)

- Standarize implementation. See QuantConnect/Lean#7837
- Adjust and fix example algorithms

QuantConnect.DataSource.csproj

@@ -10,7 +10,7 @@
 
   <ItemGroup>
     <PackageReference Include="QuantConnect.Common" Version="2.5.*" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
   </ItemGroup>
 
   <ItemGroup>

QuiverWallStreetBetsUniverse.cs

@@ -15,26 +15,20 @@
 */
 
 using System;
-using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
 using Newtonsoft.Json;
-using NodaTime;
-using QuantConnect;
 using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
+
 namespace QuantConnect.DataSource
 {
     /// <summary>
     /// Universe Selection helper class for QuiverWallStreetBets dataset
     /// </summary>
-    public class QuiverWallStreetBetsUniverse : BaseData
+    public class QuiverWallStreetBetsUniverse : BaseDataCollection
     {
         private static readonly TimeSpan _period = TimeSpan.FromDays(1);
-
-        /// <summary>
-        /// Symbol of data
-        /// </summary>
-        public Symbol Symbol { get; set; }
 
         /// <summary>
         /// The number of mentions on the given date
@@ -78,7 +72,8 @@ public override SubscriptionDataSource GetSource(SubscriptionDataConfig config,
                     "universe",
                     $"{date.ToStringInvariant(DateFormat.EightCharacter)}.csv"
                 ),
-                SubscriptionTransportMedium.LocalFile
+                SubscriptionTransportMedium.LocalFile,
+                FileFormat.FoldingCollection
             );
         }
 
@@ -106,5 +101,23 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date
                 Value = mentions
             };
         }
+
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public override BaseData Clone()
+        {
+            return new QuiverWallStreetBetsUniverse
+            {
+                Symbol = Symbol,
+                Time = Time,
+                Value = Value,
+                Data = Data,
+
+                Mentions = Mentions,
+                Rank = Rank,
+                Sentiment = Sentiment
+            };
+        }
     }
 }

QuiverWallStreetBetsUniverseSelectionAlgorithm.cs

@@ -34,18 +34,32 @@ public override void Initialize()
             SetCash(100000);
 
             // add a custom universe data source (defaults to usa-equity)
-            AddUniverse<QuiverWallStreetBetsUniverse>("QuiverWallStreetBetsUniverse", Resolution.Daily, data =>
+            var universe = AddUniverse<QuiverWallStreetBetsUniverse>(data =>
             {
-                foreach (var datum in data)
+                foreach (QuiverWallStreetBetsUniverse datum in data)
                 {
                     Log($"{datum.Symbol},{datum.Mentions},{datum.Rank},{datum.Sentiment}");
                 }
 
                 // define our selection criteria
-                return from d in data
+                return from QuiverWallStreetBetsUniverse d in data
                        where d.Mentions > 10 && d.Rank > 10
                        select d.Symbol;
             });
+
+            var history = History(universe, 1).ToList();
+            if (history.Count != 1)
+            {
+                throw new System.Exception($"Unexpected historical data count!");
+            }
+            foreach (var dataForDate in history)
+            {
+                var coarseData = dataForDate.ToList();
+                if (coarseData.Count < 100)
+                {
+                    throw new System.Exception($"Unexpected historical universe data!");
+                }
+            }
         }
 
         public override void OnSecuritiesChanged(SecurityChanges changes)

QuiverWallStreetBetsUniverseSelectionAlgorithm.py

@@ -13,7 +13,7 @@
 
 from AlgorithmImports import *
 
-class QuiverWallStreetBetsUniverseAlgorithm(QCAlgorithm): 
+class QuiverWallStreetBetsUniverseAlgorithm(QCAlgorithm):
     def Initialize(self):
         # Data ADDED via universe selection is added with Daily resolution.
         self.UniverseSettings.Resolution = Resolution.Daily
@@ -23,8 +23,16 @@ def Initialize(self):
         self.SetCash(100000)
 
         # add a custom universe data source (defaults to usa-equity)
-        self.AddUniverse(QuiverWallStreetBetsUniverse, "QuiverWallStreetBetsUniverse", Resolution.Daily, self.UniverseSelection)
-
+        universe = self.AddUniverse(QuiverWallStreetBetsUniverse, self.UniverseSelection)
+
+        history = self.History(universe, TimeSpan(1, 0, 0, 0))
+        if len(history) != 1:
+            raise ValueError(f"Unexpected history count {len(history)}! Expected 1")
+
+        for dataForDate in history:
+            if len(dataForDate) < 100:
+                raise ValueError(f"Unexpected historical universe data!")
+
     def UniverseSelection(self, data):
         for datum in data:
             self.Log(f"{datum.Symbol},{datum.Mentions},{datum.Rank},{datum.Sentiment}")