Standarize implementation (#17)

- Standarize implementation. See QuantConnect/Lean#7837
- Adjust and fix example algorithms

BrainCompanyFilingLanguageMetricsUniverse.cs

@@ -21,13 +21,14 @@
 using System.IO;
 using NodaTime;
 using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
 
 namespace QuantConnect.DataSource
 {
     /// <summary>
     /// Universe Selection helper class for BrainCompanyFilingLanguageMetrics dataset
     /// </summary>
-    public class BrainCompanyFilingLanguageMetricsUniverse<T> : BaseData
+    public class BrainCompanyFilingLanguageMetricsUniverse<T> : BaseDataCollection
         where T : BrainCompanyFilingLanguageMetricsUniverse<T>, new()
     {
         private static readonly TimeSpan _period = TimeSpan.FromDays(1);
@@ -75,7 +76,8 @@ public override SubscriptionDataSource GetSource(SubscriptionDataConfig config,
                     "universe",
                     $"{date:yyyyMMdd}.csv"
                 ),
-                SubscriptionTransportMedium.LocalFile
+                SubscriptionTransportMedium.LocalFile,
+                FileFormat.FoldingCollection
             );
         }
 
@@ -152,7 +154,6 @@ public override List<Resolution> SupportedResolutions()
         /// <returns>The <see cref="T:NodaTime.DateTimeZone" /> of this data type</returns>
         public override DateTimeZone DataTimeZone()
         {
-            // TODO: AddUniverse does not take this into account
             return TimeZones.Utc;
         }
     }

BrainCompanyFilingLanguageMetricsUniverse10K.cs

@@ -14,6 +14,8 @@
  *
 */
 
+using QuantConnect.Data;
+
 namespace QuantConnect.DataSource
 {
     /// <summary>
@@ -22,5 +24,23 @@ namespace QuantConnect.DataSource
     public class BrainCompanyFilingLanguageMetricsUniverse10K : BrainCompanyFilingLanguageMetricsUniverse<BrainCompanyFilingLanguageMetricsUniverse10K>
     {
         protected override string ReportType { get; set; } = "10k";
+
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public override BaseData Clone()
+        {
+            return new BrainCompanyFilingLanguageMetricsUniverse10K()
+            {
+                Symbol = Symbol,
+                Time = Time,
+                Data = Data,
+                Value = Value,
+
+                ReportSentiment = ReportSentiment,
+                RiskFactorsStatementSentiment = RiskFactorsStatementSentiment,
+                ManagementDiscussionAnalyasisOfFinancialConditionAndResultsOfOperations = ManagementDiscussionAnalyasisOfFinancialConditionAndResultsOfOperations
+            };
+        }
     }
 }

BrainCompanyFilingLanguageMetricsUniverseAll.cs

@@ -14,6 +14,8 @@
  *
 */
 
+using QuantConnect.Data;
+
 namespace QuantConnect.DataSource
 {
     /// <summary>
@@ -22,5 +24,23 @@ namespace QuantConnect.DataSource
     public class BrainCompanyFilingLanguageMetricsUniverseAll : BrainCompanyFilingLanguageMetricsUniverse<BrainCompanyFilingLanguageMetricsUniverseAll>
     {
         protected override string ReportType { get; set; } = "all";
+
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public override BaseData Clone()
+        {
+            return new BrainCompanyFilingLanguageMetricsUniverseAll()
+            {
+                Symbol = Symbol,
+                Time = Time,
+                Data = Data,
+                Value = Value,
+
+                ReportSentiment = ReportSentiment,
+                RiskFactorsStatementSentiment = RiskFactorsStatementSentiment,
+                ManagementDiscussionAnalyasisOfFinancialConditionAndResultsOfOperations = ManagementDiscussionAnalyasisOfFinancialConditionAndResultsOfOperations
+            };
+        }
     }
 }

BrainCompanyFilingLanguageMetricsUniverseSelectionAlgorithm.cs

@@ -34,9 +34,9 @@ public override void Initialize()
             SetCash(100000);
 
             // add a custom universe data source (defaults to usa-equity)
-            AddUniverse<BrainCompanyFilingLanguageMetricsUniverseAll>("BrainCompanyFilingLanguageMetricsUniverseAll", Resolution.Daily, data =>
+            var universe = AddUniverse<BrainCompanyFilingLanguageMetricsUniverseAll>(data =>
             {
-                foreach (var datum in data)
+                foreach (BrainCompanyFilingLanguageMetricsUniverseAll datum in data)
                 {
                     Log($@"{datum.Symbol},
                     {datum.ReportSentiment.Sentiment},
@@ -53,10 +53,24 @@ public override void Initialize()
                 }
 
                 // define our selection criteria
-                return from d in data 
-                    where d.ReportSentiment.Sentiment > 0m && d.ManagementDiscussionAnalyasisOfFinancialConditionAndResultsOfOperations.Sentiment > 0m
-                    select d.Symbol;
+                return from BrainCompanyFilingLanguageMetricsUniverseAll d in data
+                       where d.ReportSentiment.Sentiment > 0m && d.ManagementDiscussionAnalyasisOfFinancialConditionAndResultsOfOperations.Sentiment > 0m
+                       select d.Symbol;
             });
+
+            var history = History(universe, 1).ToList();
+            if (history.Count != 1)
+            {
+                throw new System.Exception($"Unexpected historical data count!");
+            }
+            foreach (var dataForDate in history)
+            {
+                var coarseData = dataForDate.ToList();
+                if (coarseData.Count < 100)
+                {
+                    throw new System.Exception($"Unexpected historical universe data!");
+                }
+            }
         }
 
         public override void OnSecuritiesChanged(SecurityChanges changes)

BrainCompanyFilingLanguageMetricsUniverseSelectionAlgorithm.py

@@ -13,7 +13,7 @@
 
 from AlgorithmImports import *
 
-class BrainCompanyFilingLanguageMetricsUniverseAlgorithm(QCAlgorithm): 
+class BrainCompanyFilingLanguageMetricsUniverseAlgorithm(QCAlgorithm):
     def Initialize(self):
         # Data ADDED via universe selection is added with Daily resolution.
         self.UniverseSettings.Resolution = Resolution.Daily
@@ -23,7 +23,15 @@ def Initialize(self):
         self.SetCash(100000)
 
         # add a custom universe data source (defaults to usa-equity)
-        self.AddUniverse(BrainCompanyFilingLanguageMetricsUniverseAll, "BrainCompanyFilingLanguageMetricsUniverseAll", Resolution.Daily, self.UniverseSelection)
+        universe = self.AddUniverse(BrainCompanyFilingLanguageMetricsUniverseAll, self.UniverseSelection)
+
+        history = self.History(universe, TimeSpan(1, 0, 0, 0))
+        if len(history) != 1:
+            raise ValueError(f"Unexpected history count {len(history)}! Expected 1")
+
+        for dataForDate in history:
+            if len(dataForDate) < 100:
+                raise ValueError(f"Unexpected historical universe data!")
 
     def UniverseSelection(self, data):
         for datum in data:
@@ -42,8 +50,9 @@ def UniverseSelection(self, data):
 
         # define our selection criteria
         return [d.Symbol for d in data \
-                    if d.ReportSentiment.Sentiment > 0 \
-                    and d.ManagementDiscussionAnalyasisOfFinancialConditionAndResultsOfOperations.Sentiment > 0]
+                    if d.ReportSentiment.Sentiment and d.ReportSentiment.Sentiment > 0 \
+                    and d.ManagementDiscussionAnalyasisOfFinancialConditionAndResultsOfOperations.Sentiment and \
+                    d.ManagementDiscussionAnalyasisOfFinancialConditionAndResultsOfOperations.Sentiment > 0]
 
     def OnSecuritiesChanged(self, changes):
         self.Log(changes.ToString())

BrainSentimentIndicatorUniverse.cs

@@ -20,13 +20,14 @@
 using System.IO;
 using NodaTime;
 using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
 
 namespace QuantConnect.DataSource
 {
     /// <summary>
     /// Universe Selection helper class for Brain Sentiment dataset
     /// </summary>
-    public class BrainSentimentIndicatorUniverse : BaseData
+    public class BrainSentimentIndicatorUniverse : BaseDataCollection
     {
         private static readonly TimeSpan _period = TimeSpan.FromDays(1);
 
@@ -103,7 +104,8 @@ public override SubscriptionDataSource GetSource(SubscriptionDataConfig config,
                     "universe",
                     $"{date:yyyyMMdd}.csv"
                 ),
-                SubscriptionTransportMedium.LocalFile
+                SubscriptionTransportMedium.LocalFile,
+                FileFormat.FoldingCollection
             );
         }
 
@@ -160,6 +162,31 @@ public override string ToString()
                     {SentimentalBuzzVolume30Days}";
         }
 
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public override BaseData Clone()
+        {
+            return new BrainSentimentIndicatorUniverse()
+            {
+                Symbol = Symbol,
+                Time = Time,
+                Data = Data,
+                Value = Value,
+
+                TotalArticleMentions7Days = TotalArticleMentions7Days,
+                SentimentalArticleMentions7Days = SentimentalArticleMentions7Days,
+                Sentiment7Days = Sentiment7Days,
+                TotalBuzzVolume7Days = TotalBuzzVolume7Days,
+                SentimentalBuzzVolume7Days = SentimentalBuzzVolume7Days,
+                TotalArticleMentions30Days = TotalArticleMentions30Days,
+                SentimentalArticleMentions30Days = SentimentalArticleMentions30Days,
+                Sentiment30Days = Sentiment30Days,
+                TotalBuzzVolume30Days = TotalBuzzVolume30Days,
+                SentimentalBuzzVolume30Days = SentimentalBuzzVolume30Days
+            };
+        }
+
         /// <summary>
         /// Gets the default resolution for this data and security type
         /// </summary>
@@ -182,7 +209,6 @@ public override List<Resolution> SupportedResolutions()
         /// <returns>The <see cref="T:NodaTime.DateTimeZone" /> of this data type</returns>
         public override DateTimeZone DataTimeZone()
         {
-            // TODO: AddUniverse does not take this into account
             return TimeZones.Utc;
         }
     }

BrainSentimentIndicatorUniverseSelectionAlgorithm.cs

@@ -34,9 +34,9 @@ public override void Initialize()
             SetCash(100000);
 
             // add a custom universe data source (defaults to usa-equity)
-            AddUniverse<BrainSentimentIndicatorUniverse>("BrainSentimentIndicatorUniverse", Resolution.Daily, data =>
+            var universe = AddUniverse<BrainSentimentIndicatorUniverse>(data =>
             {
-                foreach (var datum in data)
+                foreach (BrainSentimentIndicatorUniverse datum in data)
                 {
                     Log($@"{datum.Symbol},
                     {datum.TotalArticleMentions7Days},
@@ -52,10 +52,24 @@ public override void Initialize()
                 }
 
                 // define our selection criteria
-                return from d in data 
-                    where d.TotalArticleMentions7Days > 0m && d.Sentiment7Days > 0m
-                    select d.Symbol;
+                return from BrainSentimentIndicatorUniverse d in data
+                       where d.TotalArticleMentions7Days > 0m && d.Sentiment7Days > 0m
+                       select d.Symbol;
             });
+
+            var history = History(universe, 1).ToList();
+            if (history.Count != 1)
+            {
+                throw new System.Exception($"Unexpected historical data count!");
+            }
+            foreach (var dataForDate in history)
+            {
+                var coarseData = dataForDate.ToList();
+                if (coarseData.Count < 1000)
+                {
+                    throw new System.Exception($"Unexpected historical universe data!");
+                }
+            }
         }
 
         public override void OnSecuritiesChanged(SecurityChanges changes)

BrainSentimentIndicatorUniverseSelectionAlgorithm.py

@@ -13,7 +13,7 @@
 
 from AlgorithmImports import *
 
-class BrainSentimentIndicatorUniverseAlgorithm(QCAlgorithm): 
+class BrainSentimentIndicatorUniverseAlgorithm(QCAlgorithm):
     def Initialize(self):
         # Data ADDED via universe selection is added with Daily resolution.
         self.UniverseSettings.Resolution = Resolution.Daily
@@ -23,7 +23,15 @@ def Initialize(self):
         self.SetCash(100000)
 
         # add a custom universe data source (defaults to usa-equity)
-        self.AddUniverse(BrainSentimentIndicatorUniverse, "BrainSentimentIndicatorUniverse", Resolution.Daily, self.UniverseSelection)
+        universe = self.AddUniverse(BrainSentimentIndicatorUniverse, self.UniverseSelection)
+
+        history = self.History(universe, TimeSpan(1, 0, 0, 0))
+        if len(history) != 1:
+            raise ValueError(f"Unexpected history count {len(history)}! Expected 1")
+
+        for dataForDate in history:
+            if len(dataForDate) < 1000:
+                raise ValueError(f"Unexpected historical universe data!")
 
     def UniverseSelection(self, data):
         for datum in data:
@@ -41,7 +49,7 @@ def UniverseSelection(self, data):
 
         # define our selection criteria
         return [d.Symbol for d in data \
-                    if d.TotalArticleMentions7Days > 0 \
+                    if d.TotalArticleMentions7Days and d.TotalArticleMentions7Days > 0 \
                     and d.Sentiment7Days]
 
     def OnSecuritiesChanged(self, changes):

BrainStockRankingUniverse.cs

@@ -20,13 +20,14 @@
 using System.IO;
 using NodaTime;
 using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
 
 namespace QuantConnect.DataSource
 {
     /// <summary>
     /// Universe Selection helper class for Brain ML Stock Ranking dataset
     /// </summary>
-    public class BrainStockRankingUniverse : BaseData
+    public class BrainStockRankingUniverse : BaseDataCollection
     {
         private static readonly TimeSpan _period = TimeSpan.FromDays(1);
 
@@ -78,7 +79,8 @@ public override SubscriptionDataSource GetSource(SubscriptionDataConfig config,
                     "universe",
                     $"{date:yyyyMMdd}.csv"
                 ),
-                SubscriptionTransportMedium.LocalFile
+                SubscriptionTransportMedium.LocalFile,
+                FileFormat.FoldingCollection
             );
         }
 
@@ -119,6 +121,26 @@ public override string ToString()
             return $"{Symbol},{Rank2Days},{Rank3Days},{Rank5Days},{Rank10Days},{Rank21Days}";
         }
 
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public override BaseData Clone()
+        {
+            return new BrainStockRankingUniverse()
+            {
+                Symbol = Symbol,
+                Time = Time,
+                Data = Data,
+                Value = Value,
+
+                Rank2Days = Rank2Days,
+                Rank3Days = Rank3Days,
+                Rank5Days = Rank5Days,
+                Rank10Days = Rank10Days,
+                Rank21Days = Rank21Days
+            };
+        }
+
         /// <summary>
         /// Gets the default resolution for this data and security type
         /// </summary>
@@ -141,7 +163,6 @@ public override List<Resolution> SupportedResolutions()
         /// <returns>The <see cref="T:NodaTime.DateTimeZone" /> of this data type</returns>
         public override DateTimeZone DataTimeZone()
         {
-            // TODO: AddUniverse does not take this into account
             return TimeZones.Utc;
         }
     }