[.NET] Integration of new mode termed "TasksMode" in DatetimeOption.cs (

#2972) * Addition of TasksMode in DatetTimeOptions * Add sekelton infra for new mode and tests * [TasksMode] Don't classify four digit numbers by themselves as daterange under tasksmode * Updated DatetimeTasksMode.json * Added all platforms in "NotSupported" tags for tasks mode * Added new specs in DatetimeModel for comparing behavior of test cases under default mode and tasksmode
microsoft · Jun 7, 2022 · 41a5f12 · 41a5f12
1 parent b6d8443
commit 41a5f12
Show file tree

Hide file tree

Showing 19 changed files with 240 additions and 5 deletions.
diff --git a/.NET/Microsoft.Recognizers.Text.DataDrivenTests/DateTime/TestDateTime_English.cs b/.NET/Microsoft.Recognizers.Text.DataDrivenTests/DateTime/TestDateTime_English.cs
@@ -257,5 +257,13 @@ public void DateTimeModelExperimentalMode(TestModel testSpec)
         {
             TestDateTimeAlt(testSpec);
         }
+
+        [NetCoreTestDataSource]
+        [TestMethod]
+        public void DateTimeModelTasksMode(TestModel testSpec)
+        {
+            TestDateTimeAlt(testSpec);
+        }
+
     }
 }
diff --git a/.NET/Microsoft.Recognizers.Text.DataDrivenTests/TestHelpers.cs b/.NET/Microsoft.Recognizers.Text.DataDrivenTests/TestHelpers.cs
@@ -53,6 +53,7 @@ public enum Models
         DateTimeExtendedTypes,
         DateTimeComplexCalendar,
         DateTimeExperimentalMode,
+        DateTimeTasksMode,
         PhoneNumber,
         IpAddress,
         Mention,
@@ -121,6 +122,7 @@ public static class TestContextExtensions
             { Models.DateTimeExtendedTypes, (test, culture) => DateTimeRecognizer.RecognizeDateTime(test.Input, culture, DateTimeOptions.ExtendedTypes, refTime: test.GetReferenceDateTime(), fallbackToDefaultCulture: false) },
             { Models.DateTimeComplexCalendar, (test, culture) => DateTimeRecognizer.RecognizeDateTime(test.Input, culture, DateTimeOptions.ExtendedTypes | DateTimeOptions.CalendarMode | DateTimeOptions.EnablePreview, refTime: test.GetReferenceDateTime(), fallbackToDefaultCulture: false) },
             { Models.DateTimeExperimentalMode, (test, culture) => DateTimeRecognizer.RecognizeDateTime(test.Input, culture, DateTimeOptions.ExperimentalMode, refTime: test.GetReferenceDateTime(), fallbackToDefaultCulture: false) },
+            { Models.DateTimeTasksMode, (test, culture) => DateTimeRecognizer.RecognizeDateTime(test.Input, culture, options: DateTimeOptions.TasksMode, refTime: test.GetReferenceDateTime(), fallbackToDefaultCulture: false) },
             { Models.PhoneNumber, (test, culture) => SequenceRecognizer.RecognizePhoneNumber(test.Input, culture, fallbackToDefaultCulture: false) },
             { Models.IpAddress, (test, culture) => SequenceRecognizer.RecognizeIpAddress(test.Input, culture, fallbackToDefaultCulture: false) },
             { Models.Mention, (test, culture) => SequenceRecognizer.RecognizeMention(test.Input, culture, fallbackToDefaultCulture: false) },

diff --git a/...crosoft.Recognizers.Text.DateTime/Arabic/Extractors/ArabicMergedExtractorConfiguration.cs b/...crosoft.Recognizers.Text.DateTime/Arabic/Extractors/ArabicMergedExtractorConfiguration.cs
@@ -52,6 +52,9 @@ public class ArabicMergedExtractorConfiguration : BaseDateTimeOptionsConfigurati
         public static readonly Regex FailFastRegex =
             new Regex(DateTimeDefinitions.FailFastRegex, RegexFlags | RegexOptions.Compiled);
 
+        public static readonly Regex YearRegex =
+           new Regex(DateTimeDefinitions.YearRegex, RegexFlags);
+
         public static readonly Regex[] TermFilterRegexes =
         {
             // one on one
@@ -162,6 +165,8 @@ public ArabicMergedExtractorConfiguration(IDateTimeOptionsConfiguration config)
 
         Regex IMergedExtractorConfiguration.UnspecificTimePeriodRegex => UnspecificTimePeriodRegex;
 
+        Regex IMergedExtractorConfiguration.YearRegex => YearRegex;
+
         Regex IMergedExtractorConfiguration.FailFastRegex => FailFastRegex;
 
         IEnumerable<Regex> IMergedExtractorConfiguration.TermFilterRegexes => TermFilterRegexes;

diff --git a/.NET/Microsoft.Recognizers.Text.DateTime/DateTimeOptions.cs b/.NET/Microsoft.Recognizers.Text.DateTime/DateTimeOptions.cs
@@ -38,6 +38,11 @@ public enum DateTimeOptions
         /// </summary>
         NoProtoCache = 16,
 
+        /// <summary>
+        /// TasksMode, specific functionality that changes default behaviour for business reasons.
+        /// </summary>
+        TasksMode = 1048576, // 2 ^20
+
         /// <summary>
         /// FailFast, mode that aborts extraction/tagging quickly for non-entity cases. May be removed later.
         /// </summary>

diff --git a/...Microsoft.Recognizers.Text.DateTime/Dutch/Extractors/DutchMergedExtractorConfiguration.cs b/...Microsoft.Recognizers.Text.DateTime/Dutch/Extractors/DutchMergedExtractorConfiguration.cs
@@ -52,6 +52,9 @@ public class DutchMergedExtractorConfiguration : BaseDateTimeOptionsConfiguratio
         public static readonly Regex PotentialAmbiguousRangeRegex =
             new Regex(DateTimeDefinitions.PotentialAmbiguousRangeRegex, RegexFlags);
 
+        public static readonly Regex YearRegex =
+           new Regex(DateTimeDefinitions.YearRegex, RegexFlags);
+
         public static readonly Regex[] TermFilterRegexes =
         {
             // one on one
@@ -152,6 +155,8 @@ public DutchMergedExtractorConfiguration(IDateTimeOptionsConfiguration config)
 
         Regex IMergedExtractorConfiguration.UnspecificTimePeriodRegex => null;
 
+        Regex IMergedExtractorConfiguration.YearRegex => YearRegex;
+
         public Regex FailFastRegex { get; } = null;
 
         IEnumerable<Regex> IMergedExtractorConfiguration.TermFilterRegexes => TermFilterRegexes;

diff --git a/...osoft.Recognizers.Text.DateTime/English/Extractors/EnglishMergedExtractorConfiguration.cs b/...osoft.Recognizers.Text.DateTime/English/Extractors/EnglishMergedExtractorConfiguration.cs
@@ -53,6 +53,10 @@ public class EnglishMergedExtractorConfiguration : BaseDateTimeOptionsConfigurat
         public static readonly Regex FailFastRegex =
             new Regex(DateTimeDefinitions.FailFastRegex, RegexFlags | RegexOptions.Compiled);
 
+        // Used to skip only year references in a text in TasksMode
+        public static readonly Regex YearRegex =
+            new Regex(DateTimeDefinitions.YearRegex, RegexFlags);
+
         public static readonly Regex[] TermFilterRegexes =
         {
             // one on one
@@ -164,6 +168,8 @@ public EnglishMergedExtractorConfiguration(IDateTimeOptionsConfiguration config)
 
         Regex IMergedExtractorConfiguration.UnspecificTimePeriodRegex => UnspecificTimePeriodRegex;
 
+        Regex IMergedExtractorConfiguration.YearRegex => YearRegex;
+
         Regex IMergedExtractorConfiguration.FailFastRegex => FailFastRegex;
 
         IEnumerable<Regex> IMergedExtractorConfiguration.TermFilterRegexes => TermFilterRegexes;

diff --git a/.NET/Microsoft.Recognizers.Text.DateTime/Extractors/BaseMergedDateTimeExtractor.cs b/.NET/Microsoft.Recognizers.Text.DateTime/Extractors/BaseMergedDateTimeExtractor.cs
@@ -223,6 +223,14 @@ private void AddTo(List<ExtractResult> dst, List<ExtractResult> src, string text
                     }
                 }
 
+                if ((config.Options & DateTimeOptions.TasksMode) != 0)
+                {
+                    if (ShouldSkipOnlyYear(result))
+                    {
+                        continue;
+                    }
+                }
+
                 var isFound = false;
                 var overlapIndexes = new List<int>();
                 var firstIndex = -1;
@@ -270,6 +278,15 @@ private bool ShouldSkipFromToMerge(ExtractResult er)
             return config.FromToRegex.IsMatch(er.Text);
         }
 
+        /*Under TasksMode: Should not treat a four-digit number as a daterange if the input text does not include a month or year reference.
+          It should not treat 2005 as a daterange in statements like "Milk 2005."
+         (The year 2005 should be treated as a number only.)
+        */
+        private bool ShouldSkipOnlyYear(ExtractResult er)
+        {
+            return config.YearRegex.Match(er.Text).Value == er.Text;
+        }
+
         private List<ExtractResult> FilterUnspecificDatePeriod(List<ExtractResult> ers)
         {
             ers.RemoveAll(o => this.config.UnspecificDatePeriodRegex.IsMatch(o.Text));

diff --git a/.NET/Microsoft.Recognizers.Text.DateTime/Extractors/IMergedExtractorConfiguration.cs b/.NET/Microsoft.Recognizers.Text.DateTime/Extractors/IMergedExtractorConfiguration.cs
@@ -67,6 +67,8 @@ public interface IMergedExtractorConfiguration : IDateTimeOptionsConfiguration
         // Regex to act as umbrella for key terms so that sentences that clearly don't have entities can be rejected quickly
         Regex FailFastRegex { get; }
 
+        Regex YearRegex { get; }
+
         StringMatcher SuperfluousWordMatcher { get; }
 
         Dictionary<Regex, Regex> AmbiguityFiltersDict { get; }

diff --git a/...crosoft.Recognizers.Text.DateTime/French/Extractors/FrenchMergedExtractorConfiguration.cs b/...crosoft.Recognizers.Text.DateTime/French/Extractors/FrenchMergedExtractorConfiguration.cs
@@ -57,6 +57,9 @@ public class FrenchMergedExtractorConfiguration : BaseDateTimeOptionsConfigurati
 
         public static readonly StringMatcher SuperfluousWordMatcher = new StringMatcher();
 
+        public static readonly Regex YearRegex =
+            new Regex(DateTimeDefinitions.YearRegex, RegexFlags);
+
         private const RegexOptions RegexFlags = RegexOptions.Singleline | RegexOptions.ExplicitCapture;
 
         public FrenchMergedExtractorConfiguration(IDateTimeOptionsConfiguration config)
@@ -141,6 +144,8 @@ public FrenchMergedExtractorConfiguration(IDateTimeOptionsConfiguration config)
 
         Regex IMergedExtractorConfiguration.UnspecificTimePeriodRegex => null;
 
+        Regex IMergedExtractorConfiguration.YearRegex => YearRegex;
+
         public Regex FailFastRegex { get; } = null;
 
         IEnumerable<Regex> IMergedExtractorConfiguration.TermFilterRegexes => TermFilterRegexes;

diff --git a/...crosoft.Recognizers.Text.DateTime/German/Extractors/GermanMergedExtractorConfiguration.cs b/...crosoft.Recognizers.Text.DateTime/German/Extractors/GermanMergedExtractorConfiguration.cs
@@ -49,6 +49,9 @@ public class GermanMergedExtractorConfiguration : BaseDateTimeOptionsConfigurati
         public static readonly Regex UnspecificDatePeriodRegex =
             new Regex(DateTimeDefinitions.UnspecificDatePeriodRegex, RegexFlags);
 
+        public static readonly Regex YearRegex =
+          new Regex(DateTimeDefinitions.YearRegex, RegexFlags);
+
         public static readonly StringMatcher SuperfluousWordMatcher = new StringMatcher();
 
         public static readonly Regex[] TermFilterRegexes =
@@ -142,6 +145,8 @@ public GermanMergedExtractorConfiguration(IDateTimeOptionsConfiguration config)
 
         Regex IMergedExtractorConfiguration.UnspecificTimePeriodRegex => null;
 
+        Regex IMergedExtractorConfiguration.YearRegex => YearRegex;
+
         public Regex FailFastRegex { get; } = null;
 
         IEnumerable<Regex> IMergedExtractorConfiguration.TermFilterRegexes => TermFilterRegexes;

diff --git a/...Microsoft.Recognizers.Text.DateTime/Hindi/Extractors/HindiMergedExtractorConfiguration.cs b/...Microsoft.Recognizers.Text.DateTime/Hindi/Extractors/HindiMergedExtractorConfiguration.cs
@@ -51,6 +51,9 @@ public class HindiMergedExtractorConfiguration : BaseDateTimeOptionsConfiguratio
         public static readonly Regex FailFastRegex =
             new Regex(DateTimeDefinitions.FailFastRegex, RegexFlags | RegexOptions.Compiled);
 
+        public static readonly Regex YearRegex =
+            new Regex(DateTimeDefinitions.YearRegex, RegexFlags);
+
         public static readonly Regex[] TermFilterRegexes =
         {
             // one on one
@@ -141,6 +144,8 @@ public HindiMergedExtractorConfiguration(IDateTimeOptionsConfiguration config)
 
         Regex IMergedExtractorConfiguration.UnspecificTimePeriodRegex => null;
 
+        Regex IMergedExtractorConfiguration.YearRegex => YearRegex;
+
         Regex IMergedExtractorConfiguration.FailFastRegex => FailFastRegex;
 
         IEnumerable<Regex> IMergedExtractorConfiguration.TermFilterRegexes => TermFilterRegexes;

diff --git a/...osoft.Recognizers.Text.DateTime/Italian/Extractors/ItalianMergedExtractorConfiguration.cs b/...osoft.Recognizers.Text.DateTime/Italian/Extractors/ItalianMergedExtractorConfiguration.cs
@@ -56,6 +56,9 @@ public class ItalianMergedExtractorConfiguration : BaseDateTimeOptionsConfigurat
 
         public static readonly StringMatcher SuperfluousWordMatcher = new StringMatcher();
 
+        public static readonly Regex YearRegex =
+          new Regex(DateTimeDefinitions.YearRegex, RegexFlags);
+
         private const RegexOptions RegexFlags = RegexOptions.Singleline | RegexOptions.ExplicitCapture;
 
         public ItalianMergedExtractorConfiguration(IDateTimeOptionsConfiguration config)
@@ -131,6 +134,8 @@ public ItalianMergedExtractorConfiguration(IDateTimeOptionsConfiguration config)
 
         Regex IMergedExtractorConfiguration.UnspecificTimePeriodRegex => null;
 
+        Regex IMergedExtractorConfiguration.YearRegex => YearRegex;
+
         public Regex FailFastRegex { get; } = null;
 
         IEnumerable<Regex> IMergedExtractorConfiguration.TermFilterRegexes => TermFilterRegexes;

diff --git a/.NET/Microsoft.Recognizers.Text.DateTime/Microsoft.Recognizers.Text.DateTime.xml b/.NET/Microsoft.Recognizers.Text.DateTime/Microsoft.Recognizers.Text.DateTime.xml
diff --git a/...Recognizers.Text.DateTime/Portuguese/Extractors/PortugueseMergedExtractorConfiguration.cs b/...Recognizers.Text.DateTime/Portuguese/Extractors/PortugueseMergedExtractorConfiguration.cs
@@ -53,6 +53,9 @@ public class PortugueseMergedExtractorConfiguration : BaseDateTimeOptionsConfigu
 
         public static readonly StringMatcher SuperfluousWordMatcher = new StringMatcher();
 
+        public static readonly Regex YearRegex =
+            new Regex(DateTimeDefinitions.YearRegex, RegexFlags);
+
         public static readonly Regex[] TermFilterRegexes = System.Array.Empty<Regex>();
 
         private const RegexOptions RegexFlags = RegexOptions.Singleline | RegexOptions.ExplicitCapture;
@@ -139,6 +142,8 @@ public PortugueseMergedExtractorConfiguration(IDateTimeOptionsConfiguration conf
 
         Regex IMergedExtractorConfiguration.UnspecificTimePeriodRegex => null;
 
+        Regex IMergedExtractorConfiguration.YearRegex => YearRegex;
+
         public Regex FailFastRegex { get; } = null;
 
         IEnumerable<Regex> IMergedExtractorConfiguration.TermFilterRegexes => TermFilterRegexes;

diff --git a/...osoft.Recognizers.Text.DateTime/Spanish/Extractors/SpanishMergedExtractorConfiguration.cs b/...osoft.Recognizers.Text.DateTime/Spanish/Extractors/SpanishMergedExtractorConfiguration.cs
@@ -51,6 +51,8 @@ public class SpanishMergedExtractorConfiguration : BaseDateTimeOptionsConfigurat
 
         public static readonly StringMatcher SuperfluousWordMatcher = new StringMatcher();
 
+        public static readonly Regex YearRegex = new Regex(DateTimeDefinitions.YearRegex, RegexFlags);
+
         private const RegexOptions RegexFlags = RegexOptions.Singleline | RegexOptions.ExplicitCapture;
 
         public SpanishMergedExtractorConfiguration(IDateTimeOptionsConfiguration config)
@@ -146,6 +148,8 @@ public SpanishMergedExtractorConfiguration(IDateTimeOptionsConfiguration config)
 
         Regex IMergedExtractorConfiguration.UnspecificTimePeriodRegex => null;
 
+        Regex IMergedExtractorConfiguration.YearRegex => YearRegex;
+
         public Regex FailFastRegex { get; } = null;
 
         IEnumerable<Regex> IMergedExtractorConfiguration.TermFilterRegexes => TermFilterRegexes;

diff --git a/...osoft.Recognizers.Text.DateTime/Swedish/Extractors/SwedishMergedExtractorConfiguration.cs b/...osoft.Recognizers.Text.DateTime/Swedish/Extractors/SwedishMergedExtractorConfiguration.cs
@@ -53,6 +53,9 @@ public class SwedishMergedExtractorConfiguration : BaseDateTimeOptionsConfigurat
         public static readonly Regex FailFastRegex =
             new Regex(DateTimeDefinitions.FailFastRegex, RegexFlags | RegexOptions.Compiled);
 
+        public static readonly Regex YearRegex =
+           new Regex(DateTimeDefinitions.YearRegex, RegexFlags);
+
         public static readonly Regex[] TermFilterRegexes =
         {
             // one on one
@@ -164,6 +167,8 @@ public SwedishMergedExtractorConfiguration(IDateTimeOptionsConfiguration config)
 
         Regex IMergedExtractorConfiguration.UnspecificTimePeriodRegex => UnspecificTimePeriodRegex;
 
+        Regex IMergedExtractorConfiguration.YearRegex => YearRegex;
+
         Regex IMergedExtractorConfiguration.FailFastRegex => FailFastRegex;
 
         IEnumerable<Regex> IMergedExtractorConfiguration.TermFilterRegexes => TermFilterRegexes;

diff --git a/...osoft.Recognizers.Text.DateTime/Turkish/Extractors/TurkishMergedExtractorConfiguration.cs b/...osoft.Recognizers.Text.DateTime/Turkish/Extractors/TurkishMergedExtractorConfiguration.cs
@@ -51,6 +51,9 @@ public class TurkishMergedExtractorConfiguration : BaseDateTimeOptionsConfigurat
         public static readonly Regex FailFastRegex =
             new Regex(DateTimeDefinitions.FailFastRegex, RegexFlags | RegexOptions.Compiled);
 
+        public static readonly Regex YearRegex =
+          new Regex(DateTimeDefinitions.YearRegex, RegexFlags);
+
         public static readonly Regex[] TermFilterRegexes =
         {
             // one on one
@@ -141,6 +144,8 @@ public TurkishMergedExtractorConfiguration(IDateTimeOptionsConfiguration config)
 
         Regex IMergedExtractorConfiguration.UnspecificTimePeriodRegex => null;
 
+        Regex IMergedExtractorConfiguration.YearRegex => YearRegex;
+
         Regex IMergedExtractorConfiguration.FailFastRegex => FailFastRegex;
 
         IEnumerable<Regex> IMergedExtractorConfiguration.TermFilterRegexes => TermFilterRegexes;