[JA DateTimeV2] Merged refinements

.NET/Microsoft.Recognizers.Text.DateTime/Chinese/Extractors/ChineseDurationExtractorConfiguration.cs

@@ -56,6 +56,8 @@ public ChineseDurationExtractorConfiguration(IDateTimeOptionsConfiguration confi
 
         public Dictionary<string, long> UnitValueMap { get; }
 
+        public Dictionary<Regex, Regex> AmbiguityDurationFiltersDict => null;
+
         Regex ICJKDurationExtractorConfiguration.DurationUnitRegex => DurationUnitRegex;
 
         Regex ICJKDurationExtractorConfiguration.DurationConnectorRegex => DurationConnectorRegex;

.NET/Microsoft.Recognizers.Text.DateTime/Constants.cs

@@ -53,6 +53,7 @@ public static class Constants
 
         // AmPm time representation for time parser
         public const string Comment_AmPm = "ampm";
+        public const string Comment_Am = "am";
 
         // Prefix early/late for time parser
         public const string Comment_Early = "early";
@@ -267,6 +268,7 @@ public static class Constants
         public const string TimexFuzzyDay = "XX";
         public const string DateTimexConnector = "-";
         public const string TimeTimexConnector = ":";
+        public const string TimexSeparator = ",";
         public const string GeneralPeriodPrefix = "P";
         public const string TimeTimexPrefix = "T";
 

.NET/Microsoft.Recognizers.Text.DateTime/Extractors/CJK/BaseCJKDurationExtractor.cs

@@ -2,7 +2,8 @@
 // Licensed under the MIT License.
 
 using System.Collections.Generic;
-
+using System.Linq;
+using System.Text.RegularExpressions;
 using Microsoft.Recognizers.Text.Utilities;
 using DateObject = System.DateTime;
 
@@ -54,9 +55,32 @@ public List<ExtractResult> Extract(string source, DateObject referenceTime)
                 res = MergeMultipleDuration(source, res);
             }
 
+            res = FilterAmbiguity(res, source);
+
             return res;
         }
 
+        private List<ExtractResult> FilterAmbiguity(List<ExtractResult> extractResults, string text)
+        {
+            if (this.config.AmbiguityDurationFiltersDict != null)
+            {
+                foreach (var regex in this.config.AmbiguityDurationFiltersDict)
+                {
+                    foreach (var extractResult in extractResults)
+                    {
+                        if (regex.Key.IsMatch(text))
+                        {
+                            var matches = regex.Value.Matches(text).Cast<Match>();
+                            extractResults = extractResults.Where(er => !matches.Any(m => m.Index < er.Start + er.Length && m.Index + m.Length > er.Start))
+                                .ToList();
+                        }
+                    }
+                }
+            }
+
+            return extractResults;
+        }
+
         private List<ExtractResult> MergeMultipleDuration(string text, List<ExtractResult> extractorResults)
         {
             if (extractorResults.Count <= 1)

.NET/Microsoft.Recognizers.Text.DateTime/Extractors/CJK/BaseCJKMergedDateTimeExtractor.cs

@@ -129,6 +129,17 @@ private void AddMod(List<ExtractResult> ers, string text)
                     er.Metadata = AssignModMetadata(er.Metadata);
                 }
 
+                match = this.config.UntilRegex.MatchBegin(afterStr, trim: true);
+
+                if (match.Success)
+                {
+                    var modLength = match.Index + match.Length;
+                    er.Length += modLength;
+                    er.Text = text.Substring(er.Start ?? 0, er.Length ?? 0);
+
+                    er.Metadata = AssignModMetadata(er.Metadata);
+                }
+
                 match = this.config.SincePrefixRegex.MatchEnd(beforeStr, trim: true);
 
                 if (match.Success && AmbiguousRangeChecker(beforeStr, text, er))

.NET/Microsoft.Recognizers.Text.DateTime/Extractors/CJK/ICJKDurationExtractorConfiguration.cs

@@ -32,5 +32,7 @@ public interface ICJKDurationExtractorConfiguration : IDateTimeOptionsConfigurat
 
         Dictionary<string, long> UnitValueMap { get; }
 
+        Dictionary<Regex, Regex> AmbiguityDurationFiltersDict { get; }
+
     }
 }

.NET/Microsoft.Recognizers.Text.DateTime/Japanese/Extractors/JapaneseDurationExtractorConfiguration.cs

@@ -6,8 +6,8 @@
 using System.Globalization;
 using System.Linq;
 using System.Text.RegularExpressions;
-
 using Microsoft.Recognizers.Definitions.Japanese;
+using Microsoft.Recognizers.Definitions.Utilities;
 using Microsoft.Recognizers.Text.NumberWithUnit;
 using Microsoft.Recognizers.Text.NumberWithUnit.Japanese;
 
@@ -48,6 +48,8 @@ public JapaneseDurationExtractorConfiguration(IDateTimeOptionsConfiguration conf
 
             UnitMap = DateTimeDefinitions.ParserConfigurationUnitMap.ToDictionary(k => k.Key, k => k.Value);
             UnitValueMap = DateTimeDefinitions.DurationUnitValueMap;
+            AmbiguityDurationFiltersDict = DefinitionLoader.LoadAmbiguityFilters(DateTimeDefinitions.AmbiguityDurationFiltersDict);
+
         }
 
         public IExtractor InternalExtractor { get; }
@@ -56,6 +58,8 @@ public JapaneseDurationExtractorConfiguration(IDateTimeOptionsConfiguration conf
 
         public Dictionary<string, long> UnitValueMap { get; }
 
+        public Dictionary<Regex, Regex> AmbiguityDurationFiltersDict { get; }
+
         Regex ICJKDurationExtractorConfiguration.DurationUnitRegex => DurationUnitRegex;
 
         Regex ICJKDurationExtractorConfiguration.DurationConnectorRegex => DurationConnectorRegex;

.NET/Microsoft.Recognizers.Text.DateTime/Korean/Extractors/KoreanDurationExtractorConfiguration.cs

@@ -56,6 +56,8 @@ public KoreanDurationExtractorConfiguration(IDateTimeOptionsConfiguration config
 
         public Dictionary<string, long> UnitValueMap { get; }
 
+        public Dictionary<Regex, Regex> AmbiguityDurationFiltersDict => null;
+
         Regex ICJKDurationExtractorConfiguration.DurationUnitRegex => DurationUnitRegex;
 
         Regex ICJKDurationExtractorConfiguration.DurationConnectorRegex => DurationConnectorRegex;

.NET/Microsoft.Recognizers.Text.DateTime/Parsers/CJK/BaseCJKDateParser.cs

@@ -318,8 +318,9 @@ protected DateTimeResolutionResult ParseImplicitDate(string text, DateObject ref
             // handle "明日から3週間" (3 weeks from tomorrow)
             var durationResult = this.config.DurationExtractor.Extract(text, referenceDate);
             var unitMatch = this.config.DurationRelativeDurationUnitRegex.Match(text);
+            var isWithin = this.config.DurationRelativeDurationUnitRegex.MatchEnd(text, trim: true).Groups[Constants.WithinGroupName].Success;
 
-            if (exactMatch.Success && unitMatch.Success && (durationResult.Count > 0) &&
+            if ((exactMatch.Success || isWithin) && unitMatch.Success && (durationResult.Count > 0) &&
                 string.IsNullOrEmpty(unitMatch.Groups["few"].Value))
             {
                 var pr = this.config.DurationParser.Parse(durationResult[0], referenceDate);

.NET/Microsoft.Recognizers.Text.DateTime/Parsers/CJK/BaseCJKDateTimeParser.cs

@@ -158,6 +158,14 @@ private DateTimeResolutionResult MergeDateAndTime(string text, DateObject refere
             var pastDate = (DateObject)((DateTimeResolutionResult)pr1.Value).PastValue;
             var time = (DateObject)((DateTimeResolutionResult)pr2.Value).FutureValue;
 
+            // handle cases with time like 25時 which resolve to the next day
+            var timexHour = TimexUtility.ParseHourFromTimeTimex(pr2.TimexStr);
+            if (timexHour > Constants.DayHourCount)
+            {
+                futureDate = futureDate.AddDays(1);
+                pastDate = pastDate.AddDays(1);
+            }
+
             var hour = time.Hour;
             var min = time.Minute;
             var sec = time.Second;
@@ -178,7 +186,6 @@ private DateTimeResolutionResult MergeDateAndTime(string text, DateObject refere
                 timeStr = timeStr.Substring(0, timeStr.Length - 4);
             }
 
-            timeStr = "T" + hour.ToString("D2", CultureInfo.InvariantCulture) + timeStr.Substring(3);
             ret.Timex = pr1.TimexStr + timeStr;
 
             var val = (DateTimeResolutionResult)pr2.Value;

.NET/Microsoft.Recognizers.Text.DateTime/Parsers/CJK/BaseCJKDateTimePeriodParser.cs

@@ -126,15 +126,31 @@ private DateTimeResolutionResult MergeDateAndTimePeriod(string text, DateObject
             var futureDate = (DateObject)((DateTimeResolutionResult)pr1.Value).FutureValue;
             var pastDate = (DateObject)((DateTimeResolutionResult)pr1.Value).PastValue;
 
+            // handle cases with time like 25時 which resolve to the next day
+            var swiftDay = 0;
+            var timexHours = TimexUtility.ParseHoursFromTimePeriodTimex(pr2.TimexStr);
+            if (timexHours.Item1 > Constants.DayHourCount)
+            {
+                pastDate = pastDate.AddDays(1);
+                futureDate = futureDate.AddDays(1);
+            }
+            else if (timexHours.Item2 > Constants.DayHourCount)
+            {
+                swiftDay++;
+            }
+
+            var pastDate1 = pastDate.AddDays(swiftDay);
+            var futureDate1 = futureDate.AddDays(swiftDay);
+
             ret.FutureValue =
                 new Tuple<DateObject, DateObject>(
                     DateObject.MinValue.SafeCreateFromValue(futureDate.Year, futureDate.Month, futureDate.Day, beginTime.Hour, beginTime.Minute, beginTime.Second),
-                    DateObject.MinValue.SafeCreateFromValue(futureDate.Year, futureDate.Month, futureDate.Day, endTime.Hour, endTime.Minute, endTime.Second));
+                    DateObject.MinValue.SafeCreateFromValue(futureDate1.Year, futureDate1.Month, futureDate1.Day, endTime.Hour, endTime.Minute, endTime.Second));
 
             ret.PastValue =
                 new Tuple<DateObject, DateObject>(
                     DateObject.MinValue.SafeCreateFromValue(pastDate.Year, pastDate.Month, pastDate.Day, beginTime.Hour, beginTime.Minute, beginTime.Second),
-                    DateObject.MinValue.SafeCreateFromValue(pastDate.Year, pastDate.Month, pastDate.Day, endTime.Hour, endTime.Minute, endTime.Second));
+                    DateObject.MinValue.SafeCreateFromValue(pastDate1.Year, pastDate1.Month, pastDate1.Day, endTime.Hour, endTime.Minute, endTime.Second));
 
             ret.Timex = TimexUtility.GenerateSplitDateTimePeriodTimex(pr1.TimexStr, pr2.TimexStr);
             ret.Success = !string.IsNullOrEmpty(ret.Timex);

.NET/Microsoft.Recognizers.Text.DateTime/Utilities/TimeFunctions.cs

@@ -79,12 +79,12 @@ public DateTimeResolutionResult PackTimeResult(DateTimeExtra<TimeType> extra, Ti
                 noDesc = false;
             }
 
-            int hour = timeResult.Hour > 0 ? timeResult.Hour % Constants.DayHourCount : 0,
+            // Hours > 24 (e.g. 25時 which resolves to the next day) are kept unnormalized in the timex
+            // to avoid ambiguity in other entities. For example, "on the 30th at 25" is resolved to
+            // "XXXX-XX-30T25" because with "XXXX-XX-30+1T01" it is not known if the day should be "31" or "01".
+            int hour = timeResult.Hour > 0 && timeResult.Hour != Constants.DayHourCount ? timeResult.Hour : 0,
                 min = timeResult.Minute > 0 ? timeResult.Minute : 0,
-                second = timeResult.Second > 0 ? timeResult.Second : 0,
-                day = referenceTime.Day,
-                month = referenceTime.Month,
-                year = referenceTime.Year;
+                second = timeResult.Second > 0 ? timeResult.Second : 0;
 
             var dateTimeResult = new DateTimeResolutionResult();
 
@@ -109,6 +109,18 @@ public DateTimeResolutionResult PackTimeResult(DateTimeExtra<TimeType> extra, Ti
                 build.Append(":" + second.ToString("D2", CultureInfo.InvariantCulture));
             }
 
+            // handle cases with time like 25時 (the hour is normalized in the past/future values)
+            if (timeResult.Hour > Constants.DayHourCount)
+            {
+                hour = timeResult.Hour - Constants.DayHourCount;
+                referenceTime = referenceTime.AddDays(1);
+                if (noDesc)
+                {
+                    dateTimeResult.Comment = Constants.Comment_Am;
+                    noDesc = false;
+                }
+            }
+
             if (noDesc && hour <= Constants.HalfDayHourCount)
             {
                 // build.Append("ampm");
@@ -117,6 +129,7 @@ public DateTimeResolutionResult PackTimeResult(DateTimeExtra<TimeType> extra, Ti
 
             dateTimeResult.Timex = build.ToString();
 
+            int day = referenceTime.Day, month = referenceTime.Month, year = referenceTime.Year;
             dateTimeResult.FutureValue = dateTimeResult.PastValue =
                 DateObject.MinValue.SafeCreateFromValue(year, month, day, hour, min, second);
             dateTimeResult.Success = true;

.NET/Microsoft.Recognizers.Text.DateTime/Utilities/TimePeriodFunctions.cs

@@ -91,31 +91,50 @@ public static DateTimeResolutionResult Handle(IDateTimeParser timeParser, DateTi
 
             int day = refTime.Day,
                 month = refTime.Month,
-                year = refTime.Year;
+                year = refTime.Year,
+                rightSwiftDay = 0,
+                leftSwiftDay = 0;
 
             // determine if the right side time is smaller than the left side, if yes, add one day
             int hour = leftResult.Hour > 0 ? leftResult.Hour : 0,
                 min = leftResult.Minute > 0 ? leftResult.Minute : 0,
                 second = leftResult.Second > 0 ? leftResult.Second : 0;
 
+            // handle cases with time like 25時 which resolve to the next day
+            if (hour > Constants.DayHourCount)
+            {
+                hour -= Constants.DayHourCount;
+                leftSwiftDay++;
+            }
+
             var leftTime = DateObject.MinValue.SafeCreateFromValue(year, month, day, hour, min, second);
 
             hour = rightResult.Hour > 0 ? rightResult.Hour : 0;
             min = rightResult.Minute > 0 ? rightResult.Minute : 0;
             second = rightResult.Second > 0 ? rightResult.Second : 0;
 
+            // handle cases with time like 25時 which resolve to the next day
+            if (hour > Constants.DayHourCount)
+            {
+                hour -= Constants.DayHourCount;
+                rightSwiftDay++;
+            }
+
             var rightTime = DateObject.MinValue.SafeCreateFromValue(year, month, day, hour, min, second);
 
-            if (rightTime.Hour < leftTime.Hour)
+            if (rightResult.Hour < leftResult.Hour)
             {
                 rightTime = rightTime.AddDays(1);
             }
 
-            ret.FutureValue = ret.PastValue = new Tuple<DateObject, DateObject>(leftTime, rightTime);
-
             var leftTimex = BuildTimex(leftResult);
             var rightTimex = BuildTimex(rightResult);
             ret.Timex = $"({leftTimex},{rightTimex},{BuildSpan(leftResult, rightResult)})";
+
+            leftTime = leftTime.AddDays(leftSwiftDay);
+            rightTime = rightTime.AddDays(rightSwiftDay);
+
+            ret.FutureValue = ret.PastValue = new Tuple<DateObject, DateObject>(leftTime, rightTime);
             return ret;
         }
 

.NET/Microsoft.Recognizers.Text.DateTime/Utilities/TimexUtility.cs

@@ -569,6 +569,30 @@ public static float ParseNumberFromDurationTimex(string timex)
             return float.Parse(numberStr);
         }
 
+        public static int ParseHourFromTimeTimex(string timex)
+        {
+            var start = timex.IndexOf(Constants.TimeTimexPrefix) + 1;
+            var end = timex.IndexOf(Constants.TimeTimexConnector);
+            end = end > 0 ? end : timex.Length;
+            var hourStr = timex.Substring(start, end - start);
+            int.TryParse(hourStr, out int hour);
+
+            return hour;
+        }
+
+        public static Tuple<int, int> ParseHoursFromTimePeriodTimex(string timex)
+        {
+            int hour1 = 0, hour2 = 0;
+            var timeList = timex.Split(Constants.TimexSeparator[0]);
+            if (timeList.Length > 2)
+            {
+                hour1 = ParseHourFromTimeTimex(timeList[0]);
+                hour2 = ParseHourFromTimeTimex(timeList[1]);
+            }
+
+            return new Tuple<int, int>(hour1, hour2);
+        }
+
         private static bool IsTimeDurationTimex(string timex)
         {
             return timex.StartsWith($"{Constants.GeneralPeriodPrefix}{Constants.TimeTimexPrefix}", StringComparison.Ordinal);