From 014f37bab747e71e69f2d1b0dafa8f70dd25b5f0 Mon Sep 17 00:00:00 2001 From: Matthew Wells Date: Fri, 30 Jun 2023 13:09:24 -0700 Subject: [PATCH 1/7] Added new functions, and alias' to PPL, including documentation, and tests Signed-off-by: Matthew Wells --- docs/user/ppl/functions/datetime.rst | 337 +++++++++++++++++- .../sql/ppl/DateTimeFunctionIT.java | 162 ++++++++- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 10 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 10 + 4 files changed, 515 insertions(+), 4 deletions(-) diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 904b99a23b..225796927a 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -627,7 +627,7 @@ Argument type: STRING/DATE/DATETIME/TIMESTAMP Return type: INTEGER -Synonyms: `DAYOFMONTH`_ +Synonyms: `DAYOFMONTH`_, `DAY_OF_MONTH`_ Example:: @@ -675,7 +675,7 @@ Argument type: STRING/DATE/DATETIME/TIMESTAMP Return type: INTEGER -Synonyms: DAY +Synonyms: `DAY`_, `DAY_OF_MONTH`_ Example:: @@ -688,6 +688,31 @@ Example:: +----------------------------------+ +DAY_OF_MONTH +---------- + +Description +>>>>>>>>>>> + +Usage: day_of_month(date) extracts the day of the month for date, in the range 1 to 31. The dates with value 0 such as '0000-00-00' or '2008-00-00' are invalid. + +Argument type: STRING/DATE/DATETIME/TIMESTAMP + +Return type: INTEGER + +Synonyms: `DAY`_, `DAYOFMONTH`_ + +Example:: + + os> source=people | eval `DAY_OF_MONTH(DATE('2020-08-26'))` = DAY_OF_MONTH(DATE('2020-08-26')) | fields `DAY_OF_MONTH(DATE('2020-08-26'))` + fetched rows / total rows = 1/1 + +----------------------------------+ + | DAY_OF_MONTH(DATE('2020-08-26')) | + |----------------------------------| + | 26 | + +----------------------------------+ + + DAYOFWEEK --------- @@ -700,6 +725,8 @@ Argument type: STRING/DATE/DATETIME/TIMESTAMP Return type: INTEGER +Synonyms: `DAY_OF_WEEK`_ + Example:: os> source=people | eval `DAYOFWEEK(DATE('2020-08-26'))` = DAYOFWEEK(DATE('2020-08-26')) | fields `DAYOFWEEK(DATE('2020-08-26'))` @@ -711,6 +738,30 @@ Example:: +---------------------------------+ +DAY_OF_WEEK +--------- + +Description +>>>>>>>>>>> + +Usage: day_of_week(date) returns the weekday index for date (1 = Sunday, 2 = Monday, ..., 7 = Saturday). + +Argument type: STRING/DATE/DATETIME/TIMESTAMP + +Return type: INTEGER + +Synonyms: `DAYOFWEEK`_ + +Example:: + + os> source=people | eval `DAYOFWEEK(DATE('2020-08-26'))` = DAYOFWEEK(DATE('2020-08-26')) | fields `DAYOFWEEK(DATE('2020-08-26'))` + fetched rows / total rows = 1/1 + +---------------------------------+ + | DAYOFWEEK(DATE('2020-08-26')) | + |---------------------------------| + | 4 | + +---------------------------------+ + DAYOFYEAR --------- @@ -724,6 +775,8 @@ Argument type: STRING/DATE/DATETIME/TIMESTAMP Return type: INTEGER +Synonyms: `DAYOFYEAR`_ + Example:: os> source=people | eval `DAYOFYEAR(DATE('2020-08-26'))` = DAYOFYEAR(DATE('2020-08-26')) | fields `DAYOFYEAR(DATE('2020-08-26'))` @@ -735,6 +788,31 @@ Example:: +---------------------------------+ +DAY_OF_YEAR +--------- + +Description +>>>>>>>>>>> + +Usage: day_of_year(date) returns the day of the year for date, in the range 1 to 366. + +Argument type: STRING/DATE/DATETIME/TIMESTAMP + +Return type: INTEGER + +Synonyms: `DAY_OF_YEAR`_ + +Example:: + + os> source=people | eval `DAY_OF_YEAR(DATE('2020-08-26'))` = DAY_OF_YEAR(DATE('2020-08-26')) | fields `DAY_OF_YEAR(DATE('2020-08-26'))` + fetched rows / total rows = 1/1 + +---------------------------------+ + | DAY_OF_YEAR(DATE('2020-08-26')) | + |---------------------------------| + | 239 | + +---------------------------------+ + + FROM_DAYS --------- @@ -806,6 +884,8 @@ Argument type: STRING/TIME/DATETIME/TIMESTAMP Return type: INTEGER +Synonyms: `HOUR_OF_DAY`_ + Example:: os> source=people | eval `HOUR(TIME('01:02:03'))` = HOUR(TIME('01:02:03')) | fields `HOUR(TIME('01:02:03'))` @@ -817,6 +897,31 @@ Example:: +--------------------------+ +HOUR_OF_DAY +---- + +Description +>>>>>>>>>>> + +Usage: hour_of_day(time) extracts the hour value for time. Different from the time of day value, the time value has a large range and can be greater than 23, so the return value of hour_of_day(time) can be also greater than 23. + +Argument type: STRING/TIME/DATETIME/TIMESTAMP + +Return type: INTEGER + +Synonyms: `HOUR`_ + +Example:: + + os> source=people | eval `HOUR_OF_DAY(TIME('01:02:03'))` = HOUR_OF_DAY(TIME('01:02:03')) | fields `HOUR_OF_DAY(TIME('01:02:03'))` + fetched rows / total rows = 1/1 + +---------------------------------+ + | HOUR_OF_DAY(TIME('01:02:03')) | + |---------------------------------| + | 1 | + +---------------------------------+ + + LOCALTIMESTAMP -------------- @@ -956,6 +1061,8 @@ Argument type: STRING/TIME/DATETIME/TIMESTAMP Return type: INTEGER +Synonyms: `MINUTE_OF_HOUR`_ + Example:: os> source=people | eval `MINUTE(TIME('01:02:03'))` = MINUTE(TIME('01:02:03')) | fields `MINUTE(TIME('01:02:03'))` @@ -967,6 +1074,54 @@ Example:: +----------------------------+ +MINUTE_OF_HOUR +------ + +Description +>>>>>>>>>>> + +Usage: minute(time) returns the minute for time, in the range 0 to 59. + +Argument type: STRING/TIME/DATETIME/TIMESTAMP + +Return type: INTEGER + +Synonyms: `MINUTE`_ + +Example:: + + os> source=people | eval `MINUTE_OF_HOUR(TIME('01:02:03'))` = MINUTE_OF_HOUR(TIME('01:02:03')) | fields `MINUTE_OF_HOUR(TIME('01:02:03'))` + fetched rows / total rows = 1/1 + +------------------------------------+ + | MINUTE_OF_HOUR(TIME('01:02:03')) | + |------------------------------------| + | 2 | + +------------------------------------+ + + +MINUTE_OF_DAY +------ + +Description +>>>>>>>>>>> + +Usage: minute(time) returns the amount of minutes in the day, in the range of 0 to 1439. + +Argument type: STRING/TIME/DATETIME/TIMESTAMP + +Return type: INTEGER + +Example:: + + os> source=people | eval MINUTE_of_day(TIME('01:02:03')) = MINUTE_of_day(TIME('01:02:03')) | fields MINUTE_of_day(TIME('01:02:03')) + fetched rows / total rows = 1/1 + +----------------------------+ + | MINUTE(TIME('01:02:03')) | + |----------------------------| + | 62 | + +----------------------------+ + + MONTH ----- @@ -979,6 +1134,9 @@ Argument type: STRING/DATE/DATETIME/TIMESTAMP Return type: INTEGER +Synonyms: `MONTH_OF_YEAR`_ + + Example:: os> source=people | eval `MONTH(DATE('2020-08-26'))` = MONTH(DATE('2020-08-26')) | fields `MONTH(DATE('2020-08-26'))` @@ -990,6 +1148,31 @@ Example:: +-----------------------------+ +MONTH_OF_YEAR +----- + +Description +>>>>>>>>>>> + +Usage: month_of_year(date) returns the month for date, in the range 1 to 12 for January to December. The dates with value 0 such as '0000-00-00' or '2008-00-00' are invalid. + +Argument type: STRING/DATE/DATETIME/TIMESTAMP + +Return type: INTEGER + +Synonyms: `MONTH`_ + +Example:: + + os> source=people | eval `MONTH_OF_YEAR(DATE('2020-08-26'))` = MONTH_OF_YEAR(DATE('2020-08-26')) | fields `MONTH_OF_YEAR(DATE('2020-08-26'))` + fetched rows / total rows = 1/1 + +-----------------------------+ + | MONTH_OF_YEAR(DATE('2020-08-26')) | + |-----------------------------| + | 8 | + +-----------------------------+ + + MONTHNAME --------- @@ -1118,6 +1301,8 @@ Argument type: STRING/TIME/DATETIME/TIMESTAMP Return type: INTEGER +Synonyms: `SECOND_OF_MINUTE`_ + Example:: os> source=people | eval `SECOND(TIME('01:02:03'))` = SECOND(TIME('01:02:03')) | fields `SECOND(TIME('01:02:03'))` @@ -1129,6 +1314,31 @@ Example:: +----------------------------+ +SECOND_OF_MINUTE +------ + +Description +>>>>>>>>>>> + +Usage: second_of_minute(time) returns the second for time, in the range 0 to 59. + +Argument type: STRING/TIME/DATETIME/TIMESTAMP + +Return type: INTEGER + +Synonyms: `SECOND`_ + +Example:: + + os> source=people | eval `SECOND_OF_MINUTE(TIME('01:02:03'))` = SECOND_OF_MINUTE(TIME('01:02:03')) | fields `SECOND_OF_MINUTE(TIME('01:02:03'))` + fetched rows / total rows = 1/1 + +--------------------------------------+ + | SECOND_OF_MINUTE(TIME('01:02:03')) | + |--------------------------------------| + | 3 | + +--------------------------------------+ + + SUBDATE ------- @@ -1298,6 +1508,61 @@ Example:: +----------------------------+ +TIME_FORMAT +----------- + +Description +>>>>>>>>>>> + +Usage: time_format(time, format) formats the time argument using the specifiers in the format argument. +This supports a subset of the time format specifiers available for the `date_format`_ function. +Using date format specifiers supported by `date_format`_ will return 0 or null. +Acceptable format specifiers are listed in the table below. +If an argument of type DATE is passed in, it is treated as a DATETIME at midnight (i.e., 00:00:00). + +.. list-table:: The following table describes the available specifier arguments. + :widths: 20 80 + :header-rows: 1 + + * - Specifier + - Description + * - %f + - Microseconds (000000..999999) + * - %H + - Hour (00..23) + * - %h + - Hour (01..12) + * - %I + - Hour (01..12) + * - %i + - Minutes, numeric (00..59) + * - %p + - AM or PM + * - %r + - Time, 12-hour (hh:mm:ss followed by AM or PM) + * - %S + - Seconds (00..59) + * - %s + - Seconds (00..59) + * - %T + - Time, 24-hour (hh:mm:ss) + + +Argument type: STRING/DATE/DATETIME/TIME/TIMESTAMP, STRING + +Return type: STRING + +Example:: + + os> source=calcs | eval `TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T')` = TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | fields `TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T')` + fetched rows / total rows = 1/1 + +------------------------------------------------------------------------------+ + | TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | + |------------------------------------------------------------------------------| + | 012345 13 01 01 14 PM 01:14:15 PM 15 15 13:14:15 | + +------------------------------------------------------------------------------+ + + TIME_TO_SEC ----------- @@ -1543,6 +1808,8 @@ Argument type: DATE/DATETIME/TIMESTAMP/STRING Return type: INTEGER +Synonyms: `WEEK_OF_YEAR`_ + Example:: os> source=people | eval `WEEK(DATE('2008-02-20'))` = WEEK(DATE('2008-02-20')), `WEEK(DATE('2008-02-20'), 1)` = WEEK(DATE('2008-02-20'), 1) | fields `WEEK(DATE('2008-02-20'))`, `WEEK(DATE('2008-02-20'), 1)` @@ -1554,6 +1821,72 @@ Example:: +----------------------------+-------------------------------+ +WEEK_OF_YEAR +---- + +Description +>>>>>>>>>>> + +Usage: week_of_year(date[, mode]) returns the week number for date. If the mode argument is omitted, the default mode 0 is used. + +.. list-table:: The following table describes how the mode argument works. + :widths: 25 50 25 75 + :header-rows: 1 + + * - Mode + - First day of week + - Range + - Week 1 is the first week ... + * - 0 + - Sunday + - 0-53 + - with a Sunday in this year + * - 1 + - Monday + - 0-53 + - with 4 or more days this year + * - 2 + - Sunday + - 1-53 + - with a Sunday in this year + * - 3 + - Monday + - 1-53 + - with 4 or more days this year + * - 4 + - Sunday + - 0-53 + - with 4 or more days this year + * - 5 + - Monday + - 0-53 + - with a Monday in this year + * - 6 + - Sunday + - 1-53 + - with 4 or more days this year + * - 7 + - Monday + - 1-53 + - with a Monday in this year + +Argument type: DATE/DATETIME/TIMESTAMP/STRING + +Return type: INTEGER + +Synonyms: `WEEK`_ + +Example:: + + os> source=people | eval `WEEK_OF_YEAR(DATE('2008-02-20'))` = WEEK(DATE('2008-02-20')), `WEEK_OF_YEAR(DATE('2008-02-20'), 1)` = WEEK_OF_YEAR(DATE('2008-02-20'), 1) | fields `WEEK_OF_YEAR(DATE('2008-02-20'))`, `WEEK_OF_YEAR(DATE('2008-02-20'), 1)` + fetched rows / total rows = 1/1 + +------------------------------------+---------------------------------------+ + | WEEK_OF_YEAR(DATE('2008-02-20')) | WEEK_OF_YEAR(DATE('2008-02-20'), 1) | + |------------------------------------+---------------------------------------| + | 7 | 8 | + +------------------------------------+---------------------------------------+ + + YEAR ---- diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index 5780dd2d44..89d87e870a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -407,6 +407,45 @@ public void testDay() throws IOException { verifySome(result.getJSONArray("datarows"), rows(16)); } + @Test + public void testDay_of_week() throws IOException { + JSONObject result = executeQuery(String.format( + "source=%s | eval f = day_of_week(date('2020-09-16')) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(4)); + + result = executeQuery(String.format( + "source=%s | eval f = day_of_week('2020-09-16') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(4)); + } + + @Test + public void testDay_of_month() throws IOException { + JSONObject result = executeQuery(String.format( + "source=%s | eval f = day_of_month(date('2020-09-16')) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(16)); + + result = executeQuery(String.format( + "source=%s | eval f = day_of_month('2020-09-16') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(16)); + } + + @Test + public void testDay_of_year() throws IOException { + JSONObject result = executeQuery(String.format( + "source=%s | eval f = day_of_year(date('2020-09-16')) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(260)); + + result = executeQuery(String.format( + "source=%s | eval f = day_of_year('2020-09-16') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(260)); + } + @Test public void testDayName() throws IOException { JSONObject result = executeQuery(String.format( @@ -490,6 +529,29 @@ public void testHour() throws IOException { verifySome(result.getJSONArray("datarows"), rows(17)); } + @Test + public void testHour_of_day() throws IOException { + JSONObject result = executeQuery(String.format( + "source=%s | eval f = hour_of_day(timestamp('2020-09-16 17:30:00')) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(17)); + + result = executeQuery(String.format( + "source=%s | eval f = hour_of_day(time('17:30:00')) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(17)); + + result = executeQuery(String.format( + "source=%s | eval f = hour_of_day('2020-09-16 17:30:00') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(17)); + + result = executeQuery(String.format( + "source=%s | eval f = hour_of_day('17:30:00') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(17)); + } + @Test public void testMicrosecond() throws IOException { JSONObject result = executeQuery(String.format( @@ -560,6 +622,52 @@ public void testMinute() throws IOException { verifySome(result.getJSONArray("datarows"), rows(30)); } + @Test + public void testMinute_of_hour() throws IOException { + JSONObject result = executeQuery(String.format( + "source=%s | eval f = minute_of_hour(timestamp('2020-09-16 17:30:00')) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(30)); + + result = executeQuery(String.format( + "source=%s | eval f = minute_of_hour(time('17:30:00')) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(30)); + + result = executeQuery(String.format( + "source=%s | eval f = minute_of_hour('2020-09-16 17:30:00') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(30)); + + result = executeQuery(String.format( + "source=%s | eval f = minute_of_hour('17:30:00') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(30)); + } + + @Test + public void testMinute_of_day() throws IOException { + JSONObject result = executeQuery(String.format( + "source=%s | eval f = minute_of_day(timestamp('2020-09-16 17:30:00')) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(1050)); + + result = executeQuery(String.format( + "source=%s | eval f = minute_of_day(time('17:30:00')) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(1050)); + + result = executeQuery(String.format( + "source=%s | eval f = minute_of_day('2020-09-16 17:30:00') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(1050)); + + result = executeQuery(String.format( + "source=%s | eval f = minute_of_day('17:30:00') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(1050)); + } + @Test public void testMonth() throws IOException { JSONObject result = executeQuery(String.format( @@ -573,6 +681,19 @@ public void testMonth() throws IOException { verifySome(result.getJSONArray("datarows"), rows(9)); } + @Test + public void testMonth_of_year() throws IOException { + JSONObject result = executeQuery(String.format( + "source=%s | eval f = month_of_year(date('2020-09-16')) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(9)); + + result = executeQuery(String.format( + "source=%s | eval f = month_of_year('2020-09-16') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(9)); + } + @Test public void testMonthName() throws IOException { JSONObject result = executeQuery(String.format( @@ -622,6 +743,29 @@ public void testSecond() throws IOException { verifySome(result.getJSONArray("datarows"), rows(0)); } + @Test + public void testSecond_of_minute() throws IOException { + JSONObject result = executeQuery(String.format( + "source=%s | eval f = second_of_minute(timestamp('2020-09-16 17:30:00')) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(0)); + + result = executeQuery(String.format( + "source=%s | eval f = second_of_minute(time('17:30:00')) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(0)); + + result = executeQuery(String.format( + "source=%s | eval f = second_of_minute('2020-09-16 17:30:00') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(0)); + + result = executeQuery(String.format( + "source=%s | eval f = second_of_minute('17:30:00') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(0)); + } + @Test public void testSubDateDays() throws IOException { var result = executeQuery(String.format("source=%s | eval " @@ -739,6 +883,20 @@ public void testWeek() throws IOException { week("2000-01-01", 2, 52); } + @Test + public void testWeek_of_year() throws IOException { + JSONObject result = executeQuery(String.format( + "source=%s | eval f = week(date('2008-02-20')) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(7)); + + week("2008-02-20", 0, 7); + week("2008-02-20", 1, 8); + week("2008-12-31", 1, 53); + week("2000-01-01", 0, 0); + week("2000-01-01", 2, 52); + } + @Test public void testYear() throws IOException { JSONObject result = executeQuery(String.format( @@ -754,13 +912,13 @@ public void testYear() throws IOException { void verifyDateFormat(String date, String type, String format, String formatted) throws IOException { JSONObject result = executeQuery(String.format( - "source=%s | eval f = date_format(%s('%s'), '%s') | fields f", + "source=%s | eval f = date_format(%s('%s'), '%s') | fields f", TEST_INDEX_DATE, type, date, format)); verifySchema(result, schema("f", null, "string")); verifySome(result.getJSONArray("datarows"), rows(formatted)); result = executeQuery(String.format( - "source=%s | eval f = date_format('%s', '%s') | fields f", + "source=%s | eval f = date_format('%s', '%s') | fields f", TEST_INDEX_DATE, date, format)); verifySchema(result, schema("f", null, "string")); verifySome(result.getJSONArray("datarows"), rows(formatted)); diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index f412f29280..2ffa67f7fc 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -95,11 +95,18 @@ INTERVAL: 'INTERVAL'; MICROSECOND: 'MICROSECOND'; MILLISECOND: 'MILLISECOND'; SECOND: 'SECOND'; +SECOND_OF_MINUTE: 'SECOND_OF_MINUTE'; MINUTE: 'MINUTE'; +MINUTE_OF_HOUR: 'MINUTE_OF_HOUR'; +MINUTE_OF_DAY: 'MINUTE_OF_DAY'; HOUR: 'HOUR'; +HOUR_OF_DAY: 'HOUR_OF_DAY'; DAY: 'DAY'; +DAY_OF_YEAR: 'DAY_OF_YEAR'; WEEK: 'WEEK'; +WEEK_OF_YEAR: 'WEEK_OF_YEAR'; MONTH: 'MONTH'; +MONTH_OF_YEAR: 'MONTH_OF_YEAR'; QUARTER: 'QUARTER'; YEAR: 'YEAR'; SECOND_MICROSECOND: 'SECOND_MICROSECOND'; @@ -247,11 +254,14 @@ ADDTIME: 'ADDTIME'; DATE: 'DATE'; DATE_ADD: 'DATE_ADD'; DATE_FORMAT: 'DATE_FORMAT'; +TIME_FORMAT: 'TIME_FORMAT'; DATE_SUB: 'DATE_SUB'; DATEDIFF: 'DATEDIFF'; DAYNAME: 'DAYNAME'; DAYOFMONTH: 'DAYOFMONTH'; +DAY_OF_MONTH: 'DAY_OF_MONTH'; DAYOFWEEK: 'DAYOFWEEK'; +DAY_OF_WEEK: 'DAY_OF_WEEK'; DAYOFYEAR: 'DAYOFYEAR'; FROM_DAYS: 'FROM_DAYS'; LOCALTIME: 'LOCALTIME'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index d98ee3cd1c..ae88495579 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -521,32 +521,41 @@ dateTimeFunctionName | DATE | DATE_ADD | DATE_FORMAT + | TIME_FORMAT | DATE_SUB | DATEDIFF | DATETIME | DAY + | DAY_OF_YEAR | DAYNAME | DAYOFMONTH + | DAY_OF_MONTH | DAYOFWEEK + | DAY_OF_WEEK | DAYOFYEAR | CURDATE | CURTIME | FROM_DAYS | FROM_UNIXTIME | HOUR + | HOUR_OF_DAY | LOCALTIME | LOCALTIMESTAMP | MAKEDATE | MAKETIME | MICROSECOND | MINUTE + | MINUTE_OF_HOUR + | MINUTE_OF_DAY | MONTH + | MONTH_OF_YEAR | MONTHNAME | NOW | PERIOD_ADD | PERIOD_DIFF | QUARTER | SECOND + | SECOND_OF_MINUTE | SUBDATE | SUBTIME | SYSDATE @@ -560,6 +569,7 @@ dateTimeFunctionName | UTC_TIME | UTC_TIMESTAMP | WEEK + | WEEK_OF_YEAR | YEAR ; From 88ae668d544ea813c0bf725d3c21e28dc2c6a4cc Mon Sep 17 00:00:00 2001 From: Matthew Wells Date: Fri, 30 Jun 2023 13:30:34 -0700 Subject: [PATCH 2/7] fixed incorrect test and updated documentation Signed-off-by: Matthew Wells --- docs/user/ppl/functions/datetime.rst | 4 ++-- .../java/org/opensearch/sql/ppl/DateTimeFunctionIT.java | 8 +------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 225796927a..6c5df01068 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -775,7 +775,7 @@ Argument type: STRING/DATE/DATETIME/TIMESTAMP Return type: INTEGER -Synonyms: `DAYOFYEAR`_ +Synonyms: `DAY_OF_YEAR`_ Example:: @@ -800,7 +800,7 @@ Argument type: STRING/DATE/DATETIME/TIMESTAMP Return type: INTEGER -Synonyms: `DAY_OF_YEAR`_ +Synonyms: `DAYOFYEAR`_ Example:: diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index 89d87e870a..a10d7f3771 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -886,15 +886,9 @@ public void testWeek() throws IOException { @Test public void testWeek_of_year() throws IOException { JSONObject result = executeQuery(String.format( - "source=%s | eval f = week(date('2008-02-20')) | fields f", TEST_INDEX_DATE)); + "source=%s | eval f = week_of_year(date('2008-02-20')) | fields f", TEST_INDEX_DATE)); verifySchema(result, schema("f", null, "integer")); verifySome(result.getJSONArray("datarows"), rows(7)); - - week("2008-02-20", 0, 7); - week("2008-02-20", 1, 8); - week("2008-12-31", 1, 53); - week("2000-01-01", 0, 0); - week("2000-01-01", 2, 52); } @Test From 3dd14b30a5681796de16fac5bb565b274b6bc28c Mon Sep 17 00:00:00 2001 From: Matthew Wells Date: Fri, 30 Jun 2023 13:38:50 -0700 Subject: [PATCH 3/7] fixed multiple doctest failures Signed-off-by: Matthew Wells --- docs/user/ppl/functions/datetime.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 6c5df01068..52d59d3327 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -706,11 +706,11 @@ Example:: os> source=people | eval `DAY_OF_MONTH(DATE('2020-08-26'))` = DAY_OF_MONTH(DATE('2020-08-26')) | fields `DAY_OF_MONTH(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +----------------------------------+ + +------------------------------------+ | DAY_OF_MONTH(DATE('2020-08-26')) | - |----------------------------------| - | 26 | - +----------------------------------+ + |------------------------------------| + | 26 | + +------------------------------------+ DAYOFWEEK @@ -806,11 +806,11 @@ Example:: os> source=people | eval `DAY_OF_YEAR(DATE('2020-08-26'))` = DAY_OF_YEAR(DATE('2020-08-26')) | fields `DAY_OF_YEAR(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +---------------------------------+ + +-----------------------------------+ | DAY_OF_YEAR(DATE('2020-08-26')) | - |---------------------------------| - | 239 | - +---------------------------------+ + |-----------------------------------| + | 239 | + +-----------------------------------+ FROM_DAYS @@ -1113,12 +1113,12 @@ Return type: INTEGER Example:: - os> source=people | eval MINUTE_of_day(TIME('01:02:03')) = MINUTE_of_day(TIME('01:02:03')) | fields MINUTE_of_day(TIME('01:02:03')) + os> source=people | eval `MINUTE_OF_DAY(TIME('01:02:03'))` = MINUTE_OF_DAY(TIME('01:02:03')) | fields `MINUTE_OF_DAY(TIME('01:02:03'))` fetched rows / total rows = 1/1 +----------------------------+ | MINUTE(TIME('01:02:03')) | |----------------------------| - | 62 | + | 62 | +----------------------------+ @@ -1166,11 +1166,11 @@ Example:: os> source=people | eval `MONTH_OF_YEAR(DATE('2020-08-26'))` = MONTH_OF_YEAR(DATE('2020-08-26')) | fields `MONTH_OF_YEAR(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +-----------------------------+ + +-------------------------------------+ | MONTH_OF_YEAR(DATE('2020-08-26')) | - |-----------------------------| - | 8 | - +-----------------------------+ + |-------------------------------------| + | 8 | + +-------------------------------------+ MONTHNAME From 50a9b437af9ae820a27f935823577dd81f96c1db Mon Sep 17 00:00:00 2001 From: Matthew Wells Date: Fri, 30 Jun 2023 13:59:13 -0700 Subject: [PATCH 4/7] Fixed another doctest failure Signed-off-by: Matthew Wells --- docs/user/ppl/functions/datetime.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 52d59d3327..4c7c77c9d7 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -1115,11 +1115,11 @@ Example:: os> source=people | eval `MINUTE_OF_DAY(TIME('01:02:03'))` = MINUTE_OF_DAY(TIME('01:02:03')) | fields `MINUTE_OF_DAY(TIME('01:02:03'))` fetched rows / total rows = 1/1 - +----------------------------+ - | MINUTE(TIME('01:02:03')) | - |----------------------------| - | 62 | - +----------------------------+ + +-----------------------------------+ + | MINUTE_OF_DAY(TIME('01:02:03')) | + |-----------------------------------| + | 62 | + +-----------------------------------+ MONTH From f267d07e7cdaf2182154952ba0b682f84149eb36 Mon Sep 17 00:00:00 2001 From: Matthew Wells Date: Fri, 30 Jun 2023 15:07:26 -0700 Subject: [PATCH 5/7] fixed time_format doctest failure Signed-off-by: Matthew Wells --- docs/user/ppl/functions/datetime.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 4c7c77c9d7..d5d367460d 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -1554,7 +1554,7 @@ Return type: STRING Example:: - os> source=calcs | eval `TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T')` = TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | fields `TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T')` + os> source=people | eval `TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T')` = TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | fields `TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T')` fetched rows / total rows = 1/1 +------------------------------------------------------------------------------+ | TIME_FORMAT('1998-01-31 13:14:15.012345', '%f %H %h %I %i %p %r %S %s %T') | From 7b927df652fd09ef0e74315eb6a2abfbffe1f8f3 Mon Sep 17 00:00:00 2001 From: Matthew Wells Date: Fri, 30 Jun 2023 15:54:46 -0700 Subject: [PATCH 6/7] Alphabetized parser and lexer, extended dashes under name in documentation Signed-off-by: Matthew Wells --- docs/user/ppl/functions/datetime.rst | 16 ++++----- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 40 +++++++++++------------ ppl/src/main/antlr/OpenSearchPPLParser.g4 | 22 ++++++------- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index d5d367460d..917eea869c 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -689,7 +689,7 @@ Example:: DAY_OF_MONTH ----------- +------------ Description >>>>>>>>>>> @@ -739,7 +739,7 @@ Example:: DAY_OF_WEEK ---------- +----------- Description >>>>>>>>>>> @@ -789,7 +789,7 @@ Example:: DAY_OF_YEAR ---------- +----------- Description >>>>>>>>>>> @@ -898,7 +898,7 @@ Example:: HOUR_OF_DAY ----- +----------- Description >>>>>>>>>>> @@ -1075,7 +1075,7 @@ Example:: MINUTE_OF_HOUR ------- +-------------- Description >>>>>>>>>>> @@ -1149,7 +1149,7 @@ Example:: MONTH_OF_YEAR ------ +------------- Description >>>>>>>>>>> @@ -1315,7 +1315,7 @@ Example:: SECOND_OF_MINUTE ------- +---------------- Description >>>>>>>>>>> @@ -1822,7 +1822,7 @@ Example:: WEEK_OF_YEAR ----- +------------ Description >>>>>>>>>>> diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 2ffa67f7fc..32dc3e99b2 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -90,37 +90,37 @@ FALSE: 'FALSE'; REGEXP: 'REGEXP'; // DATETIME, INTERVAL AND UNIT KEYWORDS +CONVERT_TZ: 'CONVERT_TZ'; DATETIME: 'DATETIME'; +DAY: 'DAY'; +DAY_HOUR: 'DAY_HOUR'; +DAY_MICROSECOND: 'DAY_MICROSECOND'; +DAY_MINUTE: 'DAY_MINUTE'; +DAY_OF_YEAR: 'DAY_OF_YEAR'; +DAY_SECOND: 'DAY_SECOND'; +HOUR: 'HOUR'; +HOUR_MICROSECOND: 'HOUR_MICROSECOND'; +HOUR_MINUTE: 'HOUR_MINUTE'; +HOUR_OF_DAY: 'HOUR_OF_DAY'; +HOUR_SECOND: 'HOUR_SECOND'; INTERVAL: 'INTERVAL'; MICROSECOND: 'MICROSECOND'; MILLISECOND: 'MILLISECOND'; -SECOND: 'SECOND'; -SECOND_OF_MINUTE: 'SECOND_OF_MINUTE'; MINUTE: 'MINUTE'; -MINUTE_OF_HOUR: 'MINUTE_OF_HOUR'; +MINUTE_MICROSECOND: 'MINUTE_MICROSECOND'; MINUTE_OF_DAY: 'MINUTE_OF_DAY'; -HOUR: 'HOUR'; -HOUR_OF_DAY: 'HOUR_OF_DAY'; -DAY: 'DAY'; -DAY_OF_YEAR: 'DAY_OF_YEAR'; -WEEK: 'WEEK'; -WEEK_OF_YEAR: 'WEEK_OF_YEAR'; +MINUTE_OF_HOUR: 'MINUTE_OF_HOUR'; +MINUTE_SECOND: 'MINUTE_SECOND'; MONTH: 'MONTH'; MONTH_OF_YEAR: 'MONTH_OF_YEAR'; QUARTER: 'QUARTER'; -YEAR: 'YEAR'; +SECOND: 'SECOND'; SECOND_MICROSECOND: 'SECOND_MICROSECOND'; -MINUTE_MICROSECOND: 'MINUTE_MICROSECOND'; -MINUTE_SECOND: 'MINUTE_SECOND'; -HOUR_MICROSECOND: 'HOUR_MICROSECOND'; -HOUR_SECOND: 'HOUR_SECOND'; -HOUR_MINUTE: 'HOUR_MINUTE'; -DAY_MICROSECOND: 'DAY_MICROSECOND'; -DAY_SECOND: 'DAY_SECOND'; -DAY_MINUTE: 'DAY_MINUTE'; -DAY_HOUR: 'DAY_HOUR'; +SECOND_OF_MINUTE: 'SECOND_OF_MINUTE'; +WEEK: 'WEEK'; +WEEK_OF_YEAR: 'WEEK_OF_YEAR'; +YEAR: 'YEAR'; YEAR_MONTH: 'YEAR_MONTH'; -CONVERT_TZ: 'CONVERT_TZ'; // DATASET TYPES DATAMODEL: 'DATAMODEL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index ae88495579..853af31443 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -515,26 +515,25 @@ dateTimeFunctionName : ADDDATE | ADDTIME | CONVERT_TZ + | CURDATE | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP + | CURTIME | DATE + | DATEDIFF + | DATETIME | DATE_ADD | DATE_FORMAT - | TIME_FORMAT | DATE_SUB - | DATEDIFF - | DATETIME | DAY - | DAY_OF_YEAR | DAYNAME | DAYOFMONTH - | DAY_OF_MONTH | DAYOFWEEK - | DAY_OF_WEEK | DAYOFYEAR - | CURDATE - | CURTIME + | DAY_OF_MONTH + | DAY_OF_WEEK + | DAY_OF_YEAR | FROM_DAYS | FROM_UNIXTIME | HOUR @@ -545,11 +544,11 @@ dateTimeFunctionName | MAKETIME | MICROSECOND | MINUTE - | MINUTE_OF_HOUR | MINUTE_OF_DAY + | MINUTE_OF_HOUR | MONTH - | MONTH_OF_YEAR | MONTHNAME + | MONTH_OF_YEAR | NOW | PERIOD_ADD | PERIOD_DIFF @@ -560,9 +559,10 @@ dateTimeFunctionName | SUBTIME | SYSDATE | TIME - | TIME_TO_SEC | TIMEDIFF | TIMESTAMP + | TIME_FORMAT + | TIME_TO_SEC | TO_DAYS | UNIX_TIMESTAMP | UTC_DATE From 6fa69170702814d04b302773f51e016cf64f7674 Mon Sep 17 00:00:00 2001 From: Matthew Wells Date: Tue, 4 Jul 2023 08:43:10 -0700 Subject: [PATCH 7/7] Alphabized lexer Signed-off-by: Matthew Wells --- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 32dc3e99b2..908dbe5262 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -245,28 +245,27 @@ TAN: 'TAN'; // DATE AND TIME FUNCTIONS ADDDATE: 'ADDDATE'; +ADDTIME: 'ADDTIME'; CURDATE: 'CURDATE'; CURRENT_DATE: 'CURRENT_DATE'; CURRENT_TIME: 'CURRENT_TIME'; CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; CURTIME: 'CURTIME'; -ADDTIME: 'ADDTIME'; DATE: 'DATE'; +DATEDIFF: 'DATEDIFF'; DATE_ADD: 'DATE_ADD'; DATE_FORMAT: 'DATE_FORMAT'; -TIME_FORMAT: 'TIME_FORMAT'; DATE_SUB: 'DATE_SUB'; -DATEDIFF: 'DATEDIFF'; DAYNAME: 'DAYNAME'; DAYOFMONTH: 'DAYOFMONTH'; -DAY_OF_MONTH: 'DAY_OF_MONTH'; DAYOFWEEK: 'DAYOFWEEK'; -DAY_OF_WEEK: 'DAY_OF_WEEK'; DAYOFYEAR: 'DAYOFYEAR'; +DAY_OF_MONTH: 'DAY_OF_MONTH'; +DAY_OF_WEEK: 'DAY_OF_WEEK'; FROM_DAYS: 'FROM_DAYS'; +FROM_UNIXTIME: 'FROM_UNIXTIME'; LOCALTIME: 'LOCALTIME'; LOCALTIMESTAMP: 'LOCALTIMESTAMP'; -FROM_UNIXTIME: 'FROM_UNIXTIME'; MAKEDATE: 'MAKEDATE'; MAKETIME: 'MAKETIME'; MONTHNAME: 'MONTHNAME'; @@ -277,14 +276,15 @@ SUBDATE: 'SUBDATE'; SUBTIME: 'SUBTIME'; SYSDATE: 'SYSDATE'; TIME: 'TIME'; -TIME_TO_SEC: 'TIME_TO_SEC'; TIMEDIFF: 'TIMEDIFF'; TIMESTAMP: 'TIMESTAMP'; +TIME_FORMAT: 'TIME_FORMAT'; +TIME_TO_SEC: 'TIME_TO_SEC'; TO_DAYS: 'TO_DAYS'; +UNIX_TIMESTAMP: 'UNIX_TIMESTAMP'; UTC_DATE: 'UTC_DATE'; UTC_TIME: 'UTC_TIME'; UTC_TIMESTAMP: 'UTC_TIMESTAMP'; -UNIX_TIMESTAMP: 'UNIX_TIMESTAMP'; // TEXT FUNCTIONS SUBSTR: 'SUBSTR';