From d33f51ddfd345ed80f80f93b07b6777d3bc755c8 Mon Sep 17 00:00:00 2001 From: psmagin Date: Tue, 19 Jul 2022 12:56:26 +0300 Subject: [PATCH 1/5] MSEARCH-393 Fix date query validation to support ISO date/time formats --- .../search/cql/CqlTermQueryConverter.java | 26 ++++++++++++++++--- .../controller/SearchAuthorityFilterIT.java | 6 +++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java b/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java index 38e65b14c..f19c3e607 100644 --- a/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java +++ b/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java @@ -3,10 +3,10 @@ import static java.util.Collections.unmodifiableMap; import static java.util.stream.Collectors.joining; import static org.apache.commons.lang3.StringUtils.lowerCase; -import static org.apache.commons.validator.GenericValidator.isDate; import static org.folio.search.utils.SearchUtils.ASTERISKS_SIGN; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -34,7 +34,16 @@ public class CqlTermQueryConverter { public static final String WILDCARD_OPERATOR = "wildcard"; private static final String MATCH_ALL_CQL_QUERY = "cql.allRecords = 1"; private static final String KEYWORD_ALL_CQL_QUERY = "keyword = *"; - private static final String STRICT_DATE_PATTERN = "yyyy-MM-dd"; + + private static final DateTimeFormatter[] SUPPORTED_DATE_FORMATS = new DateTimeFormatter[] { + DateTimeFormatter.ISO_DATE, + DateTimeFormatter.ISO_DATE_TIME, + DateTimeFormatter.ISO_LOCAL_DATE, + DateTimeFormatter.ISO_LOCAL_DATE_TIME, + DateTimeFormatter.ISO_OFFSET_DATE, + DateTimeFormatter.ISO_OFFSET_DATE_TIME, + DateTimeFormatter.ISO_INSTANT + }; private final SearchFieldProvider searchFieldProvider; private final Map termQueryBuilders; @@ -143,11 +152,22 @@ private static Map getTermQueryProvidersAsMap(List filteredSearchQueriesProvider() { arguments("(subjectHeadings==\"n\")", List.of(IDS[11], IDS[12])), arguments("(metadata.createdDate >= 2021-03-01) ", List.of(IDS[0], IDS[1], IDS[2], IDS[3])), + arguments("(metadata.createdDate >= 2021-03-01T00:00:00.000) ", List.of(IDS[0], IDS[1], IDS[2], IDS[3])), + arguments("(metadata.createdDate >= 2021-03-01T00:00:00.000Z) ", List.of(IDS[0], IDS[1], IDS[2], IDS[3])), + arguments("(metadata.createdDate >= 2021-03-01T00:00:00.000+00:00) ", List.of(IDS[0], IDS[1], IDS[2], IDS[3])), arguments("(metadata.createdDate > 2021-03-01) ", List.of(IDS[1], IDS[2], IDS[3])), arguments("(metadata.createdDate >= 2021-03-01 and metadata.createdDate < 2021-03-10) ", List.of(IDS[0], IDS[2])), @@ -158,8 +161,7 @@ private static Stream invalidDateSearchQueriesProvider() { arguments("metadata.createdDate", "2022-06-1"), arguments("metadata.createdDate", "2022-06-40"), arguments("metadata.updatedDate", "2022-15-01"), - arguments("metadata.updatedDate", "invalidDate"), - arguments("metadata.updatedDate", "2022-11-15T15:00:00.000") + arguments("metadata.updatedDate", "invalidDate") ); } From 07038b2407dce73e3b14f11a63224e9813268449 Mon Sep 17 00:00:00 2001 From: psmagin Date: Tue, 19 Jul 2022 13:02:18 +0300 Subject: [PATCH 2/5] MSEARCH-393 Fix code style --- .../java/org/folio/search/cql/CqlTermQueryConverter.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java b/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java index f19c3e607..eb4d5bdb6 100644 --- a/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java +++ b/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java @@ -35,7 +35,7 @@ public class CqlTermQueryConverter { private static final String MATCH_ALL_CQL_QUERY = "cql.allRecords = 1"; private static final String KEYWORD_ALL_CQL_QUERY = "keyword = *"; - private static final DateTimeFormatter[] SUPPORTED_DATE_FORMATS = new DateTimeFormatter[] { + private static final List SUPPORTED_DATE_FORMATS = List.of( DateTimeFormatter.ISO_DATE, DateTimeFormatter.ISO_DATE_TIME, DateTimeFormatter.ISO_LOCAL_DATE, @@ -43,7 +43,7 @@ public class CqlTermQueryConverter { DateTimeFormatter.ISO_OFFSET_DATE, DateTimeFormatter.ISO_OFFSET_DATE_TIME, DateTimeFormatter.ISO_INSTANT - }; + ); private final SearchFieldProvider searchFieldProvider; private final Map termQueryBuilders; @@ -163,7 +163,9 @@ private boolean isValidDate(String value) { try { dateFormat.parse(value); isValidDate = true; - } catch (Exception ignored) { } + } catch (Exception ignored) { + // do nothing + } } return isValidDate; } From 93968bd6a86789d447430e731a2fa838e7d3b202 Mon Sep 17 00:00:00 2001 From: psmagin Date: Tue, 19 Jul 2022 13:14:45 +0300 Subject: [PATCH 3/5] MSEARCH-393 Fix test --- .../org/folio/search/controller/SearchInstanceFilterIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/java/org/folio/search/controller/SearchInstanceFilterIT.java b/src/test/java/org/folio/search/controller/SearchInstanceFilterIT.java index 9f9f791c8..749ff9988 100644 --- a/src/test/java/org/folio/search/controller/SearchInstanceFilterIT.java +++ b/src/test/java/org/folio/search/controller/SearchInstanceFilterIT.java @@ -316,8 +316,7 @@ private static Stream invalidDateSearchQueriesProvider() { arguments("holdings.metadata.createdDate", "2022-15-01"), arguments("holdings.metadata.updatedDate", "2022-06-40"), - arguments("item.metadata.updatedDate", "invalidDate"), - arguments("item.metadata.createdDate", "2022-06-15T15:00:00.000") + arguments("item.metadata.updatedDate", "invalidDate") ); } From 29d2823fbe038717775c188ae6b2521fca9c633e Mon Sep 17 00:00:00 2001 From: psmagin Date: Tue, 19 Jul 2022 17:06:18 +0300 Subject: [PATCH 4/5] MSEARCH-393 Update README example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 05b9f83d8..0c70c802c 100644 --- a/README.md +++ b/README.md @@ -366,7 +366,7 @@ if it is defined but doesn't match. | `statusId` | term | `statusId == "123"` | Matches instances with the `123` status | | `instanceFormatIds` | term | `instanceFormatIds == "123"` | Matches instances with the `123` format id | | `languages` | term | `languages == "eng"` | Matches instances that have `eng` language | -| `metadata.createdDate` | term | `metadata.createdDate > "2020-12-12"` | Matches instances that were created after `2020-12-12` | +| `metadata.createdDate` | term | `metadata.createdDate > "2021-03-01T00:00:00.000+00:00"` | Matches instances that were created after `2020-12-12` | | `metadata.updatedDate` | term | `metadata.updatedDate > "2020-12-12"` | Matches instances that were updated after `2020-12-12` | | `modeOfIssuanceId` | term | `modeOfIssuanceId=="123"` | Matches instances that have `123` mode of issuance | | `natureOfContentTermIds` | term | `natureOfContentTermIds=="123"` | Matches instances that have `123` nature of content | From c4213c229b15d1124b9ca6a9b175c011cb0d9b3b Mon Sep 17 00:00:00 2001 From: psmagin Date: Wed, 20 Jul 2022 15:16:58 +0300 Subject: [PATCH 5/5] MSEARCH-393 Improve isValidDate method --- .../java/org/folio/search/cql/CqlTermQueryConverter.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java b/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java index eb4d5bdb6..6ba6ad0d3 100644 --- a/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java +++ b/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java @@ -158,16 +158,15 @@ private void validateIndexFormat(String index, CQLTermNode termNode) { } private boolean isValidDate(String value) { - boolean isValidDate = false; for (DateTimeFormatter dateFormat : SUPPORTED_DATE_FORMATS) { try { dateFormat.parse(value); - isValidDate = true; + return true; } catch (Exception ignored) { // do nothing } } - return isValidDate; + return false; } private static boolean isMatchAllQuery(String cqlQuery) {