diff --git a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/AbstractKqlParserTestCase.java b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/AbstractKqlParserTestCase.java index ac06a96d49eb4..588e60bd4dd75 100644 --- a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/AbstractKqlParserTestCase.java +++ b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/AbstractKqlParserTestCase.java @@ -7,9 +7,12 @@ package org.elasticsearch.xpack.kql.parser; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.core.Predicates; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.MatchPhraseQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.MultiMatchQueryBuilder; @@ -19,6 +22,7 @@ import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.test.AbstractBuilderTestCase; +import org.elasticsearch.xcontent.XContentBuilder; import java.io.BufferedReader; import java.io.IOException; @@ -36,6 +40,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; @@ -46,6 +51,43 @@ public abstract class AbstractKqlParserTestCase extends AbstractBuilderTestCase protected static final String UNSUPPORTED_QUERY_FILE_PATH = "/unsupported-queries"; protected static final Predicate BOOLEAN_QUERY_FILTER = (q) -> q.matches("(?i)[^{]*[^\\\\]*(NOT|AND|OR)[^}]*"); + protected static final String NESTED_FIELD_NAME = "mapped_nested"; + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties"); + + mapping.startObject(TEXT_FIELD_NAME).field("type", "text").endObject(); + mapping.startObject(NESTED_FIELD_NAME); + { + mapping.field("type", "nested"); + mapping.startObject("properties"); + { + mapping.startObject(TEXT_FIELD_NAME).field("type", "text").endObject(); + mapping.startObject(KEYWORD_FIELD_NAME).field("type", "keyword").endObject(); + mapping.startObject(INT_FIELD_NAME).field("type", "integer").endObject(); + mapping.startObject(NESTED_FIELD_NAME); + { + mapping.field("type", "nested"); + mapping.startObject("properties"); + { + mapping.startObject(TEXT_FIELD_NAME).field("type", "text").endObject(); + mapping.startObject(KEYWORD_FIELD_NAME).field("type", "keyword").endObject(); + mapping.startObject(INT_FIELD_NAME).field("type", "integer").endObject(); + } + mapping.endObject(); + } + mapping.endObject(); + } + mapping.endObject(); + } + mapping.endObject(); + + mapping.endObject().endObject().endObject(); + + mapperService.merge("_doc", new CompressedXContent(Strings.toString(mapping)), MapperService.MergeReason.MAPPING_UPDATE); + } + protected static String wrapWithRandomWhitespaces(String input) { return String.join("", randomWhitespaces(), input, randomWhitespaces()); } @@ -93,7 +135,18 @@ private static InputStream readFromJarUrl(URL source) throws IOException { protected List mappedLeafFields() { return Stream.concat( Arrays.stream(MAPPED_LEAF_FIELD_NAMES), - List.of(DATE_FIELD_NAME, INT_FIELD_NAME).stream().map(subfieldName -> OBJECT_FIELD_NAME + "." + subfieldName) + Stream.of( + // Adding mapped_object subfields + Strings.format("%s.%s", OBJECT_FIELD_NAME, INT_FIELD_NAME), + Strings.format("%s.%s", OBJECT_FIELD_NAME, DATE_FIELD_NAME), + // Adding mapped_nested subfields + Strings.format("%s.%s", NESTED_FIELD_NAME, TEXT_FIELD_NAME), + Strings.format("%s.%s", NESTED_FIELD_NAME, KEYWORD_FIELD_NAME), + Strings.format("%s.%s", NESTED_FIELD_NAME, INT_FIELD_NAME), + Strings.format("%s.%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME, TEXT_FIELD_NAME), + Strings.format("%s.%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME, KEYWORD_FIELD_NAME), + Strings.format("%s.%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME, INT_FIELD_NAME) + ) ).toList(); } diff --git a/x-pack/plugin/kql/src/test/resources/supported-queries b/x-pack/plugin/kql/src/test/resources/supported-queries index 4911c9e3ebecd..b659b1ae5b1db 100644 --- a/x-pack/plugin/kql/src/test/resources/supported-queries +++ b/x-pack/plugin/kql/src/test/resources/supported-queries @@ -23,54 +23,54 @@ f*oo *:"foo bar" // Querying a field -foo_field:200 -foo_field:foo -foo_field:foo bar -foo_field:(foo bar) -foo_field:foo* -foo_field: f*oo -foo_field: *foo -foo_field:"foo bar" -foo_field.subfield:foo -foo_*_field:foo -foo_field:* -foo_*:* +mapped_int:200 +mapped_string_2:foo +mapped_string:foo bar +mapped_string:(foo bar) +mapped_string:foo* +mapped_string_2: f*oo +mapped_string: *foo +mapped_string:"foo bar" +mapped_object.subfield:foo +mapped_str*:foo +mapped_string:* +mapped_str_*:* // Range queries -foo_field<200 -foo_field=200 -foo_field>=foo -foo_field>"foo bar" -foo_field<=foo -foo_field>=foo +mapped_int<200 +mapped_string_2=200 +mapped_string_alias>=foo +mapped_string>"foo bar" +mapped_string<=foo +mapped_string_2>=foo // Boolean queries NOT foo NOT foo bar -NOT foo_field:foo -NOT foo_fieldbar -(foo_field:foo) AND (foo_field:foo bar) -foo_field:foo OR foo_field:foo bar -NOT(foo_field:foo OR foo_field:foo bar) -NOT(foo_field:foo AND foo_field:foo bar) -NOT foo_field:foo AND NOT foo_field:foo bar -(NOT foo_field:foo) AND (NOT foo_field:foo bar) -NOT(foo_field:foo) AND NOT(foo_field:foo bar) -foo_field:foo AND foo_field:foo bar AND foo bar -foo_field:foo AND foo_field:foo bar OR foo bar -foo_field:foo OR foo_field:foo bar OR foo bar -foo_field:foo OR foo_field:foo bar AND foo bar -foo_field:foo AND (foo_field:foo bar OR foo bar) -foo_field:foo AND (foo_field:foo bar OR foo bar) -foo_field:foo OR (foo_field:foo bar OR foo bar) +NOT mapped_string:foo +NOT mapped_string_2bar +(mapped_string:foo) AND (mapped_string:foo bar) +mapped_string:foo OR mapped_string_2:foo bar +NOT(mapped_string:foo OR mapped_string:foo bar) +NOT(mapped_string:foo AND mapped_string:foo bar) +NOT mapped_string:foo AND NOT mapped_string_2:foo bar +(NOT mapped_string_alias:foo) AND (NOT mapped_string:foo bar) +NOT(mapped_string:foo) AND NOT(mapped_string:foo bar) +mapped_string:foo AND mapped_string_2:foo bar AND foo bar +mapped_string:foo AND mapped_string_2:foo bar OR foo bar +mapped_string:foo OR mapped_string_2:foo bar OR foo bar +mapped_string:foo OR mapped_string:foo bar AND foo bar +mapped_string:foo AND (mapped_string_2:foo bar OR foo bar) +mapped_string:foo AND (mapped_string_2:foo bar OR foo bar) +mapped_string:foo OR (mapped_string_2:foo bar OR foo bar) -foo:AND -foo:OR -foo:NOT +mapped_string:AND +mapped_string:OR +mapped_string:NOT foo AND foo OR foo NOT @@ -79,43 +79,51 @@ OR foo NOT // Nested queries -nested_field: { NOT foo } -nested_field: { NOT foo bar } -nested_field: { NOT foo_field:foo } -nested_field: { foo_field:foo AND foo_field:foo bar } -nested_field: { foo_fieldbar } -nested_field: { (foo_field:foo) AND (foo_field:foo bar) } -nested_field: { foo_field:foo OR foo_field:foo bar } -nested_field: { NOT(foo_field:foo OR foo_field:foo bar) } -nested_field: { NOT(foo_field:foo AND foo_field:foo bar) } -nested_field: { NOT foo_field:foo AND NOT foo_field:foo bar } -nested_field: { (NOT foo_field:foo) AND (NOT foo_field:foo bar) } -nested_field: { NOT(foo_field:foo) AND NOT(foo_field:foo bar) } -nested_field: { foo_field:foo AND foo_field:foo bar AND foo bar } -nested_field: { foo_field:foo AND foo_field:foo bar OR foo bar } -nested_field: { foo_field:foo OR foo_field:foo bar OR foo bar } -nested_field: { foo_field:foo OR foo_field:foo bar AND foo bar } -nested_field: { foo_field:foo AND (foo_field:foo bar OR foo bar) } -nested_field: { foo_field:foo AND (foo_field:foo bar OR foo bar) } -nested_field: { foo_field:foo OR (foo_field:foo bar OR foo bar) } -nested_field: { sub_nested_field : { foo_field:foo } AND foo_field:foo bar } +mapped_nested: { NOT foo } +mapped_nested: { NOT foo bar } +mapped_nested: { NOT mapped_string:foo } +mapped_nested: { mapped_string:foo AND mapped_string_2:foo bar } +mapped_nested: { mapped_string2 } +mapped_nested: { (mapped_string:foo) AND (mapped_string_2:foo bar) } +mapped_nested: { mapped_string:foo OR mapped_string_2:foo bar } +mapped_nested: { NOT(mapped_string:foo OR mapped_string_2:foo bar) } +mapped_nested: { NOT(mapped_string:foo AND mapped_string_2:foo bar) } +mapped_nested: { NOT mapped_string:foo AND NOT mapped_string_2:foo bar } +mapped_nested: { (NOT mapped_string:foo) AND (NOT mapped_string_2:foo bar) } +mapped_nested: { NOT(mapped_string:foo) AND NOT(mapped_string_2:foo bar) } +mapped_nested: { mapped_string:foo AND mapped_string_2:foo bar AND foo bar } +mapped_nested: { mapped_string:foo AND mapped_string_2:foo bar OR foo bar } +mapped_nested: { mapped_string:foo OR mapped_string_2:foo bar OR foo bar } +mapped_nested: { mapped_string:foo OR mapped_string_2:foo bar AND foo bar } +mapped_nested: { mapped_string:foo AND (mapped_string_2:foo bar OR foo bar) } +mapped_nested: { mapped_string:foo AND (mapped_string_2:foo bar OR foo bar) } +mapped_nested: { mapped_string:foo OR (mapped_string_2:foo bar OR foo bar) } +mapped_nested: { mapped_str*:foo } +mapped_nested: { mapped_nested : { mapped_string:foo AND mapped_int < 3 } AND mapped_string_2:foo bar } +mapped_nested: { mapped_nested.mapped_string:foo AND mapped_string_2:foo bar } + +// Inline nested queries +mapped_nested.mapped_string:foo AND mapped_nested.mapped_int < 2 +mapped_nested.mapped_nested.mapped_string:foo AND mapped_nested.mapped_int < 2 +mapped_nested.mapped_str*: foo + // Queries with escape sequences -foo_field : (foo\(bar\)) -foo_field : foo\:bar -foo_field : (foo \\and bar) -foo_field : (foo \\or bar) -foo_field : foo \\not bar -foo_field : foo \{bar\} -foo_field : foo \(bar\) -foo_field : foo \\ bar -foo_field : foo \"bar\" +mapped_string:(foo\(bar\)) +mapped_string:foo\:bar +mapped_string:(foo \\and bar) +mapped_string:(foo \\or bar) +mapped_string:foo \\not bar +mapped_string:foo \{bar\} +mapped_string:foo \(bar\) +mapped_string:foo \\ bar +mapped_string:foo \"bar\" -foo_field : "foo and bar" -foo_field : "foo not bar" -foo_field : "foo or bar" -foo_field : "foo : bar" -foo_field : "foo { bar }" -foo_field : "foo (bar)" -foo_field : "foo \\ bar" -foo_field : "foo \"bar\"" +mapped_string:"foo and bar" +mapped_string:"foo not bar" +mapped_string:"foo or bar" +mapped_string:"foo : bar" +mapped_string:"foo { bar }" +mapped_string:"foo (bar)" +mapped_string:"foo \\ bar" +mapped_string:"foo \"bar\"" diff --git a/x-pack/plugin/kql/src/test/resources/unsupported-queries b/x-pack/plugin/kql/src/test/resources/unsupported-queries index 64901891c6786..149bcf5bd2b5a 100644 --- a/x-pack/plugin/kql/src/test/resources/unsupported-queries +++ b/x-pack/plugin/kql/src/test/resources/unsupported-queries @@ -1,36 +1,36 @@ // Incomplete expressions -foo_field : -foo_field < -foo_field > -foo_field >= -foo_field <= +mapped_string : +mapped_string < +mapped_string > +mapped_string >= +mapped_string <= >= foo : "foo" : foo // Parentheses mismatch -foo_field: (foo bar -foo_field: foo bar) -NOT foo_field:foo OR foo_field:foo bar) -NOT (foo_field:foo AND) foo_field:foo bar +mapped_string: (foo bar +mapped_string: foo bar) +NOT mapped_string:foo OR mapped_string_2:foo bar) +NOT (mapped_string:foo AND) mapped_string_2:foo bar // Quotes mismatch -foo_field: "foo bar -foo_field: foo bar" +mapped_string: "foo bar +mapped_string: foo bar" // Can't nest grouping terms parentheses -foo_field:(foo (bar)) +mapped_string:(foo (bar)) // Bad syntax for nested fields: -nested_field { foo: bar } +mapped_nested { mapped_string: bar } // Missing escape sequences: -foo_field: foo:bar -foo_field: (foo and bar) -foo_field: (foo or bar) -foo_field: foo not bar -foo_field: foo { bar } -foo_field: foo (bar) -foo_field: foo "bar" -foo_field: "foo "bar"" +mapped_string: foo:bar +mapped_string: (foo and bar) +mapped_string: (foo or bar) +mapped_string: foo not bar +mapped_string: foo { bar } +mapped_string: foo (bar) +mapped_string: foo "bar" +mapped_string: "foo "bar""