From db8788e5a26575506a72c226db74e5b5ca09ce07 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Mon, 17 Aug 2020 18:24:32 +0300 Subject: [PATCH] QL: wildcard field type support (#58062) (#61205) (cherry picked from commit c874e6cdd3e051ce599b50c18642de038b84105f) --- .../whitelist/InternalQlScriptUtils.java | 2 +- .../elasticsearch/xpack/ql/type/Types.java | 6 +- .../xpack/ql/type/TypesTests.java | 13 +- .../test/resources/mapping-multi-field.json | 3 + .../src/test/resources/mapping-wildcard.json | 8 + .../qa/jdbc/PreparedStatementTestCase.java | 59 +++++ .../xpack/sql/qa/FieldExtractorTestCase.java | 33 +++ .../xpack/sql/qa/jdbc/CsvSpecTestCase.java | 2 +- .../xpack/sql/qa/jdbc/DataLoader.java | 15 ++ .../server/src/main/resources/alias.csv-spec | 2 + .../src/main/resources/command.csv-spec | 2 + .../single-node-only/command-sys.csv-spec | 6 + .../src/main/resources/wildcard.csv-spec | 236 ++++++++++++++++++ 13 files changed, 382 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugin/ql/src/test/resources/mapping-wildcard.json create mode 100644 x-pack/plugin/sql/qa/server/src/main/resources/wildcard.csv-spec diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java index 0aa9b0597d301..c27f4e6e0f68e 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java @@ -7,8 +7,8 @@ package org.elasticsearch.xpack.ql.expression.function.scalar.whitelist; import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogicProcessor.BinaryLogicOperation; import org.elasticsearch.xpack.ql.expression.function.scalar.string.StartsWithFunctionProcessor; +import org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogicProcessor.BinaryLogicOperation; import org.elasticsearch.xpack.ql.expression.predicate.logical.NotProcessor; import org.elasticsearch.xpack.ql.expression.predicate.nulls.CheckNullProcessor.CheckNullOperation; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation; diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java index 136d6e8155451..129b6d591d7f9 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java @@ -48,8 +48,12 @@ private static Map startWalking(DataTypeRegistry typeRegistry, private static DataType getType(DataTypeRegistry typeRegistry, Map content) { if (content.containsKey("type")) { + String typeName = content.get("type").toString(); + if ("wildcard".equals(typeName)) { + return KEYWORD; + } try { - return typeRegistry.fromEs(content.get("type").toString()); + return typeRegistry.fromEs(typeName); } catch (IllegalArgumentException ex) { return UNSUPPORTED; } diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java index 7adbf9306d7b8..76be9f3cfc78d 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java @@ -136,10 +136,11 @@ public void testMultiField() { assertThat(DataTypes.isPrimitive(field.getDataType()), is(true)); assertThat(field.getDataType(), is(TEXT)); Map fields = field.getProperties(); - assertThat(fields.size(), is(3)); + assertThat(fields.size(), is(4)); assertThat(fields.get("raw").getDataType(), is(KEYWORD)); assertThat(fields.get("english").getDataType(), is(TEXT)); assertThat(fields.get("constant").getDataType(), is(CONSTANT_KEYWORD)); + assertThat(fields.get("wildcard").getDataType(), is(KEYWORD)); } public void testMultiFieldTooManyOptions() { @@ -150,10 +151,11 @@ public void testMultiFieldTooManyOptions() { assertThat(DataTypes.isPrimitive(field.getDataType()), is(true)); assertThat(field, instanceOf(TextEsField.class)); Map fields = field.getProperties(); - assertThat(fields.size(), is(3)); + assertThat(fields.size(), is(4)); assertThat(fields.get("raw").getDataType(), is(KEYWORD)); assertThat(fields.get("english").getDataType(), is(TEXT)); assertThat(fields.get("constant").getDataType(), is(CONSTANT_KEYWORD)); + assertThat(fields.get("wildcard").getDataType(), is(KEYWORD)); } public void testNestedDoc() { @@ -183,6 +185,13 @@ public void testConstantKeywordField() { assertThat(dt.getDataType().typeName(), is("constant_keyword")); } + public void testWildcardField() { + Map mapping = loadMapping("mapping-wildcard.json"); + assertThat(mapping.size(), is(1)); + EsField dt = mapping.get("full_name"); + assertThat(dt.getDataType().typeName(), is("keyword")); + } + public void testUnsupportedTypes() { Map mapping = loadMapping("mapping-unsupported.json"); EsField dt = mapping.get("range"); diff --git a/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json b/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json index 8adc4c64e8476..490f1306250d4 100644 --- a/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json +++ b/x-pack/plugin/ql/src/test/resources/mapping-multi-field.json @@ -13,6 +13,9 @@ "constant" : { "type" : "constant_keyword", "value" : "some constant value" + }, + "wildcard" : { + "type" : "wildcard" } } } diff --git a/x-pack/plugin/ql/src/test/resources/mapping-wildcard.json b/x-pack/plugin/ql/src/test/resources/mapping-wildcard.json new file mode 100644 index 0000000000000..f24a4ec31a107 --- /dev/null +++ b/x-pack/plugin/ql/src/test/resources/mapping-wildcard.json @@ -0,0 +1,8 @@ +{ + "properties" : { + "full_name" : { + "type" : "wildcard", + "ignore_above" : 256 + } + } +} diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java index 3f57af6327d42..8db94e634f4d5 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java @@ -189,6 +189,65 @@ public void testOutOfRangeBigDecimal() throws SQLException { } } + public void testWildcardField() throws IOException, SQLException { + String mapping = "\"properties\":{\"id\":{\"type\":\"integer\"},\"text\":{\"type\":\"wildcard\"}}"; + createIndex("test", Settings.EMPTY, mapping); + String text = randomAlphaOfLengthBetween(1, 10); + + for (int i = 1; i <= 3; i++) { + int id = 1000 + i; + String valueToIndex = text + i; + index("test", "" + i, builder -> { + builder.field("id", id); + builder.field("text", valueToIndex); + }); + } + + try (Connection connection = esJdbc()) { + try (PreparedStatement statement = connection.prepareStatement("SELECT id, text FROM test WHERE text = ?")) { + int randomDocumentIndex = randomIntBetween(1, 3); + String randomDocumentText = text + randomDocumentIndex; + + statement.setString(1, randomDocumentText); + try (ResultSet results = statement.executeQuery()) { + assertTrue(results.next()); + assertEquals(1000 + randomDocumentIndex, results.getInt(1)); + assertEquals(randomDocumentText, results.getString(2)); + assertFalse(results.next()); + } + } + } + } + + public void testConstantKeywordField() throws IOException, SQLException { + String mapping = "\"properties\":{\"id\":{\"type\":\"integer\"},\"text\":{\"type\":\"constant_keyword\"}}"; + createIndex("test", Settings.EMPTY, mapping); + String text = randomAlphaOfLengthBetween(1, 10); + + for (int i = 1; i <= 3; i++) { + int id = 1000 + i; + index("test", "" + i, builder -> { + builder.field("id", id); + builder.field("text", text); + }); + } + + try (Connection connection = esJdbc()) { + try (PreparedStatement statement = connection.prepareStatement("SELECT id, text FROM test WHERE text = ?")) { + statement.setString(1, text); + + try (ResultSet results = statement.executeQuery()) { + for (int i = 1; i <= 3; i++) { + assertTrue(results.next()); + assertEquals(1000 + i, results.getInt(1)); + assertEquals(text, results.getString(2)); + } + assertFalse(results.next()); + } + } + } + } + public void testUnsupportedParameterUse() throws IOException, SQLException { index("library", builder -> { builder.field("name", "Don Quixote"); diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java index d18b0f52e9c4b..25dde1cef9ee4 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java @@ -134,6 +134,39 @@ public void testConstantKeywordField() throws IOException { assertResponse(expected, runSql("SELECT constant_keyword_field FROM test")); } + /* + * "wildcard_field": { + * "type": "wildcard", + * "ignore_above": 10 + * } + */ + public void testWildcardField() throws IOException { + String wildcard = randomAlphaOfLength(20); + // _source for `wildcard` fields doesn't matter, as they should be taken from docvalue_fields + boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting + boolean enableSource = randomBoolean(); // enable _source at index level + boolean ignoreAbove = randomBoolean(); + + Map indexProps = new HashMap<>(1); + indexProps.put("_source", enableSource); + + Map> fieldProps = null; + if (ignoreAbove) { + fieldProps = new HashMap<>(1); + Map fieldProp = new HashMap<>(1); + fieldProp.put("ignore_above", 10); + fieldProps.put("wildcard_field", fieldProp); + } + + createIndexWithFieldTypeAndProperties("wildcard", fieldProps, explicitSourceSetting ? indexProps : null); + index("{\"wildcard_field\":\"" + wildcard + "\"}"); + + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList(columnInfo("plain", "wildcard_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); + expected.put("rows", singletonList(singletonList(ignoreAbove ? null : wildcard))); + assertResponse(expected, runSql("SELECT wildcard_field FROM test")); + } + /* * "long/integer/short/byte_field": { * "type": "long/integer/short/byte" diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java index 00bf9030dbd7a..4dd8bb0b8111f 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java @@ -30,7 +30,7 @@ public abstract class CsvSpecTestCase extends SpecBaseIntegrationTestCase { @ParametersFactory(argumentFormatting = PARAM_FORMATTING) public static List readScriptSpec() throws Exception { List urls = JdbcTestUtils.classpathResources("/*.csv-spec"); - assertTrue("Not enough specs found " + urls.toString(), urls.size() > 15); + assertTrue("Not enough specs found (" + urls.size() + ") " + urls.toString(), urls.size() >= 23); return readScriptSpec(urls, specParser()); } diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java index bc9c87238e26a..759e5a9f33d72 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java @@ -129,6 +129,7 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String if (extraFields) { createIndex.startObject("extra_gender").field("type", "constant_keyword").endObject(); createIndex.startObject("null_constant").field("type", "constant_keyword").endObject(); + createIndex.startObject("wildcard_name").field("type", "wildcard").endObject(); createIndex.startObject("extra.info.gender").field("type", "alias").field("path", "gender").endObject(); } @@ -186,6 +187,8 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String boolean hadLastItem = false; + String wildcard_name = null; + boolean setWildcardName = true; for (int f = 0; f < fields.size(); f++) { // an empty value in the csv file is treated as 'null', thus skipping it in the bulk request if (fields.get(f).trim().length() > 0) { @@ -198,7 +201,19 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String bulk.append(",\"extra_gender\":\"Female\""); } } + if ((titles.get(f).equals("first_name") || titles.get(f).equals("last_name")) && extraFields && setWildcardName) { + if (fields.get(f).trim().length() == 0) { + setWildcardName = false; + } else { + wildcard_name = wildcard_name == null ? fields.get(f) : wildcard_name + " " + fields.get(f); + } + } } + // append the wildcard field + if (extraFields && setWildcardName) { + bulk.append(",\"wildcard_name\":\"" + wildcard_name + "\""); + } + // append department List> list = dep_emp.get(emp_no); if (!list.isEmpty()) { diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec index f3bfb6df281f3..3c72cb630b0c0 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec @@ -52,6 +52,7 @@ last_name |VARCHAR |text last_name.keyword |VARCHAR |keyword null_constant |VARCHAR |keyword salary |INTEGER |integer +wildcard_name |VARCHAR |keyword ; describePattern @@ -81,6 +82,7 @@ last_name |VARCHAR |text last_name.keyword |VARCHAR |keyword null_constant |VARCHAR |keyword salary |INTEGER |integer +wildcard_name |VARCHAR |keyword ; showAlias diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec index 457b9dd9b9ba9..39604f1edf4fe 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec @@ -295,6 +295,7 @@ last_name |VARCHAR |text last_name.keyword |VARCHAR |keyword null_constant |VARCHAR |keyword salary |INTEGER |integer +wildcard_name |VARCHAR |keyword ; describeMultiLike @@ -324,6 +325,7 @@ last_name |VARCHAR |text last_name.keyword |VARCHAR |keyword null_constant |VARCHAR |keyword salary |INTEGER |integer +wildcard_name |VARCHAR |keyword ; describeSimpleIdentifier diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec index ccd638016a625..7b3331f81aa3e 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec @@ -43,6 +43,7 @@ integTest |null |test_emp |last_name |12 integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO integTest |null |test_emp |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO ; sysColumnsWithCatalogAndLike @@ -64,6 +65,7 @@ integTest |null |test_emp_copy|last_name |12 |TEX integTest |null |test_emp_copy|last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO integTest |null |test_emp_copy|null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO integTest |null |test_emp_copy|salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO ; sysColumnsOnAliasWithTableLike @@ -85,6 +87,7 @@ integTest |null |test_alias |last_name |12 |T integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO integTest |null |test_alias |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO ; sysColumnsAllTables @@ -114,6 +117,7 @@ integTest |null |test_alias |last_name |12 integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO integTest |null |test_alias |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO integTest |null |test_alias_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO integTest |null |test_alias_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO integTest |null |test_alias_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO @@ -128,6 +132,7 @@ integTest |null |test_alias_emp |last_name |12 integTest |null |test_alias_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO integTest |null |test_alias_emp |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO integTest |null |test_alias_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO integTest |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO @@ -152,4 +157,5 @@ integTest |null |test_emp_copy |last_name |12 integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO integTest |null |test_emp_copy |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO ; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/wildcard.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/wildcard.csv-spec new file mode 100644 index 0000000000000..a0961f32bfd33 --- /dev/null +++ b/x-pack/plugin/sql/qa/server/src/main/resources/wildcard.csv-spec @@ -0,0 +1,236 @@ +// To mute tests follow example in file: example.csv-spec + +// +// Tests testing wildcard field type (introduced in ES 7.9?) +// + +// filtering + +filterEquals +SELECT last_name, first_name, wildcard_name FROM test_emp_copy WHERE wildcard_name = 'Saniya Kalloufi'; + + last_name:s | first_name:s |wildcard_name:s +---------------+---------------+--------------- +Kalloufi |Saniya |Saniya Kalloufi +; + +filterNotEquals +SELECT COUNT(*) AS c FROM "test_emp_copy" WHERE wildcard_name <> 'Saniya Kalloufi'; + + c:l +--------------- +99 +; + +aggWithNullFilter +SELECT COUNT(*) count FROM test_emp_copy WHERE wildcard_name IS NOT NULL; + + count:l +--------------- +90 +; + +aggWithNullFilter +SELECT COUNT(*) count FROM test_emp_copy WHERE wildcard_name IS NULL; + + count:l +--------------- +10 +; + +functionOverAlias +SELECT BIT_LENGTH(wildcard_name) bit, wildcard_name, length(wildcard_name) l FROM test_emp_copy ORDER BY bit DESC LIMIT 10; + + bit | wildcard_name | l +---------------+----------------------+--------------- +176 |Sudharsan Flasterstein|22 +168 |Arumugam Ossenbruggen |21 +168 |Sreekrishna Servieres |21 +160 |Kazuhito Cappelletti |20 +160 |Breannda Billingsley |20 +152 |Cristinel Bouloucos |19 +144 |Duangkaew Piveteau |18 +144 |Patricio Bridgland |18 +144 |Guoxiang Nooteboom |18 +144 |Alejandro McAlpine |18 +; + + +groupByWithPercentile +SELECT wildcard_name AS w, PERCENTILE(emp_no, 97) p1 FROM test_emp_copy GROUP BY wildcard_name LIMIT 10; + + w:s | p1:d +---------------------+--------------- +null |10039.0 +Alejandro McAlpine |10059.0 +Amabile Gomatam |10091.0 +Anneke Preusig |10006.0 +Anoosh Peyn |10062.0 +Arumugam Ossenbruggen|10094.0 +Basil Tramer |10049.0 +Berhard McFarlin |10058.0 +Berni Genin |10014.0 +Bezalel Simmel |10002.0 +; + +complexGroupByWithPercentile +SELECT CONCAT(SUBSTRING(wildcard_name, 0, 1), SUBSTRING(wildcard_name, LOCATE(' ', wildcard_name) + 1, 1)) AS w, PERCENTILE(emp_no, 97) p1, COUNT(*) c FROM test_emp_copy GROUP BY w HAVING c > 1 ORDER BY w DESC; + + w:s | p1:d | c:l +---------------+---------------+--------------- +SP |10024.0 |2 +SF |10089.0 |3 +PB |10080.0 |3 +MS |10054.0 |4 +MB |10074.0 |2 +KM |10085.0 |2 +GD |10075.0 |2 +CB |10068.0 |2 +BM |10058.0 |2 +BB |10060.0 |2 +AP |10062.0 |2 + |10039.0 |10 +; + +aggSumWithAliasWithColumnRepeatedWithOrderDesc +SELECT wildcard_name AS g, wildcard_name, SUM(salary) AS s3, SUM(salary), SUM(salary) AS s5 FROM test_emp_copy GROUP BY wildcard_name LIMIT 10; + + g:s | wildcard_name:s | s3:i | SUM(salary):i | s5:i +---------------------+---------------------+---------------+---------------+--------------- +null |null |473020 |473020 |473020 +Alejandro McAlpine |Alejandro McAlpine |44307 |44307 |44307 +Amabile Gomatam |Amabile Gomatam |38645 |38645 |38645 +Anneke Preusig |Anneke Preusig |60335 |60335 |60335 +Anoosh Peyn |Anoosh Peyn |65030 |65030 |65030 +Arumugam Ossenbruggen|Arumugam Ossenbruggen|66817 |66817 |66817 +Basil Tramer |Basil Tramer |37853 |37853 |37853 +Berhard McFarlin |Berhard McFarlin |38376 |38376 |38376 +Berni Genin |Berni Genin |37137 |37137 |37137 +Bezalel Simmel |Bezalel Simmel |56371 |56371 |56371 +; + +topHitsAsMinAndMaxAndGroupBy +schema::w:s|min:s|max:s|first:s|last:s +SELECT SUBSTRING(wildcard_name, 0, 1) w, MIN(first_name) as min, MAX(first_name) as max, FIRST(first_name) as first, LAST(first_name) as last FROM test_emp_copy GROUP BY w ORDER BY w; + + w | min | max | first | last +---------------+---------------+---------------+---------------+--------------- +null |null |null |null |null +A |Alejandro |Arumugam |Alejandro |Arumugam +B |Basil |Brendon |Basil |Brendon +C |Charlene |Cristinel |Charlene |Cristinel +D |Danel |Duangkaew |Danel |Duangkaew +E |Ebbe |Erez |Ebbe |Erez +F |Florian |Florian |Florian |Florian +G |Gao |Guoxiang |Gao |Guoxiang +H |Heping |Hisao |Heping |Hisao +J |Jayson |Jungsoon |Jayson |Jungsoon +K |Kazuhide |Kyoichi |Kazuhide |Kyoichi +L |Lillian |Lucien |Lillian |Lucien +M |Magy |Moss |Magy |Moss +O |Otmar |Otmar |Otmar |Otmar +P |Parto |Premal |Parto |Premal +R |Ramzi |Reuven |Ramzi |Reuven +S |Sailaja |Suzette |Sailaja |Suzette +T |Tse |Tzvetan |Tse |Tzvetan +U |Udi |Uri |Udi |Uri +V |Valdiodio |Vishv |Valdiodio |Vishv +W |Weiyi |Weiyi |Weiyi |Weiyi +X |Xinglin |Xinglin |Xinglin |Xinglin +Y |Yinghua |Yongqiao |Yinghua |Yongqiao +Z |Zhongwei |Zvonko |Zhongwei |Zvonko +; + +medianAbsoluteDeviationOnTwoFields +schema::wildcard_name:s|avg:l|mad_s:l|mad_l:d +SELECT wildcard_name, FLOOR(AVG(salary)) AS avg, FLOOR(MAD(salary)) AS mad_s, MAD(languages) AS mad_l FROM test_emp_copy GROUP BY wildcard_name ORDER BY wildcard_name LIMIT 3; + + wildcard_name | avg | mad_s | mad_l +------------------+-------------+-------------+--------------- +null |47302 |8213 |1 +Alejandro McAlpine|44307 |0 |0 +Amabile Gomatam |38645 |0 |0 +; + +caseGroupByAndHaving +schema::count:l|wildcard_name:s|languages:byte +SELECT count(*) AS count, wildcard_name, languages FROM test_emp_copy GROUP BY 2, 3 HAVING CASE WHEN count(*) > 1 THEN 'many' ELSE 'a few' END = 'many' ORDER BY 1, 3; + + count | wildcard_name | languages +---------------+---------------+--------------- +2 |null |1 +2 |null |2 +2 |null |3 +3 |null |4 +; + +whereFieldWithRLikeAndGroupByOrderBy +SELECT last_name l, wildcard_name g, COUNT(*) c, MAX(salary) AS sal FROM test_emp_copy WHERE emp_no < 10050 AND (last_name RLIKE 'B.*' OR wildcard_name = 'F') GROUP BY g, l ORDER BY sal; + + l:s | g:s | c:l | sal:i +---------------+-------------------+---------------+--------------- +Berztiss |Yongqiao Berztiss |1 |28336 +Brender |null |1 |36051 +Bridgland |Patricio Bridgland |1 |48942 +Bouloucos |Cristinel Bouloucos|1 |58715 +Bamford |Parto Bamford |1 |61805 +; + +multipleGroupingsAndOrderingByGroupsWithFunctions_2 +SELECT first_name f, last_name l, LCASE(wildcard_name) g, CONCAT(UCASE(first_name), LCASE(last_name)) c FROM test_emp_copy GROUP BY f, LCASE(wildcard_name), l, c ORDER BY c ASC, first_name, l ASC, g LIMIT 10; + + f:s | l:s | g:s | c:s +---------------+---------------+---------------------+-------------------- +Alejandro |McAlpine |alejandro mcalpine |ALEJANDROmcalpine +Amabile |Gomatam |amabile gomatam |AMABILEgomatam +Anneke |Preusig |anneke preusig |ANNEKEpreusig +Anoosh |Peyn |anoosh peyn |ANOOSHpeyn +Arumugam |Ossenbruggen |arumugam ossenbruggen|ARUMUGAMossenbruggen +Basil |Tramer |basil tramer |BASILtramer +Berhard |McFarlin |berhard mcfarlin |BERHARDmcfarlin +Berni |Genin |berni genin |BERNIgenin +Bezalel |Simmel |bezalel simmel |BEZALELsimmel +Bojan |Montemayor |bojan montemayor |BOJANmontemayor +; + +isNotNullWithCount +SELECT wildcard_name IS NOT NULL AS bool, COUNT(*) FROM test_emp_copy GROUP BY bool; + + bool:b | COUNT(*):l +---------------+--------------- +false |10 +true |90 +; + +like +SELECT last_name, first_name, wildcard_name FROM test_emp_copy WHERE wildcard_name LIKE '%Kalloufi'; + + last_name:s | first_name:s |wildcard_name:s +---------------+---------------+--------------- +Kalloufi |Saniya |Saniya Kalloufi +Kalloufi |Tuval |Tuval Kalloufi +; + +likeWithCount1 +SELECT COUNT(*) FROM test_emp_copy WHERE wildcard_name LIKE 'A%'; + + COUNT(*):l +--------------- +5 +; + +likeWithCount2 +SELECT COUNT(*) FROM test_emp_copy WHERE wildcard_name LIKE '%m%'; + + COUNT(*):l +--------------- +25 +; + +likeWithCount3 +SELECT COUNT(*) FROM test_emp_copy WHERE wildcard_name NOT LIKE '%m%'; + + COUNT(*):l +--------------- +75 +;