diff --git a/docs/experiment/ppl/general/identifiers.rst b/docs/experiment/ppl/general/identifiers.rst index 1a3aa70147..951accfbb6 100644 --- a/docs/experiment/ppl/general/identifiers.rst +++ b/docs/experiment/ppl/general/identifiers.rst @@ -30,6 +30,8 @@ For Elasticsearch, the following identifiers are supported extensionally: 3. Identifiers with ``-`` in the middle: this is mostly the case for index name with date information. 4. Identifiers with star ``*`` present: this is mostly an index pattern for wildcard match. +Index name with date suffix separated by dash or dots, such as ``cwl-2020.01.11`` or ``logs-7.0-2020.01.11``, is common for those created by Logstash or FileBeat ingestion. So, this kind of identifier used as index name is also supported without the need of being quoted for user convenience. In this case, wildcard within date pattern is also allowed to search for data across indices of different date range. For example, you can use ``logs-2020.1*`` to search in indices for October, November and December 2020. + Examples -------- diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/SearchCommandIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/SearchCommandIT.java index 6bdad4ed10..76c4c26500 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/SearchCommandIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/SearchCommandIT.java @@ -15,6 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.ppl; +import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -49,6 +50,15 @@ public void testSearchCommandWithoutSearchKeyword() throws IOException { executeQueryToString(String.format("source=%s", TEST_INDEX_BANK))); } + @Test + public void testSearchCommandWithSpecialIndexName() throws IOException { + executeRequest(new Request("PUT", "/logs-2021.01.11")); + verifyDataRows(executeQuery("search source=logs-2021.01.11")); + + executeRequest(new Request("PUT", "/logs-7.10.0-2021.01.11")); + verifyDataRows(executeQuery("search source=logs-7.10.0-2021.01.11")); + } + @Test public void testSearchCommandWithLogicalExpression() throws IOException { JSONObject result = diff --git a/ppl/src/main/antlr/OpenDistroPPLLexer.g4 b/ppl/src/main/antlr/OpenDistroPPLLexer.g4 index 15b8830a31..0869f15911 100644 --- a/ppl/src/main/antlr/OpenDistroPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenDistroPPLLexer.g4 @@ -241,7 +241,9 @@ ID: ID_LITERAL; INTEGER_LITERAL: DEC_DIGIT+; DECIMAL_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+; +fragment DATE_SUFFIX: ([\-.][*0-9]+)*; fragment ID_LITERAL: [@*A-Z]+?[*A-Z_\-0-9]*; +ID_DATE_SUFFIX: ID_LITERAL DATE_SUFFIX; DQUOTA_STRING: '"' ( '\\'. | '""' | ~('"'| '\\') )* '"'; SQUOTA_STRING: '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\''; BQUOTA_STRING: '`' ( '\\'. | '``' | ~('`'|'\\'))* '`'; diff --git a/ppl/src/main/antlr/OpenDistroPPLParser.g4 b/ppl/src/main/antlr/OpenDistroPPLParser.g4 index d724beb0de..b23bcdd062 100644 --- a/ppl/src/main/antlr/OpenDistroPPLParser.g4 +++ b/ppl/src/main/antlr/OpenDistroPPLParser.g4 @@ -178,6 +178,7 @@ booleanExpression /** tables */ tableSource : qualifiedName + | ID_DATE_SUFFIX ; /** fields */ @@ -311,11 +312,11 @@ valueList ; qualifiedName - : ident (DOT ident)* #identsAsQualifiedName + : ident #identsAsQualifiedName ; wcQualifiedName - : wildcard (DOT wildcard)* #identsAsWildcardQualifiedName + : wildcard #identsAsWildcardQualifiedName ; ident diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilder.java index abe8680175..8a8c011190 100644 --- a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilder.java @@ -267,7 +267,7 @@ public UnresolvedPlan visitTopCommand(TopCommandContext ctx) { */ @Override public UnresolvedPlan visitFromClause(FromClauseContext ctx) { - return new Relation(visitExpression(ctx.tableSource().qualifiedName())); + return new Relation(visitExpression(ctx.tableSource())); } /** diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java index 729b4ba0a9..0cb16a4121 100644 --- a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java @@ -18,8 +18,10 @@ import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.IS_NOT_NULL; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.IS_NULL; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.BinaryArithmeticContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.BooleanFunctionCallContext; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.BooleanLiteralContext; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.CompareExprContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.CountAllFunctionCallContext; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.DecimalLiteralContext; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.EvalClauseContext; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.EvalFunctionCallContext; @@ -38,6 +40,7 @@ import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.SortFieldContext; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.StatsFunctionCallContext; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.StringLiteralContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.TableSourceContext; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.WcFieldExpressionContext; import com.amazon.opendistroforelasticsearch.sql.ast.expression.AggregateFunction; @@ -59,7 +62,6 @@ import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Xor; import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils; -import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser; import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParserBaseVisitor; import com.amazon.opendistroforelasticsearch.sql.ppl.utils.ArgumentFactory; import com.google.common.collect.ImmutableMap; @@ -183,8 +185,7 @@ public UnresolvedExpression visitStatsFunctionCall(StatsFunctionCallContext ctx) } @Override - public UnresolvedExpression visitCountAllFunctionCall( - OpenDistroPPLParser.CountAllFunctionCallContext ctx) { + public UnresolvedExpression visitCountAllFunctionCall(CountAllFunctionCallContext ctx) { return new AggregateFunction("count", AllFields.of()); } @@ -198,8 +199,7 @@ public UnresolvedExpression visitPercentileAggFunction(PercentileAggFunctionCont * Eval function. */ @Override - public UnresolvedExpression visitBooleanFunctionCall( - OpenDistroPPLParser.BooleanFunctionCallContext ctx) { + public UnresolvedExpression visitBooleanFunctionCall(BooleanFunctionCallContext ctx) { final String functionName = ctx.conditionFunctionBase().getText(); return new Function( @@ -225,30 +225,23 @@ public UnresolvedExpression visitEvalFunctionCall(EvalFunctionCallContext ctx) { .collect(Collectors.toList())); } + @Override + public UnresolvedExpression visitTableSource(TableSourceContext ctx) { + return visitIdentifier(ctx); + } + /** * Literal and value. */ @Override public UnresolvedExpression visitIdentsAsQualifiedName(IdentsAsQualifiedNameContext ctx) { - return new QualifiedName( - ctx.ident() - .stream() - .map(ParserRuleContext::getText) - .map(StringUtils::unquoteText) - .collect(Collectors.toList()) - ); + return visitIdentifier(ctx.ident()); } @Override public UnresolvedExpression visitIdentsAsWildcardQualifiedName( IdentsAsWildcardQualifiedNameContext ctx) { - return new QualifiedName( - ctx.wildcard() - .stream() - .map(ParserRuleContext::getText) - .map(StringUtils::unquoteText) - .collect(Collectors.toList()) - ); + return visitIdentifier(ctx.wildcard()); } @Override @@ -277,4 +270,9 @@ public UnresolvedExpression visitBooleanLiteral(BooleanLiteralContext ctx) { return new Literal(Boolean.valueOf(ctx.getText()), DataType.BOOLEAN); } + private UnresolvedExpression visitIdentifier(ParserRuleContext ctx) { + return new QualifiedName(Collections.singletonList( + StringUtils.unquoteIdentifier(ctx.getText()))); + } + } diff --git a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilderTest.java index 7565f52e8c..11b035153a 100644 --- a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilderTest.java @@ -50,7 +50,6 @@ import com.amazon.opendistroforelasticsearch.sql.ast.Node; import com.amazon.opendistroforelasticsearch.sql.ast.tree.RareTopN.CommandType; -import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException; import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.PPLSyntaxParser; import org.junit.Ignore; import org.junit.Rule; @@ -379,9 +378,9 @@ public void testIdentifierAsIndexNameStartWithDot() { } @Test - public void identifierAsIndexNameWithDotInTheMiddleThrowException() { - exceptionRule.expect(SyntaxCheckException.class); - plan("source=log.2020.10.10"); + public void testIdentifierAsIndexNameWithDotInTheMiddle() { + assertEqual("source=log.2020.10.10", relation("log.2020.10.10")); + assertEqual("source=log-7.10-2020.10.10", relation("log-7.10-2020.10.10")); } @Test @@ -396,6 +395,13 @@ public void testIdentifierAsIndexNameContainStar() { relation("log-2020-10-*")); } + @Test + public void testIdentifierAsIndexNameContainStarAndDots() { + assertEqual("source=log-2020.10.*", relation("log-2020.10.*")); + assertEqual("source=log-2020.*.01", relation("log-2020.*.01")); + assertEqual("source=log-2020.*.*", relation("log-2020.*.*")); + } + @Test public void testIdentifierAsFieldNameStartWithAt() { assertEqual("source=log-2020 | fields @timestamp", diff --git a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilderTest.java index a0a754bef8..7f0f3badb2 100644 --- a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -376,6 +376,7 @@ public void testEvalFuncCallExpr() { )); } + @Ignore("Nested field is not supported in backend yet") @Test public void testNestedFieldName() { assertEqual("source=t | fields field0.field1.field2", @@ -390,6 +391,19 @@ public void testNestedFieldName() { @Test public void testFieldNameWithSpecialChars() { + assertEqual("source=t | fields `field-0`", + projectWithArg( + relation("t"), + defaultFieldsArgs(), + field( + qualifiedName("field-0") + ) + )); + } + + @Ignore("Nested field is not supported in backend yet") + @Test + public void testNestedFieldNameWithSpecialChars() { assertEqual("source=t | fields `field-0`.`field#1`.`field*2`", projectWithArg( relation("t"), @@ -486,9 +500,9 @@ public void testKeywordsAsIdentifiers() { @Test public void canBuildKeywordsAsIdentInQualifiedName() { assertEqual( - "source=test.timestamp | fields timestamp", + "source=test | fields timestamp", projectWithArg( - relation("test.timestamp"), + relation("test"), defaultFieldsArgs(), field("timestamp") ) diff --git a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 91b14172c6..2221245866 100644 --- a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -151,8 +151,8 @@ public void testNotExpression() { @Test public void testQualifiedName() { - assertEquals("source=t | fields + field0.field1", - anonymize("source=t | fields field0.field1") + assertEquals("source=t | fields + field0", + anonymize("source=t | fields field0") ); }