diff --git a/common/build.gradle b/common/build.gradle index 2b3f09a883..08de79ca76 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -11,6 +11,7 @@ dependencies { compile "org.antlr:antlr4-runtime:4.7.1" // https://github.com/google/guava/wiki/CVE-2018-10237 compile group: 'com.google.guava', name: 'guava', version: '29.0-jre' + compile group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.11.1' testCompile group: 'junit', name: 'junit', version: '4.12' } diff --git a/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/utils/LogUtils.java b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/utils/LogUtils.java new file mode 100644 index 0000000000..7e7bdac5a5 --- /dev/null +++ b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/utils/LogUtils.java @@ -0,0 +1,75 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.common.utils; + +import java.util.Map; +import java.util.UUID; +import org.apache.logging.log4j.ThreadContext; + +/** + * Utility class for generating/accessing the request id from logging context. + */ +public class LogUtils { + + /** + * The key of the request id in the context map. + */ + private static final String REQUEST_ID_KEY = "request_id"; + + /** + * Generates a random UUID and adds to the {@link ThreadContext} as the request id. + *

+ * Note: If a request id already present, this method will overwrite it with a new + * one. This is to pre-vent re-using the same request id for different requests in + * case the same thread handles both of them. But this also means one should not + * call this method twice on the same thread within the lifetime of the request. + *

+ */ + public static void addRequestId() { + ThreadContext.put(REQUEST_ID_KEY, UUID.randomUUID().toString()); + } + + /** + * Get RequestID. + * @return the current request id from {@link ThreadContext}. + */ + public static String getRequestId() { + final String requestId = ThreadContext.get(REQUEST_ID_KEY); + return requestId; + } + + /** + * Wraps a given instance of {@link Runnable} into a new one which gets all the + * entries from current ThreadContext map. + * + * @param task the instance of Runnable to wrap + * @return the new task + */ + public static Runnable withCurrentContext(final Runnable task) { + final Map currentContext = ThreadContext.getImmutableContext(); + return () -> { + ThreadContext.putAll(currentContext); + task.run(); + }; + } + + private LogUtils() { + throw new AssertionError( + getClass().getCanonicalName() + " is a utility class and must not be initialized"); + } +} diff --git a/docs/experiment/ppl/index.rst b/docs/experiment/ppl/index.rst index f0fcc5ce5a..dbb8e66928 100644 --- a/docs/experiment/ppl/index.rst +++ b/docs/experiment/ppl/index.rst @@ -38,7 +38,7 @@ The query start with search command and then flowing a set of command delimited - `eval command `_ - - `field command `_ + - `fields command `_ - `rename command `_ diff --git a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java index 015f4aff16..7f7d2300ec 100644 --- a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java +++ b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java @@ -24,6 +24,7 @@ import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException; import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; import com.amazon.opendistroforelasticsearch.sql.common.setting.Settings; +import com.amazon.opendistroforelasticsearch.sql.common.utils.LogUtils; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.response.error.ErrorMessageFactory; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.security.SecurityAccess; import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; @@ -33,7 +34,6 @@ import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse; import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName; import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; -import com.amazon.opendistroforelasticsearch.sql.legacy.utils.LogUtils; import com.amazon.opendistroforelasticsearch.sql.plugin.request.PPLQueryRequestFactory; import com.amazon.opendistroforelasticsearch.sql.ppl.PPLService; import com.amazon.opendistroforelasticsearch.sql.ppl.config.PPLServiceConfig; diff --git a/ppl/build.gradle b/ppl/build.gradle index 062ddfd606..bc82d50a31 100644 --- a/ppl/build.gradle +++ b/ppl/build.gradle @@ -31,6 +31,7 @@ dependencies { compile group: 'org.json', name: 'json', version: '20180813' compile group: 'org.springframework', name: 'spring-context', version: '5.2.5.RELEASE' compile group: 'org.springframework', name: 'spring-beans', version: '5.2.5.RELEASE' + compile group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.11.1' compile project(':common') compile project(':core') compile project(':protocol') diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLService.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLService.java index 18763c18ba..5cba6a55e6 100644 --- a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLService.java +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLService.java @@ -21,6 +21,7 @@ import com.amazon.opendistroforelasticsearch.sql.analysis.Analyzer; import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.common.utils.LogUtils; import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.ExplainResponse; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; @@ -33,10 +34,13 @@ import com.amazon.opendistroforelasticsearch.sql.ppl.domain.PPLQueryRequest; import com.amazon.opendistroforelasticsearch.sql.ppl.parser.AstBuilder; import com.amazon.opendistroforelasticsearch.sql.ppl.parser.AstExpressionBuilder; +import com.amazon.opendistroforelasticsearch.sql.ppl.utils.PPLQueryDataAnonymizer; import com.amazon.opendistroforelasticsearch.sql.ppl.utils.UnresolvedPlanHelper; import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; import lombok.RequiredArgsConstructor; import org.antlr.v4.runtime.tree.ParseTree; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; @RequiredArgsConstructor public class PPLService { @@ -50,6 +54,10 @@ public class PPLService { private final BuiltinFunctionRepository repository; + private final PPLQueryDataAnonymizer anonymizer = new PPLQueryDataAnonymizer(); + + private static final Logger LOG = LogManager.getLogger(); + /** * Execute the {@link PPLQueryRequest}, using {@link ResponseListener} to get response. * @@ -85,6 +93,8 @@ private PhysicalPlan plan(PPLQueryRequest request) { UnresolvedPlan ast = cst.accept( new AstBuilder(new AstExpressionBuilder(), request.getRequest())); + LOG.info("[{}] Incoming request {}", LogUtils.getRequestId(), anonymizer.anonymizeData(ast)); + // 2.Analyze abstract syntax to generate logical plan LogicalPlan logicalPlan = analyzer.analyze(UnresolvedPlanHelper.addSelectAll(ast), new AnalysisContext()); diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/PPLQueryDataAnonymizer.java new file mode 100644 index 0000000000..5a20beab77 --- /dev/null +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -0,0 +1,326 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.utils; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.AggregateFunction; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Alias; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.And; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Compare; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Function; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Interval; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Let; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Map; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Not; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Or; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedArgument; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Xor; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Aggregation; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Dedupe; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Eval; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Filter; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Head; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.RareTopN; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Relation; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Rename; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalAggregation; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalDedupe; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalEval; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalHead; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalProject; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRareTopN; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRemove; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRename; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalSort; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Utility class to mask sensitive information in incoming PPL queries. + */ +public class PPLQueryDataAnonymizer extends AbstractNodeVisitor { + + private static final String MASK_LITERAL = "***"; + + private final AnonymizerExpressionAnalyzer expressionAnalyzer; + + public PPLQueryDataAnonymizer() { + this.expressionAnalyzer = new AnonymizerExpressionAnalyzer(); + } + + /** + * This method is used to anonymize sensitive data in PPL query. + * Sensitive data includes user data., + * + * @return ppl query string with all user data replace with "***" + */ + public String anonymizeData(UnresolvedPlan plan) { + return plan.accept(this, null); + } + + @Override + public String visitRelation(Relation node, String context) { + return StringUtils.format("source=%s", node.getTableName()); + } + + @Override + public String visitFilter(Filter node, String context) { + String child = node.getChild().get(0).accept(this, context); + String condition = visitExpression(node.getCondition()); + return StringUtils.format("%s | where %s", child, condition); + } + + /** + * Build {@link LogicalRename}. + */ + @Override + public String visitRename(Rename node, String context) { + String child = node.getChild().get(0).accept(this, context); + ImmutableMap.Builder renameMapBuilder = new ImmutableMap.Builder<>(); + for (Map renameMap : node.getRenameList()) { + renameMapBuilder.put(visitExpression(renameMap.getOrigin()), + ((Field) renameMap.getTarget()).getField().toString()); + } + String renames = + renameMapBuilder.build().entrySet().stream().map(entry -> StringUtils.format("%s as %s", + entry.getKey(), entry.getValue())).collect(Collectors.joining(",")); + return StringUtils.format("%s | rename %s", child, renames); + } + + /** + * Build {@link LogicalAggregation}. + */ + @Override + public String visitAggregation(Aggregation node, String context) { + String child = node.getChild().get(0).accept(this, context); + final String group = visitExpressionList(node.getGroupExprList()); + return StringUtils.format("%s | stats %s", child, + String.join(" ", visitExpressionList(node.getAggExprList()), groupBy(group)).trim()); + } + + /** + * Build {@link LogicalRareTopN}. + */ + @Override + public String visitRareTopN(RareTopN node, String context) { + final String child = node.getChild().get(0).accept(this, context); + List options = node.getNoOfResults(); + Integer noOfResults = (Integer) options.get(0).getValue().getValue(); + String fields = visitFieldList(node.getFields()); + String group = visitExpressionList(node.getGroupExprList()); + return StringUtils.format("%s | %s %d %s", child, + node.getCommandType().name().toLowerCase(), + noOfResults, + String.join(" ", fields, groupBy(group)).trim() + ); + } + + /** + * Build {@link LogicalProject} or {@link LogicalRemove} from {@link Field}. + */ + @Override + public String visitProject(Project node, String context) { + String child = node.getChild().get(0).accept(this, context); + String arg = "+"; + String fields = visitExpressionList(node.getProjectList()); + + if (node.hasArgument()) { + Argument argument = node.getArgExprList().get(0); + Boolean exclude = (Boolean) argument.getValue().getValue(); + if (exclude) { + arg = "-"; + } + } + return StringUtils.format("%s | fields %s %s", child, arg, fields); + } + + /** + * Build {@link LogicalEval}. + */ + @Override + public String visitEval(Eval node, String context) { + String child = node.getChild().get(0).accept(this, context); + ImmutableList.Builder> expressionsBuilder = new ImmutableList.Builder<>(); + for (Let let : node.getExpressionList()) { + String expression = visitExpression(let.getExpression()); + String target = let.getVar().getField().toString(); + expressionsBuilder.add(ImmutablePair.of(target, expression)); + } + String expressions = expressionsBuilder.build().stream().map(pair -> StringUtils.format("%s" + + "=%s", pair.getLeft(), pair.getRight())).collect(Collectors.joining(" ")); + return StringUtils.format("%s | eval %s", child, expressions); + } + + /** + * Build {@link LogicalSort}. + */ + @Override + public String visitSort(Sort node, String context) { + String child = node.getChild().get(0).accept(this, context); + // the first options is {"count": "integer"} + Integer count = (Integer) node.getOptions().get(0).getValue().getValue(); + String sortList = visitFieldList(node.getSortList()); + return StringUtils.format("%s | sort %d %s", child, count, sortList); + } + + /** + * Build {@link LogicalDedupe}. + */ + @Override + public String visitDedupe(Dedupe node, String context) { + String child = node.getChild().get(0).accept(this, context); + String fields = visitFieldList(node.getFields()); + List options = node.getOptions(); + Integer allowedDuplication = (Integer) options.get(0).getValue().getValue(); + Boolean keepEmpty = (Boolean) options.get(1).getValue().getValue(); + Boolean consecutive = (Boolean) options.get(2).getValue().getValue(); + + return StringUtils + .format("%s | dedup %s %d keepempty=%b consecutive=%b", child, fields, allowedDuplication, + keepEmpty, + consecutive); + } + + /** + * Build {@link LogicalHead}. + */ + @Override + public String visitHead(Head node, String context) { + String child = node.getChild().get(0).accept(this, context); + List options = node.getOptions(); + Boolean keeplast = (Boolean) ((Literal) options.get(0).getValue()).getValue(); + String whileExpr = visitExpression(options.get(1).getValue()); + Integer number = (Integer) ((Literal) options.get(2).getValue()).getValue(); + + return StringUtils.format("%s | head keeplast=%b while(%s) %d", child, keeplast, whileExpr, + number); + } + + private String visitFieldList(List fieldList) { + return fieldList.stream().map(this::visitExpression).collect(Collectors.joining(",")); + } + + private String visitExpressionList(List expressionList) { + return expressionList.isEmpty() ? "" : + expressionList.stream().map(this::visitExpression).collect(Collectors.joining(",")); + } + + private String visitExpression(UnresolvedExpression expression) { + return expressionAnalyzer.analyze(expression, null); + } + + private String groupBy(String groupBy) { + return Strings.isNullOrEmpty(groupBy) ? "" : StringUtils.format("by %s", groupBy); + } + + /** + * Expression Anonymizer. + */ + private static class AnonymizerExpressionAnalyzer extends AbstractNodeVisitor { + + public String analyze(UnresolvedExpression unresolved, String context) { + return unresolved.accept(this, context); + } + + @Override + public String visitLiteral(Literal node, String context) { + return MASK_LITERAL; + } + + @Override + public String visitInterval(Interval node, String context) { + String value = node.getValue().accept(this, context); + String unit = node.getUnit().name(); + return StringUtils.format("INTERVAL %s %s", value, unit); + } + + @Override + public String visitAnd(And node, String context) { + String left = node.getLeft().accept(this, context); + String right = node.getRight().accept(this, context); + return StringUtils.format("%s and %s", left, right); + } + + @Override + public String visitOr(Or node, String context) { + String left = node.getLeft().accept(this, context); + String right = node.getRight().accept(this, context); + return StringUtils.format("%s or %s", left, right); + } + + @Override + public String visitXor(Xor node, String context) { + String left = node.getLeft().accept(this, context); + String right = node.getRight().accept(this, context); + return StringUtils.format("%s xor %s", left, right); + } + + @Override + public String visitNot(Not node, String context) { + String expr = node.getExpression().accept(this, context); + return StringUtils.format("not %s", expr); + } + + @Override + public String visitAggregateFunction(AggregateFunction node, String context) { + String arg = node.getField().accept(this, context); + return StringUtils.format("%s(%s)", node.getFuncName(), arg); + } + + @Override + public String visitFunction(Function node, String context) { + String arguments = + node.getFuncArgs().stream() + .map(unresolvedExpression -> analyze(unresolvedExpression, context)) + .collect(Collectors.joining(",")); + return StringUtils.format("%s(%s)", node.getFuncName(), arguments); + } + + @Override + public String visitCompare(Compare node, String context) { + String left = analyze(node.getLeft(), context); + String right = analyze(node.getRight(), context); + return StringUtils.format("%s %s %s", left, node.getOperator(), right); + } + + @Override + public String visitField(Field node, String context) { + return node.getField().toString(); + } + + @Override + public String visitAlias(Alias node, String context) { + String expr = node.getDelegated().accept(this, context); + return StringUtils.format("%s", expr); + } + } +} diff --git a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java new file mode 100644 index 0000000000..403c234c0b --- /dev/null +++ b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -0,0 +1,171 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.utils; + +import static org.junit.Assert.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.PPLSyntaxParser; +import com.amazon.opendistroforelasticsearch.sql.ppl.parser.AstBuilder; +import com.amazon.opendistroforelasticsearch.sql.ppl.parser.AstExpressionBuilder; +import org.junit.Test; + +public class PPLQueryDataAnonymizerTest { + + private PPLSyntaxParser parser = new PPLSyntaxParser(); + + @Test + public void testSearchCommand() { + assertEquals("source=t | where a = ***", + anonymize("search source=t a=1") + ); + } + + @Test + public void testWhereCommand() { + assertEquals("source=t | where a = ***", + anonymize("search source=t | where a=1") + ); + } + + @Test + public void testFieldsCommandWithoutArguments() { + assertEquals("source=t | fields + f,g", + anonymize("source=t | fields f,g")); + } + + @Test + public void testFieldsCommandWithIncludeArguments() { + assertEquals("source=t | fields + f,g", + anonymize("source=t | fields + f,g")); + } + + @Test + public void testFieldsCommandWithExcludeArguments() { + assertEquals("source=t | fields - f,g", + anonymize("source=t | fields - f,g")); + } + + @Test + public void testRenameCommandWithMultiFields() { + assertEquals("source=t | rename f as g,h as i,j as k", + anonymize("source=t | rename f as g,h as i,j as k")); + } + + @Test + public void testStatsCommandWithByClause() { + assertEquals("source=t | stats count(a) by b", + anonymize("source=t | stats count(a) by b")); + } + + @Test + public void testStatsCommandWithNestedFunctions() { + assertEquals("source=t | stats sum(+(a,b))", + anonymize("source=t | stats sum(a+b)")); + } + + @Test + public void testDedupCommand() { + assertEquals("source=t | dedup f1,f2 1 keepempty=false consecutive=false", + anonymize("source=t | dedup f1, f2")); + } + + @Test + public void testHeadCommandWithNumber() { + assertEquals("source=t | head keeplast=true while(***) 3", + anonymize("source=t | head 3")); + } + + @Test + public void testHeadCommandWithWhileExpr() { + assertEquals("source=t | head keeplast=true while(a < ***) 5", + anonymize("source=t | head while(a < 5) 5")); + } + + //todo, sort order is ignored, it doesn't impact the log analysis. + @Test + public void testSortCommandWithOptions() { + assertEquals("source=t | sort 100 f1,f2", + anonymize("source=t | sort 100 - f1, + f2")); + } + + @Test + public void testEvalCommand() { + assertEquals("source=t | eval r=abs(f)", + anonymize("source=t | eval r=abs(f)")); + } + + @Test + public void testRareCommandWithGroupBy() { + assertEquals("source=t | rare 10 a by b", + anonymize("source=t | rare a by b")); + } + + @Test + public void testTopCommandWithNAndGroupBy() { + assertEquals("source=t | top 1 a by b", + anonymize("source=t | top 1 a by b")); + } + + @Test + public void testAndExpression() { + assertEquals("source=t | where a = *** and b = ***", + anonymize("source=t | where a=1 and b=2") + ); + } + + @Test + public void testOrExpression() { + assertEquals("source=t | where a = *** or b = ***", + anonymize("source=t | where a=1 or b=2") + ); + } + + @Test + public void testXorExpression() { + assertEquals("source=t | where a = *** xor b = ***", + anonymize("source=t | where a=1 xor b=2") + ); + } + + @Test + public void testNotExpression() { + assertEquals("source=t | where not a = ***", + anonymize("source=t | where not a=1 ") + ); + } + + @Test + public void testQualifiedName() { + assertEquals("source=t | fields + field0.field1", + anonymize("source=t | fields field0.field1") + ); + } + + @Test + public void testDateFunction() { + assertEquals("source=t | eval date=DATE_ADD(DATE(***),INTERVAL *** HOUR)", + anonymize("source=t | eval date=DATE_ADD(DATE('2020-08-26'),INTERVAL 1 HOUR)") + ); + } + + private String anonymize(String query) { + AstBuilder astBuilder = new AstBuilder(new AstExpressionBuilder(), query); + final PPLQueryDataAnonymizer anonymizer = new PPLQueryDataAnonymizer(); + return anonymizer.anonymizeData(astBuilder.visit(parser.analyzeSyntax(query))); + } +}