Add ppl request log (#796)

* add the log for anonymized incoming request

* fix checkstyle

* ignore request id

* fix ut

common/build.gradle

@@ -11,6 +11,7 @@ dependencies {
     compile "org.antlr:antlr4-runtime:4.7.1"
     // https://github.com/google/guava/wiki/CVE-2018-10237
     compile group: 'com.google.guava', name: 'guava', version: '29.0-jre'
+    compile group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.11.1'
 
     testCompile group: 'junit', name: 'junit', version: '4.12'
 }

common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/utils/LogUtils.java

@@ -0,0 +1,75 @@
+/*
+ *
+ *    Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License").
+ *    You may not use this file except in compliance with the License.
+ *    A copy of the License is located at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    or in the "license" file accompanying this file. This file is distributed
+ *    on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *    express or implied. See the License for the specific language governing
+ *    permissions and limitations under the License.
+ *
+ */
+
+package com.amazon.opendistroforelasticsearch.sql.common.utils;
+
+import java.util.Map;
+import java.util.UUID;
+import org.apache.logging.log4j.ThreadContext;
+
+/**
+ * Utility class for generating/accessing the request id from logging context.
+ */
+public class LogUtils {
+
+  /**
+   * The key of the request id in the context map.
+   */
+  private static final String REQUEST_ID_KEY = "request_id";
+
+  /**
+   * Generates a random UUID and adds to the {@link ThreadContext} as the request id.
+   * <p>
+   * Note: If a request id already present, this method will overwrite it with a new
+   * one. This is to pre-vent re-using the same request id for different requests in
+   * case the same thread handles both of them. But this also means one should not
+   * call this method twice on the same thread within the lifetime of the request.
+   * </p>
+   */
+  public static void addRequestId() {
+    ThreadContext.put(REQUEST_ID_KEY, UUID.randomUUID().toString());
+  }
+
+  /**
+   * Get RequestID.
+   * @return the current request id from {@link ThreadContext}.
+   */
+  public static String getRequestId() {
+    final String requestId = ThreadContext.get(REQUEST_ID_KEY);
+    return requestId;
+  }
+
+  /**
+   * Wraps a given instance of {@link Runnable} into a new one which gets all the
+   * entries from current ThreadContext map.
+   *
+   * @param task the instance of Runnable to wrap
+   * @return the new task
+   */
+  public static Runnable withCurrentContext(final Runnable task) {
+    final Map<String, String> currentContext = ThreadContext.getImmutableContext();
+    return () -> {
+      ThreadContext.putAll(currentContext);
+      task.run();
+    };
+  }
+
+  private LogUtils() {
+    throw new AssertionError(
+        getClass().getCanonicalName() + " is a utility class and must not be initialized");
+  }
+}

docs/experiment/ppl/index.rst

@@ -38,7 +38,7 @@ The query start with search command and then flowing a set of command delimited
 
   - `eval command <cmd/eval.rst>`_
 
-  - `field command <cmd/fields.rst>`_
+  - `fields command <cmd/fields.rst>`_
 
   - `rename command <cmd/rename.rst>`_
 

plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java

@@ -24,6 +24,7 @@
 import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException;
 import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener;
 import com.amazon.opendistroforelasticsearch.sql.common.setting.Settings;
+import com.amazon.opendistroforelasticsearch.sql.common.utils.LogUtils;
 import com.amazon.opendistroforelasticsearch.sql.elasticsearch.response.error.ErrorMessageFactory;
 import com.amazon.opendistroforelasticsearch.sql.elasticsearch.security.SecurityAccess;
 import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException;
@@ -33,7 +34,6 @@
 import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse;
 import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName;
 import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics;
-import com.amazon.opendistroforelasticsearch.sql.legacy.utils.LogUtils;
 import com.amazon.opendistroforelasticsearch.sql.plugin.request.PPLQueryRequestFactory;
 import com.amazon.opendistroforelasticsearch.sql.ppl.PPLService;
 import com.amazon.opendistroforelasticsearch.sql.ppl.config.PPLServiceConfig;

ppl/build.gradle

@@ -31,6 +31,7 @@ dependencies {
     compile group: 'org.json', name: 'json', version: '20180813'
     compile group: 'org.springframework', name: 'spring-context', version: '5.2.5.RELEASE'
     compile group: 'org.springframework', name: 'spring-beans', version: '5.2.5.RELEASE'
+    compile group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.11.1'
     compile project(':common')
     compile project(':core')
     compile project(':protocol')

ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLService.java

@@ -21,6 +21,7 @@
 import com.amazon.opendistroforelasticsearch.sql.analysis.Analyzer;
 import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan;
 import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener;
+import com.amazon.opendistroforelasticsearch.sql.common.utils.LogUtils;
 import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine;
 import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.ExplainResponse;
 import com.amazon.opendistroforelasticsearch.sql.expression.DSL;
@@ -33,10 +34,13 @@
 import com.amazon.opendistroforelasticsearch.sql.ppl.domain.PPLQueryRequest;
 import com.amazon.opendistroforelasticsearch.sql.ppl.parser.AstBuilder;
 import com.amazon.opendistroforelasticsearch.sql.ppl.parser.AstExpressionBuilder;
+import com.amazon.opendistroforelasticsearch.sql.ppl.utils.PPLQueryDataAnonymizer;
 import com.amazon.opendistroforelasticsearch.sql.ppl.utils.UnresolvedPlanHelper;
 import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine;
 import lombok.RequiredArgsConstructor;
 import org.antlr.v4.runtime.tree.ParseTree;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 @RequiredArgsConstructor
 public class PPLService {
@@ -50,6 +54,10 @@ public class PPLService {
 
   private final BuiltinFunctionRepository repository;
 
+  private final PPLQueryDataAnonymizer anonymizer = new PPLQueryDataAnonymizer();
+
+  private static final Logger LOG = LogManager.getLogger();
+
   /**
    * Execute the {@link PPLQueryRequest}, using {@link ResponseListener} to get response.
    *
@@ -85,6 +93,8 @@ private PhysicalPlan plan(PPLQueryRequest request) {
     UnresolvedPlan ast = cst.accept(
         new AstBuilder(new AstExpressionBuilder(), request.getRequest()));
 
+    LOG.info("[{}] Incoming request {}", LogUtils.getRequestId(), anonymizer.anonymizeData(ast));
+
     // 2.Analyze abstract syntax to generate logical plan
     LogicalPlan logicalPlan = analyzer.analyze(UnresolvedPlanHelper.addSelectAll(ast),
         new AnalysisContext());