Merge branch 'master' of github.com:elastic/elasticsearch into ccr-docs

* 'master' of github.com:elastic/elasticsearch:
  Fix line length for org.elasticsearch.common.* files (elastic#34888)
  [ML] Extract common native process base class (elastic#34856)
  Refactor children aggregator into a generic ParentJoinAggregator (elastic#34845)
  [Style] Fix line lengths in action.admin.indices (elastic#34890)
  HLRC - add support for source exists API (elastic#34519)
  [CCR] Retry when no index shard stats can be found (elastic#34852)
  [Docs] audit logfile structured format (elastic#34584)
  [Test] Fix FullClusterRestartIT.testShrink() with copy_settings param (elastic#34853)
  Fix LineLength Check Suppressions: index.fielddata (elastic#34891)
  TEST: Stablize Minio Free Port Search (elastic#34894)
  Delete flaky SettingsBasedHostProviderIT test (elastic#34813)
  [ML] Include message in field_stats for text log files (elastic#34861)
  [TEST] HLRC: Expand failure messages in API checks (elastic#34838)
  Lowercase static final DeprecationLogger instance names (elastic#34887)

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -261,6 +261,18 @@ private static Request getStyleRequest(String method, GetRequest getRequest) {
 
         return request;
     }
+
+    static Request sourceExists(GetRequest getRequest) {
+        Request request = new Request(HttpHead.METHOD_NAME, endpoint(getRequest.index(), getRequest.type(), getRequest.id(), "_source"));
+
+        Params parameters = new Params(request);
+        parameters.withPreference(getRequest.preference());
+        parameters.withRouting(getRequest.routing());
+        parameters.withRefresh(getRequest.refresh());
+        parameters.withRealtime(getRequest.realtime());
+        // Version params are not currently supported by the source exists API so are not passed
+        return request;
+    }    
 
     static Request multiGet(MultiGetRequest multiGetRequest) throws IOException {
         Request request = new Request(HttpPost.METHOD_NAME, "/_mget");

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -727,6 +727,32 @@ public final void existsAsync(GetRequest getRequest, RequestOptions options, Act
                 emptySet());
     }
 
+    /**
+     * Checks for the existence of a document with a "_source" field. Returns true if it exists, false otherwise.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html#_source">Source exists API 
+     * on elastic.co</a>
+     * @param getRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return <code>true</code> if the document and _source field exists, <code>false</code> otherwise
+     * @throws IOException in case there is a problem sending the request
+     */
+    public boolean existsSource(GetRequest getRequest, RequestOptions options) throws IOException {
+        return performRequest(getRequest, RequestConverters::sourceExists, options, RestHighLevelClient::convertExistsResponse, emptySet());
+    }     
+
+    /**
+     * Asynchronously checks for the existence of a document with a "_source" field. Returns true if it exists, false otherwise.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html#_source">Source exists API 
+     * on elastic.co</a>
+     * @param getRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public final void existsSourceAsync(GetRequest getRequest, RequestOptions options, ActionListener<Boolean> listener) {
+        performRequestAsync(getRequest, RequestConverters::sourceExists, options, RestHighLevelClient::convertExistsResponse, listener,
+                emptySet());
+    }    
+
     /**
      * Index a document using the Index API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html">Index API on elastic.co</a>

client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java

@@ -194,6 +194,61 @@ public void testExists() throws IOException {
             assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
         }
     }
+
+    public void testSourceExists() throws IOException {     
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "id");
+            assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
+        }
+        IndexRequest index = new IndexRequest("index", "type", "id");
+        index.source("{\"field1\":\"value1\",\"field2\":\"value2\"}", XContentType.JSON);
+        index.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+        highLevelClient().index(index, RequestOptions.DEFAULT);
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "id");
+            assertTrue(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
+        }
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "does_not_exist");
+            assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
+        }
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "does_not_exist").version(1);
+            assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
+        }
+    }
+
+    public void testSourceDoesNotExist() throws IOException {     
+        final String noSourceIndex = "no_source";
+        {
+            // Prepare
+            Settings settings = Settings.builder()
+                .put("number_of_shards", 1)
+                .put("number_of_replicas", 0)
+                .build();
+            String mapping = "\"_doc\": { \"_source\": {\n" + 
+                    "        \"enabled\": false\n" + 
+                    "      }  }";
+            createIndex(noSourceIndex, settings, mapping);
+            assertEquals(
+                RestStatus.OK,
+                highLevelClient().bulk(
+                    new BulkRequest()
+                        .add(new IndexRequest(noSourceIndex, "_doc", "1")
+                            .source(Collections.singletonMap("foo", 1), XContentType.JSON))
+                        .add(new IndexRequest(noSourceIndex, "_doc", "2")
+                            .source(Collections.singletonMap("foo", 2), XContentType.JSON))
+                        .setRefreshPolicy(RefreshPolicy.IMMEDIATE),
+                    RequestOptions.DEFAULT
+                ).status()
+            );
+        }        
+        {
+            GetRequest getRequest = new GetRequest(noSourceIndex, "_doc", "1");
+            assertTrue(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
+            assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
+        }
+    }    
 
     public void testGet() throws IOException {
         {

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -650,7 +650,6 @@ public void testApiNamingConventions() throws Exception {
             "cluster.remote_info",
             "count",
             "create",
-            "exists_source",
             "get_source",
             "indices.delete_alias",
             "indices.delete_template",
@@ -712,41 +711,49 @@ public void testApiNamingConventions() throws Exception {
 
             assertTrue("method [" + apiName + "] is not final",
                     Modifier.isFinal(method.getClass().getModifiers()) || Modifier.isFinal(method.getModifiers()));
-            assertTrue(Modifier.isPublic(method.getModifiers()));
+            assertTrue("method [" + method + "] should be public", Modifier.isPublic(method.getModifiers()));
 
             //we convert all the method names to snake case, hence we need to look for the '_async' suffix rather than 'Async'
             if (apiName.endsWith("_async")) {
                 assertTrue("async method [" + method.getName() + "] doesn't have corresponding sync method",
                         methods.containsKey(apiName.substring(0, apiName.length() - 6)));
-                assertThat(method.getReturnType(), equalTo(Void.TYPE));
-                assertEquals(0, method.getExceptionTypes().length);
+                assertThat("async method [" + method + "] should return void", method.getReturnType(), equalTo(Void.TYPE));
+                assertEquals("async method [" + method + "] should not throw any exceptions", 0, method.getExceptionTypes().length);
                 if (apiName.equals("security.get_ssl_certificates_async")) {
                     assertEquals(2, method.getParameterTypes().length);
                     assertThat(method.getParameterTypes()[0], equalTo(RequestOptions.class));
                     assertThat(method.getParameterTypes()[1], equalTo(ActionListener.class));
                 } else {
-                    assertEquals(3, method.getParameterTypes().length);
-                    assertThat(method.getParameterTypes()[0].getSimpleName(), endsWith("Request"));
-                    assertThat(method.getParameterTypes()[1], equalTo(RequestOptions.class));
-                    assertThat(method.getParameterTypes()[2], equalTo(ActionListener.class));
+                    assertEquals("async method [" + method + "] has the wrong number of arguments", 3, method.getParameterTypes().length);
+                    assertThat("the first parameter to async method [" + method + "] should be a request type",
+                        method.getParameterTypes()[0].getSimpleName(), endsWith("Request"));
+                    assertThat("the second parameter to async method [" + method + "] is the wrong type",
+                        method.getParameterTypes()[1], equalTo(RequestOptions.class));
+                    assertThat("the third parameter to async method [" + method + "] is the wrong type",
+                        method.getParameterTypes()[2], equalTo(ActionListener.class));
                 }
             } else {
                 //A few methods return a boolean rather than a response object
                 if (apiName.equals("ping") || apiName.contains("exist")) {
-                    assertThat(method.getReturnType().getSimpleName(), equalTo("boolean"));
+                    assertThat("the return type for method [" + method + "] is incorrect",
+                        method.getReturnType().getSimpleName(), equalTo("boolean"));
                 } else {
-                    assertThat(method.getReturnType().getSimpleName(), endsWith("Response"));
+                    assertThat("the return type for method [" + method + "] is incorrect",
+                        method.getReturnType().getSimpleName(), endsWith("Response"));
                 }
 
-                assertEquals(1, method.getExceptionTypes().length);
+                assertEquals("incorrect number of exceptions for method [" + method + "]", 1, method.getExceptionTypes().length);
                 //a few methods don't accept a request object as argument
                 if (apiName.equals("ping") || apiName.equals("info") || apiName.equals("security.get_ssl_certificates")) {
-                    assertEquals(1, method.getParameterTypes().length);
-                    assertThat(method.getParameterTypes()[0], equalTo(RequestOptions.class));
+                    assertEquals("incorrect number of arguments for method [" + method + "]", 1, method.getParameterTypes().length);
+                    assertThat("the parameter to method [" + method + "] is the wrong type",
+                        method.getParameterTypes()[0], equalTo(RequestOptions.class));
                 } else {
-                    assertEquals(apiName, 2, method.getParameterTypes().length);
-                    assertThat(method.getParameterTypes()[0].getSimpleName(), endsWith("Request"));
-                    assertThat(method.getParameterTypes()[1], equalTo(RequestOptions.class));
+                    assertEquals("incorrect number of arguments for method [" + method + "]", 2, method.getParameterTypes().length);
+                    assertThat("the first parameter to method [" + method + "] is the wrong type",
+                        method.getParameterTypes()[0].getSimpleName(), endsWith("Request"));
+                    assertThat("the second parameter to method [" + method + "] is the wrong type",
+                        method.getParameterTypes()[1], equalTo(RequestOptions.class));
                 }
 
                 boolean remove = apiSpec.remove(apiName);

docs/java-rest/high-level/document/exists.asciidoc

@@ -29,3 +29,10 @@ include-tagged::{doc-tests-file}[{api}-request]
 <5> Disable fetching stored fields.
 
 include::../execution.asciidoc[]
+
+
+==== Source exists request
+A variant of the exists request is `existsSource` method which has the additional check
+that the document in question has stored the `source`. If the mapping for the index has opted
+to remove support for storing JSON source in documents then this method will return false
+for documents in this index.

docs/reference/settings/audit-settings.asciidoc

@@ -18,10 +18,18 @@ Set to `true` to enable auditing on the node. The default value is `false`.
 `xpack.security.audit.outputs`::
 Specifies where audit logs are output. For example: `[ index, logfile ]`. The
 default value is `logfile`, which puts the auditing events in a dedicated
-`<clustername>_access.log` file on the node. You can also specify `index`, which
-puts the auditing events in an {es} index that is prefixed with
-`.security_audit_log`. The index can reside on the same cluster or a separate
-cluster.
+file named `<clustername>_audit.log` on each node.
+You can also specify `index`, which puts the auditing events in an {es} index
+that is prefixed with `.security_audit_log`. The index can reside on the same
+cluster or a separate cluster.
+
+For backwards compatibility reasons, if you use the logfile output type, a
+`<clustername>_access.log` file is also created. It contains the same
+information, but it uses the older (pre-6.5.0) formatting style.
+If the backwards compatible format is not required, it should be disabled.
+To do that, change its logger level to `off` in the `log4j2.properties` file.
+For more information, see <<configuring-logging-levels>>.
+
 +
 --
 TIP: If the index is unavailable, it is possible for auditing events to
@@ -57,17 +65,27 @@ audited in plain text when including the request body in audit events.
 [[node-audit-settings]]
 ==== Local Node Info Settings
 
-`xpack.security.audit.logfile.prefix.emit_node_name`::
-Specifies whether to include the node's name in the local node info. The
-default value is `true`.
-
-`xpack.security.audit.logfile.prefix.emit_node_host_address`::
-Specifies whether to include the node's IP address in the local node info. The
-default value is `false`.
-
-`xpack.security.audit.logfile.prefix.emit_node_host_name`::
-Specifies whether to include the node's host name in the local node info. The
-default value is `false`.
+`xpack.security.audit.logfile.emit_node_name`::
+Specifies whether to include the <<node.name,node name>> as a field in
+each audit event.
+The default value is `true`.
+
+`xpack.security.audit.logfile.emit_node_host_address`::
+Specifies whether to include the node's IP address as a field in each audit event.
+The default value is `false`.
+
+`xpack.security.audit.logfile.emit_node_host_name`::
+Specifies whether to include the node's host name as a field in each audit event.
+The default value is `false`.
+
+`xpack.security.audit.logfile.emit_node_id`::
+Specifies whether to include the node id as a field in each audit event.
+This is available for the new format only. That is to say, this information
+does not exist in the `<clustername>_access.log` file.
+Unlike <<node.name,node name>>, whose value might change if the administrator
+changes the setting in the config file, the node id will persist across cluster
+restarts and the administrator cannot change it.
+The default value is `true`.
 
 [[index-audit-settings]]
 ==== Audit Log Indexing Configuration Settings

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.analysis.common;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.CharArraySet;
 import org.apache.lucene.analysis.LowerCaseFilter;
@@ -115,7 +116,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
@@ -151,7 +151,7 @@
 
 public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin, ScriptPlugin {
 
-    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(CommonAnalysisPlugin.class));
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(CommonAnalysisPlugin.class));
 
     private final SetOnce<ScriptService> scriptService = new SetOnce<>();
 
@@ -376,7 +376,7 @@ public List<PreConfiguredCharFilter> getPreConfiguredCharFilters() {
         filters.add(PreConfiguredCharFilter.singleton("html_strip", false, HTMLStripCharFilter::new));
         filters.add(PreConfiguredCharFilter.singletonWithVersion("htmlStrip", false, (reader, version) -> {
             if (version.onOrAfter(org.elasticsearch.Version.V_6_3_0)) {
-                DEPRECATION_LOGGER.deprecatedAndMaybeLog("htmlStrip_deprecation",
+                deprecationLogger.deprecatedAndMaybeLog("htmlStrip_deprecation",
                         "The [htmpStrip] char filter name is deprecated and will be removed in a future version. "
                                 + "Please change the filter name to [html_strip] instead.");
             }
@@ -414,7 +414,7 @@ public List<PreConfiguredTokenFilter> getPreConfiguredTokenFilters() {
                 new EdgeNGramTokenFilter(input, 1)));
         filters.add(PreConfiguredTokenFilter.singletonWithVersion("edgeNGram", false, (reader, version) -> {
             if (version.onOrAfter(org.elasticsearch.Version.V_6_4_0)) {
-                DEPRECATION_LOGGER.deprecatedAndMaybeLog("edgeNGram_deprecation",
+                deprecationLogger.deprecatedAndMaybeLog("edgeNGram_deprecation",
                         "The [edgeNGram] token filter name is deprecated and will be removed in a future version. "
                                 + "Please change the filter name to [edge_ngram] instead.");
             }
@@ -438,7 +438,7 @@ public List<PreConfiguredTokenFilter> getPreConfiguredTokenFilters() {
         filters.add(PreConfiguredTokenFilter.singleton("ngram", false, reader -> new NGramTokenFilter(reader, 1, 2, false)));
         filters.add(PreConfiguredTokenFilter.singletonWithVersion("nGram", false, (reader, version) -> {
             if (version.onOrAfter(org.elasticsearch.Version.V_6_4_0)) {
-                DEPRECATION_LOGGER.deprecatedAndMaybeLog("nGram_deprecation",
+                deprecationLogger.deprecatedAndMaybeLog("nGram_deprecation",
                         "The [nGram] token filter name is deprecated and will be removed in a future version. "
                                 + "Please change the filter name to [ngram] instead.");
             }

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LegacyDelimitedPayloadTokenFilterFactory.java

@@ -19,16 +19,16 @@
 
 package org.elasticsearch.analysis.common;
 
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 
 public class LegacyDelimitedPayloadTokenFilterFactory extends DelimitedPayloadTokenFilterFactory {
-    private static final DeprecationLogger DEPRECATION_LOGGER =
-        new DeprecationLogger(Loggers.getLogger(LegacyDelimitedPayloadTokenFilterFactory.class));
+    private static final DeprecationLogger deprecationLogger =
+        new DeprecationLogger(LogManager.getLogger(LegacyDelimitedPayloadTokenFilterFactory.class));
 
     LegacyDelimitedPayloadTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, env, name, settings);
@@ -37,7 +37,7 @@ public class LegacyDelimitedPayloadTokenFilterFactory extends DelimitedPayloadTo
                     "[delimited_payload_filter] is not supported for new indices, use [delimited_payload] instead");
         }
         if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_6_2_0)) {
-            DEPRECATION_LOGGER.deprecated("Deprecated [delimited_payload_filter] used, replaced by [delimited_payload]");
+            deprecationLogger.deprecated("Deprecated [delimited_payload_filter] used, replaced by [delimited_payload]");
         }
     }
 }