Merge branch 'master' into no-copy-settings-for-you

* master: (41 commits)
  Bump Gradle heap to 2 GB (elastic#30535)
  SQL: Use request flavored methods in tests (elastic#30345)
  Suppress hdfsFixture if there are spaces in the path (elastic#30302)
  Delete temporary blobs before creating index file (elastic#30528)
  Watcher: Remove TriggerEngine.getJobCount() (elastic#30395)
  [ML] Fix wire BWC for JobUpdate (elastic#30512)
  Use simpler write-once semantics for FS repository (elastic#30435)
  Derive max composite buffers from max content len
  Use simpler write-once semantics for HDFS repository (elastic#30439)
  SQL: Improve correctness of SYS COLUMNS & TYPES (elastic#30418)
  Mute two tests in FlushIT with @AwaitsFix.
  Fix incorrect template name in test case
  Build: Remove legacy bwc files from xpack (elastic#30485)
  Mute UnicastZenPingTests#testSimplePings with @AwaitsFix.
  Security: cleanup code in file stores (elastic#30348)
  Security: fix TokenMetaData equals and hashcode (elastic#30347)
  Mute two tests from SmokeTestWatcherWithSecurityClientYamlTestSuiteIT.
  Mute SharedClusterSnapshotRestoreIT#testSnapshotSucceedsAfterSnapshotFailure with @AwaitsFix.
  SQL: Improve compatibility with MS query (elastic#30516)
  SQL: Fix parsing of dates with milliseconds (elastic#30419)
  ...

benchmarks/build.gradle

@@ -24,22 +24,22 @@ buildscript {
         }
     }
     dependencies {
-        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.2'
+        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
     }
 }
 
 apply plugin: 'elasticsearch.build'
-// build an uberjar with all benchmarks
-apply plugin: 'com.github.johnrengelman.shadow'
-// have the shadow plugin provide the runShadow task
-apply plugin: 'application'
+
+// order of this seciont matters, see: https://github.com/johnrengelman/shadow/issues/336
+apply plugin: 'application' // have the shadow plugin provide the runShadow task
+mainClassName = 'org.openjdk.jmh.Main'
+apply plugin: 'com.github.johnrengelman.shadow' // build an uberjar with all benchmarks
 
 // Not published so no need to assemble
 tasks.remove(assemble)
 build.dependsOn.remove('assemble')
 
 archivesBaseName = 'elasticsearch-benchmarks'
-mainClassName = 'org.openjdk.jmh.Main'
 
 test.enabled = false
 

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.4.0-snapshot-1ed95c097b
+lucene            = 7.4.0-snapshot-6705632810
 
 # optional dependencies
 spatial4j         = 0.7

client/rest/src/main/java/org/elasticsearch/client/Request.java

@@ -19,8 +19,10 @@
 
 package org.elasticsearch.client;
 
+import org.apache.http.entity.ContentType;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
+import org.apache.http.nio.entity.NStringEntity;
 import org.apache.http.nio.protocol.HttpAsyncResponseConsumer;
 
 import java.util.Arrays;
@@ -103,6 +105,17 @@ public void setEntity(HttpEntity entity) {
         this.entity = entity;
     }
 
+    /**
+     * Set the body of the request to a string. If not set or set to
+     * {@code null} then no body is sent with the request. The
+     * {@code Content-Type} will be sent as {@code application/json}.
+     * If you need a different content type then use
+     * {@link #setEntity(HttpEntity)}.
+     */
+    public void setJsonEntity(String entity) {
+        setEntity(entity == null ? null : new NStringEntity(entity, ContentType.APPLICATION_JSON));
+    }
+
     /**
      * The body of the request. If {@code null} then no body
      * is sent with the request.

client/rest/src/test/java/org/elasticsearch/client/RequestTests.java

@@ -19,6 +19,8 @@
 
 package org.elasticsearch.client;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -27,9 +29,11 @@
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.nio.entity.NStringEntity;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.fail;
 
 public class RequestTests extends RestClientTestCase {
@@ -99,12 +103,27 @@ public void testSetEntity() {
         final String endpoint = randomAsciiLettersOfLengthBetween(1, 10);
         final HttpEntity entity =
                 randomBoolean() ? new StringEntity(randomAsciiLettersOfLengthBetween(1, 100), ContentType.TEXT_PLAIN) : null;
-        Request request = new Request(method, endpoint);
 
+        Request request = new Request(method, endpoint);
         request.setEntity(entity);
         assertEquals(entity, request.getEntity());
     }
 
+    public void testSetJsonEntity() throws IOException {
+        final String method = randomFrom(new String[] {"GET", "PUT", "POST", "HEAD", "DELETE"});
+        final String endpoint = randomAsciiLettersOfLengthBetween(1, 10);
+
+        Request request = new Request(method, endpoint);
+        assertNull(request.getEntity());
+
+        final String json = randomAsciiLettersOfLengthBetween(1, 100);
+        request.setJsonEntity(json);
+        assertEquals(ContentType.APPLICATION_JSON.toString(), request.getEntity().getContentType().getValue());
+        ByteArrayOutputStream os = new ByteArrayOutputStream();
+        request.getEntity().writeTo(os);
+        assertEquals(json, new String(os.toByteArray(), ContentType.APPLICATION_JSON.getCharset()));
+    }
+
     public void testSetHeaders() {
         final String method = randomFrom(new String[] {"GET", "PUT", "POST", "HEAD", "DELETE"});
         final String endpoint = randomAsciiLettersOfLengthBetween(1, 10);

client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java

@@ -35,7 +35,9 @@
 import org.apache.http.nio.entity.NStringEntity;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.mocksocket.MockHttpServer;
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
@@ -69,20 +71,20 @@
  */
 public class RestClientSingleHostIntegTests extends RestClientTestCase {
 
-    private static HttpServer httpServer;
-    private static RestClient restClient;
-    private static String pathPrefix;
-    private static Header[] defaultHeaders;
+    private HttpServer httpServer;
+    private RestClient restClient;
+    private String pathPrefix;
+    private Header[] defaultHeaders;
 
-    @BeforeClass
-    public static void startHttpServer() throws Exception {
+    @Before
+    public void startHttpServer() throws Exception {
         pathPrefix = randomBoolean() ? "/testPathPrefix/" + randomAsciiLettersOfLengthBetween(1, 5) : "";
         httpServer = createHttpServer();
         defaultHeaders = RestClientTestUtil.randomHeaders(getRandom(), "Header-default");
         restClient = createRestClient(false, true);
     }
 
-    private static HttpServer createHttpServer() throws Exception {
+    private HttpServer createHttpServer() throws Exception {
         HttpServer httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
         httpServer.start();
         //returns a different status code depending on the path
@@ -127,7 +129,7 @@ public void handle(HttpExchange httpExchange) throws IOException {
         }
     }
 
-    private static RestClient createRestClient(final boolean useAuth, final boolean usePreemptiveAuth) {
+    private RestClient createRestClient(final boolean useAuth, final boolean usePreemptiveAuth) {
         // provide the username/password for every request
         final BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
         credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("user", "pass"));
@@ -155,8 +157,8 @@ public HttpAsyncClientBuilder customizeHttpClient(final HttpAsyncClientBuilder h
         return restClientBuilder.build();
     }
 
-    @AfterClass
-    public static void stopHttpServers() throws IOException {
+    @After
+    public void stopHttpServers() throws IOException {
         restClient.close();
         restClient = null;
         httpServer.stop(0);

client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java

@@ -168,10 +168,13 @@ public void onFailure(Exception exception) {
             request.addParameter("pretty", "true");
             //end::rest-client-parameters
             //tag::rest-client-body
-            request.setEntity(new StringEntity(
+            request.setEntity(new NStringEntity(
                     "{\"json\":\"text\"}",
                     ContentType.APPLICATION_JSON));
             //end::rest-client-body
+            //tag::rest-client-body-shorter
+            request.setJsonEntity("{\"json\":\"text\"}");
+            //end::rest-client-body-shorter
             //tag::rest-client-headers
             request.setHeaders(
                     new BasicHeader("Accept", "text/plain"),

distribution/build.gradle

@@ -312,6 +312,22 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
 
 task run(type: RunTask) {
   distribution = System.getProperty('run.distribution', 'zip')
+  if (distribution == 'zip') {
+    String licenseType = System.getProperty("license_type", "basic")
+    if (licenseType == 'trial') {
+      setting 'xpack.ml.enabled', 'true'
+      setting 'xpack.graph.enabled', 'true'
+      setting 'xpack.watcher.enabled', 'true'
+      setting 'xpack.license.self_generated.type', 'trial'
+    } else if (licenseType != 'basic') {
+      throw new IllegalArgumentException("Unsupported self-generated license type: [" + licenseType + "[basic] or [trial].")
+    }
+    setting 'xpack.security.enabled', 'true'
+    setting 'xpack.monitoring.enabled', 'true'
+    setting 'xpack.sql.enabled', 'true'
+    setting 'xpack.rollup.enabled', 'true'
+    keystoreSetting 'bootstrap.password', 'password'
+  }
 }
 
 /**

docs/CHANGELOG.asciidoc

@@ -115,6 +115,9 @@ Rollup::
 * Validate timezone in range queries to ensure they match the selected job when
 searching ({pull}30338[#30338])
 
+SQL::
+* Fix parsing of Dates containing milliseconds ({pull}30419[#30419])
+
 [float]
 === Regressions
 Fail snapshot operations early when creating or deleting a snapshot on a repository that has been
@@ -166,6 +169,7 @@ copying source settings on index resize operations] ({pull}30255[#30255], {pull}
 Added new "Request" object flavored request methods in the RestClient. Prefer
 these instead of the multi-argument versions. ({pull}29623[#29623])
 
+Added `setJsonEntity` to `Request` object so it is marginally easier to send JSON. ({pull}30447[#30447])
 Watcher HTTP client used in watches now allows more parallel connections to the
 same endpoint and evicts long running connections. ({pull}30130[#30130])
 
@@ -201,6 +205,8 @@ Rollup::
 * Validate timezone in range queries to ensure they match the selected job when
 searching ({pull}30338[#30338])
 
+SQL::
+* Fix parsing of Dates containing milliseconds ({pull}30419[#30419])
 
 Allocation::
 
@@ -241,6 +247,9 @@ Reduce the number of object allocations made by {security} when resolving the in
 
 Respect accept header on requests with no handler ({pull}30383[#30383])
 
+SQL::
+* Fix parsing of Dates containing milliseconds ({pull}30419[#30419])
+
 //[float]
 //=== Regressions
 

docs/java-rest/low-level/usage.asciidoc

@@ -263,6 +263,14 @@ IMPORTANT: The `ContentType` specified for the `HttpEntity` is important
 because it will be used to set the `Content-Type` header so that Elasticsearch
 can properly parse the content.
 
+You can also set it to a `String` which will default to
+a `ContentType` of `application/json`.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-body-shorter]
+--------------------------------------------------
+
 And you can set a list of headers to send with the request:
 
 ["source","java",subs="attributes,callouts,macros"]

docs/reference/indices/forcemerge.asciidoc

@@ -10,6 +10,11 @@ This call will block until the merge is complete. If the http connection is
 lost, the request will continue in the background, and any new requests will
 block until the previous force merge is complete.
 
+WARNING: Force merge should only be called against *read-only indices*. Running 
+force merge against a read-write index can cause very large segments to be produced 
+(>5Gb per segment), and the merge policy will never consider it for merging again until 
+it mostly consists of deleted docs. This can cause very large segments to remain in the shards.
+
 [source,js]
 --------------------------------------------------
 POST /twitter/_forcemerge

gradle.properties

@@ -1,2 +1,2 @@
 org.gradle.daemon=false
-org.gradle.jvmargs=-Xmx1536m
+org.gradle.jvmargs=-Xmx2g

modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-6705632810.jar.sha1

@@ -0,0 +1 @@
+f72ad4b6474c2d59b0eed0ca84eddd1f99d29129

modules/reindex/build.gradle

@@ -30,11 +30,17 @@ esplugin {
 }
 
 integTestCluster {
+  // Modules who's integration is explicitly tested in integration tests
+  module project(':modules:parent-join')
+  module project(':modules:lang-painless')
   // Whitelist reindexing from the local node so we can test reindex-from-remote.
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
 }
 
 run {
+  // Modules who's integration is explicitly tested in integration tests
+  module project(':modules:parent-join')
+  module project(':modules:lang-painless')
   // Whitelist reindexing from the local node so we can test reindex-from-remote.
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
 }

modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java

@@ -19,10 +19,8 @@
 
 package org.elasticsearch.index.reindex.remote;
 
-import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.lucene.util.BytesRef;
+import org.apache.http.nio.entity.NStringEntity;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.search.SearchRequest;
@@ -151,8 +149,7 @@ static Request initialSearch(SearchRequest searchRequest, BytesReference query,
             }
 
             entity.endObject();
-            BytesRef bytes = BytesReference.bytes(entity).toBytesRef();
-            request.setEntity(new ByteArrayEntity(bytes.bytes, bytes.offset, bytes.length, ContentType.APPLICATION_JSON));
+            request.setJsonEntity(Strings.toString(entity));
         } catch (IOException e) {
             throw new ElasticsearchException("unexpected error building entity", e);
         }
@@ -199,15 +196,15 @@ static Request scroll(String scroll, TimeValue keepAlive, Version remoteVersion)
 
         if (remoteVersion.before(Version.fromId(2000099))) {
             // Versions before 2.0.0 extract the plain scroll_id from the body
-            request.setEntity(new StringEntity(scroll, ContentType.TEXT_PLAIN));
+            request.setEntity(new NStringEntity(scroll, ContentType.TEXT_PLAIN));
             return request;
         }
 
         try (XContentBuilder entity = JsonXContent.contentBuilder()) {
             entity.startObject()
                     .field("scroll_id", scroll)
                 .endObject();
-            request.setEntity(new StringEntity(Strings.toString(entity), ContentType.APPLICATION_JSON));
+            request.setJsonEntity(Strings.toString(entity));
         } catch (IOException e) {
             throw new ElasticsearchException("failed to build scroll entity", e);
         }
@@ -219,14 +216,14 @@ static Request clearScroll(String scroll, Version remoteVersion) {
 
         if (remoteVersion.before(Version.fromId(2000099))) {
             // Versions before 2.0.0 extract the plain scroll_id from the body
-            request.setEntity(new StringEntity(scroll, ContentType.TEXT_PLAIN));
+            request.setEntity(new NStringEntity(scroll, ContentType.TEXT_PLAIN));
             return request;
         }
         try (XContentBuilder entity = JsonXContent.contentBuilder()) {
             entity.startObject()
                     .array("scroll_id", scroll)
                 .endObject();
-            request.setEntity(new StringEntity(Strings.toString(entity), ContentType.APPLICATION_JSON));
+            request.setJsonEntity(Strings.toString(entity));
         } catch (IOException e) {
             throw new ElasticsearchException("failed to build clear scroll entity", e);
         }

qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yml → modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml

@@ -1,5 +1,5 @@
 ---
-"Response format search failures":
+"Response format for search failures":
   - do:
       index:
         index:   source

qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/10_script.yml → modules/reindex/src/test/resources/rest-api-spec/test/reindex/85_scripting.yml

@@ -433,3 +433,28 @@
   - match:  { docs.2._index:     index2      }
   - match:  { docs.2._type:      type2       }
   - match:  { docs.2._id:        fr_789      }
+
+---
+"Totally broken scripts report the error properly":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: request
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: twitter
+          dest:
+            index: new_twitter
+          script:
+            lang: painless
+            source: syntax errors are fun!
+  - match: {error.reason: 'compile error'}