Merge branch '6.x' into feature-jindex-6x

Resolve conflicts due to fields removed from AnalysisConfig

build.gradle

@@ -25,6 +25,9 @@ import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
+import com.carrotsearch.gradle.junit4.RandomizedTestingTask
+
+import java.util.function.Predicate
 
 plugins {
     id 'com.gradle.build-scan' version '1.13.2'
@@ -611,3 +614,19 @@ allprojects {
     }
   }
 }
+
+allprojects {
+  task checkPart1
+  task checkPart2 
+  tasks.matching { it.name == "check" }.all { check ->
+    if (check.path.startsWith(":x-pack:")) {
+      checkPart2.dependsOn check
+    } else {
+      checkPart1.dependsOn check
+    }
+  } 
+}
+
+
+
+

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -443,11 +443,11 @@ class BuildPlugin implements Plugin<Project> {
             // such that we don't have to pass hardcoded files to gradle
             repos.mavenLocal()
         }
-        repos.jcenter()
         repos.maven {
             name "elastic"
             url "https://artifacts.elastic.co/maven"
         }
+        repos.jcenter()
         String luceneVersion = VersionProperties.lucene
         if (luceneVersion.contains('-snapshot')) {
             // extract the revision number from the version with a regex matcher

client/rest-high-level/src/main/java/org/elasticsearch/client/CcrClient.java

@@ -23,6 +23,8 @@
 import org.elasticsearch.client.ccr.PauseFollowRequest;
 import org.elasticsearch.client.ccr.PutFollowRequest;
 import org.elasticsearch.client.ccr.PutFollowResponse;
+import org.elasticsearch.client.ccr.ResumeFollowRequest;
+import org.elasticsearch.client.ccr.UnfollowRequest;
 import org.elasticsearch.client.core.AcknowledgedResponse;
 
 import java.io.IOException;
@@ -89,7 +91,7 @@ public void putFollowAsync(PutFollowRequest request,
     }
 
     /**
-     * Instructs a follower index the pause the following of a leader index.
+     * Instructs a follower index to pause the following of a leader index.
      *
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-pause-follow.html">
      * the docs</a> for more.
@@ -110,7 +112,7 @@ public AcknowledgedResponse pauseFollow(PauseFollowRequest request, RequestOptio
     }
 
     /**
-     * Asynchronously instruct a follower index the pause the following of a leader index.
+     * Asynchronously instruct a follower index to pause the following of a leader index.
      *
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-pause-follow.html">
      * the docs</a> for more.
@@ -130,4 +132,91 @@ public void pauseFollowAsync(PauseFollowRequest request,
             Collections.emptySet());
     }
 
+    /**
+     * Instructs a follower index to resume the following of a leader index.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-resume-follow.html">
+     * the docs</a> for more.
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public AcknowledgedResponse resumeFollow(ResumeFollowRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(
+            request,
+            CcrRequestConverters::resumeFollow,
+            options,
+            AcknowledgedResponse::fromXContent,
+            Collections.emptySet()
+        );
+    }
+
+    /**
+     * Asynchronously instruct a follower index to resume the following of a leader index.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-resume-follow.html">
+     * the docs</a> for more.
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     */
+    public void resumeFollowAsync(ResumeFollowRequest request,
+                                  RequestOptions options,
+                                  ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(
+            request,
+            CcrRequestConverters::resumeFollow,
+            options,
+            AcknowledgedResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
+    /**
+     * Instructs a follower index to unfollow and become a regular index.
+     * Note that index following needs to be paused and the follower index needs to be closed.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-unfollow.html">
+     * the docs</a> for more.
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public AcknowledgedResponse unfollow(UnfollowRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(
+            request,
+            CcrRequestConverters::unfollow,
+            options,
+            AcknowledgedResponse::fromXContent,
+            Collections.emptySet()
+        );
+    }
+
+    /**
+     * Asynchronously instructs a follower index to unfollow and become a regular index.
+     * Note that index following needs to be paused and the follower index needs to be closed.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-unfollow.html">
+     * the docs</a> for more.
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     */
+    public void unfollowAsync(UnfollowRequest request,
+                              RequestOptions options,
+                              ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(
+            request,
+            CcrRequestConverters::unfollow,
+            options,
+            AcknowledgedResponse::fromXContent,
+            listener,
+            Collections.emptySet()
+        );
+    }
+
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java

@@ -23,6 +23,8 @@
 import org.apache.http.client.methods.HttpPut;
 import org.elasticsearch.client.ccr.PauseFollowRequest;
 import org.elasticsearch.client.ccr.PutFollowRequest;
+import org.elasticsearch.client.ccr.ResumeFollowRequest;
+import org.elasticsearch.client.ccr.UnfollowRequest;
 
 import java.io.IOException;
 
@@ -49,4 +51,22 @@ static Request pauseFollow(PauseFollowRequest pauseFollowRequest) {
         return new Request(HttpPost.METHOD_NAME, endpoint);
     }
 
+    static Request resumeFollow(ResumeFollowRequest resumeFollowRequest) throws IOException {
+        String endpoint = new RequestConverters.EndpointBuilder()
+            .addPathPart(resumeFollowRequest.getFollowerIndex())
+            .addPathPartAsIs("_ccr", "resume_follow")
+            .build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(resumeFollowRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
+    static Request unfollow(UnfollowRequest unfollowRequest) {
+        String endpoint = new RequestConverters.EndpointBuilder()
+            .addPathPart(unfollowRequest.getFollowerIndex())
+            .addPathPartAsIs("_ccr", "unfollow")
+            .build();
+        return new Request(HttpPost.METHOD_NAME, endpoint);
+    }
+
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleClient.java

@@ -76,7 +76,7 @@ public void getLifecyclePolicyAsync(GetLifecyclePolicyRequest request, RequestOp
     }
 
     /**
-     * Create or modify a lifecycle definition See <a href=
+     * Create or modify a lifecycle definition. See <a href=
      * "https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-ilm-ilm-put-lifecycle-policy.html">
      * the docs</a> for more.
      * @param request the request
@@ -91,8 +91,8 @@ public AcknowledgedResponse putLifecyclePolicy(PutLifecyclePolicyRequest request
     }
 
     /**
-     * Asynchronously create or modify a lifecycle definition
-     * See <a href="https://fix-me-when-we-have-docs.com">
+     * Asynchronously create or modify a lifecycle definition. See <a href=
+     * "https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-ilm-ilm-put-lifecycle-policy.html">
      * the docs</a> for more.
      * @param request the request
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
@@ -282,7 +282,7 @@ public void explainLifecycleAsync(ExplainLifecycleRequest request, RequestOption
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public AcknowledgedResponse retryLifecycleStep(RetryLifecyclePolicyRequest request, RequestOptions options) throws IOException {
+    public AcknowledgedResponse retryLifecyclePolicy(RetryLifecyclePolicyRequest request, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(request, IndexLifecycleRequestConverters::retryLifecycle, options,
             AcknowledgedResponse::fromXContent, emptySet());
     }
@@ -295,8 +295,8 @@ public AcknowledgedResponse retryLifecycleStep(RetryLifecyclePolicyRequest reque
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      */
-    public void retryLifecycleStepAsync(RetryLifecyclePolicyRequest request, RequestOptions options,
-        ActionListener<AcknowledgedResponse> listener) {
+    public void retryLifecyclePolicyAsync(RetryLifecyclePolicyRequest request, RequestOptions options,
+                                          ActionListener<AcknowledgedResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(request, IndexLifecycleRequestConverters::retryLifecycle, options,
             AcknowledgedResponse::fromXContent, listener, emptySet());
     }

client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java

@@ -28,6 +28,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.client.RequestConverters.EndpointBuilder;
 import org.elasticsearch.client.ml.CloseJobRequest;
+import org.elasticsearch.client.ml.DeleteCalendarJobRequest;
 import org.elasticsearch.client.ml.DeleteCalendarRequest;
 import org.elasticsearch.client.ml.DeleteDatafeedRequest;
 import org.elasticsearch.client.ml.DeleteFilterRequest;
@@ -37,6 +38,7 @@
 import org.elasticsearch.client.ml.FlushJobRequest;
 import org.elasticsearch.client.ml.ForecastJobRequest;
 import org.elasticsearch.client.ml.GetBucketsRequest;
+import org.elasticsearch.client.ml.GetCalendarEventsRequest;
 import org.elasticsearch.client.ml.GetCalendarsRequest;
 import org.elasticsearch.client.ml.GetCategoriesRequest;
 import org.elasticsearch.client.ml.GetDatafeedRequest;
@@ -49,17 +51,21 @@
 import org.elasticsearch.client.ml.GetOverallBucketsRequest;
 import org.elasticsearch.client.ml.GetRecordsRequest;
 import org.elasticsearch.client.ml.OpenJobRequest;
+import org.elasticsearch.client.ml.PostCalendarEventRequest;
 import org.elasticsearch.client.ml.PostDataRequest;
 import org.elasticsearch.client.ml.PreviewDatafeedRequest;
+import org.elasticsearch.client.ml.PutCalendarJobRequest;
 import org.elasticsearch.client.ml.PutCalendarRequest;
 import org.elasticsearch.client.ml.PutDatafeedRequest;
 import org.elasticsearch.client.ml.PutFilterRequest;
 import org.elasticsearch.client.ml.PutJobRequest;
+import org.elasticsearch.client.ml.RevertModelSnapshotRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
 import org.elasticsearch.client.ml.StopDatafeedRequest;
 import org.elasticsearch.client.ml.UpdateDatafeedRequest;
 import org.elasticsearch.client.ml.UpdateFilterRequest;
 import org.elasticsearch.client.ml.UpdateJobRequest;
+import org.elasticsearch.client.ml.UpdateModelSnapshotRequest;
 import org.elasticsearch.client.ml.job.util.PageParams;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -390,6 +396,36 @@ static Request getModelSnapshots(GetModelSnapshotsRequest getModelSnapshotsReque
         return request;
     }
 
+    static Request updateModelSnapshot(UpdateModelSnapshotRequest updateModelSnapshotRequest) throws IOException {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("ml")
+            .addPathPartAsIs("anomaly_detectors")
+            .addPathPart(updateModelSnapshotRequest.getJobId())
+            .addPathPartAsIs("model_snapshots")
+            .addPathPart(updateModelSnapshotRequest.getSnapshotId())
+            .addPathPartAsIs("_update")
+            .build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(updateModelSnapshotRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
+    static Request revertModelSnapshot(RevertModelSnapshotRequest revertModelSnapshotsRequest) throws IOException {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("ml")
+            .addPathPartAsIs("anomaly_detectors")
+            .addPathPart(revertModelSnapshotsRequest.getJobId())
+            .addPathPartAsIs("model_snapshots")
+            .addPathPart(revertModelSnapshotsRequest.getSnapshotId())
+            .addPathPart("_revert")
+            .build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(revertModelSnapshotsRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
     static Request getOverallBuckets(GetOverallBucketsRequest getOverallBucketsRequest) throws IOException {
         String endpoint = new EndpointBuilder()
                 .addPathPartAsIs("_xpack")
@@ -485,6 +521,30 @@ static Request getCalendars(GetCalendarsRequest getCalendarsRequest) throws IOEx
         return request;
     }
 
+    static Request putCalendarJob(PutCalendarJobRequest putCalendarJobRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("ml")
+            .addPathPartAsIs("calendars")
+            .addPathPart(putCalendarJobRequest.getCalendarId())
+            .addPathPartAsIs("jobs")
+            .addPathPart(Strings.collectionToCommaDelimitedString(putCalendarJobRequest.getJobIds()))
+            .build();
+        return new Request(HttpPut.METHOD_NAME, endpoint);
+    }
+
+    static Request deleteCalendarJob(DeleteCalendarJobRequest deleteCalendarJobRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("ml")
+            .addPathPartAsIs("calendars")
+            .addPathPart(deleteCalendarJobRequest.getCalendarId())
+            .addPathPartAsIs("jobs")
+            .addPathPart(Strings.collectionToCommaDelimitedString(deleteCalendarJobRequest.getJobIds()))
+            .build();
+        return new Request(HttpDelete.METHOD_NAME, endpoint);
+    }
+
     static Request deleteCalendar(DeleteCalendarRequest deleteCalendarRequest) {
         String endpoint = new EndpointBuilder()
                 .addPathPartAsIs("_xpack")
@@ -496,6 +556,34 @@ static Request deleteCalendar(DeleteCalendarRequest deleteCalendarRequest) {
         return request;
     }
 
+    static Request getCalendarEvents(GetCalendarEventsRequest getCalendarEventsRequest) throws IOException {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("ml")
+            .addPathPartAsIs("calendars")
+            .addPathPart(getCalendarEventsRequest.getCalendarId())
+            .addPathPartAsIs("events")
+            .build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(getCalendarEventsRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
+    static Request postCalendarEvents(PostCalendarEventRequest postCalendarEventRequest) throws IOException {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("ml")
+            .addPathPartAsIs("calendars")
+            .addPathPart(postCalendarEventRequest.getCalendarId())
+            .addPathPartAsIs("events")
+            .build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(postCalendarEventRequest,
+            REQUEST_BODY_CONTENT_TYPE,
+            PostCalendarEventRequest.EXCLUDE_CALENDAR_ID_PARAMS));
+        return request;
+    }
+
     static Request putFilter(PutFilterRequest putFilterRequest) throws IOException {
         String endpoint = new EndpointBuilder()
             .addPathPartAsIs("_xpack")