[ML] Deprecate anomaly detection post data endpoint

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -232,6 +232,10 @@
 
 public class MachineLearningIT extends ESRestHighLevelClientTestCase {
 
+    private static final RequestOptions POST_DATA_OPTIONS = RequestOptions.DEFAULT.toBuilder()
+        .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " +
+            "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false).build();
+
     @After
     public void cleanUp() throws IOException {
         new MlTestStateCleaner(logger, highLevelClient().machineLearning()).clearMlMetadata();
@@ -435,7 +439,8 @@ public void testForecastJob() throws Exception {
             builder.addDoc(hashMap);
         }
         PostDataRequest postDataRequest = new PostDataRequest(jobId, builder);
-        machineLearningClient.postData(postDataRequest, RequestOptions.DEFAULT);
+        // Post data is deprecated, so expect a deprecation warning
+        machineLearningClient.postData(postDataRequest, POST_DATA_OPTIONS);
         machineLearningClient.flushJob(new FlushJobRequest(jobId), RequestOptions.DEFAULT);
 
         ForecastJobRequest request = new ForecastJobRequest(jobId);
@@ -461,7 +466,9 @@ public void testPostData() throws Exception {
         }
         PostDataRequest postDataRequest = new PostDataRequest(jobId, builder);
 
-        PostDataResponse response = execute(postDataRequest, machineLearningClient::postData, machineLearningClient::postDataAsync);
+        // Post data is deprecated, so expect a deprecation warning
+        PostDataResponse response = execute(postDataRequest, machineLearningClient::postData, machineLearningClient::postDataAsync,
+            POST_DATA_OPTIONS);
         assertEquals(10, response.getDataCounts().getInputRecordCount());
         assertEquals(0, response.getDataCounts().getOutOfOrderTimeStampCount());
     }
@@ -1068,7 +1075,8 @@ public void testDeleteForecast() throws Exception {
         }
 
         PostDataRequest postDataRequest = new PostDataRequest(jobId, builder);
-        machineLearningClient.postData(postDataRequest, RequestOptions.DEFAULT);
+        // Post data is deprecated, so expect a deprecation warning
+        machineLearningClient.postData(postDataRequest, POST_DATA_OPTIONS);
         machineLearningClient.flushJob(new FlushJobRequest(jobId), RequestOptions.DEFAULT);
         ForecastJobResponse forecastJobResponse1 = machineLearningClient.forecastJob(new ForecastJobRequest(jobId), RequestOptions.DEFAULT);
         ForecastJobResponse forecastJobResponse2 = machineLearningClient.forecastJob(new ForecastJobRequest(jobId), RequestOptions.DEFAULT);

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

@@ -251,6 +251,10 @@
 
 public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
 
+    private static final RequestOptions POST_DATA_OPTIONS = RequestOptions.DEFAULT.toBuilder()
+        .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " +
+            "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false).build();
+
     @After
     public void cleanUp() throws IOException {
         new MlTestStateCleaner(logger, highLevelClient().machineLearning()).clearMlMetadata();
@@ -1382,7 +1386,8 @@ public void testDeleteForecast() throws Exception {
         }
 
         PostDataRequest postDataRequest = new PostDataRequest(job.getId(), builder);
-        client.machineLearning().postData(postDataRequest, RequestOptions.DEFAULT);
+        // Post data is deprecated, so expect a deprecation warning
+        client.machineLearning().postData(postDataRequest, POST_DATA_OPTIONS);
         client.machineLearning().flushJob(new FlushJobRequest(job.getId()), RequestOptions.DEFAULT);
 
         ForecastJobResponse forecastJobResponse = client.machineLearning().
@@ -1519,7 +1524,8 @@ public void testForecastJob() throws Exception {
             builder.addDoc(hashMap);
         }
         PostDataRequest postDataRequest = new PostDataRequest(job.getId(), builder);
-        client.machineLearning().postData(postDataRequest, RequestOptions.DEFAULT);
+        // Post data is deprecated, so expect a deprecation warning
+        client.machineLearning().postData(postDataRequest, POST_DATA_OPTIONS);
         client.machineLearning().flushJob(new FlushJobRequest(job.getId()), RequestOptions.DEFAULT);
 
         {
@@ -1788,9 +1794,14 @@ public void testPostData() throws Exception {
             postDataRequest.setResetEnd(null);
             postDataRequest.setResetStart(null);
 
+            // Post data is deprecated, so expect a deprecation warning
+            PostDataResponse postDataResponse = client.machineLearning().postData(postDataRequest, POST_DATA_OPTIONS);
+            // The end user can use the default options without it being a fatal error (this is only in the test framework)
+            /*
             // tag::post-data-execute
             PostDataResponse postDataResponse = client.machineLearning().postData(postDataRequest, RequestOptions.DEFAULT);
             // end::post-data-execute
+            */
 
             // tag::post-data-response
             DataCounts dataCounts = postDataResponse.getDataCounts(); // <1>
@@ -1822,9 +1833,14 @@ public void onFailure(Exception e) {
             final CountDownLatch latch = new CountDownLatch(1);
             listener = new LatchedActionListener<>(listener, latch);
 
+            // Post data is deprecated, so expect a deprecation warning
+            client.machineLearning().postDataAsync(postDataRequest, POST_DATA_OPTIONS, listener);
+            // The end user can use the default options without it being a fatal error (this is only in the test framework)
+            /*
             // tag::post-data-execute-async
             client.machineLearning().postDataAsync(postDataRequest, RequestOptions.DEFAULT, listener); // <1>
             // end::post-data-execute-async
+            */
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
         }

client/rest/src/main/java/org/elasticsearch/client/RequestOptions.java

@@ -219,8 +219,9 @@ public void setHttpAsyncResponseConsumerFactory(HttpAsyncResponseConsumerFactory
          * fail the request if the warnings returned don't
          * <strong>exactly</strong> match some set.
          */
-        public void setWarningsHandler(WarningsHandler warningsHandler) {
+        public Builder setWarningsHandler(WarningsHandler warningsHandler) {
             this.warningsHandler = warningsHandler;
+            return this;
         }
 
         /**

docs/reference/ml/anomaly-detection/apis/post-data.asciidoc

@@ -6,6 +6,8 @@
 <titleabbrev>Post data to jobs</titleabbrev>
 ++++
 
+deprecated::[7.11.0, "Posting data directly to anomaly detection jobs is deprecated, in a future major version a <<ml-api-datafeed-endpoint,{dfeed}>> will be required."]
+
 Sends data to an anomaly detection job for analysis.
 
 [[ml-post-data-request]]

x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java

@@ -8,14 +8,14 @@
 import org.apache.http.entity.ContentType;
 import org.apache.http.nio.entity.NStringEntity;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.yaml.snakeyaml.util.UriEncoder;
 
-import javax.print.attribute.standard.JobStateReason;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.Collections;
@@ -32,6 +32,10 @@ public class MlBasicMultiNodeIT extends ESRestTestCase {
 
     private static final String BASE_PATH = "/_ml/";
 
+    private static final RequestOptions POST_DATA_OPTIONS = RequestOptions.DEFAULT.toBuilder()
+        .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " +
+            "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false).build();
+
     public void testMachineLearningInstalled() throws Exception {
         Response response = client().performRequest(new Request("GET", "/_xpack"));
         Map<?, ?> features = (Map<?, ?>) entityAsMap(response).get("features");
@@ -66,6 +70,7 @@ public void testMiniFarequote() throws Exception {
                 "{\"airline\":\"AAL\",\"responsetime\":\"132.2046\",\"sourcetype\":\"farequote\",\"time\":\"1403481600\"}\n" +
                 "{\"airline\":\"JZA\",\"responsetime\":\"990.4628\",\"sourcetype\":\"farequote\",\"time\":\"1403481700\"}",
                 randomFrom(ContentType.APPLICATION_JSON, ContentType.create("application/x-ndjson"))));
+        addData.setOptions(POST_DATA_OPTIONS);
         Response addDataResponse = client().performRequest(addData);
         assertEquals(202, addDataResponse.getStatusLine().getStatusCode());
         Map<String, Object> responseBody = entityAsMap(addDataResponse);
@@ -165,6 +170,8 @@ public void testMiniFarequoteReopen() throws Exception {
                 "{\"airline\":\"KLM\",\"responsetime\":\"1355.4812\",\"sourcetype\":\"farequote\",\"time\":\"1403481900\"}\n" +
                 "{\"airline\":\"NKS\",\"responsetime\":\"9991.3981\",\"sourcetype\":\"farequote\",\"time\":\"1403482000\"}",
                 randomFrom(ContentType.APPLICATION_JSON, ContentType.create("application/x-ndjson"))));
+        // Post data is deprecated, so expect a deprecation warning
+        addDataRequest.setOptions(POST_DATA_OPTIONS);
         Response addDataResponse = client().performRequest(addDataRequest);
         assertEquals(202, addDataResponse.getStatusLine().getStatusCode());
         Map<String, Object> responseBody = entityAsMap(addDataResponse);
@@ -205,6 +212,8 @@ public void testMiniFarequoteReopen() throws Exception {
                 "{\"airline\":\"UAL\",\"responsetime\":\"8.4275\",\"sourcetype\":\"farequote\",\"time\":\"1407081900\"}\n" +
                 "{\"airline\":\"FFT\",\"responsetime\":\"221.8693\",\"sourcetype\":\"farequote\",\"time\":\"1407082000\"}",
                 randomFrom(ContentType.APPLICATION_JSON, ContentType.create("application/x-ndjson"))));
+        // Post data is deprecated, so expect a deprecation warning
+        addDataRequest2.setOptions(POST_DATA_OPTIONS);
         Response addDataResponse2 = client().performRequest(addDataRequest2);
         assertEquals(202, addDataResponse2.getStatusLine().getStatusCode());
         Map<String, Object> responseBody2 = entityAsMap(addDataResponse2);

x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java

@@ -28,6 +28,7 @@
 import org.junit.After;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -475,6 +476,10 @@ public void testDeleteJob_TimingStatsDocumentIsDeleted() throws Exception {
         assertThat(entityAsMap(openResponse), hasEntry("opened", true));
 
         Request postDataRequest = new Request("POST", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_data");
+        // Post data is deprecated, so expect a deprecation warning
+        postDataRequest.setOptions(RequestOptions.DEFAULT.toBuilder()
+            .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " +
+                "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false));
         postDataRequest.setJsonEntity("{ \"airline\":\"LOT\", \"response_time\":100, \"time\":\"2019-07-01 00:00:00Z\" }");
         client().performRequest(postDataRequest);
         postDataRequest.setJsonEntity("{ \"airline\":\"LOT\", \"response_time\":100, \"time\":\"2019-07-01 02:00:00Z\" }");

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.ml.MachineLearning;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
@@ -29,13 +30,13 @@ public List<Route> routes() {
         return Collections.emptyList();
     }
 
-    @Override
-    public List<ReplacedRoute> replacedRoutes() {
-        // TODO: remove deprecated endpoint in 8.0.0
-        return Collections.singletonList(
-            new ReplacedRoute(POST, MachineLearning.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_data",
-                POST, MachineLearning.PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_data")
-        );
+    public List<DeprecatedRoute> deprecatedRoutes() {
+        final String msg = "Posting data directly to anomaly detection jobs is deprecated, " +
+            "in a future major version it will be compulsory to use a datafeed";
+        return Arrays.asList(
+            new DeprecatedRoute(POST, MachineLearning.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_data", msg),
+            new DeprecatedRoute(POST, MachineLearning.PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_data",
+                msg));
     }
 
     @Override

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml

@@ -4,6 +4,10 @@ setup:
 ---
 "Test CRUD on two jobs in shared index":
 
+  - skip:
+      features:
+        - "warnings"
+
   - do:
       ml.put_job:
         job_id: index-layout-job
@@ -58,13 +62,17 @@ setup:
         job_id: index-layout-job2
 
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: index-layout-job
         body: >
           {"airline":"AAL","responsetime":"132.2046","sourcetype":"farequote","time":"1403481600"}
           {"airline":"JZA","responsetime":"990.4628","sourcetype":"farequote","time":"1403481700"}
 
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: index-layout-job2
         body: >
@@ -341,6 +349,10 @@ setup:
 ---
 "Test unrelated index":
 
+  - skip:
+      features:
+        - "warnings"
+
   - do:
       ml.put_job:
         job_id: index-layout-job
@@ -368,6 +380,8 @@ setup:
         job_id: index-layout-job
 
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: index-layout-job
         body: >
@@ -649,6 +663,10 @@ setup:
 ---
 "Test force close does not create state":
 
+  - skip:
+      features:
+        - "warnings"
+
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
@@ -680,6 +698,8 @@ setup:
         job_id: index-layout-force-close-job
 
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: index-layout-force-close-job
         body: >

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/job_cat_apis.yml

@@ -72,7 +72,13 @@ setup:
 ---
 "Test cat anomaly detector jobs":
 
+  - skip:
+      features:
+        - "warnings"
+
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: job-stats-test
         body: >

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml

@@ -657,7 +657,9 @@
 ---
 "Test close job":
   - skip:
-      features: headers
+      features:
+        - "headers"
+        - "warnings"
   - do:
       ml.put_job:
         job_id: jobs-crud-close-job
@@ -682,6 +684,8 @@
         job_id: jobs-crud-close-job
 
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: jobs-crud-close-job
         body: >
@@ -915,7 +919,9 @@
 ---
 "Test force close job":
   - skip:
-      features: headers
+      features:
+        - "headers"
+        - "warnings"
   - do:
       ml.put_job:
         job_id: jobs-crud-force-close-job
@@ -943,6 +949,8 @@
         job_id: jobs-crud-force-close-job
 
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: jobs-crud-force-close-job
         body: >

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml

@@ -72,7 +72,13 @@ setup:
 ---
 "Test get job stats after uploading data prompting the creation of some stats":
 
+  - skip:
+      features:
+        - "warnings"
+
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: job-stats-test
         body: >
@@ -110,7 +116,13 @@ setup:
 ---
 "Test get job stats for closed job":
 
+  - skip:
+      features:
+        - "warnings"
+
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: job-stats-test
         body: >
@@ -341,7 +353,13 @@ setup:
 ---
 "Test no exception on get job stats with missing index":
 
+  - skip:
+      features:
+        - "warnings"
+
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: job-stats-test
         body: >