[ML] Deprecate anomaly detection post data endpoint

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -435,7 +435,11 @@ public void testForecastJob() throws Exception {
             builder.addDoc(hashMap);
         }
         PostDataRequest postDataRequest = new PostDataRequest(jobId, builder);
-        machineLearningClient.postData(postDataRequest, RequestOptions.DEFAULT);
+        // Post data is deprecated, so expect a deprecation warning
+        RequestOptions requestOptions = RequestOptions.DEFAULT.toBuilder()
+            .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " +
+                "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false).build();
+        machineLearningClient.postData(postDataRequest, requestOptions);
         machineLearningClient.flushJob(new FlushJobRequest(jobId), RequestOptions.DEFAULT);
 
         ForecastJobRequest request = new ForecastJobRequest(jobId);
@@ -461,7 +465,12 @@ public void testPostData() throws Exception {
         }
         PostDataRequest postDataRequest = new PostDataRequest(jobId, builder);
 
-        PostDataResponse response = execute(postDataRequest, machineLearningClient::postData, machineLearningClient::postDataAsync);
+        // Post data is deprecated, so expect a deprecation warning
+        RequestOptions requestOptions = RequestOptions.DEFAULT.toBuilder()
+            .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " +
+                "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false).build();
+        PostDataResponse response = execute(postDataRequest, machineLearningClient::postData, machineLearningClient::postDataAsync,
+            requestOptions);
         assertEquals(10, response.getDataCounts().getInputRecordCount());
         assertEquals(0, response.getDataCounts().getOutOfOrderTimeStampCount());
     }
@@ -1038,7 +1047,11 @@ public void testDeleteForecast() throws Exception {
         }
 
         PostDataRequest postDataRequest = new PostDataRequest(jobId, builder);
-        machineLearningClient.postData(postDataRequest, RequestOptions.DEFAULT);
+        // Post data is deprecated, so expect a deprecation warning
+        RequestOptions requestOptions = RequestOptions.DEFAULT.toBuilder()
+            .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " +
+                "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false).build();
+        machineLearningClient.postData(postDataRequest, requestOptions);
         machineLearningClient.flushJob(new FlushJobRequest(jobId), RequestOptions.DEFAULT);
         ForecastJobResponse forecastJobResponse1 = machineLearningClient.forecastJob(new ForecastJobRequest(jobId), RequestOptions.DEFAULT);
         ForecastJobResponse forecastJobResponse2 = machineLearningClient.forecastJob(new ForecastJobRequest(jobId), RequestOptions.DEFAULT);

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.client.MlTestStateCleaner;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.WarningsHandler;
 import org.elasticsearch.client.core.PageParams;
 import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.client.ml.CloseJobRequest;
@@ -1382,7 +1383,11 @@ public void testDeleteForecast() throws Exception {
         }
 
         PostDataRequest postDataRequest = new PostDataRequest(job.getId(), builder);
-        client.machineLearning().postData(postDataRequest, RequestOptions.DEFAULT);
+        // Post data is deprecated, so expect a deprecation warning
+        RequestOptions requestOptions = RequestOptions.DEFAULT.toBuilder()
+            .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " +
+                "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false).build();
+        client.machineLearning().postData(postDataRequest, requestOptions);
         client.machineLearning().flushJob(new FlushJobRequest(job.getId()), RequestOptions.DEFAULT);
 
         ForecastJobResponse forecastJobResponse = client.machineLearning().
@@ -1519,7 +1524,11 @@ public void testForecastJob() throws Exception {
             builder.addDoc(hashMap);
         }
         PostDataRequest postDataRequest = new PostDataRequest(job.getId(), builder);
-        client.machineLearning().postData(postDataRequest, RequestOptions.DEFAULT);
+        // Post data is deprecated, so expect a deprecation warning
+        RequestOptions requestOptions = RequestOptions.DEFAULT.toBuilder()
+            .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " +
+                "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false).build();
+        client.machineLearning().postData(postDataRequest, requestOptions);
         client.machineLearning().flushJob(new FlushJobRequest(job.getId()), RequestOptions.DEFAULT);
 
         {
@@ -1789,7 +1798,10 @@ public void testPostData() throws Exception {
             postDataRequest.setResetStart(null);
 
             // tag::post-data-execute
-            PostDataResponse postDataResponse = client.machineLearning().postData(postDataRequest, RequestOptions.DEFAULT);
+            RequestOptions ignoreDeprecationsWarningsOptions = RequestOptions.DEFAULT.toBuilder()
+                .setWarningsHandler(WarningsHandler.PERMISSIVE).build();
+            PostDataResponse postDataResponse =
+                client.machineLearning().postData(postDataRequest, ignoreDeprecationsWarningsOptions);
             // end::post-data-execute
 
             // tag::post-data-response
@@ -1823,7 +1835,9 @@ public void onFailure(Exception e) {
             listener = new LatchedActionListener<>(listener, latch);
 
             // tag::post-data-execute-async
-            client.machineLearning().postDataAsync(postDataRequest, RequestOptions.DEFAULT, listener); // <1>
+            RequestOptions ignoreDeprecationsWarningsOptions = RequestOptions.DEFAULT.toBuilder()
+                .setWarningsHandler(WarningsHandler.PERMISSIVE).build();
+            client.machineLearning().postDataAsync(postDataRequest, ignoreDeprecationsWarningsOptions, listener); // <1>
             // end::post-data-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));

docs/reference/ml/anomaly-detection/apis/post-data.asciidoc

@@ -6,6 +6,7 @@
 <titleabbrev>Post data to jobs</titleabbrev>
 ++++
 
+deprecated:[7.11.0, "This API will be removed in a future major version, where it will be compulsory to use a <<ml-put-datafeed-request-body,{dfeed}>> instead."]
 Sends data to an anomaly detection job for analysis.
 
 [[ml-post-data-request]]

x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java

@@ -8,14 +8,14 @@
 import org.apache.http.entity.ContentType;
 import org.apache.http.nio.entity.NStringEntity;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.yaml.snakeyaml.util.UriEncoder;
 
-import javax.print.attribute.standard.JobStateReason;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.Collections;
@@ -32,6 +32,10 @@ public class MlBasicMultiNodeIT extends ESRestTestCase {
 
     private static final String BASE_PATH = "/_ml/";
 
+    private static final RequestOptions POST_DATA_OPTIONS = RequestOptions.DEFAULT.toBuilder()
+        .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " +
+            "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false).build();
+
     public void testMachineLearningInstalled() throws Exception {
         Response response = client().performRequest(new Request("GET", "/_xpack"));
         Map<?, ?> features = (Map<?, ?>) entityAsMap(response).get("features");
@@ -66,6 +70,7 @@ public void testMiniFarequote() throws Exception {
                 "{\"airline\":\"AAL\",\"responsetime\":\"132.2046\",\"sourcetype\":\"farequote\",\"time\":\"1403481600\"}\n" +
                 "{\"airline\":\"JZA\",\"responsetime\":\"990.4628\",\"sourcetype\":\"farequote\",\"time\":\"1403481700\"}",
                 randomFrom(ContentType.APPLICATION_JSON, ContentType.create("application/x-ndjson"))));
+        addData.setOptions(POST_DATA_OPTIONS);
         Response addDataResponse = client().performRequest(addData);
         assertEquals(202, addDataResponse.getStatusLine().getStatusCode());
         Map<String, Object> responseBody = entityAsMap(addDataResponse);
@@ -165,6 +170,8 @@ public void testMiniFarequoteReopen() throws Exception {
                 "{\"airline\":\"KLM\",\"responsetime\":\"1355.4812\",\"sourcetype\":\"farequote\",\"time\":\"1403481900\"}\n" +
                 "{\"airline\":\"NKS\",\"responsetime\":\"9991.3981\",\"sourcetype\":\"farequote\",\"time\":\"1403482000\"}",
                 randomFrom(ContentType.APPLICATION_JSON, ContentType.create("application/x-ndjson"))));
+        // Post data is deprecated, so expect a deprecation warning
+        addDataRequest.setOptions(POST_DATA_OPTIONS);
         Response addDataResponse = client().performRequest(addDataRequest);
         assertEquals(202, addDataResponse.getStatusLine().getStatusCode());
         Map<String, Object> responseBody = entityAsMap(addDataResponse);
@@ -205,6 +212,8 @@ public void testMiniFarequoteReopen() throws Exception {
                 "{\"airline\":\"UAL\",\"responsetime\":\"8.4275\",\"sourcetype\":\"farequote\",\"time\":\"1407081900\"}\n" +
                 "{\"airline\":\"FFT\",\"responsetime\":\"221.8693\",\"sourcetype\":\"farequote\",\"time\":\"1407082000\"}",
                 randomFrom(ContentType.APPLICATION_JSON, ContentType.create("application/x-ndjson"))));
+        // Post data is deprecated, so expect a deprecation warning
+        addDataRequest2.setOptions(POST_DATA_OPTIONS);
         Response addDataResponse2 = client().performRequest(addDataRequest2);
         assertEquals(202, addDataResponse2.getStatusLine().getStatusCode());
         Map<String, Object> responseBody2 = entityAsMap(addDataResponse2);

x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java

@@ -28,6 +28,7 @@
 import org.junit.After;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -475,6 +476,10 @@ public void testDeleteJob_TimingStatsDocumentIsDeleted() throws Exception {
         assertThat(entityAsMap(openResponse), hasEntry("opened", true));
 
         Request postDataRequest = new Request("POST", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_data");
+        // Post data is deprecated, so expect a deprecation warning
+        postDataRequest.setOptions(RequestOptions.DEFAULT.toBuilder()
+            .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " +
+                "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false));
         postDataRequest.setJsonEntity("{ \"airline\":\"LOT\", \"response_time\":100, \"time\":\"2019-07-01 00:00:00Z\" }");
         client().performRequest(postDataRequest);
         postDataRequest.setJsonEntity("{ \"airline\":\"LOT\", \"response_time\":100, \"time\":\"2019-07-01 02:00:00Z\" }");

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java

@@ -26,9 +26,15 @@ public class RestPostDataAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public List<DeprecatedRoute> deprecatedRoutes() {
         return Collections.singletonList(
-            new Route(POST, MachineLearning.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_data")
-        );
+            new DeprecatedRoute(POST, MachineLearning.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_data",
+            "Posting data directly to anomaly detection jobs is deprecated, " +
+                "in a future major version it will be compulsory to use a datafeed"));
     }
 
     @Override

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml

@@ -4,6 +4,10 @@ setup:
 ---
 "Test CRUD on two jobs in shared index":
 
+  - skip:
+      features:
+        - "warnings"
+
   - do:
       ml.put_job:
         job_id: index-layout-job
@@ -58,13 +62,17 @@ setup:
         job_id: index-layout-job2
 
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: index-layout-job
         body: >
           {"airline":"AAL","responsetime":"132.2046","sourcetype":"farequote","time":"1403481600"}
           {"airline":"JZA","responsetime":"990.4628","sourcetype":"farequote","time":"1403481700"}
 
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: index-layout-job2
         body: >
@@ -341,6 +349,10 @@ setup:
 ---
 "Test unrelated index":
 
+  - skip:
+      features:
+        - "warnings"
+
   - do:
       ml.put_job:
         job_id: index-layout-job
@@ -368,6 +380,8 @@ setup:
         job_id: index-layout-job
 
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: index-layout-job
         body: >
@@ -649,6 +663,10 @@ setup:
 ---
 "Test force close does not create state":
 
+  - skip:
+      features:
+        - "warnings"
+
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
@@ -680,6 +698,8 @@ setup:
         job_id: index-layout-force-close-job
 
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: index-layout-force-close-job
         body: >

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/job_cat_apis.yml

@@ -72,7 +72,13 @@ setup:
 ---
 "Test cat anomaly detector jobs":
 
+  - skip:
+      features:
+        - "warnings"
+
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: job-stats-test
         body: >

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml

@@ -657,7 +657,9 @@
 ---
 "Test close job":
   - skip:
-      features: headers
+      features:
+        - "headers"
+        - "warnings"
   - do:
       ml.put_job:
         job_id: jobs-crud-close-job
@@ -682,6 +684,8 @@
         job_id: jobs-crud-close-job
 
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: jobs-crud-close-job
         body: >
@@ -915,7 +919,9 @@
 ---
 "Test force close job":
   - skip:
-      features: headers
+      features:
+        - "headers"
+        - "warnings"
   - do:
       ml.put_job:
         job_id: jobs-crud-force-close-job
@@ -943,6 +949,8 @@
         job_id: jobs-crud-force-close-job
 
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: jobs-crud-force-close-job
         body: >

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml

@@ -72,7 +72,13 @@ setup:
 ---
 "Test get job stats after uploading data prompting the creation of some stats":
 
+  - skip:
+      features:
+        - "warnings"
+
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: job-stats-test
         body: >
@@ -110,7 +116,13 @@ setup:
 ---
 "Test get job stats for closed job":
 
+  - skip:
+      features:
+        - "warnings"
+
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: job-stats-test
         body: >
@@ -341,7 +353,13 @@ setup:
 ---
 "Test no exception on get job stats with missing index":
 
+  - skip:
+      features:
+        - "warnings"
+
   - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
       ml.post_data:
         job_id: job-stats-test
         body: >