From 96fd6a0b8c1e64de5411bc4ef307d11ab1c497de Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 15 Dec 2020 18:37:20 +0000 Subject: [PATCH 1/2] [ML] Deprecate anomaly detection post data endpoint There is little evidence of this endpoint being used and there is quite a lot of code complexity associated with the various formats that can be used to upload data and the different errors that can occur when direct data upload is open to end users. In a future release we can make this endpoint internal so that only datafeeds can use it, and remove all the options and formats that are not used by datafeeds. End users will have to store their input data for anomaly detection in Elasticsearch indices (which we believe all do today) and use a datafeed to feed it to anomaly detection jobs. Backport of #66347 --- .../client/MachineLearningIT.java | 14 +++++++-- .../MlClientDocumentationIT.java | 20 +++++++++++-- .../anomaly-detection/apis/post-data.asciidoc | 2 ++ .../ml/integration/MlBasicMultiNodeIT.java | 11 ++++++- .../xpack/ml/integration/MlJobIT.java | 5 ++++ .../xpack/ml/rest/job/RestPostDataAction.java | 15 +++++----- .../rest-api-spec/test/ml/index_layout.yml | 20 +++++++++++++ .../rest-api-spec/test/ml/job_cat_apis.yml | 6 ++++ .../rest-api-spec/test/ml/jobs_crud.yml | 12 ++++++-- .../rest-api-spec/test/ml/jobs_get_stats.yml | 18 +++++++++++ .../rest-api-spec/test/ml/post_data.yml | 30 +++++++++++++++++++ .../upgrades/MlJobSnapshotUpgradeIT.java | 14 ++++++++- .../test/mixed_cluster/30_ml_jobs_crud.yml | 6 ++++ .../test/old_cluster/30_ml_jobs_crud.yml | 21 +++++++++++++ 14 files changed, 178 insertions(+), 16 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index 812a572437950..a2ba8dd7f0923 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -232,6 +232,10 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase { + private static final RequestOptions POST_DATA_OPTIONS = RequestOptions.DEFAULT.toBuilder() + .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " + + "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false).build(); + @After public void cleanUp() throws IOException { new MlTestStateCleaner(logger, highLevelClient().machineLearning()).clearMlMetadata(); @@ -435,7 +439,8 @@ public void testForecastJob() throws Exception { builder.addDoc(hashMap); } PostDataRequest postDataRequest = new PostDataRequest(jobId, builder); - machineLearningClient.postData(postDataRequest, RequestOptions.DEFAULT); + // Post data is deprecated, so expect a deprecation warning + machineLearningClient.postData(postDataRequest, POST_DATA_OPTIONS); machineLearningClient.flushJob(new FlushJobRequest(jobId), RequestOptions.DEFAULT); ForecastJobRequest request = new ForecastJobRequest(jobId); @@ -461,7 +466,9 @@ public void testPostData() throws Exception { } PostDataRequest postDataRequest = new PostDataRequest(jobId, builder); - PostDataResponse response = execute(postDataRequest, machineLearningClient::postData, machineLearningClient::postDataAsync); + // Post data is deprecated, so expect a deprecation warning + PostDataResponse response = execute(postDataRequest, machineLearningClient::postData, machineLearningClient::postDataAsync, + POST_DATA_OPTIONS); assertEquals(10, response.getDataCounts().getInputRecordCount()); assertEquals(0, response.getDataCounts().getOutOfOrderTimeStampCount()); } @@ -1068,7 +1075,8 @@ public void testDeleteForecast() throws Exception { } PostDataRequest postDataRequest = new PostDataRequest(jobId, builder); - machineLearningClient.postData(postDataRequest, RequestOptions.DEFAULT); + // Post data is deprecated, so expect a deprecation warning + machineLearningClient.postData(postDataRequest, POST_DATA_OPTIONS); machineLearningClient.flushJob(new FlushJobRequest(jobId), RequestOptions.DEFAULT); ForecastJobResponse forecastJobResponse1 = machineLearningClient.forecastJob(new ForecastJobRequest(jobId), RequestOptions.DEFAULT); ForecastJobResponse forecastJobResponse2 = machineLearningClient.forecastJob(new ForecastJobRequest(jobId), RequestOptions.DEFAULT); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 209a483fe0c1d..749290a16501b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -251,6 +251,10 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase { + private static final RequestOptions POST_DATA_OPTIONS = RequestOptions.DEFAULT.toBuilder() + .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " + + "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false).build(); + @After public void cleanUp() throws IOException { new MlTestStateCleaner(logger, highLevelClient().machineLearning()).clearMlMetadata(); @@ -1382,7 +1386,8 @@ public void testDeleteForecast() throws Exception { } PostDataRequest postDataRequest = new PostDataRequest(job.getId(), builder); - client.machineLearning().postData(postDataRequest, RequestOptions.DEFAULT); + // Post data is deprecated, so expect a deprecation warning + client.machineLearning().postData(postDataRequest, POST_DATA_OPTIONS); client.machineLearning().flushJob(new FlushJobRequest(job.getId()), RequestOptions.DEFAULT); ForecastJobResponse forecastJobResponse = client.machineLearning(). @@ -1519,7 +1524,8 @@ public void testForecastJob() throws Exception { builder.addDoc(hashMap); } PostDataRequest postDataRequest = new PostDataRequest(job.getId(), builder); - client.machineLearning().postData(postDataRequest, RequestOptions.DEFAULT); + // Post data is deprecated, so expect a deprecation warning + client.machineLearning().postData(postDataRequest, POST_DATA_OPTIONS); client.machineLearning().flushJob(new FlushJobRequest(job.getId()), RequestOptions.DEFAULT); { @@ -1788,9 +1794,14 @@ public void testPostData() throws Exception { postDataRequest.setResetEnd(null); postDataRequest.setResetStart(null); + // Post data is deprecated, so expect a deprecation warning + PostDataResponse postDataResponse = client.machineLearning().postData(postDataRequest, POST_DATA_OPTIONS); + // The end user can use the default options without it being a fatal error (this is only in the test framework) + /* // tag::post-data-execute PostDataResponse postDataResponse = client.machineLearning().postData(postDataRequest, RequestOptions.DEFAULT); // end::post-data-execute + */ // tag::post-data-response DataCounts dataCounts = postDataResponse.getDataCounts(); // <1> @@ -1822,9 +1833,14 @@ public void onFailure(Exception e) { final CountDownLatch latch = new CountDownLatch(1); listener = new LatchedActionListener<>(listener, latch); + // Post data is deprecated, so expect a deprecation warning + client.machineLearning().postDataAsync(postDataRequest, POST_DATA_OPTIONS, listener); + // The end user can use the default options without it being a fatal error (this is only in the test framework) + /* // tag::post-data-execute-async client.machineLearning().postDataAsync(postDataRequest, RequestOptions.DEFAULT, listener); // <1> // end::post-data-execute-async + */ assertTrue(latch.await(30L, TimeUnit.SECONDS)); } diff --git a/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc b/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc index 42b385a515950..4d9bb3fce1474 100644 --- a/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc @@ -6,6 +6,8 @@ Post data to jobs ++++ +deprecated::[7.11.0, "Posting data directly to anomaly detection jobs is deprecated, in a future major version a <> will be required."] + Sends data to an anomaly detection job for analysis. [[ml-post-data-request]] diff --git a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java index 071b731bd9d59..6ca5d98714728 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java +++ b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java @@ -8,6 +8,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.nio.entity.NStringEntity; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.Strings; @@ -15,7 +16,6 @@ import org.elasticsearch.test.rest.ESRestTestCase; import org.yaml.snakeyaml.util.UriEncoder; -import javax.print.attribute.standard.JobStateReason; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collections; @@ -32,6 +32,10 @@ public class MlBasicMultiNodeIT extends ESRestTestCase { private static final String BASE_PATH = "/_ml/"; + private static final RequestOptions POST_DATA_OPTIONS = RequestOptions.DEFAULT.toBuilder() + .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " + + "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false).build(); + public void testMachineLearningInstalled() throws Exception { Response response = client().performRequest(new Request("GET", "/_xpack")); Map features = (Map) entityAsMap(response).get("features"); @@ -66,6 +70,7 @@ public void testMiniFarequote() throws Exception { "{\"airline\":\"AAL\",\"responsetime\":\"132.2046\",\"sourcetype\":\"farequote\",\"time\":\"1403481600\"}\n" + "{\"airline\":\"JZA\",\"responsetime\":\"990.4628\",\"sourcetype\":\"farequote\",\"time\":\"1403481700\"}", randomFrom(ContentType.APPLICATION_JSON, ContentType.create("application/x-ndjson")))); + addData.setOptions(POST_DATA_OPTIONS); Response addDataResponse = client().performRequest(addData); assertEquals(202, addDataResponse.getStatusLine().getStatusCode()); Map responseBody = entityAsMap(addDataResponse); @@ -165,6 +170,8 @@ public void testMiniFarequoteReopen() throws Exception { "{\"airline\":\"KLM\",\"responsetime\":\"1355.4812\",\"sourcetype\":\"farequote\",\"time\":\"1403481900\"}\n" + "{\"airline\":\"NKS\",\"responsetime\":\"9991.3981\",\"sourcetype\":\"farequote\",\"time\":\"1403482000\"}", randomFrom(ContentType.APPLICATION_JSON, ContentType.create("application/x-ndjson")))); + // Post data is deprecated, so expect a deprecation warning + addDataRequest.setOptions(POST_DATA_OPTIONS); Response addDataResponse = client().performRequest(addDataRequest); assertEquals(202, addDataResponse.getStatusLine().getStatusCode()); Map responseBody = entityAsMap(addDataResponse); @@ -205,6 +212,8 @@ public void testMiniFarequoteReopen() throws Exception { "{\"airline\":\"UAL\",\"responsetime\":\"8.4275\",\"sourcetype\":\"farequote\",\"time\":\"1407081900\"}\n" + "{\"airline\":\"FFT\",\"responsetime\":\"221.8693\",\"sourcetype\":\"farequote\",\"time\":\"1407082000\"}", randomFrom(ContentType.APPLICATION_JSON, ContentType.create("application/x-ndjson")))); + // Post data is deprecated, so expect a deprecation warning + addDataRequest2.setOptions(POST_DATA_OPTIONS); Response addDataResponse2 = client().performRequest(addDataRequest2); assertEquals(202, addDataResponse2.getStatusLine().getStatusCode()); Map responseBody2 = entityAsMap(addDataResponse2); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java index 434037a959308..4252902f110a8 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java @@ -28,6 +28,7 @@ import org.junit.After; import java.io.IOException; +import java.util.Collections; import java.util.Locale; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -475,6 +476,10 @@ public void testDeleteJob_TimingStatsDocumentIsDeleted() throws Exception { assertThat(entityAsMap(openResponse), hasEntry("opened", true)); Request postDataRequest = new Request("POST", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_data"); + // Post data is deprecated, so expect a deprecation warning + postDataRequest.setOptions(RequestOptions.DEFAULT.toBuilder() + .setWarningsHandler(warnings -> Collections.singletonList("Posting data directly to anomaly detection jobs is deprecated, " + + "in a future major version it will be compulsory to use a datafeed").equals(warnings) == false)); postDataRequest.setJsonEntity("{ \"airline\":\"LOT\", \"response_time\":100, \"time\":\"2019-07-01 00:00:00Z\" }"); client().performRequest(postDataRequest); postDataRequest.setJsonEntity("{ \"airline\":\"LOT\", \"response_time\":100, \"time\":\"2019-07-01 02:00:00Z\" }"); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java index 01ac45eb4f2b8..dd899ea6e0e91 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.ml.MachineLearning; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -29,13 +30,13 @@ public List routes() { return Collections.emptyList(); } - @Override - public List replacedRoutes() { - // TODO: remove deprecated endpoint in 8.0.0 - return Collections.singletonList( - new ReplacedRoute(POST, MachineLearning.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_data", - POST, MachineLearning.PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_data") - ); + public List deprecatedRoutes() { + final String msg = "Posting data directly to anomaly detection jobs is deprecated, " + + "in a future major version it will be compulsory to use a datafeed"; + return Arrays.asList( + new DeprecatedRoute(POST, MachineLearning.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_data", msg), + new DeprecatedRoute(POST, MachineLearning.PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_data", + msg)); } @Override diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml index 64a613bd946d9..5626f541bc2a9 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml @@ -4,6 +4,10 @@ setup: --- "Test CRUD on two jobs in shared index": + - skip: + features: + - "warnings" + - do: ml.put_job: job_id: index-layout-job @@ -58,6 +62,8 @@ setup: job_id: index-layout-job2 - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: index-layout-job body: > @@ -65,6 +71,8 @@ setup: {"airline":"JZA","responsetime":"990.4628","sourcetype":"farequote","time":"1403481700"} - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: index-layout-job2 body: > @@ -341,6 +349,10 @@ setup: --- "Test unrelated index": + - skip: + features: + - "warnings" + - do: ml.put_job: job_id: index-layout-job @@ -368,6 +380,8 @@ setup: job_id: index-layout-job - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: index-layout-job body: > @@ -649,6 +663,10 @@ setup: --- "Test force close does not create state": + - skip: + features: + - "warnings" + - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser @@ -680,6 +698,8 @@ setup: job_id: index-layout-force-close-job - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: index-layout-force-close-job body: > diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/job_cat_apis.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/job_cat_apis.yml index a82ce20032094..25a127f077c24 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/job_cat_apis.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/job_cat_apis.yml @@ -72,7 +72,13 @@ setup: --- "Test cat anomaly detector jobs": + - skip: + features: + - "warnings" + - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: job-stats-test body: > diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml index ec7528098db72..0be490fe79a49 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml @@ -657,7 +657,9 @@ --- "Test close job": - skip: - features: headers + features: + - "headers" + - "warnings" - do: ml.put_job: job_id: jobs-crud-close-job @@ -682,6 +684,8 @@ job_id: jobs-crud-close-job - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: jobs-crud-close-job body: > @@ -915,7 +919,9 @@ --- "Test force close job": - skip: - features: headers + features: + - "headers" + - "warnings" - do: ml.put_job: job_id: jobs-crud-force-close-job @@ -943,6 +949,8 @@ job_id: jobs-crud-force-close-job - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: jobs-crud-force-close-job body: > diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml index 4313d48f0e146..8910fdca0c918 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml @@ -72,7 +72,13 @@ setup: --- "Test get job stats after uploading data prompting the creation of some stats": + - skip: + features: + - "warnings" + - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: job-stats-test body: > @@ -110,7 +116,13 @@ setup: --- "Test get job stats for closed job": + - skip: + features: + - "warnings" + - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: job-stats-test body: > @@ -341,7 +353,13 @@ setup: --- "Test no exception on get job stats with missing index": + - skip: + features: + - "warnings" + - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: job-stats-test body: > diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml index 6cba27ae8dc6c..e19a8c48a042f 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml @@ -46,7 +46,13 @@ setup: --- "Test POST data job api, flush, close and verify DataCounts doc": + - skip: + features: + - "warnings" + - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: post-data-job body: @@ -123,7 +129,13 @@ setup: --- "Test flush with skip_time": + - skip: + features: + - "warnings" + - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: post-data-job body: @@ -146,6 +158,8 @@ setup: # Send some data that should be ignored - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: post-data-job body: @@ -160,6 +174,8 @@ setup: # Send data that will create results for the bucket after the skipped one - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: post-data-job body: @@ -191,8 +207,12 @@ setup: - skip: reason: "https://github.com/elastic/elasticsearch/issues/34747" version: "6.5.0 - " + features: + - "warnings" - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' catch: missing ml.post_data: job_id: not_a_job @@ -207,6 +227,8 @@ setup: time: 1403481700 - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' catch: /parse_exception/ ml.post_data: job_id: post-data-job @@ -222,6 +244,8 @@ setup: time: 1403481700 - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' catch: /parse_exception/ ml.post_data: job_id: post-data-job @@ -280,12 +304,18 @@ setup: --- "Test flushing and posting a closed job": + - skip: + features: + - "warnings" + - do: catch: /status_exception/ ml.flush_job: job_id: post-data-closed-job - do: + warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' catch: /status_exception/ ml.post_data: job_id: post-data-closed-job diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java index 7a25007efd7ac..740e529264bb3 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java @@ -248,7 +248,19 @@ protected OpenJobResponse openJob(String jobId) throws IOException { } protected PostDataResponse postData(String jobId, String data) throws IOException { - return hlrc.postData(new PostDataRequest(jobId, XContentType.JSON, new BytesArray(data)), RequestOptions.DEFAULT); + // Post data is deprecated, so a deprecation warning is possible (depending on the old version) + RequestOptions postDataOptions = RequestOptions.DEFAULT.toBuilder() + .setWarningsHandler(warnings -> { + if (warnings.isEmpty()) { + // No warning is OK - it means we hit an old node where post data is not deprecated + return false; + } else if (warnings.size() > 1) { + return true; + } + return warnings.get(0).equals("Posting data directly to anomaly detection jobs is deprecated, " + + "in a future major version it will be compulsory to use a datafeed") == false; + }).build(); + return hlrc.postData(new PostDataRequest(jobId, XContentType.JSON, new BytesArray(data)), postDataOptions); } protected FlushJobResponse flushJob(String jobId) throws IOException { diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml index 92268391fae21..f840ec80226be 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml @@ -87,6 +87,10 @@ --- "Create a job in the mixed cluster and write some data": + + - skip: + features: allowed_warnings + - do: ml.put_job: job_id: mixed-cluster-job @@ -113,6 +117,8 @@ job_id: mixed-cluster-job - do: + allowed_warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: mixed-cluster-job body: diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml index eb04ff0bf83f0..3a1618dff100c 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml @@ -15,6 +15,9 @@ setup: --- "Put job on the old cluster and post some data": + - skip: + features: allowed_warnings + - do: ml.put_job: job_id: old-cluster-job @@ -41,6 +44,8 @@ setup: job_id: old-cluster-job - do: + allowed_warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: old-cluster-job body: @@ -73,6 +78,9 @@ setup: --- "Put categorization job on the old cluster and post some data": + - skip: + features: allowed_warnings + - do: ml.put_job: job_id: old-cluster-categorization-job @@ -104,6 +112,8 @@ setup: job_id: old-cluster-categorization-job - do: + allowed_warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: old-cluster-categorization-job body: @@ -131,6 +141,10 @@ setup: --- "Put job on the old cluster with the default model memory limit and post some data": + + - skip: + features: allowed_warnings + - do: ml.put_job: job_id: no-model-memory-limit-job @@ -152,6 +166,8 @@ setup: job_id: no-model-memory-limit-job - do: + allowed_warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: no-model-memory-limit-job body: @@ -186,6 +202,9 @@ setup: --- "Put job with timing stats checking on the old cluster and post some data": + - skip: + features: allowed_warnings + - do: ml.put_job: job_id: old-cluster-job-with-ts @@ -212,6 +231,8 @@ setup: job_id: old-cluster-job-with-ts - do: + allowed_warnings: + - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed' ml.post_data: job_id: old-cluster-job-with-ts body: From e5fe86af2f3f186f1a1de1226a04f5fb51653d0f Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 15 Dec 2020 20:18:28 +0000 Subject: [PATCH 2/2] Fix compilation --- .../src/main/java/org/elasticsearch/client/RequestOptions.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/RequestOptions.java b/client/rest/src/main/java/org/elasticsearch/client/RequestOptions.java index 6e44c219cd049..3eac76e7a143a 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RequestOptions.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RequestOptions.java @@ -219,8 +219,9 @@ public void setHttpAsyncResponseConsumerFactory(HttpAsyncResponseConsumerFactory * fail the request if the warnings returned don't * exactly match some set. */ - public void setWarningsHandler(WarningsHandler warningsHandler) { + public Builder setWarningsHandler(WarningsHandler warningsHandler) { this.warningsHandler = warningsHandler; + return this; } /**