From 13c76efe898254f372747712be5569162b4f9ffa Mon Sep 17 00:00:00 2001 From: Ed Savage <32410745+edsavage@users.noreply.github.com> Date: Mon, 26 Nov 2018 16:15:54 +0000 Subject: [PATCH] [HLRC][ML] Add delete expired data API (#35906) Relates to #29827 --- .../client/MLRequestConverters.java | 12 + .../client/MachineLearningClient.java | 44 ++++ .../client/ml/DeleteExpiredDataRequest.java | 39 ++++ .../client/ml/DeleteExpiredDataResponse.java | 88 ++++++++ .../client/MLRequestConvertersTests.java | 9 + .../client/MachineLearningIT.java | 209 ++++++++++++++++++ .../MlClientDocumentationIT.java | 52 +++++ .../ml/DeleteExpiredDataResponseTests.java | 43 ++++ .../ml/delete-expired-data.asciidoc | 34 +++ .../high-level/supported-apis.asciidoc | 2 + .../ml/apis/delete-expired-data.asciidoc | 47 ++++ docs/reference/ml/apis/ml-api.asciidoc | 7 + 12 files changed, 586 insertions(+) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteExpiredDataRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteExpiredDataResponse.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteExpiredDataResponseTests.java create mode 100644 docs/java-rest/high-level/ml/delete-expired-data.asciidoc create mode 100644 docs/reference/ml/apis/delete-expired-data.asciidoc diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 5347b25c8fa42..f3844566f6e05 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -32,6 +32,7 @@ import org.elasticsearch.client.ml.DeleteCalendarJobRequest; import org.elasticsearch.client.ml.DeleteCalendarRequest; import org.elasticsearch.client.ml.DeleteDatafeedRequest; +import org.elasticsearch.client.ml.DeleteExpiredDataRequest; import org.elasticsearch.client.ml.DeleteFilterRequest; import org.elasticsearch.client.ml.DeleteForecastRequest; import org.elasticsearch.client.ml.DeleteJobRequest; @@ -155,6 +156,17 @@ static Request closeJob(CloseJobRequest closeJobRequest) throws IOException { return request; } + static Request deleteExpiredData(DeleteExpiredDataRequest deleteExpiredDataRequest) { + String endpoint = new EndpointBuilder() + .addPathPartAsIs("_xpack") + .addPathPartAsIs("ml") + .addPathPartAsIs("_delete_expired_data") + .build(); + Request request = new Request(HttpDelete.METHOD_NAME, endpoint); + + return request; + } + static Request deleteJob(DeleteJobRequest deleteJobRequest) { String endpoint = new EndpointBuilder() .addPathPartAsIs("_xpack") diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java index a4d8f1b9aa366..fa204c5bccedd 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java @@ -26,6 +26,8 @@ import org.elasticsearch.client.ml.DeleteCalendarJobRequest; import org.elasticsearch.client.ml.DeleteCalendarRequest; import org.elasticsearch.client.ml.DeleteDatafeedRequest; +import org.elasticsearch.client.ml.DeleteExpiredDataRequest; +import org.elasticsearch.client.ml.DeleteExpiredDataResponse; import org.elasticsearch.client.ml.DeleteFilterRequest; import org.elasticsearch.client.ml.DeleteForecastRequest; import org.elasticsearch.client.ml.DeleteJobRequest; @@ -227,6 +229,48 @@ public void getJobStatsAsync(GetJobStatsRequest request, RequestOptions options, Collections.emptySet()); } + /** + * Deletes expired data from Machine Learning Jobs + *

+ * For additional info + * see ML Delete Expired Data + * documentation + * + * @param request The request to delete expired ML data + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return The action response which contains the acknowledgement or the task id depending on whether the action was set to wait for + * completion + * @throws IOException when there is a serialization issue sending the request or receiving the response + */ + public DeleteExpiredDataResponse deleteExpiredData(DeleteExpiredDataRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, + MLRequestConverters::deleteExpiredData, + options, + DeleteExpiredDataResponse::fromXContent, + Collections.emptySet()); + } + + /** + * Deletes expired data from Machine Learning Jobs asynchronously and notifies the listener on completion + *

+ * For additional info + * see ML Delete Expired Data + * documentation + * + * @param request The request to delete expired ML data + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener Listener to be notified upon request completion + */ + public void deleteExpiredDataAsync(DeleteExpiredDataRequest request, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, + MLRequestConverters::deleteExpiredData, + options, + DeleteExpiredDataResponse::fromXContent, + listener, + Collections.emptySet()); + } + /** * Deletes the given Machine Learning Job *

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteExpiredDataRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteExpiredDataRequest.java new file mode 100644 index 0000000000000..25e340a8bab15 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteExpiredDataRequest.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; + +/** + * Request to delete expired model snapshots and forecasts + */ +public class DeleteExpiredDataRequest extends ActionRequest { + + /** + * Create a new request to delete expired data + */ + public DeleteExpiredDataRequest() { + } + + @Override + public ActionRequestValidationException validate() { + return null; + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteExpiredDataResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteExpiredDataResponse.java new file mode 100644 index 0000000000000..7a9ecbde32513 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteExpiredDataResponse.java @@ -0,0 +1,88 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + + +/** + * A response acknowledging the deletion of expired data + */ +public class DeleteExpiredDataResponse extends ActionResponse implements ToXContentObject { + + private static final ParseField DELETED = new ParseField("deleted"); + + public DeleteExpiredDataResponse(boolean deleted) { + this.deleted = deleted; + } + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("delete_expired_data_response", true, + a -> new DeleteExpiredDataResponse((Boolean) a[0])); + + static { + PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), DELETED); + } + + public static DeleteExpiredDataResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + private final Boolean deleted; + + public Boolean getDeleted() { + return deleted; + } + + @Override + public int hashCode() { + return Objects.hash(deleted); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + if (deleted != null) { + builder.field(DELETED.getPreferredName(), deleted); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + DeleteExpiredDataResponse response = (DeleteExpiredDataResponse) obj; + return Objects.equals(deleted, response.deleted); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java index 1685c62a2963f..107f4614505b2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.client.ml.DeleteCalendarJobRequest; import org.elasticsearch.client.ml.DeleteCalendarRequest; import org.elasticsearch.client.ml.DeleteDatafeedRequest; +import org.elasticsearch.client.ml.DeleteExpiredDataRequest; import org.elasticsearch.client.ml.DeleteFilterRequest; import org.elasticsearch.client.ml.DeleteForecastRequest; import org.elasticsearch.client.ml.DeleteJobRequest; @@ -183,6 +184,14 @@ public void testCloseJob() throws Exception { requestEntityToString(request)); } + public void testDeleteExpiredData() { + DeleteExpiredDataRequest deleteExpiredDataRequest = new DeleteExpiredDataRequest(); + + Request request = MLRequestConverters.deleteExpiredData(deleteExpiredDataRequest); + assertEquals(HttpDelete.METHOD_NAME, request.getMethod()); + assertEquals("/_xpack/ml/_delete_expired_data", request.getEndpoint()); + } + public void testDeleteJob() { String jobId = randomAlphaOfLength(10); DeleteJobRequest deleteJobRequest = new DeleteJobRequest(jobId); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index 66f9acc065e3c..4bc4f3641ac09 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -27,12 +27,15 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.ml.CloseJobRequest; import org.elasticsearch.client.ml.CloseJobResponse; import org.elasticsearch.client.ml.DeleteCalendarEventRequest; import org.elasticsearch.client.ml.DeleteCalendarJobRequest; import org.elasticsearch.client.ml.DeleteCalendarRequest; import org.elasticsearch.client.ml.DeleteDatafeedRequest; +import org.elasticsearch.client.ml.DeleteExpiredDataRequest; +import org.elasticsearch.client.ml.DeleteExpiredDataResponse; import org.elasticsearch.client.ml.DeleteFilterRequest; import org.elasticsearch.client.ml.DeleteForecastRequest; import org.elasticsearch.client.ml.DeleteJobRequest; @@ -110,6 +113,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHit; import org.junit.After; import java.io.IOException; @@ -130,6 +134,7 @@ import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @@ -772,6 +777,142 @@ public void testPreviewDatafeed() throws Exception { assertThat(totalTotals, containsInAnyOrder(totals)); } + public void testDeleteExpiredDataGivenNothingToDelete() throws Exception { + // Tests that nothing goes wrong when there's nothing to delete + MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); + + DeleteExpiredDataResponse response = execute(new DeleteExpiredDataRequest(), + machineLearningClient::deleteExpiredData, + machineLearningClient::deleteExpiredDataAsync); + + assertTrue(response.getDeleted()); + } + + private String createExpiredData(String jobId) throws Exception { + String indexId = jobId + "-data"; + // Set up the index and docs + CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexId); + createIndexRequest.mapping("doc", "timestamp", "type=date,format=epoch_millis", "total", "type=long"); + highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT); + BulkRequest bulk = new BulkRequest(); + bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + long nowMillis = System.currentTimeMillis(); + int totalBuckets = 2 * 24; + int normalRate = 10; + int anomalousRate = 100; + int anomalousBucket = 30; + for (int bucket = 0; bucket < totalBuckets; bucket++) { + long timestamp = nowMillis - TimeValue.timeValueHours(totalBuckets - bucket).getMillis(); + int bucketRate = bucket == anomalousBucket ? anomalousRate : normalRate; + for (int point = 0; point < bucketRate; point++) { + IndexRequest indexRequest = new IndexRequest(indexId, "doc"); + indexRequest.source(XContentType.JSON, "timestamp", timestamp, "total", randomInt(1000)); + bulk.add(indexRequest); + } + } + highLevelClient().bulk(bulk, RequestOptions.DEFAULT); + + { + // Index a randomly named unused state document + String docId = "non_existing_job_" + randomFrom("model_state_1234567#1", "quantiles", "categorizer_state#1"); + IndexRequest indexRequest = new IndexRequest(".ml-state", "doc", docId); + indexRequest.source(Collections.emptyMap(), XContentType.JSON); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + highLevelClient().index(indexRequest, RequestOptions.DEFAULT); + } + + Job job = buildJobForExpiredDataTests(jobId); + putJob(job); + openJob(job); + String datafeedId = createAndPutDatafeed(jobId, indexId); + + startDatafeed(datafeedId, String.valueOf(0), String.valueOf(nowMillis - TimeValue.timeValueHours(24).getMillis())); + + waitForJobToClose(jobId); + + // Update snapshot timestamp to force it out of snapshot retention window + long oneDayAgo = nowMillis - TimeValue.timeValueHours(24).getMillis() - 1; + updateModelSnapshotTimestamp(jobId, String.valueOf(oneDayAgo)); + + openJob(job); + + MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); + ForecastJobRequest forecastJobRequest = new ForecastJobRequest(jobId); + forecastJobRequest.setDuration(TimeValue.timeValueHours(3)); + forecastJobRequest.setExpiresIn(TimeValue.timeValueSeconds(1)); + ForecastJobResponse forecastJobResponse = machineLearningClient.forecastJob(forecastJobRequest, RequestOptions.DEFAULT); + + waitForForecastToComplete(jobId, forecastJobResponse.getForecastId()); + + // Wait for the forecast to expire + awaitBusy(() -> false, 1, TimeUnit.SECONDS); + + // Run up to now + startDatafeed(datafeedId, String.valueOf(0), String.valueOf(nowMillis)); + + waitForJobToClose(jobId); + + return forecastJobResponse.getForecastId(); + } + + public void testDeleteExpiredData() throws Exception { + + String jobId = "test-delete-expired-data"; + + String forecastId = createExpiredData(jobId); + + MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); + + GetModelSnapshotsRequest getModelSnapshotsRequest = new GetModelSnapshotsRequest(jobId); + GetModelSnapshotsResponse getModelSnapshotsResponse = execute(getModelSnapshotsRequest, machineLearningClient::getModelSnapshots, + machineLearningClient::getModelSnapshotsAsync); + + assertEquals(2L, getModelSnapshotsResponse.count()); + + assertTrue(forecastExists(jobId, forecastId)); + + { + // Verify .ml-state contains the expected unused state document + Iterable hits = searchAll(".ml-state"); + List target = new ArrayList<>(); + hits.forEach(target::add); + long numMatches = target.stream() + .filter(c -> c.getId().startsWith("non_existing_job")) + .count(); + + assertThat(numMatches, equalTo(1L)); + } + + DeleteExpiredDataRequest request = new DeleteExpiredDataRequest(); + DeleteExpiredDataResponse response = execute(request, machineLearningClient::deleteExpiredData, + machineLearningClient::deleteExpiredDataAsync); + + assertTrue(response.getDeleted()); + + awaitBusy(() -> false, 1, TimeUnit.SECONDS); + + GetModelSnapshotsRequest getModelSnapshotsRequest1 = new GetModelSnapshotsRequest(jobId); + GetModelSnapshotsResponse getModelSnapshotsResponse1 = execute(getModelSnapshotsRequest1, machineLearningClient::getModelSnapshots, + machineLearningClient::getModelSnapshotsAsync); + + assertEquals(1L, getModelSnapshotsResponse1.count()); + + assertFalse(forecastExists(jobId, forecastId)); + + { + // Verify .ml-state doesn't contain unused state documents + Iterable hits = searchAll(".ml-state"); + List hitList = new ArrayList<>(); + hits.forEach(hitList::add); + long numMatches = hitList.stream() + .filter(c -> c.getId().startsWith("non_existing_job")) + .count(); + + assertThat(numMatches, equalTo(0L)); + } + } + public void testDeleteForecast() throws Exception { String jobId = "test-delete-forecast"; @@ -1146,6 +1287,27 @@ public static String randomValidJobId() { return generator.ofCodePointsLength(random(), 10, 10); } + private static Job buildJobForExpiredDataTests(String jobId) { + Job.Builder builder = new Job.Builder(jobId); + builder.setDescription(randomAlphaOfLength(10)); + + Detector detector = new Detector.Builder() + .setFunction("count") + .setDetectorDescription(randomAlphaOfLength(10)) + .build(); + AnalysisConfig.Builder configBuilder = new AnalysisConfig.Builder(Arrays.asList(detector)); + //should not be random, see:https://github.com/elastic/ml-cpp/issues/208 + configBuilder.setBucketSpan(new TimeValue(1, TimeUnit.HOURS)); + builder.setAnalysisConfig(configBuilder); + + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setTimeFormat(DataDescription.EPOCH_MS); + dataDescription.setTimeField("timestamp"); + builder.setDataDescription(dataDescription); + + return builder.build(); + } + public static Job buildJob(String jobId) { Job.Builder builder = new Job.Builder(jobId); builder.setDescription(randomAlphaOfLength(10)); @@ -1176,6 +1338,53 @@ private void openJob(Job job) throws IOException { highLevelClient().machineLearning().openJob(new OpenJobRequest(job.getId()), RequestOptions.DEFAULT); } + private void waitForJobToClose(String jobId) throws Exception { + MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); + + assertBusy(() -> { + JobStats stats = machineLearningClient.getJobStats(new GetJobStatsRequest(jobId), RequestOptions.DEFAULT).jobStats().get(0); + assertEquals(JobState.CLOSED, stats.getState()); + }, 30, TimeUnit.SECONDS); + } + + private void startDatafeed(String datafeedId, String start, String end) throws Exception { + MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); + + StartDatafeedRequest startDatafeedRequest = new StartDatafeedRequest(datafeedId); + startDatafeedRequest.setStart(start); + startDatafeedRequest.setEnd(end); + StartDatafeedResponse response = execute(startDatafeedRequest, + machineLearningClient::startDatafeed, + machineLearningClient::startDatafeedAsync); + + assertTrue(response.isStarted()); + } + + private void updateModelSnapshotTimestamp(String jobId, String timestamp) throws Exception { + MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); + + GetModelSnapshotsRequest getModelSnapshotsRequest = new GetModelSnapshotsRequest(jobId); + GetModelSnapshotsResponse getModelSnapshotsResponse = execute(getModelSnapshotsRequest, machineLearningClient::getModelSnapshots, + machineLearningClient::getModelSnapshotsAsync); + + assertThat(getModelSnapshotsResponse.count(), greaterThanOrEqualTo(1L)); + + ModelSnapshot modelSnapshot = getModelSnapshotsResponse.snapshots().get(0); + + String snapshotId = modelSnapshot.getSnapshotId(); + String documentId = jobId + "_model_snapshot_" + snapshotId; + + String snapshotUpdate = "{ \"timestamp\": " + timestamp + "}"; + UpdateRequest updateSnapshotRequest = new UpdateRequest(".ml-anomalies-" + jobId, "doc", documentId); + updateSnapshotRequest.doc(snapshotUpdate.getBytes(StandardCharsets.UTF_8), XContentType.JSON); + highLevelClient().update(updateSnapshotRequest, RequestOptions.DEFAULT); + + // Wait a second to ensure subsequent model snapshots will have a different ID (it depends on epoch seconds) + awaitBusy(() -> false, 1, TimeUnit.SECONDS); + } + + + private String createAndPutDatafeed(String jobId, String indexName) throws IOException { String datafeedId = jobId + "-feed"; DatafeedConfig datafeed = DatafeedConfig.builder(datafeedId, jobId) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index eea4ddc2fd610..e0fbf47281f26 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -39,6 +39,8 @@ import org.elasticsearch.client.ml.DeleteCalendarJobRequest; import org.elasticsearch.client.ml.DeleteCalendarRequest; import org.elasticsearch.client.ml.DeleteDatafeedRequest; +import org.elasticsearch.client.ml.DeleteExpiredDataRequest; +import org.elasticsearch.client.ml.DeleteExpiredDataResponse; import org.elasticsearch.client.ml.DeleteFilterRequest; import org.elasticsearch.client.ml.DeleteForecastRequest; import org.elasticsearch.client.ml.DeleteJobRequest; @@ -1959,6 +1961,56 @@ public void onFailure(Exception e) { } } + public void testDeleteExpiredData() throws IOException, InterruptedException { + RestHighLevelClient client = highLevelClient(); + + String jobId = "test-delete-expired-data"; + MachineLearningIT.buildJob(jobId); + { + // tag::delete-expired-data-request + DeleteExpiredDataRequest request = new DeleteExpiredDataRequest(); // <1> + // end::delete-expired-data-request + + // tag::delete-expired-data-execute + DeleteExpiredDataResponse response = client.machineLearning().deleteExpiredData(request, RequestOptions.DEFAULT); + // end::delete-expired-data-execute + + // tag::delete-expired-data-response + boolean deleted = response.getDeleted(); // <1> + // end::delete-expired-data-response + + assertTrue(deleted); + } + { + // tag::delete-expired-data-execute-listener + ActionListener listener = new ActionListener() { + @Override + public void onResponse(DeleteExpiredDataResponse deleteExpiredDataResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::delete-expired-data-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + DeleteExpiredDataRequest deleteExpiredDataRequest = new DeleteExpiredDataRequest(); + + // tag::delete-expired-data-execute-async + client.machineLearning().deleteExpiredDataAsync(deleteExpiredDataRequest, RequestOptions.DEFAULT, listener); // <1> + // end::delete-expired-data-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } + + public void testDeleteModelSnapshot() throws IOException, InterruptedException { RestHighLevelClient client = highLevelClient(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteExpiredDataResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteExpiredDataResponseTests.java new file mode 100644 index 0000000000000..7df554eea96d1 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteExpiredDataResponseTests.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; + + +public class DeleteExpiredDataResponseTests extends AbstractXContentTestCase { + + @Override + protected DeleteExpiredDataResponse createTestInstance() { + return new DeleteExpiredDataResponse(randomBoolean()); + } + + @Override + protected DeleteExpiredDataResponse doParseInstance(XContentParser parser) throws IOException { + return DeleteExpiredDataResponse.PARSER.apply(parser, null); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } +} diff --git a/docs/java-rest/high-level/ml/delete-expired-data.asciidoc b/docs/java-rest/high-level/ml/delete-expired-data.asciidoc new file mode 100644 index 0000000000000..03bd013b2abde --- /dev/null +++ b/docs/java-rest/high-level/ml/delete-expired-data.asciidoc @@ -0,0 +1,34 @@ + +-- +:api: delete-expired-data +:request: DeleteExpiredRequest +:response: DeleteExpiredResponse +-- +[id="{upid}-{api}"] +=== Delete Expired Data API +Delete expired {ml} data. +The API accepts a +{request}+ and responds +with a +{response}+ object. + +[id="{upid}-{api}-request"] +==== Delete Expired Data Request + +A `DeleteExpiredDataRequest` object does not require any arguments. + +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request] +--------------------------------------------------- +<1> Constructing a new request. + +[id="{upid}-{api}-response"] +==== Delete Expired Data Response + +The returned +{response}+ object indicates the acknowledgement of the request: +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +--------------------------------------------------- +<1> `getDeleted` acknowledges the deletion request. + +include::../execution.asciidoc[] diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index cf86952951478..d7c8de6838b40 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -280,6 +280,7 @@ The Java High Level REST Client supports the following Machine Learning APIs: * <<{upid}-delete-model-snapshot>> * <<{upid}-revert-model-snapshot>> * <<{upid}-update-model-snapshot>> +* <<{upid}-delete-expired-data>> include::ml/put-job.asciidoc[] include::ml/get-job.asciidoc[] @@ -321,6 +322,7 @@ include::ml/get-model-snapshots.asciidoc[] include::ml/delete-model-snapshot.asciidoc[] include::ml/revert-model-snapshot.asciidoc[] include::ml/update-model-snapshot.asciidoc[] +include::ml/delete-expired-data.asciidoc[] == Migration APIs diff --git a/docs/reference/ml/apis/delete-expired-data.asciidoc b/docs/reference/ml/apis/delete-expired-data.asciidoc new file mode 100644 index 0000000000000..310a17ff10304 --- /dev/null +++ b/docs/reference/ml/apis/delete-expired-data.asciidoc @@ -0,0 +1,47 @@ +[role="xpack"] +[testenv="platinum"] +[[ml-delete-expired-data]] +=== Delete Expired Data API +++++ +Delete Expired Data +++++ + +Deletes expired and unused machine learning data. + +==== Request + +`DELETE _xpack/ml/_delete_expired_data` + +==== Description + +Deletes all job results, model snapshots and forecast data that have exceeded their +`retention days` period. +Machine Learning state documents that are not associated with any job are also deleted. + +==== Authorization + +You must have `manage_ml`, or `manage` cluster privileges to use this API. +For more information, see +{stack-ov}/security-privileges.html[Security Privileges] and +{stack-ov}/built-in-roles.html[Built-in Roles]. + + +==== Examples + +The endpoint takes no arguments: + +[source,js] +-------------------------------------------------- +DELETE _xpack/ml/_delete_expired_data +-------------------------------------------------- +// CONSOLE +// TEST + +When the expired data is deleted, you receive the following response: +[source,js] +---- +{ + "deleted": true +} +---- +// TESTRESPONSE diff --git a/docs/reference/ml/apis/ml-api.asciidoc b/docs/reference/ml/apis/ml-api.asciidoc index d3d1c42d0a8e3..dd8a54f73f634 100644 --- a/docs/reference/ml/apis/ml-api.asciidoc +++ b/docs/reference/ml/apis/ml-api.asciidoc @@ -82,6 +82,12 @@ machine learning APIs and in advanced job configuration options in Kibana. * <> +[float] +[[ml-api-delete-expired-data-endpoint]] +=== Delete Expired Data + +* <> + //ADD include::post-calendar-event.asciidoc[] include::put-calendar-job.asciidoc[] @@ -101,6 +107,7 @@ include::delete-forecast.asciidoc[] include::delete-job.asciidoc[] include::delete-calendar-job.asciidoc[] include::delete-snapshot.asciidoc[] +include::delete-expired-data.asciidoc[] //FIND include::find-file-structure.asciidoc[] //FLUSH