From 5b3b52a652da51fee45cf4d8a3c2ed389265138d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Wed, 5 Jan 2022 09:49:37 +0100 Subject: [PATCH] [7.16] [Transform] Report _preview warning in the face of pipeline failure. (#81972) (#82202) --- .../integration/TransformPivotRestIT.java | 57 +++++++++++++++++++ .../TransportPreviewTransformAction.java | 13 ++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java index 49f86fe370736..0e16c628f729a 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java @@ -10,8 +10,10 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.xcontent.XContentBuilder; @@ -29,7 +31,9 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasSize; @@ -949,6 +953,59 @@ public void testPreviewTransformWithPipeline() throws Exception { }); } + @SuppressWarnings("unchecked") + public void testPreviewTransformWithPipelineScript() throws Exception { + String pipelineId = "my-preview-pivot-pipeline-script"; + Request pipelineRequest = new Request("PUT", "/_ingest/pipeline/" + pipelineId); + pipelineRequest.setJsonEntity( + "{\n" + + " \"description\" : \"my pivot preview pipeline\",\n" + + " \"processors\" : [\n" + + " {\n" + + " \"script\" : {\n" + + " \"lang\": \"painless\",\n" + + " \"source\": \"ctx._id = ctx['non']['existing'];\"\n" + + " }\n" + + " }\n" + + " ]\n" + + "}" + ); + client().performRequest(pipelineRequest); + + setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME); + final Request createPreviewRequest = createRequestWithAuth("POST", getTransformEndpoint() + "_preview", null); + createPreviewRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); + + String config = "{ \"source\": {\"index\":\"" + + REVIEWS_INDEX_NAME + + "\"} ," + + "\"dest\": {\"pipeline\": \"" + + pipelineId + + "\"}," + + " \"pivot\": {" + + " \"group_by\": {" + + " \"user.id\": {\"terms\": { \"field\": \"user_id\" }}," + + " \"by_day\": {\"date_histogram\": {\"fixed_interval\": \"1d\",\"field\":\"timestamp\"}}}," + + " \"aggregations\": {" + + " \"user.avg_rating\": {" + + " \"avg\": {" + + " \"field\": \"stars\"" + + " } } } }" + + "}"; + createPreviewRequest.setJsonEntity(config); + + Response createPreviewResponse = client().performRequest(createPreviewRequest); + Map previewTransformResponse = entityAsMap(createPreviewResponse); + List> preview = (List>) previewTransformResponse.get("preview"); + // Pipeline failed for all the docs so the preview is empty + assertThat(preview, is(empty())); + assertThat(createPreviewResponse.getWarnings(), is(not(empty()))); + assertThat( + createPreviewResponse.getWarnings().get(createPreviewResponse.getWarnings().size() - 1), + allOf(containsString("Pipeline returned 100 errors, first error:"), containsString("type=script_exception")) + ); + } + public void testPivotWithMaxOnDateField() throws Exception { String transformId = "simple_date_histogram_pivot_with_max_time"; String transformIndex = "pivot_reviews_via_date_histogram_with_max_time"; diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java index 83bca0a55b5f9..3731e7fa0d18c 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java @@ -230,13 +230,24 @@ private void getPreview( ActionListener pipelineResponseActionListener = ActionListener.wrap(simulatePipelineResponse -> { List> docs = new ArrayList<>(simulatePipelineResponse.getResults().size()); + List> errors = new ArrayList<>(); for (SimulateDocumentResult simulateDocumentResult : simulatePipelineResponse.getResults()) { try (XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()) { XContentBuilder content = simulateDocumentResult.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); Map tempMap = XContentHelper.convertToMap(BytesReference.bytes(content), true, XContentType.JSON).v2(); - docs.add((Map) XContentMapValues.extractValue("doc._source", tempMap)); + Map doc = (Map) XContentMapValues.extractValue("doc._source", tempMap); + if (doc != null) { + docs.add(doc); + } + Map error = (Map) XContentMapValues.extractValue("error", tempMap); + if (error != null) { + errors.add(error); + } } } + if (errors.isEmpty() == false) { + HeaderWarning.addWarning("Pipeline returned " + errors.size() + " errors, first error: " + errors.get(0)); + } TransformDestIndexSettings generatedDestIndexSettings = TransformIndex.createTransformDestIndexSettings( mappings.get(), transformId,