From 6614d4e5ff1f746fa018951ab94023a42645cbfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Tue, 4 Jan 2022 14:25:12 +0100 Subject: [PATCH] [Transform] Report _preview warning in the face of pipeline failure. (#81972) --- .../integration/TransformPivotRestIT.java | 72 +++++++++++++++++++ .../TransportPreviewTransformAction.java | 13 +++- 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java index 49f86fe370736..d04c19bd7af74 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java @@ -10,8 +10,10 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.xcontent.XContentBuilder; @@ -29,7 +31,9 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasSize; @@ -949,6 +953,74 @@ public void testPreviewTransformWithPipeline() throws Exception { }); } + @SuppressWarnings("unchecked") + public void testPreviewTransformWithPipelineScript() throws Exception { + String pipelineId = "my-preview-pivot-pipeline-script"; + Request pipelineRequest = new Request("PUT", "/_ingest/pipeline/" + pipelineId); + pipelineRequest.setJsonEntity(""" + { + "description": "my pivot preview pipeline", + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx._id = ctx['non']['existing'];" + } + } + ] + } + """); + client().performRequest(pipelineRequest); + + setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME); + final Request createPreviewRequest = createRequestWithAuth("POST", getTransformEndpoint() + "_preview", null); + createPreviewRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); + + String config = """ + { + "source": { + "index": "%s" + }, + "dest": { + "pipeline": "%s" + }, + "pivot": { + "group_by": { + "user.id": { + "terms": { + "field": "user_id" + } + }, + "by_day": { + "date_histogram": { + "fixed_interval": "1d", + "field": "timestamp" + } + } + }, + "aggregations": { + "user.avg_rating": { + "avg": { + "field": "stars" + } + } + } + } + }""".formatted(REVIEWS_INDEX_NAME, pipelineId); + createPreviewRequest.setJsonEntity(config); + + Response createPreviewResponse = client().performRequest(createPreviewRequest); + Map previewTransformResponse = entityAsMap(createPreviewResponse); + List> preview = (List>) previewTransformResponse.get("preview"); + // Pipeline failed for all the docs so the preview is empty + assertThat(preview, is(empty())); + assertThat(createPreviewResponse.getWarnings(), hasSize(1)); + assertThat( + createPreviewResponse.getWarnings().get(0), + allOf(containsString("Pipeline returned 100 errors, first error:"), containsString("type=script_exception")) + ); + } + public void testPivotWithMaxOnDateField() throws Exception { String transformId = "simple_date_histogram_pivot_with_max_time"; String transformIndex = "pivot_reviews_via_date_histogram_with_max_time"; diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java index 83bca0a55b5f9..3731e7fa0d18c 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java @@ -230,13 +230,24 @@ private void getPreview( ActionListener pipelineResponseActionListener = ActionListener.wrap(simulatePipelineResponse -> { List> docs = new ArrayList<>(simulatePipelineResponse.getResults().size()); + List> errors = new ArrayList<>(); for (SimulateDocumentResult simulateDocumentResult : simulatePipelineResponse.getResults()) { try (XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()) { XContentBuilder content = simulateDocumentResult.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); Map tempMap = XContentHelper.convertToMap(BytesReference.bytes(content), true, XContentType.JSON).v2(); - docs.add((Map) XContentMapValues.extractValue("doc._source", tempMap)); + Map doc = (Map) XContentMapValues.extractValue("doc._source", tempMap); + if (doc != null) { + docs.add(doc); + } + Map error = (Map) XContentMapValues.extractValue("error", tempMap); + if (error != null) { + errors.add(error); + } } } + if (errors.isEmpty() == false) { + HeaderWarning.addWarning("Pipeline returned " + errors.size() + " errors, first error: " + errors.get(0)); + } TransformDestIndexSettings generatedDestIndexSettings = TransformIndex.createTransformDestIndexSettings( mappings.get(), transformId,