From c63163e96ae8115f173afff9823b2d2427d06201 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Fri, 7 Jun 2019 07:46:58 -0500 Subject: [PATCH] [ML][Data frame] make sure that fields exist when creating progress (#42943) --- .../DataFrameTransformProgressIT.java | 19 +++++++++++++++++-- ...FrameTransformPersistentTasksExecutor.java | 2 +- .../transforms/TransformProgressGatherer.java | 14 +++++++++++++- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java index fea225ced3bd9..7f320ff9aaf27 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.dataframe.integration; -import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -44,7 +43,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -@LuceneTestCase.AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/42344") public class DataFrameTransformProgressIT extends ESRestTestCase { protected void createReviewsIndex() throws Exception { final int numDocs = 1000; @@ -163,6 +161,23 @@ public void testGetProgress() throws Exception { assertThat(progress.getRemainingDocs(), equalTo(35L)); assertThat(progress.getPercentComplete(), equalTo(0.0)); + histgramGroupConfig = new GroupConfig(Collections.emptyMap(), + Collections.singletonMap("every_50", new HistogramGroupSource("missing_field", 50.0))); + pivotConfig = new PivotConfig(histgramGroupConfig, aggregationConfig, null); + config = new DataFrameTransformConfig("get_progress_transform", + sourceConfig, + destConfig, + null, + pivotConfig, + null); + + response = restClient.search(TransformProgressGatherer.getSearchRequest(config), RequestOptions.DEFAULT); + progress = TransformProgressGatherer.searchResponseToDataFrameTransformProgressFunction().apply(response); + + assertThat(progress.getTotalDocs(), equalTo(0L)); + assertThat(progress.getRemainingDocs(), equalTo(0L)); + assertThat(progress.getPercentComplete(), equalTo(100.0)); + deleteIndex(REVIEWS_INDEX_NAME); } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java index 97d4f9a818bbd..a215570f25dc0 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java @@ -131,7 +131,7 @@ protected void nodeOperation(AllocatedPersistentTask task, @Nullable DataFrameTr .setInitialPosition(stateAndStats.getTransformState().getPosition()) .setProgress(stateAndStats.getTransformState().getProgress()) .setIndexerState(currentIndexerState(stateAndStats.getTransformState())); - logger.info("[{}] Loading existing state: [{}], position [{}]", + logger.debug("[{}] Loading existing state: [{}], position [{}]", transformId, stateAndStats.getTransformState(), stateAndStats.getTransformState().getPosition()); diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/TransformProgressGatherer.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/TransformProgressGatherer.java index 18a341e217294..ab03d94acfac8 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/TransformProgressGatherer.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/TransformProgressGatherer.java @@ -11,6 +11,8 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig; @@ -49,10 +51,20 @@ public static void getInitialProgress(Client client, public static SearchRequest getSearchRequest(DataFrameTransformConfig config) { SearchRequest request = new SearchRequest(config.getSource().getIndex()); request.allowPartialSearchResults(false); + BoolQueryBuilder existsClauses = QueryBuilders.boolQuery(); + config.getPivotConfig() + .getGroupConfig() + .getGroups() + .values() + // TODO change once we allow missing_buckets + .forEach(src -> existsClauses.must(QueryBuilders.existsQuery(src.getField()))); + request.source(new SearchSourceBuilder() .size(0) .trackTotalHits(true) - .query(config.getSource().getQueryConfig().getQuery())); + .query(QueryBuilders.boolQuery() + .filter(config.getSource().getQueryConfig().getQuery()) + .filter(existsClauses))); return request; }