From f3f4ec6b18bd118aaf732fae3b68b8f1da242fb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Thu, 21 Jan 2021 13:27:42 +0100 Subject: [PATCH] [Transform] Add support for search-time runtime fields. (#67643) --- .../transform/transforms/SourceConfig.java | 38 +- .../transforms/SourceConfigTests.java | 14 +- docs/reference/rest-api/common-parms.asciidoc | 10 +- .../transform/apis/preview-transform.asciidoc | 18 +- .../transform/apis/put-transform.asciidoc | 20 +- .../transform/transforms/SourceConfig.java | 40 ++- .../UpdateTransformActionRequestTests.java | 23 +- .../UpdateTransformsActionResponseTests.java | 66 +++- .../transforms/SourceConfigTests.java | 33 +- .../transforms/TransformConfigTests.java | 6 +- .../test/transform/preview_transforms.yml | 85 +++++ .../xpack/transform/integration/LatestIT.java | 21 +- .../transform/integration/TransformIT.java | 21 +- .../integration/TransformIntegTestCase.java | 20 +- .../TransformUsingSearchRuntimeFieldsIT.java | 329 ++++++++++++++++++ .../transforms/TransformIndexer.java | 11 +- .../transform/transforms/latest/Latest.java | 26 +- .../transform/transforms/pivot/Pivot.java | 15 +- .../transforms/pivot/PivotTests.java | 9 +- 19 files changed, 678 insertions(+), 127 deletions(-) create mode 100644 x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUsingSearchRuntimeFieldsIT.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SourceConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SourceConfig.java index 157a637040c45..508c4f3a7b934 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SourceConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SourceConfig.java @@ -24,10 +24,12 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; @@ -49,15 +51,19 @@ public class SourceConfig implements ToXContentObject { String[] index = ((List)args[0]).toArray(new String[0]); // default handling: if the user does not specify a query, we default to match_all QueryConfig queryConfig = (QueryConfig) args[1]; - return new SourceConfig(index, queryConfig); + @SuppressWarnings("unchecked") + Map runtimeMappings = (Map) args[2]; + return new SourceConfig(index, queryConfig, runtimeMappings); }); static { PARSER.declareStringArray(constructorArg(), INDEX); PARSER.declareObject(optionalConstructorArg(), (p, c) -> QueryConfig.fromXContent(p), QUERY); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.map(), SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD); } private final String[] index; private final QueryConfig queryConfig; + private final Map runtimeMappings; /** * Create a new SourceConfig for the provided indices. @@ -67,8 +73,7 @@ public class SourceConfig implements ToXContentObject { * @param index Any number of indices. At least one non-null, non-empty, index should be provided */ public SourceConfig(String... index) { - this.index = index; - this.queryConfig = null; + this(index, null, null); } /** @@ -76,10 +81,12 @@ public SourceConfig(String... index) { * * @param index Any number of indices. At least one non-null, non-empty, index should be provided * @param queryConfig A QueryConfig object that contains the desired query. Defaults to MatchAll query. + * @param runtimeMappings Search-time runtime fields that can be used by the transform */ - SourceConfig(String[] index, QueryConfig queryConfig) { + SourceConfig(String[] index, QueryConfig queryConfig, Map runtimeMappings) { this.index = index; this.queryConfig = queryConfig; + this.runtimeMappings = runtimeMappings; } public String[] getIndex() { @@ -90,6 +97,10 @@ public QueryConfig getQueryConfig() { return queryConfig; } + public Map getRuntimeMappings() { + return runtimeMappings; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -99,6 +110,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (queryConfig != null) { builder.field(QUERY.getPreferredName(), queryConfig); } + if (runtimeMappings != null) { + builder.field(SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD.getPreferredName(), runtimeMappings); + } builder.endObject(); return builder; } @@ -113,14 +127,16 @@ public boolean equals(Object other) { } SourceConfig that = (SourceConfig) other; - return Arrays.equals(index, that.index) && Objects.equals(queryConfig, that.queryConfig); + return Arrays.equals(index, that.index) + && Objects.equals(queryConfig, that.queryConfig) + && Objects.equals(runtimeMappings, that.runtimeMappings); } @Override public int hashCode(){ // Using Arrays.hashCode as Objects.hash does not deeply hash nested arrays. Since we are doing Array.equals, this is necessary - int hash = Arrays.hashCode(index); - return 31 * hash + (queryConfig == null ? 0 : queryConfig.hashCode()); + int indexArrayHash = Arrays.hashCode(index); + return Objects.hash(indexArrayHash, queryConfig, runtimeMappings); } public static Builder builder() { @@ -130,6 +146,7 @@ public static Builder builder() { public static class Builder { private String[] index; private QueryConfig queryConfig; + private Map runtimeMappings; /** * Sets what indices from which to fetch data @@ -160,8 +177,13 @@ public Builder setQuery(QueryBuilder query) { return this.setQueryConfig(new QueryConfig(query)); } + public Builder setRuntimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; + return this; + } + public SourceConfig build() { - return new SourceConfig(index, queryConfig); + return new SourceConfig(index, queryConfig, runtimeMappings); } } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SourceConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SourceConfigTests.java index dede3d2f2566e..ce0040626ac93 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SourceConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SourceConfigTests.java @@ -26,18 +26,28 @@ import org.elasticsearch.test.AbstractXContentTestCase; import java.io.IOException; +import java.util.Map; import java.util.function.Predicate; import static java.util.Collections.emptyList; +import static java.util.Collections.singletonMap; +import static java.util.stream.Collectors.toMap; public class SourceConfigTests extends AbstractXContentTestCase { public static SourceConfig randomSourceConfig() { - return new SourceConfig(generateRandomStringArray(10, 10, false, false), - QueryConfigTests.randomQueryConfig()); + return new SourceConfig( + generateRandomStringArray(10, 10, false, false), + QueryConfigTests.randomQueryConfig(), + randomRuntimeMappings()); } + private static Map randomRuntimeMappings() { + return randomList(0, 10, () -> randomAlphaOfLengthBetween(1, 10)).stream() + .distinct() + .collect(toMap(f -> f, f -> singletonMap("type", randomFrom("boolean", "date", "double", "keyword", "long")))); + } @Override protected SourceConfig doParseInstance(XContentParser parser) throws IOException { diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 282ed5471aa52..12bcd45394f7c 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -700,7 +700,7 @@ objects define the `group by` fields and the aggregation to reduce the data. end::pivot[] tag::pivot-aggs[] -Defines how to aggregate the grouped data. The following aggregations are +Defines how to aggregate the grouped data. The following aggregations are currently supported: + -- @@ -727,7 +727,7 @@ currently supported: end::pivot-aggs[] tag::pivot-group-by[] -Defines how to group the data. More than one grouping can be defined per pivot. +Defines how to group the data. More than one grouping can be defined per pivot. The following groupings are currently supported: + -- @@ -939,6 +939,10 @@ A query clause that retrieves a subset of data from the source index. See <>. end::source-query-transforms[] +tag::source-runtime-mappings-transforms[] +Definitions of search-time runtime fields that can be used by the transform. +end::source-runtime-mappings-transforms[] + tag::state-transform[] The status of the {transform}, which can be one of the following values: + @@ -990,7 +994,7 @@ default value is `60s`. end::sync-time-delay[] tag::transform-latest[] -The `latest` method transforms the data by finding the latest document for each +The `latest` method transforms the data by finding the latest document for each unique key. end::transform-latest[] diff --git a/docs/reference/transform/apis/preview-transform.asciidoc b/docs/reference/transform/apis/preview-transform.asciidoc index 20467ae3a2dde..a46a48dfc07bc 100644 --- a/docs/reference/transform/apis/preview-transform.asciidoc +++ b/docs/reference/transform/apis/preview-transform.asciidoc @@ -18,10 +18,10 @@ Previews a {transform}. [[preview-transform-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following +If the {es} {security-features} are enabled, you must have the following privileges: -* `manage_transform` +* `manage_transform` * source index: `read`, `view_index_metadata` The built-in `transform_admin` role has the `manage_transform` privilege. @@ -136,6 +136,10 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-index-transform `query`::: (Optional, object) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-query-transforms] + +`runtime_mappings`::: +(Optional, object) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-runtime-mappings-transforms] ==== //End source @@ -160,7 +164,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time] `delay`:::: (Optional, <>) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-delay] - + `field`:::: (Optional, string) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-field] @@ -191,11 +195,11 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max == {api-response-body-title} `preview`:: -(array) An array of documents. In particular, they are the JSON representation -of the documents that would be created in the destination index by the +(array) An array of documents. In particular, they are the JSON representation +of the documents that would be created in the destination index by the {transform}. -//Begin generated_dest_index +//Begin generated_dest_index `generated_dest_index`:: (object) Contains details about the destination index. + @@ -207,7 +211,7 @@ of the documents that would be created in the destination index by the (object) The aliases for the destination index. `mappings`::: -(object) The <> for each document in the destination index. +(object) The <> for each document in the destination index. `settings`::: (object) The <> for the destination index. diff --git a/docs/reference/transform/apis/put-transform.asciidoc b/docs/reference/transform/apis/put-transform.asciidoc index 71d59acc08efa..ec9032d72335f 100644 --- a/docs/reference/transform/apis/put-transform.asciidoc +++ b/docs/reference/transform/apis/put-transform.asciidoc @@ -33,10 +33,10 @@ For more information, see <>, <>, and == {api-description-title} This API defines a {transform}, which copies data from source indices, -transforms it, and persists it into an entity-centric destination index. If you -choose to use the pivot method for your {transform}, the entities are defined by -the set of `group_by` fields in the `pivot` object. If you choose to use the -latest method, the entities are defined by the `unique_key` field values in the +transforms it, and persists it into an entity-centric destination index. If you +choose to use the pivot method for your {transform}, the entities are defined by +the set of `group_by` fields in the `pivot` object. If you choose to use the +latest method, the entities are defined by the `unique_key` field values in the `latest` object. You can also think of the destination index as a two-dimensional tabular data structure (known as a {dataframe}). The ID for each document in the @@ -56,12 +56,12 @@ it had at the time of creation and uses those same roles. If those roles do not have the required privileges on the source and destination indices, the {transform} fails when it attempts unauthorized operations. -IMPORTANT: You must use {kib} or this API to create a {transform}. Do not put a +IMPORTANT: You must use {kib} or this API to create a {transform}. Do not put a {transform} directly into any `.transform-internal*` indices using the {es} index API. If {es} {security-features} are enabled, do not give users any privileges on `.transform-internal*` indices. If you used {transforms} prior to 7.5, also do not give users any privileges on `.data-frame-internal*` indices. - + You must choose either the latest or pivot method for your {transform}; you cannot use both in a single {transform}. @@ -187,6 +187,10 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-index-transform `query`::: (Optional, object) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-query-transforms] + +`runtime_mappings`::: +(Optional, object) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-runtime-mappings-transforms] ==== //End source @@ -289,7 +293,7 @@ When the {transform} is created, you receive the following results: } ---- -The following {transform} uses the `latest` method: +The following {transform} uses the `latest` method: [source,console] -------------------------------------------------- @@ -315,4 +319,4 @@ PUT _transform/ecommerce_transform2 } } -------------------------------------------------- -// TEST[setup:kibana_sample_data_ecommerce] \ No newline at end of file +// TEST[setup:kibana_sample_data_ecommerce] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfig.java index 3e30482daef0c..5fc0fdafcf477 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfig.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.transform.transforms; +import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -16,12 +17,15 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.license.RemoteClusterLicenseChecker; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper; import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; @@ -44,15 +48,18 @@ private static ConstructingObjectParser createParser(boolean String[] index = ((List)args[0]).toArray(new String[0]); // default handling: if the user does not specify a query, we default to match_all QueryConfig queryConfig = args[1] == null ? QueryConfig.matchAll() : (QueryConfig) args[1]; - return new SourceConfig(index, queryConfig); + Map runtimeMappings = args[2] == null ? Collections.emptyMap() : (Map) args[2]; + return new SourceConfig(index, queryConfig, runtimeMappings); }); parser.declareStringArray(constructorArg(), INDEX); parser.declareObject(optionalConstructorArg(), (p, c) -> QueryConfig.fromXContent(p, lenient), QUERY); + parser.declareObject(optionalConstructorArg(), (p, c) -> p.map(), SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD); return parser; } private final String[] index; private final QueryConfig queryConfig; + private final Map runtimeMappings; /** * Create a new SourceConfig for the provided indices. @@ -62,7 +69,7 @@ private static ConstructingObjectParser createParser(boolean * @param index Any number of indices. At least one non-null, non-empty, index should be provided */ public SourceConfig(String... index) { - this(index, QueryConfig.matchAll()); + this(index, QueryConfig.matchAll(), Collections.emptyMap()); } /** @@ -70,8 +77,9 @@ public SourceConfig(String... index) { * * @param index Any number of indices. At least one non-null, non-empty, index should be provided * @param queryConfig A QueryConfig object that contains the desired query, needs to be non-null + * @param runtimeMappings Search-time runtime fields that can be used by the transform */ - public SourceConfig(String[] index, QueryConfig queryConfig) { + public SourceConfig(String[] index, QueryConfig queryConfig, Map runtimeMappings) { ExceptionsHelper.requireNonNull(index, INDEX.getPreferredName()); if (index.length == 0) { throw new IllegalArgumentException("must specify at least one index"); @@ -81,11 +89,19 @@ public SourceConfig(String[] index, QueryConfig queryConfig) { } this.index = index; this.queryConfig = ExceptionsHelper.requireNonNull(queryConfig, QUERY.getPreferredName()); + this.runtimeMappings = + Collections.unmodifiableMap( + ExceptionsHelper.requireNonNull(runtimeMappings, SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD.getPreferredName())); } public SourceConfig(final StreamInput in) throws IOException { index = in.readStringArray(); queryConfig = new QueryConfig(in); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + runtimeMappings = in.readMap(); + } else { + runtimeMappings = Collections.emptyMap(); + } } public String[] getIndex() { @@ -96,6 +112,10 @@ public QueryConfig getQueryConfig() { return queryConfig; } + public Map getRuntimeMappings() { + return runtimeMappings; + } + public boolean isValid() { return queryConfig.isValid(); } @@ -108,6 +128,9 @@ public boolean requiresRemoteCluster() { public void writeTo(StreamOutput out) throws IOException { out.writeStringArray(index); queryConfig.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeMap(runtimeMappings); + } } @Override @@ -119,6 +142,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } else if(queryConfig.equals(QueryConfig.matchAll()) == false) { builder.field(QUERY.getPreferredName(), queryConfig); } + if (runtimeMappings.isEmpty() == false) { + builder.field(SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD.getPreferredName(), runtimeMappings); + } builder.endObject(); return builder; } @@ -133,14 +159,16 @@ public boolean equals(Object other) { } SourceConfig that = (SourceConfig) other; - return Arrays.equals(index, that.index) && Objects.equals(queryConfig, that.queryConfig); + return Arrays.equals(index, that.index) + && Objects.equals(queryConfig, that.queryConfig) + && Objects.equals(runtimeMappings, that.runtimeMappings); } @Override public int hashCode(){ // Using Arrays.hashCode as Objects.hash does not deeply hash nested arrays. Since we are doing Array.equals, this is necessary - int hash = Arrays.hashCode(index); - return 31 * hash + (queryConfig == null ? 0 : queryConfig.hashCode()); + int indexArrayHash = Arrays.hashCode(index); + return Objects.hash(indexArrayHash, queryConfig, runtimeMappings); } public static SourceConfig fromXContent(final XContentParser parser, boolean lenient) throws IOException { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformActionRequestTests.java index 5279160b18b69..8d06f77d235a0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformActionRequestTests.java @@ -15,6 +15,9 @@ import java.io.IOException; import static org.elasticsearch.xpack.core.transform.transforms.TransformConfigUpdateTests.randomTransformConfigUpdate; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; public class UpdateTransformActionRequestTests extends AbstractWireSerializingTransformTestCase { @@ -26,7 +29,6 @@ protected Writeable.Reader instanceReader() { @Override protected Request createTestInstance() { Request request = new Request(randomTransformConfigUpdate(), randomAlphaOfLength(10), randomBoolean()); - if (randomBoolean()) { request.setConfig(TransformConfigTests.randomTransformConfig()); } @@ -46,7 +48,15 @@ public void testBWCPre78() throws IOException { assertEquals(newRequest.getId(), oldRequest.getId()); assertEquals(newRequest.getUpdate().getDestination(), oldRequest.getUpdate().getDestination()); assertEquals(newRequest.getUpdate().getFrequency(), oldRequest.getUpdate().getFrequency()); - assertEquals(newRequest.getUpdate().getSource(), oldRequest.getUpdate().getSource()); + + if (newRequest.getUpdate().getSource() != null) { + assertThat(oldRequest.getUpdate().getSource().getIndex(), is(equalTo(newRequest.getUpdate().getSource().getIndex()))); + assertThat( + oldRequest.getUpdate().getSource().getQueryConfig(), + is(equalTo(newRequest.getUpdate().getSource().getQueryConfig()))); + // runtime_mappings was added in 7.12 so it is always empty after deserializing from 7.7 + assertThat(oldRequest.getUpdate().getSource().getRuntimeMappings(), is(anEmptyMap())); + } assertEquals(newRequest.getUpdate().getSyncConfig(), oldRequest.getUpdate().getSyncConfig()); assertEquals(newRequest.isDeferValidation(), oldRequest.isDeferValidation()); @@ -61,7 +71,14 @@ public void testBWCPre78() throws IOException { assertEquals(newRequest.getId(), newRequestFromOld.getId()); assertEquals(newRequest.getUpdate().getDestination(), newRequestFromOld.getUpdate().getDestination()); assertEquals(newRequest.getUpdate().getFrequency(), newRequestFromOld.getUpdate().getFrequency()); - assertEquals(newRequest.getUpdate().getSource(), newRequestFromOld.getUpdate().getSource()); + if (newRequest.getUpdate().getSource() != null) { + assertThat(newRequestFromOld.getUpdate().getSource().getIndex(), is(equalTo(newRequest.getUpdate().getSource().getIndex()))); + assertThat( + newRequestFromOld.getUpdate().getSource().getQueryConfig(), + is(equalTo(newRequest.getUpdate().getSource().getQueryConfig()))); + // runtime_mappings was added in 7.12 so it is always empty after deserializing from 7.7 + assertThat(newRequestFromOld.getUpdate().getSource().getRuntimeMappings(), is(anEmptyMap())); + } assertEquals(newRequest.getUpdate().getSyncConfig(), newRequestFromOld.getUpdate().getSyncConfig()); assertEquals(newRequest.isDeferValidation(), newRequestFromOld.isDeferValidation()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java index d8ee90eb8adc7..fedfdb962ddb5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java @@ -13,8 +13,14 @@ import org.elasticsearch.xpack.core.transform.action.compat.UpdateTransformActionPre78; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests; +import org.elasticsearch.xpack.core.transform.transforms.pivot.SingleGroupSource; import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; public class UpdateTransformsActionResponseTests extends AbstractSerializingTransformTestCase { @@ -35,7 +41,7 @@ protected Response doParseInstance(XContentParser parser) throws IOException { public void testBWCPre78() throws IOException { Response newResponse = new Response( - TransformConfigTests.randomTransformConfigWithoutHeaders(Version.V_7_8_0, randomAlphaOfLengthBetween(1, 10)) + TransformConfigTests.randomTransformConfigWithoutHeaders(randomAlphaOfLengthBetween(1, 10)) ); UpdateTransformActionPre78.Response oldResponse = writeAndReadBWCObject( newResponse, @@ -49,8 +55,33 @@ public void testBWCPre78() throws IOException { assertEquals(newResponse.getConfig().getCreateTime(), oldResponse.getConfig().getCreateTime()); assertEquals(newResponse.getConfig().getDestination(), oldResponse.getConfig().getDestination()); assertEquals(newResponse.getConfig().getFrequency(), oldResponse.getConfig().getFrequency()); - assertEquals(newResponse.getConfig().getPivotConfig(), oldResponse.getConfig().getPivotConfig()); - assertEquals(newResponse.getConfig().getSource(), oldResponse.getConfig().getSource()); + assertEquals( + newResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet(), + oldResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet()); + for (Map.Entry oldResponseGroupEntry + : oldResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().entrySet()) { + SingleGroupSource oldResponseGroup = oldResponseGroupEntry.getValue(); + SingleGroupSource newResponseGroup = + newResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().get(oldResponseGroupEntry.getKey()); + assertThat(oldResponseGroup.getField(), is(equalTo(newResponseGroup.getField()))); + assertThat(oldResponseGroup.getScriptConfig(), is(equalTo(newResponseGroup.getScriptConfig()))); + // missing_bucket was added in 7.10 so it is always false after deserializing from 7.7 + assertThat(oldResponseGroup.getMissingBucket(), is(false)); + } + assertEquals( + newResponse.getConfig().getPivotConfig().getAggregationConfig(), + oldResponse.getConfig().getPivotConfig().getAggregationConfig()); + assertEquals( + newResponse.getConfig().getPivotConfig().getMaxPageSearchSize(), + oldResponse.getConfig().getPivotConfig().getMaxPageSearchSize()); + if (newResponse.getConfig().getSource() != null) { + assertThat(newResponse.getConfig().getSource().getIndex(), is(equalTo(newResponse.getConfig().getSource().getIndex()))); + assertThat( + newResponse.getConfig().getSource().getQueryConfig(), + is(equalTo(newResponse.getConfig().getSource().getQueryConfig()))); + // runtime_mappings was added in 7.12 so it is always empty after deserializing from 7.7 + assertThat(oldResponse.getConfig().getSource().getRuntimeMappings(), is(anEmptyMap())); + } assertEquals(newResponse.getConfig().getSyncConfig(), oldResponse.getConfig().getSyncConfig()); assertEquals(newResponse.getConfig().getVersion(), oldResponse.getConfig().getVersion()); @@ -68,8 +99,33 @@ public void testBWCPre78() throws IOException { assertEquals(newResponse.getConfig().getCreateTime(), newRequestFromOld.getConfig().getCreateTime()); assertEquals(newResponse.getConfig().getDestination(), newRequestFromOld.getConfig().getDestination()); assertEquals(newResponse.getConfig().getFrequency(), newRequestFromOld.getConfig().getFrequency()); - assertEquals(newResponse.getConfig().getPivotConfig(), newRequestFromOld.getConfig().getPivotConfig()); - assertEquals(newResponse.getConfig().getSource(), newRequestFromOld.getConfig().getSource()); + assertEquals( + newResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet(), + newRequestFromOld.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet()); + for (Map.Entry newRequestFromOldGroupEntry + : newRequestFromOld.getConfig().getPivotConfig().getGroupConfig().getGroups().entrySet()) { + SingleGroupSource newRequestFromOldGroup = newRequestFromOldGroupEntry.getValue(); + SingleGroupSource newResponseGroup = + newResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().get(newRequestFromOldGroupEntry.getKey()); + assertThat(newRequestFromOldGroup.getField(), is(equalTo(newResponseGroup.getField()))); + assertThat(newRequestFromOldGroup.getScriptConfig(), is(equalTo(newResponseGroup.getScriptConfig()))); + // missing_bucket was added in 7.10 so it is always false after deserializing from 7.7 + assertThat(newRequestFromOldGroup.getMissingBucket(), is(false)); + } + assertEquals( + newResponse.getConfig().getPivotConfig().getAggregationConfig(), + newRequestFromOld.getConfig().getPivotConfig().getAggregationConfig()); + assertEquals( + newResponse.getConfig().getPivotConfig().getMaxPageSearchSize(), + newRequestFromOld.getConfig().getPivotConfig().getMaxPageSearchSize()); + if (newResponse.getConfig().getSource() != null) { + assertThat(newRequestFromOld.getConfig().getSource().getIndex(), is(equalTo(newResponse.getConfig().getSource().getIndex()))); + assertThat( + newRequestFromOld.getConfig().getSource().getQueryConfig(), + is(equalTo(newResponse.getConfig().getSource().getQueryConfig()))); + // runtime_mappings was added in 7.12 so it is always empty after deserializing from 7.7 + assertThat(newRequestFromOld.getConfig().getSource().getRuntimeMappings(), is(anEmptyMap())); + } assertEquals(newResponse.getConfig().getSyncConfig(), newRequestFromOld.getConfig().getSyncConfig()); assertEquals(newResponse.getConfig().getVersion(), newRequestFromOld.getConfig().getVersion()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java index 629a9485a190f..0a5894a2a826a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java @@ -11,21 +11,35 @@ import org.junit.Before; import java.io.IOException; +import java.util.Map; import java.util.function.Predicate; +import static java.util.Collections.singletonMap; +import static java.util.stream.Collectors.toMap; + public class SourceConfigTests extends AbstractSerializingTransformTestCase { private boolean lenient; public static SourceConfig randomSourceConfig() { - return new SourceConfig(generateRandomStringArray(10, 10, false, false), - QueryConfigTests.randomQueryConfig()); + return new SourceConfig( + generateRandomStringArray(10, 10, false, false), + QueryConfigTests.randomQueryConfig(), + randomRuntimeMappings()); } public static SourceConfig randomInvalidSourceConfig() { // create something broken but with a source - return new SourceConfig(generateRandomStringArray(10, 10, false, false), - QueryConfigTests.randomInvalidQueryConfig()); + return new SourceConfig( + generateRandomStringArray(10, 10, false, false), + QueryConfigTests.randomInvalidQueryConfig(), + randomRuntimeMappings()); + } + + private static Map randomRuntimeMappings() { + return randomList(0, 10, () -> randomAlphaOfLengthBetween(1, 10)).stream() + .distinct() + .collect(toMap(f -> f, f -> singletonMap("type", randomFrom("boolean", "date", "double", "keyword", "long")))); } @Before @@ -61,19 +75,18 @@ protected Reader instanceReader() { public void testRequiresRemoteCluster() { assertFalse(new SourceConfig(new String [] {"index1", "index2", "index3"}, - QueryConfigTests.randomQueryConfig()).requiresRemoteCluster()); + QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); assertTrue(new SourceConfig(new String [] {"index1", "remote2:index2", "index3"}, - QueryConfigTests.randomQueryConfig()).requiresRemoteCluster()); + QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); assertTrue(new SourceConfig(new String [] {"index1", "index2", "remote3:index3"}, - QueryConfigTests.randomQueryConfig()).requiresRemoteCluster()); + QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); assertTrue(new SourceConfig(new String [] {"index1", "remote2:index2", "remote3:index3"}, - QueryConfigTests.randomQueryConfig()).requiresRemoteCluster()); + QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); assertTrue(new SourceConfig(new String [] {"remote1:index1"}, - QueryConfigTests.randomQueryConfig()).requiresRemoteCluster()); + QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); } - } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index fa7828e797bd9..c3b80fb1e12f3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -42,10 +42,6 @@ public static TransformConfig randomTransformConfigWithoutHeaders() { } public static TransformConfig randomTransformConfigWithoutHeaders(String id) { - return randomTransformConfigWithoutHeaders(Version.CURRENT, id); - } - - public static TransformConfig randomTransformConfigWithoutHeaders(Version version, String id) { return new TransformConfig( id, randomSourceConfig(), @@ -53,7 +49,7 @@ public static TransformConfig randomTransformConfigWithoutHeaders(Version versio randomBoolean() ? null : TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), randomBoolean() ? null : randomSyncConfig(), null, - PivotConfigTests.randomPivotConfig(version), + PivotConfigTests.randomPivotConfig(Version.CURRENT), null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomSettingsConfig(), diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/preview_transforms.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/preview_transforms.yml index 70498ce9da257..5f0ccb22d78ee 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/preview_transforms.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/preview_transforms.yml @@ -162,6 +162,7 @@ setup: "sort": "time" } } + - length: { $body: 2 } - match: { preview.0.airline: bar } - match: { preview.0.time: "2017-02-18T01:00:00Z" } - match: { preview.1.airline: foo } @@ -194,6 +195,7 @@ setup: "sort": "time" } } + - length: { $body: 2 } - match: { preview.0.airline: bar } - match: { preview.0.time: "2017-02-18T01:00:00Z" } - match: { preview.0.my_field: 42 } @@ -334,3 +336,86 @@ setup: } } } + +--- +"Test preview transform pivot with search runtime fields": + + - do: + transform.preview_transform: + body: > + { + "source": { + "index": "airline-data", + "runtime_mappings" : { + "airline-upper": { + "type": "keyword", + "script": "emit(doc['airline'].value.toUpperCase())" + }, + "responsetime_x_2": { + "type": "double", + "script": "emit(doc['responsetime'].value * 2.0)" + }, + "time-5m": { + "type": "date", + "script": "emit(doc['time'].value.toInstant().minus(5, ChronoUnit.MINUTES).toEpochMilli())" + } + } + }, + "pivot": { + "group_by": { + "airline": {"terms": {"field": "airline-upper"}}, + "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}}, + "aggs": { + "avg_response": {"avg": {"field": "responsetime_x_2"}}, + "time.max": {"max": {"field": "time-5m"}}, + "time.min": {"min": {"field": "time-5m"}} + } + } + } + - match: { preview.0.airline: FOO } + - match: { preview.0.by-hour: "2017-02-18T00:00:00.000Z" } + - match: { preview.0.avg_response: 2.0 } + - match: { preview.0.time.max: "2017-02-18T00:25:00.000Z" } + - match: { preview.0.time.min: "2017-02-17T23:55:00.000Z" } + - match: { preview.1.airline: BAR } + - match: { preview.1.by-hour: "2017-02-18T01:00:00.000Z" } + - match: { preview.1.avg_response: 84.0 } + - match: { preview.1.time.max: "2017-02-18T00:55:00.000Z" } + - match: { preview.1.time.min: "2017-02-18T00:55:00.000Z" } + - match: { preview.2.airline: FOO } + - match: { preview.2.by-hour: "2017-02-18T01:00:00.000Z" } + - match: { preview.2.avg_response: 84.0 } + - match: { preview.2.time.max: "2017-02-18T00:56:00.000Z" } + - match: { preview.2.time.min: "2017-02-18T00:56:00.000Z" } + - match: { generated_dest_index.mappings.properties.airline.type: "keyword" } + - match: { generated_dest_index.mappings.properties.by-hour.type: "date" } + - match: { generated_dest_index.mappings.properties.avg_response.type: "double" } + - is_false: generated_dest_index.mappings.properties.time\.max.type + - is_false: generated_dest_index.mappings.properties.time\.min.type + +--- +"Test preview transform latest with search runtime fields": + + - do: + transform.preview_transform: + body: > + { + "source": { + "index": "airline-data", + "runtime_mappings" : { + "time-5m": { + "type": "date", + "script": "emit(doc['time'].value.toInstant().minus(5, ChronoUnit.MINUTES).toEpochMilli())" + } + } + }, + "latest": { + "unique_key": ["airline"], + "sort": "time-5m" + } + } + - length: { $body: 2 } + - match: { preview.0.airline: bar } + - match: { preview.0.time: "2017-02-18T01:00:00Z" } + - match: { preview.1.airline: foo } + - match: { preview.1.time: "2017-02-18T01:01:00Z" } diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/LatestIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/LatestIT.java index 7599884e0db9d..8aad83f313756 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/LatestIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/LatestIT.java @@ -13,10 +13,10 @@ import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; -import org.elasticsearch.client.transform.PreviewTransformRequest; import org.elasticsearch.client.transform.PreviewTransformResponse; import org.elasticsearch.client.transform.transforms.TransformConfig; import org.elasticsearch.client.transform.transforms.latest.LatestConfig; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.After; @@ -121,7 +121,7 @@ public void testLatest() throws Exception { String destIndexName = "reviews-latest"; TransformConfig transformConfig = - createTransformConfigBuilder(TRANSFORM_NAME, destIndexName, SOURCE_INDEX_NAME) + createTransformConfigBuilder(TRANSFORM_NAME, destIndexName, QueryBuilders.matchAllQuery(), SOURCE_INDEX_NAME) .setLatestConfig( LatestConfig.builder() .setUniqueKey(USER_ID) @@ -153,7 +153,7 @@ public void testLatestPreview() throws Exception { createReviewsIndex(SOURCE_INDEX_NAME, 100, NUM_USERS, LatestIT::getUserIdForRow, LatestIT::getDateStringForRow); TransformConfig transformConfig = - createTransformConfigBuilder(TRANSFORM_NAME, "dummy", SOURCE_INDEX_NAME) + createTransformConfigBuilder(TRANSFORM_NAME, "dummy", QueryBuilders.matchAllQuery(), SOURCE_INDEX_NAME) .setLatestConfig( LatestConfig.builder() .setUniqueKey(USER_ID) @@ -161,14 +161,11 @@ public void testLatestPreview() throws Exception { .build()) .build(); - try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { - PreviewTransformResponse previewResponse = - restClient.transform().previewTransform(new PreviewTransformRequest(transformConfig), RequestOptions.DEFAULT); - // Verify preview mappings - assertThat(previewResponse.getMappings(), allOf(hasKey("_meta"), hasEntry("properties", emptyMap()))); - // Verify preview contents - assertThat(previewResponse.getDocs(), hasSize(NUM_USERS + 1)); - assertThat(previewResponse.getDocs(), containsInAnyOrder(EXPECTED_DEST_INDEX_ROWS)); - } + PreviewTransformResponse previewResponse = previewTransform(transformConfig, RequestOptions.DEFAULT); + // Verify preview mappings + assertThat(previewResponse.getMappings(), allOf(hasKey("_meta"), hasEntry("properties", emptyMap()))); + // Verify preview contents + assertThat(previewResponse.getDocs(), hasSize(NUM_USERS + 1)); + assertThat(previewResponse.getDocs(), containsInAnyOrder(EXPECTED_DEST_INDEX_ROWS)); } } diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java index 40c0c8540e945..2f6d29d3f14ab 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java @@ -96,7 +96,7 @@ public void testTransformCrud() throws Exception { .addAggregator(AggregationBuilders.max("timestamp").field("timestamp")); TransformConfig config = - createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryBuilders.matchAllQuery(), null, indexName) + createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryBuilders.matchAllQuery(), indexName) .setPivotConfig(createPivotConfig(groups, aggs)) .build(); @@ -132,7 +132,7 @@ public void testContinuousTransformCrud() throws Exception { .addAggregator(AggregationBuilders.max("timestamp").field("timestamp")); TransformConfig config = - createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryBuilders.matchAllQuery(), null, indexName) + createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryBuilders.matchAllQuery(), indexName) .setPivotConfig(createPivotConfig(groups, aggs)) .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1))) .build(); @@ -180,7 +180,7 @@ public void testContinuousTransformUpdate() throws Exception { String id = "transform-to-update"; String dest = "reviews-by-user-business-day-to-update"; TransformConfig config = - createTransformConfigBuilder(id, dest, QueryBuilders.matchAllQuery(), null, indexName) + createTransformConfigBuilder(id, dest, QueryBuilders.matchAllQuery(), indexName) .setPivotConfig(createPivotConfig(groups, aggs)) .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1))) .build(); @@ -266,7 +266,7 @@ public void testStopWaitForCheckpoint() throws Exception { .addAggregator(AggregationBuilders.max("timestamp").field("timestamp")); TransformConfig config = - createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryBuilders.matchAllQuery(), null, indexName) + createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryBuilders.matchAllQuery(), indexName) .setPivotConfig(createPivotConfig(groups, aggs)) .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1))) .build(); @@ -332,15 +332,12 @@ public void testContinuousTransformRethrottle() throws Exception { .addAggregator(AggregationBuilders.max("timestamp").field("timestamp")); TransformConfig config = - createTransformConfigBuilder( - transformId, - "reviews-by-user-business-day", - QueryBuilders.matchAllQuery(), - // set requests per second and page size low enough to fail the test if update does not succeed - SettingsConfig.builder().setRequestsPerSecond(1F).setMaxPageSearchSize(10), - indexName) + createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryBuilders.matchAllQuery(), indexName) .setPivotConfig(createPivotConfig(groups, aggs)) - .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1))).build(); + .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1))) + // set requests per second and page size low enough to fail the test if update does not succeed, + .setSettings(SettingsConfig.builder().setRequestsPerSecond(1F).setMaxPageSearchSize(10).build()) + .build(); assertTrue(putTransform(config, RequestOptions.DEFAULT).isAcknowledged()); assertTrue(startTransform(config.getId(), RequestOptions.DEFAULT).isAcknowledged()); diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIntegTestCase.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIntegTestCase.java index 993abb23598a2..7d7ea4e05f83d 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIntegTestCase.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIntegTestCase.java @@ -25,6 +25,8 @@ import org.elasticsearch.client.transform.GetTransformResponse; import org.elasticsearch.client.transform.GetTransformStatsRequest; import org.elasticsearch.client.transform.GetTransformStatsResponse; +import org.elasticsearch.client.transform.PreviewTransformRequest; +import org.elasticsearch.client.transform.PreviewTransformResponse; import org.elasticsearch.client.transform.PutTransformRequest; import org.elasticsearch.client.transform.StartTransformRequest; import org.elasticsearch.client.transform.StartTransformResponse; @@ -33,7 +35,6 @@ import org.elasticsearch.client.transform.UpdateTransformRequest; import org.elasticsearch.client.transform.transforms.DestConfig; import org.elasticsearch.client.transform.transforms.QueryConfig; -import org.elasticsearch.client.transform.transforms.SettingsConfig; import org.elasticsearch.client.transform.transforms.SourceConfig; import org.elasticsearch.client.transform.transforms.TransformConfig; import org.elasticsearch.client.transform.transforms.TransformConfigUpdate; @@ -55,7 +56,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchModule; @@ -200,6 +200,12 @@ protected AcknowledgedResponse putTransform(TransformConfig config, RequestOptio } } + protected PreviewTransformResponse previewTransform(TransformConfig config, RequestOptions options) throws IOException { + try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { + return restClient.transform().previewTransform(new PreviewTransformRequest(config), options); + } + } + protected GetTransformStatsResponse getTransformStats(String id) throws IOException { try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { return restClient.transform().getTransformStats(new GetTransformStatsRequest(id), RequestOptions.DEFAULT); @@ -277,19 +283,10 @@ protected PivotConfig createPivotConfig( .build(); } - protected TransformConfig.Builder createTransformConfigBuilder( - String id, - String destinationIndex, - String... sourceIndices - ) throws Exception { - return createTransformConfigBuilder(id, destinationIndex, QueryBuilders.matchAllQuery(), null, sourceIndices); - } - protected TransformConfig.Builder createTransformConfigBuilder( String id, String destinationIndex, QueryBuilder queryBuilder, - SettingsConfig.Builder settingsBuilder, String... sourceIndices ) throws Exception { return TransformConfig.builder() @@ -297,7 +294,6 @@ protected TransformConfig.Builder createTransformConfigBuilder( .setSource(SourceConfig.builder().setIndex(sourceIndices).setQueryConfig(createQueryConfig(queryBuilder)).build()) .setDest(DestConfig.builder().setIndex(destinationIndex).build()) .setFrequency(TimeValue.timeValueSeconds(10)) - .setSettings(settingsBuilder != null ? settingsBuilder.build() : null) .setDescription("Test transform config id: " + id); } diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUsingSearchRuntimeFieldsIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUsingSearchRuntimeFieldsIT.java new file mode 100644 index 0000000000000..8a962e56edef7 --- /dev/null +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUsingSearchRuntimeFieldsIT.java @@ -0,0 +1,329 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.transform.integration; + +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.GetMappingsRequest; +import org.elasticsearch.client.indices.GetMappingsResponse; +import org.elasticsearch.client.transform.PreviewTransformResponse; +import org.elasticsearch.client.transform.transforms.SourceConfig; +import org.elasticsearch.client.transform.transforms.TransformConfig; +import org.elasticsearch.client.transform.transforms.TransformStats; +import org.elasticsearch.client.transform.transforms.latest.LatestConfig; +import org.elasticsearch.client.transform.transforms.pivot.SingleGroupSource; +import org.elasticsearch.client.transform.transforms.pivot.TermsGroupSource; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +import static java.util.Collections.singletonMap; +import static java.util.stream.Collectors.toList; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +public class TransformUsingSearchRuntimeFieldsIT extends TransformIntegTestCase { + + private static final String REVIEWS_INDEX_NAME = "basic-crud-reviews"; + private static final int NUM_USERS = 28; + + private static final Integer getUserIdForRow(int row) { + return row % NUM_USERS; + } + + private static final String getDateStringForRow(int row) { + int day = (11 + (row / 100)) % 28; + int hour = 10 + (row % 13); + int min = 10 + (row % 49); + int sec = 10 + (row % 49); + return "2017-01-" + (day < 10 ? "0" + day : day) + "T" + hour + ":" + min + ":" + sec + "Z"; + } + + private static Map createRuntimeMappings() { + return new HashMap<>() {{ + put("user-upper", new HashMap<>() {{ + put("type", "keyword"); + put("script", singletonMap("source", "if (params._source.user_id != null) {emit(params._source.user_id.toUpperCase())}")); + }}); + put("stars", new HashMap<>() {{ + put("type", "long"); + }}); + put("stars-x2", new HashMap<>() {{ + put("type", "long"); + put("script", singletonMap("source", "if (params._source.stars != null) {emit(2 * params._source.stars)}")); + }}); + put("timestamp-5m", new HashMap<>() {{ + put("type", "date"); + put("script", singletonMap( + "source", "emit(doc['timestamp'].value.toInstant().minus(5, ChronoUnit.MINUTES).toEpochMilli())")); + }}); + }}; + } + + @Before + public void createReviewsIndex() throws Exception { + createReviewsIndex( + REVIEWS_INDEX_NAME, + 100, + NUM_USERS, + TransformUsingSearchRuntimeFieldsIT::getUserIdForRow, + TransformUsingSearchRuntimeFieldsIT::getDateStringForRow); + } + + @After + public void cleanTransforms() throws IOException { + cleanUp(); + } + + public void testPivotTransform() throws Exception { + String destIndexName = "reviews-by-user-pivot"; + String transformId = "transform-with-st-rt-fields-pivot"; + Map runtimeMappings = createRuntimeMappings(); + + Map groups = singletonMap("by-user", TermsGroupSource.builder().setField("user-upper").build()); + AggregatorFactories.Builder aggs = AggregatorFactories.builder() + .addAggregator(AggregationBuilders.avg("review_score").field("stars")) + .addAggregator(AggregationBuilders.avg("review_score_rt").field("stars-x2")) + .addAggregator(AggregationBuilders.max("timestamp").field("timestamp")) + .addAggregator(AggregationBuilders.max("timestamp_rt").field("timestamp-5m")); + TransformConfig config = + createTransformConfigBuilder(transformId, destIndexName, QueryBuilders.matchAllQuery(), "dummy") + .setSource(SourceConfig.builder() + .setIndex(REVIEWS_INDEX_NAME) + .setQuery(QueryBuilders.matchAllQuery()) + .setRuntimeMappings(runtimeMappings) + .build()) + .setPivotConfig(createPivotConfig(groups, aggs)) + .build(); + + PreviewTransformResponse previewResponse = previewTransform(config, RequestOptions.DEFAULT); + // Verify preview mappings + assertThat(previewResponse.getMappings(), allOf(hasKey("_meta"), hasKey("properties"))); + // Verify preview contents + assertThat(previewResponse.getDocs(), hasSize(NUM_USERS)); + previewResponse.getDocs().forEach( + doc -> { + assertThat((String) doc.get("by-user"), isUpperCase()); + assertThat(doc.get("review_score_rt"), is(equalTo(2 * (double) doc.get("review_score")))); + assertThat( + Instant.parse((String) doc.get("timestamp_rt")), + is(equalTo(Instant.parse((String) doc.get("timestamp")).minus(5, ChronoUnit.MINUTES)))); + } + ); + + assertTrue(putTransform(config, RequestOptions.DEFAULT).isAcknowledged()); + assertTrue(startTransform(config.getId(), RequestOptions.DEFAULT).isAcknowledged()); + + waitUntilCheckpoint(config.getId(), 1L); + + stopTransform(config.getId()); + assertBusy(() -> { + assertEquals(TransformStats.State.STOPPED, getTransformStats(config.getId()).getTransformsStats().get(0).getState()); + }); + + try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { + restClient.indices().refresh(new RefreshRequest(destIndexName), RequestOptions.DEFAULT); + // Verify destination index mappings + GetMappingsResponse destIndexMapping = + restClient.indices().getMapping(new GetMappingsRequest().indices(destIndexName), RequestOptions.DEFAULT); + assertThat(destIndexMapping.mappings().get(destIndexName).sourceAsMap(), allOf(hasKey("_meta"), hasKey("properties"))); + // Verify destination index contents + SearchResponse searchResponse = + restClient.search(new SearchRequest(destIndexName).source(new SearchSourceBuilder().size(1000)), RequestOptions.DEFAULT); + assertThat(searchResponse.getHits().getTotalHits().value, is(equalTo(Long.valueOf(NUM_USERS)))); + assertThat( + Stream.of(searchResponse.getHits().getHits()).map(SearchHit::getSourceAsMap).collect(toList()), + is(equalTo(previewResponse.getDocs()))); + } + } + + public void testPivotTransform_BadRuntimeFieldScript() throws Exception { + String destIndexName = "reviews-by-user-pivot"; + String transformId = "transform-with-st-rt-fields-pivot"; + Map runtimeMappings = new HashMap<>() {{ + put("user-upper", new HashMap<>() {{ + put("type", "keyword"); + // Method name used in the script is misspelled, i.e.: "toUperCase" instead of "toUpperCase" + put("script", singletonMap("source", "if (params._source.user_id != null) {emit(params._source.user_id.toUperCase())}")); + }}); + }}; + + Map groups = singletonMap("by-user", TermsGroupSource.builder().setField("user-upper").build()); + AggregatorFactories.Builder aggs = AggregatorFactories.builder() + .addAggregator(AggregationBuilders.avg("review_score").field("stars")) + .addAggregator(AggregationBuilders.avg("review_score_rt").field("stars-x2")) + .addAggregator(AggregationBuilders.max("timestamp").field("timestamp")) + .addAggregator(AggregationBuilders.max("timestamp_rt").field("timestamp-5m")); + TransformConfig config = + createTransformConfigBuilder(transformId, destIndexName, QueryBuilders.matchAllQuery(), "dummy") + .setSource(SourceConfig.builder() + .setIndex(REVIEWS_INDEX_NAME) + .setQuery(QueryBuilders.matchAllQuery()) + .setRuntimeMappings(runtimeMappings) + .build()) + .setPivotConfig(createPivotConfig(groups, aggs)) + .build(); + + Exception e = expectThrows(Exception.class, () -> previewTransform(config, RequestOptions.DEFAULT)); + assertThat( + ExceptionsHelper.stackTrace(e), + allOf(containsString("script_exception"), containsString("dynamic method [java.lang.String, toUperCase/0] not found"))); + + e = expectThrows(Exception.class, () -> putTransform(config, RequestOptions.DEFAULT)); + assertThat( + ExceptionsHelper.stackTrace(e), + allOf(containsString("script_exception"), containsString("dynamic method [java.lang.String, toUperCase/0] not found"))); + } + + public void testLatestTransform() throws Exception { + String destIndexName = "reviews-by-user-latest"; + String transformId = "transform-with-st-rt-fields-latest"; + Map runtimeMappings = createRuntimeMappings(); + + SourceConfig sourceConfig = + SourceConfig.builder() + .setIndex(REVIEWS_INDEX_NAME) + .setQuery(QueryBuilders.matchAllQuery()) + .setRuntimeMappings(runtimeMappings) + .build(); + TransformConfig configWithOrdinaryFields = + createTransformConfigBuilder(transformId, destIndexName, QueryBuilders.matchAllQuery(), "dummy") + .setSource(sourceConfig) + .setLatestConfig(LatestConfig.builder() + .setUniqueKey("user_id") + .setSort("timestamp") + .build()) + .build(); + + PreviewTransformResponse previewWithOrdinaryFields = previewTransform(configWithOrdinaryFields, RequestOptions.DEFAULT); + // Verify preview mappings + assertThat(previewWithOrdinaryFields.getMappings(), allOf(hasKey("_meta"), hasKey("properties"))); + // Verify preview contents + assertThat("Got preview: " + previewWithOrdinaryFields, previewWithOrdinaryFields.getDocs(), hasSize(NUM_USERS)); + previewWithOrdinaryFields.getDocs().forEach( + doc -> { + assertThat(doc, hasKey("user_id")); + assertThat(doc, not(hasKey("user-upper"))); + } + ); + + TransformConfig configWithRuntimeFields = + createTransformConfigBuilder(transformId, destIndexName, QueryBuilders.matchAllQuery(), "dummy") + .setSource(sourceConfig) + .setLatestConfig(LatestConfig.builder() + .setUniqueKey("user-upper") + .setSort("timestamp-5m") + .build()) + .build(); + + PreviewTransformResponse previewWithRuntimeFields = previewTransform(configWithRuntimeFields, RequestOptions.DEFAULT); + assertThat(previewWithRuntimeFields.getDocs(), is(equalTo(previewWithOrdinaryFields.getDocs()))); + + assertTrue(putTransform(configWithRuntimeFields, RequestOptions.DEFAULT).isAcknowledged()); + assertTrue(startTransform(configWithRuntimeFields.getId(), RequestOptions.DEFAULT).isAcknowledged()); + + waitUntilCheckpoint(configWithRuntimeFields.getId(), 1L); + + stopTransform(configWithRuntimeFields.getId()); + assertBusy(() -> { + assertEquals( + TransformStats.State.STOPPED, + getTransformStats(configWithRuntimeFields.getId()).getTransformsStats().get(0).getState()); + }); + + try (RestHighLevelClient restClient = new TestRestHighLevelClient()) { + restClient.indices().refresh(new RefreshRequest(destIndexName), RequestOptions.DEFAULT); + // Verify destination index mappings + GetMappingsResponse destIndexMapping = + restClient.indices().getMapping(new GetMappingsRequest().indices(destIndexName), RequestOptions.DEFAULT); + assertThat(destIndexMapping.mappings().get(destIndexName).sourceAsMap(), allOf(hasKey("_meta"), hasKey("properties"))); + // Verify destination index contents + SearchResponse searchResponse = + restClient.search(new SearchRequest(destIndexName).source(new SearchSourceBuilder().size(1000)), RequestOptions.DEFAULT); + assertThat(searchResponse.getHits().getTotalHits().value, is(equalTo(Long.valueOf(NUM_USERS)))); + assertThat( + Stream.of(searchResponse.getHits().getHits()).map(SearchHit::getSourceAsMap).collect(toList()), + is(equalTo(previewWithOrdinaryFields.getDocs()))); + } + } + + public void testLatestTransform_BadRuntimeFieldScript() throws Exception { + String destIndexName = "reviews-by-user-latest"; + String transformId = "transform-with-st-rt-fields-latest"; + Map runtimeMappings = new HashMap<>() {{ + put("user-upper", new HashMap<>() {{ + put("type", "keyword"); + // Method name used in the script is misspelled, i.e.: "toUperCase" instead of "toUpperCase" + put("script", singletonMap("source", "if (params._source.user_id != null) {emit(params._source.user_id.toUperCase())}")); + }}); + }}; + + SourceConfig sourceConfig = + SourceConfig.builder() + .setIndex(REVIEWS_INDEX_NAME) + .setQuery(QueryBuilders.matchAllQuery()) + .setRuntimeMappings(runtimeMappings) + .build(); + TransformConfig configWithRuntimeFields = + createTransformConfigBuilder(transformId, destIndexName, QueryBuilders.matchAllQuery(), "dummy") + .setSource(sourceConfig) + .setLatestConfig(LatestConfig.builder() + .setUniqueKey("user-upper") + .setSort("timestamp") + .build()) + .build(); + + Exception e = expectThrows(Exception.class, () -> previewTransform(configWithRuntimeFields, RequestOptions.DEFAULT)); + assertThat( + ExceptionsHelper.stackTrace(e), + allOf(containsString("script_exception"), containsString("dynamic method [java.lang.String, toUperCase/0] not found"))); + + e = expectThrows(Exception.class, () -> putTransform(configWithRuntimeFields, RequestOptions.DEFAULT)); + assertThat( + ExceptionsHelper.stackTrace(e), + allOf(containsString("script_exception"), containsString("dynamic method [java.lang.String, toUperCase/0] not found"))); + } + + private static IsUpperCaseMatcher isUpperCase() { + return new IsUpperCaseMatcher(); + } + + private static class IsUpperCaseMatcher extends TypeSafeMatcher { + + @Override + protected boolean matchesSafely(String item) { + return item.chars().noneMatch(Character::isLowerCase); + } + + @Override + public void describeTo(Description description) { + description.appendText("an upper-case string"); + } + } +} diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index ccdda9aedf447..e33890b2871e5 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -829,10 +829,7 @@ protected QueryBuilder buildFilterQuery() { protected SearchRequest buildSearchRequest() { assert nextCheckpoint != null; - SearchRequest searchRequest = new SearchRequest(getConfig().getSource().getIndex()).allowPartialSearchResults(false) - .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); // .size(0); - + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().runtimeMappings(getConfig().getSource().getRuntimeMappings()); switch (runState) { case APPLY_RESULTS: buildUpdateQuery(sourceBuilder); @@ -846,8 +843,10 @@ protected SearchRequest buildSearchRequest() { throw new IllegalStateException("Transform indexer job encountered an illegal state [" + runState + "]"); } - searchRequest.source(sourceBuilder); - return searchRequest; + return new SearchRequest(getConfig().getSource().getIndex()) + .allowPartialSearchResults(false) + .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN) + .source(sourceBuilder); } private SearchSourceBuilder buildChangedBucketsQuery(SearchSourceBuilder sourceBuilder) { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/latest/Latest.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/latest/Latest.java index 24b1ddf86cce7..731bb709d4262 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/latest/Latest.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/latest/Latest.java @@ -84,6 +84,16 @@ public int getInitialPageSize() { return DEFAULT_INITIAL_MAX_PAGE_SEARCH_SIZE; } + private SearchRequest buildSearchRequest(SourceConfig sourceConfig, int pageSize) { + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query(sourceConfig.getQueryConfig().getQuery()) + .runtimeMappings(sourceConfig.getRuntimeMappings()); + buildSearchQuery(sourceBuilder, null, pageSize); + return new SearchRequest(sourceConfig.getIndex()) + .source(sourceBuilder) + .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); + } + @Override public SearchSourceBuilder buildSearchQuery(SearchSourceBuilder builder, Map position, int pageSize) { cachedCompositeAggregation.aggregateAfter(position); @@ -148,13 +158,7 @@ public Tuple, Map> processSearchResponse( @Override public void validateQuery(Client client, SourceConfig sourceConfig, ActionListener listener) { - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(sourceConfig.getQueryConfig().getQuery()); - buildSearchQuery(sourceBuilder, null, TEST_QUERY_PAGE_SIZE); - SearchRequest searchRequest = - new SearchRequest(sourceConfig.getIndex()) - .source(sourceBuilder) - .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); - + SearchRequest searchRequest = buildSearchRequest(sourceConfig, TEST_QUERY_PAGE_SIZE); client.execute(SearchAction.INSTANCE, searchRequest, ActionListener.wrap(response -> { if (response == null) { listener.onFailure( @@ -198,13 +202,7 @@ public void preview( int numberOfBuckets, ActionListener>> listener ) { - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(sourceConfig.getQueryConfig().getQuery()); - buildSearchQuery(sourceBuilder, null, numberOfBuckets); - SearchRequest searchRequest = - new SearchRequest(sourceConfig.getIndex()) - .source(sourceBuilder) - .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); - + SearchRequest searchRequest = buildSearchRequest(sourceConfig, numberOfBuckets); ClientHelper.assertNoAuthorizationHeader(headers); ClientHelper.executeWithHeadersAsync( headers, diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java index f51cee6fd0814..78b7e0a54a9cc 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.aggregations.AggregationBuilder; @@ -182,15 +181,13 @@ public int getInitialPageSize() { } private SearchRequest buildSearchRequest(SourceConfig sourceConfig, Map position, int pageSize) { - QueryBuilder queryBuilder = sourceConfig.getQueryConfig().getQuery(); - - SearchRequest searchRequest = new SearchRequest(sourceConfig.getIndex()); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query(sourceConfig.getQueryConfig().getQuery()) + .runtimeMappings(sourceConfig.getRuntimeMappings()); buildSearchQuery(sourceBuilder, null, pageSize); - sourceBuilder.query(queryBuilder); - searchRequest.source(sourceBuilder); - searchRequest.indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); - return searchRequest; + return new SearchRequest(sourceConfig.getIndex()) + .source(sourceBuilder) + .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); } @Override diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java index 3cf90305b2ee0..7ee105610c18b 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; -import org.elasticsearch.xpack.core.transform.transforms.QueryConfig; import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; import org.elasticsearch.xpack.core.transform.transforms.pivot.AggregationConfig; @@ -94,14 +93,14 @@ protected NamedXContentRegistry xContentRegistry() { } public void testValidateExistingIndex() throws Exception { - SourceConfig source = new SourceConfig(new String[] { "existing_source_index" }, QueryConfig.matchAll()); + SourceConfig source = new SourceConfig("existing_source_index"); Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10), new SettingsConfig(), Version.CURRENT); assertValidTransform(client, source, pivot); } public void testValidateNonExistingIndex() throws Exception { - SourceConfig source = new SourceConfig(new String[] { "non_existing_source_index" }, QueryConfig.matchAll()); + SourceConfig source = new SourceConfig("non_existing_source_index"); Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10), new SettingsConfig(), Version.CURRENT); assertInvalidTransform(client, source, pivot); @@ -132,7 +131,7 @@ public void testInitialPageSize() throws Exception { public void testSearchFailure() throws Exception { // test a failure during the search operation, transform creation fails if // search has failures although they might just be temporary - SourceConfig source = new SourceConfig(new String[] { "existing_source_index_with_failing_shards" }, QueryConfig.matchAll()); + SourceConfig source = new SourceConfig("existing_source_index_with_failing_shards"); Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10), new SettingsConfig(), Version.CURRENT); @@ -142,7 +141,7 @@ public void testSearchFailure() throws Exception { public void testValidateAllSupportedAggregations() throws Exception { for (String agg : supportedAggregations) { AggregationConfig aggregationConfig = getAggregationConfig(agg); - SourceConfig source = new SourceConfig(new String[] { "existing_source" }, QueryConfig.matchAll()); + SourceConfig source = new SourceConfig("existing_source"); Function pivot = new Pivot( getValidPivotConfig(aggregationConfig),