From 7be74a8046762f7b857200292367a837c6e8759e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Tue, 18 Jan 2022 09:01:23 +0100 Subject: [PATCH] Introduce `deduce_mappings` transform setting (#82256) --- .../transform/transforms/SettingsConfig.java | 64 +++++++++++++-- .../org/elasticsearch/client/TransformIT.java | 38 ++++++++- .../transforms/SettingsConfigTests.java | 8 ++ .../transforms/hlrc/SettingsConfigTests.java | 2 + docs/reference/rest-api/common-parms.asciidoc | 5 ++ .../transform/apis/preview-transform.asciidoc | 6 ++ .../transform/apis/put-transform.asciidoc | 3 + .../transform/apis/update-transform.asciidoc | 3 + .../xpack/core/transform/TransformField.java | 1 + .../transform/transforms/SettingsConfig.java | 77 +++++++++++++++++-- .../transform/transforms/TransformConfig.java | 9 ++- .../transforms/SettingsConfigTests.java | 26 ++++++- .../TransformConfigUpdateTests.java | 10 +-- .../schema/transform_config.schema.json | 6 ++ .../test/transform/preview_transforms.yml | 23 ++++++ .../test/transform/transforms_start_stop.yml | 39 +++++++++- .../transform/transforms/pivot/Pivot.java | 5 ++ .../TransformIndexerFailureHandlingTests.java | 6 +- .../TransformIndexerStateTests.java | 2 +- 19 files changed, 302 insertions(+), 31 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java index 3d60badb0d627..ce64cdb50fb3f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java @@ -27,6 +27,7 @@ public class SettingsConfig implements ToXContentObject { private static final ParseField DATES_AS_EPOCH_MILLIS = new ParseField("dates_as_epoch_millis"); private static final ParseField ALIGN_CHECKPOINTS = new ParseField("align_checkpoints"); private static final ParseField USE_PIT = new ParseField("use_point_in_time"); + private static final ParseField DEDUCE_MAPPINGS = new ParseField("deduce_mappings"); private static final int DEFAULT_MAX_PAGE_SEARCH_SIZE = -1; private static final float DEFAULT_DOCS_PER_SECOND = -1F; @@ -39,16 +40,27 @@ public class SettingsConfig implements ToXContentObject { // use an integer as we need to code 4 states: true, false, null (unchanged), default (defined server side) private static final int DEFAULT_USE_PIT = -1; + // use an integer as we need to code 4 states: true, false, null (unchanged), default (defined server side) + private static final int DEFAULT_DEDUCE_MAPPINGS = -1; + private final Integer maxPageSearchSize; private final Float docsPerSecond; private final Integer datesAsEpochMillis; private final Integer alignCheckpoints; private final Integer usePit; + private final Integer deduceMappings; private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "settings_config", true, - args -> new SettingsConfig((Integer) args[0], (Float) args[1], (Integer) args[2], (Integer) args[3], (Integer) args[4]) + args -> new SettingsConfig( + (Integer) args[0], + (Float) args[1], + (Integer) args[2], + (Integer) args[3], + (Integer) args[4], + (Integer) args[5] + ) ); static { @@ -75,18 +87,33 @@ public class SettingsConfig implements ToXContentObject { USE_PIT, ValueType.BOOLEAN_OR_NULL ); + // this boolean requires 4 possible values: true, false, not_specified, default, therefore using a custom parser + PARSER.declareField( + optionalConstructorArg(), + p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_DEDUCE_MAPPINGS : p.booleanValue() ? 1 : 0, + DEDUCE_MAPPINGS, + ValueType.BOOLEAN_OR_NULL + ); } public static SettingsConfig fromXContent(final XContentParser parser) { return PARSER.apply(parser, null); } - SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Integer datesAsEpochMillis, Integer alignCheckpoints, Integer usePit) { + SettingsConfig( + Integer maxPageSearchSize, + Float docsPerSecond, + Integer datesAsEpochMillis, + Integer alignCheckpoints, + Integer usePit, + Integer deduceMappings + ) { this.maxPageSearchSize = maxPageSearchSize; this.docsPerSecond = docsPerSecond; this.datesAsEpochMillis = datesAsEpochMillis; this.alignCheckpoints = alignCheckpoints; this.usePit = usePit; + this.deduceMappings = deduceMappings; } @Override @@ -127,6 +154,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(USE_PIT.getPreferredName(), usePit > 0 ? true : false); } } + if (deduceMappings != null) { + if (deduceMappings.equals(DEFAULT_DEDUCE_MAPPINGS)) { + builder.field(DEDUCE_MAPPINGS.getPreferredName(), (Boolean) null); + } else { + builder.field(DEDUCE_MAPPINGS.getPreferredName(), deduceMappings > 0 ? true : false); + } + } builder.endObject(); return builder; } @@ -151,6 +185,10 @@ public Boolean getUsePit() { return usePit != null ? usePit > 0 : null; } + public Boolean getDeduceMappings() { + return deduceMappings != null ? deduceMappings > 0 : null; + } + @Override public boolean equals(Object other) { if (other == this) { @@ -165,12 +203,13 @@ public boolean equals(Object other) { && Objects.equals(docsPerSecond, that.docsPerSecond) && Objects.equals(datesAsEpochMillis, that.datesAsEpochMillis) && Objects.equals(alignCheckpoints, that.alignCheckpoints) - && Objects.equals(usePit, that.usePit); + && Objects.equals(usePit, that.usePit) + && Objects.equals(deduceMappings, that.deduceMappings); } @Override public int hashCode() { - return Objects.hash(maxPageSearchSize, docsPerSecond, datesAsEpochMillis, alignCheckpoints, usePit); + return Objects.hash(maxPageSearchSize, docsPerSecond, datesAsEpochMillis, alignCheckpoints, usePit, deduceMappings); } public static Builder builder() { @@ -183,6 +222,7 @@ public static class Builder { private Integer datesAsEpochMillis; private Integer alignCheckpoints; private Integer usePit; + private Integer deduceMappings; /** * Sets the paging maximum paging maxPageSearchSize that transform can use when @@ -256,8 +296,22 @@ public Builder setUsePit(Boolean usePit) { return this; } + /** + * Whether the destination index mappings should be deduced from the transform config. + * It is used per default. + * + * An explicit `null` resets to default. + * + * @param deduceMappings true if the transform should try deducing mappings from the config. + * @return the {@link Builder} with deduceMappings set. + */ + public Builder setDeduceMappings(Boolean deduceMappings) { + this.deduceMappings = deduceMappings == null ? DEFAULT_DEDUCE_MAPPINGS : deduceMappings ? 1 : 0; + return this; + } + public SettingsConfig build() { - return new SettingsConfig(maxPageSearchSize, docsPerSecond, datesAsEpochMillis, alignCheckpoints, usePit); + return new SettingsConfig(maxPageSearchSize, docsPerSecond, datesAsEpochMillis, alignCheckpoints, usePit, deduceMappings); } } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java index 4c31be8bb57cc..fc5bb00b3efe3 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java @@ -34,6 +34,7 @@ import org.elasticsearch.client.transform.UpdateTransformRequest; import org.elasticsearch.client.transform.UpdateTransformResponse; import org.elasticsearch.client.transform.transforms.DestConfig; +import org.elasticsearch.client.transform.transforms.SettingsConfig; import org.elasticsearch.client.transform.transforms.SourceConfig; import org.elasticsearch.client.transform.transforms.TimeSyncConfig; import org.elasticsearch.client.transform.transforms.TransformConfig; @@ -66,6 +67,7 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -353,6 +355,30 @@ public void testPreview() throws IOException { assertExpectedPreview(preview); } + @SuppressWarnings("unchecked") + public void testPreviewWithoutMappingDeduction() throws IOException { + String sourceIndex = "transform-source"; + createIndex(sourceIndex); + indexData(sourceIndex); + + TransformConfig transform = validTransformConfigBuilder("test-preview", sourceIndex, null).setSettings( + SettingsConfig.builder().setDeduceMappings(false).build() + ).build(); + + TransformClient client = highLevelClient().transform(); + PreviewTransformResponse preview = execute( + new PreviewTransformRequest(transform), + client::previewTransform, + client::previewTransformAsync + ); + + assertExpectedPreviewDocs(preview.getDocs()); + + assertThat(preview.getMappings(), hasKey("properties")); + Map fields = (Map) preview.getMappings().get("properties"); + assertThat(fields, anEmptyMap()); + } + public void testPreviewById() throws IOException { String sourceIndex = "transform-source"; createIndex(sourceIndex); @@ -371,9 +397,13 @@ public void testPreviewById() throws IOException { assertExpectedPreview(preview); } - @SuppressWarnings("unchecked") private static void assertExpectedPreview(PreviewTransformResponse preview) { - List> docs = preview.getDocs(); + assertExpectedPreviewDocs(preview.getDocs()); + assertExpectedPreviewMappings(preview.getMappings()); + } + + @SuppressWarnings("unchecked") + private static void assertExpectedPreviewDocs(List> docs) { assertThat(docs, hasSize(2)); Optional> theresa = docs.stream().filter(doc -> "theresa".equals(doc.get("reviewer"))).findFirst(); assertTrue(theresa.isPresent()); @@ -382,8 +412,10 @@ private static void assertExpectedPreview(PreviewTransformResponse preview) { Optional> michel = docs.stream().filter(doc -> "michel".equals(doc.get("reviewer"))).findFirst(); assertTrue(michel.isPresent()); assertEquals(3.6d, (double) michel.get().get("avg_rating"), 0.1d); + } - Map mappings = preview.getMappings(); + @SuppressWarnings("unchecked") + private static void assertExpectedPreviewMappings(Map mappings) { assertThat(mappings, hasKey("properties")); Map fields = (Map) mappings.get("properties"); assertThat(fields.get("reviewer"), equalTo(Map.of("type", "keyword"))); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java index ef3389ba9e5db..07553178b22eb 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java @@ -32,6 +32,7 @@ public static SettingsConfig randomSettingsConfig() { randomBoolean() ? null : randomFloat(), randomBoolean() ? null : randomIntBetween(-1, 1), randomBoolean() ? null : randomIntBetween(-1, 1), + randomBoolean() ? null : randomIntBetween(-1, 1), randomBoolean() ? null : randomIntBetween(-1, 1) ); } @@ -76,6 +77,7 @@ public void testExplicitNullOnWriteParser() throws IOException { assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set")); assertThat(settingsAsMap.getOrDefault("align_checkpoints", "not_set"), equalTo("not_set")); assertThat(settingsAsMap.getOrDefault("use_point_in_time", "not_set"), equalTo("not_set")); + assertThat(settingsAsMap.getOrDefault("deduce_mappings", "not_set"), equalTo("not_set")); config = fromString("{\"dates_as_epoch_millis\" : null}"); assertFalse(config.getDatesAsEpochMillis()); @@ -86,6 +88,7 @@ public void testExplicitNullOnWriteParser() throws IOException { assertNull(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set")); assertThat(settingsAsMap.getOrDefault("align_checkpoints", "not_set"), equalTo("not_set")); assertThat(settingsAsMap.getOrDefault("use_point_in_time", "not_set"), equalTo("not_set")); + assertThat(settingsAsMap.getOrDefault("deduce_mappings", "not_set"), equalTo("not_set")); config = fromString("{\"align_checkpoints\" : null}"); assertFalse(config.getAlignCheckpoints()); @@ -96,6 +99,7 @@ public void testExplicitNullOnWriteParser() throws IOException { assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set")); assertNull(settingsAsMap.getOrDefault("align_checkpoints", "not_set")); assertThat(settingsAsMap.getOrDefault("use_point_in_time", "not_set"), equalTo("not_set")); + assertThat(settingsAsMap.getOrDefault("deduce_mappings", "not_set"), equalTo("not_set")); config = fromString("{\"use_point_in_time\" : null}"); assertFalse(config.getUsePit()); @@ -112,6 +116,7 @@ public void testExplicitNullOnWriteBuilder() throws IOException { assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set")); assertThat(settingsAsMap.getOrDefault("align_checkpoints", "not_set"), equalTo("not_set")); assertThat(settingsAsMap.getOrDefault("use_point_in_time", "not_set"), equalTo("not_set")); + assertThat(settingsAsMap.getOrDefault("deduce_mappings", "not_set"), equalTo("not_set")); SettingsConfig emptyConfig = new SettingsConfig.Builder().build(); assertNull(emptyConfig.getMaxPageSearchSize()); @@ -130,6 +135,7 @@ public void testExplicitNullOnWriteBuilder() throws IOException { assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set")); assertThat(settingsAsMap.getOrDefault("align_checkpoints", "not_set"), equalTo("not_set")); assertThat(settingsAsMap.getOrDefault("use_point_in_time", "not_set"), equalTo("not_set")); + assertThat(settingsAsMap.getOrDefault("deduce_mappings", "not_set"), equalTo("not_set")); config = new SettingsConfig.Builder().setDatesAsEpochMillis(null).build(); // returns false, however it's `null` as in "use default", checked next @@ -140,6 +146,7 @@ public void testExplicitNullOnWriteBuilder() throws IOException { assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set")); assertNull(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set")); assertThat(settingsAsMap.getOrDefault("align_checkpoints", "not_set"), equalTo("not_set")); + assertThat(settingsAsMap.getOrDefault("deduce_mappings", "not_set"), equalTo("not_set")); config = new SettingsConfig.Builder().setAlignCheckpoints(null).build(); // returns false, however it's `null` as in "use default", checked next @@ -150,6 +157,7 @@ public void testExplicitNullOnWriteBuilder() throws IOException { assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set")); assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set")); assertNull(settingsAsMap.getOrDefault("align_checkpoints", "not_set")); + assertThat(settingsAsMap.getOrDefault("deduce_mappings", "not_set"), equalTo("not_set")); } private Map xContentToMap(ToXContent xcontent) throws IOException { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/SettingsConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/SettingsConfigTests.java index bb1ab2fffba2f..ddbf0816def78 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/SettingsConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/SettingsConfigTests.java @@ -25,6 +25,7 @@ public static org.elasticsearch.xpack.core.transform.transforms.SettingsConfig r randomBoolean() ? null : randomFloat(), randomBoolean() ? null : randomIntBetween(0, 1), randomBoolean() ? null : randomIntBetween(0, 1), + randomBoolean() ? null : randomIntBetween(0, 1), randomBoolean() ? null : randomIntBetween(0, 1) ); } @@ -38,6 +39,7 @@ public static void assertHlrcEquals( assertEquals(serverTestInstance.getDatesAsEpochMillis(), clientInstance.getDatesAsEpochMillis()); assertEquals(serverTestInstance.getAlignCheckpoints(), clientInstance.getAlignCheckpoints()); assertEquals(serverTestInstance.getUsePit(), clientInstance.getUsePit()); + assertEquals(serverTestInstance.getDeduceMappings(), clientInstance.getDeduceMappings()); } @Override diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 8571cf31230e3..323aaadb2f76b 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -1034,6 +1034,11 @@ destination index will be performed thus improving overall performance. The default value is `true`, which means the checkpoint ranges will be optimized if possible. end::transform-settings-align-checkpoints[] +tag::transform-settings-deduce-mappings[] +Specifies whether the transform should deduce the destination index mappings from the transform config. +The default value is `true`, which means the destination index mappings will be deduced if possible. +end::transform-settings-deduce-mappings[] + tag::transform-settings-max-page-search-size[] Defines the initial page size to use for the composite aggregation for each checkpoint. If circuit breaker exceptions occur, the page size is dynamically diff --git a/docs/reference/transform/apis/preview-transform.asciidoc b/docs/reference/transform/apis/preview-transform.asciidoc index 55be298997cbc..241143877c674 100644 --- a/docs/reference/transform/apis/preview-transform.asciidoc +++ b/docs/reference/transform/apis/preview-transform.asciidoc @@ -227,12 +227,18 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings] .Properties of `settings` [%collapsible%open] ==== +`dates_as_epoch_millis`::: +(Optional, boolean) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli] `docs_per_second`::: (Optional, float) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second] `align_checkpoints`::: (Optional, boolean) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-align-checkpoints] +`deduce_mappings`::: +(Optional, boolean) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-deduce-mappings] `max_page_search_size`::: (Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] diff --git a/docs/reference/transform/apis/put-transform.asciidoc b/docs/reference/transform/apis/put-transform.asciidoc index 98d43b805afa5..6cd40a341f703 100644 --- a/docs/reference/transform/apis/put-transform.asciidoc +++ b/docs/reference/transform/apis/put-transform.asciidoc @@ -201,6 +201,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-doc `align_checkpoints`::: (Optional, boolean) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-align-checkpoints] +`deduce_mappings`::: +(Optional, boolean) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-deduce-mappings] `max_page_search_size`::: (Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] diff --git a/docs/reference/transform/apis/update-transform.asciidoc b/docs/reference/transform/apis/update-transform.asciidoc index c2fb81f60bddb..22c52e3083b60 100644 --- a/docs/reference/transform/apis/update-transform.asciidoc +++ b/docs/reference/transform/apis/update-transform.asciidoc @@ -151,6 +151,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-doc `align_checkpoints`::: (Optional, boolean) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-align-checkpoints] +`deduce_mappings`::: +(Optional, boolean) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-deduce-mappings] `max_page_search_size`::: (Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java index 9b745f60e9025..a2baae6c32df8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java @@ -40,6 +40,7 @@ public final class TransformField { public static final ParseField DATES_AS_EPOCH_MILLIS = new ParseField("dates_as_epoch_millis"); public static final ParseField ALIGN_CHECKPOINTS = new ParseField("align_checkpoints"); public static final ParseField USE_PIT = new ParseField("use_point_in_time"); + public static final ParseField DEDUCE_MAPPINGS = new ParseField("deduce_mappings"); public static final ParseField FIELD = new ParseField("field"); public static final ParseField SYNC = new ParseField("sync"); public static final ParseField TIME = new ParseField("time"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java index ec57b86e4190e..fb6b921faca43 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java @@ -39,12 +39,20 @@ public class SettingsConfig implements Writeable, ToXContentObject { private static final int DEFAULT_DATES_AS_EPOCH_MILLIS = -1; private static final int DEFAULT_ALIGN_CHECKPOINTS = -1; private static final int DEFAULT_USE_PIT = -1; + private static final int DEFAULT_DEDUCE_MAPPINGS = -1; private static ConstructingObjectParser createParser(boolean lenient) { ConstructingObjectParser parser = new ConstructingObjectParser<>( "transform_config_settings", lenient, - args -> new SettingsConfig((Integer) args[0], (Float) args[1], (Integer) args[2], (Integer) args[3], (Integer) args[4]) + args -> new SettingsConfig( + (Integer) args[0], + (Float) args[1], + (Integer) args[2], + (Integer) args[3], + (Integer) args[4], + (Integer) args[5] + ) ); parser.declareIntOrNull(optionalConstructorArg(), DEFAULT_MAX_PAGE_SEARCH_SIZE, TransformField.MAX_PAGE_SEARCH_SIZE); parser.declareFloatOrNull(optionalConstructorArg(), DEFAULT_DOCS_PER_SECOND, TransformField.DOCS_PER_SECOND); @@ -69,6 +77,13 @@ private static ConstructingObjectParser createParser(boole TransformField.USE_PIT, ValueType.BOOLEAN_OR_NULL ); + // this boolean requires 4 possible values: true, false, not_specified, default, therefore using a custom parser + parser.declareField( + optionalConstructorArg(), + p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_DEDUCE_MAPPINGS : p.booleanValue() ? 1 : 0, + TransformField.DEDUCE_MAPPINGS, + ValueType.BOOLEAN_OR_NULL + ); return parser; } @@ -77,9 +92,10 @@ private static ConstructingObjectParser createParser(boole private final Integer datesAsEpochMillis; private final Integer alignCheckpoints; private final Integer usePit; + private final Integer deduceMappings; public SettingsConfig() { - this(null, null, (Integer) null, (Integer) null, (Integer) null); + this(null, null, (Integer) null, (Integer) null, (Integer) null, (Integer) null); } public SettingsConfig( @@ -87,14 +103,16 @@ public SettingsConfig( Float docsPerSecond, Boolean datesAsEpochMillis, Boolean alignCheckpoints, - Boolean usePit + Boolean usePit, + Boolean deduceMappings ) { this( maxPageSearchSize, docsPerSecond, datesAsEpochMillis == null ? null : datesAsEpochMillis ? 1 : 0, alignCheckpoints == null ? null : alignCheckpoints ? 1 : 0, - usePit == null ? null : usePit ? 1 : 0 + usePit == null ? null : usePit ? 1 : 0, + deduceMappings == null ? null : deduceMappings ? 1 : 0 ); } @@ -103,13 +121,15 @@ public SettingsConfig( Float docsPerSecond, Integer datesAsEpochMillis, Integer alignCheckpoints, - Integer usePit + Integer usePit, + Integer deduceMappings ) { this.maxPageSearchSize = maxPageSearchSize; this.docsPerSecond = docsPerSecond; this.datesAsEpochMillis = datesAsEpochMillis; this.alignCheckpoints = alignCheckpoints; this.usePit = usePit; + this.deduceMappings = deduceMappings; } public SettingsConfig(final StreamInput in) throws IOException { @@ -130,6 +150,11 @@ public SettingsConfig(final StreamInput in) throws IOException { } else { this.usePit = DEFAULT_USE_PIT; } + if (in.getVersion().onOrAfter(Version.V_8_1_0)) { + deduceMappings = in.readOptionalInt(); + } else { + deduceMappings = DEFAULT_DEDUCE_MAPPINGS; + } } public Integer getMaxPageSearchSize() { @@ -164,6 +189,14 @@ public Integer getUsePitForUpdate() { return usePit; } + public Boolean getDeduceMappings() { + return deduceMappings != null ? (deduceMappings > 0) || (deduceMappings == DEFAULT_DEDUCE_MAPPINGS) : null; + } + + public Integer getDeduceMappingsForUpdate() { + return deduceMappings; + } + public ActionRequestValidationException validate(ActionRequestValidationException validationException) { if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > MultiBucketConsumerService.DEFAULT_MAX_BUCKETS)) { validationException = addValidationError( @@ -193,6 +226,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_16_1)) { out.writeOptionalInt(usePit); } + if (out.getVersion().onOrAfter(Version.V_8_1_0)) { + out.writeOptionalInt(deduceMappings); + } } @Override @@ -214,6 +250,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (usePit != null && (usePit.equals(DEFAULT_USE_PIT) == false)) { builder.field(TransformField.USE_PIT.getPreferredName(), usePit > 0 ? true : false); } + if (deduceMappings != null && (deduceMappings.equals(DEFAULT_DEDUCE_MAPPINGS) == false)) { + builder.field(TransformField.DEDUCE_MAPPINGS.getPreferredName(), deduceMappings > 0 ? true : false); + } builder.endObject(); return builder; } @@ -232,12 +271,13 @@ public boolean equals(Object other) { && Objects.equals(docsPerSecond, that.docsPerSecond) && Objects.equals(datesAsEpochMillis, that.datesAsEpochMillis) && Objects.equals(alignCheckpoints, that.alignCheckpoints) - && Objects.equals(usePit, that.usePit); + && Objects.equals(usePit, that.usePit) + && Objects.equals(deduceMappings, that.deduceMappings); } @Override public int hashCode() { - return Objects.hash(maxPageSearchSize, docsPerSecond, datesAsEpochMillis, alignCheckpoints, usePit); + return Objects.hash(maxPageSearchSize, docsPerSecond, datesAsEpochMillis, alignCheckpoints, usePit, deduceMappings); } @Override @@ -255,6 +295,7 @@ public static class Builder { private Integer datesAsEpochMillis; private Integer alignCheckpoints; private Integer usePit; + private Integer deduceMappings; /** * Default builder @@ -272,6 +313,7 @@ public Builder(SettingsConfig base) { this.datesAsEpochMillis = base.datesAsEpochMillis; this.alignCheckpoints = base.alignCheckpoints; this.usePit = base.usePit; + this.deduceMappings = base.deduceMappings; } /** @@ -346,6 +388,20 @@ public Builder setUsePit(Boolean usePit) { return this; } + /** + * Whether the destination index mappings should be deduced from the transform config. + * It is used per default. + * + * An explicit `null` resets to default. + * + * @param deduceMappings true if the transform should try deducing mappings from the config. + * @return the {@link Builder} with deduceMappings set. + */ + public Builder setDeduceMappings(Boolean deduceMappings) { + this.deduceMappings = deduceMappings == null ? DEFAULT_DEDUCE_MAPPINGS : deduceMappings ? 1 : 0; + return this; + } + /** * Update settings according to given settings config. * @@ -376,12 +432,17 @@ public Builder update(SettingsConfig update) { if (update.getUsePitForUpdate() != null) { this.usePit = update.getUsePitForUpdate().equals(DEFAULT_USE_PIT) ? null : update.getUsePitForUpdate(); } + if (update.getDeduceMappingsForUpdate() != null) { + this.deduceMappings = update.getDeduceMappingsForUpdate().equals(DEFAULT_DEDUCE_MAPPINGS) + ? null + : update.getDeduceMappingsForUpdate(); + } return this; } public SettingsConfig build() { - return new SettingsConfig(maxPageSearchSize, docsPerSecond, datesAsEpochMillis, alignCheckpoints, usePit); + return new SettingsConfig(maxPageSearchSize, docsPerSecond, datesAsEpochMillis, alignCheckpoints, usePit, deduceMappings); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index 3526875c40612..d57853a1d4ad8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -622,7 +622,8 @@ private static TransformConfig applyRewriteForUpdate(Builder builder) { builder.getSettings().getDocsPerSecond(), builder.getSettings().getDatesAsEpochMillis(), builder.getSettings().getAlignCheckpoints(), - builder.getSettings().getUsePit() + builder.getSettings().getUsePit(), + builder.getSettings().getDeduceMappings() ) ); } @@ -635,7 +636,8 @@ private static TransformConfig applyRewriteForUpdate(Builder builder) { builder.getSettings().getDocsPerSecond(), true, builder.getSettings().getAlignCheckpoints(), - builder.getSettings().getUsePit() + builder.getSettings().getUsePit(), + builder.getSettings().getDeduceMappings() ) ); } @@ -648,7 +650,8 @@ private static TransformConfig applyRewriteForUpdate(Builder builder) { builder.getSettings().getDocsPerSecond(), builder.getSettings().getDatesAsEpochMillis(), false, - builder.getSettings().getUsePit() + builder.getSettings().getUsePit(), + builder.getSettings().getDeduceMappings() ) ); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java index f7bc060750a82..f2682a982200d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java @@ -35,6 +35,7 @@ public static SettingsConfig randomSettingsConfig() { randomBoolean() ? null : randomFloat(), randomBoolean() ? null : randomIntBetween(0, 1), randomBoolean() ? null : randomIntBetween(0, 1), + randomBoolean() ? null : randomIntBetween(0, 1), randomBoolean() ? null : randomIntBetween(0, 1) ); } @@ -45,6 +46,7 @@ public static SettingsConfig randomNonEmptySettingsConfig() { randomFloat(), randomIntBetween(0, 1), randomIntBetween(0, 1), + randomIntBetween(0, 1), randomIntBetween(0, 1) ); } @@ -92,6 +94,9 @@ public void testExplicitNullParsing() throws IOException { assertThat(fromString("{\"use_point_in_time\" : null}").getUsePitForUpdate(), equalTo(-1)); assertNull(fromString("{}").getUsePitForUpdate()); + + assertThat(fromString("{\"deduce_mappings\" : null}").getDeduceMappingsForUpdate(), equalTo(-1)); + assertNull(fromString("{}").getDeduceMappingsForUpdate()); } public void testUpdateUsingBuilder() throws IOException { @@ -100,7 +105,8 @@ public void testUpdateUsingBuilder() throws IOException { + "\"docs_per_second\" :42, " + "\"dates_as_epoch_millis\": true, " + "\"align_checkpoints\": false," - + "\"use_point_in_time\": false}" + + "\"use_point_in_time\": false," + + "\"deduce_mappings\": false}" ); SettingsConfig.Builder builder = new SettingsConfig.Builder(config); @@ -111,6 +117,7 @@ public void testUpdateUsingBuilder() throws IOException { assertThat(builder.build().getDatesAsEpochMillisForUpdate(), equalTo(1)); assertThat(builder.build().getAlignCheckpointsForUpdate(), equalTo(0)); assertThat(builder.build().getUsePitForUpdate(), equalTo(0)); + assertThat(builder.build().getDeduceMappingsForUpdate(), equalTo(0)); builder.update(fromString("{\"max_page_search_size\" : null}")); assertNull(builder.build().getMaxPageSearchSize()); @@ -118,6 +125,7 @@ public void testUpdateUsingBuilder() throws IOException { assertThat(builder.build().getDatesAsEpochMillisForUpdate(), equalTo(1)); assertThat(builder.build().getAlignCheckpointsForUpdate(), equalTo(0)); assertThat(builder.build().getUsePitForUpdate(), equalTo(0)); + assertThat(builder.build().getDeduceMappingsForUpdate(), equalTo(0)); builder.update( fromString( @@ -125,7 +133,8 @@ public void testUpdateUsingBuilder() throws IOException { + "\"docs_per_second\" :null, " + "\"dates_as_epoch_millis\": null, " + "\"align_checkpoints\": null," - + "\"use_point_in_time\": null}" + + "\"use_point_in_time\": null," + + "\"deduce_mappings\": null}" ) ); assertThat(builder.build().getMaxPageSearchSize(), equalTo(77)); @@ -133,6 +142,7 @@ public void testUpdateUsingBuilder() throws IOException { assertNull(builder.build().getDatesAsEpochMillisForUpdate()); assertNull(builder.build().getAlignCheckpointsForUpdate()); assertNull(builder.build().getUsePitForUpdate()); + assertNull(builder.build().getDeduceMappingsForUpdate()); } public void testOmmitDefaultsOnWriteParser() throws IOException { @@ -172,6 +182,12 @@ public void testOmmitDefaultsOnWriteParser() throws IOException { settingsAsMap = xContentToMap(config); assertTrue(settingsAsMap.isEmpty()); + + config = fromString("{\"deduce_mappings\" : null}"); + assertThat(config.getDeduceMappingsForUpdate(), equalTo(-1)); + + settingsAsMap = xContentToMap(config); + assertTrue(settingsAsMap.isEmpty()); } public void testOmmitDefaultsOnWriteBuilder() throws IOException { @@ -211,6 +227,12 @@ public void testOmmitDefaultsOnWriteBuilder() throws IOException { settingsAsMap = xContentToMap(config); assertTrue(settingsAsMap.isEmpty()); + + config = new SettingsConfig.Builder().setDeduceMappings(null).build(); + assertThat(config.getDeduceMappingsForUpdate(), equalTo(-1)); + + settingsAsMap = xContentToMap(config); + assertTrue(settingsAsMap.isEmpty()); } private Map xContentToMap(ToXContent xcontent) throws IOException { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java index c7ec82ab727f4..589f73a241ba3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java @@ -113,7 +113,7 @@ public void testApply() { TimeValue frequency = TimeValue.timeValueSeconds(10); SyncConfig syncConfig = new TimeSyncConfig("time_field", TimeValue.timeValueSeconds(30)); String newDescription = "new description"; - SettingsConfig settings = new SettingsConfig(4_000, 4_000.400F, true, true, true); + SettingsConfig settings = new SettingsConfig(4_000, 4_000.400F, true, true, true, true); Map newMetadata = randomMetadata(); RetentionPolicyConfig retentionPolicyConfig = new TimeRetentionPolicyConfig("time_field", new TimeValue(60_000)); update = new TransformConfigUpdate( @@ -168,7 +168,7 @@ public void testApplySettings() { null, null, null, - new SettingsConfig(4_000, null, (Boolean) null, null, null), + new SettingsConfig(4_000, null, (Boolean) null, null, null, null), null, null ); @@ -187,7 +187,7 @@ public void testApplySettings() { null, null, null, - new SettingsConfig(null, 43.244F, (Boolean) null, null, null), + new SettingsConfig(null, 43.244F, (Boolean) null, null, null, null), null, null ); @@ -204,7 +204,7 @@ public void testApplySettings() { null, null, null, - new SettingsConfig(-1, null, (Boolean) null, null, null), + new SettingsConfig(-1, null, (Boolean) null, null, null, null), null, null ); @@ -220,7 +220,7 @@ public void testApplySettings() { null, null, null, - new SettingsConfig(-1, -1F, (Boolean) null, null, null), + new SettingsConfig(-1, -1F, (Boolean) null, null, null, null), null, null ); diff --git a/x-pack/plugin/core/src/test/resources/rest-api-spec/schema/transform_config.schema.json b/x-pack/plugin/core/src/test/resources/rest-api-spec/schema/transform_config.schema.json index 4f6accd936d1c..18f96b4146485 100644 --- a/x-pack/plugin/core/src/test/resources/rest-api-spec/schema/transform_config.schema.json +++ b/x-pack/plugin/core/src/test/resources/rest-api-spec/schema/transform_config.schema.json @@ -197,6 +197,12 @@ "title": "use_point_in_time", "type": "boolean", "default": true + }, + "deduce_mappings": { + "$id": "#root/settings/deduce_mappings", + "title": "deduce mappings", + "type": "boolean", + "default": true } } }, diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/preview_transforms.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/preview_transforms.yml index dddd026c4be33..06ddbb21d89f3 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/preview_transforms.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/preview_transforms.yml @@ -150,6 +150,29 @@ setup: - match: { generated_dest_index.mappings.properties.by-hour.type: "date" } - match: { generated_dest_index.mappings.properties.avg_response.type: "double" } +--- +"Test preview transform with disabled mapping deduction": + - do: + transform.preview_transform: + body: > + { + "source": { "index": "airline-data" }, + "pivot": { + "group_by": { + "airline": {"terms": {"field": "airline"}}, + "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}}, + "aggs": { + "avg_response": {"avg": {"field": "responsetime"}}, + "time.max": {"max": {"field": "time"}}, + "time.min": {"min": {"field": "time"}} + } + }, + "settings": { + "deduce_mappings": false + } + } + - match: { generated_dest_index.mappings.properties: {} } + --- "Test preview transform by id": - do: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_start_stop.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_start_stop.yml index 38268e499246e..de1af5936d2e1 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_start_stop.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_start_stop.yml @@ -28,6 +28,21 @@ setup: "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} } } + - do: + transform.put_transform: + transform_id: "airline-transform-start-stop-no-deduce" + body: > + { + "source": { "index": "airline-data" }, + "dest": { "index": "airline-data-by-airline-start-stop-no-deduce" }, + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + }, + "settings": { + "deduce_mappings": false + } + } - do: transform.put_transform: transform_id: "airline-transform-start-stop-continuous" @@ -58,6 +73,15 @@ teardown: - do: transform.delete_transform: transform_id: "airline-transform-start-stop" + - do: + transform.stop_transform: + wait_for_checkpoint: false + transform_id: "airline-transform-start-stop-no-deduce" + timeout: "10m" + wait_for_completion: true + - do: + transform.delete_transform: + transform_id: "airline-transform-start-stop-no-deduce" - do: transform.stop_transform: wait_for_checkpoint: false @@ -105,6 +129,17 @@ teardown: - match: { airline-data-by-airline-start-stop.mappings.properties.airline.type: keyword } - match: { airline-data-by-airline-start-stop.mappings.properties.avg_response.type: double } +--- +"Verify start transform creates destination index without mapping deduction": + - do: + transform.start_transform: + transform_id: "airline-transform-start-stop-no-deduce" + - match: { acknowledged: true } + - do: + indices.get_mapping: + index: airline-data-by-airline-start-stop-no-deduce + - match: { airline-data-by-airline-start-stop-no-deduce.mappings.properties: null } + --- "Verify start transform reuses destination index": - do: @@ -345,9 +380,11 @@ teardown: - do: transform.get_transform_stats: transform_id: "*" - - match: { count: 3 } + - match: { count: 4 } - match: { transforms.0.state: "stopped" } - match: { transforms.1.state: "stopped" } + - match: { transforms.2.state: "stopped" } + - match: { transforms.3.state: "stopped" } - do: transform.delete_transform: diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java index 8254c3809ff85..55002c09bcdc2 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java @@ -44,6 +44,7 @@ import java.util.Set; import java.util.stream.Stream; +import static java.util.Collections.emptyMap; import static java.util.stream.Collectors.toList; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; @@ -92,6 +93,10 @@ public List getPerformanceCriticalFields() { @Override public void deduceMappings(Client client, SourceConfig sourceConfig, final ActionListener> listener) { + if (Boolean.FALSE.equals(settings.getDeduceMappings())) { + listener.onResponse(emptyMap()); + return; + } SchemaUtil.deduceMappings(client, config, sourceConfig.getIndex(), sourceConfig.getRuntimeMappings(), listener); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java index d14155b50ce71..cabbcc341bd61 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java @@ -309,7 +309,7 @@ public void testPageSizeAdapt() throws Exception { randomPivotConfig(), null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), - new SettingsConfig(pageSize, null, (Boolean) null, null, null), + new SettingsConfig(pageSize, null, (Boolean) null, null, null, null), null, null, null, @@ -384,7 +384,7 @@ public void testDoProcessAggNullCheck() { randomPivotConfig(), null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), - new SettingsConfig(pageSize, null, (Boolean) null, null, null), + new SettingsConfig(pageSize, null, (Boolean) null, null, null, null), null, null, null, @@ -448,7 +448,7 @@ public void testScriptError() throws Exception { randomPivotConfig(), null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), - new SettingsConfig(pageSize, null, (Boolean) null, null, null), + new SettingsConfig(pageSize, null, (Boolean) null, null, null, null), null, null, null, diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java index 3c6232889411f..7d4ee0f082d31 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java @@ -506,7 +506,7 @@ public void testStopAtCheckpointForThrottledTransform() throws Exception { randomPivotConfig(), null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), - new SettingsConfig(null, Float.valueOf(1.0f), (Boolean) null, (Boolean) null, null), + new SettingsConfig(null, Float.valueOf(1.0f), (Boolean) null, (Boolean) null, null, null), null, null, null,