From aee722dd3914bc771a50f20146a81bfcef4ec3a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Fri, 15 Oct 2021 12:14:10 +0200 Subject: [PATCH] [7.x] [Transform] Add _meta field to TransformConfig (#79003) (#79219) --- .../transform/transforms/TransformConfig.java | 36 ++++++-- .../transforms/TransformConfigUpdate.java | 32 +++++-- .../transforms/TransformConfigTests.java | 27 ++++++ .../TransformConfigUpdateTests.java | 2 + docs/reference/rest-api/common-parms.asciidoc | 4 + .../transform/apis/put-transform.asciidoc | 6 ++ .../transform/apis/update-transform.asciidoc | 6 ++ .../xpack/core/transform/TransformField.java | 1 + .../transform/transforms/TransformConfig.java | 52 +++++++++-- .../transforms/TransformConfigUpdate.java | 37 ++++++-- .../PreviewTransformActionRequestTests.java | 1 + .../transforms/TransformConfigTests.java | 87 ++++++++++++++++++- .../TransformConfigUpdateTests.java | 71 ++++++++++++--- .../schema/transform_config.schema.json | 5 ++ .../test/transform/transforms_update.yml | 72 +++++++++++++++ .../integration/TransformProgressIT.java | 1 + .../integration/TransformInternalIndexIT.java | 2 +- .../TransformNoRemoteClusterClientNodeIT.java | 4 +- .../TransformNoTransformNodeIT.java | 4 +- .../TransportUpgradeTransformsAction.java | 2 +- .../persistence/TransformInternalIndex.java | 3 + .../rest/action/RestPutTransformAction.java | 16 ++++ .../action/RestUpdateTransformAction.java | 7 ++ .../TransformIndexerFailureHandlingTests.java | 6 ++ .../TransformIndexerStateTests.java | 2 + .../transforms/TransformIndexerTests.java | 3 + 26 files changed, 445 insertions(+), 44 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java index 159ab5b08200c..8c59c2bac3d4e 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.time.Instant; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; @@ -38,6 +39,7 @@ public class TransformConfig implements ToXContentObject { public static final ParseField DESCRIPTION = new ParseField("description"); public static final ParseField SYNC = new ParseField("sync"); public static final ParseField SETTINGS = new ParseField("settings"); + public static final ParseField METADATA = new ParseField("_meta"); public static final ParseField VERSION = new ParseField("version"); public static final ParseField CREATE_TIME = new ParseField("create_time"); public static final ParseField RETENTION_POLICY = new ParseField("retention_policy"); @@ -51,6 +53,7 @@ public class TransformConfig implements ToXContentObject { private final TimeValue frequency; private final SyncConfig syncConfig; private final SettingsConfig settings; + private final Map metadata; private final PivotConfig pivotConfig; private final LatestConfig latestConfig; private final String description; @@ -71,9 +74,11 @@ public class TransformConfig implements ToXContentObject { LatestConfig latestConfig = (LatestConfig) args[6]; String description = (String) args[7]; SettingsConfig settings = (SettingsConfig) args[8]; - RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[9]; - Instant createTime = (Instant) args[10]; - String transformVersion = (String) args[11]; + @SuppressWarnings("unchecked") + Map metadata = (Map) args[9]; + RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[10]; + Instant createTime = (Instant) args[11]; + String transformVersion = (String) args[12]; return new TransformConfig( id, source, @@ -84,6 +89,7 @@ public class TransformConfig implements ToXContentObject { latestConfig, description, settings, + metadata, retentionPolicyConfig, createTime, transformVersion @@ -106,6 +112,7 @@ public class TransformConfig implements ToXContentObject { PARSER.declareObject(optionalConstructorArg(), (p, c) -> LatestConfig.fromXContent(p), LATEST_TRANSFORM); PARSER.declareString(optionalConstructorArg(), DESCRIPTION); PARSER.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p), SETTINGS); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.mapOrdered(), METADATA); PARSER.declareNamedObject( optionalConstructorArg(), (p, c, n) -> p.namedObject(RetentionPolicyConfig.class, n, c), @@ -136,7 +143,7 @@ public static TransformConfig fromXContent(final XContentParser parser) { * @return A TransformConfig to preview, NOTE it will have a {@code null} id, destination and index. */ public static TransformConfig forPreview(final SourceConfig source, final PivotConfig pivotConfig) { - return new TransformConfig(null, source, null, null, null, pivotConfig, null, null, null, null, null, null); + return new TransformConfig(null, source, null, null, null, pivotConfig, null, null, null, null, null, null, null); } /** @@ -151,7 +158,7 @@ public static TransformConfig forPreview(final SourceConfig source, final PivotC * @return A TransformConfig to preview, NOTE it will have a {@code null} id, destination and index. */ public static TransformConfig forPreview(final SourceConfig source, final LatestConfig latestConfig) { - return new TransformConfig(null, source, null, null, null, null, latestConfig, null, null, null, null, null); + return new TransformConfig(null, source, null, null, null, null, latestConfig, null, null, null, null, null, null); } TransformConfig( @@ -164,6 +171,7 @@ public static TransformConfig forPreview(final SourceConfig source, final Latest final LatestConfig latestConfig, final String description, final SettingsConfig settings, + final Map metadata, final RetentionPolicyConfig retentionPolicyConfig, final Instant createTime, final String version @@ -177,6 +185,7 @@ public static TransformConfig forPreview(final SourceConfig source, final Latest this.latestConfig = latestConfig; this.description = description; this.settings = settings; + this.metadata = metadata; this.retentionPolicyConfig = retentionPolicyConfig; this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli()); this.transformVersion = version == null ? null : Version.fromString(version); @@ -228,6 +237,11 @@ public SettingsConfig getSettings() { return settings; } + @Nullable + public Map getMetadata() { + return metadata; + } + @Nullable public RetentionPolicyConfig getRetentionPolicyConfig() { return retentionPolicyConfig; @@ -265,6 +279,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (settings != null) { builder.field(SETTINGS.getPreferredName(), settings); } + if (metadata != null) { + builder.field(METADATA.getPreferredName(), metadata); + } if (retentionPolicyConfig != null) { builder.startObject(RETENTION_POLICY.getPreferredName()); builder.field(retentionPolicyConfig.getName(), retentionPolicyConfig); @@ -300,6 +317,7 @@ public boolean equals(Object other) { && Objects.equals(this.syncConfig, that.syncConfig) && Objects.equals(this.transformVersion, that.transformVersion) && Objects.equals(this.settings, that.settings) + && Objects.equals(this.metadata, that.metadata) && Objects.equals(this.createTime, that.createTime) && Objects.equals(this.pivotConfig, that.pivotConfig) && Objects.equals(this.latestConfig, that.latestConfig) @@ -315,6 +333,7 @@ public int hashCode() { frequency, syncConfig, settings, + metadata, createTime, transformVersion, pivotConfig, @@ -343,6 +362,7 @@ public static class Builder { private PivotConfig pivotConfig; private LatestConfig latestConfig; private SettingsConfig settings; + private Map metadata; private String description; private RetentionPolicyConfig retentionPolicyConfig; @@ -391,6 +411,11 @@ public Builder setSettings(SettingsConfig settings) { return this; } + public Builder setMetadata(Map metadata) { + this.metadata = metadata; + return this; + } + public Builder setRetentionPolicyConfig(RetentionPolicyConfig retentionPolicyConfig) { this.retentionPolicyConfig = retentionPolicyConfig; return this; @@ -407,6 +432,7 @@ public TransformConfig build() { latestConfig, description, settings, + metadata, retentionPolicyConfig, null, null diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfigUpdate.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfigUpdate.java index b07c09cc3dc78..a706e35ae93b2 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfigUpdate.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfigUpdate.java @@ -17,6 +17,7 @@ import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; @@ -39,8 +40,10 @@ public class TransformConfigUpdate implements ToXContentObject { SyncConfig syncConfig = (SyncConfig) args[3]; String description = (String) args[4]; SettingsConfig settings = (SettingsConfig) args[5]; - RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[6]; - return new TransformConfigUpdate(source, dest, frequency, syncConfig, description, settings, retentionPolicyConfig); + @SuppressWarnings("unchecked") + Map metadata = (Map) args[6]; + RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[7]; + return new TransformConfigUpdate(source, dest, frequency, syncConfig, description, settings, metadata, retentionPolicyConfig); } ); @@ -51,6 +54,7 @@ public class TransformConfigUpdate implements ToXContentObject { PARSER.declareNamedObject(optionalConstructorArg(), (p, c, n) -> p.namedObject(SyncConfig.class, n, c), TransformConfig.SYNC); PARSER.declareString(optionalConstructorArg(), TransformConfig.DESCRIPTION); PARSER.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p), TransformConfig.SETTINGS); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.mapOrdered(), TransformConfig.METADATA); PARSER.declareNamedObject( optionalConstructorArg(), (p, c, n) -> p.namedObject(RetentionPolicyConfig.class, n, c), @@ -64,6 +68,7 @@ public class TransformConfigUpdate implements ToXContentObject { private final SyncConfig syncConfig; private final String description; private final SettingsConfig settings; + private final Map metadata; public TransformConfigUpdate( final SourceConfig source, @@ -72,6 +77,7 @@ public TransformConfigUpdate( final SyncConfig syncConfig, final String description, final SettingsConfig settings, + final Map metadata, final RetentionPolicyConfig retentionPolicyConfig ) { this.source = source; @@ -80,6 +86,7 @@ public TransformConfigUpdate( this.syncConfig = syncConfig; this.description = description; this.settings = settings; + this.metadata = metadata; } public SourceConfig getSource() { @@ -108,6 +115,11 @@ public SettingsConfig getSettings() { return settings; } + @Nullable + public Map getMetadata() { + return metadata; + } + @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { builder.startObject(); @@ -131,6 +143,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (settings != null) { builder.field(TransformConfig.SETTINGS.getPreferredName(), settings); } + if (metadata != null) { + builder.field(TransformConfig.METADATA.getPreferredName(), metadata); + } builder.endObject(); return builder; @@ -153,12 +168,13 @@ public boolean equals(Object other) { && Objects.equals(this.frequency, that.frequency) && Objects.equals(this.syncConfig, that.syncConfig) && Objects.equals(this.description, that.description) - && Objects.equals(this.settings, that.settings); + && Objects.equals(this.settings, that.settings) + && Objects.equals(this.metadata, that.metadata); } @Override public int hashCode() { - return Objects.hash(source, dest, frequency, syncConfig, description, settings); + return Objects.hash(source, dest, frequency, syncConfig, description, settings, metadata); } @Override @@ -182,6 +198,7 @@ public static class Builder { private SyncConfig syncConfig; private String description; private SettingsConfig settings; + private Map metdata; private RetentionPolicyConfig retentionPolicyConfig; public Builder setSource(SourceConfig source) { @@ -214,13 +231,18 @@ public Builder setSettings(SettingsConfig settings) { return this; } + public Builder setMetadata(Map metadata) { + this.metdata = metdata; + return this; + } + public Builder setRetentionPolicyConfig(RetentionPolicyConfig retentionPolicyConfig) { this.retentionPolicyConfig = retentionPolicyConfig; return this; } public TransformConfigUpdate build() { - return new TransformConfigUpdate(source, dest, frequency, syncConfig, description, settings, retentionPolicyConfig); + return new TransformConfigUpdate(source, dest, frequency, syncConfig, description, settings, metdata, retentionPolicyConfig); } } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java index 8b61748b9b272..92a3965207944 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.client.transform.transforms.pivot.PivotConfigTests; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.search.SearchModule; @@ -25,6 +26,7 @@ import java.time.Instant; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.function.Predicate; import static org.elasticsearch.client.transform.transforms.DestConfigTests.randomDestConfig; @@ -52,6 +54,7 @@ public static TransformConfig randomTransformConfig() { latestConfig, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100), SettingsConfigTests.randomSettingsConfig(), + randomMetadata(), randomBoolean() ? null : randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.CURRENT.toString() @@ -66,6 +69,30 @@ public static RetentionPolicyConfig randomRetentionPolicyConfig() { return TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig(); } + public static Map randomMetadata() { + return randomMap(0, 10, () -> { + String key = randomAlphaOfLengthBetween(1, 10); + Object value; + switch (randomIntBetween(0, 3)) { + case 0: + value = null; + break; + case 1: + value = randomLong(); + break; + case 2: + value = randomAlphaOfLengthBetween(1, 10); + break; + case 3: + value = randomMap(0, 10, () -> Tuple.tuple(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10))); + break; + default: + throw new AssertionError(); + } + return Tuple.tuple(key, value); + }); + } + @Override protected TransformConfig createTestInstance() { return randomTransformConfig(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigUpdateTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigUpdateTests.java index 23614ab7d03f0..9b2b435391326 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigUpdateTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigUpdateTests.java @@ -23,6 +23,7 @@ import static org.elasticsearch.client.transform.transforms.DestConfigTests.randomDestConfig; import static org.elasticsearch.client.transform.transforms.SettingsConfigTests.randomSettingsConfig; import static org.elasticsearch.client.transform.transforms.SourceConfigTests.randomSourceConfig; +import static org.elasticsearch.client.transform.transforms.TransformConfigTests.randomMetadata; import static org.elasticsearch.client.transform.transforms.TransformConfigTests.randomRetentionPolicyConfig; import static org.elasticsearch.client.transform.transforms.TransformConfigTests.randomSyncConfig; @@ -36,6 +37,7 @@ public static TransformConfigUpdate randomTransformConfigUpdate() { randomBoolean() ? null : randomSyncConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), randomBoolean() ? null : randomSettingsConfig(), + randomBoolean() ? null : randomMetadata(), randomBoolean() ? null : randomRetentionPolicyConfig() ); } diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index d5c9d193ff1d0..19c020af00faf 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -984,6 +984,10 @@ The `latest` method transforms the data by finding the latest document for each unique key. end::transform-latest[] +tag::transform-metadata[] +Defines optional {transform} metadata. +end::transform-metadata[] + tag::transform-retention[] Defines a retention policy for the {transform}. Data that meets the defined criteria is deleted from the destination index. diff --git a/docs/reference/transform/apis/put-transform.asciidoc b/docs/reference/transform/apis/put-transform.asciidoc index d60fdff28d07e..3b69cc1996d5d 100644 --- a/docs/reference/transform/apis/put-transform.asciidoc +++ b/docs/reference/transform/apis/put-transform.asciidoc @@ -126,6 +126,12 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-unique-key] ==== //End latest +//Begin _meta +`_meta`:: +(Optional, object) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-metadata] +//End _meta + //Begin pivot `pivot`:: (Required^*^, object) diff --git a/docs/reference/transform/apis/update-transform.asciidoc b/docs/reference/transform/apis/update-transform.asciidoc index c601bd31f47fb..bfc514fdb10ec 100644 --- a/docs/reference/transform/apis/update-transform.asciidoc +++ b/docs/reference/transform/apis/update-transform.asciidoc @@ -98,6 +98,12 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-pipeline] (Optional, <>) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=frequency] +//Begin _meta +`_meta`:: +(Optional, object) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-metadata] +//End _meta + //Begin retention policy `retention_policy`:: (Optional, object) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java index c1ca4a534d4cf..76527e69ee5c8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java @@ -32,6 +32,7 @@ public final class TransformField { public static final ParseField CREATE_TIME = new ParseField("create_time"); public static final ParseField DESTINATION = new ParseField("dest"); public static final ParseField SETTINGS = new ParseField("settings"); + public static final ParseField METADATA = new ParseField("_meta"); public static final ParseField FREQUENCY = new ParseField("frequency"); public static final ParseField FORCE = new ParseField("force"); public static final ParseField MAX_PAGE_SEARCH_SIZE = new ParseField("max_page_search_size"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index a22b3893ddfc6..9bc3eeefcbd3c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -84,6 +84,7 @@ public ParseField getParseField() { private final TimeValue frequency; private final SyncConfig syncConfig; private final SettingsConfig settings; + private final Map metadata; private final RetentionPolicyConfig retentionPolicyConfig; private final String description; // headers store the user context from the creating user, which allows us to run the transform as this user @@ -127,8 +128,8 @@ private static ConstructingObjectParser createParser(bo if (lenient == false) { // on strict parsing do not allow injection of headers, transform version, or create time validateStrictParsingParams(args[6], HEADERS.getPreferredName()); - validateStrictParsingParams(args[12], TransformField.CREATE_TIME.getPreferredName()); - validateStrictParsingParams(args[13], TransformField.VERSION.getPreferredName()); + validateStrictParsingParams(args[13], TransformField.CREATE_TIME.getPreferredName()); + validateStrictParsingParams(args[14], TransformField.VERSION.getPreferredName()); // exactly one function must be defined if ((args[7] == null) == (args[8] == null)) { throw new IllegalArgumentException(TransformMessages.TRANSFORM_CONFIGURATION_BAD_FUNCTION_COUNT); @@ -143,7 +144,13 @@ private static ConstructingObjectParser createParser(bo LatestConfig latestConfig = (LatestConfig) args[8]; String description = (String) args[9]; SettingsConfig settings = (SettingsConfig) args[10]; - RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[11]; + + @SuppressWarnings("unchecked") + Map metadata = (Map) args[11]; + + RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[12]; + Instant createTime = (Instant) args[13]; + String version = (String) args[14]; return new TransformConfig( id, @@ -156,9 +163,10 @@ private static ConstructingObjectParser createParser(bo latestConfig, description, settings, + metadata, retentionPolicyConfig, - (Instant) args[12], - (String) args[13] + createTime, + version ); }); @@ -173,6 +181,7 @@ private static ConstructingObjectParser createParser(bo parser.declareObject(optionalConstructorArg(), (p, c) -> LatestConfig.fromXContent(p, lenient), Function.LATEST.getParseField()); parser.declareString(optionalConstructorArg(), TransformField.DESCRIPTION); parser.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p, lenient), TransformField.SETTINGS); + parser.declareObject(optionalConstructorArg(), (p, c) -> p.mapOrdered(), TransformField.METADATA); parser.declareNamedObject( optionalConstructorArg(), (p, c, n) -> p.namedObject(RetentionPolicyConfig.class, n, c), @@ -203,6 +212,7 @@ public TransformConfig( final LatestConfig latestConfig, final String description, final SettingsConfig settings, + final Map metadata, final RetentionPolicyConfig retentionPolicyConfig, final Instant createTime, final String version @@ -217,6 +227,7 @@ public TransformConfig( this.latestConfig = latestConfig; this.description = description; this.settings = settings == null ? new SettingsConfig() : settings; + this.metadata = metadata; this.retentionPolicyConfig = retentionPolicyConfig; if (this.description != null && this.description.length() > MAX_DESCRIPTION_LENGTH) { throw new IllegalArgumentException("[description] must be less than 1000 characters in length."); @@ -256,6 +267,11 @@ public TransformConfig(final StreamInput in) throws IOException { } else { settings = new SettingsConfig(); } + if (in.getVersion().onOrAfter(Version.V_7_16_0)) { + metadata = in.readMap(); + } else { + metadata = null; + } if (in.getVersion().onOrAfter(Version.V_7_12_0)) { retentionPolicyConfig = in.readOptionalNamedWriteable(RetentionPolicyConfig.class); } else { @@ -328,6 +344,10 @@ public SettingsConfig getSettings() { return settings; } + public Map getMetadata() { + return metadata; + } + @Nullable public RetentionPolicyConfig getRetentionPolicyConfig() { return retentionPolicyConfig; @@ -432,6 +452,9 @@ public void writeTo(final StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_8_0)) { settings.writeTo(out); } + if (out.getVersion().onOrAfter(Version.V_7_16_0)) { + out.writeMap(metadata); + } if (out.getVersion().onOrAfter(Version.V_7_12_0)) { out.writeOptionalNamedWriteable(retentionPolicyConfig); } @@ -483,6 +506,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(TransformField.DESCRIPTION.getPreferredName(), description); } builder.field(TransformField.SETTINGS.getPreferredName(), settings); + if (metadata != null) { + builder.field(TransformField.METADATA.getPreferredName(), metadata); + } if (retentionPolicyConfig != null) { builder.startObject(TransformField.RETENTION_POLICY.getPreferredName()); builder.field(retentionPolicyConfig.getWriteableName(), retentionPolicyConfig); @@ -514,6 +540,7 @@ public boolean equals(Object other) { && Objects.equals(this.latestConfig, that.latestConfig) && Objects.equals(this.description, that.description) && Objects.equals(this.settings, that.settings) + && Objects.equals(this.metadata, that.metadata) && Objects.equals(this.retentionPolicyConfig, that.retentionPolicyConfig) && Objects.equals(this.createTime, that.createTime) && Objects.equals(this.transformVersion, that.transformVersion); @@ -532,6 +559,7 @@ public int hashCode() { latestConfig, description, settings, + metadata, retentionPolicyConfig, createTime, transformVersion @@ -639,6 +667,7 @@ public static class Builder { private PivotConfig pivotConfig; private LatestConfig latestConfig; private SettingsConfig settings; + private Map metadata; private RetentionPolicyConfig retentionPolicyConfig; public Builder() {} @@ -655,6 +684,7 @@ public Builder(TransformConfig config) { this.pivotConfig = config.pivotConfig; this.latestConfig = config.latestConfig; this.settings = config.settings; + this.metadata = config.metadata; this.retentionPolicyConfig = config.retentionPolicyConfig; } @@ -721,6 +751,15 @@ SettingsConfig getSettings() { return settings; } + public Builder setMetadata(Map metadata) { + this.metadata = metadata; + return this; + } + + Map getMetadata() { + return metadata; + } + public Builder setHeaders(Map headers) { this.headers = headers; return this; @@ -774,6 +813,7 @@ public TransformConfig build() { latestConfig, description, settings, + metadata, retentionPolicyConfig, createTime, transformVersion == null ? null : transformVersion.toString() @@ -802,6 +842,7 @@ public boolean equals(Object other) { && Objects.equals(this.latestConfig, that.latestConfig) && Objects.equals(this.description, that.description) && Objects.equals(this.settings, that.settings) + && Objects.equals(this.metadata, that.metadata) && Objects.equals(this.retentionPolicyConfig, that.retentionPolicyConfig) && Objects.equals(this.createTime, that.createTime) && Objects.equals(this.transformVersion, that.transformVersion); @@ -820,6 +861,7 @@ public int hashCode() { latestConfig, description, settings, + metadata, retentionPolicyConfig, createTime, transformVersion diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdate.java index 55e1c38c8ee0a..cb221e15b0fa6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdate.java @@ -12,11 +12,11 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xcontent.ConstructingObjectParser; -import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.TransformMessages; @@ -34,7 +34,7 @@ public class TransformConfigUpdate implements Writeable { public static final String NAME = "data_frame_transform_config_update"; - public static TransformConfigUpdate EMPTY = new TransformConfigUpdate(null, null, null, null, null, null, null); + public static TransformConfigUpdate EMPTY = new TransformConfigUpdate(null, null, null, null, null, null, null, null); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( NAME, @@ -48,8 +48,10 @@ public class TransformConfigUpdate implements Writeable { SyncConfig syncConfig = (SyncConfig) args[3]; String description = (String) args[4]; SettingsConfig settings = (SettingsConfig) args[5]; - RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[6]; - return new TransformConfigUpdate(source, dest, frequency, syncConfig, description, settings, retentionPolicyConfig); + @SuppressWarnings("unchecked") + Map metadata = (Map) args[6]; + RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[7]; + return new TransformConfigUpdate(source, dest, frequency, syncConfig, description, settings, metadata, retentionPolicyConfig); } ); @@ -60,6 +62,7 @@ public class TransformConfigUpdate implements Writeable { PARSER.declareNamedObject(optionalConstructorArg(), (p, c, n) -> p.namedObject(SyncConfig.class, n, c), TransformField.SYNC); PARSER.declareString(optionalConstructorArg(), TransformField.DESCRIPTION); PARSER.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p, false), TransformField.SETTINGS); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.mapOrdered(), TransformField.METADATA); PARSER.declareNamedObject( optionalConstructorArg(), (p, c, n) -> p.namedObject(RetentionPolicyConfig.class, n, c), @@ -73,6 +76,7 @@ public class TransformConfigUpdate implements Writeable { private final SyncConfig syncConfig; private final String description; private final SettingsConfig settings; + private final Map metadata; private final RetentionPolicyConfig retentionPolicyConfig; private Map headers; @@ -83,6 +87,7 @@ public TransformConfigUpdate( final SyncConfig syncConfig, final String description, final SettingsConfig settings, + final Map metadata, final RetentionPolicyConfig retentionPolicyConfig ) { this.source = source; @@ -94,6 +99,7 @@ public TransformConfigUpdate( throw new IllegalArgumentException("[description] must be less than 1000 characters in length."); } this.settings = settings; + this.metadata = metadata; this.retentionPolicyConfig = retentionPolicyConfig; } @@ -111,6 +117,11 @@ public TransformConfigUpdate(final StreamInput in) throws IOException { } else { settings = null; } + if (in.getVersion().onOrAfter(Version.V_7_16_0)) { + metadata = in.readMap(); + } else { + metadata = null; + } if (in.getVersion().onOrAfter(Version.V_7_12_0)) { retentionPolicyConfig = in.readOptionalNamedWriteable(RetentionPolicyConfig.class); } else { @@ -144,6 +155,11 @@ public SettingsConfig getSettings() { return settings; } + @Nullable + public Map getMetadata() { + return metadata; + } + @Nullable public RetentionPolicyConfig getRetentionPolicyConfig() { return retentionPolicyConfig; @@ -173,6 +189,9 @@ public void writeTo(final StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_8_0)) { out.writeOptionalWriteable(settings); } + if (out.getVersion().onOrAfter(Version.V_7_16_0)) { + out.writeMap(metadata); + } if (out.getVersion().onOrAfter(Version.V_7_12_0)) { out.writeOptionalNamedWriteable(retentionPolicyConfig); } @@ -196,13 +215,14 @@ public boolean equals(Object other) { && Objects.equals(this.syncConfig, that.syncConfig) && Objects.equals(this.description, that.description) && Objects.equals(this.settings, that.settings) + && Objects.equals(this.metadata, that.metadata) && Objects.equals(this.retentionPolicyConfig, that.retentionPolicyConfig) && Objects.equals(this.headers, that.headers); } @Override public int hashCode() { - return Objects.hash(source, dest, frequency, syncConfig, description, settings, retentionPolicyConfig, headers); + return Objects.hash(source, dest, frequency, syncConfig, description, settings, metadata, retentionPolicyConfig, headers); } public static TransformConfigUpdate fromXContent(final XContentParser parser) { @@ -220,6 +240,7 @@ && isNullOrEqual(frequency, config.getFrequency()) && isNullOrEqual(syncConfig, config.getSyncConfig()) && isNullOrEqual(description, config.getDescription()) && isNullOrEqual(settings, config.getSettings()) + && isNullOrEqual(metadata, config.getMetadata()) && isNullOrEqual(retentionPolicyConfig, config.getRetentionPolicyConfig()) && isNullOrEqual(headers, config.getHeaders()); } @@ -273,6 +294,10 @@ public TransformConfig apply(TransformConfig config) { settingsBuilder.update(settings); builder.setSettings(settingsBuilder.build()); } + if (metadata != null) { + // Unlike with settings, we fully replace the old metadata with the new metadata + builder.setMetadata(metadata); + } if (retentionPolicyConfig != null) { builder.setRetentionPolicyConfig(retentionPolicyConfig); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java index 337721f21a063..998598114c738 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java @@ -57,6 +57,7 @@ protected Request createTestInstance() { null, null, null, + null, null ); return new Request(config); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index 2327054c82248..1febd8d0cf733 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -10,15 +10,19 @@ import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.core.common.validation.SourceDestValidator.RemoteClusterMinimumVersionValidation; import org.elasticsearch.xpack.core.common.validation.SourceDestValidator.SourceDestValidation; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; @@ -84,6 +88,7 @@ public static TransformConfig randomTransformConfigWithoutHeaders(String id, Piv latestConfig, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomSettingsConfig(), + randomBoolean() ? null : randomMetadata(), randomBoolean() ? null : randomRetentionPolicyConfig(), null, null @@ -128,6 +133,7 @@ public static TransformConfig randomTransformConfig(String id, Version version, latestConfig, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), randomBoolean() ? null : SettingsConfigTests.randomSettingsConfig(), + randomBoolean() ? null : randomMetadata(), randomBoolean() ? null : randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), version == null ? null : version.toString() @@ -157,6 +163,7 @@ public static TransformConfig randomInvalidTransformConfig() { latestConfig, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), null, + randomBoolean() ? null : randomMetadata(), randomBoolean() ? null : randomRetentionPolicyConfig(), null, null @@ -173,6 +180,7 @@ public static TransformConfig randomInvalidTransformConfig() { null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), null, + randomBoolean() ? null : randomMetadata(), randomBoolean() ? null : randomRetentionPolicyConfig(), null, null @@ -187,6 +195,30 @@ public static RetentionPolicyConfig randomRetentionPolicyConfig() { return TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig(); } + public static Map randomMetadata() { + return randomMap(0, 10, () -> { + String key = randomAlphaOfLengthBetween(1, 10); + Object value; + switch (randomIntBetween(0, 3)) { + case 0: + value = null; + break; + case 1: + value = randomLong(); + break; + case 2: + value = randomAlphaOfLengthBetween(1, 10); + break; + case 3: + value = randomMap(0, 10, () -> Tuple.tuple(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10))); + break; + default: + throw new AssertionError(); + } + return Tuple.tuple(key, value); + }); + } + @Before public void setUpOptionalId() { transformId = randomAlphaOfLengthBetween(1, 10); @@ -390,6 +422,7 @@ public void testMaxLengthDescription() { null, null, null, + null, null ) ); @@ -408,6 +441,7 @@ public void testMaxLengthDescription() { null, null, null, + null, null ); assertThat(description, equalTo(config.getDescription())); @@ -775,7 +809,55 @@ public void testCheckForDeprecations() { ) ) ); + } + public void testSerializingMetadataPreservesOrder() throws IOException { + String json = "{" + + " \"id\" : \"" + + transformId + + "\"," + + " \"_meta\": {" + + " \"d\": 4," + + " \"a\": 1," + + " \"c\": 3," + + " \"e\": 5," + + " \"b\": 2" + + "}," + + " \"source\" : {\"index\":\"src\"}," + + " \"dest\" : {\"index\": \"dest\"}," + + " \"pivot\" : {" + + " \"group_by\": {" + + " \"time\": {" + + " \"date_histogram\": {" + + " \"field\": \"timestamp\"," + + " \"fixed_interval\": \"1d\"" + + "} } }," + + " \"aggs\": {" + + " \"avg\": {" + + " \"avg\": {" + + " \"field\": \"points\"" + + "} } } } }"; + + // Read TransformConfig from JSON and verify that metadata keys are in the same order as in JSON + TransformConfig transformConfig = createTransformConfigFromString(json, transformId, true); + assertThat( + new ArrayList<>(transformConfig.getMetadata().keySet()), is(equalTo(org.elasticsearch.core.List.of("d", "a", "c", "e", "b")))); + + // Write TransformConfig to JSON, read it again and verify that metadata keys are still in the same order + json = XContentHelper.toXContent(transformConfig, XContentType.JSON, TO_XCONTENT_PARAMS, false).utf8ToString(); + transformConfig = createTransformConfigFromString(json, transformId, true); + assertThat( + new ArrayList<>(transformConfig.getMetadata().keySet()), is(equalTo(org.elasticsearch.core.List.of("d", "a", "c", "e", "b")))); + + // Write TransformConfig to wire, read it again and verify that metadata keys are still in the same order + try (BytesStreamOutput output = new BytesStreamOutput()) { + transformConfig.writeTo(output); + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), getNamedWriteableRegistry())) { + transformConfig = new TransformConfig(in); + } + } + assertThat( + new ArrayList<>(transformConfig.getMetadata().keySet()), is(equalTo(org.elasticsearch.core.List.of("d", "a", "c", "e", "b")))); } private TransformConfig createTransformConfigFromString(String json, String id) throws IOException { @@ -787,5 +869,4 @@ private TransformConfig createTransformConfigFromString(String json, String id, .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json); return TransformConfig.fromXContent(parser, id, lenient); } - } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java index c7f30a9b966f5..8b3f492d5bf72 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java @@ -21,11 +21,15 @@ import java.io.IOException; import java.time.Instant; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; import static org.elasticsearch.xpack.core.transform.transforms.DestConfigTests.randomDestConfig; import static org.elasticsearch.xpack.core.transform.transforms.SourceConfigTests.randomSourceConfig; +import static org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests.randomMetadata; +import static org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests.randomRetentionPolicyConfig; +import static org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests.randomSyncConfig; import static org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests.randomTransformConfig; import static org.hamcrest.Matchers.equalTo; @@ -39,6 +43,7 @@ public static TransformConfigUpdate randomTransformConfigUpdate() { randomBoolean() ? null : randomSyncConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), randomBoolean() ? null : SettingsConfigTests.randomSettingsConfig(), + randomBoolean() ? null : randomMetadata(), randomBoolean() ? null : randomRetentionPolicyConfig() ); } @@ -65,6 +70,7 @@ public void testIsNoop() { config.getSyncConfig(), config.getDescription(), config.getSettings(), + config.getMetadata(), config.getRetentionPolicyConfig() ); assertTrue("equal update is not noop", update.isNoop(config)); @@ -76,6 +82,7 @@ public void testIsNoop() { config.getSyncConfig(), "this is a new description", config.getSettings(), + config.getMetadata(), config.getRetentionPolicyConfig() ); assertFalse("true update is noop", update.isNoop(config)); @@ -94,11 +101,12 @@ public void testApply() { null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomNonEmptySettingsConfig(), + randomMetadata(), randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.V_7_2_0.toString() ); - TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, null, null); + TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, null, null, null); assertThat(config, equalTo(update.apply(config))); SourceConfig sourceConfig = new SourceConfig("the_new_index"); @@ -107,6 +115,7 @@ public void testApply() { SyncConfig syncConfig = new TimeSyncConfig("time_field", TimeValue.timeValueSeconds(30)); String newDescription = "new description"; SettingsConfig settings = new SettingsConfig(4_000, 4_000.400F, true, true); + Map newMetadata = randomMetadata(); RetentionPolicyConfig retentionPolicyConfig = new TimeRetentionPolicyConfig("time_field", new TimeValue(60_000)); update = new TransformConfigUpdate( sourceConfig, @@ -115,6 +124,7 @@ public void testApply() { syncConfig, newDescription, settings, + newMetadata, retentionPolicyConfig ); @@ -128,6 +138,8 @@ public void testApply() { assertThat(updatedConfig.getSyncConfig(), equalTo(syncConfig)); assertThat(updatedConfig.getDescription(), equalTo(newDescription)); assertThat(updatedConfig.getSettings(), equalTo(settings)); + // We only check for the existence of new entries. The map can also contain the old (random) entries. + assertThat(updatedConfig.getMetadata(), equalTo(newMetadata)); assertThat(updatedConfig.getRetentionPolicyConfig(), equalTo(retentionPolicyConfig)); assertThat(updatedConfig.getHeaders(), equalTo(headers)); assertThat(updatedConfig.getVersion(), equalTo(Version.CURRENT)); @@ -145,13 +157,14 @@ public void testApplySettings() { null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomNonEmptySettingsConfig(), + randomMetadata(), randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.V_7_2_0.toString() ); TransformConfigUpdate update = - new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(4_000, null, (Boolean) null, null), null); + new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(4_000, null, (Boolean) null, null), null, null); TransformConfig updatedConfig = update.apply(config); // for settings we allow partial updates, so changing 1 setting should not overwrite the other @@ -161,7 +174,8 @@ public void testApplySettings() { assertThat(updatedConfig.getSettings().getDatesAsEpochMillis(), equalTo(config.getSettings().getDatesAsEpochMillis())); assertThat(updatedConfig.getSettings().getAlignCheckpoints(), equalTo(config.getSettings().getAlignCheckpoints())); - update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(null, 43.244F, (Boolean) null, null), null); + update = + new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(null, 43.244F, (Boolean) null, null), null, null); updatedConfig = update.apply(updatedConfig); assertThat(updatedConfig.getSettings().getMaxPageSearchSize(), equalTo(4_000)); assertThat(updatedConfig.getSettings().getDocsPerSecond(), equalTo(43.244F)); @@ -169,14 +183,14 @@ public void testApplySettings() { assertThat(updatedConfig.getSettings().getAlignCheckpoints(), equalTo(config.getSettings().getAlignCheckpoints())); // now reset to default using the magic -1 - update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, null, (Boolean) null, null), null); + update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, null, (Boolean) null, null), null, null); updatedConfig = update.apply(updatedConfig); assertNull(updatedConfig.getSettings().getMaxPageSearchSize()); assertThat(updatedConfig.getSettings().getDocsPerSecond(), equalTo(43.244F)); assertThat(updatedConfig.getSettings().getDatesAsEpochMillis(), equalTo(config.getSettings().getDatesAsEpochMillis())); assertThat(updatedConfig.getSettings().getAlignCheckpoints(), equalTo(config.getSettings().getAlignCheckpoints())); - update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, -1F, (Boolean) null, null), null); + update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, -1F, (Boolean) null, null), null, null); updatedConfig = update.apply(updatedConfig); assertNull(updatedConfig.getSettings().getMaxPageSearchSize()); assertNull(updatedConfig.getSettings().getDocsPerSecond()); @@ -184,6 +198,37 @@ public void testApplySettings() { assertThat(updatedConfig.getSettings().getAlignCheckpoints(), equalTo(config.getSettings().getAlignCheckpoints())); } + public void testApplyMetadata() { + Map oldMetadata = new HashMap(); + oldMetadata.put("foo", 123); + oldMetadata.put("bar", 456); + TransformConfig config = new TransformConfig( + "time-transform", + randomSourceConfig(), + randomDestConfig(), + TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), + randomSyncConfig(), + Collections.singletonMap("key", "value"), + PivotConfigTests.randomPivotConfig(), + null, + randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), + SettingsConfigTests.randomNonEmptySettingsConfig(), + oldMetadata, + randomRetentionPolicyConfig(), + randomBoolean() ? null : Instant.now(), + randomBoolean() ? null : Version.V_7_2_0.toString() + ); + + Map newMetadata = new HashMap(); + newMetadata.put("bar", 789); + newMetadata.put("baz", 1000); + TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, null, newMetadata, null); + TransformConfig updatedConfig = update.apply(config); + + // For metadata we apply full replace rather than partial update, so "foo" disappears. + assertThat(updatedConfig.getMetadata(), equalTo(newMetadata)); + } + public void testApplyWithSyncChange() { TransformConfig batchConfig = new TransformConfig( "batch-transform", @@ -196,6 +241,7 @@ public void testApplyWithSyncChange() { null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomNonEmptySettingsConfig(), + randomMetadata(), randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.CURRENT.toString() @@ -208,6 +254,7 @@ public void testApplyWithSyncChange() { TimeSyncConfigTests.randomTimeSyncConfig(), null, null, + null, null ); @@ -228,12 +275,13 @@ public void testApplyWithSyncChange() { null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomNonEmptySettingsConfig(), + randomMetadata(), randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.CURRENT.toString() ); - TransformConfigUpdate fooSyncUpdate = new TransformConfigUpdate(null, null, null, new FooSync(), null, null, null); + TransformConfigUpdate fooSyncUpdate = new TransformConfigUpdate(null, null, null, new FooSync(), null, null, null, null); ex = expectThrows(ElasticsearchStatusException.class, () -> fooSyncUpdate.apply(timeSyncedConfig)); assertThat( ex.getMessage(), @@ -273,6 +321,9 @@ private void toXContent(TransformConfigUpdate update, XContentBuilder builder) t if (update.getSettings() != null) { builder.field(TransformField.SETTINGS.getPreferredName(), update.getSettings()); } + if (update.getMetadata() != null) { + builder.field(TransformField.METADATA.getPreferredName(), update.getMetadata()); + } if (update.getRetentionPolicyConfig() != null) { builder.startObject(TransformField.RETENTION_POLICY.getPreferredName()); builder.field(update.getRetentionPolicyConfig().getWriteableName(), update.getRetentionPolicyConfig()); @@ -282,14 +333,6 @@ private void toXContent(TransformConfigUpdate update, XContentBuilder builder) t builder.endObject(); } - private static SyncConfig randomSyncConfig() { - return TimeSyncConfigTests.randomTimeSyncConfig(); - } - - private static RetentionPolicyConfig randomRetentionPolicyConfig() { - return TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig(); - } - static class FooSync implements SyncConfig { @Override diff --git a/x-pack/plugin/core/src/test/resources/rest-api-spec/schema/transform_config.schema.json b/x-pack/plugin/core/src/test/resources/rest-api-spec/schema/transform_config.schema.json index a7cb9fb13bc15..4a18e647b74c2 100644 --- a/x-pack/plugin/core/src/test/resources/rest-api-spec/schema/transform_config.schema.json +++ b/x-pack/plugin/core/src/test/resources/rest-api-spec/schema/transform_config.schema.json @@ -194,6 +194,11 @@ } } }, + "_meta": { + "$id": "#root/_meta", + "title": "Metadata", + "type": "object" + }, "retention_policy": { "$id": "#root/retention_policy", "additionalProperties": false, diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_update.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_update.yml index 5ac837032b06c..2ef844cdc1d90 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_update.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_update.yml @@ -315,3 +315,75 @@ setup: { "dest": { "index": "destination#dest" } } + +--- +"Test update transform metadata": + - do: + transform.get_transform: + transform_id: "updating-airline-transform" + - match: { count: 1 } + - match: { transforms.0.id: "updating-airline-transform" } + - is_false: transforms.0._meta # no metadata exists yet + + - do: + transform.update_transform: + transform_id: "updating-airline-transform" + body: > + { + "_meta": { + "foo": 123, + "bar": 456, + "baz": { + "a1": 11, + "a2": 22 + } + } + } + - match: { id: "updating-airline-transform" } + - match: { _meta.foo: 123 } + - match: { _meta.bar: 456 } + - match: { _meta.baz.a1: 11 } + - match: { _meta.baz.a2: 22 } + - is_false: _meta.baz.a3 + + - do: + transform.get_transform: + transform_id: "updating-airline-transform" + - match: { count: 1 } + - match: { transforms.0.id: "updating-airline-transform" } + - match: { transforms.0._meta.foo: 123 } + - match: { transforms.0._meta.bar: 456 } + - match: { transforms.0._meta.baz.a1: 11 } + - match: { transforms.0._meta.baz.a2: 22 } + - is_false: transforms.0._meta.baz.a3 + + - do: + transform.update_transform: + transform_id: "updating-airline-transform" + body: > + { + "_meta": { + "bar": "some bar note", + "baz": { + "a2": 222, + "a3": 333 + } + } + } + - match: { id: "updating-airline-transform" } + - is_false: _meta.foo # "foo" disappeared as the metadata update is implemented as full replace + - match: { _meta.bar: "some bar note" } # "bar" value type has changed from int to string + - is_false: _meta.baz.a1 # "baz.a1" disappeared as the metadata update is implemented as full replace + - match: { _meta.baz.a2: 222 } + - match: { _meta.baz.a3: 333 } + + - do: + transform.get_transform: + transform_id: "updating-airline-transform" + - match: { count: 1 } + - match: { transforms.0.id: "updating-airline-transform" } + - is_false: transforms.0._meta.foo + - match: { transforms.0._meta.bar: "some bar note" } + - is_false: transforms.0._meta.baz.a1 + - match: { transforms.0._meta.baz.a2: 222 } + - match: { transforms.0._meta.baz.a3: 333 } diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java index 38bcc325b7147..d7132a29ccd17 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java @@ -165,6 +165,7 @@ public void assertGetProgress(int userWithMissingBuckets) throws Exception { null, null, null, + null, null ); diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java index 5cf1411384efc..d403392905144 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java @@ -85,7 +85,7 @@ public void testUpdateDeletesOldTransformConfig() throws Exception { assertThat(getTransformResponse.getTransformConfigurations().get(0).getId(), equalTo(transformId)); UpdateTransformAction.Request updateTransformActionRequest = new UpdateTransformAction.Request( - new TransformConfigUpdate(null, null, null, null, "updated", null, null), + new TransformConfigUpdate(null, null, null, null, "updated", null, null, null), transformId, false); UpdateTransformAction.Response updateTransformActionResponse = client().execute(UpdateTransformAction.INSTANCE, updateTransformActionRequest).actionGet(); diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoRemoteClusterClientNodeIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoRemoteClusterClientNodeIT.java index 0a803cbcb04b4..691c9e8bbbe6b 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoRemoteClusterClientNodeIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoRemoteClusterClientNodeIT.java @@ -78,7 +78,7 @@ public void testUpdateTransformWithRemoteIndex_DeferValidation() { } TransformConfigUpdate update = - new TransformConfigUpdate(new SourceConfig("remote_cluster:my-index"), null, null, null, null, null, null); + new TransformConfigUpdate(new SourceConfig("remote_cluster:my-index"), null, null, null, null, null, null, null); UpdateTransformAction.Request request = new UpdateTransformAction.Request(update, transformId, true); client().execute(UpdateTransformAction.INSTANCE, request).actionGet(); } @@ -93,7 +93,7 @@ public void testUpdateTransformWithRemoteIndex_NoDeferValidation() { } TransformConfigUpdate update = - new TransformConfigUpdate(new SourceConfig("remote_cluster:my-index"), null, null, null, null, null, null); + new TransformConfigUpdate(new SourceConfig("remote_cluster:my-index"), null, null, null, null, null, null, null); UpdateTransformAction.Request request = new UpdateTransformAction.Request(update, transformId, false); ElasticsearchStatusException e = expectThrows( diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoTransformNodeIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoTransformNodeIT.java index 933d35ff07c8e..bbcd776ab4b4d 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoTransformNodeIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoTransformNodeIT.java @@ -93,7 +93,7 @@ public void testUpdateTransform_DeferValidation() { } TransformConfigUpdate update = - new TransformConfigUpdate(new SourceConfig("my-index", "my-index-2"), null, null, null, null, null, null); + new TransformConfigUpdate(new SourceConfig("my-index", "my-index-2"), null, null, null, null, null, null, null); UpdateTransformAction.Request request = new UpdateTransformAction.Request(update, transformId, true); client().execute(UpdateTransformAction.INSTANCE, request).actionGet(); @@ -111,7 +111,7 @@ public void testUpdateTransform_NoDeferValidation() { } TransformConfigUpdate update = - new TransformConfigUpdate(new SourceConfig("my-index", "my-index-2"), null, null, null, null, null, null); + new TransformConfigUpdate(new SourceConfig("my-index", "my-index-2"), null, null, null, null, null, null, null); UpdateTransformAction.Request request = new UpdateTransformAction.Request(update, transformId, false); ElasticsearchStatusException e = expectThrows( diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java index ea925b6b9451e..58f7f0c8519f0 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java @@ -161,7 +161,7 @@ private void updateOneTransform(String id, boolean dryRun, ActionListener { - TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, null, null); + TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, null, null, null); TransformConfig config = configAndVersion.v1(); /* diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java index 7eb3d92c1ad97..ac1cddc167128 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java @@ -354,6 +354,9 @@ public static XContentBuilder addTransformsConfigMappings(XContentBuilder builde .startObject(TransformConfig.Function.LATEST.getParseField().getPreferredName()) .field(TYPE, FLATTENED) .endObject() + .startObject(TransformField.METADATA.getPreferredName()) + .field(TYPE, FLATTENED) + .endObject() .startObject(TransformField.RETENTION_POLICY.getPreferredName()) .field(TYPE, FLATTENED) .endObject() diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestPutTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestPutTransformAction.java index ce97da904b55e..f03a470fb1e52 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestPutTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestPutTransformAction.java @@ -8,10 +8,12 @@ package org.elasticsearch.xpack.transform.rest.action; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.action.PutTransformAction; @@ -23,6 +25,15 @@ public class RestPutTransformAction extends BaseRestHandler { + /** + * Maximum allowed size of the REST request. + * + * It is set so that the user is able to provide elaborate painless scripts but not able to provide TransformConfig._meta map of + * arbitrary size. Such transform configs of an arbitrary size could be a problem upon fetch so it's better to prevent them on Put and + * Update actions. + */ + static final ByteSizeValue MAX_REQUEST_SIZE = ByteSizeValue.ofMb(5); + @Override public List routes() { return singletonList(new Route(PUT, TransformField.REST_BASE_PATH_TRANSFORMS_BY_ID)); @@ -35,6 +46,11 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + if (restRequest.contentLength() > MAX_REQUEST_SIZE.getBytes()) { + throw ExceptionsHelper.badRequestException( + "Request is too large: was [{}b], expected at most [{}]", restRequest.contentLength(), MAX_REQUEST_SIZE); + } + String id = restRequest.param(TransformField.ID.getPreferredName()); XContentParser parser = restRequest.contentParser(); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestUpdateTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestUpdateTransformAction.java index 40d207e348df1..86e295c7cba7c 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestUpdateTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestUpdateTransformAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.action.UpdateTransformAction; @@ -20,6 +21,7 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.xpack.transform.rest.action.RestPutTransformAction.MAX_REQUEST_SIZE; public class RestUpdateTransformAction extends BaseRestHandler { @@ -35,6 +37,11 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + if (restRequest.contentLength() > MAX_REQUEST_SIZE.getBytes()) { + throw ExceptionsHelper.badRequestException( + "Request is too large: was [{}b], expected at most [{}]", restRequest.contentLength(), MAX_REQUEST_SIZE); + } + String id = restRequest.param(TransformField.ID.getPreferredName()); boolean deferValidation = restRequest.paramAsBoolean(TransformField.DEFER_VALIDATION.getPreferredName(), false); XContentParser parser = restRequest.contentParser(); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java index 61f04b54cb910..fe141e046ef59 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java @@ -312,6 +312,7 @@ public void testPageSizeAdapt() throws Exception { new SettingsConfig(pageSize, null, (Boolean) null, null), null, null, + null, null ); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); @@ -386,6 +387,7 @@ public void testDoProcessAggNullCheck() { new SettingsConfig(pageSize, null, (Boolean) null, null), null, null, + null, null ); SearchResponse searchResponse = new SearchResponse( @@ -449,6 +451,7 @@ public void testScriptError() throws Exception { new SettingsConfig(pageSize, null, (Boolean) null, null), null, null, + null, null ); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); @@ -530,6 +533,7 @@ public void testRetentionPolicyDeleteByQueryThrowsIrrecoverable() throws Excepti null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), null, + null, new TimeRetentionPolicyConfig(randomAlphaOfLength(10), TimeValue.timeValueSeconds(10)), null, null @@ -631,6 +635,7 @@ public void testRetentionPolicyDeleteByQueryThrowsTemporaryProblem() throws Exce null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), null, + null, new TimeRetentionPolicyConfig(randomAlphaOfLength(10), TimeValue.timeValueSeconds(10)), null, null @@ -732,6 +737,7 @@ public void testFailureCounterIsResetOnSuccess() throws Exception { null, null, null, + null, null ); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java index f0ae6bfd95f84..be5d4cda2bd15 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java @@ -276,6 +276,7 @@ public void testStopAtCheckpoint() throws Exception { null, null, null, + null, null ); @@ -508,6 +509,7 @@ public void testStopAtCheckpointForThrottledTransform() throws Exception { new SettingsConfig(null, Float.valueOf(1.0f), (Boolean) null, (Boolean) null), null, null, + null, null ); AtomicReference state = new AtomicReference<>(IndexerState.STARTED); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java index f54b071c72eb0..e0aa1ba0e4c1a 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java @@ -300,6 +300,7 @@ public void testRetentionPolicyExecution() throws Exception { null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), null, + null, TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig(), null, null @@ -344,6 +345,7 @@ public void testRetentionPolicyExecution() throws Exception { null, null, null, + null, null ); @@ -397,6 +399,7 @@ public void testInterActionWhileIndexerShutsdown() throws Exception { null, null, null, + null, null ); AtomicReference state = new AtomicReference<>(IndexerState.STARTED);