From f6c1bba51aa8a4a368022ea1c57d3b1c3ef7efc9 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Wed, 19 Jun 2019 07:01:45 -0500 Subject: [PATCH 1/3] [ML][Data Frame] Add version and create_time to transform config --- .../transforms/DataFrameTransformConfig.java | 51 ++++++-- .../dataframe/transforms/util/TimeUtil.java | 48 ++++++++ .../client/DataFrameTransformIT.java | 4 +- .../DataFrameTransformConfigTests.java | 11 +- .../DataFrameTransformDocumentationIT.java | 2 - .../transforms/DataFrameTransformConfig.java | 114 +++++++++++++++--- .../DataFrameTransformConfigTests.java | 64 +++++++++- .../integration/DataFrameIntegTestCase.java | 14 ++- .../integration/DataFrameTransformIT.java | 8 ++ .../TransportPutDataFrameTransformAction.java | 8 +- .../test/data_frame/transforms_crud.yml | 2 + 11 files changed, 289 insertions(+), 37 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java index 8465ae8342827..4dbe7e09f2587 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java @@ -19,16 +19,20 @@ package org.elasticsearch.client.dataframe.transforms; +import org.elasticsearch.Version; import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfig; +import org.elasticsearch.client.dataframe.transforms.util.TimeUtil; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; +import java.util.Date; import java.util.Objects; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; @@ -40,6 +44,8 @@ public class DataFrameTransformConfig implements ToXContentObject { public static final ParseField SOURCE = new ParseField("source"); public static final ParseField DEST = new ParseField("dest"); public static final ParseField DESCRIPTION = new ParseField("description"); + public static final ParseField TRANSFORM_VERSION = new ParseField("transform_version"); + public static final ParseField CREATE_TIME = new ParseField("create_time"); // types of transforms public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot"); @@ -48,6 +54,8 @@ public class DataFrameTransformConfig implements ToXContentObject { private final DestConfig dest; private final PivotConfig pivotConfig; private final String description; + private final Version transformVersion; + private final Date createTime; public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_frame_transform", true, @@ -57,7 +65,9 @@ public class DataFrameTransformConfig implements ToXContentObject { DestConfig dest = (DestConfig) args[2]; PivotConfig pivotConfig = (PivotConfig) args[3]; String description = (String)args[4]; - return new DataFrameTransformConfig(id, source, dest, pivotConfig, description); + Date createTime = (Date)args[5]; + String transformVersion = (String)args[6]; + return new DataFrameTransformConfig(id, source, dest, pivotConfig, description, createTime, transformVersion); }); static { @@ -66,6 +76,9 @@ public class DataFrameTransformConfig implements ToXContentObject { PARSER.declareObject(constructorArg(), (p, c) -> DestConfig.PARSER.apply(p, null), DEST); PARSER.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p), PIVOT_TRANSFORM); PARSER.declareString(optionalConstructorArg(), DESCRIPTION); + PARSER.declareField(optionalConstructorArg(), + p -> TimeUtil.parseTimeField(p, CREATE_TIME.getPreferredName()), CREATE_TIME, ObjectParser.ValueType.VALUE); + PARSER.declareString(optionalConstructorArg(), TRANSFORM_VERSION); } public static DataFrameTransformConfig fromXContent(final XContentParser parser) { @@ -84,19 +97,23 @@ public static DataFrameTransformConfig fromXContent(final XContentParser parser) * @return A DataFrameTransformConfig to preview, NOTE it will have a {@code null} id, destination and index. */ public static DataFrameTransformConfig forPreview(final SourceConfig source, final PivotConfig pivotConfig) { - return new DataFrameTransformConfig(null, source, null, pivotConfig, null); + return new DataFrameTransformConfig(null, source, null, pivotConfig, null, null, null); } DataFrameTransformConfig(final String id, final SourceConfig source, final DestConfig dest, final PivotConfig pivotConfig, - final String description) { + final String description, + final Date createTime, + final String version) { this.id = id; this.source = source; this.dest = dest; this.pivotConfig = pivotConfig; this.description = description; + this.createTime = createTime; + this.transformVersion = version == null ? null : Version.fromString(version); } public String getId() { @@ -115,6 +132,14 @@ public PivotConfig getPivotConfig() { return pivotConfig; } + public Version getTransformVersion() { + return transformVersion; + } + + public Date getCreateTime() { + return createTime; + } + @Nullable public String getDescription() { return description; @@ -138,6 +163,12 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (description != null) { builder.field(DESCRIPTION.getPreferredName(), description); } + if (createTime != null) { + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.getTime()); + } + if (transformVersion != null) { + builder.field(TRANSFORM_VERSION.getPreferredName(), transformVersion); + } builder.endObject(); return builder; } @@ -155,15 +186,17 @@ public boolean equals(Object other) { final DataFrameTransformConfig that = (DataFrameTransformConfig) other; return Objects.equals(this.id, that.id) - && Objects.equals(this.source, that.source) - && Objects.equals(this.dest, that.dest) - && Objects.equals(this.description, that.description) - && Objects.equals(this.pivotConfig, that.pivotConfig); + && Objects.equals(this.source, that.source) + && Objects.equals(this.dest, that.dest) + && Objects.equals(this.description, that.description) + && Objects.equals(this.transformVersion, that.transformVersion) + && Objects.equals(this.createTime, that.createTime) + && Objects.equals(this.pivotConfig, that.pivotConfig); } @Override public int hashCode() { - return Objects.hash(id, source, dest, pivotConfig, description); + return Objects.hash(id, source, dest, pivotConfig, description, createTime, transformVersion); } @Override @@ -209,7 +242,7 @@ public Builder setDescription(String description) { } public DataFrameTransformConfig build() { - return new DataFrameTransformConfig(id, source, dest, pivotConfig, description); + return new DataFrameTransformConfig(id, source, dest, pivotConfig, description, null, null); } } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java new file mode 100644 index 0000000000000..b35cb44a7422e --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.dataframe.transforms.util; + +import org.elasticsearch.common.time.DateFormatters; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.time.format.DateTimeFormatter; +import java.util.Date; + +public final class TimeUtil { + + /** + * Parse out a Date object given the current parser and field name. + * + * @param parser current XContentParser + * @param fieldName the field's preferred name (utilized in exception) + * @return parsed Date object + * @throws IOException from XContentParser + */ + public static Date parseTimeField(XContentParser parser, String fieldName) throws IOException { + if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) { + return new Date(parser.longValue()); + } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) { + return new Date(DateFormatters.from(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant().toEpochMilli()); + } + throw new IllegalArgumentException( + "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]"); + } + +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java index 5e00dfb8ed3c9..c82cd2f540ff1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java @@ -195,7 +195,7 @@ public void testGetTransform() throws IOException { client::getDataFrameTransformAsync); assertNull(getResponse.getInvalidTransforms()); assertThat(getResponse.getTransformConfigurations(), hasSize(1)); - assertEquals(transform, getResponse.getTransformConfigurations().get(0)); + assertEquals(transform.getId(), getResponse.getTransformConfigurations().get(0).getId()); } public void testGetAllAndPageTransforms() throws IOException { @@ -219,7 +219,7 @@ public void testGetAllAndPageTransforms() throws IOException { client::getDataFrameTransformAsync); assertNull(getResponse.getInvalidTransforms()); assertThat(getResponse.getTransformConfigurations(), hasSize(2)); - assertEquals(transform, getResponse.getTransformConfigurations().get(1)); + assertEquals(transform.getId(), getResponse.getTransformConfigurations().get(1).getId()); getRequest.setPageParams(new PageParams(0,1)); getResponse = execute(getRequest, client::getDataFrameTransform, diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java index e8f030fe79607..7930b4adbdff3 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.client.dataframe.transforms; +import org.elasticsearch.Version; import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfigTests; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -28,6 +29,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.Date; import java.util.function.Predicate; import static org.elasticsearch.client.dataframe.transforms.DestConfigTests.randomDestConfig; @@ -36,8 +38,13 @@ public class DataFrameTransformConfigTests extends AbstractXContentTestCase { public static DataFrameTransformConfig randomDataFrameTransformConfig() { - return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(), - randomDestConfig(), PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100)); + return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), + randomSourceConfig(), + randomDestConfig(), + PivotConfigTests.randomPivotConfig(), + randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100), + randomBoolean() ? null : new Date(), + randomBoolean() ? null : Version.CURRENT.toString()); } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java index f0dfda1d58902..b419fa356bf75 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java @@ -473,7 +473,6 @@ public void testGetStats() throws IOException, InterruptedException { RestHighLevelClient client = highLevelClient(); - QueryConfig queryConfig = new QueryConfig(new MatchAllQueryBuilder()); GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer", TermsGroupSource.builder().setField("user_id").build()).build(); AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder(); @@ -559,7 +558,6 @@ public void onFailure(Exception e) { public void testGetDataFrameTransform() throws IOException, InterruptedException { createIndex("source-data"); - QueryConfig queryConfig = new QueryConfig(new MatchAllQueryBuilder()); GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer", TermsGroupSource.builder().setField("user_id").build()).build(); AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java index ee35fe3d21ec7..d94c22461dffc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.dataframe.transforms; +import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; @@ -14,6 +15,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -21,9 +23,11 @@ import org.elasticsearch.xpack.core.dataframe.DataFrameMessages; import org.elasticsearch.xpack.core.dataframe.transforms.pivot.PivotConfig; import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.dataframe.utils.TimeUtils; import java.io.IOException; import java.util.Collections; +import java.util.Date; import java.util.Map; import java.util.Objects; @@ -42,6 +46,8 @@ public class DataFrameTransformConfig extends AbstractDiffable STRICT_PARSER = createParser(false); private static final ConstructingObjectParser LENIENT_PARSER = createParser(true); private static final int MAX_DESCRIPTION_LENGTH = 1_000; @@ -53,9 +59,17 @@ public class DataFrameTransformConfig extends AbstractDiffable headers; + private Version transformVersion; + private Date createTime; private final PivotConfig pivotConfig; + private static void validateStrictParsingParams(Object arg, String parameterName) { + if (arg != null) { + throw new IllegalArgumentException("Found [" + parameterName + "], not allowed for strict parsing"); + } + } + private static ConstructingObjectParser createParser(boolean lenient) { ConstructingObjectParser parser = new ConstructingObjectParser<>(NAME, lenient, (args, optionalId) -> { @@ -74,9 +88,11 @@ private static ConstructingObjectParser create // ignored, only for internal storage: String docType = (String) args[3]; - // on strict parsing do not allow injection of headers - if (lenient == false && args[4] != null) { - throw new IllegalArgumentException("Found [headers], not allowed for strict parsing"); + // on strict parsing do not allow injection of headers, transform version, or create time + if (lenient == false) { + validateStrictParsingParams(args[4], HEADERS.getPreferredName()); + validateStrictParsingParams(args[7], CREATE_TIME.getPreferredName()); + validateStrictParsingParams(args[8], TRANSFORM_VERSION.getPreferredName()); } @SuppressWarnings("unchecked") @@ -84,7 +100,14 @@ private static ConstructingObjectParser create PivotConfig pivotConfig = (PivotConfig) args[5]; String description = (String)args[6]; - return new DataFrameTransformConfig(id, source, dest, headers, pivotConfig, description); + return new DataFrameTransformConfig(id, + source, + dest, + headers, + pivotConfig, + description, + (Date)args[7], + (String)args[8]); }); parser.declareString(optionalConstructorArg(), DataFrameField.ID); @@ -95,7 +118,9 @@ private static ConstructingObjectParser create parser.declareObject(optionalConstructorArg(), (p, c) -> p.mapStrings(), HEADERS); parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), PIVOT_TRANSFORM); parser.declareString(optionalConstructorArg(), DESCRIPTION); - + parser.declareField(optionalConstructorArg(), + p -> TimeUtils.parseTimeField(p, CREATE_TIME.getPreferredName()), CREATE_TIME, ObjectParser.ValueType.VALUE); + parser.declareString(optionalConstructorArg(), TRANSFORM_VERSION); return parser; } @@ -103,12 +128,14 @@ public static String documentId(String transformId) { return NAME + "-" + transformId; } - public DataFrameTransformConfig(final String id, - final SourceConfig source, - final DestConfig dest, - final Map headers, - final PivotConfig pivotConfig, - final String description) { + DataFrameTransformConfig(final String id, + final SourceConfig source, + final DestConfig dest, + final Map headers, + final PivotConfig pivotConfig, + final String description, + final Date createTime, + final String version){ this.id = ExceptionsHelper.requireNonNull(id, DataFrameField.ID.getPreferredName()); this.source = ExceptionsHelper.requireNonNull(source, DataFrameField.SOURCE.getPreferredName()); this.dest = ExceptionsHelper.requireNonNull(dest, DataFrameField.DESTINATION.getPreferredName()); @@ -123,6 +150,17 @@ public DataFrameTransformConfig(final String id, if (this.description != null && this.description.length() > MAX_DESCRIPTION_LENGTH) { throw new IllegalArgumentException("[description] must be less than 1000 characters in length."); } + this.createTime = createTime; + this.transformVersion = version == null ? null : Version.fromString(version); + } + + public DataFrameTransformConfig(final String id, + final SourceConfig source, + final DestConfig dest, + final Map headers, + final PivotConfig pivotConfig, + final String description) { + this(id, source, dest, headers, pivotConfig, description, null, null); } public DataFrameTransformConfig(final StreamInput in) throws IOException { @@ -132,6 +170,13 @@ public DataFrameTransformConfig(final StreamInput in) throws IOException { setHeaders(in.readMap(StreamInput::readString, StreamInput::readString)); pivotConfig = in.readOptionalWriteable(PivotConfig::new); description = in.readOptionalString(); + if (in.getVersion().onOrAfter(Version.V_7_3_0)) { + createTime = in.readBoolean() ? new Date(in.readVLong()) : null; + transformVersion = in.readBoolean() ? Version.readVersion(in) : null; + } else { + createTime = null; + transformVersion = null; + } } public String getId() { @@ -150,8 +195,27 @@ public Map getHeaders() { return headers; } - public void setHeaders(Map headers) { + public DataFrameTransformConfig setHeaders(Map headers) { this.headers = headers; + return this; + } + + public Version getTransformVersion() { + return transformVersion; + } + + public DataFrameTransformConfig setTransformVersion(Version transformVersion) { + this.transformVersion = transformVersion; + return this; + } + + public Date getCreateTime() { + return createTime; + } + + public DataFrameTransformConfig setCreateTime(Date createTime) { + this.createTime = createTime; + return this; } public PivotConfig getPivotConfig() { @@ -179,6 +243,20 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeString); out.writeOptionalWriteable(pivotConfig); out.writeOptionalString(description); + if (out.getVersion().onOrAfter(Version.V_7_3_0)) { + if (createTime != null) { + out.writeBoolean(true); + out.writeVLong(createTime.getTime()); + } else { + out.writeBoolean(false); + } + if (transformVersion != null) { + out.writeBoolean(true); + Version.writeVersion(transformVersion, out); + } else { + out.writeBoolean(false); + } + } } @Override @@ -199,6 +277,12 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (description != null) { builder.field(DESCRIPTION.getPreferredName(), description); } + if (transformVersion != null) { + builder.field(TRANSFORM_VERSION.getPreferredName(), transformVersion); + } + if (createTime != null) { + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.getTime()); + } builder.endObject(); return builder; } @@ -220,12 +304,14 @@ public boolean equals(Object other) { && Objects.equals(this.dest, that.dest) && Objects.equals(this.headers, that.headers) && Objects.equals(this.pivotConfig, that.pivotConfig) - && Objects.equals(this.description, that.description); + && Objects.equals(this.description, that.description) + && Objects.equals(this.createTime, that.createTime) + && Objects.equals(this.transformVersion, that.transformVersion); } @Override public int hashCode(){ - return Objects.hash(id, source, dest, headers, pivotConfig, description); + return Objects.hash(id, source, dest, headers, pivotConfig, description, createTime, transformVersion); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java index a735b5a02acb8..a0f924287d361 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.dataframe.transforms; +import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.DeprecationHandler; @@ -18,6 +19,7 @@ import org.junit.Before; import java.io.IOException; +import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -41,13 +43,25 @@ public static DataFrameTransformConfig randomDataFrameTransformConfig() { } public static DataFrameTransformConfig randomDataFrameTransformConfigWithoutHeaders(String id) { - return new DataFrameTransformConfig(id, randomSourceConfig(), randomDestConfig(), null, - PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000)); + return new DataFrameTransformConfig(id, + randomSourceConfig(), + randomDestConfig(), + null, + PivotConfigTests.randomPivotConfig(), + randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), + null, + null); } public static DataFrameTransformConfig randomDataFrameTransformConfig(String id) { - return new DataFrameTransformConfig(id, randomSourceConfig(), randomDestConfig(), randomHeaders(), - PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000)); + return new DataFrameTransformConfig(id, + randomSourceConfig(), + randomDestConfig(), + randomHeaders(), + PivotConfigTests.randomPivotConfig(), + randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), + randomBoolean() ? null : new Date(), + randomBoolean() ? null : Version.CURRENT.toString()); } public static DataFrameTransformConfig randomInvalidDataFrameTransformConfig() { @@ -147,6 +161,48 @@ public void testPreventHeaderInjection() throws IOException { () -> createDataFrameTransformConfigFromString(pivotTransform, "test_header_injection")); } + public void testPreventCreateTimeInjection() throws IOException { + String pivotTransform = "{" + + " \"create_time\" : " + new Date().toInstant().toEpochMilli() + " }," + + " \"source\" : {\"index\":\"src\"}," + + " \"dest\" : {\"index\": \"dest\"}," + + " \"pivot\" : {" + + " \"group_by\": {" + + " \"id\": {" + + " \"terms\": {" + + " \"field\": \"id\"" + + "} } }," + + " \"aggs\": {" + + " \"avg\": {" + + " \"avg\": {" + + " \"field\": \"points\"" + + "} } } } }"; + + expectThrows(IllegalArgumentException.class, + () -> createDataFrameTransformConfigFromString(pivotTransform, "test_createTime_injection")); + } + + public void testPreventVersionInjection() throws IOException { + String pivotTransform = "{" + + " \"transform_version\" : \"7.3.0\"," + + " \"source\" : {\"index\":\"src\"}," + + " \"dest\" : {\"index\": \"dest\"}," + + " \"pivot\" : {" + + " \"group_by\": {" + + " \"id\": {" + + " \"terms\": {" + + " \"field\": \"id\"" + + "} } }," + + " \"aggs\": {" + + " \"avg\": {" + + " \"avg\": {" + + " \"field\": \"points\"" + + "} } } } }"; + + expectThrows(IllegalArgumentException.class, + () -> createDataFrameTransformConfigFromString(pivotTransform, "test_createTime_injection")); + } + public void testXContentForInternalStorage() throws IOException { DataFrameTransformConfig dataFrameTransformConfig = randomDataFrameTransformConfig(); diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java index 456bdf7ae6d90..f520d72cc3793 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java @@ -15,6 +15,8 @@ import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.core.AcknowledgedResponse; import org.elasticsearch.client.dataframe.DeleteDataFrameTransformRequest; +import org.elasticsearch.client.dataframe.GetDataFrameTransformRequest; +import org.elasticsearch.client.dataframe.GetDataFrameTransformResponse; import org.elasticsearch.client.dataframe.GetDataFrameTransformStatsRequest; import org.elasticsearch.client.dataframe.GetDataFrameTransformStatsResponse; import org.elasticsearch.client.dataframe.PutDataFrameTransformRequest; @@ -57,6 +59,7 @@ import java.util.Base64; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -118,6 +121,11 @@ protected GetDataFrameTransformStatsResponse getDataFrameTransformStats(String i return restClient.dataFrame().getDataFrameTransformStats(new GetDataFrameTransformStatsRequest(id), RequestOptions.DEFAULT); } + protected GetDataFrameTransformResponse getDataFrameTransform(String id) throws IOException { + RestHighLevelClient restClient = new TestRestHighLevelClient(); + return restClient.dataFrame().getDataFrameTransform(new GetDataFrameTransformRequest(id), RequestOptions.DEFAULT); + } + protected void waitUntilCheckpoint(String id, long checkpoint) throws Exception { waitUntilCheckpoint(id, checkpoint, TimeValue.timeValueSeconds(30)); } @@ -321,9 +329,11 @@ protected Settings restClientSettings() { .build(); } - private class TestRestHighLevelClient extends RestHighLevelClient { + private static class TestRestHighLevelClient extends RestHighLevelClient { + private static final List X_CONTENT_ENTRIES = + new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents(); TestRestHighLevelClient() { - super(client(), restClient -> {}, Collections.emptyList()); + super(client(), restClient -> {}, X_CONTENT_ENTRIES); } } } diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java index c4c5ca3c13004..320b993808192 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.dataframe.integration; +import org.elasticsearch.Version; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.core.IndexerState; import org.elasticsearch.client.dataframe.transforms.DataFrameTransformConfig; @@ -17,6 +18,7 @@ import org.junit.After; import java.io.IOException; +import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -58,6 +60,12 @@ public void testDataFrameTransformCrud() throws Exception { assertThat(getDataFrameTransformStats(config.getId()).getTransformsStateAndStats().get(0).getTransformState().getIndexerState(), equalTo(IndexerState.STOPPED))); stopDataFrameTransform(config.getId()); + + DataFrameTransformConfig storedConfig = getDataFrameTransform(config.getId()).getTransformConfigurations().get(0); + assertThat(storedConfig.getTransformVersion(), equalTo(Version.CURRENT)); + Date now = new Date(); + assertTrue(storedConfig.getCreateTime().before(now)); + deleteDataFrameTransform(config.getId()); } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java index 049b0804f4519..77cbeec817e63 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java @@ -8,6 +8,7 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.IndicesOptions; @@ -53,6 +54,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -110,8 +112,10 @@ protected void masterOperation(Request request, ClusterState clusterState, Actio .filter(e -> ClientHelper.SECURITY_HEADER_FILTERS.contains(e.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - DataFrameTransformConfig config = request.getConfig(); - config.setHeaders(filteredHeaders); + DataFrameTransformConfig config = request.getConfig() + .setHeaders(filteredHeaders) + .setCreateTime(new Date()) + .setTransformVersion(Version.CURRENT); String transformId = config.getId(); // quick check whether a transform has already been created under that name diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml index ffba67f879145..713426321e747 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml @@ -90,6 +90,8 @@ setup: - match: { transforms.0.source.index.0: "airline-data" } - match: { transforms.0.dest.index: "airline-data-by-airline" } - is_true: transforms.0.source.query.match_all + - is_true: transforms.0.create_time + - is_true: transforms.0.transform_version - match: { transforms.0.pivot.group_by.airline.terms.field: "airline" } - match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" } - match: { transforms.0.description: "yaml test transform on airline-data" } From a7bfbfc61e9dd07056a49c06a49c0019dac66821 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Thu, 20 Jun 2019 11:32:29 -0500 Subject: [PATCH 2/3] s/transform_version/version s/Date/Instant --- .../transforms/DataFrameTransformConfig.java | 22 +++++----- .../dataframe/transforms/util/TimeUtil.java | 11 +++++ .../DataFrameTransformConfigTests.java | 4 +- .../transforms/DataFrameTransformConfig.java | 40 +++++++++---------- .../xpack/core/dataframe/utils/TimeUtils.java | 11 +++++ .../DataFrameTransformConfigTests.java | 8 ++-- .../integration/DataFrameTransformIT.java | 7 ++-- .../TransportPutDataFrameTransformAction.java | 4 +- .../test/data_frame/transforms_crud.yml | 2 +- 9 files changed, 63 insertions(+), 46 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java index 4dbe7e09f2587..23aa47a0a9df1 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java @@ -32,7 +32,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; -import java.util.Date; +import java.time.Instant; import java.util.Objects; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; @@ -44,7 +44,7 @@ public class DataFrameTransformConfig implements ToXContentObject { public static final ParseField SOURCE = new ParseField("source"); public static final ParseField DEST = new ParseField("dest"); public static final ParseField DESCRIPTION = new ParseField("description"); - public static final ParseField TRANSFORM_VERSION = new ParseField("transform_version"); + public static final ParseField VERSION = new ParseField("version"); public static final ParseField CREATE_TIME = new ParseField("create_time"); // types of transforms public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot"); @@ -55,7 +55,7 @@ public class DataFrameTransformConfig implements ToXContentObject { private final PivotConfig pivotConfig; private final String description; private final Version transformVersion; - private final Date createTime; + private final Instant createTime; public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_frame_transform", true, @@ -65,7 +65,7 @@ public class DataFrameTransformConfig implements ToXContentObject { DestConfig dest = (DestConfig) args[2]; PivotConfig pivotConfig = (PivotConfig) args[3]; String description = (String)args[4]; - Date createTime = (Date)args[5]; + Instant createTime = (Instant)args[5]; String transformVersion = (String)args[6]; return new DataFrameTransformConfig(id, source, dest, pivotConfig, description, createTime, transformVersion); }); @@ -77,8 +77,8 @@ public class DataFrameTransformConfig implements ToXContentObject { PARSER.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p), PIVOT_TRANSFORM); PARSER.declareString(optionalConstructorArg(), DESCRIPTION); PARSER.declareField(optionalConstructorArg(), - p -> TimeUtil.parseTimeField(p, CREATE_TIME.getPreferredName()), CREATE_TIME, ObjectParser.ValueType.VALUE); - PARSER.declareString(optionalConstructorArg(), TRANSFORM_VERSION); + p -> TimeUtil.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()), CREATE_TIME, ObjectParser.ValueType.VALUE); + PARSER.declareString(optionalConstructorArg(), VERSION); } public static DataFrameTransformConfig fromXContent(final XContentParser parser) { @@ -105,14 +105,14 @@ public static DataFrameTransformConfig forPreview(final SourceConfig source, fin final DestConfig dest, final PivotConfig pivotConfig, final String description, - final Date createTime, + final Instant createTime, final String version) { this.id = id; this.source = source; this.dest = dest; this.pivotConfig = pivotConfig; this.description = description; - this.createTime = createTime; + this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli()); this.transformVersion = version == null ? null : Version.fromString(version); } @@ -136,7 +136,7 @@ public Version getTransformVersion() { return transformVersion; } - public Date getCreateTime() { + public Instant getCreateTime() { return createTime; } @@ -164,10 +164,10 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(DESCRIPTION.getPreferredName(), description); } if (createTime != null) { - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.getTime()); + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); } if (transformVersion != null) { - builder.field(TRANSFORM_VERSION.getPreferredName(), transformVersion); + builder.field(VERSION.getPreferredName(), transformVersion); } builder.endObject(); return builder; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java index b35cb44a7422e..2470c3f7a4aa8 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; +import java.time.Instant; import java.time.format.DateTimeFormatter; import java.util.Date; @@ -45,4 +46,14 @@ public static Date parseTimeField(XContentParser parser, String fieldName) throw "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]"); } + public static Instant parseTimeFieldToInstant(XContentParser parser, String fieldName) throws IOException { + if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) { + return Instant.ofEpochMilli(parser.longValue()); + } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) { + return DateFormatters.from(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant(); + } + throw new IllegalArgumentException( + "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]"); + } + } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java index 7930b4adbdff3..9146bd39d1336 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java @@ -28,8 +28,8 @@ import org.elasticsearch.test.AbstractXContentTestCase; import java.io.IOException; +import java.time.Instant; import java.util.Collections; -import java.util.Date; import java.util.function.Predicate; import static org.elasticsearch.client.dataframe.transforms.DestConfigTests.randomDestConfig; @@ -43,7 +43,7 @@ public static DataFrameTransformConfig randomDataFrameTransformConfig() { randomDestConfig(), PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100), - randomBoolean() ? null : new Date(), + randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.CURRENT.toString()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java index d94c22461dffc..b2bbaad294819 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java @@ -26,8 +26,8 @@ import org.elasticsearch.xpack.core.dataframe.utils.TimeUtils; import java.io.IOException; +import java.time.Instant; import java.util.Collections; -import java.util.Date; import java.util.Map; import java.util.Objects; @@ -46,7 +46,7 @@ public class DataFrameTransformConfig extends AbstractDiffable STRICT_PARSER = createParser(false); private static final ConstructingObjectParser LENIENT_PARSER = createParser(true); @@ -60,7 +60,7 @@ public class DataFrameTransformConfig extends AbstractDiffable headers; private Version transformVersion; - private Date createTime; + private Instant createTime; private final PivotConfig pivotConfig; @@ -92,7 +92,7 @@ private static ConstructingObjectParser create if (lenient == false) { validateStrictParsingParams(args[4], HEADERS.getPreferredName()); validateStrictParsingParams(args[7], CREATE_TIME.getPreferredName()); - validateStrictParsingParams(args[8], TRANSFORM_VERSION.getPreferredName()); + validateStrictParsingParams(args[8], VERSION.getPreferredName()); } @SuppressWarnings("unchecked") @@ -106,7 +106,7 @@ private static ConstructingObjectParser create headers, pivotConfig, description, - (Date)args[7], + (Instant)args[7], (String)args[8]); }); @@ -119,8 +119,8 @@ private static ConstructingObjectParser create parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), PIVOT_TRANSFORM); parser.declareString(optionalConstructorArg(), DESCRIPTION); parser.declareField(optionalConstructorArg(), - p -> TimeUtils.parseTimeField(p, CREATE_TIME.getPreferredName()), CREATE_TIME, ObjectParser.ValueType.VALUE); - parser.declareString(optionalConstructorArg(), TRANSFORM_VERSION); + p -> TimeUtils.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()), CREATE_TIME, ObjectParser.ValueType.VALUE); + parser.declareString(optionalConstructorArg(), VERSION); return parser; } @@ -134,7 +134,7 @@ public static String documentId(String transformId) { final Map headers, final PivotConfig pivotConfig, final String description, - final Date createTime, + final Instant createTime, final String version){ this.id = ExceptionsHelper.requireNonNull(id, DataFrameField.ID.getPreferredName()); this.source = ExceptionsHelper.requireNonNull(source, DataFrameField.SOURCE.getPreferredName()); @@ -150,7 +150,7 @@ public static String documentId(String transformId) { if (this.description != null && this.description.length() > MAX_DESCRIPTION_LENGTH) { throw new IllegalArgumentException("[description] must be less than 1000 characters in length."); } - this.createTime = createTime; + this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli()); this.transformVersion = version == null ? null : Version.fromString(version); } @@ -171,7 +171,7 @@ public DataFrameTransformConfig(final StreamInput in) throws IOException { pivotConfig = in.readOptionalWriteable(PivotConfig::new); description = in.readOptionalString(); if (in.getVersion().onOrAfter(Version.V_7_3_0)) { - createTime = in.readBoolean() ? new Date(in.readVLong()) : null; + createTime = in.readOptionalInstant(); transformVersion = in.readBoolean() ? Version.readVersion(in) : null; } else { createTime = null; @@ -209,12 +209,13 @@ public DataFrameTransformConfig setTransformVersion(Version transformVersion) { return this; } - public Date getCreateTime() { + public Instant getCreateTime() { return createTime; } - public DataFrameTransformConfig setCreateTime(Date createTime) { - this.createTime = createTime; + public DataFrameTransformConfig setCreateTime(Instant createTime) { + ExceptionsHelper.requireNonNull(createTime, CREATE_TIME.getPreferredName()); + this.createTime = Instant.ofEpochMilli(createTime.toEpochMilli()); return this; } @@ -244,13 +245,8 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeOptionalWriteable(pivotConfig); out.writeOptionalString(description); if (out.getVersion().onOrAfter(Version.V_7_3_0)) { - if (createTime != null) { - out.writeBoolean(true); - out.writeVLong(createTime.getTime()); - } else { - out.writeBoolean(false); - } - if (transformVersion != null) { + out.writeOptionalInstant(createTime); + if (transformVersion != null) { out.writeBoolean(true); Version.writeVersion(transformVersion, out); } else { @@ -278,10 +274,10 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(DESCRIPTION.getPreferredName(), description); } if (transformVersion != null) { - builder.field(TRANSFORM_VERSION.getPreferredName(), transformVersion); + builder.field(VERSION.getPreferredName(), transformVersion); } if (createTime != null) { - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.getTime()); + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); } builder.endObject(); return builder; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/utils/TimeUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/utils/TimeUtils.java index 4cf3859380cd8..21a4692f547af 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/utils/TimeUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/utils/TimeUtils.java @@ -12,6 +12,7 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import java.io.IOException; +import java.time.Instant; import java.util.Date; import java.util.concurrent.TimeUnit; @@ -31,6 +32,16 @@ public static Date parseTimeField(XContentParser parser, String fieldName) throw "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]"); } + public static Instant parseTimeFieldToInstant(XContentParser parser, String fieldName) throws IOException { + if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) { + return Instant.ofEpochMilli(parser.longValue()); + } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) { + return Instant.ofEpochMilli(TimeUtils.dateStringToEpoch(parser.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]"); + } + /** * First tries to parse the date first as a Long and convert that to an * epoch time. If the long number has more than 10 digits it is considered a diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java index a0f924287d361..907c8eb98e69f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java @@ -19,7 +19,7 @@ import org.junit.Before; import java.io.IOException; -import java.util.Date; +import java.time.Instant; import java.util.HashMap; import java.util.Map; @@ -60,7 +60,7 @@ public static DataFrameTransformConfig randomDataFrameTransformConfig(String id) randomHeaders(), PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), - randomBoolean() ? null : new Date(), + randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.CURRENT.toString()); } @@ -163,7 +163,7 @@ public void testPreventHeaderInjection() throws IOException { public void testPreventCreateTimeInjection() throws IOException { String pivotTransform = "{" - + " \"create_time\" : " + new Date().toInstant().toEpochMilli() + " }," + + " \"create_time\" : " + Instant.now().toEpochMilli() + " }," + " \"source\" : {\"index\":\"src\"}," + " \"dest\" : {\"index\": \"dest\"}," + " \"pivot\" : {" @@ -184,7 +184,7 @@ public void testPreventCreateTimeInjection() throws IOException { public void testPreventVersionInjection() throws IOException { String pivotTransform = "{" - + " \"transform_version\" : \"7.3.0\"," + + " \"version\" : \"7.3.0\"," + " \"source\" : {\"index\":\"src\"}," + " \"dest\" : {\"index\": \"dest\"}," + " \"pivot\" : {" diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java index 320b993808192..03606eb3bc5e9 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java @@ -18,7 +18,7 @@ import org.junit.After; import java.io.IOException; -import java.util.Date; +import java.time.Instant; import java.util.HashMap; import java.util.Map; @@ -63,9 +63,8 @@ public void testDataFrameTransformCrud() throws Exception { DataFrameTransformConfig storedConfig = getDataFrameTransform(config.getId()).getTransformConfigurations().get(0); assertThat(storedConfig.getTransformVersion(), equalTo(Version.CURRENT)); - Date now = new Date(); - assertTrue(storedConfig.getCreateTime().before(now)); - + Instant now = Instant.now(); + assertTrue("[create_time] is not before current time", storedConfig.getCreateTime().isBefore(now)); deleteDataFrameTransform(config.getId()); } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java index 77cbeec817e63..2bc6b044f8cc3 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java @@ -52,9 +52,9 @@ import org.elasticsearch.xpack.dataframe.transforms.pivot.Pivot; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; -import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -114,7 +114,7 @@ protected void masterOperation(Request request, ClusterState clusterState, Actio DataFrameTransformConfig config = request.getConfig() .setHeaders(filteredHeaders) - .setCreateTime(new Date()) + .setCreateTime(Instant.now()) .setTransformVersion(Version.CURRENT); String transformId = config.getId(); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml index 713426321e747..3efa608e8f54c 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml @@ -91,7 +91,7 @@ setup: - match: { transforms.0.dest.index: "airline-data-by-airline" } - is_true: transforms.0.source.query.match_all - is_true: transforms.0.create_time - - is_true: transforms.0.transform_version + - is_true: transforms.0.version - match: { transforms.0.pivot.group_by.airline.terms.field: "airline" } - match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" } - match: { transforms.0.description: "yaml test transform on airline-data" } From a92527050cf0857f219c1cec12a15a79ebcfd945 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Thu, 20 Jun 2019 11:40:26 -0500 Subject: [PATCH 3/3] fixing getter/setter for version --- .../client/dataframe/transforms/DataFrameTransformConfig.java | 2 +- .../core/dataframe/transforms/DataFrameTransformConfig.java | 4 ++-- .../xpack/dataframe/integration/DataFrameTransformIT.java | 2 +- .../action/TransportPutDataFrameTransformAction.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java index 23aa47a0a9df1..34bcb595c206e 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java @@ -132,7 +132,7 @@ public PivotConfig getPivotConfig() { return pivotConfig; } - public Version getTransformVersion() { + public Version getVersion() { return transformVersion; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java index b2bbaad294819..19d4d6ab6eed1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java @@ -200,11 +200,11 @@ public DataFrameTransformConfig setHeaders(Map headers) { return this; } - public Version getTransformVersion() { + public Version getVersion() { return transformVersion; } - public DataFrameTransformConfig setTransformVersion(Version transformVersion) { + public DataFrameTransformConfig setVersion(Version transformVersion) { this.transformVersion = transformVersion; return this; } diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java index 03606eb3bc5e9..174a956eb3c45 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java @@ -62,7 +62,7 @@ public void testDataFrameTransformCrud() throws Exception { stopDataFrameTransform(config.getId()); DataFrameTransformConfig storedConfig = getDataFrameTransform(config.getId()).getTransformConfigurations().get(0); - assertThat(storedConfig.getTransformVersion(), equalTo(Version.CURRENT)); + assertThat(storedConfig.getVersion(), equalTo(Version.CURRENT)); Instant now = Instant.now(); assertTrue("[create_time] is not before current time", storedConfig.getCreateTime().isBefore(now)); deleteDataFrameTransform(config.getId()); diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java index 2bc6b044f8cc3..36023c0f7372c 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java @@ -115,7 +115,7 @@ protected void masterOperation(Request request, ClusterState clusterState, Actio DataFrameTransformConfig config = request.getConfig() .setHeaders(filteredHeaders) .setCreateTime(Instant.now()) - .setTransformVersion(Version.CURRENT); + .setVersion(Version.CURRENT); String transformId = config.getId(); // quick check whether a transform has already been created under that name