Skip to content

Commit

Permalink
[7.x] [ML][Data Frame] Add version and create_time to transform config (
Browse files Browse the repository at this point in the history 
#43384) (#43480)

* [ML][Data Frame] Add version and create_time to transform config (#43384)

* [ML][Data Frame] Add version and create_time to transform config

* s/transform_version/version s/Date/Instant

* fixing getter/setter for version

* adjusting for backport
  • Loading branch information
@benwtrent
benwtrent authored Jun 21, 2019
1 parent e4fd0ce commit f4b75d6
Show file tree
Hide file tree
Showing 12 changed files with 306 additions and 37 deletions.
51 changes: 42 additions & 9 deletions ...src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,20 @@

package org.elasticsearch.client.dataframe.transforms;

import org.elasticsearch.Version;
import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfig;
import org.elasticsearch.client.dataframe.transforms.util.TimeUtil;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;

import java.io.IOException;
import java.time.Instant;
import java.util.Objects;

import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
Expand All @@ -40,6 +44,8 @@ public class DataFrameTransformConfig implements ToXContentObject {
public static final ParseField SOURCE = new ParseField("source");
public static final ParseField DEST = new ParseField("dest");
public static final ParseField DESCRIPTION = new ParseField("description");
public static final ParseField VERSION = new ParseField("version");
public static final ParseField CREATE_TIME = new ParseField("create_time");
// types of transforms
public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot");

Expand All @@ -48,6 +54,8 @@ public class DataFrameTransformConfig implements ToXContentObject {
private final DestConfig dest;
private final PivotConfig pivotConfig;
private final String description;
private final Version transformVersion;
private final Instant createTime;

public static final ConstructingObjectParser<DataFrameTransformConfig, Void> PARSER =
new ConstructingObjectParser<>("data_frame_transform", true,
Expand All @@ -57,7 +65,9 @@ public class DataFrameTransformConfig implements ToXContentObject {
DestConfig dest = (DestConfig) args[2];
PivotConfig pivotConfig = (PivotConfig) args[3];
String description = (String)args[4];
return new DataFrameTransformConfig(id, source, dest, pivotConfig, description);
Instant createTime = (Instant)args[5];
String transformVersion = (String)args[6];
return new DataFrameTransformConfig(id, source, dest, pivotConfig, description, createTime, transformVersion);
});

static {
Expand All @@ -66,6 +76,9 @@ public class DataFrameTransformConfig implements ToXContentObject {
PARSER.declareObject(constructorArg(), (p, c) -> DestConfig.PARSER.apply(p, null), DEST);
PARSER.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p), PIVOT_TRANSFORM);
PARSER.declareString(optionalConstructorArg(), DESCRIPTION);
PARSER.declareField(optionalConstructorArg(),
p -> TimeUtil.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()), CREATE_TIME, ObjectParser.ValueType.VALUE);
PARSER.declareString(optionalConstructorArg(), VERSION);
}

public static DataFrameTransformConfig fromXContent(final XContentParser parser) {
Expand All @@ -84,19 +97,23 @@ public static DataFrameTransformConfig fromXContent(final XContentParser parser)
* @return A DataFrameTransformConfig to preview, NOTE it will have a {@code null} id, destination and index.
*/
public static DataFrameTransformConfig forPreview(final SourceConfig source, final PivotConfig pivotConfig) {
return new DataFrameTransformConfig(null, source, null, pivotConfig, null);
return new DataFrameTransformConfig(null, source, null, pivotConfig, null, null, null);
}

DataFrameTransformConfig(final String id,
final SourceConfig source,
final DestConfig dest,
final PivotConfig pivotConfig,
final String description) {
final String description,
final Instant createTime,
final String version) {
this.id = id;
this.source = source;
this.dest = dest;
this.pivotConfig = pivotConfig;
this.description = description;
this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli());
this.transformVersion = version == null ? null : Version.fromString(version);
}

public String getId() {
Expand All @@ -115,6 +132,14 @@ public PivotConfig getPivotConfig() {
return pivotConfig;
}

public Version getVersion() {
return transformVersion;
}

public Instant getCreateTime() {
return createTime;
}

@Nullable
public String getDescription() {
return description;
Expand All @@ -138,6 +163,12 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
if (description != null) {
builder.field(DESCRIPTION.getPreferredName(), description);
}
if (createTime != null) {
builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli());
}
if (transformVersion != null) {
builder.field(VERSION.getPreferredName(), transformVersion);
}
builder.endObject();
return builder;
}
Expand All @@ -155,15 +186,17 @@ public boolean equals(Object other) {
final DataFrameTransformConfig that = (DataFrameTransformConfig) other;

return Objects.equals(this.id, that.id)
&& Objects.equals(this.source, that.source)
&& Objects.equals(this.dest, that.dest)
&& Objects.equals(this.description, that.description)
&& Objects.equals(this.pivotConfig, that.pivotConfig);
&& Objects.equals(this.source, that.source)
&& Objects.equals(this.dest, that.dest)
&& Objects.equals(this.description, that.description)
&& Objects.equals(this.transformVersion, that.transformVersion)
&& Objects.equals(this.createTime, that.createTime)
&& Objects.equals(this.pivotConfig, that.pivotConfig);
}

@Override
public int hashCode() {
return Objects.hash(id, source, dest, pivotConfig, description);
return Objects.hash(id, source, dest, pivotConfig, description, createTime, transformVersion);
}

@Override
Expand Down Expand Up @@ -209,7 +242,7 @@ public Builder setDescription(String description) {
}

public DataFrameTransformConfig build() {
return new DataFrameTransformConfig(id, source, dest, pivotConfig, description);
return new DataFrameTransformConfig(id, source, dest, pivotConfig, description, null, null);
}
}
}
59 changes: 59 additions & 0 deletions ...high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.dataframe.transforms.util;

import org.elasticsearch.common.time.DateFormatters;
import org.elasticsearch.common.xcontent.XContentParser;

import java.io.IOException;
import java.time.Instant;
import java.time.format.DateTimeFormatter;
import java.util.Date;

public final class TimeUtil {

/**
* Parse out a Date object given the current parser and field name.
*
* @param parser current XContentParser
* @param fieldName the field's preferred name (utilized in exception)
* @return parsed Date object
* @throws IOException from XContentParser
*/
public static Date parseTimeField(XContentParser parser, String fieldName) throws IOException {
if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
return new Date(parser.longValue());
} else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
return new Date(DateFormatters.from(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant().toEpochMilli());
}
throw new IllegalArgumentException(
"unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]");
}

public static Instant parseTimeFieldToInstant(XContentParser parser, String fieldName) throws IOException {
if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
return Instant.ofEpochMilli(parser.longValue());
} else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
return DateFormatters.from(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant();
}
throw new IllegalArgumentException(
"unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]");
}

}
4 changes: 2 additions & 2 deletions client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ public void testGetTransform() throws IOException {
client::getDataFrameTransformAsync);
assertNull(getResponse.getInvalidTransforms());
assertThat(getResponse.getTransformConfigurations(), hasSize(1));
assertEquals(transform, getResponse.getTransformConfigurations().get(0));
assertEquals(transform.getId(), getResponse.getTransformConfigurations().get(0).getId());
}

public void testGetAllAndPageTransforms() throws IOException {
Expand All @@ -219,7 +219,7 @@ public void testGetAllAndPageTransforms() throws IOException {
client::getDataFrameTransformAsync);
assertNull(getResponse.getInvalidTransforms());
assertThat(getResponse.getTransformConfigurations(), hasSize(2));
assertEquals(transform, getResponse.getTransformConfigurations().get(1));
assertEquals(transform.getId(), getResponse.getTransformConfigurations().get(1).getId());

getRequest.setPageParams(new PageParams(0,1));
getResponse = execute(getRequest, client::getDataFrameTransform,
Expand Down
11 changes: 9 additions & 2 deletions ...est/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.client.dataframe.transforms;

import org.elasticsearch.Version;
import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfigTests;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
Expand All @@ -27,6 +28,7 @@
import org.elasticsearch.test.AbstractXContentTestCase;

import java.io.IOException;
import java.time.Instant;
import java.util.Collections;
import java.util.function.Predicate;

Expand All @@ -36,8 +38,13 @@
public class DataFrameTransformConfigTests extends AbstractXContentTestCase<DataFrameTransformConfig> {

public static DataFrameTransformConfig randomDataFrameTransformConfig() {
return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(),
randomDestConfig(), PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100));
return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10),
randomSourceConfig(),
randomDestConfig(),
PivotConfigTests.randomPivotConfig(),
randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100),
randomBoolean() ? null : Instant.now(),
randomBoolean() ? null : Version.CURRENT.toString());
}

@Override
Expand Down
2 changes: 0 additions & 2 deletions ...c/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -478,7 +478,6 @@ public void testGetStats() throws IOException, InterruptedException {

RestHighLevelClient client = highLevelClient();

QueryConfig queryConfig = new QueryConfig(new MatchAllQueryBuilder());
GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer",
TermsGroupSource.builder().setField("user_id").build()).build();
AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder();
Expand Down Expand Up @@ -564,7 +563,6 @@ public void onFailure(Exception e) {
public void testGetDataFrameTransform() throws IOException, InterruptedException {
createIndex("source-data");

QueryConfig queryConfig = new QueryConfig(new MatchAllQueryBuilder());
GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer",
TermsGroupSource.builder().setField("user_id").build()).build();
AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder();
Expand Down
Loading

0 comments on commit f4b75d6

Please sign in to comment.