diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsRequest.java index 14950a74c9187..2624b68a98318 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsRequest.java @@ -22,6 +22,7 @@ import org.elasticsearch.client.Validatable; import org.elasticsearch.client.ValidationException; import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -67,4 +68,9 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(config); } + + @Override + public String toString() { + return Strings.toString(this); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java index b1309e66afcd4..62adb06294558 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java @@ -19,11 +19,14 @@ package org.elasticsearch.client.ml.dataframe; +import org.elasticsearch.Version; +import org.elasticsearch.client.dataframe.transforms.util.TimeUtil; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -31,11 +34,9 @@ import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import java.io.IOException; +import java.time.Instant; import java.util.Objects; -import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING; -import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.VALUE; - public class DataFrameAnalyticsConfig implements ToXContentObject { public static DataFrameAnalyticsConfig fromXContent(XContentParser parser) { @@ -52,6 +53,8 @@ public static Builder builder(String id) { private static final ParseField ANALYSIS = new ParseField("analysis"); private static final ParseField ANALYZED_FIELDS = new ParseField("analyzed_fields"); private static final ParseField MODEL_MEMORY_LIMIT = new ParseField("model_memory_limit"); + private static final ParseField CREATE_TIME = new ParseField("create_time"); + private static final ParseField VERSION = new ParseField("version"); private static ObjectParser PARSER = new ObjectParser<>("data_frame_analytics_config", true, Builder::new); @@ -63,9 +66,24 @@ public static Builder builder(String id) { PARSER.declareField(Builder::setAnalyzedFields, (p, c) -> FetchSourceContext.fromXContent(p), ANALYZED_FIELDS, - OBJECT_ARRAY_BOOLEAN_OR_STRING); + ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING); PARSER.declareField(Builder::setModelMemoryLimit, - (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MODEL_MEMORY_LIMIT.getPreferredName()), MODEL_MEMORY_LIMIT, VALUE); + (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MODEL_MEMORY_LIMIT.getPreferredName()), + MODEL_MEMORY_LIMIT, + ValueType.VALUE); + PARSER.declareField(Builder::setCreateTime, + p -> TimeUtil.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()), + CREATE_TIME, + ValueType.VALUE); + PARSER.declareField(Builder::setVersion, + p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return Version.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, + VERSION, + ValueType.STRING); } private static DataFrameAnalysis parseAnalysis(XContentParser parser) throws IOException { @@ -82,15 +100,20 @@ private static DataFrameAnalysis parseAnalysis(XContentParser parser) throws IOE private final DataFrameAnalysis analysis; private final FetchSourceContext analyzedFields; private final ByteSizeValue modelMemoryLimit; + private final Instant createTime; + private final Version version; private DataFrameAnalyticsConfig(String id, DataFrameAnalyticsSource source, DataFrameAnalyticsDest dest, DataFrameAnalysis analysis, - @Nullable FetchSourceContext analyzedFields, @Nullable ByteSizeValue modelMemoryLimit) { + @Nullable FetchSourceContext analyzedFields, @Nullable ByteSizeValue modelMemoryLimit, + @Nullable Instant createTime, @Nullable Version version) { this.id = Objects.requireNonNull(id); this.source = Objects.requireNonNull(source); this.dest = Objects.requireNonNull(dest); this.analysis = Objects.requireNonNull(analysis); this.analyzedFields = analyzedFields; this.modelMemoryLimit = modelMemoryLimit; + this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli());; + this.version = version; } public String getId() { @@ -117,6 +140,14 @@ public ByteSizeValue getModelMemoryLimit() { return modelMemoryLimit; } + public Instant getCreateTime() { + return createTime; + } + + public Version getVersion() { + return version; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -132,6 +163,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (modelMemoryLimit != null) { builder.field(MODEL_MEMORY_LIMIT.getPreferredName(), modelMemoryLimit.getStringRep()); } + if (createTime != null) { + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); + } + if (version != null) { + builder.field(VERSION.getPreferredName(), version); + } builder.endObject(); return builder; } @@ -147,12 +184,14 @@ public boolean equals(Object o) { && Objects.equals(dest, other.dest) && Objects.equals(analysis, other.analysis) && Objects.equals(analyzedFields, other.analyzedFields) - && Objects.equals(modelMemoryLimit, other.modelMemoryLimit); + && Objects.equals(modelMemoryLimit, other.modelMemoryLimit) + && Objects.equals(createTime, other.createTime) + && Objects.equals(version, other.version); } @Override public int hashCode() { - return Objects.hash(id, source, dest, analysis, analyzedFields, getModelMemoryLimit()); + return Objects.hash(id, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version); } @Override @@ -168,6 +207,8 @@ public static class Builder { private DataFrameAnalysis analysis; private FetchSourceContext analyzedFields; private ByteSizeValue modelMemoryLimit; + private Instant createTime; + private Version version; private Builder() {} @@ -201,8 +242,18 @@ public Builder setModelMemoryLimit(ByteSizeValue modelMemoryLimit) { return this; } + public Builder setCreateTime(Instant createTime) { + this.createTime = createTime; + return this; + } + + public Builder setVersion(Version version) { + this.version = version; + return this; + } + public DataFrameAnalyticsConfig build() { - return new DataFrameAnalyticsConfig(id, source, dest, analysis, analyzedFields, modelMemoryLimit); + return new DataFrameAnalyticsConfig(id, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version); } } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java index 4eba642401054..fa8df5bfee94e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.client.ml.dataframe; +import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -29,6 +30,7 @@ import org.elasticsearch.test.AbstractXContentTestCase; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -54,6 +56,12 @@ public static DataFrameAnalyticsConfig randomDataFrameAnalyticsConfig() { if (randomBoolean()) { builder.setModelMemoryLimit(new ByteSizeValue(randomIntBetween(1, 16), randomFrom(ByteSizeUnit.MB, ByteSizeUnit.GB))); } + if (randomBoolean()) { + builder.setCreateTime(Instant.now()); + } + if (randomBoolean()) { + builder.setVersion(Version.CURRENT); + } return builder.build(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java index 0e9acdd44a2fe..99460a6883a56 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -5,7 +5,9 @@ */ package org.elasticsearch.xpack.core.ml.dataframe; +import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -17,12 +19,14 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.xpack.core.common.time.TimeUtils; import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import java.io.IOException; +import java.time.Instant; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -47,6 +51,8 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { public static final ParseField ANALYZED_FIELDS = new ParseField("analyzed_fields"); public static final ParseField MODEL_MEMORY_LIMIT = new ParseField("model_memory_limit"); public static final ParseField HEADERS = new ParseField("headers"); + public static final ParseField CREATE_TIME = new ParseField("create_time"); + public static final ParseField VERSION = new ParseField("version"); public static final ObjectParser STRICT_PARSER = createParser(false); public static final ObjectParser LENIENT_PARSER = createParser(true); @@ -69,6 +75,18 @@ public static ObjectParser createParser(boolean ignoreUnknownFiel // Headers are not parsed by the strict (config) parser, so headers supplied in the _body_ of a REST request will be rejected. // (For config, headers are explicitly transferred from the auth headers by code in the put data frame actions.) parser.declareObject(Builder::setHeaders, (p, c) -> p.mapStrings(), HEADERS); + // Creation time is set automatically during PUT, so create_time supplied in the _body_ of a REST request will be rejected. + parser.declareField(Builder::setCreateTime, + p -> TimeUtils.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()), + CREATE_TIME, + ObjectParser.ValueType.VALUE); + // Version is set automatically during PUT, so version supplied in the _body_ of a REST request will be rejected. + parser.declareField(Builder::setVersion, p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return Version.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, VERSION, ObjectParser.ValueType.STRING); } return parser; } @@ -96,10 +114,12 @@ private static DataFrameAnalysis parseAnalysis(XContentParser parser, boolean ig */ private final ByteSizeValue modelMemoryLimit; private final Map headers; + private final Instant createTime; + private final Version version; public DataFrameAnalyticsConfig(String id, DataFrameAnalyticsSource source, DataFrameAnalyticsDest dest, DataFrameAnalysis analysis, Map headers, ByteSizeValue modelMemoryLimit, - FetchSourceContext analyzedFields) { + FetchSourceContext analyzedFields, Instant createTime, Version version) { this.id = ExceptionsHelper.requireNonNull(id, ID); this.source = ExceptionsHelper.requireNonNull(source, SOURCE); this.dest = ExceptionsHelper.requireNonNull(dest, DEST); @@ -107,16 +127,25 @@ public DataFrameAnalyticsConfig(String id, DataFrameAnalyticsSource source, Data this.analyzedFields = analyzedFields; this.modelMemoryLimit = modelMemoryLimit; this.headers = Collections.unmodifiableMap(headers); + this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli());; + this.version = version; } public DataFrameAnalyticsConfig(StreamInput in) throws IOException { - id = in.readString(); - source = new DataFrameAnalyticsSource(in); - dest = new DataFrameAnalyticsDest(in); - analysis = in.readNamedWriteable(DataFrameAnalysis.class); + this.id = in.readString(); + this.source = new DataFrameAnalyticsSource(in); + this.dest = new DataFrameAnalyticsDest(in); + this.analysis = in.readNamedWriteable(DataFrameAnalysis.class); this.analyzedFields = in.readOptionalWriteable(FetchSourceContext::new); this.modelMemoryLimit = in.readOptionalWriteable(ByteSizeValue::new); this.headers = Collections.unmodifiableMap(in.readMap(StreamInput::readString, StreamInput::readString)); + if (in.getVersion().onOrAfter(Version.V_7_3_0)) { + createTime = in.readOptionalInstant(); + version = in.readBoolean() ? Version.readVersion(in) : null; + } else { + createTime = null; + version = null; + } } public String getId() { @@ -147,6 +176,14 @@ public Map getHeaders() { return headers; } + public Instant getCreateTime() { + return createTime; + } + + public Version getVersion() { + return version; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -168,6 +205,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { builder.field(HEADERS.getPreferredName(), headers); } + if (createTime != null) { + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); + } + if (version != null) { + builder.field(VERSION.getPreferredName(), version); + } builder.endObject(); return builder; } @@ -181,6 +224,15 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(analyzedFields); out.writeOptionalWriteable(modelMemoryLimit); out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeString); + if (out.getVersion().onOrAfter(Version.V_7_3_0)) { + out.writeOptionalInstant(createTime); + if (version != null) { + out.writeBoolean(true); + Version.writeVersion(version, out); + } else { + out.writeBoolean(false); + } + } } @Override @@ -195,12 +247,19 @@ public boolean equals(Object o) { && Objects.equals(analysis, other.analysis) && Objects.equals(headers, other.headers) && Objects.equals(getModelMemoryLimit(), other.getModelMemoryLimit()) - && Objects.equals(analyzedFields, other.analyzedFields); + && Objects.equals(analyzedFields, other.analyzedFields) + && Objects.equals(createTime, other.createTime) + && Objects.equals(version, other.version); } @Override public int hashCode() { - return Objects.hash(id, source, dest, analysis, headers, getModelMemoryLimit(), analyzedFields); + return Objects.hash(id, source, dest, analysis, headers, getModelMemoryLimit(), analyzedFields, createTime, version); + } + + @Override + public String toString() { + return Strings.toString(this); } public static String documentId(String id) { @@ -217,6 +276,8 @@ public static class Builder { private ByteSizeValue modelMemoryLimit; private ByteSizeValue maxModelMemoryLimit; private Map headers = Collections.emptyMap(); + private Instant createTime; + private Version version; public Builder() {} @@ -243,6 +304,8 @@ public Builder(DataFrameAnalyticsConfig config, ByteSizeValue maxModelMemoryLimi if (config.analyzedFields != null) { this.analyzedFields = new FetchSourceContext(true, config.analyzedFields.includes(), config.analyzedFields.excludes()); } + this.createTime = config.createTime; + this.version = config.version; } public String getId() { @@ -304,9 +367,19 @@ private void applyMaxModelMemoryLimit() { } } + public Builder setCreateTime(Instant createTime) { + this.createTime = createTime; + return this; + } + + public Builder setVersion(Version version) { + this.version = version; + return this; + } + public DataFrameAnalyticsConfig build() { applyMaxModelMemoryLimit(); - return new DataFrameAnalyticsConfig(id, source, dest, analysis, headers, modelMemoryLimit, analyzedFields); + return new DataFrameAnalyticsConfig(id, source, dest, analysis, headers, modelMemoryLimit, analyzedFields, createTime, version); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index bc69f4b5d5e20..75ce2d53315c3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -391,6 +391,10 @@ public static void addDatafeedConfigFields(XContentBuilder builder) throws IOExc .endObject(); } + /** + * {@link DataFrameAnalyticsConfig} mapping. + * Does not include mapping for CREATE_TIME as this mapping is added by {@link #addJobConfigFields} method. + */ public static void addDataFrameAnalyticsFields(XContentBuilder builder) throws IOException { builder.startObject(DataFrameAnalyticsConfig.ID.getPreferredName()) .field(TYPE, KEYWORD) @@ -434,6 +438,10 @@ public static void addDataFrameAnalyticsFields(XContentBuilder builder) throws I .endObject() .endObject() .endObject() + .endObject() + // re-used: CREATE_TIME + .startObject(DataFrameAnalyticsConfig.VERSION.getPreferredName()) + .field(TYPE, KEYWORD) .endObject(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index 39036abb693b0..eff33a37d9773 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -277,6 +277,8 @@ public final class ReservedFieldNames { DataFrameAnalyticsConfig.DEST.getPreferredName(), DataFrameAnalyticsConfig.ANALYSIS.getPreferredName(), DataFrameAnalyticsConfig.ANALYZED_FIELDS.getPreferredName(), + DataFrameAnalyticsConfig.CREATE_TIME.getPreferredName(), + DataFrameAnalyticsConfig.VERSION.getPreferredName(), DataFrameAnalyticsDest.INDEX.getPreferredName(), DataFrameAnalyticsDest.RESULTS_FIELD.getPreferredName(), DataFrameAnalyticsSource.INDEX.getPreferredName(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java index a5df1f83c3d37..518950b675c19 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java @@ -8,6 +8,7 @@ import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; @@ -17,6 +18,7 @@ import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; @@ -30,16 +32,18 @@ import org.elasticsearch.xpack.core.ml.dataframe.analyses.MlDataFrameAnalysisNamedXContentProvider; import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetectionTests; import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; +import org.junit.Before; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasSize; @@ -49,7 +53,11 @@ public class DataFrameAnalyticsConfigTests extends AbstractSerializingTestCase dataFrameAnalyticsConfigParser = + lenient + ? DataFrameAnalyticsConfig.LENIENT_PARSER + : DataFrameAnalyticsConfig.STRICT_PARSER; + return dataFrameAnalyticsConfigParser.apply(parser, null).build(); } @Override @@ -70,7 +78,7 @@ protected NamedXContentRegistry xContentRegistry() { @Override protected DataFrameAnalyticsConfig createTestInstance() { - return createRandom(randomValidId()); + return createRandom(randomValidId(), lenient); } @Override @@ -79,10 +87,18 @@ protected Writeable.Reader instanceReader() { } public static DataFrameAnalyticsConfig createRandom(String id) { - return createRandomBuilder(id).build(); + return createRandom(id, false); + } + + public static DataFrameAnalyticsConfig createRandom(String id, boolean withGeneratedFields) { + return createRandomBuilder(id, withGeneratedFields).build(); } public static DataFrameAnalyticsConfig.Builder createRandomBuilder(String id) { + return createRandomBuilder(id, false); + } + + public static DataFrameAnalyticsConfig.Builder createRandomBuilder(String id, boolean withGeneratedFields) { DataFrameAnalyticsSource source = DataFrameAnalyticsSourceTests.createRandom(); DataFrameAnalyticsDest dest = DataFrameAnalyticsDestTests.createRandom(); DataFrameAnalyticsConfig.Builder builder = new DataFrameAnalyticsConfig.Builder() @@ -98,6 +114,14 @@ public static DataFrameAnalyticsConfig.Builder createRandomBuilder(String id) { if (randomBoolean()) { builder.setModelMemoryLimit(new ByteSizeValue(randomIntBetween(1, 16), randomFrom(ByteSizeUnit.MB, ByteSizeUnit.GB))); } + if (withGeneratedFields) { + if (randomBoolean()) { + builder.setCreateTime(Instant.now()); + } + if (randomBoolean()) { + builder.setVersion(Version.CURRENT); + } + } return builder; } @@ -122,6 +146,13 @@ public static String randomValidId() { " \"analysis\": {\"outlier_detection\": {\"n_neighbors\": 10}}\n" + "}"; + private boolean lenient; + + @Before + public void chooseStrictOrLenient() { + lenient = randomBoolean(); + } + public void testQueryConfigStoresUserInputOnly() throws IOException { try (XContentParser parser = XContentFactory.xContent(XContentType.JSON) .createParser(xContentRegistry(), @@ -245,6 +276,36 @@ public void testExplicitModelMemoryLimitTooHigh() { assertThat(e.getMessage(), containsString("must be less than the value of the xpack.ml.max_model_memory_limit setting")); } + public void testPreventCreateTimeInjection() throws IOException { + String json = "{" + + " \"create_time\" : 123456789 }," + + " \"source\" : {\"index\":\"src\"}," + + " \"dest\" : {\"index\": \"dest\"}," + + "}"; + + try (XContentParser parser = + XContentFactory.xContent(XContentType.JSON).createParser( + xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json)) { + Exception e = expectThrows(IllegalArgumentException.class, () -> DataFrameAnalyticsConfig.STRICT_PARSER.apply(parser, null)); + assertThat(e.getMessage(), containsString("unknown field [create_time], parser not found")); + } + } + + public void testPreventVersionInjection() throws IOException { + String json = "{" + + " \"version\" : \"7.3.0\"," + + " \"source\" : {\"index\":\"src\"}," + + " \"dest\" : {\"index\": \"dest\"}," + + "}"; + + try (XContentParser parser = + XContentFactory.xContent(XContentType.JSON).createParser( + xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json)) { + Exception e = expectThrows(IllegalArgumentException.class, () -> DataFrameAnalyticsConfig.STRICT_PARSER.apply(parser, null)); + assertThat(e.getMessage(), containsString("unknown field [version], parser not found")); + } + } + public void assertTooSmall(IllegalArgumentException e) { assertThat(e.getMessage(), is("[model_memory_limit] must be at least [1mb]")); } diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index af500c1dd8568..643d736e6b1de 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -39,6 +39,8 @@ integTest.runner { 'ml/datafeeds_crud/Test put datafeed with security headers in the body', 'ml/datafeeds_crud/Test update datafeed with missing id', 'ml/data_frame_analytics_crud/Test put config with security headers in the body', + 'ml/data_frame_analytics_crud/Test put config with create_time in the body', + 'ml/data_frame_analytics_crud/Test put config with version in the body', 'ml/data_frame_analytics_crud/Test put config with inconsistent body/param ids', 'ml/data_frame_analytics_crud/Test put config with invalid id', 'ml/data_frame_analytics_crud/Test put config with invalid dest index name', diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java index 0f709b4e16680..d8f5dbb469f5f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.ml.action; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; @@ -41,6 +42,7 @@ import org.elasticsearch.xpack.ml.dataframe.persistence.DataFrameAnalyticsConfigProvider; import java.io.IOException; +import java.time.Instant; import java.util.Objects; import java.util.function.Supplier; @@ -91,7 +93,10 @@ protected void doExecute(Task task, PutDataFrameAnalyticsAction.Request request, } validateConfig(request.getConfig()); DataFrameAnalyticsConfig memoryCappedConfig = - new DataFrameAnalyticsConfig.Builder(request.getConfig(), maxModelMemoryLimit).build(); + new DataFrameAnalyticsConfig.Builder(request.getConfig(), maxModelMemoryLimit) + .setCreateTime(Instant.now()) + .setVersion(Version.CURRENT) + .build(); if (licenseState.isAuthAllowed()) { final String username = securityContext.getUser().principal(); RoleDescriptor.IndicesPrivileges sourceIndexPrivileges = RoleDescriptor.IndicesPrivileges.builder() @@ -156,5 +161,6 @@ private void validateConfig(DataFrameAnalyticsConfig config) { } config.getDest().validate(); new SourceDestValidator(clusterService.state(), indexNameExpressionResolver).check(config); + } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml index 307ecda231b16..bfde8128b491c 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml @@ -453,3 +453,41 @@ setup: "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} } } + +--- +"Test put valid config with create_time in the body": + + - do: + catch: /Found \[create_time\], not allowed for strict parsing/ + data_frame.put_data_frame_transform: + transform_id: "airline-transform-with-create-time" + body: > + { + "source": { "index": "airline-data" }, + "dest": { "index": "airline-data-by-airline" }, + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + }, + "description": "yaml test transform on airline-data", + "create_time": 123456789 + } + +--- +"Test put valid config with version in the body": + + - do: + catch: /Found \[version\], not allowed for strict parsing/ + data_frame.put_data_frame_transform: + transform_id: "airline-transform-with-version" + body: > + { + "source": { "index": "airline-data" }, + "dest": { "index": "airline-data-by-airline" }, + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + }, + "description": "yaml test transform on airline-data", + "version": "7.3.0" + } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml index e5a68fb33834e..01afb7714f395 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml @@ -55,6 +55,21 @@ setup: - match: { dest.index: "index-dest" } - match: { analysis: {"outlier_detection":{}} } - match: { analyzed_fields: {"includes" : ["obj1.*", "obj2.*" ], "excludes": [] } } + - is_true: create_time + - is_true: version + + - do: + ml.get_data_frame_analytics: + id: "simple-outlier-detection-with-query" + - match: { count: 1 } + - match: { data_frame_analytics.0.id: "simple-outlier-detection-with-query" } + - match: { data_frame_analytics.0.source.index: "index-source" } + - match: { data_frame_analytics.0.source.query: {"term" : { "user" : "Kimchy"} } } + - match: { data_frame_analytics.0.dest.index: "index-dest" } + - match: { data_frame_analytics.0.analysis: {"outlier_detection":{}} } + - match: { data_frame_analytics.0.analyzed_fields: {"includes" : ["obj1.*", "obj2.*" ], "excludes": [] } } + - is_true: data_frame_analytics.0.create_time + - is_true: data_frame_analytics.0.version --- "Test put config with security headers in the body": @@ -75,6 +90,44 @@ setup: "headers":{ "a_security_header" : "secret" } } +--- +"Test put config with create_time in the body": + + - do: + catch: /unknown field \[create_time\], parser not found/ + ml.put_data_frame_analytics: + id: "data_frame_with_create_time" + body: > + { + "source": { + "index": "index-source" + }, + "dest": { + "index": "index-dest" + }, + "analysis": {"outlier_detection":{}}, + "create_time": 123456789 + } + +--- +"Test put config with version in the body": + + - do: + catch: /unknown field \[version\], parser not found/ + ml.put_data_frame_analytics: + id: "data_frame_with_version" + body: > + { + "source": { + "index": "index-source" + }, + "dest": { + "index": "index-dest" + }, + "analysis": {"outlier_detection":{}}, + "version": "7.3.0" + } + --- "Test put valid config with default outlier detection": @@ -96,6 +149,8 @@ setup: - match: { source.query: {"match_all" : {} } } - match: { dest.index: "index-dest" } - match: { analysis: {"outlier_detection":{}} } + - is_true: create_time + - is_true: version --- "Test put valid config with custom outlier detection": @@ -126,6 +181,8 @@ setup: - match: { analysis.outlier_detection.n_neighbors: 5 } - match: { analysis.outlier_detection.method: "lof" } - match: { analysis.outlier_detection.minimum_score_to_write_feature_influence: 0.0 } + - is_true: create_time + - is_true: version --- "Test put config with inconsistent body/param ids":