Skip to content

Commit

Permalink
[7.x] [Transform] Add _meta field to TransformConfig (elastic#79003) (e…
Browse files Browse the repository at this point in the history 
…lastic#79219)
  • Loading branch information
@przemekwitek
przemekwitek authored Oct 15, 2021
1 parent 54c3716 commit aee722d
Show file tree
Hide file tree
Showing 26 changed files with 445 additions and 44 deletions.
36 changes: 31 additions & 5 deletions ...gh-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

import java.io.IOException;
import java.time.Instant;
import java.util.Map;
import java.util.Objects;

import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
Expand All @@ -38,6 +39,7 @@ public class TransformConfig implements ToXContentObject {
public static final ParseField DESCRIPTION = new ParseField("description");
public static final ParseField SYNC = new ParseField("sync");
public static final ParseField SETTINGS = new ParseField("settings");
public static final ParseField METADATA = new ParseField("_meta");
public static final ParseField VERSION = new ParseField("version");
public static final ParseField CREATE_TIME = new ParseField("create_time");
public static final ParseField RETENTION_POLICY = new ParseField("retention_policy");
Expand All @@ -51,6 +53,7 @@ public class TransformConfig implements ToXContentObject {
private final TimeValue frequency;
private final SyncConfig syncConfig;
private final SettingsConfig settings;
private final Map<String, Object> metadata;
private final PivotConfig pivotConfig;
private final LatestConfig latestConfig;
private final String description;
Expand All @@ -71,9 +74,11 @@ public class TransformConfig implements ToXContentObject {
LatestConfig latestConfig = (LatestConfig) args[6];
String description = (String) args[7];
SettingsConfig settings = (SettingsConfig) args[8];
RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[9];
Instant createTime = (Instant) args[10];
String transformVersion = (String) args[11];
@SuppressWarnings("unchecked")
Map<String, Object> metadata = (Map<String, Object>) args[9];
RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[10];
Instant createTime = (Instant) args[11];
String transformVersion = (String) args[12];
return new TransformConfig(
id,
source,
Expand All @@ -84,6 +89,7 @@ public class TransformConfig implements ToXContentObject {
latestConfig,
description,
settings,
metadata,
retentionPolicyConfig,
createTime,
transformVersion
Expand All @@ -106,6 +112,7 @@ public class TransformConfig implements ToXContentObject {
PARSER.declareObject(optionalConstructorArg(), (p, c) -> LatestConfig.fromXContent(p), LATEST_TRANSFORM);
PARSER.declareString(optionalConstructorArg(), DESCRIPTION);
PARSER.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p), SETTINGS);
PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.mapOrdered(), METADATA);
PARSER.declareNamedObject(
optionalConstructorArg(),
(p, c, n) -> p.namedObject(RetentionPolicyConfig.class, n, c),
Expand Down Expand Up @@ -136,7 +143,7 @@ public static TransformConfig fromXContent(final XContentParser parser) {
* @return A TransformConfig to preview, NOTE it will have a {@code null} id, destination and index.
*/
public static TransformConfig forPreview(final SourceConfig source, final PivotConfig pivotConfig) {
return new TransformConfig(null, source, null, null, null, pivotConfig, null, null, null, null, null, null);
return new TransformConfig(null, source, null, null, null, pivotConfig, null, null, null, null, null, null, null);
}

/**
Expand All @@ -151,7 +158,7 @@ public static TransformConfig forPreview(final SourceConfig source, final PivotC
* @return A TransformConfig to preview, NOTE it will have a {@code null} id, destination and index.
*/
public static TransformConfig forPreview(final SourceConfig source, final LatestConfig latestConfig) {
return new TransformConfig(null, source, null, null, null, null, latestConfig, null, null, null, null, null);
return new TransformConfig(null, source, null, null, null, null, latestConfig, null, null, null, null, null, null);
}

TransformConfig(
Expand All @@ -164,6 +171,7 @@ public static TransformConfig forPreview(final SourceConfig source, final Latest
final LatestConfig latestConfig,
final String description,
final SettingsConfig settings,
final Map<String, Object> metadata,
final RetentionPolicyConfig retentionPolicyConfig,
final Instant createTime,
final String version
Expand All @@ -177,6 +185,7 @@ public static TransformConfig forPreview(final SourceConfig source, final Latest
this.latestConfig = latestConfig;
this.description = description;
this.settings = settings;
this.metadata = metadata;
this.retentionPolicyConfig = retentionPolicyConfig;
this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli());
this.transformVersion = version == null ? null : Version.fromString(version);
Expand Down Expand Up @@ -228,6 +237,11 @@ public SettingsConfig getSettings() {
return settings;
}

@Nullable
public Map<String, Object> getMetadata() {
return metadata;
}

@Nullable
public RetentionPolicyConfig getRetentionPolicyConfig() {
return retentionPolicyConfig;
Expand Down Expand Up @@ -265,6 +279,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
if (settings != null) {
builder.field(SETTINGS.getPreferredName(), settings);
}
if (metadata != null) {
builder.field(METADATA.getPreferredName(), metadata);
}
if (retentionPolicyConfig != null) {
builder.startObject(RETENTION_POLICY.getPreferredName());
builder.field(retentionPolicyConfig.getName(), retentionPolicyConfig);
Expand Down Expand Up @@ -300,6 +317,7 @@ public boolean equals(Object other) {
&& Objects.equals(this.syncConfig, that.syncConfig)
&& Objects.equals(this.transformVersion, that.transformVersion)
&& Objects.equals(this.settings, that.settings)
&& Objects.equals(this.metadata, that.metadata)
&& Objects.equals(this.createTime, that.createTime)
&& Objects.equals(this.pivotConfig, that.pivotConfig)
&& Objects.equals(this.latestConfig, that.latestConfig)
Expand All @@ -315,6 +333,7 @@ public int hashCode() {
frequency,
syncConfig,
settings,
metadata,
createTime,
transformVersion,
pivotConfig,
Expand Down Expand Up @@ -343,6 +362,7 @@ public static class Builder {
private PivotConfig pivotConfig;
private LatestConfig latestConfig;
private SettingsConfig settings;
private Map<String, Object> metadata;
private String description;
private RetentionPolicyConfig retentionPolicyConfig;

Expand Down Expand Up @@ -391,6 +411,11 @@ public Builder setSettings(SettingsConfig settings) {
return this;
}

public Builder setMetadata(Map<String, Object> metadata) {
this.metadata = metadata;
return this;
}

public Builder setRetentionPolicyConfig(RetentionPolicyConfig retentionPolicyConfig) {
this.retentionPolicyConfig = retentionPolicyConfig;
return this;
Expand All @@ -407,6 +432,7 @@ public TransformConfig build() {
latestConfig,
description,
settings,
metadata,
retentionPolicyConfig,
null,
null
Expand Down
32 changes: 27 additions & 5 deletions ...el/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfigUpdate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.elasticsearch.xcontent.XContentParser;

import java.io.IOException;
import java.util.Map;
import java.util.Objects;

import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
Expand All @@ -39,8 +40,10 @@ public class TransformConfigUpdate implements ToXContentObject {
SyncConfig syncConfig = (SyncConfig) args[3];
String description = (String) args[4];
SettingsConfig settings = (SettingsConfig) args[5];
RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[6];
return new TransformConfigUpdate(source, dest, frequency, syncConfig, description, settings, retentionPolicyConfig);
@SuppressWarnings("unchecked")
Map<String, Object> metadata = (Map<String, Object>) args[6];
RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[7];
return new TransformConfigUpdate(source, dest, frequency, syncConfig, description, settings, metadata, retentionPolicyConfig);
}
);

Expand All @@ -51,6 +54,7 @@ public class TransformConfigUpdate implements ToXContentObject {
PARSER.declareNamedObject(optionalConstructorArg(), (p, c, n) -> p.namedObject(SyncConfig.class, n, c), TransformConfig.SYNC);
PARSER.declareString(optionalConstructorArg(), TransformConfig.DESCRIPTION);
PARSER.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p), TransformConfig.SETTINGS);
PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.mapOrdered(), TransformConfig.METADATA);
PARSER.declareNamedObject(
optionalConstructorArg(),
(p, c, n) -> p.namedObject(RetentionPolicyConfig.class, n, c),
Expand All @@ -64,6 +68,7 @@ public class TransformConfigUpdate implements ToXContentObject {
private final SyncConfig syncConfig;
private final String description;
private final SettingsConfig settings;
private final Map<String, Object> metadata;

public TransformConfigUpdate(
final SourceConfig source,
Expand All @@ -72,6 +77,7 @@ public TransformConfigUpdate(
final SyncConfig syncConfig,
final String description,
final SettingsConfig settings,
final Map<String, Object> metadata,
final RetentionPolicyConfig retentionPolicyConfig
) {
this.source = source;
Expand All @@ -80,6 +86,7 @@ public TransformConfigUpdate(
this.syncConfig = syncConfig;
this.description = description;
this.settings = settings;
this.metadata = metadata;
}

public SourceConfig getSource() {
Expand Down Expand Up @@ -108,6 +115,11 @@ public SettingsConfig getSettings() {
return settings;
}

@Nullable
public Map<String, Object> getMetadata() {
return metadata;
}

@Override
public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
builder.startObject();
Expand All @@ -131,6 +143,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
if (settings != null) {
builder.field(TransformConfig.SETTINGS.getPreferredName(), settings);
}
if (metadata != null) {
builder.field(TransformConfig.METADATA.getPreferredName(), metadata);
}

builder.endObject();
return builder;
Expand All @@ -153,12 +168,13 @@ public boolean equals(Object other) {
&& Objects.equals(this.frequency, that.frequency)
&& Objects.equals(this.syncConfig, that.syncConfig)
&& Objects.equals(this.description, that.description)
&& Objects.equals(this.settings, that.settings);
&& Objects.equals(this.settings, that.settings)
&& Objects.equals(this.metadata, that.metadata);
}

@Override
public int hashCode() {
return Objects.hash(source, dest, frequency, syncConfig, description, settings);
return Objects.hash(source, dest, frequency, syncConfig, description, settings, metadata);
}

@Override
Expand All @@ -182,6 +198,7 @@ public static class Builder {
private SyncConfig syncConfig;
private String description;
private SettingsConfig settings;
private Map<String, Object> metdata;
private RetentionPolicyConfig retentionPolicyConfig;

public Builder setSource(SourceConfig source) {
Expand Down Expand Up @@ -214,13 +231,18 @@ public Builder setSettings(SettingsConfig settings) {
return this;
}

public Builder setMetadata(Map<String, Object> metadata) {
this.metdata = metdata;
return this;
}

public Builder setRetentionPolicyConfig(RetentionPolicyConfig retentionPolicyConfig) {
this.retentionPolicyConfig = retentionPolicyConfig;
return this;
}

public TransformConfigUpdate build() {
return new TransformConfigUpdate(source, dest, frequency, syncConfig, description, settings, retentionPolicyConfig);
return new TransformConfigUpdate(source, dest, frequency, syncConfig, description, settings, metdata, retentionPolicyConfig);
}
}
}
27 changes: 27 additions & 0 deletions ...vel/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.elasticsearch.client.transform.transforms.pivot.PivotConfigTests;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.xcontent.NamedXContentRegistry;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.search.SearchModule;
Expand All @@ -25,6 +26,7 @@
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;

import static org.elasticsearch.client.transform.transforms.DestConfigTests.randomDestConfig;
Expand Down Expand Up @@ -52,6 +54,7 @@ public static TransformConfig randomTransformConfig() {
latestConfig,
randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100),
SettingsConfigTests.randomSettingsConfig(),
randomMetadata(),
randomBoolean() ? null : randomRetentionPolicyConfig(),
randomBoolean() ? null : Instant.now(),
randomBoolean() ? null : Version.CURRENT.toString()
Expand All @@ -66,6 +69,30 @@ public static RetentionPolicyConfig randomRetentionPolicyConfig() {
return TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig();
}

public static Map<String, Object> randomMetadata() {
return randomMap(0, 10, () -> {
String key = randomAlphaOfLengthBetween(1, 10);
Object value;
switch (randomIntBetween(0, 3)) {
case 0:
value = null;
break;
case 1:
value = randomLong();
break;
case 2:
value = randomAlphaOfLengthBetween(1, 10);
break;
case 3:
value = randomMap(0, 10, () -> Tuple.tuple(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)));
break;
default:
throw new AssertionError();
}
return Tuple.tuple(key, value);
});
}

@Override
protected TransformConfig createTestInstance() {
return randomTransformConfig();
Expand Down
2 changes: 2 additions & 0 deletions ...c/test/java/org/elasticsearch/client/transform/transforms/TransformConfigUpdateTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import static org.elasticsearch.client.transform.transforms.DestConfigTests.randomDestConfig;
import static org.elasticsearch.client.transform.transforms.SettingsConfigTests.randomSettingsConfig;
import static org.elasticsearch.client.transform.transforms.SourceConfigTests.randomSourceConfig;
import static org.elasticsearch.client.transform.transforms.TransformConfigTests.randomMetadata;
import static org.elasticsearch.client.transform.transforms.TransformConfigTests.randomRetentionPolicyConfig;
import static org.elasticsearch.client.transform.transforms.TransformConfigTests.randomSyncConfig;

Expand All @@ -36,6 +37,7 @@ public static TransformConfigUpdate randomTransformConfigUpdate() {
randomBoolean() ? null : randomSyncConfig(),
randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000),
randomBoolean() ? null : randomSettingsConfig(),
randomBoolean() ? null : randomMetadata(),
randomBoolean() ? null : randomRetentionPolicyConfig()
);
}
Expand Down
4 changes: 4 additions & 0 deletions docs/reference/rest-api/common-parms.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -984,6 +984,10 @@ The `latest` method transforms the data by finding the latest document for each
unique key.
end::transform-latest[]

tag::transform-metadata[]
Defines optional {transform} metadata.
end::transform-metadata[]

tag::transform-retention[]
Defines a retention policy for the {transform}. Data that meets the defined
criteria is deleted from the destination index.
Expand Down
6 changes: 6 additions & 0 deletions docs/reference/transform/apis/put-transform.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,12 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-unique-key]
====
//End latest

//Begin _meta
`_meta`::
(Optional, object)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-metadata]
//End _meta

//Begin pivot
`pivot`::
(Required^*^, object)
Expand Down
6 changes: 6 additions & 0 deletions docs/reference/transform/apis/update-transform.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,12 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-pipeline]
(Optional, <<time-units, time units>>)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=frequency]

//Begin _meta
`_meta`::
(Optional, object)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-metadata]
//End _meta

//Begin retention policy
`retention_policy`::
(Optional, object)
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public final class TransformField {
public static final ParseField CREATE_TIME = new ParseField("create_time");
public static final ParseField DESTINATION = new ParseField("dest");
public static final ParseField SETTINGS = new ParseField("settings");
public static final ParseField METADATA = new ParseField("_meta");
public static final ParseField FREQUENCY = new ParseField("frequency");
public static final ParseField FORCE = new ParseField("force");
public static final ParseField MAX_PAGE_SEARCH_SIZE = new ParseField("max_page_search_size");
Expand Down
Loading

0 comments on commit aee722d

Please sign in to comment.