diff --git a/docs/reference/rollup/apis/rollup-api.asciidoc b/docs/reference/rollup/apis/rollup-api.asciidoc index f02a70f4f477c..853df9d7584d1 100644 --- a/docs/reference/rollup/apis/rollup-api.asciidoc +++ b/docs/reference/rollup/apis/rollup-api.asciidoc @@ -11,9 +11,8 @@ For example, you can roll up hourly data into daily or weekly summaries. [source,console] ---- -POST /my-index-000001/_rollup +POST /my-index-000001/_rollup/my-rollup-index { - "rollup_index": "my-rollup-index", "groups": { "date_histogram": { "field": "@timestamp", @@ -44,7 +43,7 @@ POST /my-index-000001/_rollup [[rollup-api-request]] ==== {api-request-title} -`PUT //_rollup` +`PUT //_rollup/` [[rollup-api-prereqs]] ==== {api-prereq-title} @@ -61,11 +60,7 @@ Index to roll up. Cannot be a <> or <>. Does not support <> or wildcards (`*`). -[role="child_attributes"] -[[rollup-api-request-body]] -==== {api-request-body-title} - -`rollup_index`:: +``:: (Required, string) New index that stores the rollup results. Cannot be an existing index, a <>, or an <>. @@ -75,6 +70,10 @@ The request creates this index with `` is a backing index for a data stream, this index is a backing index for the same stream. +[role="child_attributes"] +[[rollup-api-request-body]] +==== {api-request-body-title} + `groups`:: (Required, object) Aggregates and stores fields in the rollup. @@ -105,7 +104,7 @@ Time interval used to group documents. For differences between TIP: Choose this value carefully. You won't be able to use a smaller interval later. For example, you can't aggregate daily rollups into hourly summaries. However, smaller time intervals can greatly increase the size of your -`rollup_index`. +``. `time_zone`:: (Optional, string) @@ -147,7 +146,7 @@ Array of <> and <> fields to store. If you specify a `terms` object, this property is required. + TIP: Avoid storing high-cardinality fields. High-cardinality fields can greatly -increase the size of your `rollup_index`. +increase the size of your ``. ===== ==== diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/v2/RollupAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/v2/RollupAction.java index d1f7458ce9d95..5439ec62ea2a7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/v2/RollupAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/v2/RollupAction.java @@ -35,10 +35,12 @@ private RollupAction() { public static class Request extends ActionRequest implements ToXContentObject { private String sourceIndex; + private String rollupIndex; private RollupActionConfig rollupConfig; - public Request(String sourceIndex, RollupActionConfig rollupConfig) { + public Request(String sourceIndex, String rollupIndex, RollupActionConfig rollupConfig) { this.sourceIndex = sourceIndex; + this.rollupIndex = rollupIndex; this.rollupConfig = rollupConfig; } @@ -47,18 +49,20 @@ public Request() {} public Request(StreamInput in) throws IOException { super(in); sourceIndex = in.readString(); + rollupIndex = in.readString(); rollupConfig = new RollupActionConfig(in); } @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { - return new RollupTask(id, type, action, parentTaskId, rollupConfig, headers); + return new RollupTask(id, type, action, parentTaskId, rollupIndex, rollupConfig, headers); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(sourceIndex); + out.writeString(rollupIndex); rollupConfig.writeTo(out); } @@ -66,6 +70,10 @@ public String getSourceIndex() { return sourceIndex; } + public String getRollupIndex() { + return rollupIndex; + } + public RollupActionConfig getRollupConfig() { return rollupConfig; } @@ -79,6 +87,7 @@ public ActionRequestValidationException validate() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("source_index", sourceIndex); + builder.field("rollup_index", rollupIndex); rollupConfig.toXContent(builder, params); builder.endObject(); return builder; @@ -86,7 +95,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public int hashCode() { - return Objects.hash(sourceIndex, rollupConfig); + return Objects.hash(sourceIndex, rollupIndex, rollupConfig); } @Override @@ -99,6 +108,7 @@ public boolean equals(Object obj) { } Request other = (Request) obj; return Objects.equals(sourceIndex, other.sourceIndex) + && Objects.equals(rollupIndex, other.rollupIndex) && Objects.equals(rollupConfig, other.rollupConfig); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/v2/RollupActionConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/v2/RollupActionConfig.java index 48622e4618815..d7b68775c9188 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/v2/RollupActionConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/v2/RollupActionConfig.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.xpack.core.rollup.RollupField; import org.elasticsearch.xpack.core.rollup.job.GroupConfig; import org.elasticsearch.xpack.core.rollup.job.MetricConfig; @@ -31,7 +30,6 @@ import java.util.Objects; import java.util.Set; -import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; /** @@ -43,59 +41,41 @@ public class RollupActionConfig implements NamedWriteable, ToXContentObject { private static final String NAME = "xpack/rollup/action/config"; private static final TimeValue DEFAULT_TIMEOUT = TimeValue.timeValueSeconds(20); private static final String TIMEOUT = "timeout"; - private static final String ROLLUP_INDEX = "rollup_index"; private final GroupConfig groupConfig; private final List metricsConfig; private final TimeValue timeout; - private String rollupIndex; private static final ConstructingObjectParser PARSER; static { PARSER = new ConstructingObjectParser<>(NAME, false, (args) -> { - String rollupIndex = (String) args[0]; - GroupConfig groupConfig = (GroupConfig) args[1]; + GroupConfig groupConfig = (GroupConfig) args[0]; @SuppressWarnings("unchecked") - List metricsConfig = (List) args[2]; - TimeValue timeout = (TimeValue) args[3]; - return new RollupActionConfig(groupConfig, metricsConfig, timeout, rollupIndex); + List metricsConfig = (List) args[1]; + TimeValue timeout = (TimeValue) args[2]; + return new RollupActionConfig(groupConfig, metricsConfig, timeout); }); - PARSER.declareString(constructorArg(), new ParseField(ROLLUP_INDEX)); PARSER.declareObject(optionalConstructorArg(), (p, c) -> GroupConfig.fromXContent(p), new ParseField(GroupConfig.NAME)); PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> MetricConfig.fromXContent(p), new ParseField(MetricConfig.NAME)); PARSER.declareField(optionalConstructorArg(), (p, c) -> TimeValue.parseTimeValue(p.textOrNull(), TIMEOUT), new ParseField(TIMEOUT), ObjectParser.ValueType.STRING_OR_NULL); } - public RollupActionConfig(final GroupConfig groupConfig, final List metricsConfig, - final @Nullable TimeValue timeout, final String rollupIndex) { - if (rollupIndex == null || rollupIndex.isEmpty()) { - throw new IllegalArgumentException("Rollup index must be a non-null, non-empty string"); - } + public RollupActionConfig(final GroupConfig groupConfig, final List metricsConfig, final @Nullable TimeValue timeout) { if (groupConfig == null && (metricsConfig == null || metricsConfig.isEmpty())) { throw new IllegalArgumentException("At least one grouping or metric must be configured"); } - this.rollupIndex = rollupIndex; this.groupConfig = groupConfig; this.metricsConfig = metricsConfig != null ? metricsConfig : Collections.emptyList(); this.timeout = timeout != null ? timeout : DEFAULT_TIMEOUT; } public RollupActionConfig(final StreamInput in) throws IOException { - rollupIndex = in.readString(); groupConfig = in.readOptionalWriteable(GroupConfig::new); metricsConfig = in.readList(MetricConfig::new); timeout = in.readTimeValue(); } - public String getId() { - return RollupField.NAME + "_" + rollupIndex; - } - - public void setRollupIndex(String rollupIndex) { - this.rollupIndex = rollupIndex; - } - public GroupConfig getGroupConfig() { return groupConfig; } @@ -108,10 +88,6 @@ public TimeValue getTimeout() { return timeout; } - public String getRollupIndex() { - return rollupIndex; - } - @Override public String getWriteableName() { return NAME; @@ -142,7 +118,6 @@ public void validateMappings(final Map> f public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { builder.startObject(); { - builder.field(ROLLUP_INDEX, rollupIndex); if (groupConfig != null) { builder.field(GroupConfig.NAME, groupConfig); } @@ -163,7 +138,6 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa @Override public void writeTo(final StreamOutput out) throws IOException { - out.writeString(rollupIndex); out.writeOptionalWriteable(groupConfig); out.writeList(metricsConfig); out.writeTimeValue(timeout); @@ -179,15 +153,14 @@ public boolean equals(Object other) { } final RollupActionConfig that = (RollupActionConfig) other; - return Objects.equals(this.rollupIndex, that.rollupIndex) - && Objects.equals(this.groupConfig, that.groupConfig) + return Objects.equals(this.groupConfig, that.groupConfig) && Objects.equals(this.metricsConfig, that.metricsConfig) && Objects.equals(this.timeout, that.timeout); } @Override public int hashCode() { - return Objects.hash(rollupIndex, groupConfig, metricsConfig, timeout); + return Objects.hash(groupConfig, metricsConfig, timeout); } @Override @@ -195,13 +168,6 @@ public String toString() { return Strings.toString(this, true, true); } - /** - * Same as toString() but more explicitly named so the caller knows this is turned into JSON - */ - public String toJSONString() { - return toString(); - } - public static RollupActionConfig fromXContent(final XContentParser parser) throws IOException { return PARSER.parse(parser, null); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/v2/RollupTask.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/v2/RollupTask.java index 6e6f7b68ba04d..6b2874d67bec5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/v2/RollupTask.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/v2/RollupTask.java @@ -5,8 +5,6 @@ */ package org.elasticsearch.xpack.core.rollup.v2; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.xpack.core.rollup.RollupField; @@ -19,16 +17,21 @@ * which drives the indexing, and periodically updates it's parent PersistentTask with the indexing's current position. */ public class RollupTask extends CancellableTask { - private static final Logger logger = LogManager.getLogger(RollupTask.class.getName()); - + private String rollupIndex; private RollupActionConfig config; private RollupJobStatus status; - RollupTask(long id, String type, String action, TaskId parentTask, RollupActionConfig config, Map headers) { - super(id, type, action, RollupField.NAME + "_" + config.getRollupIndex(), parentTask, headers); + RollupTask(long id, String type, String action, TaskId parentTask, String rollupIndex, RollupActionConfig config, + Map headers) { + super(id, type, action, RollupField.NAME + "_" + rollupIndex, parentTask, headers); + this.rollupIndex = rollupIndex; this.config = config; } + public String getRollupIndex() { + return rollupIndex; + } + public RollupActionConfig config() { return config; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/v2/RollupActionConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/v2/RollupActionConfigTests.java index ecb6b937ae68a..8cb3f93d62dba 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/v2/RollupActionConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/v2/RollupActionConfigTests.java @@ -29,11 +29,10 @@ protected RollupActionConfig createTestInstance() { } public static RollupActionConfig randomConfig(Random random) { - final String rollupIndex = randomAlphaOfLength(5); final TimeValue timeout = random.nextBoolean() ? null : ConfigTestHelpers.randomTimeout(random); final GroupConfig groupConfig = ConfigTestHelpers.randomGroupConfig(random); final List metricConfigs = ConfigTestHelpers.randomMetricsConfigs(random); - return new RollupActionConfig(groupConfig, metricConfigs, timeout, rollupIndex); + return new RollupActionConfig(groupConfig, metricConfigs, timeout); } @Override @@ -46,19 +45,10 @@ protected RollupActionConfig doParseInstance(final XContentParser parser) throws return RollupActionConfig.fromXContent(parser); } - public void testEmptyRollupIndex() { - final RollupActionConfig sample = createTestInstance(); - Exception e = expectThrows(IllegalArgumentException.class, () -> - new RollupActionConfig(sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout(), - randomBoolean() ? null : "")); - assertThat(e.getMessage(), equalTo("Rollup index must be a non-null, non-empty string")); - } - public void testEmptyGroupAndMetrics() { final RollupActionConfig sample = createTestInstance(); Exception e = expectThrows(IllegalArgumentException.class, () -> - new RollupActionConfig(null, randomBoolean() ? null : emptyList(), sample.getTimeout(), - sample.getRollupIndex())); + new RollupActionConfig(null, randomBoolean() ? null : emptyList(), sample.getTimeout())); assertThat(e.getMessage(), equalTo("At least one grouping or metric must be configured")); } } diff --git a/x-pack/plugin/rollup/qa/rest/src/test/resources/rest-api-spec/test/rollup/10_basic.yml b/x-pack/plugin/rollup/qa/rest/src/test/resources/rest-api-spec/test/rollup/10_basic.yml index d04f04d232fb4..1684dd43eff2e 100644 --- a/x-pack/plugin/rollup/qa/rest/src/test/resources/rest-api-spec/test/rollup/10_basic.yml +++ b/x-pack/plugin/rollup/qa/rest/src/test/resources/rest-api-spec/test/rollup/10_basic.yml @@ -54,9 +54,9 @@ setup: - do: rollup.rollup: index: docs + rollup_index: rollup_docs body: > { - "rollup_index": "rollup_docs", "groups" : { "date_histogram": { "field": "timestamp", diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RestRollupAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RestRollupAction.java index da67e7d91d4d5..44b970e54ee39 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RestRollupAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RestRollupAction.java @@ -22,14 +22,15 @@ public class RestRollupAction extends BaseRestHandler { @Override public List routes() { - return List.of(new Route(POST, "/{index}/_rollup")); + return List.of(new Route(POST, "/{index}/_rollup/{rollup_index}")); } @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { String index = restRequest.param("index"); + String rollupIndex = restRequest.param("rollup_index"); RollupActionConfig config = RollupActionConfig.fromXContent(restRequest.contentParser()); - RollupAction.Request request = new RollupAction.Request(index, config); + RollupAction.Request request = new RollupAction.Request(index, rollupIndex, config); return channel -> client.execute(RollupAction.INSTANCE, request, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupV2Indexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupV2Indexer.java index b50d753f134f4..add7386c5f2b1 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupV2Indexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupV2Indexer.java @@ -106,13 +106,13 @@ public class RollupV2Indexer extends AsyncTwoPhaseIndexer, R this.request = request; this.headers = headers; this.compositeBuilder = createCompositeBuilder(this.request.getRollupConfig()); - this.tmpIndex = ".rolluptmp-" + this.request.getRollupConfig().getRollupIndex(); + this.tmpIndex = ".rolluptmp-" + this.request.getRollupIndex(); this.completionListener = completionListener; } @Override protected String getJobId() { - return request.getRollupConfig().getId(); + return "rollup_" + request.getRollupIndex(); } @Override @@ -196,7 +196,7 @@ protected void onFailure(Exception exc) { @Override protected void onFinish(ActionListener listener) { // "shrink index" - ResizeRequest resizeRequest = new ResizeRequest(request.getRollupConfig().getRollupIndex(), tmpIndex); + ResizeRequest resizeRequest = new ResizeRequest(request.getRollupIndex(), tmpIndex); resizeRequest.setResizeType(ResizeType.CLONE); resizeRequest.getTargetIndexRequest() .settings(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, false).build()); diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/TransportRollupAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/TransportRollupAction.java index 083bd8b54667f..e12a3c1bf3148 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/TransportRollupAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/TransportRollupAction.java @@ -71,7 +71,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS @Override public ClusterState execute(ClusterState currentState) throws Exception { - String rollupIndexName = rollupTask.config().getRollupIndex(); + String rollupIndexName = rollupTask.getRollupIndex(); IndexMetadata rollupIndexMetadata = currentState.getMetadata().index(rollupIndexName); Index rollupIndex = rollupIndexMetadata.getIndex(); // TODO(talevy): find better spot to get the original index name diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.rollup.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.rollup.json index 8423ab3e5a997..3774df023843d 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.rollup.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.rollup.json @@ -13,7 +13,7 @@ "url": { "paths": [ { - "path": "/{index}/_rollup", + "path": "/{index}/_rollup/{rollup_index}", "methods": [ "POST" ], @@ -22,6 +22,11 @@ "type": "string", "description": "The index to roll up", "required": true + }, + "rollup_index": { + "type": "string", + "description": "The name of the rollup index to create", + "required": true } } }