Merge remote-tracking branch 'es/7.x' into backport_58582

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java

@@ -49,7 +49,7 @@ protected void doExecute(Task task, SearchRequest request, ActionListener<Search
         listener.onResponse(new SearchResponse(new InternalSearchResponse(
             new SearchHits(
                 new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f),
-            new InternalAggregations(Collections.emptyList()),
+            InternalAggregations.EMPTY,
             new Suggest(Collections.emptyList()),
             new SearchProfileShardResults(Collections.emptyMap()), false, false, 1),
             "", 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY));

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/MlInferenceNamedXContentProvider.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.client.ml.inference.trainedmodel.RegressionConfig;
 import org.elasticsearch.client.ml.inference.trainedmodel.TrainedModel;
 import org.elasticsearch.client.ml.inference.trainedmodel.ensemble.Ensemble;
+import org.elasticsearch.client.ml.inference.trainedmodel.ensemble.Exponent;
 import org.elasticsearch.client.ml.inference.trainedmodel.ensemble.LogisticRegression;
 import org.elasticsearch.client.ml.inference.trainedmodel.ensemble.OutputAggregator;
 import org.elasticsearch.client.ml.inference.trainedmodel.ensemble.WeightedMode;
@@ -82,6 +83,9 @@ public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
         namedXContent.add(new NamedXContentRegistry.Entry(OutputAggregator.class,
             new ParseField(LogisticRegression.NAME),
             LogisticRegression::fromXContent));
+        namedXContent.add(new NamedXContentRegistry.Entry(OutputAggregator.class,
+            new ParseField(Exponent.NAME),
+            Exponent::fromXContent));
 
         return namedXContent;
     }

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Exponent.java

@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.inference.trainedmodel.ensemble;
+
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+
+public class Exponent implements OutputAggregator {
+
+    public static final String NAME = "exponent";
+    public static final ParseField WEIGHTS = new ParseField("weights");
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<Exponent, Void> PARSER = new ConstructingObjectParser<>(
+        NAME,
+        true,
+        a -> new Exponent((List<Double>)a[0]));
+    static {
+        PARSER.declareDoubleArray(ConstructingObjectParser.optionalConstructorArg(), WEIGHTS);
+    }
+
+    public static Exponent fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    private final List<Double> weights;
+
+    public Exponent(List<Double> weights) {
+        this.weights = weights;
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (weights != null) {
+            builder.field(WEIGHTS.getPreferredName(), weights);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Exponent that = (Exponent) o;
+        return Objects.equals(weights, that.weights);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(weights);
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -80,6 +80,7 @@
 import org.elasticsearch.client.ml.inference.trainedmodel.ClassificationConfig;
 import org.elasticsearch.client.ml.inference.trainedmodel.RegressionConfig;
 import org.elasticsearch.client.ml.inference.trainedmodel.ensemble.Ensemble;
+import org.elasticsearch.client.ml.inference.trainedmodel.ensemble.Exponent;
 import org.elasticsearch.client.ml.inference.trainedmodel.ensemble.LogisticRegression;
 import org.elasticsearch.client.ml.inference.trainedmodel.ensemble.WeightedMode;
 import org.elasticsearch.client.ml.inference.trainedmodel.ensemble.WeightedSum;
@@ -703,7 +704,7 @@ public void testDefaultNamedXContents() {
 
     public void testProvidedNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(68, namedXContents.size());
+        assertEquals(69, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         List<String> names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -788,9 +789,9 @@ public void testProvidedNamedXContents() {
         assertThat(names, hasItems(FrequencyEncoding.NAME, OneHotEncoding.NAME, TargetMeanEncoding.NAME, CustomWordEmbedding.NAME));
         assertEquals(Integer.valueOf(3), categories.get(org.elasticsearch.client.ml.inference.trainedmodel.TrainedModel.class));
         assertThat(names, hasItems(Tree.NAME, Ensemble.NAME, LangIdentNeuralNetwork.NAME));
-        assertEquals(Integer.valueOf(3),
+        assertEquals(Integer.valueOf(4),
             categories.get(org.elasticsearch.client.ml.inference.trainedmodel.ensemble.OutputAggregator.class));
-        assertThat(names, hasItems(WeightedMode.NAME, WeightedSum.NAME, LogisticRegression.NAME));
+        assertThat(names, hasItems(WeightedMode.NAME, WeightedSum.NAME, LogisticRegression.NAME, Exponent.NAME));
         assertEquals(Integer.valueOf(2),
             categories.get(org.elasticsearch.client.ml.inference.trainedmodel.InferenceConfig.class));
         assertThat(names, hasItems(ClassificationConfig.NAME.getPreferredName(), RegressionConfig.NAME.getPreferredName()));

client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/EnsembleTests.java

@@ -73,7 +73,8 @@ public static Ensemble createRandom(TargetType targetType) {
             categoryLabels = randomList(2, randomIntBetween(3, 10), () -> randomAlphaOfLength(10));
         }
         List<Double> weights = Stream.generate(ESTestCase::randomDouble).limit(numberOfModels).collect(Collectors.toList());
-        OutputAggregator outputAggregator = targetType == TargetType.REGRESSION ? new WeightedSum(weights) :
+        OutputAggregator outputAggregator = targetType == TargetType.REGRESSION ?
+            randomFrom(new WeightedSum(weights), new Exponent(weights)) :
             randomFrom(
                 new WeightedMode(
                     categoryLabels != null ? categoryLabels.size() : randomIntBetween(2, 10),

client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/ExponentTests.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.inference.trainedmodel.ensemble;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+
+public class ExponentTests extends AbstractXContentTestCase<Exponent> {
+
+    Exponent createTestInstance(int numberOfWeights) {
+        return new Exponent(Stream.generate(ESTestCase::randomDouble).limit(numberOfWeights).collect(Collectors.toList()));
+    }
+
+    @Override
+    protected Exponent doParseInstance(XContentParser parser) throws IOException {
+        return Exponent.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected Exponent createTestInstance() {
+        return randomBoolean() ? new Exponent(null) : createTestInstance(randomIntBetween(1, 100));
+    }
+
+}

docs/reference/glossary.asciidoc

@@ -268,7 +268,6 @@ shard will never be started on the same node as its primary shard.
 --
 // tag::rollover-def[]
 // tag::rollover-def-short[]
-
 Creates a new index for a rollover target when the existing index reaches a certain size, number of docs, or age.
 A rollover target can be either an <<indices-aliases, index alias>> or a <<data-streams, data stream>>.
 // end::rollover-def-short[]

docs/reference/ilm/example-index-lifecycle-policy.asciidoc

@@ -0,0 +1,165 @@
+[role="xpack"]
+
+[[example-using-index-lifecycle-policy]]
+=== Tutorial: Manage {filebeat} time-based indices
+++++
+<titleabbrev>Manage {filebeat} time-based indices</titleabbrev>
+++++
+
+With {ilm} ({ilm-init}), you can create policies that perform actions automatically 
+on indices as they age and grow. {ilm-init} policies help you to manage
+performance, resilience, and retention of your data during its lifecycle. This tutorial shows
+you how to use {kib}’s *Index Lifecycle Policies* to modify and create {ilm-init}
+policies. You can learn more about all of the actions, benefits, and lifecycle
+phases in the <<overview-index-lifecycle-management, {ilm-init} overview>>.
+
+
+[discrete]
+[[example-using-index-lifecycle-policy-scenario]]
+==== Scenario
+
+You’re tasked with sending syslog files to an {es} cluster. This
+log data has the following data retention guidelines:
+
+* Keep logs on hot data nodes for 30 days
+* Roll over to a new index if the size reaches 50GB
+* After 30 days:
+** Move the logs to warm data nodes
+** Set <<glossary-replica-shard, replica shards>> to 1
+** <<indices-forcemerge, Force merge>> multiple index segments to free up the space used by deleted documents
+* Delete logs after 90 days
+
+
+[discrete]
+[[example-using-index-lifecycle-policy-prerequisites]]
+==== Prerequisites
+
+To complete this tutorial, you'll need:
+
+* An {es} cluster with hot and warm nodes configured for shard allocation
+awareness. 
+
+** {ess}: 
+Choose the {cloud}/ec-getting-started-templates-hot-warm.html[hot-warm architecture] deployment template.
+
+** Self-managed cluster: 
+Add node attributes as described for {ref}/shard-allocation-filtering.html[shard allocation filtering].
++
+For example, you can set this in your `elasticsearch.yml` for each data node:
++
+[source,yaml]
+--------------------------------------------------------------------------------
+node.attr.data: "warm"
+--------------------------------------------------------------------------------
+
+* A server with {filebeat} installed and configured to send logs to the `elasticsearch`
+output as described in {filebeat-ref}/filebeat-getting-started.html[Getting Started with {filebeat}].
+
+[discrete]
+[[example-using-index-lifecycle-policy-view-fb-ilm-policy]]
+==== View the {filebeat} {ilm-init} policy
+
+{filebeat} includes a default {ilm-init} policy that enables rollover. {ilm-init}
+is enabled automatically if you’re using the default `filebeat.yml` and index template.
+
+To view the default policy in {kib}:
+
+. Go to Management and select *Index Lifecycle Policies*.
+. Search for _filebeat_
+. Select the _filebeat-version_ policy.
+
+This policy initiates the rollover action when the index size reaches 50GB or
+becomes 30 days old.
+
+[role="screenshot"]
+image::images/ilm/tutorial-ilm-hotphaserollover-default.png["Default policy"]
+
+
+[discrete]
+==== Modify the policy
+
+The default policy is enough to prevent the creation of many tiny daily indices.
+You can modify the policy to meet more complex requirements.
+
+. Activate the warm phase.
++
+--
+[role="screenshot"]
+image::images/ilm/tutorial-ilm-modify-default-warm-phase-rollover.png["Modify to add warm phase"]
+
+.. Set one of the following options to control when the index moves to the warm phase:
+
+*** Provide a value for *Timing for warm phase*. Setting this to *15* keeps the
+indices on hot nodes for a range of  15-45 days, depending on when the initial
+rollover occurred.
+
+*** Enable *Move to warm phase on rollover*. The index might move to the warm phase
+more quickly than intended if it reaches the *Maximum index size* before the
+the *Maximum age*.
+
+.. In the *Select a node attribute to control shard allocation* dropdown, select
+*data:warm(2)* to migrate shards to warm data nodes.
+
+.. Change *Number of replicas* to *1*.
+
+.. Enable *Force merge data* and set *Number of segments* to *1*.
+
+NOTE:  When rollover is enabled in the hot phase, action timing in the other phases
+is based on the rollover date.
+--
+
+. Activate the delete phase and set *Timing for delete phase* to *90* days.
++
+[role="screenshot"]
+image::images/ilm/tutorial-ilm-delete-rollover.png["Add a delete phase"]
+
+[discrete]
+==== Create a custom policy
+
+If meeting a specific retention time period is most important, you can create a
+custom policy.  For this option, you use {filebeat} daily indices without
+rollover.
+
+To create a custom policy:
+
+. Go to Management and select *Index Lifecycle Policies*.
+. Click *Create policy*.
+. Activate the warm phase and configure it as follows:
++
+--
+**Timing for warm phase**: 30 days from index creation
+
+**Node attribute**: `data:warm`
+
+**Number of replicas**: 1
+
+**Force merge data**: enable
+
+**Number of segments**: 1
+
+[role="screenshot"]
+image::images/ilm/tutorial-ilm-custom-policy.png["Modify the custom policy to add a warm phase"]
+--
+
+. Activate the delete phase and set the timing to 90 days.
++
+[role="screenshot"]
+image::images/ilm/tutorial-ilm-delete-phase-creation.png["Delete phase"]
+
+To configure the index to use the new policy:
+
+. Go to Management and select *Index Lifecycle Policies*.
+. Find your {ilm-init} policy and click its *Actions* link.
+. Choose *Add policy to index template*.
+. Select your {filebeat} index template name from the *Index template* list. For example, `filebeat-7.5.x`.
+. Click *Add Policy* to save the changes.
++
+NOTE: If you initially used the default {filebeat} {ilm-init} policy, you will
+see a notice that the template already has a policy associated with it. Confirm
+that you want to overwrite that configuration.
+
+When you change the policy associated with the index template, the active
+index will continue to use the policy it was associated with at index creation
+unless you manually update it. The next new index will use the updated policy.
+For more reasons that your {ilm-init} policy changes might be delayed, see
+<<update-lifecycle-policy, Update Lifecycle Policy>>.

docs/reference/ilm/ilm-overview.asciidoc

@@ -9,6 +9,7 @@
 
 You can create and apply {ilm-cap} ({ilm-init}) policies to automatically manage your indices 
 according to your performance, resiliency, and retention requirements. 
+
 Index lifecycle policies can trigger actions such as:
 
 * **Rollover**: 
@@ -39,9 +40,9 @@ For example, if you are indexing metrics data from a fleet of ATMs into
 Elasticsearch, you might define a policy that says:
 
 . When the index reaches 50GB, roll over to a new index.
-. Move the old index into the warm stage, mark it read only, and shrink it down
+. Move the old index into the warm phase, mark it read only, and shrink it down
 to a single shard.
-. After 7 days, move the index into the cold stage and move it to less expensive
+. After 7 days, move the index into the cold phase and move it to less expensive
 hardware.
 . Delete the index once the required 30 day retention period is reached.