Merge remote-tracking branch 'elastic/master' into geoshape-doc-values

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -424,23 +424,23 @@ public synchronized void start() {
 
         if (plugins.isEmpty() == false) {
             logToProcessStdout("Installing " + plugins.size() + " plugins");
-            plugins.forEach(plugin -> runElaticsearchBinScript(
+            plugins.forEach(plugin -> runElasticsearchBinScript(
                 "elasticsearch-plugin",
                 "install", "--batch", plugin.toString())
             );
         }
 
         if (getVersion().before("6.3.0") && testDistribution == TestDistribution.DEFAULT) {
             LOGGER.info("emulating the {} flavor for {} by installing x-pack", testDistribution, getVersion());
-            runElaticsearchBinScript(
+            runElasticsearchBinScript(
                 "elasticsearch-plugin",
                 "install", "--batch", "x-pack"
             );
         }
 
         if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
             logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files");
-            runElaticsearchBinScript("elasticsearch-keystore", "create");
+            runElasticsearchBinScript("elasticsearch-keystore", "create");
 
             keystoreSettings.forEach((key, value) ->
                 runElasticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
@@ -452,7 +452,7 @@ public synchronized void start() {
                 if (file.exists() == false) {
                     throw new TestClustersException("supplied keystore file " + file + " does not exist, require for " + this);
                 }
-                runElaticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
+                runElasticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
             }
         }
 
@@ -467,7 +467,7 @@ public synchronized void start() {
         if (credentials.isEmpty() == false) {
             logToProcessStdout("Setting up " + credentials.size() + " users");
 
-            credentials.forEach(paramMap -> runElaticsearchBinScript(
+            credentials.forEach(paramMap -> runElasticsearchBinScript(
                 getVersion().onOrAfter("6.3.0") ? "elasticsearch-users" : "x-pack/users",
                 paramMap.entrySet().stream()
                     .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue()))
@@ -663,7 +663,7 @@ private void runElasticsearchBinScriptWithInput(String input, String tool, Strin
         }
     }
 
-    private void runElaticsearchBinScript(String tool, String... args) {
+    private void runElasticsearchBinScript(String tool, String... args) {
         runElasticsearchBinScriptWithInput("", tool, args);
     }
 

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.4.0-snapshot-e648d601efb
+lucene            = 8.4.0-snapshot-662c455
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 13.0.1+9

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java

@@ -49,6 +49,7 @@ public static Builder builder(String dependentVariable) {
     static final ParseField PREDICTION_FIELD_NAME = new ParseField("prediction_field_name");
     static final ParseField TRAINING_PERCENT = new ParseField("training_percent");
     static final ParseField NUM_TOP_CLASSES = new ParseField("num_top_classes");
+    static final ParseField RANDOMIZE_SEED = new ParseField("randomize_seed");
 
     private static final ConstructingObjectParser<Classification, Void> PARSER =
         new ConstructingObjectParser<>(
@@ -63,7 +64,8 @@ public static Builder builder(String dependentVariable) {
                 (Double) a[5],
                 (String) a[6],
                 (Double) a[7],
-                (Integer) a[8]));
+                (Integer) a[8],
+                (Long) a[9]));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), DEPENDENT_VARIABLE);
@@ -75,6 +77,7 @@ public static Builder builder(String dependentVariable) {
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PREDICTION_FIELD_NAME);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), TRAINING_PERCENT);
         PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), NUM_TOP_CLASSES);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), RANDOMIZE_SEED);
     }
 
     private final String dependentVariable;
@@ -86,10 +89,11 @@ public static Builder builder(String dependentVariable) {
     private final String predictionFieldName;
     private final Double trainingPercent;
     private final Integer numTopClasses;
+    private final Long randomizeSeed;
 
     private Classification(String dependentVariable, @Nullable Double lambda, @Nullable Double gamma, @Nullable Double eta,
                            @Nullable Integer maximumNumberTrees, @Nullable Double featureBagFraction, @Nullable String predictionFieldName,
-                           @Nullable Double trainingPercent, @Nullable Integer numTopClasses) {
+                           @Nullable Double trainingPercent, @Nullable Integer numTopClasses, @Nullable Long randomizeSeed) {
         this.dependentVariable = Objects.requireNonNull(dependentVariable);
         this.lambda = lambda;
         this.gamma = gamma;
@@ -99,6 +103,7 @@ private Classification(String dependentVariable, @Nullable Double lambda, @Nulla
         this.predictionFieldName = predictionFieldName;
         this.trainingPercent = trainingPercent;
         this.numTopClasses = numTopClasses;
+        this.randomizeSeed = randomizeSeed;
     }
 
     @Override
@@ -138,6 +143,10 @@ public Double getTrainingPercent() {
         return trainingPercent;
     }
 
+    public Long getRandomizeSeed() {
+        return randomizeSeed;
+    }
+
     public Integer getNumTopClasses() {
         return numTopClasses;
     }
@@ -167,6 +176,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (trainingPercent != null) {
             builder.field(TRAINING_PERCENT.getPreferredName(), trainingPercent);
         }
+        if (randomizeSeed != null) {
+            builder.field(RANDOMIZE_SEED.getPreferredName(), randomizeSeed);
+        }
         if (numTopClasses != null) {
             builder.field(NUM_TOP_CLASSES.getPreferredName(), numTopClasses);
         }
@@ -177,7 +189,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     @Override
     public int hashCode() {
         return Objects.hash(dependentVariable, lambda, gamma, eta, maximumNumberTrees, featureBagFraction, predictionFieldName,
-            trainingPercent, numTopClasses);
+            trainingPercent, randomizeSeed, numTopClasses);
     }
 
     @Override
@@ -193,6 +205,7 @@ public boolean equals(Object o) {
             && Objects.equals(featureBagFraction, that.featureBagFraction)
             && Objects.equals(predictionFieldName, that.predictionFieldName)
             && Objects.equals(trainingPercent, that.trainingPercent)
+            && Objects.equals(randomizeSeed, that.randomizeSeed)
             && Objects.equals(numTopClasses, that.numTopClasses);
     }
 
@@ -211,6 +224,7 @@ public static class Builder {
         private String predictionFieldName;
         private Double trainingPercent;
         private Integer numTopClasses;
+        private Long randomizeSeed;
 
         private Builder(String dependentVariable) {
             this.dependentVariable = Objects.requireNonNull(dependentVariable);
@@ -251,14 +265,19 @@ public Builder setTrainingPercent(Double trainingPercent) {
             return this;
         }
 
+        public Builder setRandomizeSeed(Long randomizeSeed) {
+            this.randomizeSeed = randomizeSeed;
+            return this;
+        }
+
         public Builder setNumTopClasses(Integer numTopClasses) {
             this.numTopClasses = numTopClasses;
             return this;
         }
 
         public Classification build() {
             return new Classification(dependentVariable, lambda, gamma, eta, maximumNumberTrees, featureBagFraction, predictionFieldName,
-                trainingPercent, numTopClasses);
+                trainingPercent, numTopClasses, randomizeSeed);
         }
     }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Regression.java

@@ -48,6 +48,7 @@ public static Builder builder(String dependentVariable) {
     static final ParseField FEATURE_BAG_FRACTION = new ParseField("feature_bag_fraction");
     static final ParseField PREDICTION_FIELD_NAME = new ParseField("prediction_field_name");
     static final ParseField TRAINING_PERCENT = new ParseField("training_percent");
+    static final ParseField RANDOMIZE_SEED = new ParseField("randomize_seed");
 
     private static final ConstructingObjectParser<Regression, Void> PARSER =
         new ConstructingObjectParser<>(
@@ -61,7 +62,8 @@ public static Builder builder(String dependentVariable) {
                 (Integer) a[4],
                 (Double) a[5],
                 (String) a[6],
-                (Double) a[7]));
+                (Double) a[7],
+                (Long) a[8]));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), DEPENDENT_VARIABLE);
@@ -72,6 +74,7 @@ public static Builder builder(String dependentVariable) {
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), FEATURE_BAG_FRACTION);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PREDICTION_FIELD_NAME);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), TRAINING_PERCENT);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), RANDOMIZE_SEED);
     }
 
     private final String dependentVariable;
@@ -82,10 +85,11 @@ public static Builder builder(String dependentVariable) {
     private final Double featureBagFraction;
     private final String predictionFieldName;
     private final Double trainingPercent;
+    private final Long randomizeSeed;
 
     private Regression(String dependentVariable,  @Nullable Double lambda, @Nullable Double gamma, @Nullable Double eta,
                        @Nullable Integer maximumNumberTrees, @Nullable Double featureBagFraction, @Nullable String predictionFieldName,
-                       @Nullable Double trainingPercent) {
+                       @Nullable Double trainingPercent, @Nullable Long randomizeSeed) {
         this.dependentVariable = Objects.requireNonNull(dependentVariable);
         this.lambda = lambda;
         this.gamma = gamma;
@@ -94,6 +98,7 @@ private Regression(String dependentVariable,  @Nullable Double lambda, @Nullable
         this.featureBagFraction = featureBagFraction;
         this.predictionFieldName = predictionFieldName;
         this.trainingPercent = trainingPercent;
+        this.randomizeSeed = randomizeSeed;
     }
 
     @Override
@@ -133,6 +138,10 @@ public Double getTrainingPercent() {
         return trainingPercent;
     }
 
+    public Long getRandomizeSeed() {
+        return randomizeSeed;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -158,14 +167,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (trainingPercent != null) {
             builder.field(TRAINING_PERCENT.getPreferredName(), trainingPercent);
         }
+        if (randomizeSeed != null) {
+            builder.field(RANDOMIZE_SEED.getPreferredName(), randomizeSeed);
+        }
         builder.endObject();
         return builder;
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(dependentVariable, lambda, gamma, eta, maximumNumberTrees, featureBagFraction, predictionFieldName,
-            trainingPercent);
+            trainingPercent, randomizeSeed);
     }
 
     @Override
@@ -180,7 +192,8 @@ public boolean equals(Object o) {
             && Objects.equals(maximumNumberTrees, that.maximumNumberTrees)
             && Objects.equals(featureBagFraction, that.featureBagFraction)
             && Objects.equals(predictionFieldName, that.predictionFieldName)
-            && Objects.equals(trainingPercent, that.trainingPercent);
+            && Objects.equals(trainingPercent, that.trainingPercent)
+            && Objects.equals(randomizeSeed, that.randomizeSeed);
     }
 
     @Override
@@ -197,6 +210,7 @@ public static class Builder {
         private Double featureBagFraction;
         private String predictionFieldName;
         private Double trainingPercent;
+        private Long randomizeSeed;
 
         private Builder(String dependentVariable) {
             this.dependentVariable = Objects.requireNonNull(dependentVariable);
@@ -237,9 +251,14 @@ public Builder setTrainingPercent(Double trainingPercent) {
             return this;
         }
 
+        public Builder setRandomizeSeed(Long randomizeSeed) {
+            this.randomizeSeed = randomizeSeed;
+            return this;
+        }
+
         public Regression build() {
             return new Regression(dependentVariable, lambda, gamma, eta, maximumNumberTrees, featureBagFraction, predictionFieldName,
-                trainingPercent);
+                trainingPercent, randomizeSeed);
         }
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -1291,6 +1291,7 @@ public void testPutDataFrameAnalyticsConfig_GivenRegression() throws Exception {
             .setAnalysis(org.elasticsearch.client.ml.dataframe.Regression.builder("my_dependent_variable")
                 .setPredictionFieldName("my_dependent_variable_prediction")
                 .setTrainingPercent(80.0)
+                .setRandomizeSeed(42L)
                 .build())
             .setDescription("this is a regression")
             .build();
@@ -1326,6 +1327,7 @@ public void testPutDataFrameAnalyticsConfig_GivenClassification() throws Excepti
             .setAnalysis(org.elasticsearch.client.ml.dataframe.Classification.builder("my_dependent_variable")
                 .setPredictionFieldName("my_dependent_variable_prediction")
                 .setTrainingPercent(80.0)
+                .setRandomizeSeed(42L)
                 .setNumTopClasses(1)
                 .build())
             .setDescription("this is a classification")

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

@@ -2975,7 +2975,8 @@ public void testPutDataFrameAnalytics() throws Exception {
                 .setFeatureBagFraction(0.4) // <6>
                 .setPredictionFieldName("my_prediction_field_name") // <7>
                 .setTrainingPercent(50.0) // <8>
-                .setNumTopClasses(1) // <9>
+                .setRandomizeSeed(1234L) // <9>
+                .setNumTopClasses(1) // <10>
                 .build();
             // end::put-data-frame-analytics-classification
 
@@ -2988,6 +2989,7 @@ public void testPutDataFrameAnalytics() throws Exception {
                 .setFeatureBagFraction(0.4) // <6>
                 .setPredictionFieldName("my_prediction_field_name") // <7>
                 .setTrainingPercent(50.0) // <8>
+                .setRandomizeSeed(1234L) // <9>
                 .build();
             // end::put-data-frame-analytics-regression
 

client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/ClassificationTests.java

@@ -34,6 +34,7 @@ public static Classification randomClassification() {
             .setFeatureBagFraction(randomBoolean() ? null : randomDoubleBetween(0.0, 1.0, false))
             .setPredictionFieldName(randomBoolean() ? null : randomAlphaOfLength(10))
             .setTrainingPercent(randomBoolean() ? null : randomDoubleBetween(1.0, 100.0, true))
+            .setRandomizeSeed(randomBoolean() ? null : randomLong())
             .setNumTopClasses(randomBoolean() ? null : randomIntBetween(0, 10))
             .build();
     }

docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc

@@ -119,7 +119,8 @@ include-tagged::{doc-tests-file}[{api}-classification]
 <6> The fraction of features which will be used when selecting a random bag for each candidate split. A double in (0, 1].
 <7> The name of the prediction field in the results object.
 <8> The percentage of training-eligible rows to be used in training. Defaults to 100%.
-<9> The number of top classes to be reported in the results. Defaults to 2.
+<9> The seed to be used by the random generator that picks which rows are used in training.
+<10> The number of top classes to be reported in the results. Defaults to 2.
 
 ===== Regression
 
@@ -138,6 +139,7 @@ include-tagged::{doc-tests-file}[{api}-regression]
 <6> The fraction of features which will be used when selecting a random bag for each candidate split. A double in (0, 1].
 <7> The name of the prediction field in the results object.
 <8> The percentage of training-eligible rows to be used in training. Defaults to 100%.
+<9> The seed to be used by the random generator that picks which rows are used in training.
 
 ==== Analyzed fields
 

docs/reference/cat/shards.asciidoc

@@ -257,7 +257,7 @@ Reason the shard is unassigned. Returned values are:
 * `NEW_INDEX_RESTORED`: Unassigned as a result of restoring into a new index.
 * `NODE_LEFT`: Unassigned as a result of the node hosting it leaving the cluster.
 * `REALLOCATED_REPLICA`: A better replica location is identified and causes the existing replica allocation to be cancelled.
-* `REINITIALIZED`: When a shard moves from started back to initializing, for example, with shadow replicas.
+* `REINITIALIZED`: When a shard moves from started back to initializing.
 * `REPLICA_ADDED`: Unassigned as a result of explicit addition of a replica.
 * `REROUTE_CANCELLED`: Unassigned as a result of explicit cancel reroute command.
 

docs/reference/ingest/ingest-node.asciidoc

@@ -825,6 +825,7 @@ include::processors/append.asciidoc[]
 include::processors/bytes.asciidoc[]
 include::processors/circle.asciidoc[]
 include::processors/convert.asciidoc[]
+include::processors/csv.asciidoc[]
 include::processors/date.asciidoc[]
 include::processors/date-index-name.asciidoc[]
 include::processors/dissect.asciidoc[]

docs/reference/ingest/processors/csv.asciidoc

@@ -0,0 +1,33 @@
+[[csv-processor]]
+=== CSV Processor
+Extracts fields from CSV line out of a single text field within a document. Any empty field in CSV will be skipped.
+
+[[csv-options]]
+.CSV Options
+[options="header"]
+|======
+| Name              | Required  | Default  | Description
+| `field`           | yes       | -        | The field to extract data from
+| `target_fields`   | yes       | -        | The array of fields to assign extracted values to
+| `separator`       | no        | ,        | Separator used in CSV, has to be single character string
+| `quote`           | no        | "        | Quote used in CSV, has to be single character string
+| `ignore_missing`  | no        | `true`   | If `true` and `field` does not exist, the processor quietly exits without modifying the document
+| `trim`            | no        | `false`  | Trim whitespaces in unquoted fields
+include::common-options.asciidoc[]
+|======
+
+[source,js]
+--------------------------------------------------
+{
+  "csv": {
+    "field": "my_field",
+    "target_fields": ["field1, field2"],
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+If the `trim` option is enabled then any whitespace in the beginning and in the end of each unquoted field will be trimmed.
+For example with configuration above, a value of `A, B` will result in field `field2`
+having value `{nbsp}B` (with space at the beginning). If `trim` is enabled `A, B` will result in field `field2`
+having value `B` (no whitespace). Quoted fields will be left untouched.