Make TDigestState configurable

benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java

@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License.
+ */
+
+package org.elasticsearch.benchmark.tdigest;
+
+import org.elasticsearch.tdigest.Sort;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/** Explores the performance of Sort on pathological input data. */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 10, time = 3, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 20, time = 2, timeUnit = TimeUnit.SECONDS)
+@Fork(1)
+@Threads(1)
+@State(Scope.Thread)
+public class SortBench {
+    private final int size = 100000;
+    private final double[] values = new double[size];
+
+    @Param({ "0", "1", "-1" })
+    public int sortDirection;
+
+    @Setup
+    public void setup() {
+        Random prng = new Random(999983);
+        for (int i = 0; i < size; i++) {
+            values[i] = prng.nextDouble();
+        }
+        if (sortDirection > 0) {
+            Arrays.sort(values);
+        } else if (sortDirection < 0) {
+            Arrays.sort(values);
+            Sort.reverse(values, 0, values.length);
+        }
+    }
+
+    @Benchmark
+    public void quicksort() {
+        int[] order = new int[size];
+        for (int i = 0; i < size; i++) {
+            order[i] = i;
+        }
+        Sort.sort(order, values, null, values.length);
+    }
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java

@@ -0,0 +1,131 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License.
+ */
+
+package org.elasticsearch.benchmark.tdigest;
+
+import org.elasticsearch.tdigest.AVLTreeDigest;
+import org.elasticsearch.tdigest.MergingDigest;
+import org.elasticsearch.tdigest.TDigest;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.profile.GCProfiler;
+import org.openjdk.jmh.profile.StackProfiler;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 3, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
+@Fork(1)
+@Threads(1)
+@State(Scope.Thread)
+public class TDigestBench {
+
+    public enum TDigestFactory {
+        MERGE {
+            @Override
+            TDigest create(double compression) {
+                return new MergingDigest(compression, (int) (10 * compression));
+            }
+        },
+        AVL_TREE {
+            @Override
+            TDigest create(double compression) {
+                return new AVLTreeDigest(compression);
+            }
+        };
+
+        abstract TDigest create(double compression);
+    }
+
+    @Param({ "100", "300" })
+    double compression;
+
+    @Param({ "MERGE", "AVL_TREE" })
+    TDigestFactory tdigestFactory;
+
+    @Param({ "NORMAL", "GAUSSIAN" })
+    String distribution;
+
+    Random random;
+    TDigest tdigest;
+
+    double[] data = new double[1000000];
+
+    @Setup
+    public void setUp() {
+        random = ThreadLocalRandom.current();
+        tdigest = tdigestFactory.create(compression);
+
+        Supplier<Double> nextRandom = () -> distribution.equals("GAUSSIAN") ? random.nextGaussian() : random.nextDouble();
+        for (int i = 0; i < 10000; ++i) {
+            tdigest.add(nextRandom.get());
+        }
+
+        for (int i = 0; i < data.length; ++i) {
+            data[i] = nextRandom.get();
+        }
+    }
+
+    @State(Scope.Thread)
+    public static class ThreadState {
+        int index = 0;
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.AverageTime)
+    @OutputTimeUnit(TimeUnit.MICROSECONDS)
+    public void add(ThreadState state) {
+        if (state.index >= data.length) {
+            state.index = 0;
+        }
+        tdigest.add(data[state.index++]);
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder().include(".*" + TDigestBench.class.getSimpleName() + ".*")
+            .warmupIterations(5)
+            .measurementIterations(5)
+            .addProfiler(GCProfiler.class)
+            .addProfiler(StackProfiler.class)
+            .build();
+
+        new Runner(opt).run();
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -61,6 +61,7 @@ public class InternalDistributionModuleCheckTaskProvider {
         "org.elasticsearch.preallocate",
         "org.elasticsearch.securesm",
         "org.elasticsearch.server",
+        "org.elasticsearch.tdigest",
         "org.elasticsearch.xcontent"
     );
 
@@ -75,7 +76,7 @@ public class InternalDistributionModuleCheckTaskProvider {
 
     private static final Function<ModuleReference, String> toName = mref -> mref.descriptor().name();
 
-    private InternalDistributionModuleCheckTaskProvider() {};
+    private InternalDistributionModuleCheckTaskProvider() {}
 
     /** Registers the checkModules tasks, which contains all checks relevant to ES Java Modules. */
     static TaskProvider<Task> registerCheckModulesTask(Project project, TaskProvider<Copy> checkExtraction) {

docs/reference/aggregations/metrics/boxplot-aggregation.asciidoc

@@ -55,9 +55,9 @@ The response will look like this:
     "load_time_boxplot": {
       "min": 0.0,
       "max": 990.0,
-      "q1": 165.0,
+      "q1": 167.5,
       "q2": 445.0,
-      "q3": 725.0,
+      "q3": 722.5,
       "lower": 0.0,
       "upper": 990.0
     }
@@ -114,9 +114,9 @@ GET latency/_search
     "load_time_boxplot": {
       "min": 0.0,
       "max": 99.0,
-      "q1": 16.5,
+      "q1": 16.75,
       "q2": 44.5,
-      "q3": 72.5,
+      "q3": 72.25,
       "lower": 0.0,
       "upper": 99.0
     }

docs/reference/aggregations/metrics/percentile-aggregation.asciidoc

@@ -54,19 +54,26 @@ percentiles: `[ 1, 5, 25, 50, 75, 95, 99 ]`. The response will look like this:
  "aggregations": {
     "load_time_outlier": {
       "values": {
-        "1.0": 5.0,
-        "5.0": 25.0,
-        "25.0": 165.0,
+        "1.0": 10.0,
+        "5.0": 30.0,
+        "25.0": 170.0,
         "50.0": 445.0,
-        "75.0": 725.0,
-        "95.0": 945.0,
-        "99.0": 985.0
+        "75.0": 720.0,
+        "95.0": 940.0,
+        "99.0": 980.0
       }
     }
   }
 }
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/"1.0": 10.0/"1.0": 9.9/]
+// TESTRESPONSE[s/"5.0": 30.0/"5.0": 29.500000000000004/]
+// TESTRESPONSE[s/"25.0": 170.0/"25.0": 167.5/]
+// TESTRESPONSE[s/"50.0": 445.0/"50.0": 445.0/]
+// TESTRESPONSE[s/"75.0": 720.0/"75.0": 722.5/]
+// TESTRESPONSE[s/"95.0": 940.0/"95.0": 940.5/]
+// TESTRESPONSE[s/"99.0": 980.0/"99.0": 980.1000000000001/]
 
 As you can see, the aggregation will return a calculated value for each percentile
 in the default range. If we assume response times are in milliseconds, it is
@@ -128,38 +135,45 @@ Response:
       "values": [
         {
           "key": 1.0,
-          "value": 5.0
+          "value": 10.0
         },
         {
           "key": 5.0,
-          "value": 25.0
+          "value": 30.0
         },
         {
           "key": 25.0,
-          "value": 165.0
+          "value": 170.0
         },
         {
           "key": 50.0,
           "value": 445.0
         },
         {
           "key": 75.0,
-          "value": 725.0
+          "value": 720.0
         },
         {
           "key": 95.0,
-          "value": 945.0
+          "value": 940.0
         },
         {
           "key": 99.0,
-          "value": 985.0
+          "value": 980.0
         }
       ]
     }
   }
 }
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/"value": 10.0/"value": 9.9/]
+// TESTRESPONSE[s/"value": 30.0/"value": 29.500000000000004/]
+// TESTRESPONSE[s/"value": 170.0/"value": 167.5/]
+// TESTRESPONSE[s/"value": 445.0/"value": 445.0/]
+// TESTRESPONSE[s/"value": 720.0/"value": 722.5/]
+// TESTRESPONSE[s/"value": 940.0/"value": 940.5/]
+// TESTRESPONSE[s/"value": 980.0/"value": 980.1000000000001/]
 
 ==== Script
 
@@ -203,13 +217,13 @@ GET latency/_search
  "aggregations": {
     "load_time_outlier": {
       "values": {
-        "1.0": 0.5,
-        "5.0": 2.5,
-        "25.0": 16.5,
+        "1.0": 0.99,
+        "5.0": 2.95,
+        "25.0": 16.75,
         "50.0": 44.5,
-        "75.0": 72.5,
-        "95.0": 94.5,
-        "99.0": 98.5
+        "75.0": 72.25,
+        "95.0": 94.05,
+        "99.0": 98.01
       }
     }
   }