From a6c7a938ea4b6a08caf2b524fc6a38e8adee923a Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 13 Oct 2021 09:50:39 -0500 Subject: [PATCH 01/26] Script: Time series compile and cache evict metrics Collects compilation and cache eviction metrics for each script context. Metrics are available in _nodes/stats in 5m/15m/1d buckets. Refs: #62899 --- .../elasticsearch/script/ScriptMetrics.java | 16 +- .../script/TimeSeriesCounter.java | 378 ++++++++++++++++++ .../script/TimeSeriesCounterTest.java | 290 ++++++++++++++ 3 files changed, 682 insertions(+), 2 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java create mode 100644 server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTest.java diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java index f853305c3cd45..32a6a552e1575 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java @@ -14,13 +14,22 @@ public class ScriptMetrics { final CounterMetric compilationsMetric = new CounterMetric(); final CounterMetric cacheEvictionsMetric = new CounterMetric(); final CounterMetric compilationLimitTriggered = new CounterMetric(); + final TimeSeriesCounter compilationsHistory = timeSeriesCounter(); + final TimeSeriesCounter cacheEvictionsHistory = timeSeriesCounter(); + final int[] TIME_PERIODS = { 5 * TimeSeriesCounter.MINUTE, 15 * TimeSeriesCounter.MINUTE, 24 * TimeSeriesCounter.HOUR }; + + public static TimeSeriesCounter timeSeriesCounter() { + return TimeSeriesCounter.nestedCounter(24 * 4, 15 * TimeSeriesCounter.MINUTE, 60, 15); + } public void onCompilation() { compilationsMetric.inc(); + compilationsHistory.inc(); } public void onCacheEviction() { cacheEvictionsMetric.inc(); + cacheEvictionsHistory.inc(); } public void onCompilationLimit() { @@ -28,12 +37,15 @@ public void onCompilationLimit() { } public ScriptContextStats stats(String context) { + long timestamp = compilationsHistory.timestamp(); + int[] compilations = compilationsHistory.counts(timestamp, TIME_PERIODS); + int[] cacheEvictions = cacheEvictionsHistory.counts(timestamp, TIME_PERIODS); return new ScriptContextStats( context, compilationsMetric.count(), cacheEvictionsMetric.count(), compilationLimitTriggered.count(), - new ScriptContextStats.TimeSeries(), - new ScriptContextStats.TimeSeries() + new ScriptContextStats.TimeSeries(compilations[0], compilations[1], compilations[2]), + new ScriptContextStats.TimeSeries(cacheEvictions[0], cacheEvictions[1], cacheEvictions[2]) ); }} diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java new file mode 100644 index 0000000000000..b7bddd8647d9c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -0,0 +1,378 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.LongAdder; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.LongSupplier; + +/** + * A counter that keeps a time series history of (resolutionSecs * numEpochs) seconds. + * {@code inc} always updates the accumulator for the current epoch. If the given timestamp indicates + * an epoch rollover, the previous value of the accumulator is stored in the appropriate epoch. + */ +public class TimeSeriesCounter { + public final static LongSupplier SYSTEM_SECONDS_TIME_PROVIDER = () -> System.currentTimeMillis() / 1000; + public final static int MINUTE = 60; + public final static int HOUR = 60 * MINUTE; + + protected final long resolutionSecs; + protected final int[] epochs; + + // TOOD(stu): Use ClusterService.getClusterApplierService().threadPool().absoluteTimeInMillis() + protected final LongSupplier timeProvider; + + protected final ReentrantReadWriteLock lock; + + protected long currentEpochStart; + protected final LongAdder currentEpochAdder; + protected final TimeSeriesCounter currentEpochTimeSeries; + + TimeSeriesCounter(int numEpochs, int resolutionSecs, LongSupplier timeProvider) { + this(numEpochs, resolutionSecs, timeProvider, new ReentrantReadWriteLock()); + } + + private TimeSeriesCounter(int numEpochs, int resolutionSecs, LongSupplier timeProvider, ReentrantReadWriteLock lock) { + assert numEpochs > 0; + assert resolutionSecs > 0; + this.epochs = new int[numEpochs]; + this.resolutionSecs = resolutionSecs; + this.timeProvider = timeProvider; + this.lock = lock; + this.currentEpochAdder = new LongAdder(); + this.currentEpochTimeSeries = null; + } + + TimeSeriesCounter(int numEpochs, int resolutionSecs, LongSupplier timeProvider, int subNumEpochs, int subResolutionSecs) { + assert numEpochs > 0; + assert resolutionSecs > 0; + if (subNumEpochs * subResolutionSecs != resolutionSecs) { + throw new IllegalArgumentException("sub counter with" + + " resolution [" + subResolutionSecs + "] and numEpochs [" + subNumEpochs + "]" + + " does not cover one epoch for TimeSeriesCounter with " + + " resolution [" + resolutionSecs + "] and numEpochs [" + numEpochs + "]"); + } + this.epochs = new int[numEpochs]; + this.resolutionSecs = resolutionSecs; + this.timeProvider = timeProvider; + this.lock = new ReentrantReadWriteLock(); + this.currentEpochAdder = null; + this.currentEpochTimeSeries = new TimeSeriesCounter(subNumEpochs, subResolutionSecs, timeProvider, null); + } + + public static TimeSeriesCounter nestedCounter(int numEpochs, int resolutionSecs, int subNumEpochs, int subResolutionSecs) { + return new TimeSeriesCounter(numEpochs, resolutionSecs, SYSTEM_SECONDS_TIME_PROVIDER, subNumEpochs, subResolutionSecs); + } + + /** + * Increment the counter at the current time + */ + public void inc() { + inc(timeProvider.getAsLong()); + } + + /** + * Increment the counter at the given time + */ + public void inc(long nowSecs) { + assert nowSecs >= 0; + if (lock != null) { + lock.writeLock().lock(); + } + try { + int gap = epochsBetween(nowSecs); + + if (gap < -1) { // Excessive negative gap + start(nowSecs); + } else if (gap == -1) { // Clamp small negative jitter to current epoch + incAccumulator(currentEpochStart); + } else if (gap == 0) { + incAccumulator(nowSecs); + } else if (gap > epochs.length) { // Initialization or history expired + start(nowSecs); + } else { + int currentEpochIndex = storeAccumulator(); + for (int i = 1; i <= gap; i++) { + epochs[(currentEpochIndex + i) % epochs.length] = 0; + } + incAccumulator(nowSecs); + currentEpochStart = epochStartMillis(nowSecs); + } + } finally { + if (lock != null) { + lock.writeLock().unlock(); + } + } + } + + /** + * Get the current timestamp from the timeProvider, for consistent reads across different counters + */ + public long timestamp() { + return timeProvider.getAsLong(); + } + + /** + * Get the counts at time {@code now} for each timePeriod, as number of seconds ago until now. + */ + public int[] counts(long now, int ... timePeriods) { + if (lock != null) { + lock.readLock().lock(); + } + try { + int[] countsForPeriod = new int[timePeriods.length]; + for (int i = 0; i < countsForPeriod.length; i++) { + countsForPeriod[i] = count(now - timePeriods[i], now); + } + return countsForPeriod; + } finally { + if (lock != null) { + lock.readLock().unlock(); + } + } + } + + /** + * Get the counts for each timePeriod (as seconds ago) in timePeriods + */ + public int[] counts(int ... timePeriods) { + return counts(timeProvider.getAsLong(), timePeriods); + } + + /** + * Count the entire contents of the time series, for testing + */ + int count() { + if (lock != null) { + lock.readLock().lock(); + } + try { + int count = 0; + for (int j : epochs) { + count += j; + } + return count + currentEpochCount(); + } finally { + if (lock != null) { + lock.readLock().unlock(); + } + } + } + + /** + * Get the count between two times, clamped to the resolution of the counters. + */ + protected int count(long start, long end) { + if (end < start) { + throw new IllegalArgumentException("start [" + start + "] must be before end [" + end + "]"); + } + + // Clamp to range + long earliestTimeStamp = beginningOfTimeSeries(); + if (start < earliestTimeStamp) { + if (end < earliestTimeStamp) { + return 0; + } + start = earliestTimeStamp; + } + long latestTimeStamp = endOfTimeSeries(); + if (end > latestTimeStamp) { + if (start > latestTimeStamp) { + return 0; + } + end = latestTimeStamp; + } + + int total = 0; + // handle the current bucket + if (start <= latestTimeStamp && end >= currentEpochStart) { + + } + if (end >= currentEpochStart) { + if (currentEpochTimeSeries != null) { + total += currentEpochTimeSeries.count(currentEpochStart, end); + } else { + total += currentEpochAdderCount(); + } + end = currentEpochStart - 1; + // only covers one bucket + if (end < start) { + return total; + } + } + + // handle the rest of the buckets, end guaranteed to stop before current bucket + int numEpochs = epochsBetween(start, end); + if (numEpochs < 0 || numEpochs >= epochs.length) { + return 0; + } + int startEpoch = epochIndex(start); + for (int i = 0; i < numEpochs; i++) { + total += epochs[(startEpoch + i) % epochs.length]; + } + return total; + } + + /** + * The earliest millisecond valid for this time series. + */ + public long beginningOfTimeSeries() { + return currentEpochStart - (resolutionSecs * (epochs.length - 1)); + } + + /** + * The latest millisecond valid for this time series. + */ + public long endOfTimeSeries() { + return currentEpochStart + resolutionSecs - 1; + } + + long getCurrentEpochStart() { + return currentEpochStart; + } + + /** + * reset the accumulator and all arrays + */ + protected void reset() { + clearAccumulator(); + Arrays.fill(epochs, 0); + } + + /** + * get the count of the current epoch accumulator, long adder or nested TimeSeriesCounter + */ + protected int currentEpochCount() { + if (currentEpochAdder != null) { + long sum = currentEpochAdder.sum(); + if (sum > Integer.MAX_VALUE) { + return Integer.MAX_VALUE; + } + return (int) sum; + } else { + assert currentEpochTimeSeries != null; + return currentEpochTimeSeries.count(); + } + } + + protected int currentEpochAdderCount() { + if (currentEpochAdder == null) { + return 0; + } + long sum = currentEpochAdder.sum(); + if (sum > Integer.MAX_VALUE) { + return Integer.MAX_VALUE; + } else if (sum < 0) { + return 0; + } + return (int) sum; + } + + /** + * increment current epoch accumulator, long adder or nested TimeSeriesCounter + */ + protected void incAccumulator(long time) { + if (currentEpochAdder != null) { + currentEpochAdder.increment(); + } else { + assert currentEpochTimeSeries != null; + currentEpochTimeSeries.inc(time); + } + } + + /** + * clear the current epoch accumulator, long adder or nested TimeSeriesCounter + */ + protected void clearAccumulator() { + if (currentEpochAdder != null) { + currentEpochAdder.reset(); + } else { + assert currentEpochTimeSeries != null; + currentEpochTimeSeries.reset(); + } + } + + /** + * How many milliseconds of history does this {@code TimeSeriesCounter} cover? + */ + protected long duration() { + return epochs.length * resolutionSecs; + } + + /** + * Index in the epoch array for the given time + */ + protected int epochIndex(long millis) { + return (int)((millis / resolutionSecs) % epochs.length); + } + + /** + * The beginning of the epoch given by {@code nowMillis} + */ + protected long epochStartMillis(long nowMillis) { + return (nowMillis / resolutionSecs) * resolutionSecs; + } + + /** + * Starts the TimeSeries at {@code nowMillis} + */ + protected void start(long nowMillis) { + reset(); + currentEpochStart = epochStartMillis(nowMillis); + incAccumulator(nowMillis); + } + + /** + * Store the {@code currentEpochAccumulator} into the {@code epoch} history array at the appropriate index before + * moving on to a new epoch. + * + * This overrides the previous value in that index (expected to be zero), so calling this multiple times for the + * same epoch will lose counts. + */ + protected int storeAccumulator() { + int currentEpochIndex = epochIndex(currentEpochStart); + epochs[currentEpochIndex] = currentEpochCount(); + clearAccumulator(); + return currentEpochIndex; + } + + /** + * The number of epochs between {@code currentEpochStart} and the given time. Clamped to the range [Int.MAX, Int.Min]. + */ + protected int epochsBetween(long nowMillis) { + return epochsBetween(currentEpochStart, nowMillis); + } + + /** + * The number of epochs between {@code start} and {@code end}. Clamped to the range [Int.MAX, Int.Min]. + */ + protected int epochsBetween(long start, long end) { + long gap = (end / resolutionSecs) - (start / resolutionSecs); + if (gap > Integer.MAX_VALUE) { + return Integer.MAX_VALUE; + } else if (gap < Integer.MIN_VALUE) { + return Integer.MIN_VALUE; + } + return (int) gap; + } + + @Override + public String toString() { + return "TimeSeriesCounter{" + + "resolutionSecs=" + resolutionSecs + + ", epochs=" + Arrays.toString(epochs) + + ", currentEpochStart=" + currentEpochStart + + ", currentEpochAdder=" + currentEpochAdder + + ", currentEpochTimeSeries=" + currentEpochTimeSeries + + '}'; + } +} diff --git a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTest.java b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTest.java new file mode 100644 index 0000000000000..a0cf6234ea98a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTest.java @@ -0,0 +1,290 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.LongSupplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class TimeSeriesCounterTest extends ESTestCase { + protected final int RES = 15; + protected final long NOW = randomLongBetween(1632935764L, 16329357645L + randomLongBetween(1L, RES * 1_000_000)); + protected final TimeProvider time = new TimeProvider(); + protected final int NUM_ENTRIES = 5; + protected TimeSeriesCounter ts; + protected boolean denseSeries; + + @Override + public void setUp() throws Exception { + setUpLongAdder(); + denseSeries = randomBoolean(); + super.setUp(); + } + + protected void setUpLongAdder() { + ts = new TimeSeriesCounter(NUM_ENTRIES, RES, time); + } + + protected void setUpSubCounter() { + ts = new TimeSeriesCounter(NUM_ENTRIES, RES, time, RES / 3, 3); + } + + // The start of the time series + protected long start() { + return (NOW / RES) * RES; + } + + // Generate a sorted random series of events for a given bucket + protected int bucket(int bucket, int count) { + List longs = new ArrayList<>(); + long start = start() + ((long) bucket * RES); + long end = start + RES - 1; + for (int i = 0; i < count; i++) { + longs.add(randomLongBetween(start, end)); + } + longs.sort(Long::compare); + time.times.addAll(longs); + return count; + } + + protected int[] randomSeries() { + int[] counts = new int[NUM_ENTRIES]; + counts[0] = bucket(0, randomIntBetween(0, 5)); + for (int i = 1; i < NUM_ENTRIES; i += denseSeries ? 1 : randomIntBetween(1, NUM_ENTRIES)) { + counts[i] = bucket(i, randomIntBetween(1, 20)); + } + return counts; + } + + /** + * Test that increments in the current bucket count correctly + */ + public void testCurrentBucket() { + bucket(0, 2); + ts.inc(); + ts.inc(); + assertEquals(2, ts.count()); + } + + public void testCurrentBucketSubCounter() { + setUpSubCounter(); + bucket(0, 2); + ts.inc(); + ts.inc(); + assertEquals(2, ts.count()); + } + + /** + * Test that increments that roll over to the next bucket count correctly + */ + public void testNextBucket() { + int total = bucket(0, randomIntBetween(1, 20)); + total += bucket(1, randomIntBetween(1, 20)); + incTS(total); + assertEquals(total, ts.count()); + } + + /** + * Test that increments that roll over to the next bucket count correctly + */ + public void testNextBucketSubCounter() { + setUpSubCounter(); + int total = bucket(0, randomIntBetween(1, 20)); + total += bucket(1, randomIntBetween(1, 20)); + incTS(total); + assertEquals(total, ts.count()); + } + + /** + * Test that buckets are skipped + */ + public void testGapBucket() { + int total = bucket(0, randomIntBetween(1, 20)); + for (int i = 1; i < NUM_ENTRIES; i += randomIntBetween(1, 3)) { + total += bucket(i, randomIntBetween(1, 5)); + } + for (long t : time.times) { + ts.inc(t); + } + assertEquals(total, ts.count()); + } + + /** + * Test that buckets are skipped + */ + public void testGapBucketSubCounter() { + setUpSubCounter(); + int total = bucket(0, randomIntBetween(1, 20)); + for (int i = 1; i < NUM_ENTRIES; i += randomIntBetween(1, 3)) { + total += bucket(i, randomIntBetween(1, 5)); + } + for (long t : time.times) { + ts.inc(t); + } + assertEquals(total, ts.count()); + } + + /** + * Test that a big gap forward in time clears the old history + */ + public void testHistoryExpired() { + int[] oldCount = randomSeries(); + incTS(oldCount); + int nextBucket = randomIntBetween(2 * NUM_ENTRIES - 1, 3 * NUM_ENTRIES); + int total = bucket(nextBucket, randomIntBetween(0, 20)); + incTS(total); + assertEquals(total, ts.count()); + } + + /** + * Test that a big gap forward in time clears the old history + */ + public void testHistoryExpiredSubCounter() { + setUpSubCounter(); + int[] oldCount = randomSeries(); + incTS(oldCount); + int nextBucket = randomIntBetween(2 * NUM_ENTRIES - 1, 3 * NUM_ENTRIES); + int total = bucket(nextBucket, randomIntBetween(0, 20)); + incTS(total); + assertEquals(total, ts.count()); + } + + /** + * Test that epochs roll out of a full history as later epochs are added + */ + public void testHistoryRollover() { + for (int i = 0; i < NUM_ENTRIES; i++) { + // Fill history + int[] oldCount = randomSeries(); + incTS(oldCount); + // Add more epochs + int[] updates = new int[i + 1]; + int total = 0; + for (int j = 0; j <= i; j++) { + updates[j] = bucket(NUM_ENTRIES + j, randomIntBetween(1, 20)); + total += updates[j]; + } + incTS(updates); + // New epochs should cause old ones to be skipped + assertEquals(IntStream.of(oldCount).skip(i + 1).sum() + total, ts.count()); + } + } + + /** + * Test that epochs roll out of a full history as later epochs are added + */ + public void testHistoryRolloverSubCounter() { + setUpSubCounter(); + for (int i = 0; i < NUM_ENTRIES; i++) { + // Fill history + int[] oldCount = randomSeries(); + incTS(oldCount); + // Add more epochs + int[] updates = new int[i + 1]; + int total = 0; + for (int j = 0; j <= i; j++) { + updates[j] = bucket(NUM_ENTRIES + j, randomIntBetween(1, 20)); + total += updates[j]; + } + incTS(updates); + // New epochs should cause old ones to be skipped + assertEquals(IntStream.of(oldCount).skip(i + 1).sum() + total, ts.count()); + } + } + + /** + * Test that a gap backwards of more than one epoch resets + */ + public void testExcessiveNegativeGap() { + int[] count = randomSeries(); + incTS(count); + int total = IntStream.of(count).sum(); + assertEquals(total, ts.count()); + time.times.add(ts.getCurrentEpochStart() + randomIntBetween(-4 * RES, -1 * RES - 1)); + ts.inc(); + assertEquals(1, ts.count()); + } + + /** + * Test that a gap backwards of more than one epoch resets + */ + public void testExcessiveNegativeGapSubCounter() { + setUpSubCounter(); + int[] count = randomSeries(); + incTS(count); + int total = IntStream.of(count).sum(); + assertEquals(total, ts.count()); + time.times.add(ts.getCurrentEpochStart() + randomIntBetween(-4 * RES, -1 * RES - 1)); + ts.inc(); + assertEquals(1, ts.count()); + } + + /** + * Test that a gap backwards of more at most one epoch resets + */ + public void testSmallNegativeGap() { + int[] count = randomSeries(); + incTS(count); + int total = IntStream.of(count).sum(); + assertEquals(total, ts.count()); + int backwards = randomIntBetween(1, 5); + for (int i = 0; i < backwards; i ++) { + time.times.add(ts.getCurrentEpochStart() + randomIntBetween(-1 * RES, 0)); + } + incTS(backwards); + assertEquals(total + backwards, ts.count()); + } + + /** + * Test that a gap backwards of more at most one epoch resets + */ + public void testSmallNegativeGapSubCounter() { + setUpSubCounter(); + int[] count = randomSeries(); + incTS(count); + int total = IntStream.of(count).sum(); + assertEquals(total, ts.count()); + int backwards = randomIntBetween(1, 5); + for (int i = 0; i < backwards; i ++) { + time.times.add(ts.getCurrentEpochStart() + randomIntBetween(-1 * RES, 0)); + } + incTS(backwards); + assertEquals(total + backwards, ts.count()); + } + + protected void incTS(int[] count) { + for (int cnt : count) { + incTS(cnt); + } + } + + protected void incTS(int count) { + for (int i = 0; i < count; i++) { + ts.inc(); + } + } + + public static class TimeProvider implements LongSupplier { + public final List times = new ArrayList<>(); + public int i = 0; + + @Override + public long getAsLong() { + assert times.size() > 0; + if (i >= times.size()) { + return times.get(times.size() - 1); + } + return times.get(i++); + } + } +} From 829523e8d1317214bfd89e499a143a4963c063dd Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 13 Oct 2021 10:07:53 -0500 Subject: [PATCH 02/26] checkstyle --- .../java/org/elasticsearch/script/TimeSeriesCounter.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index b7bddd8647d9c..8c96b4fc75cf6 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -8,9 +8,7 @@ package org.elasticsearch.script; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.LongSupplier; @@ -21,9 +19,9 @@ * an epoch rollover, the previous value of the accumulator is stored in the appropriate epoch. */ public class TimeSeriesCounter { - public final static LongSupplier SYSTEM_SECONDS_TIME_PROVIDER = () -> System.currentTimeMillis() / 1000; - public final static int MINUTE = 60; - public final static int HOUR = 60 * MINUTE; + public static final LongSupplier SYSTEM_SECONDS_TIME_PROVIDER = () -> System.currentTimeMillis() / 1000; + public static final int MINUTE = 60; + public static final int HOUR = 60 * MINUTE; protected final long resolutionSecs; protected final int[] epochs; From 610b95d4d7dd33e6983d3d29be3e1fa34138a02e Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 13 Oct 2021 13:24:18 -0500 Subject: [PATCH 03/26] remove empty if --- .../main/java/org/elasticsearch/script/TimeSeriesCounter.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index 8c96b4fc75cf6..b0d678f259dab 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -191,10 +191,6 @@ protected int count(long start, long end) { } int total = 0; - // handle the current bucket - if (start <= latestTimeStamp && end >= currentEpochStart) { - - } if (end >= currentEpochStart) { if (currentEpochTimeSeries != null) { total += currentEpochTimeSeries.count(currentEpochStart, end); From c36d06087f912a2034b2ac8716a662ad58e24285 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 13 Oct 2021 13:40:35 -0500 Subject: [PATCH 04/26] Collectors import unused --- .../java/org/elasticsearch/script/TimeSeriesCounterTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTest.java b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTest.java index a0cf6234ea98a..96bbff6ff670e 100644 --- a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTest.java +++ b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTest.java @@ -13,7 +13,6 @@ import java.util.ArrayList; import java.util.List; import java.util.function.LongSupplier; -import java.util.stream.Collectors; import java.util.stream.IntStream; public class TimeSeriesCounterTest extends ESTestCase { From 795baaa3aeb79874f8b289b243bcfd3ce8419ebd Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 13 Oct 2021 15:59:08 -0500 Subject: [PATCH 05/26] TimeSeriesCounterTest -> TimeSeriesCounterTests --- .../{TimeSeriesCounterTest.java => TimeSeriesCounterTests.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename server/src/test/java/org/elasticsearch/script/{TimeSeriesCounterTest.java => TimeSeriesCounterTests.java} (99%) diff --git a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTest.java b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java similarity index 99% rename from server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTest.java rename to server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java index 96bbff6ff670e..171cc582cc976 100644 --- a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTest.java +++ b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java @@ -15,7 +15,7 @@ import java.util.function.LongSupplier; import java.util.stream.IntStream; -public class TimeSeriesCounterTest extends ESTestCase { +public class TimeSeriesCounterTests extends ESTestCase { protected final int RES = 15; protected final long NOW = randomLongBetween(1632935764L, 16329357645L + randomLongBetween(1L, RES * 1_000_000)); protected final TimeProvider time = new TimeProvider(); From 937037dffd2b791533cc3ddd576c45ba09aace26 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 13 Oct 2021 20:00:14 -0500 Subject: [PATCH 06/26] correctly name seconds, clamp sub series to latest --- .../script/TimeSeriesCounter.java | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index b0d678f259dab..64d4510289af9 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -92,9 +92,9 @@ public void inc(long nowSecs) { if (gap < -1) { // Excessive negative gap start(nowSecs); } else if (gap == -1) { // Clamp small negative jitter to current epoch - incAccumulator(currentEpochStart); + incLatestEpoch(); } else if (gap == 0) { - incAccumulator(nowSecs); + incLatestEpoch(); } else if (gap > epochs.length) { // Initialization or history expired start(nowSecs); } else { @@ -103,7 +103,7 @@ public void inc(long nowSecs) { epochs[(currentEpochIndex + i) % epochs.length] = 0; } incAccumulator(nowSecs); - currentEpochStart = epochStartMillis(nowSecs); + currentEpochStart = epochStartSeconds(nowSecs); } } finally { if (lock != null) { @@ -272,14 +272,30 @@ protected int currentEpochAdderCount() { } /** - * increment current epoch accumulator, long adder or nested TimeSeriesCounter + * increment current epoch accumulator, if nested, increment at a time, ensuring no erasure + */ + protected void incAccumulator(long now) { + if (currentEpochAdder != null) { + currentEpochAdder.increment(); + } else { + assert currentEpochTimeSeries != null; + if (now > currentEpochTimeSeries.currentEpochStart) { + currentEpochTimeSeries.inc(now); + } else { + currentEpochTimeSeries.incAccumulator(now); + } + } + } + + /** + * increment current epoch accumulator at the */ - protected void incAccumulator(long time) { + protected void incLatestEpoch() { if (currentEpochAdder != null) { currentEpochAdder.increment(); } else { assert currentEpochTimeSeries != null; - currentEpochTimeSeries.inc(time); + currentEpochTimeSeries.incLatestEpoch(); } } @@ -305,24 +321,24 @@ protected long duration() { /** * Index in the epoch array for the given time */ - protected int epochIndex(long millis) { - return (int)((millis / resolutionSecs) % epochs.length); + protected int epochIndex(long seconds) { + return (int)((seconds / resolutionSecs) % epochs.length); } /** - * The beginning of the epoch given by {@code nowMillis} + * The beginning of the epoch given by {@code startSeconds} */ - protected long epochStartMillis(long nowMillis) { - return (nowMillis / resolutionSecs) * resolutionSecs; + protected long epochStartSeconds(long startSeconds) { + return (startSeconds / resolutionSecs) * resolutionSecs; } /** - * Starts the TimeSeries at {@code nowMillis} + * Starts the TimeSeries at {@code startSeconds} */ - protected void start(long nowMillis) { + protected void start(long startSeconds) { reset(); - currentEpochStart = epochStartMillis(nowMillis); - incAccumulator(nowMillis); + currentEpochStart = epochStartSeconds(startSeconds); + incAccumulator(startSeconds); } /** @@ -342,8 +358,8 @@ protected int storeAccumulator() { /** * The number of epochs between {@code currentEpochStart} and the given time. Clamped to the range [Int.MAX, Int.Min]. */ - protected int epochsBetween(long nowMillis) { - return epochsBetween(currentEpochStart, nowMillis); + protected int epochsBetween(long nowSeconds) { + return epochsBetween(currentEpochStart, nowSeconds); } /** From ee1b1a7bbdcb75b9c671c8cab0274c7dc0d4cbf8 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 18 Oct 2021 09:11:04 -0500 Subject: [PATCH 07/26] Use threadpool for time and time ranges for counters --- .../PredicateTokenScriptFilterTests.java | 2 +- .../ScriptedConditionTokenFilterTests.java | 2 +- .../common/ScriptProcessorFactoryTests.java | 2 +- .../ingest/common/ScriptProcessorTests.java | 4 +- .../ingest/AbstractScriptTestCase.java | 2 +- .../java/org/elasticsearch/node/Node.java | 9 +- .../org/elasticsearch/script/ScriptCache.java | 6 +- .../elasticsearch/script/ScriptMetrics.java | 50 +- .../elasticsearch/script/ScriptService.java | 8 +- .../script/TimeSeriesCounter.java | 651 ++++++++++-------- .../action/update/UpdateRequestTests.java | 2 +- .../elasticsearch/index/IndexModuleTests.java | 2 +- .../index/mapper/MappingParserTests.java | 2 +- .../index/mapper/TestScriptEngine.java | 2 +- .../ingest/ConditionalProcessorTests.java | 12 +- .../ingest/IngestServiceTests.java | 5 +- .../ingest/TrackingResultProcessorTests.java | 16 +- .../script/ScriptCacheTests.java | 17 +- .../script/ScriptLanguagesInfoTests.java | 6 +- .../script/ScriptServiceTests.java | 2 +- .../script/TimeSeriesCounterTests.java | 320 +++------ .../missing/MissingAggregatorTests.java | 2 +- .../bucket/nested/NestedAggregatorTests.java | 2 +- .../bucket/terms/TermsAggregatorTests.java | 2 +- .../metrics/AvgAggregatorTests.java | 2 +- .../metrics/InternalScriptedMetricTests.java | 2 +- .../metrics/MaxAggregatorTests.java | 2 +- ...edianAbsoluteDeviationAggregatorTests.java | 2 +- .../metrics/MinAggregatorTests.java | 2 +- .../ScriptedMetricAggregatorTests.java | 4 +- .../metrics/StatsAggregatorTests.java | 2 +- .../metrics/SumAggregatorTests.java | 2 +- .../metrics/ValueCountAggregatorTests.java | 2 +- .../pipeline/BucketScriptAggregatorTests.java | 2 +- .../pipeline/MovFnAggrgatorTests.java | 2 +- .../search/sort/AbstractSortTestCase.java | 3 +- .../snapshots/SnapshotResiliencyTests.java | 2 +- .../ingest/TestTemplateService.java | 2 +- .../java/org/elasticsearch/node/MockNode.java | 6 +- .../script/MockScriptService.java | 2 +- .../boxplot/BoxplotAggregatorTests.java | 2 +- .../multiterms/MultiTermsAggregatorTests.java | 2 +- .../analytics/rate/RateAggregatorTests.java | 2 +- .../StringStatsAggregatorTests.java | 2 +- .../topmetrics/TopMetricsAggregatorTests.java | 2 +- .../analytics/ttest/TTestAggregatorTests.java | 2 +- .../support/mapper/TemplateRoleNameTests.java | 16 +- .../WildcardServiceProviderResolverTests.java | 2 +- .../authc/ldap/ActiveDirectoryRealmTests.java | 2 +- .../security/authc/ldap/LdapRealmTests.java | 2 +- .../mapper/NativeRoleMappingStoreTests.java | 2 +- .../watcher/support/WatcherTemplateTests.java | 2 +- .../watcher/test/WatcherMockScriptPlugin.java | 2 +- .../test/integration/SearchInputTests.java | 2 +- .../integration/SearchTransformTests.java | 2 +- .../script/ScriptTransformTests.java | 2 +- 56 files changed, 588 insertions(+), 625 deletions(-) diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java index 411d62d971cfa..23bffb476f73d 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java @@ -50,7 +50,7 @@ public boolean execute(Token token) { }; @SuppressWarnings("unchecked") - ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap()){ + ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L){ @Override public FactoryType compile(Script script, ScriptContext context) { assertEquals(context, AnalysisPredicateScript.CONTEXT); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java index 0475df5671a4b..8db0ab815fafb 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java @@ -50,7 +50,7 @@ public boolean execute(Token token) { }; @SuppressWarnings("unchecked") - ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap()){ + ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L){ @Override public FactoryType compile(Script script, ScriptContext context) { assertEquals(context, AnalysisPredicateScript.CONTEXT); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java index 676d16cbe26f0..978dd27a9fd19 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java @@ -136,7 +136,7 @@ Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine( }), Collections.emptyMap() ) - ), new HashMap<>(ScriptModule.CORE_CONTEXTS)); + ), new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L); factory = new ScriptProcessor.Factory(scriptService); Map configMap = new HashMap<>(); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java index bae8ed6f3023f..cb44c034053f1 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java @@ -53,8 +53,8 @@ Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine( Collections.emptyMap() ) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) - ); + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L); script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()); ingestScript = scriptService.compile(script, IngestScript.CONTEXT).newInstance(script.getParams()); } diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java index fb2bf9e2aee2b..eca978353fb0d 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java +++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java @@ -32,7 +32,7 @@ public abstract class AbstractScriptTestCase extends ESTestCase { public void init() throws Exception { MustacheScriptEngine engine = new MustacheScriptEngine(); Map engines = Collections.singletonMap(engine.getType(), engine); - scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } protected TemplateScript.Factory compile(String template) { diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 4135ef10fe57c..39f6ec049b976 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -195,6 +195,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Function; +import java.util.function.LongSupplier; import java.util.function.UnaryOperator; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -380,7 +381,8 @@ protected Node(final Environment initialEnvironment, client = new NodeClient(settings, threadPool); final ScriptModule scriptModule = new ScriptModule(settings, pluginsService.filterPlugins(ScriptPlugin.class)); - final ScriptService scriptService = newScriptService(settings, scriptModule.engines, scriptModule.contexts); + final ScriptService scriptService = + newScriptService(settings, scriptModule.engines, scriptModule.contexts, threadPool::absoluteTimeInMillis); AnalysisModule analysisModule = new AnalysisModule(this.environment, pluginsService.filterPlugins(AnalysisPlugin.class)); // this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool // so we might be late here already @@ -1199,8 +1201,9 @@ protected SearchService newSearchService(ClusterService clusterService, IndicesS /** * Creates a new the ScriptService. This method can be overwritten by tests to inject mock implementations. */ - protected ScriptService newScriptService(Settings settings, Map engines, Map> contexts) { - return new ScriptService(settings, engines, contexts); + protected ScriptService newScriptService(Settings settings, Map engines, Map> contexts, + LongSupplier timeProvider) { + return new ScriptService(settings, engines, contexts, timeProvider); } /** diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCache.java b/server/src/main/java/org/elasticsearch/script/ScriptCache.java index 622e9910ac372..161c110981692 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptCache.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptCache.java @@ -23,6 +23,7 @@ import java.util.Objects; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.LongSupplier; /** * Script cache and compilation rate limiter. @@ -48,7 +49,8 @@ public class ScriptCache { int cacheMaxSize, TimeValue cacheExpire, CompilationRate maxCompilationRate, - String contextRateSetting + String contextRateSetting, + LongSupplier timeProvider ) { this.cacheSize = cacheMaxSize; this.cacheExpire = cacheExpire; @@ -68,7 +70,7 @@ public class ScriptCache { this.rate = maxCompilationRate; this.compilesAllowedPerNano = ((double) rate.count) / rate.time.nanos(); - this.scriptMetrics = new ScriptMetrics(); + this.scriptMetrics = new ScriptMetrics(timeProvider); this.tokenBucketState = new AtomicReference(new TokenBucketState(this.rate.count)); } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java index 32a6a552e1575..883b1c6164c95 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java @@ -9,27 +9,40 @@ package org.elasticsearch.script; import org.elasticsearch.common.metrics.CounterMetric; +import static org.elasticsearch.script.TimeSeriesCounter.Snapshot; +import static org.elasticsearch.script.TimeSeriesCounter.SECOND; +import static org.elasticsearch.script.TimeSeriesCounter.MINUTE; +import static org.elasticsearch.script.TimeSeriesCounter.HOUR; + +import java.util.function.LongSupplier; public class ScriptMetrics { - final CounterMetric compilationsMetric = new CounterMetric(); - final CounterMetric cacheEvictionsMetric = new CounterMetric(); + protected static final int FIVE_MINUTES = 5 * MINUTE; + protected static final int FIFTEEN_MINUTES = 15 * MINUTE; + protected static final int TWENTY_FOUR_HOURS = 24 * HOUR; + final CounterMetric compilationLimitTriggered = new CounterMetric(); - final TimeSeriesCounter compilationsHistory = timeSeriesCounter(); - final TimeSeriesCounter cacheEvictionsHistory = timeSeriesCounter(); - final int[] TIME_PERIODS = { 5 * TimeSeriesCounter.MINUTE, 15 * TimeSeriesCounter.MINUTE, 24 * TimeSeriesCounter.HOUR }; + final TimeSeriesCounter compilations; + final TimeSeriesCounter cacheEvictions; + + protected final LongSupplier timeProvider; + + public ScriptMetrics(LongSupplier timeProvider) { + this.timeProvider = timeProvider; + this.compilations = timeSeriesCounter(); + this.cacheEvictions = timeSeriesCounter(); + } - public static TimeSeriesCounter timeSeriesCounter() { - return TimeSeriesCounter.nestedCounter(24 * 4, 15 * TimeSeriesCounter.MINUTE, 60, 15); + TimeSeriesCounter timeSeriesCounter() { + return new TimeSeriesCounter(TWENTY_FOUR_HOURS, 30 * MINUTE, 15 * SECOND, timeProvider); } public void onCompilation() { - compilationsMetric.inc(); - compilationsHistory.inc(); + compilations.inc(); } public void onCacheEviction() { - cacheEvictionsMetric.inc(); - cacheEvictionsHistory.inc(); + cacheEvictions.inc(); } public void onCompilationLimit() { @@ -37,15 +50,16 @@ public void onCompilationLimit() { } public ScriptContextStats stats(String context) { - long timestamp = compilationsHistory.timestamp(); - int[] compilations = compilationsHistory.counts(timestamp, TIME_PERIODS); - int[] cacheEvictions = cacheEvictionsHistory.counts(timestamp, TIME_PERIODS); + Snapshot compilation = compilations.snapshot(FIVE_MINUTES, FIFTEEN_MINUTES, TWENTY_FOUR_HOURS); + Snapshot cacheEviction = cacheEvictions.snapshot(compilation); return new ScriptContextStats( context, - compilationsMetric.count(), - cacheEvictionsMetric.count(), + compilation.total, + cacheEviction.total, compilationLimitTriggered.count(), - new ScriptContextStats.TimeSeries(compilations[0], compilations[1], compilations[2]), - new ScriptContextStats.TimeSeries(cacheEvictions[0], cacheEvictions[1], cacheEvictions[2]) + new ScriptContextStats.TimeSeries(compilation.getTime(FIVE_MINUTES), compilation.getTime(FIFTEEN_MINUTES), + compilation.getTime(TWENTY_FOUR_HOURS)), + new ScriptContextStats.TimeSeries(cacheEviction.getTime(FIVE_MINUTES), cacheEviction.getTime(FIFTEEN_MINUTES), + cacheEviction.getTime(TWENTY_FOUR_HOURS)) ); }} diff --git a/server/src/main/java/org/elasticsearch/script/ScriptService.java b/server/src/main/java/org/elasticsearch/script/ScriptService.java index d0e9f65425498..8c2374bf4262e 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptService.java @@ -44,6 +44,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import java.util.function.LongSupplier; import java.util.stream.Collectors; public class ScriptService implements Closeable, ClusterStateApplier, ScriptCompiler { @@ -96,6 +97,7 @@ public class ScriptService implements Closeable, ClusterStateApplier, ScriptComp private final Map engines; private final Map> contexts; + private final LongSupplier timeProvider; private ClusterState clusterState; @@ -104,7 +106,8 @@ public class ScriptService implements Closeable, ClusterStateApplier, ScriptComp // package private for tests final AtomicReference cacheHolder = new AtomicReference<>(); - public ScriptService(Settings settings, Map engines, Map> contexts) { + public ScriptService(Settings settings, Map engines, Map> contexts, + LongSupplier timeProvider) { this.engines = Collections.unmodifiableMap(Objects.requireNonNull(engines)); this.contexts = Collections.unmodifiableMap(Objects.requireNonNull(contexts)); @@ -182,6 +185,7 @@ public ScriptService(Settings settings, Map engines, Map context) { rate = new ScriptCache.CompilationRate(context.maxCompilationRateDefault); } - return new ScriptCache(cacheSize, cacheExpire, rate, rateSetting.getKey()); + return new ScriptCache(cacheSize, cacheExpire, rate, rateSetting.getKey(), timeProvider); } /** diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index 64d4510289af9..aab796432bccc 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -9,380 +9,475 @@ package org.elasticsearch.script; import java.util.Arrays; +import java.util.Objects; import java.util.concurrent.atomic.LongAdder; +import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.LongSupplier; /** - * A counter that keeps a time series history of (resolutionSecs * numEpochs) seconds. - * {@code inc} always updates the accumulator for the current epoch. If the given timestamp indicates - * an epoch rollover, the previous value of the accumulator is stored in the appropriate epoch. + * {@link TimeSeriesCounter} implements an event counter for keeping running stats at fixed intervals, such as 5m/15m/24h. + * Callers use {@link #inc()} to increment the counter at the current time, as supplied by {@link #timeProvider}. + * To fetch a count for a given time range, callers use {@link #count(long, long)}, providing an end time and a duration. + * + * {@link TimeSeriesCounter} has counts for the given duration at two different resolutions. From the last updated time, + * {@link #latestSec}, for {@link #lowSec} ({@code lowSecPerEpoch} in the constructor) previous seconds, the resolution is + * {@link #highSec} ({@code highSecPerEpoch} in the constructor). + * + * Outside of that range, but within {@code totalDuration} seconds, the resolution is {@link #lowSec}. + * + * This two ranges balance recent high resolution, historical range and memory usage. + * + * This class is implemented with two arrays, {@link #high}, {@link #low} and a {@link LongAdder}. + * + * All recent updates hit the LongAdder, {@link #adder} unless an increment causes a roll over to a new in high epoch or both + * a new high and new low epoch. + * + * The two arrays have overlapping time ranges. {@link #low} delegates its most recent low resolution epoch to {@link #high} + * and {@link #adder}. Similarly, {@link #high} delegates its most recent high resolution epoch to {@link #adder}. + * + * There are some useful time ranges to think about when reading this code: + * Total authority range - The range of validity of the time series. Ends within low resolution seconds of the last update and + * starts duration before the last update (plus or minus low resolution seconds. + * Adder authority range - {@link #adder} must be consulted if the queried time overlaps this range. + * High authority range - The {@link #high} array must be consulted if the queried time overlaps this range. + * This range may partially overlap the previous {@link #low} epoch because the high array + * does not necessarily reset when rolling over the low epoch. If we are only a few seconds + * into the new epoch, high keeps higher resolution counts. + * Low authority range - The {@link #low} array is the authority for counts within this range. + * High as low delegate - The latest low epoch is represented by a combination of the high array and adder. + * This range occurs immediately after the Low authority range. + * The two ranges together equal the Total authority range. + * Use {@link #sumHighDelegate} to get the correct count out of this range when combining with + * counts from previous low epochs. As mentioned above, high frequently overlaps with the + * last low epoch and naively counting all of high will lead to double counting. */ public class TimeSeriesCounter { - public static final LongSupplier SYSTEM_SECONDS_TIME_PROVIDER = () -> System.currentTimeMillis() / 1000; + public static final int SECOND = 1; public static final int MINUTE = 60; public static final int HOUR = 60 * MINUTE; - protected final long resolutionSecs; - protected final int[] epochs; + protected final long highSec; // high resolution in seconds + protected final int[] high; - // TOOD(stu): Use ClusterService.getClusterApplierService().threadPool().absoluteTimeInMillis() - protected final LongSupplier timeProvider; + protected final long lowSec; // low resolution in seconds + protected final int[] low; - protected final ReentrantReadWriteLock lock; + protected final ReadWriteLock lock = new ReentrantReadWriteLock(); + protected final LongSupplier timeProvider; - protected long currentEpochStart; - protected final LongAdder currentEpochAdder; - protected final TimeSeriesCounter currentEpochTimeSeries; + protected final LongAdder adder = new LongAdder(); // most recent high epoch + protected final LongAdder total = new LongAdder(); // the total number of increments + protected long latestSec; // most recent update time - TimeSeriesCounter(int numEpochs, int resolutionSecs, LongSupplier timeProvider) { - this(numEpochs, resolutionSecs, timeProvider, new ReentrantReadWriteLock()); - } - - private TimeSeriesCounter(int numEpochs, int resolutionSecs, LongSupplier timeProvider, ReentrantReadWriteLock lock) { - assert numEpochs > 0; - assert resolutionSecs > 0; - this.epochs = new int[numEpochs]; - this.resolutionSecs = resolutionSecs; - this.timeProvider = timeProvider; - this.lock = lock; - this.currentEpochAdder = new LongAdder(); - this.currentEpochTimeSeries = null; - } - - TimeSeriesCounter(int numEpochs, int resolutionSecs, LongSupplier timeProvider, int subNumEpochs, int subResolutionSecs) { - assert numEpochs > 0; - assert resolutionSecs > 0; - if (subNumEpochs * subResolutionSecs != resolutionSecs) { - throw new IllegalArgumentException("sub counter with" - + " resolution [" + subResolutionSecs + "] and numEpochs [" + subNumEpochs + "]" - + " does not cover one epoch for TimeSeriesCounter with " - + " resolution [" + resolutionSecs + "] and numEpochs [" + numEpochs + "]"); + /** + * Create a new time series that covers the given {@code totalDuration} in seconds, with the low resolution epochs covering + * {@code lowSecPerEpoch} seconds and the high resolution epochs cover {@code highSecPerEpoch}. + * + * Because the most recent low resolution epoch is covered completely by the high resolution epochs, {@code lowSecPerEpoch} + * must be divisible by {@code highSecPerEpoch}. + */ + public TimeSeriesCounter(long totalDuration, long lowSecPerEpoch, long highSecPerEpoch, LongSupplier timeProvider) { + if (totalDuration <= 0 || lowSecPerEpoch <= 0 || highSecPerEpoch <= 0) { + throw new IllegalArgumentException("totalDuration [" + totalDuration + "], lowSecPerEpoch [" + lowSecPerEpoch + + "], highSecPerEpoch[" + highSecPerEpoch + "] must be greater than zero"); + } else if (highSecPerEpoch > lowSecPerEpoch) { + throw new IllegalArgumentException("highSecPerEpoch [" + highSecPerEpoch + "] must be less than lowSecPerEpoch [" + + lowSecPerEpoch + "]"); + } else if (totalDuration % lowSecPerEpoch != 0) { + throw new IllegalArgumentException( + "totalDuration [" + totalDuration + "] must be divisible by lowSecPerEpoch [" + lowSecPerEpoch + "]"); + } else if (lowSecPerEpoch % highSecPerEpoch != 0) { + throw new IllegalArgumentException( + "lowSecPerEpoch [" + lowSecPerEpoch + "] must be divisible by highSecPerEpoch [" + highSecPerEpoch + "]"); } - this.epochs = new int[numEpochs]; - this.resolutionSecs = resolutionSecs; - this.timeProvider = timeProvider; - this.lock = new ReentrantReadWriteLock(); - this.currentEpochAdder = null; - this.currentEpochTimeSeries = new TimeSeriesCounter(subNumEpochs, subResolutionSecs, timeProvider, null); - } - - public static TimeSeriesCounter nestedCounter(int numEpochs, int resolutionSecs, int subNumEpochs, int subResolutionSecs) { - return new TimeSeriesCounter(numEpochs, resolutionSecs, SYSTEM_SECONDS_TIME_PROVIDER, subNumEpochs, subResolutionSecs); + this.timeProvider = Objects.requireNonNull(timeProvider); + this.lowSec = lowSecPerEpoch; + this.low = new int[(int) (totalDuration / lowSecPerEpoch)]; + this.highSec = highSecPerEpoch; + this.high = new int[(int) (lowSecPerEpoch / highSecPerEpoch)]; + assert high.length * highSecPerEpoch == lowSecPerEpoch; } /** - * Increment the counter at the current time + * Increment the counter at the current time. */ public void inc() { - inc(timeProvider.getAsLong()); + inc(now()); } /** - * Increment the counter at the given time + * Increment the counter at the given time. + * + * If {@code nowSec} is less than the last update, it is treated as an increment at the time of the last update + * unless it's before the begininng of this time series, in which case the time series is reset. */ - public void inc(long nowSecs) { - assert nowSecs >= 0; - if (lock != null) { - lock.writeLock().lock(); + public void inc(long nowSec) { + if (nowSec < 0) { + // The math below relies on now being positive + return; } + + lock.writeLock().lock(); + total.increment(); try { - int gap = epochsBetween(nowSecs); - - if (gap < -1) { // Excessive negative gap - start(nowSecs); - } else if (gap == -1) { // Clamp small negative jitter to current epoch - incLatestEpoch(); - } else if (gap == 0) { - incLatestEpoch(); - } else if (gap > epochs.length) { // Initialization or history expired - start(nowSecs); - } else { - int currentEpochIndex = storeAccumulator(); - for (int i = 1; i <= gap; i++) { - epochs[(currentEpochIndex + i) % epochs.length] = 0; + // Handle the busy case quickly + if (nowSec <= adderAuthorityEnd(latestSec) && nowSec >= adderAuthorityStart(latestSec)) { + adder.increment(); + } else if (nowSec < latestSec) { + if (nowSec < totalAuthorityStart(latestSec)) { + reset(nowSec); + } else { + adder.increment(); } - incAccumulator(nowSecs); - currentEpochStart = epochStartSeconds(nowSecs); + } else if (nowSec <= highDelegateEnd(latestSec)) { + rollForwardHigh(nowSec); + latestSec = nowSec; + adder.increment(); + } else { + rollForwardLow(nowSec); + rollForwardHigh(nowSec); + latestSec = nowSec; + adder.increment(); } } finally { - if (lock != null) { - lock.writeLock().unlock(); - } + lock.writeLock().unlock(); } } - /** - * Get the current timestamp from the timeProvider, for consistent reads across different counters - */ - public long timestamp() { - return timeProvider.getAsLong(); + // roll low forward such that t lowAuthorityEnd(t) + 1 = highDelegateStart(t) + // Assumes t >= latestSec. + void rollForwardLow(long t) { + if (totalAuthorityEnd(latestSec) < lowAuthorityStart(t)) { + // complete rollover + Arrays.fill(low, 0); + return; + } + int cur = lowIndex(latestSec); + int dst = lowIndex(t); + if (cur == dst) { + // no rollover + return; + } + + // grab the high + adder version of the current epoch + low[cur] = sumHighDelegate(latestSec); + cur = nextLowIndex(cur); + while (cur != dst) { + low[cur] = 0; + cur = nextLowIndex(cur); + } + // low[dst]'s contents is delegated to highDelegate + low[dst] = 0; } - /** - * Get the counts at time {@code now} for each timePeriod, as number of seconds ago until now. - */ - public int[] counts(long now, int ... timePeriods) { - if (lock != null) { - lock.readLock().lock(); + void rollForwardHigh(long t) { + if (highDelegateEnd(latestSec) < highAuthorityStart(t)) { + Arrays.fill(high, 0); + adder.reset(); + return; } - try { - int[] countsForPeriod = new int[timePeriods.length]; - for (int i = 0; i < countsForPeriod.length; i++) { - countsForPeriod[i] = count(now - timePeriods[i], now); - } - return countsForPeriod; - } finally { - if (lock != null) { - lock.readLock().unlock(); - } + int cur = highIndex(latestSec); + int dst = highIndex(t); + if (cur == dst) { + // no rollover + return; + } + + high[cur] = sumThenResetAdder(); + cur = nextHighIndex(cur); + while (cur != dst) { + high[cur] = 0; + cur = nextHighIndex(cur); } + // high[dst]'s contents is delegated to adder + high[dst] = 0; } /** - * Get the counts for each timePeriod (as seconds ago) in timePeriods + * reset the accumulator and all arrays, setting the latestSet to t and incrementing the adder. */ - public int[] counts(int ... timePeriods) { - return counts(timeProvider.getAsLong(), timePeriods); + protected void reset(long t) { + adder.reset(); + Arrays.fill(high, 0); + Arrays.fill(low, 0); + adder.increment(); + latestSec = t; } /** - * Count the entire contents of the time series, for testing + * Get the count for the time series ending at {@code end} for {@code duration} seconds beforehand. */ - int count() { - if (lock != null) { - lock.readLock().lock(); + public int count(long end, long duration) { + if (duration < 0 || end - duration < 0) { + return 0; // invalid range } + + lock.readLock().lock(); try { - int count = 0; - for (int j : epochs) { - count += j; + long start = end - duration; + + if (start >= highAuthorityStart(latestSec)) { + // entirely within high authority + return sumHighAuthority(start, end); } - return count + currentEpochCount(); - } finally { - if (lock != null) { - lock.readLock().unlock(); + int total = 0; + if (end >= highDelegateStart(latestSec)) { + total = sumHighDelegate(end); + end = lowAuthorityEnd(latestSec); } + return total + sumLow(start, end); + } finally { + lock.readLock().unlock(); } } /** - * Get the count between two times, clamped to the resolution of the counters. + * Get the total number of increments for all time. */ - protected int count(long start, long end) { - if (end < start) { - throw new IllegalArgumentException("start [" + start + "] must be before end [" + end + "]"); + public long total() { + long sum = total.sum(); + return sum >= 0 ? sum : 0; + } + + // sum high range representing the current low resolution epoch. + int sumHighDelegate(long t) { + int delegateIndex = highIndex(Math.min(t, latestSec)); + int total = sumAdder(); + for (int i = 0; i < delegateIndex; i++) { + total += high[i]; } + return total; + } - // Clamp to range - long earliestTimeStamp = beginningOfTimeSeries(); - if (start < earliestTimeStamp) { - if (end < earliestTimeStamp) { - return 0; - } - start = earliestTimeStamp; + // sum within the high range's authority. Should not be combined with any + // low epoch counts as the high range's authority may overlap with the + // previous low epoch. + int sumHighAuthority(long start, long end) { + if (start > end) { + return 0; } - long latestTimeStamp = endOfTimeSeries(); - if (end > latestTimeStamp) { - if (start > latestTimeStamp) { - return 0; - } - end = latestTimeStamp; + + long authorityStart = highAuthorityStart(latestSec); + long authorityEnd = adderAuthorityEnd(latestSec); + + if (end < authorityStart) { + return 0; + } else if (end > authorityEnd) { + end = authorityEnd; + } + + if (start > authorityEnd) { + return 0; + } else if (start < authorityStart) { + start = authorityStart; } + int delegateIndex = highIndex(latestSec); + int cur = highIndex(start); + int dst = highIndex(end); int total = 0; - if (end >= currentEpochStart) { - if (currentEpochTimeSeries != null) { - total += currentEpochTimeSeries.count(currentEpochStart, end); - } else { - total += currentEpochAdderCount(); - } - end = currentEpochStart - 1; - // only covers one bucket - if (end < start) { - return total; - } + while (cur != dst) { + total += cur == delegateIndex ? sumAdder() : high[cur]; + cur = (cur + 1) % high.length; } + return total + (cur == delegateIndex ? sumAdder() : high[cur]); + } - // handle the rest of the buckets, end guaranteed to stop before current bucket - int numEpochs = epochsBetween(start, end); - if (numEpochs < 0 || numEpochs >= epochs.length) { + // sum the low epochs represented by the given range + public int sumLow(long start, long end) { + if (start > end) { return 0; } - int startEpoch = epochIndex(start); - for (int i = 0; i < numEpochs; i++) { - total += epochs[(startEpoch + i) % epochs.length]; + + long authorityStart = lowAuthorityStart(latestSec); + long authorityEnd = lowAuthorityEnd(latestSec); + + if (end < authorityStart) { + return 0; + } else if (end > authorityEnd) { + end = authorityEnd; } - return total; + + if (start > authorityEnd) { + return 0; + } else if (start < authorityStart) { + start = authorityStart; + } + + int cur = lowIndex(start); + int dst = lowIndex(end); + int total = 0; + while (cur != dst) { + total += low[cur]; + cur = (cur + 1) % low.length; + } + return total + low[cur]; } - /** - * The earliest millisecond valid for this time series. - */ - public long beginningOfTimeSeries() { - return currentEpochStart - (resolutionSecs * (epochs.length - 1)); + // get the current time represented by timeProvider + protected long now() { + return timeProvider.getAsLong() / 1000; } - /** - * The latest millisecond valid for this time series. - */ - public long endOfTimeSeries() { - return currentEpochStart + resolutionSecs - 1; + // get the current sum from adder, clamped to the range [0, Integer.MAX_VALUE]. + // then reset the adder. This should only be called when rolling over. + protected int sumThenResetAdder() { + long sum = adder.sumThenReset(); + return sum > 0 ? (int) sum : 0; } - long getCurrentEpochStart() { - return currentEpochStart; + // get the current sum from adder, clamped to the range [0, Integer.MAX_VALUE]. + protected int sumAdder() { + long sum = adder.sum(); + return sum > 0 ? (int) sum : 0; } - /** - * reset the accumulator and all arrays - */ - protected void reset() { - clearAccumulator(); - Arrays.fill(epochs, 0); + // adder is the authority from this time until adderAuthorityEnd. + // Preceded by high authority range [highAuthorityStart, highAuthorityEnd]. + long adderAuthorityStart(long t) { + // authority for what _would be_ the latest high epoch + return (t / highSec) * highSec; } - /** - * get the count of the current epoch accumulator, long adder or nested TimeSeriesCounter - */ - protected int currentEpochCount() { - if (currentEpochAdder != null) { - long sum = currentEpochAdder.sum(); - if (sum > Integer.MAX_VALUE) { - return Integer.MAX_VALUE; - } - return (int) sum; - } else { - assert currentEpochTimeSeries != null; - return currentEpochTimeSeries.count(); - } + long adderAuthorityEnd(long t) { + return adderAuthorityStart(t) + highSec - 1; } - protected int currentEpochAdderCount() { - if (currentEpochAdder == null) { - return 0; - } - long sum = currentEpochAdder.sum(); - if (sum > Integer.MAX_VALUE) { - return Integer.MAX_VALUE; - } else if (sum < 0) { - return 0; - } - return (int) sum; + // high is the authority from his time until highAuthorityEnd. + // This range is proceeded by the adder authority range [adderAuthorityStart, adderAuthorityEnd] + long highAuthorityStart(long t) { + return adderAuthorityStart(t) - ((high.length - 1) * highSec); } - /** - * increment current epoch accumulator, if nested, increment at a time, ensuring no erasure - */ - protected void incAccumulator(long now) { - if (currentEpochAdder != null) { - currentEpochAdder.increment(); - } else { - assert currentEpochTimeSeries != null; - if (now > currentEpochTimeSeries.currentEpochStart) { - currentEpochTimeSeries.inc(now); - } else { - currentEpochTimeSeries.incAccumulator(now); - } - } + long highAuthorityEnd(long t) { + return adderAuthorityStart(t) - 1; } - /** - * increment current epoch accumulator at the - */ - protected void incLatestEpoch() { - if (currentEpochAdder != null) { - currentEpochAdder.increment(); - } else { - assert currentEpochTimeSeries != null; - currentEpochTimeSeries.incLatestEpoch(); - } + // The beginning of the range where high can combine with low to provide accurate counts. + // This range is preceded by the low authority range [lowAuthorityStart, lowAuthorityEnd] + long highDelegateStart(long t) { + return (t / lowSec) * lowSec; } - /** - * clear the current epoch accumulator, long adder or nested TimeSeriesCounter - */ - protected void clearAccumulator() { - if (currentEpochAdder != null) { - currentEpochAdder.reset(); - } else { - assert currentEpochTimeSeries != null; - currentEpochTimeSeries.reset(); - } + // The end of the high delegate range [highDelegateStart, highDelegateEnd]. There may + // not be counts for all parts of this range. This range only has valid counts until + // latestSec. + long highDelegateEnd(long t) { + return highDelegateStart(t) + lowSec - 1; } - /** - * How many milliseconds of history does this {@code TimeSeriesCounter} cover? - */ - protected long duration() { - return epochs.length * resolutionSecs; + // The beginning of the range where low has counts. + long lowAuthorityStart(long t) { + return totalAuthorityStart(t); } - /** - * Index in the epoch array for the given time - */ - protected int epochIndex(long seconds) { - return (int)((seconds / resolutionSecs) % epochs.length); + long lowAuthorityEnd(long t) { + return highDelegateStart(t) - 1; } - /** - * The beginning of the epoch given by {@code startSeconds} - */ - protected long epochStartSeconds(long startSeconds) { - return (startSeconds / resolutionSecs) * resolutionSecs; + // The range of times valid for this TimeSeriesCounter. + // Equal to [lowAuthorityStart, lowAuthorityEnd] + [highDelegateStart, highDelegateEnd] + long totalAuthorityStart(long t) { + return ((t / lowSec) * lowSec) - ((low.length - 1) * lowSec); } - /** - * Starts the TimeSeries at {@code startSeconds} - */ - protected void start(long startSeconds) { - reset(); - currentEpochStart = epochStartSeconds(startSeconds); - incAccumulator(startSeconds); + long totalAuthorityEnd(long t) { + return ((t / lowSec) * lowSec) + (lowSec - 1); } - /** - * Store the {@code currentEpochAccumulator} into the {@code epoch} history array at the appropriate index before - * moving on to a new epoch. - * - * This overrides the previous value in that index (expected to be zero), so calling this multiple times for the - * same epoch will lose counts. - */ - protected int storeAccumulator() { - int currentEpochIndex = epochIndex(currentEpochStart); - epochs[currentEpochIndex] = currentEpochCount(); - clearAccumulator(); - return currentEpochIndex; + // the index within the high array of the given time. + int highIndex(long t) { + return (int)((t / highSec) % high.length); } - /** - * The number of epochs between {@code currentEpochStart} and the given time. Clamped to the range [Int.MAX, Int.Min]. - */ - protected int epochsBetween(long nowSeconds) { - return epochsBetween(currentEpochStart, nowSeconds); + // the index within the low array of the given time. + int lowIndex(long t) { + return (int)((t / lowSec) % low.length); } - /** - * The number of epochs between {@code start} and {@code end}. Clamped to the range [Int.MAX, Int.Min]. - */ - protected int epochsBetween(long start, long end) { - long gap = (end / resolutionSecs) - (start / resolutionSecs); - if (gap > Integer.MAX_VALUE) { - return Integer.MAX_VALUE; - } else if (gap < Integer.MIN_VALUE) { - return Integer.MIN_VALUE; + // the next index within the high array, wrapping as appropriate + int nextHighIndex(int index) { + return (index + 1) % high.length; + } + + // the previous index within the high array, wrapping as appropriate + int prevHighIndex(int index) { + return index == 0 ? high.length - 1 : (index - 1) % high.length; + } + + // the next index within the low array, wrapping as appropriate + int nextLowIndex(int index) { + return (index + 1) % low.length; + } + + // the previous index within the low array, wrapping as appropriate + int prevLowIndex(int index) { + return index == 0 ? low.length - 1 : (index - 1) % low.length; + } + + public Snapshot snapshot(long ... times) { + return snapshot(now(), times); + } + + public Snapshot snapshot(Snapshot snapshot) { + return snapshot(snapshot.now, snapshot.times[0]); + } + + Snapshot snapshot(long now, long[] times) { + lock.readLock().lock(); + try { + Snapshot snapshot = new Snapshot(now, total(), times); + for (int i = 0; i < snapshot.times.length; i++) { + snapshot.counts[i] = count(snapshot.now, snapshot.times[i]); + } + return snapshot; + } finally { + lock.readLock().unlock(); } - return (int) gap; - } - - @Override - public String toString() { - return "TimeSeriesCounter{" + - "resolutionSecs=" + resolutionSecs + - ", epochs=" + Arrays.toString(epochs) + - ", currentEpochStart=" + currentEpochStart + - ", currentEpochAdder=" + currentEpochAdder + - ", currentEpochTimeSeries=" + currentEpochTimeSeries + - '}'; + } + + public static class Snapshot { + public final long now; + public final long total; + public final long[] times; + public final long[] counts; + public Snapshot(long now, long total, long ... times) { + this.now = now; + this.total = total; + this.times = new long[times.length]; + this.counts = new long[times.length]; + System.arraycopy(times, 0, this.times, 0, times.length); + } + + public long getTime(long time) { + for (int i = 0; i < times.length; i++) { + if (times[i] == time) { + return counts[i]; + } + } + return 0; + } + } + + // Testing and debugging methods + int getAdder() { + return adder.intValue(); + } + + int getLowLength() { + return low.length; + } + + int getHighLength() { + return high.length; + } + + long getHighSec() { + return highSec; + } + + long getLowSec() { + return lowSec; } } diff --git a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index a7b8f735d42c3..d4c87875892aa 100644 --- a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -126,7 +126,7 @@ public void setUp() throws Exception { scripts.put("return", vars -> null); final MockScriptEngine engine = new MockScriptEngine("mock", scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(engine.getType(), engine); - ScriptService scriptService = new ScriptService(baseSettings, engines, ScriptModule.CORE_CONTEXTS); + ScriptService scriptService = new ScriptService(baseSettings, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); updateHelper = new UpdateHelper(scriptService); } diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index dbb159953a4c6..87fe9da3ce22a 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -154,7 +154,7 @@ public void setUp() throws Exception { circuitBreakerService = new NoneCircuitBreakerService(); PageCacheRecycler pageCacheRecycler = new PageCacheRecycler(settings); bigArrays = new BigArrays(pageCacheRecycler, circuitBreakerService, CircuitBreaker.REQUEST); - scriptService = new ScriptService(settings, Collections.emptyMap(), Collections.emptyMap()); + scriptService = new ScriptService(settings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L); clusterService = ClusterServiceUtils.createClusterService(threadPool); nodeEnvironment = new NodeEnvironment(settings, environment); mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java index 209ee41fd07f1..47bfefee2e577 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java @@ -29,7 +29,7 @@ public class MappingParserTests extends MapperServiceTestCase { private static MappingParser createMappingParser(Settings settings) { - ScriptService scriptService = new ScriptService(settings, Collections.emptyMap(), Collections.emptyMap()); + ScriptService scriptService = new ScriptService(settings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L); IndexSettings indexSettings = createIndexSettings(Version.CURRENT, settings); IndexAnalyzers indexAnalyzers = createIndexAnalyzers(); SimilarityService similarityService = new SimilarityService(indexSettings, scriptService, Collections.emptyMap()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TestScriptEngine.java b/server/src/test/java/org/elasticsearch/index/mapper/TestScriptEngine.java index 7a4c16fe6b95a..5fd2ed03d9ff7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TestScriptEngine.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TestScriptEngine.java @@ -28,7 +28,7 @@ protected Object buildScriptFactory(ScriptContext context) { public Set> getSupportedContexts() { return Set.of(context); } - }), Map.of(context.name, context)); + }), Map.of(context.name, context), () -> 1L); } @Override diff --git a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java index 87687b4b2cdb5..67488a51e3b9e 100644 --- a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java @@ -60,8 +60,8 @@ public void testChecksCondition() throws Exception { Collections.emptyMap() ) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) - ); + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L); Map document = new HashMap<>(); LongSupplier relativeTimeProvider = mock(LongSupplier.class); when(relativeTimeProvider.getAsLong()).thenReturn(0L, TimeUnit.MILLISECONDS.toNanos(1), 0L, TimeUnit.MILLISECONDS.toNanos(2)); @@ -155,8 +155,8 @@ public void testTypeDeprecation() throws Exception { Collections.emptyMap() ) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) - ); + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L); LongSupplier relativeTimeProvider = mock(LongSupplier.class); when(relativeTimeProvider.getAsLong()).thenReturn(0L, TimeUnit.MILLISECONDS.toNanos(1), 0L, TimeUnit.MILLISECONDS.toNanos(2)); @@ -267,8 +267,8 @@ private static void assertMutatingCtxThrows(Consumer> mutati Collections.emptyMap() ) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) - ); + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L); Map document = new HashMap<>(); ConditionalProcessor processor = new ConditionalProcessor( randomAlphaOfLength(10), diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index e239b2dd987f7..a1f7d2d5d2428 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -91,7 +91,6 @@ import static org.hamcrest.Matchers.containsString; import static org.elasticsearch.core.Tuple.tuple; import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.emptyIterable; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -490,8 +489,8 @@ public void testGetProcessorsInPipelineComplexConditional() throws Exception { Collections.emptyMap() ) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS) - ); + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L); Map processors = new HashMap<>(); processors.put("complexSet", (factories, tag, description, config) -> { diff --git a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java index 0ee0928871ec5..f3142597b13e6 100644 --- a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java @@ -135,8 +135,8 @@ public void testActualCompoundProcessorWithOnFailureAndTrueCondition() throws Ex String scriptName = "conditionalScript"; ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> true), Collections.emptyMap())), - new HashMap<>(ScriptModule.CORE_CONTEXTS) - ); + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L); RuntimeException exception = new RuntimeException("fail"); TestProcessor failProcessor = new TestProcessor("fail", "test", null, exception); ConditionalProcessor conditionalProcessor = new ConditionalProcessor( @@ -204,8 +204,8 @@ public void testActualCompoundProcessorWithFalseConditional() throws Exception { String scriptName = "conditionalScript"; ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> false), Collections.emptyMap())), - new HashMap<>(ScriptModule.CORE_CONTEXTS) - ); + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L); CompoundProcessor compoundProcessor = new CompoundProcessor( new TestProcessor(ingestDocument -> {ingestDocument.setFieldValue(key1, randomInt()); }), @@ -309,8 +309,8 @@ public void testActualPipelineProcessorWithTrueConditional() throws Exception { ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> true), Collections.emptyMap())), - new HashMap<>(ScriptModule.CORE_CONTEXTS) - ); + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L); Pipeline pipeline1 = new Pipeline( pipelineId1, null, null, null, new CompoundProcessor( @@ -389,8 +389,8 @@ public void testActualPipelineProcessorWithFalseConditional() throws Exception { ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> false), Collections.emptyMap())), - new HashMap<>(ScriptModule.CORE_CONTEXTS) - ); + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L); Pipeline pipeline1 = new Pipeline( pipelineId1, null, null, null, new CompoundProcessor( diff --git a/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java b/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java index 6ee7149a460ec..82fccda1b700c 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java @@ -13,9 +13,11 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; +import java.util.function.LongSupplier; import java.util.stream.Collectors; public class ScriptCacheTests extends ESTestCase { + private static final LongSupplier time = () -> 1L; // even though circuit breaking is allowed to be configured per minute, we actually weigh this over five minutes // simply by multiplying by five, so even setting it to one, requires five compilations to break public void testCompilationCircuitBreaking() throws Exception { @@ -32,23 +34,26 @@ public void testCompilationCircuitBreaking() throws Exception { ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY); String rateSettingName = rateSetting.getKey(); ScriptCache cache = new ScriptCache(size, expire, - new ScriptCache.CompilationRate(1, TimeValue.timeValueMinutes(1)), rateSettingName); + new ScriptCache.CompilationRate(1, TimeValue.timeValueMinutes(1)), rateSettingName, time); cache.checkCompilationLimit(); // should pass expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), rateSettingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), rateSettingName, + time); cache.checkCompilationLimit(); // should pass cache.checkCompilationLimit(); // should pass expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); int count = randomIntBetween(5, 50); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(count, TimeValue.timeValueMinutes(1)), rateSettingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(count, TimeValue.timeValueMinutes(1)), rateSettingName, time); for (int i = 0; i < count; i++) { cache.checkCompilationLimit(); // should pass } expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), rateSettingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), rateSettingName, + time); expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); cache = new ScriptCache(size, expire, - new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)), rateSettingName); + new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)), rateSettingName, + time); int largeLimit = randomIntBetween(1000, 10000); for (int i = 0; i < largeLimit; i++) { cache.checkCompilationLimit(); @@ -64,7 +69,7 @@ public void testUnlimitedCompilationRate() { final Integer size = ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY); final TimeValue expire = ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY); String settingName = ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context).getKey(); - ScriptCache cache = new ScriptCache(size, expire, ScriptCache.UNLIMITED_COMPILATION_RATE, settingName); + ScriptCache cache = new ScriptCache(size, expire, ScriptCache.UNLIMITED_COMPILATION_RATE, settingName, time); ScriptCache.TokenBucketState initialState = cache.tokenBucketState.get(); for(int i=0; i < 3000; i++) { cache.checkCompilationLimit(); diff --git a/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java b/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java index e4ca0af2bb874..0d3532b009119 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java @@ -60,7 +60,7 @@ private ScriptService getMockScriptService(Settings settings) { Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(settings, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(settings, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } @@ -85,7 +85,7 @@ public void testOnlyScriptEngineContextsReturned() { Map> mockAndMiscContexts = new HashMap<>(mockContexts); mockAndMiscContexts.put(miscContext, new ScriptContext<>(miscContext, MiscContext.class)); - ScriptService ss = new ScriptService(Settings.EMPTY, engines, mockAndMiscContexts); + ScriptService ss = new ScriptService(Settings.EMPTY, engines, mockAndMiscContexts, () -> 1L); ScriptLanguagesInfo info = ss.getScriptLanguages(); assertTrue(info.languageContexts.containsKey(MockScriptEngine.NAME)); @@ -113,7 +113,7 @@ public void testContextsAllowedSettingRespected() { Map> mockAndMiscContexts = new HashMap<>(mockContexts); mockAndMiscContexts.put(miscContext, new ScriptContext<>(miscContext, MiscContext.class)); - ScriptService ss = new ScriptService(settings.build(), engines, mockAndMiscContexts); + ScriptService ss = new ScriptService(settings.build(), engines, mockAndMiscContexts, () -> 1L); ScriptLanguagesInfo info = ss.getScriptLanguages(); assertTrue(info.languageContexts.containsKey(MockScriptEngine.NAME)); diff --git a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java index 1ea9780c762a6..3b21b1ec0b07c 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java @@ -72,7 +72,7 @@ public void setup() throws IOException { private void buildScriptService(Settings additionalSettings) throws IOException { Settings finalSettings = Settings.builder().put(baseSettings).put(additionalSettings).build(); - scriptService = new ScriptService(finalSettings, engines, contexts) { + scriptService = new ScriptService(finalSettings, engines, contexts, () -> 1L) { @Override Map getScriptsFromClusterState() { Map scripts = new HashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java index 171cc582cc976..6e58787f7c29f 100644 --- a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java +++ b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java @@ -9,266 +9,100 @@ package org.elasticsearch.script; import org.elasticsearch.test.ESTestCase; +import org.junit.Before; +import static org.elasticsearch.script.TimeSeriesCounter.Snapshot; +import static org.elasticsearch.script.TimeSeriesCounter.MINUTE; +import static org.elasticsearch.script.TimeSeriesCounter.HOUR; import java.util.ArrayList; import java.util.List; import java.util.function.LongSupplier; -import java.util.stream.IntStream; public class TimeSeriesCounterTests extends ESTestCase { - protected final int RES = 15; - protected final long NOW = randomLongBetween(1632935764L, 16329357645L + randomLongBetween(1L, RES * 1_000_000)); - protected final TimeProvider time = new TimeProvider(); - protected final int NUM_ENTRIES = 5; + protected static final int totalDuration = 24 * HOUR; + protected static final int lowResSecPerEpoch = 30 * MINUTE; + protected static final int highResSecPerEpoch = 15; + protected static final int FIVE = 5 * MINUTE; + protected static final int FIFTEEN = 15 * MINUTE; + protected static final int TWENTY_FOUR = 24 * HOUR; + protected long now; protected TimeSeriesCounter ts; - protected boolean denseSeries; + protected TimeProvider t; @Override + @Before public void setUp() throws Exception { - setUpLongAdder(); - denseSeries = randomBoolean(); super.setUp(); + now = 16345080831234L; + t = new TimeProvider(); + ts = new TimeSeriesCounter(totalDuration, lowResSecPerEpoch, highResSecPerEpoch, t); } - protected void setUpLongAdder() { - ts = new TimeSeriesCounter(NUM_ENTRIES, RES, time); - } - - protected void setUpSubCounter() { - ts = new TimeSeriesCounter(NUM_ENTRIES, RES, time, RES / 3, 3); - } - - // The start of the time series - protected long start() { - return (NOW / RES) * RES; - } - - // Generate a sorted random series of events for a given bucket - protected int bucket(int bucket, int count) { - List longs = new ArrayList<>(); - long start = start() + ((long) bucket * RES); - long end = start + RES - 1; - for (int i = 0; i < count; i++) { - longs.add(randomLongBetween(start, end)); + public void testIncAdder() { + long start = ts.adderAuthorityStart(now); + t.add(now); + long highSec = ts.getHighSec(); + for (int i = 0; i < highSec; i++) { + t.add(start + i); } - longs.sort(Long::compare); - time.times.addAll(longs); - return count; + inc(); + assertEquals(highSec + 1, ts.count(now + highSec - 1, highSec - 1)); + assertEquals(highSec + 1, ts.getAdder()); } - protected int[] randomSeries() { - int[] counts = new int[NUM_ENTRIES]; - counts[0] = bucket(0, randomIntBetween(0, 5)); - for (int i = 1; i < NUM_ENTRIES; i += denseSeries ? 1 : randomIntBetween(1, NUM_ENTRIES)) { - counts[i] = bucket(i, randomIntBetween(1, 20)); + public void testIncAdderRollover() { + long start = ts.adderAuthorityStart(now); + long highSec = ts.getHighSec(); + t.add(now); + for (int i = 0; i < 2 * highSec; i++) { + t.add(start + i); } - return counts; - } - - /** - * Test that increments in the current bucket count correctly - */ - public void testCurrentBucket() { - bucket(0, 2); - ts.inc(); - ts.inc(); - assertEquals(2, ts.count()); - } - - public void testCurrentBucketSubCounter() { - setUpSubCounter(); - bucket(0, 2); - ts.inc(); - ts.inc(); - assertEquals(2, ts.count()); - } - - /** - * Test that increments that roll over to the next bucket count correctly - */ - public void testNextBucket() { - int total = bucket(0, randomIntBetween(1, 20)); - total += bucket(1, randomIntBetween(1, 20)); - incTS(total); - assertEquals(total, ts.count()); - } - - /** - * Test that increments that roll over to the next bucket count correctly - */ - public void testNextBucketSubCounter() { - setUpSubCounter(); - int total = bucket(0, randomIntBetween(1, 20)); - total += bucket(1, randomIntBetween(1, 20)); - incTS(total); - assertEquals(total, ts.count()); - } - - /** - * Test that buckets are skipped - */ - public void testGapBucket() { - int total = bucket(0, randomIntBetween(1, 20)); - for (int i = 1; i < NUM_ENTRIES; i += randomIntBetween(1, 3)) { - total += bucket(i, randomIntBetween(1, 5)); - } - for (long t : time.times) { - ts.inc(t); - } - assertEquals(total, ts.count()); - } - - /** - * Test that buckets are skipped - */ - public void testGapBucketSubCounter() { - setUpSubCounter(); - int total = bucket(0, randomIntBetween(1, 20)); - for (int i = 1; i < NUM_ENTRIES; i += randomIntBetween(1, 3)) { - total += bucket(i, randomIntBetween(1, 5)); - } - for (long t : time.times) { - ts.inc(t); - } - assertEquals(total, ts.count()); - } - - /** - * Test that a big gap forward in time clears the old history - */ - public void testHistoryExpired() { - int[] oldCount = randomSeries(); - incTS(oldCount); - int nextBucket = randomIntBetween(2 * NUM_ENTRIES - 1, 3 * NUM_ENTRIES); - int total = bucket(nextBucket, randomIntBetween(0, 20)); - incTS(total); - assertEquals(total, ts.count()); - } - - /** - * Test that a big gap forward in time clears the old history - */ - public void testHistoryExpiredSubCounter() { - setUpSubCounter(); - int[] oldCount = randomSeries(); - incTS(oldCount); - int nextBucket = randomIntBetween(2 * NUM_ENTRIES - 1, 3 * NUM_ENTRIES); - int total = bucket(nextBucket, randomIntBetween(0, 20)); - incTS(total); - assertEquals(total, ts.count()); - } - - /** - * Test that epochs roll out of a full history as later epochs are added - */ - public void testHistoryRollover() { - for (int i = 0; i < NUM_ENTRIES; i++) { - // Fill history - int[] oldCount = randomSeries(); - incTS(oldCount); - // Add more epochs - int[] updates = new int[i + 1]; - int total = 0; - for (int j = 0; j <= i; j++) { - updates[j] = bucket(NUM_ENTRIES + j, randomIntBetween(1, 20)); - total += updates[j]; - } - incTS(updates); - // New epochs should cause old ones to be skipped - assertEquals(IntStream.of(oldCount).skip(i + 1).sum() + total, ts.count()); - } - } - - /** - * Test that epochs roll out of a full history as later epochs are added - */ - public void testHistoryRolloverSubCounter() { - setUpSubCounter(); - for (int i = 0; i < NUM_ENTRIES; i++) { - // Fill history - int[] oldCount = randomSeries(); - incTS(oldCount); - // Add more epochs - int[] updates = new int[i + 1]; - int total = 0; - for (int j = 0; j <= i; j++) { - updates[j] = bucket(NUM_ENTRIES + j, randomIntBetween(1, 20)); - total += updates[j]; + inc(); + assertEquals(2 * highSec + 1, ts.count(now + 2 * highSec - 1, 2 * highSec - 1)); + assertEquals(highSec, ts.getAdder()); + } + + public void testIncHighRollover() { + long start = ts.adderAuthorityStart(now); + long highSec = ts.getHighSec(); + int highLength = ts.getHighLength(); + int count = 0; + t.add(now); + for (int i = 0; i < highLength + 1; i++) { + t.add(start + (i * highSec)); + if (i == highLength / 2 + 1) { + count = i + 1; } - incTS(updates); - // New epochs should cause old ones to be skipped - assertEquals(IntStream.of(oldCount).skip(i + 1).sum() + total, ts.count()); - } - } - - /** - * Test that a gap backwards of more than one epoch resets - */ - public void testExcessiveNegativeGap() { - int[] count = randomSeries(); - incTS(count); - int total = IntStream.of(count).sum(); - assertEquals(total, ts.count()); - time.times.add(ts.getCurrentEpochStart() + randomIntBetween(-4 * RES, -1 * RES - 1)); - ts.inc(); - assertEquals(1, ts.count()); - } - - /** - * Test that a gap backwards of more than one epoch resets - */ - public void testExcessiveNegativeGapSubCounter() { - setUpSubCounter(); - int[] count = randomSeries(); - incTS(count); - int total = IntStream.of(count).sum(); - assertEquals(total, ts.count()); - time.times.add(ts.getCurrentEpochStart() + randomIntBetween(-4 * RES, -1 * RES - 1)); - ts.inc(); - assertEquals(1, ts.count()); - } - - /** - * Test that a gap backwards of more at most one epoch resets - */ - public void testSmallNegativeGap() { - int[] count = randomSeries(); - incTS(count); - int total = IntStream.of(count).sum(); - assertEquals(total, ts.count()); - int backwards = randomIntBetween(1, 5); - for (int i = 0; i < backwards; i ++) { - time.times.add(ts.getCurrentEpochStart() + randomIntBetween(-1 * RES, 0)); - } - incTS(backwards); - assertEquals(total + backwards, ts.count()); - } - - /** - * Test that a gap backwards of more at most one epoch resets - */ - public void testSmallNegativeGapSubCounter() { - setUpSubCounter(); - int[] count = randomSeries(); - incTS(count); - int total = IntStream.of(count).sum(); - assertEquals(total, ts.count()); - int backwards = randomIntBetween(1, 5); - for (int i = 0; i < backwards; i ++) { - time.times.add(ts.getCurrentEpochStart() + randomIntBetween(-1 * RES, 0)); - } - incTS(backwards); - assertEquals(total + backwards, ts.count()); - } - - protected void incTS(int[] count) { - for (int cnt : count) { - incTS(cnt); } - } - - protected void incTS(int count) { - for (int i = 0; i < count; i++) { + inc(); + assertEquals(highLength + 2, ts.count(now + (highSec * highLength), (highSec * highLength))); + assertEquals(1, ts.getAdder()); + assertEquals(count, ts.count(now + (highSec * (highLength / 2)), highSec * (highLength / 2))); + } + + public void testSnapshot() { + t.add(now); + inc(); + t.add(now + 10); + Snapshot s = ts.snapshot(FIVE, FIFTEEN, TWENTY_FOUR); + assertEquals(1, s.getTime(FIVE)); + assertEquals(1, s.getTime(FIFTEEN)); + assertEquals(1, s.getTime(TWENTY_FOUR)); + } + + public void testRolloverCount() { + t.add(now); + inc(); + assertEquals(1, ts.count(now + 1, FIVE)); + assertEquals(0, ts.count(now + (2 * FIVE) + highResSecPerEpoch, FIVE)); + assertEquals(1, ts.count(now + 1, FIFTEEN)); + assertEquals(0, ts.count(now + (2 * FIFTEEN) + highResSecPerEpoch, FIFTEEN)); + assertEquals(1, ts.count(now + 1, HOUR)); + assertEquals(0, ts.count(now + (2 * HOUR) + highResSecPerEpoch, HOUR)); + } + + void inc() { + for (int i = 0; i < t.times.size(); i++) { ts.inc(); } } @@ -277,6 +111,10 @@ public static class TimeProvider implements LongSupplier { public final List times = new ArrayList<>(); public int i = 0; + public void add(long time) { + times.add(time * 1000); + } + @Override public long getAsLong() { assert times.size() > 0; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java index 3d38b40cb0945..b96c5e8f0d355 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java @@ -371,7 +371,7 @@ protected ScriptService getMockScriptService() { }); final MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, deterministicScripts, emptyMap(), emptyMap()); final Map engines = singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } private static List threshold(String fieldName, long threshold, Map vars) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java index f4935805cd5c8..271faf4776f0f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java @@ -131,7 +131,7 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testNoDocs() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index b5bb03405d957..0eb3aba58283c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -169,7 +169,7 @@ protected ScriptService getMockScriptService() { ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } protected A createAggregator(AggregationBuilder aggregationBuilder, AggregationContext context) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java index 754c14b9ca87b..55a344dd8c1e8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java @@ -122,7 +122,7 @@ protected ScriptService getMockScriptService() { ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testNoDocs() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java index ec91f60b63d0c..1e5cbf5a95191 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java @@ -142,7 +142,7 @@ protected ScriptService mockScriptService() { Collections.emptyMap() ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } @Override diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java index 5c6d8606f6e80..52652893ecc1d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java @@ -148,7 +148,7 @@ protected ScriptService getMockScriptService() { ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } @Override diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorTests.java index bbd209ea06b8b..22275cfc4876a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorTests.java @@ -325,6 +325,6 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java index 49824e84f130e..152e278b2c081 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java @@ -147,7 +147,7 @@ protected ScriptService getMockScriptService() { ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testNoMatchingField() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java index c92ce332dde7a..efc5946620716 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java @@ -276,7 +276,7 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } @SuppressWarnings("unchecked") @@ -427,7 +427,7 @@ public void testScriptParamsPassedThrough() throws IOException { public void testAggParamsPassedToReduceScript() throws IOException { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java index cce5f5cfb6afb..05c1667a5cc37 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java @@ -448,6 +448,6 @@ protected ScriptService getMockScriptService() { ); final MockScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts, emptyMap()); final Map engines = singletonMap(engine.getType(), engine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java index 58074ca1efe33..8cd8358cae043 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorTests.java @@ -406,7 +406,7 @@ protected ScriptService getMockScriptService() { ); final MockScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts, emptyMap()); final Map engines = singletonMap(engine.getType(), engine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } private static MappedFieldType defaultFieldType() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java index 068d65dd240a3..63e8b0ce7e651 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java @@ -96,7 +96,7 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testGeoField() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptAggregatorTests.java index d7a98379547af..61d3014251212 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptAggregatorTests.java @@ -57,7 +57,7 @@ protected ScriptService getMockScriptService() { ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testScript() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java index 541ef1cf36000..7dae864ee3340 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnAggrgatorTests.java @@ -77,7 +77,7 @@ protected ScriptService getMockScriptService() { ); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testMatchAllDocs() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 545b23e92ecc2..a28167b34d2ba 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -78,7 +78,8 @@ public static void init() { .build(); Map, Object>> scripts = Collections.singletonMap(MOCK_SCRIPT_NAME, p -> null); ScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); - scriptService = new ScriptService(baseSettings, Collections.singletonMap(engine.getType(), engine), ScriptModule.CORE_CONTEXTS); + scriptService = new ScriptService(baseSettings, Collections.singletonMap(engine.getType(), engine), ScriptModule.CORE_CONTEXTS, + () -> 1L); SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables()); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 05d77044ea94f..6f895d2d39271 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1720,7 +1720,7 @@ protected void assertSnapshotOrGenericThread() { ); nodeEnv = new NodeEnvironment(settings, environment); final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(Collections.emptyList()); - final ScriptService scriptService = new ScriptService(settings, emptyMap(), emptyMap()); + final ScriptService scriptService = new ScriptService(settings, emptyMap(), emptyMap(), () -> 1L); client = new NodeClient(settings, threadPool); final SetOnce rerouteServiceSetOnce = new SetOnce<>(); final SnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestTemplateService.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestTemplateService.java index 5093828384a54..b01f46a113d88 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/TestTemplateService.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestTemplateService.java @@ -32,7 +32,7 @@ public static ScriptService instance(boolean compilationException) { } private TestTemplateService(boolean compilationException) { - super(Settings.EMPTY, Collections.singletonMap(DEFAULT_TEMPLATE_LANG, new MockScriptEngine()), Collections.emptyMap()); + super(Settings.EMPTY, Collections.singletonMap(DEFAULT_TEMPLATE_LANG, new MockScriptEngine()), Collections.emptyMap(), () -> 1L); this.compilationException = compilationException; } diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java index 2d89b59a14f5c..e305b8bb88ef7 100644 --- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java +++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java @@ -49,6 +49,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; +import java.util.function.LongSupplier; /** * A node for testing which allows: @@ -129,9 +130,10 @@ protected SearchService newSearchService(ClusterService clusterService, IndicesS } @Override - protected ScriptService newScriptService(Settings settings, Map engines, Map> contexts) { + protected ScriptService newScriptService(Settings settings, Map engines, Map> contexts, + LongSupplier timeProvider) { if (getPluginsService().filterPlugins(MockScriptService.TestPlugin.class).isEmpty()) { - return super.newScriptService(settings, engines, contexts); + return super.newScriptService(settings, engines, contexts, timeProvider); } return new MockScriptService(settings, engines, contexts); } diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptService.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptService.java index 7cefdaa200d79..40437140b0b4c 100644 --- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptService.java +++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptService.java @@ -23,7 +23,7 @@ public class MockScriptService extends ScriptService { public static class TestPlugin extends Plugin {} public MockScriptService(Settings settings, Map engines, Map> contexts) { - super(settings, engines, contexts); + super(settings, engines, contexts, () -> 1L); } @Override diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java index bed0939510c6d..0ad234c425891 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java @@ -79,7 +79,7 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testNoMatchingField() throws IOException { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregatorTests.java index 926878d167ffb..d14bccd0f6058 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregatorTests.java @@ -112,7 +112,7 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testIntegersFloatsAndStrings() throws IOException { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java index b8cb03a8be5a2..4abfafa6cee02 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java @@ -107,7 +107,7 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testNoMatchingField() throws IOException { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java index 5b3509e6e59ef..21e75fbbd1dc2 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java @@ -423,6 +423,6 @@ protected ScriptService getMockScriptService() { ); final MockScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts, emptyMap()); final Map engines = singletonMap(engine.getType(), engine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java index ac30d0e079d63..042f85b8abd9c 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java @@ -585,7 +585,7 @@ protected ScriptService getMockScriptService() { return field.getValue(); }), emptyMap()); Map engines = singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } @Override diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java index 70f59362c4081..3ef0f4b64a3b4 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java @@ -138,7 +138,7 @@ protected ScriptService getMockScriptService() { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); - return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } public void testNoMatchingField() throws IOException { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java index df934673b3d40..aba002895dae2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java @@ -88,7 +88,7 @@ public void testEqualsAndHashCode() throws Exception { public void testEvaluateRoles() throws Exception { final ScriptService scriptService = new ScriptService(Settings.EMPTY, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS); + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS, () -> 1L); final ExpressionModel model = new ExpressionModel(); model.defineField("username", "hulk"); model.defineField("groups", Arrays.asList("avengers", "defenders", "panthenon")); @@ -139,7 +139,7 @@ public void tryEquals(TemplateRoleName original) { public void testValidate() { final ScriptService scriptService = new ScriptService(Settings.EMPTY, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS); + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS, () -> 1L); final TemplateRoleName plainString = new TemplateRoleName(new BytesArray("{ \"source\":\"heroes\" }"), Format.STRING); plainString.validate(scriptService); @@ -160,7 +160,7 @@ public void testValidate() { public void testValidateWillPassWithEmptyContext() { final ScriptService scriptService = new ScriptService(Settings.EMPTY, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS); + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS, () -> 1L); final BytesReference template = new BytesArray("{ \"source\":\"" + "{{username}}/{{dn}}/{{realm}}/{{metadata}}" + @@ -185,7 +185,7 @@ public void testValidateWillPassWithEmptyContext() { public void testValidateWillFailForSyntaxError() { final ScriptService scriptService = new ScriptService(Settings.EMPTY, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS); + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS, () -> 1L); final BytesReference template = new BytesArray("{ \"source\":\" {{#not-closed}} {{other-variable}} \" }"); @@ -210,7 +210,7 @@ public void testValidateWillCompileButNotExecutePainlessScript() { .when(scriptEngine).compile(eq("invalid"), eq("bad syntax"), any(), eq(Map.of())); final ScriptService scriptService = new ScriptService(Settings.EMPTY, - Map.of("painless", scriptEngine), ScriptModule.CORE_CONTEXTS) { + Map.of("painless", scriptEngine), ScriptModule.CORE_CONTEXTS, () -> 1L) { @Override protected StoredScriptSource getScriptFromClusterState(String id) { if ("valid".equals(id)) { @@ -235,7 +235,7 @@ protected StoredScriptSource getScriptFromClusterState(String id) { public void testValidationWillFailWhenInlineScriptIsNotEnabled() { final Settings settings = Settings.builder().put("script.allowed_types", ScriptService.ALLOW_NONE).build(); final ScriptService scriptService = new ScriptService(settings, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS); + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS, () -> 1L); final BytesReference inlineScript = new BytesArray("{ \"source\":\"\" }"); final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new TemplateRoleName(inlineScript, Format.STRING).validate(scriptService)); @@ -245,7 +245,7 @@ public void testValidationWillFailWhenInlineScriptIsNotEnabled() { public void testValidateWillFailWhenStoredScriptIsNotEnabled() { final Settings settings = Settings.builder().put("script.allowed_types", ScriptService.ALLOW_NONE).build(); final ScriptService scriptService = new ScriptService(settings, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS); + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS, () -> 1L); final ClusterChangedEvent clusterChangedEvent = mock(ClusterChangedEvent.class); final ClusterState clusterState = mock(ClusterState.class); final Metadata metadata = mock(Metadata.class); @@ -267,7 +267,7 @@ public void testValidateWillFailWhenStoredScriptIsNotEnabled() { public void testValidateWillFailWhenStoredScriptIsNotFound() { final ScriptService scriptService = new ScriptService(Settings.EMPTY, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS); + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS, () -> 1L); final ClusterChangedEvent clusterChangedEvent = mock(ClusterChangedEvent.class); final ClusterState clusterState = mock(ClusterState.class); final Metadata metadata = mock(Metadata.class); diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProviderResolverTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProviderResolverTests.java index baa8441f1f45f..add443622b35c 100644 --- a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProviderResolverTests.java +++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProviderResolverTests.java @@ -93,7 +93,7 @@ public class WildcardServiceProviderResolverTests extends IdpSamlTestCase { public void setUpResolver() { final Settings settings = Settings.EMPTY; final ScriptService scriptService = new ScriptService(settings, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS); + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS, () -> 1L); final ServiceProviderDefaults samlDefaults = new ServiceProviderDefaults("elastic-cloud", NameID.TRANSIENT, Duration.ofMinutes(15)); resolver = new WildcardServiceProviderResolver(settings, scriptService, new SamlServiceProviderFactory(samlDefaults)); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java index 63634fed0eaaf..64d9634be9d4d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java @@ -386,7 +386,7 @@ public void testRealmWithTemplatedRoleMapping() throws Exception { when(mockClient.threadPool()).thenReturn(threadPool); final ScriptService scriptService = new ScriptService(settings, Collections.singletonMap(MustacheScriptEngine.NAME, - new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS); + new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS, () -> 1L); NativeRoleMappingStore roleMapper = new NativeRoleMappingStore(settings, mockClient, mockSecurityIndex, scriptService) { @Override protected void loadMappings(ActionListener> listener) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java index f60dce4110bc4..d2123631e02f9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java @@ -449,7 +449,7 @@ public void testLdapRealmWithTemplatedRoleMapping() throws Exception { when(mockClient.threadPool()).thenReturn(threadPool); final ScriptService scriptService = new ScriptService(defaultGlobalSettings, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS); + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS, () -> 1L); NativeRoleMappingStore roleMapper = new NativeRoleMappingStore(defaultGlobalSettings, mockClient, mockSecurityIndex, scriptService) { @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index 2bea6f09aced0..356a20178ced1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -92,7 +92,7 @@ public void testResolveRoles() throws Exception { final Client client = mock(Client.class); SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); ScriptService scriptService = new ScriptService(Settings.EMPTY, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS); + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), ScriptModule.CORE_CONTEXTS, () -> 1L); when(securityIndex.isAvailable()).thenReturn(true); final NativeRoleMappingStore store = new NativeRoleMappingStore(Settings.EMPTY, client, securityIndex, scriptService) { diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java index 586157fff6d88..7655a4b10c03f 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java @@ -41,7 +41,7 @@ public void init() throws Exception { Map engines = Collections.singletonMap(engine.getType(), engine); Map> contexts = Collections.singletonMap(Watcher.SCRIPT_TEMPLATE_CONTEXT.name, Watcher.SCRIPT_TEMPLATE_CONTEXT); - ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, contexts); + ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, contexts, () -> 1L); textTemplateEngine = new TextTemplateEngine(scriptService); } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherMockScriptPlugin.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherMockScriptPlugin.java index b2f5751c34e31..fcc0c4f3f9c00 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherMockScriptPlugin.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherMockScriptPlugin.java @@ -64,6 +64,6 @@ public static ScriptService newMockScriptService(Map> contexts = CONTEXTS.stream().collect(Collectors.toMap(o -> o.name, Function.identity())); - return new ScriptService(Settings.EMPTY, engines, contexts); + return new ScriptService(Settings.EMPTY, engines, contexts, () -> 1L); } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java index a41dd6ff70665..9b097a6355c5e 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java @@ -78,7 +78,7 @@ public void setup() { Map> contexts = new HashMap<>(); contexts.put(Watcher.SCRIPT_TEMPLATE_CONTEXT.name, Watcher.SCRIPT_TEMPLATE_CONTEXT); contexts.put(WatcherTransformScript.CONTEXT.name, WatcherTransformScript.CONTEXT); - scriptService = new ScriptService(Settings.EMPTY, engines, contexts); + scriptService = new ScriptService(Settings.EMPTY, engines, contexts, () -> 1L); ThreadPool threadPool = mock(ThreadPool.class); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java index b6a501902be36..807610d1327e7 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java @@ -73,7 +73,7 @@ public void testParser() throws Exception { final MockScriptEngine engine = new MockScriptEngine("mock", Collections.emptyMap(), Collections.emptyMap()); Map engines = Collections.singletonMap(engine.getType(), engine); - ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); Client client = mock(Client.class); SearchTransformFactory transformFactory = new SearchTransformFactory(Settings.EMPTY, client, xContentRegistry(), scriptService); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/script/ScriptTransformTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/script/ScriptTransformTests.java index 8f1a74c9af980..f00019c89c992 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/script/ScriptTransformTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/script/ScriptTransformTests.java @@ -200,6 +200,6 @@ public static ScriptService createScriptService() throws Exception { Map> contexts = new HashMap<>(ScriptModule.CORE_CONTEXTS); contexts.put(WatcherTransformScript.CONTEXT.name, WatcherTransformScript.CONTEXT); contexts.put(Watcher.SCRIPT_TEMPLATE_CONTEXT.name, Watcher.SCRIPT_TEMPLATE_CONTEXT); - return new ScriptService(settings, Collections.emptyMap(), Collections.emptyMap()); + return new ScriptService(settings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L); } } From dfe5b4c4c129bafce9248cae54a9b5b4b2d54c2b Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 18 Oct 2021 09:45:35 -0500 Subject: [PATCH 08/26] Remove spurious newlines --- .../ingest/common/ScriptProcessorTests.java | 3 +-- .../java/org/elasticsearch/script/ScriptService.java | 2 +- .../ingest/ConditionalProcessorTests.java | 9 +++------ .../org/elasticsearch/ingest/IngestServiceTests.java | 3 +-- .../ingest/TrackingResultProcessorTests.java | 12 ++++-------- 5 files changed, 10 insertions(+), 19 deletions(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java index cb44c034053f1..ce5daf1a81fc9 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java @@ -53,8 +53,7 @@ Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine( Collections.emptyMap() ) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS), - () -> 1L); + new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L); script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()); ingestScript = scriptService.compile(script, IngestScript.CONTEXT).newInstance(script.getParams()); } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptService.java b/server/src/main/java/org/elasticsearch/script/ScriptService.java index 8c2374bf4262e..2d0543610eccb 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptService.java @@ -185,7 +185,7 @@ public ScriptService(Settings settings, Map engines, Map 20000L; //timeProvider; // Validation requires knowing which contexts exist. this.validateCacheSettings(settings); diff --git a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java index 67488a51e3b9e..7f64f11bbc9e6 100644 --- a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java @@ -60,8 +60,7 @@ public void testChecksCondition() throws Exception { Collections.emptyMap() ) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS), - () -> 1L); + new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L); Map document = new HashMap<>(); LongSupplier relativeTimeProvider = mock(LongSupplier.class); when(relativeTimeProvider.getAsLong()).thenReturn(0L, TimeUnit.MILLISECONDS.toNanos(1), 0L, TimeUnit.MILLISECONDS.toNanos(2)); @@ -155,8 +154,7 @@ public void testTypeDeprecation() throws Exception { Collections.emptyMap() ) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS), - () -> 1L); + new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L); LongSupplier relativeTimeProvider = mock(LongSupplier.class); when(relativeTimeProvider.getAsLong()).thenReturn(0L, TimeUnit.MILLISECONDS.toNanos(1), 0L, TimeUnit.MILLISECONDS.toNanos(2)); @@ -267,8 +265,7 @@ private static void assertMutatingCtxThrows(Consumer> mutati Collections.emptyMap() ) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS), - () -> 1L); + new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L); Map document = new HashMap<>(); ConditionalProcessor processor = new ConditionalProcessor( randomAlphaOfLength(10), diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 532ef88cb5cd7..da517254f9f1c 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -489,8 +489,7 @@ public void testGetProcessorsInPipelineComplexConditional() throws Exception { Collections.emptyMap() ) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS), - () -> 1L); + new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L); Map processors = new HashMap<>(); processors.put("complexSet", (factories, tag, description, config) -> { diff --git a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java index f3142597b13e6..62954f9cb46af 100644 --- a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java @@ -135,8 +135,7 @@ public void testActualCompoundProcessorWithOnFailureAndTrueCondition() throws Ex String scriptName = "conditionalScript"; ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> true), Collections.emptyMap())), - new HashMap<>(ScriptModule.CORE_CONTEXTS), - () -> 1L); + new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L); RuntimeException exception = new RuntimeException("fail"); TestProcessor failProcessor = new TestProcessor("fail", "test", null, exception); ConditionalProcessor conditionalProcessor = new ConditionalProcessor( @@ -204,8 +203,7 @@ public void testActualCompoundProcessorWithFalseConditional() throws Exception { String scriptName = "conditionalScript"; ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> false), Collections.emptyMap())), - new HashMap<>(ScriptModule.CORE_CONTEXTS), - () -> 1L); + new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L); CompoundProcessor compoundProcessor = new CompoundProcessor( new TestProcessor(ingestDocument -> {ingestDocument.setFieldValue(key1, randomInt()); }), @@ -309,8 +307,7 @@ public void testActualPipelineProcessorWithTrueConditional() throws Exception { ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> true), Collections.emptyMap())), - new HashMap<>(ScriptModule.CORE_CONTEXTS), - () -> 1L); + new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L); Pipeline pipeline1 = new Pipeline( pipelineId1, null, null, null, new CompoundProcessor( @@ -389,8 +386,7 @@ public void testActualPipelineProcessorWithFalseConditional() throws Exception { ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> false), Collections.emptyMap())), - new HashMap<>(ScriptModule.CORE_CONTEXTS), - () -> 1L); + new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L); Pipeline pipeline1 = new Pipeline( pipelineId1, null, null, null, new CompoundProcessor( From 67d3a43d5a471846c40af4b1a2a97aef320fe887 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 18 Oct 2021 10:10:46 -0500 Subject: [PATCH 09/26] Update comments --- .../elasticsearch/script/ScriptMetrics.java | 9 +-- .../elasticsearch/script/ScriptService.java | 2 +- .../script/TimeSeriesCounter.java | 72 +++++++++++-------- 3 files changed, 48 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java index 883b1c6164c95..e099a83bd8bab 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java @@ -25,15 +25,12 @@ public class ScriptMetrics { final TimeSeriesCounter compilations; final TimeSeriesCounter cacheEvictions; - protected final LongSupplier timeProvider; - public ScriptMetrics(LongSupplier timeProvider) { - this.timeProvider = timeProvider; - this.compilations = timeSeriesCounter(); - this.cacheEvictions = timeSeriesCounter(); + this.compilations = timeSeriesCounter(timeProvider); + this.cacheEvictions = timeSeriesCounter(timeProvider); } - TimeSeriesCounter timeSeriesCounter() { + TimeSeriesCounter timeSeriesCounter(LongSupplier timeProvider) { return new TimeSeriesCounter(TWENTY_FOUR_HOURS, 30 * MINUTE, 15 * SECOND, timeProvider); } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptService.java b/server/src/main/java/org/elasticsearch/script/ScriptService.java index 2d0543610eccb..8c2374bf4262e 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptService.java @@ -185,7 +185,7 @@ public ScriptService(Settings settings, Map engines, Map 20000L; //timeProvider; + this.timeProvider = timeProvider; // Validation requires knowing which contexts exist. this.validateCacheSettings(settings); diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index aab796432bccc..16816af774959 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -38,19 +38,20 @@ * * There are some useful time ranges to think about when reading this code: * Total authority range - The range of validity of the time series. Ends within low resolution seconds of the last update and - * starts duration before the last update (plus or minus low resolution seconds. + * starts {@code duration} before the last update (plus or minus low resolution seconds). * Adder authority range - {@link #adder} must be consulted if the queried time overlaps this range. * High authority range - The {@link #high} array must be consulted if the queried time overlaps this range. * This range may partially overlap the previous {@link #low} epoch because the high array - * does not necessarily reset when rolling over the low epoch. If we are only a few seconds - * into the new epoch, high keeps higher resolution counts. + * does not necessarily reset when rolling over the low epoch. If we are at least {@link #highSec} before + * the end of the current epoch, {@link #high} still has high resolution counts overlapping with the + * previous low epochs count. * Low authority range - The {@link #low} array is the authority for counts within this range. - * High as low delegate - The latest low epoch is represented by a combination of the high array and adder. + * High as low delegate - The latest low epoch is represented by a combination of the {@link #high} array and {@link #adder}. * This range occurs immediately after the Low authority range. * The two ranges together equal the Total authority range. * Use {@link #sumHighDelegate} to get the correct count out of this range when combining with * counts from previous low epochs. As mentioned above, high frequently overlaps with the - * last low epoch and naively counting all of high will lead to double counting. + * last low epoch and naively counting all of {@link #high} will lead to double counting. */ public class TimeSeriesCounter { public static final int SECOND = 1; @@ -71,8 +72,8 @@ public class TimeSeriesCounter { protected long latestSec; // most recent update time /** - * Create a new time series that covers the given {@code totalDuration} in seconds, with the low resolution epochs covering - * {@code lowSecPerEpoch} seconds and the high resolution epochs cover {@code highSecPerEpoch}. + * Create a new time series that covers the given {@code totalDuration} in seconds, with the low resolution epochs each + * covering {@code lowSecPerEpoch} seconds and each high resolution epoch covering {@code highSecPerEpoch}. * * Because the most recent low resolution epoch is covered completely by the high resolution epochs, {@code lowSecPerEpoch} * must be divisible by {@code highSecPerEpoch}. @@ -205,6 +206,39 @@ protected void reset(long t) { latestSec = t; } + /** + * Generate a {@link Snapshot} at the current time, provided by {@link #timeProvider}, for the + * given times. + */ + public Snapshot snapshot(long ... times) { + return snapshot(now(), times); + } + + /** + * Use another {@link TimeSeriesCounter}s snapshot to determine the time of the counts and + * the time ranges to count. + * + * Useful when a consistent count is necessary across multiple TimeSeriesCounters. This + * method does not ensure zero updates (and thus potential rollovers) have occurred during + * the counts, but it does ensure the counts happen for the same time. + */ + public Snapshot snapshot(Snapshot snapshot) { + return snapshot(snapshot.now, snapshot.times[0]); + } + + Snapshot snapshot(long now, long[] times) { + lock.readLock().lock(); + try { + Snapshot snapshot = new Snapshot(now, total(), times); + for (int i = 0; i < snapshot.times.length; i++) { + snapshot.counts[i] = count(snapshot.now, snapshot.times[i]); + } + return snapshot; + } finally { + lock.readLock().unlock(); + } + } + /** * Get the count for the time series ending at {@code end} for {@code duration} seconds beforehand. */ @@ -416,27 +450,9 @@ int prevLowIndex(int index) { return index == 0 ? low.length - 1 : (index - 1) % low.length; } - public Snapshot snapshot(long ... times) { - return snapshot(now(), times); - } - - public Snapshot snapshot(Snapshot snapshot) { - return snapshot(snapshot.now, snapshot.times[0]); - } - - Snapshot snapshot(long now, long[] times) { - lock.readLock().lock(); - try { - Snapshot snapshot = new Snapshot(now, total(), times); - for (int i = 0; i < snapshot.times.length; i++) { - snapshot.counts[i] = count(snapshot.now, snapshot.times[i]); - } - return snapshot; - } finally { - lock.readLock().unlock(); - } - } - + /** + * A snapshot of the TimeSeriesCounter covering multiple time ranges. + */ public static class Snapshot { public final long now; public final long total; From 7c5af25d9e531762179d53c7de4f209699b1a87f Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 18 Oct 2021 10:51:58 -0500 Subject: [PATCH 10/26] Test coverage --- .../script/TimeSeriesCounter.java | 2 +- .../script/TimeSeriesCounterTests.java | 65 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index 16816af774959..9768c9c8a4a79 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -173,7 +173,7 @@ void rollForwardLow(long t) { } void rollForwardHigh(long t) { - if (highDelegateEnd(latestSec) < highAuthorityStart(t)) { + if (highAuthorityEnd(latestSec) < highAuthorityStart(t)) { Arrays.fill(high, 0); adder.reset(); return; diff --git a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java index 6e58787f7c29f..4cfbc63fbe203 100644 --- a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java +++ b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java @@ -101,6 +101,71 @@ public void testRolloverCount() { assertEquals(0, ts.count(now + (2 * HOUR) + highResSecPerEpoch, HOUR)); } + public void testRolloverHigh() { + for (int i = 0; i < ts.getHighLength(); i++) { + t.add(now + ((long) i * highResSecPerEpoch)); + } + inc(); + assertEquals(ts.getHighLength(), ts.count(now + lowResSecPerEpoch, lowResSecPerEpoch)); + } + + public void testRolloverHighWithGaps() { + long gap = 3; + for (int i = 0; i < ts.getHighLength(); i++) { + t.add(now + (gap * i * highResSecPerEpoch)); + } + inc(); + assertEquals(ts.getHighLength(), ts.count(now + (gap * lowResSecPerEpoch), (gap * lowResSecPerEpoch))); + } + + public void testRolloverLow() { + for (int i = 0; i < ts.getLowLength(); i++) { + t.add(now + ((long) i * lowResSecPerEpoch)); + } + inc(); + assertEquals(ts.getLowLength(), ts.count(now + totalDuration, totalDuration)); + } + + public void testRolloverLowWithGaps() { + long gap = 3; + for (int i = 0; i < ts.getLowLength() / 4; i++) { + t.add(now + (gap * i * lowResSecPerEpoch)); + } + inc(); + assertEquals(ts.getLowLength() / 4, ts.count(now + totalDuration, totalDuration)); + } + + public void testHighLowOverlap() { + int highPerLow = ts.getHighLength() / 5; + int numLow = ts.getLowLength() / 5; + long latest = 0; + for (long i = 0; i < numLow; i++) { + for (long j = 0; j < highPerLow; j++) { + latest = now + (i * lowResSecPerEpoch) + (j * highResSecPerEpoch); + t.add(latest); + } + } + inc(); + assertEquals(highPerLow * numLow, ts.count(latest, totalDuration)); + } + + public void testBackwardsInc() { + t.add(now); + t.add(now - highResSecPerEpoch); + t.add(now - lowResSecPerEpoch); + inc(); + assertEquals(3, ts.count(now + highResSecPerEpoch, totalDuration)); + } + + public void testBackwardsIncReset() { + long twoDays = now + 2 * totalDuration; + ts.inc(twoDays); + assertEquals(1, ts.count(twoDays, totalDuration)); + ts.inc(now); + assertEquals(0, ts.count(twoDays, totalDuration)); + assertEquals(1, ts.count(now, totalDuration)); + } + void inc() { for (int i = 0; i < t.times.size(); i++) { ts.inc(); From 3d4d4a501646b247877cad20ba355b1e06378210 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 18 Oct 2021 11:22:15 -0500 Subject: [PATCH 11/26] Rename internal snapshot to timeSuppliedSnapshot to avoid var args collision --- .../script/TimeSeriesCounter.java | 6 +-- .../script/TimeSeriesCounterTests.java | 44 ++++++++++++++++--- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index 9768c9c8a4a79..07713cf79807b 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -211,7 +211,7 @@ protected void reset(long t) { * given times. */ public Snapshot snapshot(long ... times) { - return snapshot(now(), times); + return timeSuppliedSnapshot(now(), times); } /** @@ -223,10 +223,10 @@ public Snapshot snapshot(long ... times) { * the counts, but it does ensure the counts happen for the same time. */ public Snapshot snapshot(Snapshot snapshot) { - return snapshot(snapshot.now, snapshot.times[0]); + return timeSuppliedSnapshot(snapshot.now, snapshot.times); } - Snapshot snapshot(long now, long[] times) { + Snapshot timeSuppliedSnapshot(long now, long[] times) { lock.readLock().lock(); try { Snapshot snapshot = new Snapshot(now, total(), times); diff --git a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java index 4cfbc63fbe203..7987ff238f8d3 100644 --- a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java +++ b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java @@ -83,11 +83,30 @@ public void testIncHighRollover() { public void testSnapshot() { t.add(now); inc(); + Snapshot s1 = ts.snapshot(FIVE, FIFTEEN, TWENTY_FOUR); + assertEquals(1, s1.getTime(FIVE)); + assertEquals(1, s1.getTime(FIFTEEN)); + assertEquals(1, s1.getTime(TWENTY_FOUR)); t.add(now + 10); - Snapshot s = ts.snapshot(FIVE, FIFTEEN, TWENTY_FOUR); - assertEquals(1, s.getTime(FIVE)); - assertEquals(1, s.getTime(FIFTEEN)); - assertEquals(1, s.getTime(TWENTY_FOUR)); + t.add(now + FIVE + highResSecPerEpoch); + inc(); + t.add(now + 2 * (FIVE + highResSecPerEpoch)); + Snapshot s2 = ts.snapshot(FIVE, FIFTEEN, TWENTY_FOUR); + assertEquals(0, s2.getTime(FIVE)); + assertEquals(3, s2.getTime(FIFTEEN)); + assertEquals(3, s2.getTime(TWENTY_FOUR)); + Snapshot s3 = ts.snapshot(s1); + assertEquals(1, s3.getTime(FIVE)); + assertEquals(1, s3.getTime(FIFTEEN)); + assertEquals(1, s3.getTime(TWENTY_FOUR)); + + // check out of range times + Snapshot s4 = ts.snapshot(FIVE, TWENTY_FOUR, (2 * TWENTY_FOUR)); + assertEquals(0, s4.getTime(FIVE)); + assertEquals(0, s4.getTime(FIFTEEN)); // not requested + assertEquals(3, s4.getTime(TWENTY_FOUR)); + assertEquals(0, s4.getTime(TWENTY_FOUR + FIFTEEN)); // not requested + assertEquals(3, s4.getTime(2 * TWENTY_FOUR)); } public void testRolloverCount() { @@ -101,6 +120,15 @@ public void testRolloverCount() { assertEquals(0, ts.count(now + (2 * HOUR) + highResSecPerEpoch, HOUR)); } + public void testInvalidCount() { + t.add(now); + inc(); + assertEquals(0, ts.count(now + 1, -1L)); + assertEquals(0, ts.count(now + 1, now + 2)); + assertEquals(1, ts.count(now + 1, now + 1)); + assertEquals(1, ts.count(now + 1, now)); + } + public void testRolloverHigh() { for (int i = 0; i < ts.getHighLength(); i++) { t.add(now + ((long) i * highResSecPerEpoch)); @@ -166,8 +194,14 @@ public void testBackwardsIncReset() { assertEquals(1, ts.count(now, totalDuration)); } + public void testNegativeConstructor() { + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, + () -> new TimeSeriesCounter(-1L, -2L, -3L, t)); + assertEquals("totalDuration [-1], lowSecPerEpoch [-2], highSecPerEpoch[-3] must be greater than zero", err.getMessage()); + } + void inc() { - for (int i = 0; i < t.times.size(); i++) { + for (int i = t.i; i < t.times.size(); i++) { ts.inc(); } } From 3384062cad9b590a3f84395ef0aea990b5140706 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 18 Oct 2021 11:39:24 -0500 Subject: [PATCH 12/26] Constructor test coverage --- .../script/TimeSeriesCounterTests.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java index 7987ff238f8d3..4cc02f4651c7a 100644 --- a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java +++ b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java @@ -194,12 +194,40 @@ public void testBackwardsIncReset() { assertEquals(1, ts.count(now, totalDuration)); } + public void testSumFuture() { + ts.inc(now); + assertEquals(0, ts.count(now + (3 * TWENTY_FOUR), TWENTY_FOUR)); + } + + public void testSumPast() { + ts.inc(now); + assertEquals(0, ts.count(now - (2 * TWENTY_FOUR), TWENTY_FOUR)); + } + public void testNegativeConstructor() { IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> new TimeSeriesCounter(-1L, -2L, -3L, t)); assertEquals("totalDuration [-1], lowSecPerEpoch [-2], highSecPerEpoch[-3] must be greater than zero", err.getMessage()); } + public void testHighEpochTooSmallConstructor() { + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, + () -> new TimeSeriesCounter(TWENTY_FOUR, FIFTEEN, FIFTEEN + 1, t)); + assertEquals("highSecPerEpoch [" + (FIFTEEN + 1) + "] must be less than lowSecPerEpoch [" + FIFTEEN + "]", err.getMessage()); + } + + public void testDurationNotDivisibleByLow() { + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, + () -> new TimeSeriesCounter(TWENTY_FOUR, 25 * MINUTE, FIVE, t)); + assertEquals("totalDuration [" + TWENTY_FOUR + "] must be divisible by lowSecPerEpoch [" + (25 * MINUTE) + "]", err.getMessage()); + } + + public void testLowDivisibleByHigh() { + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, + () -> new TimeSeriesCounter(TWENTY_FOUR, FIFTEEN, 10 * MINUTE, t)); + assertEquals("lowSecPerEpoch [" + FIFTEEN + "] must be divisible by highSecPerEpoch [" + (10 * MINUTE) + "]", err.getMessage()); + } + void inc() { for (int i = t.i; i < t.times.size(); i++) { ts.inc(); From bb6187eaf5d3da62cf3581e7fc83d5e82af7f65d Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 18 Oct 2021 16:45:49 -0500 Subject: [PATCH 13/26] External now, TimeSeries moved up, Remove authority verbage --- .../script/ScriptContextStats.java | 59 ------ .../elasticsearch/script/ScriptMetrics.java | 38 ++-- .../org/elasticsearch/script/TimeSeries.java | 74 ++++++++ .../script/TimeSeriesCounter.java | 177 ++++++------------ .../cluster/node/stats/NodeStatsTests.java | 7 +- .../script/ScriptStatsTests.java | 32 ++-- .../script/TimeSeriesCounterTests.java | 55 +++--- 7 files changed, 196 insertions(+), 246 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/script/TimeSeries.java diff --git a/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java b/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java index 2668979f158ef..708afbef914fd 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java @@ -62,65 +62,6 @@ public void writeTo(StreamOutput out) throws IOException { } } - public static class TimeSeries implements Writeable, ToXContentFragment { - public final long fiveMinutes; - public final long fifteenMinutes; - public final long twentyFourHours; - - public TimeSeries() { - this.fiveMinutes = 0; - this.fifteenMinutes = 0; - this.twentyFourHours = 0; - } - - public TimeSeries(long fiveMinutes, long fifteenMinutes, long twentyFourHours) { - assert fiveMinutes >= 0; - this.fiveMinutes = fiveMinutes; - assert fifteenMinutes >= fiveMinutes; - this.fifteenMinutes = fifteenMinutes; - assert twentyFourHours >= fifteenMinutes; - this.twentyFourHours = twentyFourHours; - } - - public TimeSeries(StreamInput in) throws IOException { - fiveMinutes = in.readVLong(); - fifteenMinutes = in.readVLong(); - twentyFourHours = in.readVLong(); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(Fields.FIVE_MINUTES, fiveMinutes); - builder.field(Fields.FIFTEEN_MINUTES, fifteenMinutes); - builder.field(Fields.TWENTY_FOUR_HOURS, twentyFourHours); - return builder; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVLong(fiveMinutes); - out.writeVLong(fifteenMinutes); - out.writeVLong(twentyFourHours); - } - - public boolean isEmpty() { - return twentyFourHours == 0; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TimeSeries that = (TimeSeries) o; - return fiveMinutes == that.fiveMinutes && fifteenMinutes == that.fifteenMinutes && twentyFourHours == that.twentyFourHours; - } - - @Override - public int hashCode() { - return Objects.hash(fiveMinutes, fifteenMinutes, twentyFourHours); - } - } - public String getContext() { return context; } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java index e099a83bd8bab..617b99fe1413f 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java @@ -9,7 +9,6 @@ package org.elasticsearch.script; import org.elasticsearch.common.metrics.CounterMetric; -import static org.elasticsearch.script.TimeSeriesCounter.Snapshot; import static org.elasticsearch.script.TimeSeriesCounter.SECOND; import static org.elasticsearch.script.TimeSeriesCounter.MINUTE; import static org.elasticsearch.script.TimeSeriesCounter.HOUR; @@ -17,46 +16,47 @@ import java.util.function.LongSupplier; public class ScriptMetrics { - protected static final int FIVE_MINUTES = 5 * MINUTE; - protected static final int FIFTEEN_MINUTES = 15 * MINUTE; + protected static final int FIFTEEN_SECONDS = 15 * SECOND; + protected static final int THIRTY_MINUTES = 30 * MINUTE; protected static final int TWENTY_FOUR_HOURS = 24 * HOUR; final CounterMetric compilationLimitTriggered = new CounterMetric(); - final TimeSeriesCounter compilations; - final TimeSeriesCounter cacheEvictions; + final TimeSeriesCounter compilations = timeSeriesCounter(); + final TimeSeriesCounter cacheEvictions = timeSeriesCounter(); + final LongSupplier timeProvider; public ScriptMetrics(LongSupplier timeProvider) { - this.compilations = timeSeriesCounter(timeProvider); - this.cacheEvictions = timeSeriesCounter(timeProvider); + this.timeProvider = timeProvider; } - TimeSeriesCounter timeSeriesCounter(LongSupplier timeProvider) { - return new TimeSeriesCounter(TWENTY_FOUR_HOURS, 30 * MINUTE, 15 * SECOND, timeProvider); + TimeSeriesCounter timeSeriesCounter() { + return new TimeSeriesCounter(TWENTY_FOUR_HOURS, THIRTY_MINUTES, FIFTEEN_SECONDS); } public void onCompilation() { - compilations.inc(); + compilations.inc(now()); } public void onCacheEviction() { - cacheEvictions.inc(); + cacheEvictions.inc(now()); } public void onCompilationLimit() { compilationLimitTriggered.inc(); } + protected long now() { + return timeProvider.getAsLong() / 1000; + } + public ScriptContextStats stats(String context) { - Snapshot compilation = compilations.snapshot(FIVE_MINUTES, FIFTEEN_MINUTES, TWENTY_FOUR_HOURS); - Snapshot cacheEviction = cacheEvictions.snapshot(compilation); + long t = now(); return new ScriptContextStats( context, - compilation.total, - cacheEviction.total, + compilations.total(), + cacheEvictions.total(), compilationLimitTriggered.count(), - new ScriptContextStats.TimeSeries(compilation.getTime(FIVE_MINUTES), compilation.getTime(FIFTEEN_MINUTES), - compilation.getTime(TWENTY_FOUR_HOURS)), - new ScriptContextStats.TimeSeries(cacheEviction.getTime(FIVE_MINUTES), cacheEviction.getTime(FIFTEEN_MINUTES), - cacheEviction.getTime(TWENTY_FOUR_HOURS)) + compilations.timeSeries(t), + cacheEvictions.timeSeries(t) ); }} diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeries.java b/server/src/main/java/org/elasticsearch/script/TimeSeries.java new file mode 100644 index 0000000000000..0b3ac2d9815d3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/TimeSeries.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +public class TimeSeries implements Writeable, ToXContentFragment { + public final long fiveMinutes; + public final long fifteenMinutes; + public final long twentyFourHours; + + public TimeSeries() { + this.fiveMinutes = 0; + this.fifteenMinutes = 0; + this.twentyFourHours = 0; + } + + public TimeSeries(long fiveMinutes, long fifteenMinutes, long twentyFourHours) { + this.fiveMinutes = fiveMinutes; + this.fifteenMinutes = fifteenMinutes; + this.twentyFourHours = twentyFourHours; + } + + public TimeSeries(StreamInput in) throws IOException { + fiveMinutes = in.readVLong(); + fifteenMinutes = in.readVLong(); + twentyFourHours = in.readVLong(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(ScriptContextStats.Fields.FIVE_MINUTES, fiveMinutes); + builder.field(ScriptContextStats.Fields.FIFTEEN_MINUTES, fifteenMinutes); + builder.field(ScriptContextStats.Fields.TWENTY_FOUR_HOURS, twentyFourHours); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(fiveMinutes); + out.writeVLong(fifteenMinutes); + out.writeVLong(twentyFourHours); + } + + public boolean isEmpty() { + return twentyFourHours == 0; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TimeSeries that = (TimeSeries) o; + return fiveMinutes == that.fiveMinutes && fifteenMinutes == that.fifteenMinutes && twentyFourHours == that.twentyFourHours; + } + + @Override + public int hashCode() { + return Objects.hash(fiveMinutes, fifteenMinutes, twentyFourHours); + } +} diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index 07713cf79807b..0518e40be1176 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -9,15 +9,13 @@ package org.elasticsearch.script; import java.util.Arrays; -import java.util.Objects; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.LongSupplier; /** * {@link TimeSeriesCounter} implements an event counter for keeping running stats at fixed intervals, such as 5m/15m/24h. - * Callers use {@link #inc()} to increment the counter at the current time, as supplied by {@link #timeProvider}. + * Callers use {@link #inc(long)} to increment the counter at the current time, as supplied by the caller. * To fetch a count for a given time range, callers use {@link #count(long, long)}, providing an end time and a duration. * * {@link TimeSeriesCounter} has counts for the given duration at two different resolutions. From the last updated time, @@ -37,18 +35,18 @@ * and {@link #adder}. Similarly, {@link #high} delegates its most recent high resolution epoch to {@link #adder}. * * There are some useful time ranges to think about when reading this code: - * Total authority range - The range of validity of the time series. Ends within low resolution seconds of the last update and + * Total range - The range of validity of the time series. Ends within low resolution seconds of the last update and * starts {@code duration} before the last update (plus or minus low resolution seconds). - * Adder authority range - {@link #adder} must be consulted if the queried time overlaps this range. - * High authority range - The {@link #high} array must be consulted if the queried time overlaps this range. - * This range may partially overlap the previous {@link #low} epoch because the high array - * does not necessarily reset when rolling over the low epoch. If we are at least {@link #highSec} before - * the end of the current epoch, {@link #high} still has high resolution counts overlapping with the - * previous low epochs count. - * Low authority range - The {@link #low} array is the authority for counts within this range. + * Adder range - {@link #adder} must be consulted if the queried time overlaps this range. + * High range - The {@link #high} array must be consulted if the queried time overlaps this range. + * This range may partially overlap the previous {@link #low} epoch because the high array + * does not necessarily reset when rolling over the low epoch. If we are at least {@link #highSec} before + * the end of the current epoch, {@link #high} still has high resolution counts overlapping with the + * previous low epochs count. + * Low range - The {@link #low} array should be exclusively consulted for times falling within this range. * High as low delegate - The latest low epoch is represented by a combination of the {@link #high} array and {@link #adder}. - * This range occurs immediately after the Low authority range. - * The two ranges together equal the Total authority range. + * This range occurs immediately after the Low range. + * The two ranges together equal the Total range. * Use {@link #sumHighDelegate} to get the correct count out of this range when combining with * counts from previous low epochs. As mentioned above, high frequently overlaps with the * last low epoch and naively counting all of {@link #high} will lead to double counting. @@ -65,7 +63,6 @@ public class TimeSeriesCounter { protected final int[] low; protected final ReadWriteLock lock = new ReentrantReadWriteLock(); - protected final LongSupplier timeProvider; protected final LongAdder adder = new LongAdder(); // most recent high epoch protected final LongAdder total = new LongAdder(); // the total number of increments @@ -78,7 +75,7 @@ public class TimeSeriesCounter { * Because the most recent low resolution epoch is covered completely by the high resolution epochs, {@code lowSecPerEpoch} * must be divisible by {@code highSecPerEpoch}. */ - public TimeSeriesCounter(long totalDuration, long lowSecPerEpoch, long highSecPerEpoch, LongSupplier timeProvider) { + public TimeSeriesCounter(long totalDuration, long lowSecPerEpoch, long highSecPerEpoch) { if (totalDuration <= 0 || lowSecPerEpoch <= 0 || highSecPerEpoch <= 0) { throw new IllegalArgumentException("totalDuration [" + totalDuration + "], lowSecPerEpoch [" + lowSecPerEpoch + "], highSecPerEpoch[" + highSecPerEpoch + "] must be greater than zero"); @@ -92,7 +89,6 @@ public TimeSeriesCounter(long totalDuration, long lowSecPerEpoch, long highSecPe throw new IllegalArgumentException( "lowSecPerEpoch [" + lowSecPerEpoch + "] must be divisible by highSecPerEpoch [" + highSecPerEpoch + "]"); } - this.timeProvider = Objects.requireNonNull(timeProvider); this.lowSec = lowSecPerEpoch; this.low = new int[(int) (totalDuration / lowSecPerEpoch)]; this.highSec = highSecPerEpoch; @@ -100,13 +96,6 @@ public TimeSeriesCounter(long totalDuration, long lowSecPerEpoch, long highSecPe assert high.length * highSecPerEpoch == lowSecPerEpoch; } - /** - * Increment the counter at the current time. - */ - public void inc() { - inc(now()); - } - /** * Increment the counter at the given time. * @@ -123,10 +112,10 @@ public void inc(long nowSec) { total.increment(); try { // Handle the busy case quickly - if (nowSec <= adderAuthorityEnd(latestSec) && nowSec >= adderAuthorityStart(latestSec)) { + if (nowSec <= adderEnd(latestSec) && nowSec >= adderStart(latestSec)) { adder.increment(); } else if (nowSec < latestSec) { - if (nowSec < totalAuthorityStart(latestSec)) { + if (nowSec < totalStart(latestSec)) { reset(nowSec); } else { adder.increment(); @@ -149,7 +138,7 @@ public void inc(long nowSec) { // roll low forward such that t lowAuthorityEnd(t) + 1 = highDelegateStart(t) // Assumes t >= latestSec. void rollForwardLow(long t) { - if (totalAuthorityEnd(latestSec) < lowAuthorityStart(t)) { + if (totalEnd(latestSec) < lowStart(t)) { // complete rollover Arrays.fill(low, 0); return; @@ -173,7 +162,7 @@ void rollForwardLow(long t) { } void rollForwardHigh(long t) { - if (highAuthorityEnd(latestSec) < highAuthorityStart(t)) { + if (highEnd(latestSec) < highStart(t)) { Arrays.fill(high, 0); adder.reset(); return; @@ -207,33 +196,16 @@ protected void reset(long t) { } /** - * Generate a {@link Snapshot} at the current time, provided by {@link #timeProvider}, for the - * given times. - */ - public Snapshot snapshot(long ... times) { - return timeSuppliedSnapshot(now(), times); - } - - /** - * Use another {@link TimeSeriesCounter}s snapshot to determine the time of the counts and - * the time ranges to count. - * - * Useful when a consistent count is necessary across multiple TimeSeriesCounters. This - * method does not ensure zero updates (and thus potential rollovers) have occurred during - * the counts, but it does ensure the counts happen for the same time. + * Generate a {@link TimeSeries} at the given time for 5m/15m/24h durations */ - public Snapshot snapshot(Snapshot snapshot) { - return timeSuppliedSnapshot(snapshot.now, snapshot.times); - } - - Snapshot timeSuppliedSnapshot(long now, long[] times) { + public TimeSeries timeSeries(long now) { lock.readLock().lock(); try { - Snapshot snapshot = new Snapshot(now, total(), times); - for (int i = 0; i < snapshot.times.length; i++) { - snapshot.counts[i] = count(snapshot.now, snapshot.times[i]); - } - return snapshot; + return new TimeSeries( + count(now, 5 * MINUTE), + count(now, 15 * MINUTE), + count(now, 24 * HOUR) + ); } finally { lock.readLock().unlock(); } @@ -242,7 +214,7 @@ Snapshot timeSuppliedSnapshot(long now, long[] times) { /** * Get the count for the time series ending at {@code end} for {@code duration} seconds beforehand. */ - public int count(long end, long duration) { + int count(long end, long duration) { if (duration < 0 || end - duration < 0) { return 0; // invalid range } @@ -251,14 +223,14 @@ public int count(long end, long duration) { try { long start = end - duration; - if (start >= highAuthorityStart(latestSec)) { - // entirely within high authority - return sumHighAuthority(start, end); + if (start >= highStart(latestSec)) { + // entirely within high + return sumHigh(start, end); } int total = 0; if (end >= highDelegateStart(latestSec)) { total = sumHighDelegate(end); - end = lowAuthorityEnd(latestSec); + end = lowEnd(latestSec); } return total + sumLow(start, end); } finally { @@ -284,27 +256,27 @@ int sumHighDelegate(long t) { return total; } - // sum within the high range's authority. Should not be combined with any - // low epoch counts as the high range's authority may overlap with the + // sum within the high range. Should not be combined with any + // low epoch counts as the high range may overlap with the // previous low epoch. - int sumHighAuthority(long start, long end) { + int sumHigh(long start, long end) { if (start > end) { return 0; } - long authorityStart = highAuthorityStart(latestSec); - long authorityEnd = adderAuthorityEnd(latestSec); + long hStart = highStart(latestSec); + long hEnd = adderEnd(latestSec); - if (end < authorityStart) { + if (end < hStart) { return 0; - } else if (end > authorityEnd) { - end = authorityEnd; + } else if (end > hEnd) { + end = hEnd; } - if (start > authorityEnd) { + if (start > hEnd) { return 0; - } else if (start < authorityStart) { - start = authorityStart; + } else if (start < hStart) { + start = hStart; } int delegateIndex = highIndex(latestSec); @@ -324,19 +296,19 @@ public int sumLow(long start, long end) { return 0; } - long authorityStart = lowAuthorityStart(latestSec); - long authorityEnd = lowAuthorityEnd(latestSec); + long lStart = lowStart(latestSec); + long lEnd = lowEnd(latestSec); - if (end < authorityStart) { + if (end < lStart) { return 0; - } else if (end > authorityEnd) { - end = authorityEnd; + } else if (end > lEnd) { + end = lEnd; } - if (start > authorityEnd) { + if (start > lEnd) { return 0; - } else if (start < authorityStart) { - start = authorityStart; + } else if (start < lStart) { + start = lStart; } int cur = lowIndex(start); @@ -349,11 +321,6 @@ public int sumLow(long start, long end) { return total + low[cur]; } - // get the current time represented by timeProvider - protected long now() { - return timeProvider.getAsLong() / 1000; - } - // get the current sum from adder, clamped to the range [0, Integer.MAX_VALUE]. // then reset the adder. This should only be called when rolling over. protected int sumThenResetAdder() { @@ -369,23 +336,23 @@ protected int sumAdder() { // adder is the authority from this time until adderAuthorityEnd. // Preceded by high authority range [highAuthorityStart, highAuthorityEnd]. - long adderAuthorityStart(long t) { + long adderStart(long t) { // authority for what _would be_ the latest high epoch return (t / highSec) * highSec; } - long adderAuthorityEnd(long t) { - return adderAuthorityStart(t) + highSec - 1; + long adderEnd(long t) { + return adderStart(t) + highSec - 1; } // high is the authority from his time until highAuthorityEnd. // This range is proceeded by the adder authority range [adderAuthorityStart, adderAuthorityEnd] - long highAuthorityStart(long t) { - return adderAuthorityStart(t) - ((high.length - 1) * highSec); + long highStart(long t) { + return adderStart(t) - ((high.length - 1) * highSec); } - long highAuthorityEnd(long t) { - return adderAuthorityStart(t) - 1; + long highEnd(long t) { + return adderStart(t) - 1; } // The beginning of the range where high can combine with low to provide accurate counts. @@ -402,21 +369,21 @@ long highDelegateEnd(long t) { } // The beginning of the range where low has counts. - long lowAuthorityStart(long t) { - return totalAuthorityStart(t); + long lowStart(long t) { + return totalStart(t); } - long lowAuthorityEnd(long t) { + long lowEnd(long t) { return highDelegateStart(t) - 1; } // The range of times valid for this TimeSeriesCounter. // Equal to [lowAuthorityStart, lowAuthorityEnd] + [highDelegateStart, highDelegateEnd] - long totalAuthorityStart(long t) { + long totalStart(long t) { return ((t / lowSec) * lowSec) - ((low.length - 1) * lowSec); } - long totalAuthorityEnd(long t) { + long totalEnd(long t) { return ((t / lowSec) * lowSec) + (lowSec - 1); } @@ -450,32 +417,6 @@ int prevLowIndex(int index) { return index == 0 ? low.length - 1 : (index - 1) % low.length; } - /** - * A snapshot of the TimeSeriesCounter covering multiple time ranges. - */ - public static class Snapshot { - public final long now; - public final long total; - public final long[] times; - public final long[] counts; - public Snapshot(long now, long total, long ... times) { - this.now = now; - this.total = total; - this.times = new long[times.length]; - this.counts = new long[times.length]; - System.arraycopy(times, 0, this.times, 0, times.length); - } - - public long getTime(long time) { - for (int i = 0; i < times.length; i++) { - if (times[i] == time) { - return counts[i]; - } - } - return 0; - } - } - // Testing and debugging methods int getAdder() { return adder.intValue(); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index 4272dfa77f2cc..cd9c5ab4be20b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.node.ResponseCollectorService; import org.elasticsearch.script.ScriptContextStats; import org.elasticsearch.script.ScriptStats; +import org.elasticsearch.script.TimeSeries; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPoolStats; @@ -694,14 +695,14 @@ public static NodeStats createNodeStats() { ingestStats, adaptiveSelectionStats, null); } - private static ScriptContextStats.TimeSeries randomTimeSeries() { + private static TimeSeries randomTimeSeries() { if (randomBoolean()) { long day = randomLongBetween(0, 1024); long fifteen = day >= 1 ? randomLongBetween(0, day) : 0; long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; - return new ScriptContextStats.TimeSeries(five, fifteen, day); + return new TimeSeries(five, fifteen, day); } else { - return new ScriptContextStats.TimeSeries(); + return new TimeSeries(); } } diff --git a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java index f7fe489a5fd38..996dea3a37ee6 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java @@ -29,10 +29,10 @@ public class ScriptStatsTests extends ESTestCase { public void testXContent() throws IOException { List contextStats = List.of( new ScriptContextStats("contextB", 100, 201, 302, - new ScriptContextStats.TimeSeries(1000, 1001, 1002), - new ScriptContextStats.TimeSeries(2000, 2001, 2002) + new TimeSeries(1000, 1001, 1002), + new TimeSeries(2000, 2001, 2002) ), - new ScriptContextStats("contextA", 1000, 2010, 3020, null, new ScriptContextStats.TimeSeries(0, 0, 0)) + new ScriptContextStats("contextA", 1000, 2010, 3020, null, new TimeSeries(0, 0, 0)) ); ScriptStats stats = new ScriptStats(contextStats); final XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); @@ -75,7 +75,7 @@ public void testXContent() throws IOException { } public void testSerializeEmptyTimeSeries() throws IOException { - ScriptContextStats.TimeSeries empty = new ScriptContextStats.TimeSeries(); + TimeSeries empty = new TimeSeries(); ScriptContextStats stats = new ScriptContextStats("c", 1111, 2222, 3333, null, empty); XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); @@ -93,10 +93,10 @@ public void testSerializeEmptyTimeSeries() throws IOException { } public void testSerializeTimeSeries() throws IOException { - Function mkContextStats = + Function mkContextStats = (ts) -> new ScriptContextStats("c", 1111, 2222, 3333, null, ts); - ScriptContextStats.TimeSeries series = new ScriptContextStats.TimeSeries(0, 0, 5); + TimeSeries series = new TimeSeries(0, 0, 5); String format = "{\n" + " \"context\" : \"c\",\n" + @@ -115,29 +115,29 @@ public void testSerializeTimeSeries() throws IOException { assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 0, 0, 5))); - series = new ScriptContextStats.TimeSeries(0, 7, 1234); + series = new TimeSeries(0, 7, 1234); builder = XContentFactory.jsonBuilder().prettyPrint(); mkContextStats.apply(series).toXContent(builder, ToXContent.EMPTY_PARAMS); assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 0, 7, 1234))); - series = new ScriptContextStats.TimeSeries(123, 456, 789); + series = new TimeSeries(123, 456, 789); builder = XContentFactory.jsonBuilder().prettyPrint(); mkContextStats.apply(series).toXContent(builder, ToXContent.EMPTY_PARAMS); assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 123, 456, 789))); } public void testTimeSeriesAssertions() { - expectThrows(AssertionError.class, () -> new ScriptContextStats.TimeSeries(-1, 1, 2)); - expectThrows(AssertionError.class, () -> new ScriptContextStats.TimeSeries(1, 0, 2)); - expectThrows(AssertionError.class, () -> new ScriptContextStats.TimeSeries(1, 3, 2)); + expectThrows(AssertionError.class, () -> new TimeSeries(-1, 1, 2)); + expectThrows(AssertionError.class, () -> new TimeSeries(1, 0, 2)); + expectThrows(AssertionError.class, () -> new TimeSeries(1, 3, 2)); } public void testTimeSeriesIsEmpty() { - assertTrue((new ScriptContextStats.TimeSeries(0, 0, 0)).isEmpty()); + assertTrue((new TimeSeries(0, 0, 0)).isEmpty()); long day = randomLongBetween(1, 1024); long fifteen = day >= 1 ? randomLongBetween(0, day) : 0; long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; - assertFalse((new ScriptContextStats.TimeSeries(five, fifteen, day)).isEmpty()); + assertFalse((new TimeSeries(five, fifteen, day)).isEmpty()); } public void testTimeSeriesSerialization() throws IOException { @@ -171,15 +171,15 @@ public ScriptContextStats serDeser(Version outVersion, Version inVersion, Script public ScriptContextStats randomStats() { long[] histStats = {randomLongBetween(0, 2048), randomLongBetween(0, 2048)}; - List timeSeries = new ArrayList<>(); + List timeSeries = new ArrayList<>(); for (int j = 0; j < 2; j++) { if (randomBoolean() && histStats[j] > 0) { long day = randomLongBetween(0, histStats[j]); long fifteen = day >= 1 ? randomLongBetween(0, day) : 0; long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; - timeSeries.add(new ScriptContextStats.TimeSeries(five, fifteen, day)); + timeSeries.add(new TimeSeries(five, fifteen, day)); } else { - timeSeries.add(new ScriptContextStats.TimeSeries()); + timeSeries.add(new TimeSeries()); } } return new ScriptContextStats( diff --git a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java index 4cc02f4651c7a..ed51c7b58d623 100644 --- a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java +++ b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.test.ESTestCase; import org.junit.Before; -import static org.elasticsearch.script.TimeSeriesCounter.Snapshot; import static org.elasticsearch.script.TimeSeriesCounter.MINUTE; import static org.elasticsearch.script.TimeSeriesCounter.HOUR; @@ -35,11 +34,11 @@ public void setUp() throws Exception { super.setUp(); now = 16345080831234L; t = new TimeProvider(); - ts = new TimeSeriesCounter(totalDuration, lowResSecPerEpoch, highResSecPerEpoch, t); + ts = new TimeSeriesCounter(totalDuration, lowResSecPerEpoch, highResSecPerEpoch); } public void testIncAdder() { - long start = ts.adderAuthorityStart(now); + long start = ts.adderStart(now); t.add(now); long highSec = ts.getHighSec(); for (int i = 0; i < highSec; i++) { @@ -51,7 +50,7 @@ public void testIncAdder() { } public void testIncAdderRollover() { - long start = ts.adderAuthorityStart(now); + long start = ts.adderStart(now); long highSec = ts.getHighSec(); t.add(now); for (int i = 0; i < 2 * highSec; i++) { @@ -63,7 +62,7 @@ public void testIncAdderRollover() { } public void testIncHighRollover() { - long start = ts.adderAuthorityStart(now); + long start = ts.adderStart(now); long highSec = ts.getHighSec(); int highLength = ts.getHighLength(); int count = 0; @@ -83,30 +82,24 @@ public void testIncHighRollover() { public void testSnapshot() { t.add(now); inc(); - Snapshot s1 = ts.snapshot(FIVE, FIFTEEN, TWENTY_FOUR); - assertEquals(1, s1.getTime(FIVE)); - assertEquals(1, s1.getTime(FIFTEEN)); - assertEquals(1, s1.getTime(TWENTY_FOUR)); + TimeSeries ts1 = ts.timeSeries(now); + assertEquals(1, ts1.fiveMinutes); + assertEquals(1, ts1.fifteenMinutes); + assertEquals(1, ts1.twentyFourHours); t.add(now + 10); t.add(now + FIVE + highResSecPerEpoch); inc(); - t.add(now + 2 * (FIVE + highResSecPerEpoch)); - Snapshot s2 = ts.snapshot(FIVE, FIFTEEN, TWENTY_FOUR); - assertEquals(0, s2.getTime(FIVE)); - assertEquals(3, s2.getTime(FIFTEEN)); - assertEquals(3, s2.getTime(TWENTY_FOUR)); - Snapshot s3 = ts.snapshot(s1); - assertEquals(1, s3.getTime(FIVE)); - assertEquals(1, s3.getTime(FIFTEEN)); - assertEquals(1, s3.getTime(TWENTY_FOUR)); - - // check out of range times - Snapshot s4 = ts.snapshot(FIVE, TWENTY_FOUR, (2 * TWENTY_FOUR)); - assertEquals(0, s4.getTime(FIVE)); - assertEquals(0, s4.getTime(FIFTEEN)); // not requested - assertEquals(3, s4.getTime(TWENTY_FOUR)); - assertEquals(0, s4.getTime(TWENTY_FOUR + FIFTEEN)); // not requested - assertEquals(3, s4.getTime(2 * TWENTY_FOUR)); + long latest = now + 2 * (FIVE + highResSecPerEpoch); + t.add(latest); + TimeSeries ts2 = ts.timeSeries(latest); + assertEquals(0, ts2.fiveMinutes); + assertEquals(3, ts2.fifteenMinutes); + assertEquals(3, ts2.twentyFourHours); + TimeSeries ts3 = ts.timeSeries(now); + assertEquals(1, ts3.fiveMinutes); + assertEquals(1, ts3.fifteenMinutes); + assertEquals(1, ts3.twentyFourHours); + assertEquals(3, ts.total()); } public void testRolloverCount() { @@ -206,31 +199,31 @@ public void testSumPast() { public void testNegativeConstructor() { IllegalArgumentException err = expectThrows(IllegalArgumentException.class, - () -> new TimeSeriesCounter(-1L, -2L, -3L, t)); + () -> new TimeSeriesCounter(-1L, -2L, -3L)); assertEquals("totalDuration [-1], lowSecPerEpoch [-2], highSecPerEpoch[-3] must be greater than zero", err.getMessage()); } public void testHighEpochTooSmallConstructor() { IllegalArgumentException err = expectThrows(IllegalArgumentException.class, - () -> new TimeSeriesCounter(TWENTY_FOUR, FIFTEEN, FIFTEEN + 1, t)); + () -> new TimeSeriesCounter(TWENTY_FOUR, FIFTEEN, FIFTEEN + 1)); assertEquals("highSecPerEpoch [" + (FIFTEEN + 1) + "] must be less than lowSecPerEpoch [" + FIFTEEN + "]", err.getMessage()); } public void testDurationNotDivisibleByLow() { IllegalArgumentException err = expectThrows(IllegalArgumentException.class, - () -> new TimeSeriesCounter(TWENTY_FOUR, 25 * MINUTE, FIVE, t)); + () -> new TimeSeriesCounter(TWENTY_FOUR, 25 * MINUTE, FIVE)); assertEquals("totalDuration [" + TWENTY_FOUR + "] must be divisible by lowSecPerEpoch [" + (25 * MINUTE) + "]", err.getMessage()); } public void testLowDivisibleByHigh() { IllegalArgumentException err = expectThrows(IllegalArgumentException.class, - () -> new TimeSeriesCounter(TWENTY_FOUR, FIFTEEN, 10 * MINUTE, t)); + () -> new TimeSeriesCounter(TWENTY_FOUR, FIFTEEN, 10 * MINUTE)); assertEquals("lowSecPerEpoch [" + FIFTEEN + "] must be divisible by highSecPerEpoch [" + (10 * MINUTE) + "]", err.getMessage()); } void inc() { for (int i = t.i; i < t.times.size(); i++) { - ts.inc(); + ts.inc(t.getAsLong() / 1000); } } From 7410f9a212dbb5db84440857bc83c98d1bc1147a Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 18 Oct 2021 16:51:49 -0500 Subject: [PATCH 14/26] update comments --- .../main/java/org/elasticsearch/script/TimeSeriesCounter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index 0518e40be1176..c4343f57f8c1b 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -100,7 +100,7 @@ public TimeSeriesCounter(long totalDuration, long lowSecPerEpoch, long highSecPe * Increment the counter at the given time. * * If {@code nowSec} is less than the last update, it is treated as an increment at the time of the last update - * unless it's before the begininng of this time series, in which case the time series is reset. + * unless it's before the beginning of this time series, in which case the time series is reset. */ public void inc(long nowSec) { if (nowSec < 0) { @@ -111,7 +111,7 @@ public void inc(long nowSec) { lock.writeLock().lock(); total.increment(); try { - // Handle the busy case quickly + // Handle the rapid update case quickly if (nowSec <= adderEnd(latestSec) && nowSec >= adderStart(latestSec)) { adder.increment(); } else if (nowSec < latestSec) { From 82833ba4b29e64c4b9fb2c7732c704d8c117ec13 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 18 Oct 2021 16:59:55 -0500 Subject: [PATCH 15/26] adder int instead of LongAdder --- .../script/TimeSeriesCounter.java | 42 ++++++++----------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index c4343f57f8c1b..8ba001bf7c0b6 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -64,7 +64,7 @@ public class TimeSeriesCounter { protected final ReadWriteLock lock = new ReentrantReadWriteLock(); - protected final LongAdder adder = new LongAdder(); // most recent high epoch + protected int adder = 0; // most recent high epoch protected final LongAdder total = new LongAdder(); // the total number of increments protected long latestSec; // most recent update time @@ -113,22 +113,22 @@ public void inc(long nowSec) { try { // Handle the rapid update case quickly if (nowSec <= adderEnd(latestSec) && nowSec >= adderStart(latestSec)) { - adder.increment(); + adder++; } else if (nowSec < latestSec) { if (nowSec < totalStart(latestSec)) { reset(nowSec); } else { - adder.increment(); + adder++; } } else if (nowSec <= highDelegateEnd(latestSec)) { rollForwardHigh(nowSec); latestSec = nowSec; - adder.increment(); + adder++; } else { rollForwardLow(nowSec); rollForwardHigh(nowSec); latestSec = nowSec; - adder.increment(); + adder++; } } finally { lock.writeLock().unlock(); @@ -164,7 +164,7 @@ void rollForwardLow(long t) { void rollForwardHigh(long t) { if (highEnd(latestSec) < highStart(t)) { Arrays.fill(high, 0); - adder.reset(); + adder = 0; return; } int cur = highIndex(latestSec); @@ -174,7 +174,7 @@ void rollForwardHigh(long t) { return; } - high[cur] = sumThenResetAdder(); + high[cur] = resetAdder(); cur = nextHighIndex(cur); while (cur != dst) { high[cur] = 0; @@ -188,10 +188,9 @@ void rollForwardHigh(long t) { * reset the accumulator and all arrays, setting the latestSet to t and incrementing the adder. */ protected void reset(long t) { - adder.reset(); Arrays.fill(high, 0); Arrays.fill(low, 0); - adder.increment(); + adder = 1; latestSec = t; } @@ -249,7 +248,7 @@ public long total() { // sum high range representing the current low resolution epoch. int sumHighDelegate(long t) { int delegateIndex = highIndex(Math.min(t, latestSec)); - int total = sumAdder(); + int total = adder; for (int i = 0; i < delegateIndex; i++) { total += high[i]; } @@ -284,10 +283,10 @@ int sumHigh(long start, long end) { int dst = highIndex(end); int total = 0; while (cur != dst) { - total += cur == delegateIndex ? sumAdder() : high[cur]; + total += cur == delegateIndex ? adder : high[cur]; cur = (cur + 1) % high.length; } - return total + (cur == delegateIndex ? sumAdder() : high[cur]); + return total + (cur == delegateIndex ? adder : high[cur]); } // sum the low epochs represented by the given range @@ -321,17 +320,12 @@ public int sumLow(long start, long end) { return total + low[cur]; } - // get the current sum from adder, clamped to the range [0, Integer.MAX_VALUE]. - // then reset the adder. This should only be called when rolling over. - protected int sumThenResetAdder() { - long sum = adder.sumThenReset(); - return sum > 0 ? (int) sum : 0; - } - - // get the current sum from adder, clamped to the range [0, Integer.MAX_VALUE]. - protected int sumAdder() { - long sum = adder.sum(); - return sum > 0 ? (int) sum : 0; + // get the current value from adder then reset it. + // This should only be called when rolling over. + protected int resetAdder() { + int sum = adder; + adder = 0; + return sum; } // adder is the authority from this time until adderAuthorityEnd. @@ -419,7 +413,7 @@ int prevLowIndex(int index) { // Testing and debugging methods int getAdder() { - return adder.intValue(); + return adder; } int getLowLength() { From 046a105469849456381347726c5d523a08f8bb15 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 18 Oct 2021 17:23:06 -0500 Subject: [PATCH 16/26] fix time series tests --- .../java/org/elasticsearch/script/ScriptStatsTests.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java index 996dea3a37ee6..e244030b1a821 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java @@ -126,12 +126,6 @@ public void testSerializeTimeSeries() throws IOException { assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 123, 456, 789))); } - public void testTimeSeriesAssertions() { - expectThrows(AssertionError.class, () -> new TimeSeries(-1, 1, 2)); - expectThrows(AssertionError.class, () -> new TimeSeries(1, 0, 2)); - expectThrows(AssertionError.class, () -> new TimeSeries(1, 3, 2)); - } - public void testTimeSeriesIsEmpty() { assertTrue((new TimeSeries(0, 0, 0)).isEmpty()); long day = randomLongBetween(1, 1024); From 760b832b9a80b43f5b1f00686ca5c89b1eeeee6b Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 27 Oct 2021 09:49:29 -0500 Subject: [PATCH 17/26] Simplify Counters, add example documentation --- .../org/elasticsearch/script/ScriptCache.java | 2 +- .../elasticsearch/script/ScriptMetrics.java | 27 +- .../script/TimeSeriesCounter.java | 683 +++++++++--------- .../script/TimeSeriesCounterTests.java | 518 ++++++++----- 4 files changed, 656 insertions(+), 574 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCache.java b/server/src/main/java/org/elasticsearch/script/ScriptCache.java index 2525c4e0e000c..6255fea51e15f 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptCache.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptCache.java @@ -45,7 +45,7 @@ public class ScriptCache { private final double compilesAllowedPerNano; private final String contextRateSetting; - ScriptCache(int cacheMaxSize, TimeValue cacheExpire, CompilationRate maxCompilationRate, String contextRateSetting, + ScriptCache(int cacheMaxSize, TimeValue cacheExpire, CompilationRate maxCompilationRate, String contextRateSetting LongSupplier timeProvider) { this.cacheSize = cacheMaxSize; this.cacheExpire = cacheExpire; diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java index 3bf5fc487a4f9..750fd309582af 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java @@ -9,30 +9,19 @@ package org.elasticsearch.script; import org.elasticsearch.common.metrics.CounterMetric; -import static org.elasticsearch.script.TimeSeriesCounter.SECOND; -import static org.elasticsearch.script.TimeSeriesCounter.MINUTE; -import static org.elasticsearch.script.TimeSeriesCounter.HOUR; import java.util.function.LongSupplier; public class ScriptMetrics { - protected static final int FIFTEEN_SECONDS = 15 * SECOND; - protected static final int THIRTY_MINUTES = 30 * MINUTE; - protected static final int TWENTY_FOUR_HOURS = 24 * HOUR; - final CounterMetric compilationLimitTriggered = new CounterMetric(); - final TimeSeriesCounter compilations = timeSeriesCounter(); - final TimeSeriesCounter cacheEvictions = timeSeriesCounter(); + final TimeSeriesCounter compilations = new TimeSeriesCounter(); + final TimeSeriesCounter cacheEvictions = new TimeSeriesCounter(); final LongSupplier timeProvider; public ScriptMetrics(LongSupplier timeProvider) { this.timeProvider = timeProvider; } - TimeSeriesCounter timeSeriesCounter() { - return new TimeSeriesCounter(TWENTY_FOUR_HOURS, THIRTY_MINUTES, FIFTEEN_SECONDS); - } - public void onCompilation() { compilations.inc(now()); } @@ -45,20 +34,20 @@ public void onCompilationLimit() { compilationLimitTriggered.inc(); } - public ScriptStats stats() { - return new ScriptStats(compilationsMetric.count(), cacheEvictionsMetric.count(), compilationLimitTriggered.count()); - } - protected long now() { return timeProvider.getAsLong() / 1000; } + public ScriptStats stats() { + return new ScriptStats(compilationsMetric.count(), cacheEvictionsMetric.count(), compilationLimitTriggered.count()); + } + public ScriptContextStats stats(String context) { long t = now(); return new ScriptContextStats( context, - compilations.total(), - cacheEvictions.total(), + compilations.count(), + cacheEvictions.count(), compilationLimitTriggered.count(), compilations.timeSeries(t), cacheEvictions.timeSeries(t) diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index 8ba001bf7c0b6..5366ee96aec79 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -14,421 +14,382 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; /** - * {@link TimeSeriesCounter} implements an event counter for keeping running stats at fixed intervals, such as 5m/15m/24h. - * Callers use {@link #inc(long)} to increment the counter at the current time, as supplied by the caller. - * To fetch a count for a given time range, callers use {@link #count(long, long)}, providing an end time and a duration. + * Provides a counter with a history of 5m/15m/24h. * - * {@link TimeSeriesCounter} has counts for the given duration at two different resolutions. From the last updated time, - * {@link #latestSec}, for {@link #lowSec} ({@code lowSecPerEpoch} in the constructor) previous seconds, the resolution is - * {@link #highSec} ({@code highSecPerEpoch} in the constructor). - * - * Outside of that range, but within {@code totalDuration} seconds, the resolution is {@link #lowSec}. - * - * This two ranges balance recent high resolution, historical range and memory usage. - * - * This class is implemented with two arrays, {@link #high}, {@link #low} and a {@link LongAdder}. - * - * All recent updates hit the LongAdder, {@link #adder} unless an increment causes a roll over to a new in high epoch or both - * a new high and new low epoch. - * - * The two arrays have overlapping time ranges. {@link #low} delegates its most recent low resolution epoch to {@link #high} - * and {@link #adder}. Similarly, {@link #high} delegates its most recent high resolution epoch to {@link #adder}. - * - * There are some useful time ranges to think about when reading this code: - * Total range - The range of validity of the time series. Ends within low resolution seconds of the last update and - * starts {@code duration} before the last update (plus or minus low resolution seconds). - * Adder range - {@link #adder} must be consulted if the queried time overlaps this range. - * High range - The {@link #high} array must be consulted if the queried time overlaps this range. - * This range may partially overlap the previous {@link #low} epoch because the high array - * does not necessarily reset when rolling over the low epoch. If we are at least {@link #highSec} before - * the end of the current epoch, {@link #high} still has high resolution counts overlapping with the - * previous low epochs count. - * Low range - The {@link #low} array should be exclusively consulted for times falling within this range. - * High as low delegate - The latest low epoch is represented by a combination of the {@link #high} array and {@link #adder}. - * This range occurs immediately after the Low range. - * The two ranges together equal the Total range. - * Use {@link #sumHighDelegate} to get the correct count out of this range when combining with - * counts from previous low epochs. As mentioned above, high frequently overlaps with the - * last low epoch and naively counting all of {@link #high} will lead to double counting. + * Callers increment the counter and query */ public class TimeSeriesCounter { public static final int SECOND = 1; public static final int MINUTE = 60; public static final int HOUR = 60 * MINUTE; + protected LongAdder adder = new LongAdder(); - protected final long highSec; // high resolution in seconds - protected final int[] high; - - protected final long lowSec; // low resolution in seconds - protected final int[] low; + protected ReadWriteLock lock = new ReentrantReadWriteLock(); - protected final ReadWriteLock lock = new ReentrantReadWriteLock(); - - protected int adder = 0; // most recent high epoch - protected final LongAdder total = new LongAdder(); // the total number of increments - protected long latestSec; // most recent update time - - /** - * Create a new time series that covers the given {@code totalDuration} in seconds, with the low resolution epochs each - * covering {@code lowSecPerEpoch} seconds and each high resolution epoch covering {@code highSecPerEpoch}. - * - * Because the most recent low resolution epoch is covered completely by the high resolution epochs, {@code lowSecPerEpoch} - * must be divisible by {@code highSecPerEpoch}. - */ - public TimeSeriesCounter(long totalDuration, long lowSecPerEpoch, long highSecPerEpoch) { - if (totalDuration <= 0 || lowSecPerEpoch <= 0 || highSecPerEpoch <= 0) { - throw new IllegalArgumentException("totalDuration [" + totalDuration + "], lowSecPerEpoch [" + lowSecPerEpoch - + "], highSecPerEpoch[" + highSecPerEpoch + "] must be greater than zero"); - } else if (highSecPerEpoch > lowSecPerEpoch) { - throw new IllegalArgumentException("highSecPerEpoch [" + highSecPerEpoch + "] must be less than lowSecPerEpoch [" - + lowSecPerEpoch + "]"); - } else if (totalDuration % lowSecPerEpoch != 0) { - throw new IllegalArgumentException( - "totalDuration [" + totalDuration + "] must be divisible by lowSecPerEpoch [" + lowSecPerEpoch + "]"); - } else if (lowSecPerEpoch % highSecPerEpoch != 0) { - throw new IllegalArgumentException( - "lowSecPerEpoch [" + lowSecPerEpoch + "] must be divisible by highSecPerEpoch [" + highSecPerEpoch + "]"); - } - this.lowSec = lowSecPerEpoch; - this.low = new int[(int) (totalDuration / lowSecPerEpoch)]; - this.highSec = highSecPerEpoch; - this.high = new int[(int) (lowSecPerEpoch / highSecPerEpoch)]; - assert high.length * highSecPerEpoch == lowSecPerEpoch; - } + protected Counter fiveMinutes = new Counter(15 * SECOND, 5 * MINUTE); + protected Counter fifteenMinutes = new Counter(90 * SECOND, 15 * MINUTE); + protected Counter twentyFourHours = new Counter(15 * MINUTE, 24 * HOUR); /** - * Increment the counter at the given time. - * - * If {@code nowSec} is less than the last update, it is treated as an increment at the time of the last update - * unless it's before the beginning of this time series, in which case the time series is reset. + * Increment counters at timestamp t, any increment more than 24hours before the current time + * series resets all historical counters, but the total counter is still increments. */ - public void inc(long nowSec) { - if (nowSec < 0) { - // The math below relies on now being positive - return; - } - + public void inc(long t) { + adder.increment(); lock.writeLock().lock(); - total.increment(); try { - // Handle the rapid update case quickly - if (nowSec <= adderEnd(latestSec) && nowSec >= adderStart(latestSec)) { - adder++; - } else if (nowSec < latestSec) { - if (nowSec < totalStart(latestSec)) { - reset(nowSec); - } else { - adder++; - } - } else if (nowSec <= highDelegateEnd(latestSec)) { - rollForwardHigh(nowSec); - latestSec = nowSec; - adder++; + if (t < twentyFourHours.earliestTimeInCounter()) { + fiveMinutes.reset(t); + fifteenMinutes.reset(t); + twentyFourHours.reset(t); } else { - rollForwardLow(nowSec); - rollForwardHigh(nowSec); - latestSec = nowSec; - adder++; + fiveMinutes.inc(t); + fifteenMinutes.inc(t); + twentyFourHours.inc(t); } } finally { lock.writeLock().unlock(); } } - // roll low forward such that t lowAuthorityEnd(t) + 1 = highDelegateStart(t) - // Assumes t >= latestSec. - void rollForwardLow(long t) { - if (totalEnd(latestSec) < lowStart(t)) { - // complete rollover - Arrays.fill(low, 0); - return; - } - int cur = lowIndex(latestSec); - int dst = lowIndex(t); - if (cur == dst) { - // no rollover - return; - } - - // grab the high + adder version of the current epoch - low[cur] = sumHighDelegate(latestSec); - cur = nextLowIndex(cur); - while (cur != dst) { - low[cur] = 0; - cur = nextLowIndex(cur); - } - // low[dst]'s contents is delegated to highDelegate - low[dst] = 0; - } - - void rollForwardHigh(long t) { - if (highEnd(latestSec) < highStart(t)) { - Arrays.fill(high, 0); - adder = 0; - return; - } - int cur = highIndex(latestSec); - int dst = highIndex(t); - if (cur == dst) { - // no rollover - return; - } - - high[cur] = resetAdder(); - cur = nextHighIndex(cur); - while (cur != dst) { - high[cur] = 0; - cur = nextHighIndex(cur); - } - // high[dst]'s contents is delegated to adder - high[dst] = 0; - } - /** - * reset the accumulator and all arrays, setting the latestSet to t and incrementing the adder. + * Get the value of the counters for the last 5 minutes, last 15 minutes and the last 24 hours from + * t. May include events up to resolution before those durations due to counter granularity. */ - protected void reset(long t) { - Arrays.fill(high, 0); - Arrays.fill(low, 0); - adder = 1; - latestSec = t; - } - - /** - * Generate a {@link TimeSeries} at the given time for 5m/15m/24h durations - */ - public TimeSeries timeSeries(long now) { + public TimeSeries timeSeries(long t) { lock.readLock().lock(); try { - return new TimeSeries( - count(now, 5 * MINUTE), - count(now, 15 * MINUTE), - count(now, 24 * HOUR) - ); + return new TimeSeries(fiveMinutes.sum(t), fifteenMinutes.sum(t), twentyFourHours.sum(t)); } finally { lock.readLock().unlock(); } } /** - * Get the count for the time series ending at {@code end} for {@code duration} seconds beforehand. + * The total number of events for all time covered gby the counters. */ - int count(long end, long duration) { - if (duration < 0 || end - duration < 0) { - return 0; // invalid range - } + public long count() { + long total = adder.sum(); + return total < 0 ? 0 : total; + } - lock.readLock().lock(); - try { - long start = end - duration; - if (start >= highStart(latestSec)) { - // entirely within high - return sumHigh(start, end); - } - int total = 0; - if (end >= highDelegateStart(latestSec)) { - total = sumHighDelegate(end); - end = lowEnd(latestSec); + /* + * Keeps track event counts over a duration. Events are clamped to buckets, either the current bucket or a future + * bucket. A bucket represents all events over a period of resolution number of seconds. + */ + public static class Counter { + + /* + * In the following diagrams, we take a duration of 100 and resolution of 20. + * + * |_______________________________________| + * duration = 100 + * |_______|_______|_______|_______|_______| + * buckets = 5 + * |_______| + * resolution = 20 + * + * Action: inc(235) + * + * Past + * [_] [_] [_] [3] [4] + * |___________|___________|___________|___________|___________| + * 140[e]-> 160-> 180-> 199 + * + * Present + * [0] [1][b] [2][g] [3] [4] + * |___________|_____1_____|___________|___________|___________| + * 200[a]-> 220-> 240[d]-> 260-> 280-> 299 + * + * Future + * [0] [_] [_] [_] [_] + * |___________|___________|___________|___________|___________| + * 300[c]-> 320[f] + * + * [a] Beginning of the current epoch + * startOfEpoch = 200 = (t / duration) * duration = (235 / 100) * 100 + * Start time of bucket zero, this is used to anchor the bucket ring in time. Without `startOfEpoch`, + * it would be impossible to distinguish between two times that are `duration` away from each other. + * In this example, the first inc used time 235, since startOfEpoch is rounded down to the nearest + * duration (100), it is 200. + * + * [b] The current bucket + * curBucket = 1 = (t / resolution) % buckets.length = (235 / 20) % 5 + * The latest active bucket in the bucket ring. The bucket of a timestamp is determined by the `resolution`. + * In this case the `resolution` is 20, so each bucket covers 20 seconds, the first covers 200-219, the + * second covers 220->239, the third 240->259 and so on. 235 is in the second bucket, at index 1. + * + * [c] Beginning of the next epoch + * nextEpoch() = 300 = startOfEpoch + duration = 200 + 100 + * The first time of the next epoch, this indicates when `startOfEpoch` should be updated. When `curBucket` + * advances to or past zero, `startOfEpoch` must be updated to `nextEpoch()` + * + * [d] Beginning of the next bucket + * nextBucketStartTime() = 240 = startOfEpoch + ((curBucket + 1) * resolution) = 200 + ((1 + 1) * 20 + * The first time of the next bucket, when a timestamp is greater than or equal to this time, we must update + * the `curBucket` and potentially the `startOfEpoch`. + * + * [e] The earliest time to sum + * earliestTimeInCounter() = 140 = nextBucketStartTime() - duration = 240 - 100 + * `curBucket` covers the latest timestamp seen by the `Counter`. Since the counter keeps a history, when a + * caller calls `sum(t)`, the `Counter` must clamp the range to the earliest time covered by its current state. + * The times proceed backwards for `buckets.length - 1`. + * **Important** this is likely _before_ the `startOfEpoch`. `startOfEpoch` is the timestamp of bucket[0]. + * + * [f] The counter is no longer valid at this time + * counterExpired() = 320 = startOfEpoch + (curBucket * resolution) + duration = 200 + (1 * 20) + 100 + * Where `earliestTimeInCounter()` is looking backwards, to the history covered by the counter, `counterExpired()` + * looks forward to when current counter has expired. Since `curBucket` represents the latest time in this counter, + * `counterExpired()` is `duration` from the start of the time covered from `curBucket` + * + * [g] The next bucket in the bucket ring + * nextBucket(curBucket) = 2 = (i + 1) % buckets.length = (1 + 1) % 5 + * Since `buckets` is a ring, the next bucket may wrap around. + * + * ------------------------------------------------------------------------------------------------------------------ + * + * Action: inc(238) - since this inc is within the current bucket, it is incremented and nothing else changes + * + * Present + * [0] [1][b] [2][g] [3] [4] + * |___________|_____2_____|___________|___________|___________| + * 200[a]-> 220-> 240[d]-> 260-> 280-> 299 + * + * ------------------------------------------------------------------------------------------------------------------ + * + * Action: inc(165) - only the current bucket is incremented, so increments from a timestamp in the past are + * clamped to the current bucket. This makes `inc(long)` dependent on the ordering of timestamps, + * but it avoids revising a history that may have already been exposed via `sum(long)`. + * + * Present + * [0] [1][b] [2][g] [3] [4] + * |___________|_____3_____|___________|___________|___________| + * 200[a]-> 220-> 240[d]-> 260-> 280-> 299 + * + * ------------------------------------------------------------------------------------------------------------------ + * + * Action: inc(267) - 267 is in bucket 3, so bucket 2 is zeroed and skipped. Bucket 2 is zeroed because it may have + * had contents that were relevant for timestamps 140 - 159. + * + * The `startOfEpoch`[a], does not change while `curBucket`[b] is now bucket 3. + * + * `nextEpoch()`[c] does not change as there hasn't been a rollover. + * + * `nextBucketStartTime()`[d] is now 280, the start time of bucket 4. + * + * `earliestTimeInCounter()`[e] is now 180, bucket 2 was zeroed, erasing the history from 140-159 and + * bucket 3 was set to 1, now representing 260-279 rather than 160-179. + * + * `counterExpired()`[f] is now 360. Bucket 3 in the current epoch represents 260->279, an + * `inc(long)` any of time (260 + `duration`) or beyond would require clearing all `buckets` in the + * `Counter` and any `sum(long)` that starts at 360 or later does not cover the valid time range for + * this state of the counter. + * + * `nextBucket(curBucket)`[g] is now 4, the bucket after 3. + * + * + * Past + * [_] [_] [_] [_] [4] + * |___________|___________|___________|___________|___________| + * 180[e]-> 199 + * + * Present + * [0] [1] [2] [3][b] [4][g] + * |___________|_____3_____|___________|______1____|___________| + * 200[a]-> 220-> 240-> 260-> 280[d]-> 299 + * + * Future + * [0] [1] [2] [_] [_] + * |___________|___________|___________|___________|___________| + * 300[c]-> 320-> 340-> 360[f]-> + * + * ------------------------------------------------------------------------------------------------------------------ + * + * Action: inc(310) - 310 is in bucket 0, so bucket 4 is zeroed and skipped, as it may have had contents + * for timestamps 180-199. + * + * The `startOfEpoch`[a], is now 300 as the `Counter` has rolled through bucket 0. + * + * `curBucket`[b] is now bucket 0. + * + * `nextEpoch()`[c] is now 400 because `startOfEpoch`[a] has changed. + * + * `nextBucketStartTime()`[d] is now 320, the start time of bucket 1 in this new epoch. + * + * `earliestTimeInCounter()`[e] is now 220, bucket 4 was zeroed, erasing the history from 180-199 and + * bucket 0 was set to 1, now representing 300-319 due to the epoch change, rather than 200-219, so + * 220 is the earliest time available in the `Counter`. + * + * `counterExpired()`[f] is now 400. Bucket 0 in the current epoch represents 300-319, an + * `inc(long)` any of time (300 + `duration`) or beyond would require clearing all `buckets` in the + * `Counter` and any `sum(long)` that starts at 400 or later does not cover the valid time range for + * this state of the counter. + * + * `nextBucket(curBucket)`[g] is now 1, the bucket after 0. + * + * + * Past + * [_] [1] [2] [3] [4] + * |___________|_____3_____|___________|______1____|___________| + * 220[e]-> 240-> 260-> 280-> 299 + * + * Present + * [0][b] [1][g] [2] [3] [4] + * |_____1_____|___________|___________|___________|___________| + * 300[a]-> 320[d]-> 340-> 360-> 380-> 399 + * + * Future + * [_] [_] [_] [_] [_] + * |_____0_____|___________|___________|___________|___________| + * 400[c][f]-> + * + * ------------------------------------------------------------------------------------------------------------------ + * + * Action: inc(321) - 321 is in bucket 1, so the previous contents of bucket 1 is replaced with the value 1. + * + * The `startOfEpoch`[a] remains 300. + * + * `curBucket`[b] is now bucket 1. + * + * `nextEpoch()`[c] remains 400. + * + * `nextBucketStartTime()`[d] is now 340, the start time of bucket 2. + * + * `earliestTimeInCounter()`[e] is now 240 as bucket 1 now represents 320-339 rather than 220-239. + * + * `counterExpired()`[f] is now 420. Bucket 1 in the current epoch represents 320-339, an + * `inc(long)` any of time (320 + `duration`) or beyond would require clearing all `buckets` in the + * `Counter` and any `sum(long)` that starts at 420 or later does not cover the valid time range for + * this state of the counter. + * + * `nextBucket(curBucket)`[g] is now 2, the bucket after 1. + * + * Past + * [_] [_] [2] [3] [4] + * |___________|___________|___________|______1____|___________| + * 240[e]-> 260-> 280-> 299 + * + * Present + * [0] [1][b] [2][g] [3] [4] + * |_____1_____|_____1_____|___________|___________|___________| + * 300[a]-> 320-> 340[d]-> 360-> 380-> 399 + * + * Future + * [0] [_] [_] [_] [_] + * |_____0_____|___________|___________|___________|___________| + * 400[c]-> 420[f]-> + * + */ + protected final long resolution; + protected final long duration; + protected final long[] buckets; + + // The start time of buckets[0]. bucket(t + (i * duration)) is the same for all i. startOfEpoch + protected long startOfEpoch; + protected int curBucket = 0; + + /** + * Create a Counter that covers duration seconds at the given resolution. Duration must be divisible by resolution. + */ + public Counter(long resolution, long duration) { + if (resolution <= 0) { + throw new IllegalArgumentException("resolution [" + resolution + "] must be greater than zero"); + } else if (duration <= 0) { + throw new IllegalArgumentException("duration [" + duration + "] must be greater than zero"); + } else if (duration % resolution != 0) { + throw new IllegalArgumentException("duration [" + duration + "] must divisible by resolution [" + resolution + "]"); } - return total + sumLow(start, end); - } finally { - lock.readLock().unlock(); + this.resolution = resolution; + this.duration = duration; + this.buckets = new long[(int) (duration / resolution)]; + assert buckets.length > 0; } - } - - /** - * Get the total number of increments for all time. - */ - public long total() { - long sum = total.sum(); - return sum >= 0 ? sum : 0; - } - // sum high range representing the current low resolution epoch. - int sumHighDelegate(long t) { - int delegateIndex = highIndex(Math.min(t, latestSec)); - int total = adder; - for (int i = 0; i < delegateIndex; i++) { - total += high[i]; + /** + * Increment the counter at time {@code t}, expressed in seconds. + */ + public void inc(long t) { + if (t < nextBucketStartTime()) { + buckets[curBucket]++; + } else if (t >= counterExpired()) { + reset(t); + } else { + int dstBucket = bucket(t); + for (int i = nextBucket(curBucket); i != dstBucket; i = nextBucket(i)) { + buckets[i] = 0; + } + curBucket = dstBucket; + buckets[curBucket] = 1; + if (t >= nextEpoch()) { + startOfEpoch = epoch(t); + } + } } - return total; - } - // sum within the high range. Should not be combined with any - // low epoch counts as the high range may overlap with the - // previous low epoch. - int sumHigh(long start, long end) { - if (start > end) { - return 0; - } + /** + * sum for the duration of the counter until {@code end}. + */ + public long sum(long end) { + long start = end - duration; + if (start >= nextBucketStartTime() || start < 0) { + return 0; + } - long hStart = highStart(latestSec); - long hEnd = adderEnd(latestSec); + if (start < earliestTimeInCounter()) { + start = earliestTimeInCounter(); + } - if (end < hStart) { - return 0; - } else if (end > hEnd) { - end = hEnd; + long total = 0; + for (int i = bucket(start); i != curBucket; i = nextBucket(i)) { + total += buckets[i]; + } + return total + buckets[curBucket]; } - if (start > hEnd) { - return 0; - } else if (start < hStart) { - start = hStart; + /** + * Reset the counter. Next counter begins at t. + */ + void reset(long t) { + Arrays.fill(buckets, 0); + startOfEpoch = epoch(t); + curBucket = bucket(t); + buckets[curBucket] = 1; } - int delegateIndex = highIndex(latestSec); - int cur = highIndex(start); - int dst = highIndex(end); - int total = 0; - while (cur != dst) { - total += cur == delegateIndex ? adder : high[cur]; - cur = (cur + 1) % high.length; + // The time at bucket[0] for the given timestamp. + long epoch(long t) { + return (t / duration) * duration; } - return total + (cur == delegateIndex ? adder : high[cur]); - } - // sum the low epochs represented by the given range - public int sumLow(long start, long end) { - if (start > end) { - return 0; + // What is the start time of the next epoch? + long nextEpoch() { + return startOfEpoch + duration; } - long lStart = lowStart(latestSec); - long lEnd = lowEnd(latestSec); - - if (end < lStart) { - return 0; - } else if (end > lEnd) { - end = lEnd; + // What is the earliest time covered by this counter? + long earliestTimeInCounter() { + return nextBucketStartTime() - duration; } - if (start > lEnd) { - return 0; - } else if (start < lStart) { - start = lStart; + // When does this entire counter expire? + long counterExpired() { + return startOfEpoch + (curBucket * resolution) + duration; } - int cur = lowIndex(start); - int dst = lowIndex(end); - int total = 0; - while (cur != dst) { - total += low[cur]; - cur = (cur + 1) % low.length; + // bucket for the given time + int bucket(long t) { + return (int) (t / resolution) % buckets.length; } - return total + low[cur]; - } - - // get the current value from adder then reset it. - // This should only be called when rolling over. - protected int resetAdder() { - int sum = adder; - adder = 0; - return sum; - } - - // adder is the authority from this time until adderAuthorityEnd. - // Preceded by high authority range [highAuthorityStart, highAuthorityEnd]. - long adderStart(long t) { - // authority for what _would be_ the latest high epoch - return (t / highSec) * highSec; - } - - long adderEnd(long t) { - return adderStart(t) + highSec - 1; - } - - // high is the authority from his time until highAuthorityEnd. - // This range is proceeded by the adder authority range [adderAuthorityStart, adderAuthorityEnd] - long highStart(long t) { - return adderStart(t) - ((high.length - 1) * highSec); - } - - long highEnd(long t) { - return adderStart(t) - 1; - } - - // The beginning of the range where high can combine with low to provide accurate counts. - // This range is preceded by the low authority range [lowAuthorityStart, lowAuthorityEnd] - long highDelegateStart(long t) { - return (t / lowSec) * lowSec; - } - - // The end of the high delegate range [highDelegateStart, highDelegateEnd]. There may - // not be counts for all parts of this range. This range only has valid counts until - // latestSec. - long highDelegateEnd(long t) { - return highDelegateStart(t) + lowSec - 1; - } - - // The beginning of the range where low has counts. - long lowStart(long t) { - return totalStart(t); - } - - long lowEnd(long t) { - return highDelegateStart(t) - 1; - } - // The range of times valid for this TimeSeriesCounter. - // Equal to [lowAuthorityStart, lowAuthorityEnd] + [highDelegateStart, highDelegateEnd] - long totalStart(long t) { - return ((t / lowSec) * lowSec) - ((low.length - 1) * lowSec); - } - - long totalEnd(long t) { - return ((t / lowSec) * lowSec) + (lowSec - 1); - } - - // the index within the high array of the given time. - int highIndex(long t) { - return (int)((t / highSec) % high.length); - } - - // the index within the low array of the given time. - int lowIndex(long t) { - return (int)((t / lowSec) % low.length); - } - - // the next index within the high array, wrapping as appropriate - int nextHighIndex(int index) { - return (index + 1) % high.length; - } - - // the previous index within the high array, wrapping as appropriate - int prevHighIndex(int index) { - return index == 0 ? high.length - 1 : (index - 1) % high.length; - } - - // the next index within the low array, wrapping as appropriate - int nextLowIndex(int index) { - return (index + 1) % low.length; - } - - // the previous index within the low array, wrapping as appropriate - int prevLowIndex(int index) { - return index == 0 ? low.length - 1 : (index - 1) % low.length; - } - - // Testing and debugging methods - int getAdder() { - return adder; - } - - int getLowLength() { - return low.length; - } - - int getHighLength() { - return high.length; - } - - long getHighSec() { - return highSec; - } + // the next bucket in the circular bucket array + int nextBucket(int i) { + return (i + 1) % buckets.length; + } - long getLowSec() { - return lowSec; + // When does the next bucket start? + long nextBucketStartTime() { + return startOfEpoch + ((curBucket + 1) * resolution); + } } } diff --git a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java index ed51c7b58d623..0aaf3a5d4e7f5 100644 --- a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java +++ b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java @@ -9,239 +9,371 @@ package org.elasticsearch.script; import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matcher; import org.junit.Before; -import static org.elasticsearch.script.TimeSeriesCounter.MINUTE; -import static org.elasticsearch.script.TimeSeriesCounter.HOUR; import java.util.ArrayList; import java.util.List; -import java.util.function.LongSupplier; + +import static org.elasticsearch.script.TimeSeriesCounter.Counter; +import static org.elasticsearch.script.TimeSeriesCounter.HOUR; +import static org.hamcrest.Matchers.lessThan; public class TimeSeriesCounterTests extends ESTestCase { - protected static final int totalDuration = 24 * HOUR; - protected static final int lowResSecPerEpoch = 30 * MINUTE; - protected static final int highResSecPerEpoch = 15; - protected static final int FIVE = 5 * MINUTE; - protected static final int FIFTEEN = 15 * MINUTE; - protected static final int TWENTY_FOUR = 24 * HOUR; protected long now; - protected TimeSeriesCounter ts; - protected TimeProvider t; + protected long customCounterResolution; + protected long customCounterDuration; + protected TimeSeriesCounter tsc = new TimeSeriesCounter(); + protected final Matcher fiveDelta = lessThan(tsc.fiveMinutes.resolution); + protected final Matcher fifteenDelta = lessThan(tsc.fifteenMinutes.resolution); + protected final Matcher twentyFourDelta = lessThan(tsc.twentyFourHours.resolution); + protected List events; + protected Counter counter; @Override @Before public void setUp() throws Exception { super.setUp(); - now = 16345080831234L; - t = new TimeProvider(); - ts = new TimeSeriesCounter(totalDuration, lowResSecPerEpoch, highResSecPerEpoch); - } - - public void testIncAdder() { - long start = ts.adderStart(now); - t.add(now); - long highSec = ts.getHighSec(); - for (int i = 0; i < highSec; i++) { - t.add(start + i); - } - inc(); - assertEquals(highSec + 1, ts.count(now + highSec - 1, highSec - 1)); - assertEquals(highSec + 1, ts.getAdder()); - } - - public void testIncAdderRollover() { - long start = ts.adderStart(now); - long highSec = ts.getHighSec(); - t.add(now); - for (int i = 0; i < 2 * highSec; i++) { - t.add(start + i); - } - inc(); - assertEquals(2 * highSec + 1, ts.count(now + 2 * highSec - 1, 2 * highSec - 1)); - assertEquals(highSec, ts.getAdder()); - } - - public void testIncHighRollover() { - long start = ts.adderStart(now); - long highSec = ts.getHighSec(); - int highLength = ts.getHighLength(); - int count = 0; - t.add(now); - for (int i = 0; i < highLength + 1; i++) { - t.add(start + (i * highSec)); - if (i == highLength / 2 + 1) { - count = i + 1; + now = 1635182590; + customCounterResolution = 45; + customCounterDuration = 900; + reset(); + } + + protected void reset() { + tsc = new TimeSeriesCounter(); + events = new ArrayList<>(); + counter = new Counter(customCounterResolution, customCounterDuration); + } + + public void testCounterNegativeResolution() { + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> new Counter(-20, 200)); + assertEquals("resolution [-20] must be greater than zero", iae.getMessage()); + } + + public void testCounterNegativeDuration() { + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> new Counter(20, -200)); + assertEquals("duration [-200] must be greater than zero", iae.getMessage()); + } + + public void testCounterIndivisibleResolution() { + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> new Counter(3, 101)); + assertEquals("duration [101] must divisible by resolution [3]", iae.getMessage()); + } + + public void testOnePerSecond() { + long time = now; + long t; + long next = randomLongBetween(1, HOUR); + long twentyFive = 25 * HOUR; + for (int i=0; i < twentyFive; i++) { + t = time + i; + inc(t); + + if (i == next) { + TimeSeries ts = tsc.timeSeries(t); + assertThat(five(t) - ts.fiveMinutes, fiveDelta); + assertThat(fifteen(t) - ts.fifteenMinutes, fifteenDelta); + assertThat(twentyFour(t) - ts.twentyFourHours, twentyFourDelta); + assertEquals(i + 1, tsc.count()); + + next = Math.min(twentyFive, next + randomLongBetween(HOUR, 3 * HOUR)); } } - inc(); - assertEquals(highLength + 2, ts.count(now + (highSec * highLength), (highSec * highLength))); - assertEquals(1, ts.getAdder()); - assertEquals(count, ts.count(now + (highSec * (highLength / 2)), highSec * (highLength / 2))); - } - - public void testSnapshot() { - t.add(now); - inc(); - TimeSeries ts1 = ts.timeSeries(now); - assertEquals(1, ts1.fiveMinutes); - assertEquals(1, ts1.fifteenMinutes); - assertEquals(1, ts1.twentyFourHours); - t.add(now + 10); - t.add(now + FIVE + highResSecPerEpoch); - inc(); - long latest = now + 2 * (FIVE + highResSecPerEpoch); - t.add(latest); - TimeSeries ts2 = ts.timeSeries(latest); - assertEquals(0, ts2.fiveMinutes); - assertEquals(3, ts2.fifteenMinutes); - assertEquals(3, ts2.twentyFourHours); - TimeSeries ts3 = ts.timeSeries(now); - assertEquals(1, ts3.fiveMinutes); - assertEquals(1, ts3.fifteenMinutes); - assertEquals(1, ts3.twentyFourHours); - assertEquals(3, ts.total()); - } - - public void testRolloverCount() { - t.add(now); - inc(); - assertEquals(1, ts.count(now + 1, FIVE)); - assertEquals(0, ts.count(now + (2 * FIVE) + highResSecPerEpoch, FIVE)); - assertEquals(1, ts.count(now + 1, FIFTEEN)); - assertEquals(0, ts.count(now + (2 * FIFTEEN) + highResSecPerEpoch, FIFTEEN)); - assertEquals(1, ts.count(now + 1, HOUR)); - assertEquals(0, ts.count(now + (2 * HOUR) + highResSecPerEpoch, HOUR)); - } - - public void testInvalidCount() { - t.add(now); - inc(); - assertEquals(0, ts.count(now + 1, -1L)); - assertEquals(0, ts.count(now + 1, now + 2)); - assertEquals(1, ts.count(now + 1, now + 1)); - assertEquals(1, ts.count(now + 1, now)); - } - - public void testRolloverHigh() { - for (int i = 0; i < ts.getHighLength(); i++) { - t.add(now + ((long) i * highResSecPerEpoch)); - } - inc(); - assertEquals(ts.getHighLength(), ts.count(now + lowResSecPerEpoch, lowResSecPerEpoch)); - } - - public void testRolloverHighWithGaps() { - long gap = 3; - for (int i = 0; i < ts.getHighLength(); i++) { - t.add(now + (gap * i * highResSecPerEpoch)); - } - inc(); - assertEquals(ts.getHighLength(), ts.count(now + (gap * lowResSecPerEpoch), (gap * lowResSecPerEpoch))); - } - - public void testRolloverLow() { - for (int i = 0; i < ts.getLowLength(); i++) { - t.add(now + ((long) i * lowResSecPerEpoch)); - } - inc(); - assertEquals(ts.getLowLength(), ts.count(now + totalDuration, totalDuration)); - } - - public void testRolloverLowWithGaps() { - long gap = 3; - for (int i = 0; i < ts.getLowLength() / 4; i++) { - t.add(now + (gap * i * lowResSecPerEpoch)); - } - inc(); - assertEquals(ts.getLowLength() / 4, ts.count(now + totalDuration, totalDuration)); - } - - public void testHighLowOverlap() { - int highPerLow = ts.getHighLength() / 5; - int numLow = ts.getLowLength() / 5; - long latest = 0; - for (long i = 0; i < numLow; i++) { - for (long j = 0; j < highPerLow; j++) { - latest = now + (i * lowResSecPerEpoch) + (j * highResSecPerEpoch); - t.add(latest); - } + } + + public void testCounterIncrementSameBucket() { + long resolution = 45; + long duration = 900; + counter.inc(now); + long count = randomLongBetween(resolution / 2, resolution * 2); + long start = (now / resolution) * resolution; + for (int i = 1; i < count; i++) { + counter.inc(start + randomLongBetween(0, resolution - 1)); + } + + assertEquals(count, counter.sum(start)); + assertEquals(count, counter.sum(now)); + + long t = 0; + for (; t <= duration; t += resolution) { + assertEquals(count, counter.sum(start + t)); } - inc(); - assertEquals(highPerLow * numLow, ts.count(latest, totalDuration)); + assertEquals(0, counter.sum(start + t)); + assertEquals(0, counter.sum(start + duration + resolution)); + assertEquals(count, counter.sum(start + duration + resolution - 1)); } - public void testBackwardsInc() { - t.add(now); - t.add(now - highResSecPerEpoch); - t.add(now - lowResSecPerEpoch); - inc(); - assertEquals(3, ts.count(now + highResSecPerEpoch, totalDuration)); + + public void testFiveMinuteSameBucket() { + inc(now); + long resolution = tsc.fiveMinutes.resolution; + long duration = tsc.fiveMinutes.duration; + long count = randomLongBetween(1, resolution); + long start = (now / resolution) * resolution; + for (int i = 1; i < count; i++) { + inc(start + i); + } + assertEquals(count, tsc.count()); + assertEquals(count, tsc.timeSeries(now).fiveMinutes); + + long t = 0; + for (; t <= duration; t += resolution) { + assertEquals(count, tsc.timeSeries(start + t).fiveMinutes); + } + + TimeSeries series = tsc.timeSeries(start + t); + assertEquals(0, series.fiveMinutes); + assertEquals(count, series.fifteenMinutes); + assertEquals(count, series.twentyFourHours); + + series = tsc.timeSeries(start + duration + resolution); + assertEquals(0, series.fiveMinutes); + assertEquals(count, series.fifteenMinutes); + assertEquals(count, series.twentyFourHours); + assertEquals(count, tsc.timeSeries(start + duration + resolution - 1).fiveMinutes); } - public void testBackwardsIncReset() { - long twoDays = now + 2 * totalDuration; - ts.inc(twoDays); - assertEquals(1, ts.count(twoDays, totalDuration)); - ts.inc(now); - assertEquals(0, ts.count(twoDays, totalDuration)); - assertEquals(1, ts.count(now, totalDuration)); + public void testFifteenMinuteSameBucket() { + inc(now); + long resolution = tsc.fifteenMinutes.resolution; + long duration = tsc.fifteenMinutes.duration; + long start = (now / resolution) * resolution; + long count = randomLongBetween(1, resolution); + for (int i = 1; i < count; i++) { + inc(start + i); + } + assertEquals(count, tsc.count()); + assertEquals(count, tsc.timeSeries(now).fifteenMinutes); + + long t = 0; + for (; t <= duration; t += resolution) { + assertEquals(count, tsc.timeSeries(start + t).fifteenMinutes); + } + + TimeSeries series = tsc.timeSeries(start + t); + assertEquals(0, series.fiveMinutes); + assertEquals(0, series.fifteenMinutes); + assertEquals(count, series.twentyFourHours); + + series = tsc.timeSeries(start + duration + resolution); + assertEquals(0, series.fiveMinutes); + assertEquals(0, series.fifteenMinutes); + assertEquals(count, series.twentyFourHours); + assertEquals(count, tsc.timeSeries(start + duration + resolution - 1).fifteenMinutes); } - public void testSumFuture() { - ts.inc(now); - assertEquals(0, ts.count(now + (3 * TWENTY_FOUR), TWENTY_FOUR)); + public void testCounterIncrementBucket() { + long count = customCounterDuration / customCounterResolution; + for (int i = 0; i < count; i++) { + counter.inc(now + i * customCounterResolution); + } + assertEquals(count, counter.sum(now + customCounterDuration)); + assertEquals(count - 1, counter.sum(now + customCounterDuration + customCounterResolution)); + assertEquals(count - 2, counter.sum(now + customCounterDuration + (2 * customCounterResolution))); + counter.inc(now + customCounterDuration); + assertEquals(count, counter.sum(now + customCounterDuration + customCounterResolution)); } - public void testSumPast() { - ts.inc(now); - assertEquals(0, ts.count(now - (2 * TWENTY_FOUR), TWENTY_FOUR)); + public void testFiveMinuteIncrementBucket() { + int count = tsc.fiveMinutes.buckets.length; + long resolution = tsc.fiveMinutes.resolution; + long duration = tsc.fiveMinutes.duration; + for (int i = 0; i < count; i++) { + inc(now + i * resolution); + } + + TimeSeries ts = tsc.timeSeries(now + duration); + assertEquals(count, ts.fiveMinutes); + assertEquals(count, ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + assertEquals(count, tsc.count()); + + ts = tsc.timeSeries(now + duration + resolution); + assertEquals(count - 1, ts.fiveMinutes); + assertEquals(count, ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + + ts = tsc.timeSeries(now + duration + (2 * resolution)); + assertEquals(count - 2, ts.fiveMinutes); + assertEquals(count, ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + + inc(now + duration); + ts = tsc.timeSeries(now + duration + resolution); + assertEquals(count, ts.fiveMinutes); + assertEquals(count + 1, ts.fifteenMinutes); + assertEquals(count + 1, ts.twentyFourHours); + assertEquals(count + 1, tsc.count()); } - public void testNegativeConstructor() { - IllegalArgumentException err = expectThrows(IllegalArgumentException.class, - () -> new TimeSeriesCounter(-1L, -2L, -3L)); - assertEquals("totalDuration [-1], lowSecPerEpoch [-2], highSecPerEpoch[-3] must be greater than zero", err.getMessage()); + public void testFifteenMinuteIncrementBucket() { + int count = tsc.fifteenMinutes.buckets.length; + long resolution = tsc.fifteenMinutes.resolution; + long duration = tsc.fifteenMinutes.duration; + for (int i = 0; i < count; i++) { + long t = now + i * resolution; + inc(t); + } + long t = now + duration; + TimeSeries ts = tsc.timeSeries(t); + assertEquals(five(t), ts.fiveMinutes); + assertEquals(count, ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + + t = now + duration + resolution; + ts = tsc.timeSeries(t); + assertEquals(five(t), ts.fiveMinutes); + assertEquals(count - 1, ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + + t = now + duration + (2 * resolution); + ts = tsc.timeSeries(t); + assertEquals(five(t), ts.fiveMinutes); + assertEquals(count - 2, ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + + inc(now + duration); + t = now + duration + resolution; + ts = tsc.timeSeries(t); + assertEquals(five(t), ts.fiveMinutes); + assertEquals(count, ts.fifteenMinutes); + assertEquals(count + 1, ts.twentyFourHours); + assertEquals(count + 1, tsc.count()); } - public void testHighEpochTooSmallConstructor() { - IllegalArgumentException err = expectThrows(IllegalArgumentException.class, - () -> new TimeSeriesCounter(TWENTY_FOUR, FIFTEEN, FIFTEEN + 1)); - assertEquals("highSecPerEpoch [" + (FIFTEEN + 1) + "] must be less than lowSecPerEpoch [" + FIFTEEN + "]", err.getMessage()); + public void testCounterSkipBuckets() { + int count = (int) (customCounterDuration / customCounterResolution); + for (int skip = 1; skip <= count; skip++) { + reset(); + int increments = 0; + for (int i = 0; (i * skip * customCounterResolution) < customCounterDuration; i++) { + counter.inc(now + (i * skip * customCounterResolution)); + increments++; + } + assertEquals(increments, counter.sum(now + customCounterDuration)); + } } - public void testDurationNotDivisibleByLow() { - IllegalArgumentException err = expectThrows(IllegalArgumentException.class, - () -> new TimeSeriesCounter(TWENTY_FOUR, 25 * MINUTE, FIVE)); - assertEquals("totalDuration [" + TWENTY_FOUR + "] must be divisible by lowSecPerEpoch [" + (25 * MINUTE) + "]", err.getMessage()); + public void testFiveMinuteSkipBucket() { + int count = tsc.fiveMinutes.buckets.length; + long resolution = tsc.fiveMinutes.resolution; + long duration = tsc.fiveMinutes.duration; + for (int skip = 1; skip <= count; skip++) { + tsc = new TimeSeriesCounter(); + long increments = 0; + for (int i = 0; (i * skip * resolution) < duration; i++) { + inc(now + (i * skip * resolution)); + increments++; + } + + TimeSeries series = tsc.timeSeries(now + duration); + assertEquals(increments, series.fiveMinutes); + assertEquals(increments, series.fifteenMinutes); + assertEquals(increments, series.twentyFourHours); + assertEquals(increments, tsc.count()); + } } - public void testLowDivisibleByHigh() { - IllegalArgumentException err = expectThrows(IllegalArgumentException.class, - () -> new TimeSeriesCounter(TWENTY_FOUR, FIFTEEN, 10 * MINUTE)); - assertEquals("lowSecPerEpoch [" + FIFTEEN + "] must be divisible by highSecPerEpoch [" + (10 * MINUTE) + "]", err.getMessage()); + public void testFifteenMinuteSkipBuckets() { + int count = tsc.fifteenMinutes.buckets.length; + long resolution = tsc.fifteenMinutes.resolution; + long duration = tsc.fifteenMinutes.duration; + for (int skip = 1; skip <= count; skip++) { + reset(); + for (int i = 0; (i * skip * resolution) < duration; i++) { + inc(now + (i * skip * resolution)); + } + TimeSeries ts = tsc.timeSeries(now + duration); + assertEquals(five(now + duration), ts.fiveMinutes); + assertEquals(events.size(), ts.fifteenMinutes); + assertEquals(events.size(), ts.twentyFourHours); + assertEquals(events.size(), tsc.count()); + } } - void inc() { - for (int i = t.i; i < t.times.size(); i++) { - ts.inc(t.getAsLong() / 1000); + public void testCounterReset() { + long time = now; + for (int i=0; i < 20; i++) { + long count = 0; + long withinBucket = randomIntBetween(1, (int) (customCounterResolution / 2)); + time += customCounterResolution + (i * customCounterDuration); + long last = time; + for (int j=0; j < withinBucket; j++) { + long bucketTime = (time / customCounterResolution) * customCounterResolution; + last = bucketTime + randomLongBetween(0, customCounterResolution - 1); + counter.inc(last); + count++; + } + assertEquals(count, counter.sum(last)); } } - public static class TimeProvider implements LongSupplier { - public final List times = new ArrayList<>(); - public int i = 0; + public void testFiveMinuteReset() { + long time = now; + long resolution = tsc.fiveMinutes.resolution; + long duration = tsc.fiveMinutes.duration; + for (int i=0; i < 20; i++) { + long withinBucket = randomLongBetween(1, resolution); + time += resolution + (i * duration); + for (int j=0; j < withinBucket; j++) { + inc(time + j); + } + TimeSeries ts = tsc.timeSeries(time); + assertThat(five(time) - ts.fiveMinutes, fiveDelta); + assertThat(fifteen(time) - ts.fifteenMinutes, fifteenDelta); + assertThat(twentyFour(time) - ts.twentyFourHours, twentyFourDelta); + assertEquals(events.size(), tsc.count()); + } + } - public void add(long time) { - times.add(time * 1000); + public void testFifteenMinuteReset() { + long time = now; + long resolution = tsc.fifteenMinutes.resolution; + long duration = tsc.fifteenMinutes.duration; + for (int i=0; i < 20; i++) { + long withinBucket = randomLongBetween(1, resolution); + time += resolution + (i * duration); + for (int j=0; j < withinBucket; j++) { + inc(time + j); + } + TimeSeries ts = tsc.timeSeries(time); + assertThat(five(time) - ts.fiveMinutes, fiveDelta); + assertThat(fifteen(time) - ts.fifteenMinutes, fifteenDelta); + assertThat(twentyFour(time) - ts.twentyFourHours, twentyFourDelta); + assertEquals(events.size(), tsc.count()); } + } - @Override - public long getAsLong() { - assert times.size() > 0; - if (i >= times.size()) { - return times.get(times.size() - 1); + // Count the last five minutes of events before t + public long five(long t) { + return countLast(t, tsc.fiveMinutes, events); + } + + // Count the last fifteen minutes of events before t + public long fifteen(long t) { + return countLast(t, tsc.fifteenMinutes, events); + } + + // Count the last twenty-four hours of events before t + public long twentyFour(long t) { + return countLast(t, tsc.twentyFourHours, events); + } + + // Count the last set of events that would be recorded by counter + public long countLast(long t, Counter counter, List events) { + long count = 0; + long after = ((t - counter.duration) / counter.resolution) * counter.resolution; + for (long event : events) { + if (event > after) { + count++; } - return times.get(i++); } + return count; + } + + private void inc(long t) { + tsc.inc(t); + events.add(t); } } From 3c8ea05448f601950ed191a873cf3adcd92e7043 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 27 Oct 2021 10:02:32 -0500 Subject: [PATCH 18/26] Update merge from master --- .../org/elasticsearch/script/ScriptCache.java | 2 +- .../org/elasticsearch/script/ScriptMetrics.java | 2 +- .../org/elasticsearch/script/ScriptService.java | 7 ++++--- .../ingest/IngestServiceTests.java | 15 +++++++++------ .../elasticsearch/script/ScriptCacheTests.java | 17 +++++++++++------ 5 files changed, 26 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCache.java b/server/src/main/java/org/elasticsearch/script/ScriptCache.java index 6255fea51e15f..2525c4e0e000c 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptCache.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptCache.java @@ -45,7 +45,7 @@ public class ScriptCache { private final double compilesAllowedPerNano; private final String contextRateSetting; - ScriptCache(int cacheMaxSize, TimeValue cacheExpire, CompilationRate maxCompilationRate, String contextRateSetting + ScriptCache(int cacheMaxSize, TimeValue cacheExpire, CompilationRate maxCompilationRate, String contextRateSetting, LongSupplier timeProvider) { this.cacheSize = cacheMaxSize; this.cacheExpire = cacheExpire; diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java index 750fd309582af..b1cb745116ece 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java @@ -39,7 +39,7 @@ protected long now() { } public ScriptStats stats() { - return new ScriptStats(compilationsMetric.count(), cacheEvictionsMetric.count(), compilationLimitTriggered.count()); + return new ScriptStats(compilations.count(), cacheEvictions.count(), compilationLimitTriggered.count()); } public ScriptContextStats stats(String context) { diff --git a/server/src/main/java/org/elasticsearch/script/ScriptService.java b/server/src/main/java/org/elasticsearch/script/ScriptService.java index 02c2204e5468c..9775cb0a80231 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptService.java @@ -601,7 +601,7 @@ CacheHolder generalCacheHolder(Settings settings) { rate = SCRIPT_COMPILATION_RATE_ZERO; } return new CacheHolder(SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(settings), SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(settings), rate, - SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey()); + SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), timeProvider); } CacheHolder contextCacheHolder(Settings settings) { @@ -643,9 +643,10 @@ static class CacheHolder { final ScriptCache general; final Map> contextCache; - CacheHolder(int cacheMaxSize, TimeValue cacheExpire, ScriptCache.CompilationRate maxCompilationRate, String contextRateSetting) { + CacheHolder(int cacheMaxSize, TimeValue cacheExpire, ScriptCache.CompilationRate maxCompilationRate, String contextRateSetting, + LongSupplier timeProvider) { contextCache = null; - general = new ScriptCache(cacheMaxSize, cacheExpire, maxCompilationRate, contextRateSetting); + general = new ScriptCache(cacheMaxSize, cacheExpire, maxCompilationRate, contextRateSetting, timeProvider); } CacheHolder(Map context) { diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 090f1958bb008..26d48c4639516 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -89,6 +89,7 @@ import static java.util.Collections.emptySet; import static org.elasticsearch.core.Tuple.tuple; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.emptyIterable; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -97,11 +98,11 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -487,7 +488,9 @@ public void testGetProcessorsInPipelineComplexConditional() throws Exception { Collections.emptyMap() ) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L); + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L + ); Map processors = new HashMap<>(); processors.put("complexSet", (factories, tag, description, config) -> { diff --git a/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java b/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java index b60b27249b806..337609a3e10b5 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java @@ -64,23 +64,28 @@ public void testGeneralCompilationCircuitBreaking() throws Exception { final TimeValue expire = ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(Settings.EMPTY); final Integer size = ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(Settings.EMPTY); String settingName = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(); - ScriptCache cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(1, TimeValue.timeValueMinutes(1)), settingName); + ScriptCache cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(1, TimeValue.timeValueMinutes(1)), settingName, + () -> 1L); cache.checkCompilationLimit(); // should pass expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), settingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), settingName, + () -> 1L); cache.checkCompilationLimit(); // should pass cache.checkCompilationLimit(); // should pass expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); int count = randomIntBetween(5, 50); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(count, TimeValue.timeValueMinutes(1)), settingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(count, TimeValue.timeValueMinutes(1)), settingName, + () -> 1L); for (int i = 0; i < count; i++) { cache.checkCompilationLimit(); // should pass } expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), settingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), settingName, + () -> 1L); expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); cache = new ScriptCache(size, expire, - new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)), settingName); + new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)), settingName, + () -> 1L); int largeLimit = randomIntBetween(1000, 10000); for (int i = 0; i < largeLimit; i++) { cache.checkCompilationLimit(); @@ -110,7 +115,7 @@ public void testGeneralUnlimitedCompilationRate() { final Integer size = ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(Settings.EMPTY); final TimeValue expire = ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(Settings.EMPTY); String settingName = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(); - ScriptCache cache = new ScriptCache(size, expire, ScriptCache.UNLIMITED_COMPILATION_RATE, settingName); + ScriptCache cache = new ScriptCache(size, expire, ScriptCache.UNLIMITED_COMPILATION_RATE, settingName, () -> 1L); ScriptCache.TokenBucketState initialState = cache.tokenBucketState.get(); for(int i=0; i < 3000; i++) { cache.checkCompilationLimit(); From d078d19a410011319a828bbc41a1bf4803a51fe6 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 27 Oct 2021 10:11:04 -0500 Subject: [PATCH 19/26] Revert imports IngestServiceTests --- .../org/elasticsearch/ingest/IngestServiceTests.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 26d48c4639516..ce39076e0b58b 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -98,11 +98,11 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyInt; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.argThat; -import static org.mockito.Matchers.eq; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; From 916a4a155d545820f207c0247357a34260210a11 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 27 Oct 2021 11:19:53 -0500 Subject: [PATCH 20/26] spotless apply --- .../ingest/common/ScriptProcessorTests.java | 4 ++- .../java/org/elasticsearch/node/Node.java | 16 +++++++++--- .../org/elasticsearch/script/ScriptCache.java | 9 +++++-- .../elasticsearch/script/ScriptService.java | 17 ++++++++++--- .../script/TimeSeriesCounter.java | 3 +-- .../ingest/ConditionalProcessorTests.java | 8 ++++-- .../script/ScriptCacheTests.java | 25 ++++++++++--------- .../script/ScriptStatsTests.java | 18 ++----------- .../script/TimeSeriesCounterTests.java | 15 ++++++----- .../search/sort/AbstractSortTestCase.java | 8 ++++-- .../java/org/elasticsearch/node/MockNode.java | 8 ++++-- 11 files changed, 76 insertions(+), 55 deletions(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java index 45092efe4e33b..6add04b9b9b61 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java @@ -49,7 +49,9 @@ public void setupScripting() { return null; }), Collections.emptyMap()) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L); + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L + ); script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()); ingestScript = scriptService.compile(script, IngestScript.CONTEXT).newInstance(script.getParams()); } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index bbc5b18a6596e..c79f39225bd14 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -421,8 +421,12 @@ protected Node( client = new NodeClient(settings, threadPool); final ScriptModule scriptModule = new ScriptModule(settings, pluginsService.filterPlugins(ScriptPlugin.class)); - final ScriptService scriptService = - newScriptService(settings, scriptModule.engines, scriptModule.contexts, threadPool::absoluteTimeInMillis); + final ScriptService scriptService = newScriptService( + settings, + scriptModule.engines, + scriptModule.contexts, + threadPool::absoluteTimeInMillis + ); AnalysisModule analysisModule = new AnalysisModule(this.environment, pluginsService.filterPlugins(AnalysisPlugin.class)); // this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool // so we might be late here already @@ -1489,8 +1493,12 @@ protected SearchService newSearchService( /** * Creates a new the ScriptService. This method can be overwritten by tests to inject mock implementations. */ - protected ScriptService newScriptService(Settings settings, Map engines, Map> contexts, - LongSupplier timeProvider) { + protected ScriptService newScriptService( + Settings settings, + Map engines, + Map> contexts, + LongSupplier timeProvider + ) { return new ScriptService(settings, engines, contexts, timeProvider); } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCache.java b/server/src/main/java/org/elasticsearch/script/ScriptCache.java index cfd2dd08140a0..95aad9b086671 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptCache.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptCache.java @@ -45,8 +45,13 @@ public class ScriptCache { private final double compilesAllowedPerNano; private final String contextRateSetting; - ScriptCache(int cacheMaxSize, TimeValue cacheExpire, CompilationRate maxCompilationRate, String contextRateSetting, - LongSupplier timeProvider) { + ScriptCache( + int cacheMaxSize, + TimeValue cacheExpire, + CompilationRate maxCompilationRate, + String contextRateSetting, + LongSupplier timeProvider + ) { this.cacheSize = cacheMaxSize; this.cacheExpire = cacheExpire; this.contextRateSetting = contextRateSetting; diff --git a/server/src/main/java/org/elasticsearch/script/ScriptService.java b/server/src/main/java/org/elasticsearch/script/ScriptService.java index 9f15c3a4b8865..b741692618988 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptService.java @@ -179,8 +179,12 @@ public class ScriptService implements Closeable, ClusterStateApplier, ScriptComp // package private for tests final AtomicReference cacheHolder = new AtomicReference<>(); - public ScriptService(Settings settings, Map engines, Map> contexts, - LongSupplier timeProvider) { + public ScriptService( + Settings settings, + Map engines, + Map> contexts, + LongSupplier timeProvider + ) { this.engines = Collections.unmodifiableMap(Objects.requireNonNull(engines)); this.contexts = Collections.unmodifiableMap(Objects.requireNonNull(contexts)); @@ -764,8 +768,13 @@ static class CacheHolder { final ScriptCache general; final Map> contextCache; - CacheHolder(int cacheMaxSize, TimeValue cacheExpire, ScriptCache.CompilationRate maxCompilationRate, String contextRateSetting, - LongSupplier timeProvider) { + CacheHolder( + int cacheMaxSize, + TimeValue cacheExpire, + ScriptCache.CompilationRate maxCompilationRate, + String contextRateSetting, + LongSupplier timeProvider + ) { contextCache = null; general = new ScriptCache(cacheMaxSize, cacheExpire, maxCompilationRate, contextRateSetting, timeProvider); } diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index 5366ee96aec79..d18c03a22d349 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -73,7 +73,6 @@ public long count() { return total < 0 ? 0 : total; } - /* * Keeps track event counts over a duration. Events are clamped to buckets, either the current bucket or a future * bucket. A bucket represents all events over a period of resolution number of seconds. @@ -285,7 +284,7 @@ public static class Counter { protected final long duration; protected final long[] buckets; - // The start time of buckets[0]. bucket(t + (i * duration)) is the same for all i. startOfEpoch + // The start time of buckets[0]. bucket(t + (i * duration)) is the same for all i. startOfEpoch protected long startOfEpoch; protected int curBucket = 0; diff --git a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java index 9acd1cb0ddcff..e44a790a1fd0d 100644 --- a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java @@ -59,7 +59,9 @@ public void testChecksCondition() throws Exception { Collections.emptyMap() ) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L); + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L + ); Map document = new HashMap<>(); LongSupplier relativeTimeProvider = mock(LongSupplier.class); when(relativeTimeProvider.getAsLong()).thenReturn(0L, TimeUnit.MILLISECONDS.toNanos(1), 0L, TimeUnit.MILLISECONDS.toNanos(2)); @@ -258,7 +260,9 @@ private static void assertMutatingCtxThrows(Consumer> mutati return false; }), Collections.emptyMap()) ), - new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L); + new HashMap<>(ScriptModule.CORE_CONTEXTS), + () -> 1L + ); Map document = new HashMap<>(); ConditionalProcessor processor = new ConditionalProcessor( randomAlphaOfLength(10), diff --git a/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java b/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java index c6bd3d1c1ba9f..9e4c77ed7be40 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java @@ -18,6 +18,7 @@ public class ScriptCacheTests extends ESTestCase { private static final LongSupplier time = () -> 1L; + // even though circuit breaking is allowed to be configured per minute, we actually weigh this over five minutes // simply by multiplying by five, so even setting it to one, requires five compilations to break public void testCompilationCircuitBreaking() throws Exception { @@ -40,8 +41,7 @@ public void testCompilationCircuitBreaking() throws Exception { ); cache.checkCompilationLimit(); // should pass expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), rateSettingName, - time); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), rateSettingName, time); cache.checkCompilationLimit(); // should pass cache.checkCompilationLimit(); // should pass expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); @@ -51,8 +51,7 @@ public void testCompilationCircuitBreaking() throws Exception { cache.checkCompilationLimit(); // should pass } expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), rateSettingName, - time); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), rateSettingName, time); expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); cache = new ScriptCache( size, @@ -71,24 +70,26 @@ public void testGeneralCompilationCircuitBreaking() throws Exception { final TimeValue expire = ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(Settings.EMPTY); final Integer size = ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(Settings.EMPTY); String settingName = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(); - ScriptCache cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(1, TimeValue.timeValueMinutes(1)), settingName, - () -> 1L); + ScriptCache cache = new ScriptCache( + size, + expire, + new ScriptCache.CompilationRate(1, TimeValue.timeValueMinutes(1)), + settingName, + () -> 1L + ); cache.checkCompilationLimit(); // should pass expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), settingName, - () -> 1L); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), settingName, () -> 1L); cache.checkCompilationLimit(); // should pass cache.checkCompilationLimit(); // should pass expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); int count = randomIntBetween(5, 50); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(count, TimeValue.timeValueMinutes(1)), settingName, - () -> 1L); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(count, TimeValue.timeValueMinutes(1)), settingName, () -> 1L); for (int i = 0; i < count; i++) { cache.checkCompilationLimit(); // should pass } expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), settingName, - () -> 1L); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), settingName, () -> 1L); expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); cache = new ScriptCache( size, diff --git a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java index 864331f35e28c..c01c558f1682c 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java @@ -28,14 +28,7 @@ public class ScriptStatsTests extends ESTestCase { public void testXContent() throws IOException { List contextStats = List.of( - new ScriptContextStats( - "contextB", - 100, - 201, - 302, - new TimeSeries(1000, 1001, 1002), - new TimeSeries(2000, 2001, 2002) - ), + new ScriptContextStats("contextB", 100, 201, 302, new TimeSeries(1000, 1001, 1002), new TimeSeries(2000, 2001, 2002)), new ScriptContextStats("contextA", 1000, 2010, 3020, null, new TimeSeries(0, 0, 0)) ); ScriptStats stats = new ScriptStats(contextStats); @@ -96,14 +89,7 @@ public void testSerializeEmptyTimeSeries() throws IOException { } public void testSerializeTimeSeries() throws IOException { - Function mkContextStats = (ts) -> new ScriptContextStats( - "c", - 1111, - 2222, - 3333, - null, - ts - ); + Function mkContextStats = (ts) -> new ScriptContextStats("c", 1111, 2222, 3333, null, ts); TimeSeries series = new TimeSeries(0, 0, 5); String format = "{\n" diff --git a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java index 0aaf3a5d4e7f5..b8611f354ecec 100644 --- a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java +++ b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java @@ -66,7 +66,7 @@ public void testOnePerSecond() { long t; long next = randomLongBetween(1, HOUR); long twentyFive = 25 * HOUR; - for (int i=0; i < twentyFive; i++) { + for (int i = 0; i < twentyFive; i++) { t = time + i; inc(t); @@ -104,7 +104,6 @@ public void testCounterIncrementSameBucket() { assertEquals(count, counter.sum(start + duration + resolution - 1)); } - public void testFiveMinuteSameBucket() { inc(now); long resolution = tsc.fiveMinutes.resolution; @@ -294,12 +293,12 @@ public void testFifteenMinuteSkipBuckets() { public void testCounterReset() { long time = now; - for (int i=0; i < 20; i++) { + for (int i = 0; i < 20; i++) { long count = 0; long withinBucket = randomIntBetween(1, (int) (customCounterResolution / 2)); time += customCounterResolution + (i * customCounterDuration); long last = time; - for (int j=0; j < withinBucket; j++) { + for (int j = 0; j < withinBucket; j++) { long bucketTime = (time / customCounterResolution) * customCounterResolution; last = bucketTime + randomLongBetween(0, customCounterResolution - 1); counter.inc(last); @@ -313,10 +312,10 @@ public void testFiveMinuteReset() { long time = now; long resolution = tsc.fiveMinutes.resolution; long duration = tsc.fiveMinutes.duration; - for (int i=0; i < 20; i++) { + for (int i = 0; i < 20; i++) { long withinBucket = randomLongBetween(1, resolution); time += resolution + (i * duration); - for (int j=0; j < withinBucket; j++) { + for (int j = 0; j < withinBucket; j++) { inc(time + j); } TimeSeries ts = tsc.timeSeries(time); @@ -331,10 +330,10 @@ public void testFifteenMinuteReset() { long time = now; long resolution = tsc.fifteenMinutes.resolution; long duration = tsc.fifteenMinutes.duration; - for (int i=0; i < 20; i++) { + for (int i = 0; i < 20; i++) { long withinBucket = randomLongBetween(1, resolution); time += resolution + (i * duration); - for (int j=0; j < withinBucket; j++) { + for (int j = 0; j < withinBucket; j++) { inc(time + j); } TimeSeries ts = tsc.timeSeries(time); diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index db1cf0414968f..09a36db0fa8b8 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -76,8 +76,12 @@ public static void init() { Settings baseSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); Map, Object>> scripts = Collections.singletonMap(MOCK_SCRIPT_NAME, p -> null); ScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap()); - scriptService = new ScriptService(baseSettings, Collections.singletonMap(engine.getType(), engine), ScriptModule.CORE_CONTEXTS, - () -> 1L); + scriptService = new ScriptService( + baseSettings, + Collections.singletonMap(engine.getType(), engine), + ScriptModule.CORE_CONTEXTS, + () -> 1L + ); SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables()); diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java index 1ea8672dc516b..fa041d51005f2 100644 --- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java +++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java @@ -158,8 +158,12 @@ protected SearchService newSearchService( } @Override - protected ScriptService newScriptService(Settings settings, Map engines, Map> contexts, - LongSupplier timeProvider) { + protected ScriptService newScriptService( + Settings settings, + Map engines, + Map> contexts, + LongSupplier timeProvider + ) { if (getPluginsService().filterPlugins(MockScriptService.TestPlugin.class).isEmpty()) { return super.newScriptService(settings, engines, contexts, timeProvider); } From 2822f35888bd9c58714c4c856c37f72b01c42504 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 27 Oct 2021 13:58:56 -0500 Subject: [PATCH 21/26] Tests and total --- .../elasticsearch/script/ScriptMetrics.java | 10 +- .../org/elasticsearch/script/TimeSeries.java | 21 +++- .../script/TimeSeriesCounter.java | 2 +- .../cluster/node/stats/NodeStatsTests.java | 4 +- .../script/ScriptStatsTests.java | 16 +-- .../script/TimeSeriesCounterTests.java | 115 ++++++++++++++++-- 6 files changed, 143 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java index b1cb745116ece..0f18feb499d58 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java @@ -44,13 +44,15 @@ public ScriptStats stats() { public ScriptContextStats stats(String context) { long t = now(); + TimeSeries compilationsTimeSeries = compilations.timeSeries(t); + TimeSeries cacheEvictionsTimeSeries = cacheEvictions.timeSeries(t); return new ScriptContextStats( context, - compilations.count(), - cacheEvictions.count(), + compilationsTimeSeries.total, + cacheEvictionsTimeSeries.total, compilationLimitTriggered.count(), - compilations.timeSeries(t), - cacheEvictions.timeSeries(t) + compilationsTimeSeries, + cacheEvictionsTimeSeries ); } } diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeries.java b/server/src/main/java/org/elasticsearch/script/TimeSeries.java index 0b3ac2d9815d3..3007e74d5b959 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeries.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeries.java @@ -8,6 +8,7 @@ package org.elasticsearch.script; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -21,23 +22,31 @@ public class TimeSeries implements Writeable, ToXContentFragment { public final long fiveMinutes; public final long fifteenMinutes; public final long twentyFourHours; + public final long total; public TimeSeries() { this.fiveMinutes = 0; this.fifteenMinutes = 0; this.twentyFourHours = 0; + this.total = 0; } - public TimeSeries(long fiveMinutes, long fifteenMinutes, long twentyFourHours) { + public TimeSeries(long fiveMinutes, long fifteenMinutes, long twentyFourHours, long total) { this.fiveMinutes = fiveMinutes; this.fifteenMinutes = fifteenMinutes; this.twentyFourHours = twentyFourHours; + this.total = total; } public TimeSeries(StreamInput in) throws IOException { fiveMinutes = in.readVLong(); fifteenMinutes = in.readVLong(); twentyFourHours = in.readVLong(); + if (in.getVersion().onOrAfter(Version.V_8_1_0)) { + total = in.readVLong(); + } else { + total = 0; + } } @Override @@ -53,6 +62,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(fiveMinutes); out.writeVLong(fifteenMinutes); out.writeVLong(twentyFourHours); + if (out.getVersion().onOrAfter(Version.V_8_1_0)) { + out.writeVLong(total); + } } public boolean isEmpty() { @@ -64,11 +76,14 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; TimeSeries that = (TimeSeries) o; - return fiveMinutes == that.fiveMinutes && fifteenMinutes == that.fifteenMinutes && twentyFourHours == that.twentyFourHours; + return fiveMinutes == that.fiveMinutes + && fifteenMinutes == that.fifteenMinutes + && twentyFourHours == that.twentyFourHours + && total == that.total; } @Override public int hashCode() { - return Objects.hash(fiveMinutes, fifteenMinutes, twentyFourHours); + return Objects.hash(fiveMinutes, fifteenMinutes, twentyFourHours, total); } } diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index d18c03a22d349..7c5c8405cd796 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -59,7 +59,7 @@ public void inc(long t) { public TimeSeries timeSeries(long t) { lock.readLock().lock(); try { - return new TimeSeries(fiveMinutes.sum(t), fifteenMinutes.sum(t), twentyFourHours.sum(t)); + return new TimeSeries(fiveMinutes.sum(t), fifteenMinutes.sum(t), twentyFourHours.sum(t), count()); } finally { lock.readLock().unlock(); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index 838f2f2ccb9df..581728e7350f2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -286,8 +286,8 @@ public void testSerialization() throws IOException { compilations += generatedStats.getCompilations(); assertEquals(generatedStats.getCompilations(), deserStats.getCompilations()); - assertEquals(generatedStats.getCacheEvictionsHistory(), deserStats.getCacheEvictionsHistory()); - assertEquals(generatedStats.getCompilationsHistory(), deserStats.getCompilationsHistory()); + assertEquals(generatedStats.getCacheEvictions(), deserStats.getCacheEvictions()); + assertEquals(generatedStats.getCompilations(), deserStats.getCompilations()); } assertEquals(evictions, scriptStats.getCacheEvictions()); assertEquals(limited, scriptStats.getCompilationLimitTriggered()); diff --git a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java index c01c558f1682c..4e250b6da18a8 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java @@ -28,8 +28,8 @@ public class ScriptStatsTests extends ESTestCase { public void testXContent() throws IOException { List contextStats = List.of( - new ScriptContextStats("contextB", 100, 201, 302, new TimeSeries(1000, 1001, 1002), new TimeSeries(2000, 2001, 2002)), - new ScriptContextStats("contextA", 1000, 2010, 3020, null, new TimeSeries(0, 0, 0)) + new ScriptContextStats("contextB", 100, 201, 302, new TimeSeries(1000, 1001, 1002, 100), new TimeSeries(2000, 2001, 2002, 201)), + new ScriptContextStats("contextA", 1000, 2010, 3020, null, new TimeSeries(0, 0, 0, 0)) ); ScriptStats stats = new ScriptStats(contextStats); final XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); @@ -91,7 +91,7 @@ public void testSerializeEmptyTimeSeries() throws IOException { public void testSerializeTimeSeries() throws IOException { Function mkContextStats = (ts) -> new ScriptContextStats("c", 1111, 2222, 3333, null, ts); - TimeSeries series = new TimeSeries(0, 0, 5); + TimeSeries series = new TimeSeries(0, 0, 5, 2222); String format = "{\n" + " \"context\" : \"c\",\n" + " \"compilations\" : 1111,\n" @@ -109,23 +109,23 @@ public void testSerializeTimeSeries() throws IOException { assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 0, 0, 5))); - series = new TimeSeries(0, 7, 1234); + series = new TimeSeries(0, 7, 1234, 1234); builder = XContentFactory.jsonBuilder().prettyPrint(); mkContextStats.apply(series).toXContent(builder, ToXContent.EMPTY_PARAMS); assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 0, 7, 1234))); - series = new TimeSeries(123, 456, 789); + series = new TimeSeries(123, 456, 789, 1234); builder = XContentFactory.jsonBuilder().prettyPrint(); mkContextStats.apply(series).toXContent(builder, ToXContent.EMPTY_PARAMS); assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 123, 456, 789))); } public void testTimeSeriesIsEmpty() { - assertTrue((new TimeSeries(0, 0, 0)).isEmpty()); + assertTrue((new TimeSeries(0, 0, 0, 0)).isEmpty()); long day = randomLongBetween(1, 1024); long fifteen = day >= 1 ? randomLongBetween(0, day) : 0; long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; - assertFalse((new TimeSeries(five, fifteen, day)).isEmpty()); + assertFalse((new TimeSeries(five, fifteen, day, 0)).isEmpty()); } public void testTimeSeriesSerialization() throws IOException { @@ -165,7 +165,7 @@ public ScriptContextStats randomStats() { long day = randomLongBetween(0, histStats[j]); long fifteen = day >= 1 ? randomLongBetween(0, day) : 0; long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; - timeSeries.add(new TimeSeries(five, fifteen, day)); + timeSeries.add(new TimeSeries(five, fifteen, day, day)); } else { timeSeries.add(new TimeSeries()); } diff --git a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java index b8611f354ecec..ca0124a8c3616 100644 --- a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java +++ b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java @@ -162,6 +162,35 @@ public void testFifteenMinuteSameBucket() { assertEquals(count, tsc.timeSeries(start + duration + resolution - 1).fifteenMinutes); } + public void testTwentyFourHourSameBucket() { + inc(now); + long resolution = tsc.twentyFourHours.resolution; + long duration = tsc.twentyFourHours.duration; + long start = (now / resolution) * resolution; + long count = randomLongBetween(1, resolution); + for (int i = 1; i < count; i++) { + inc(start + i); + } + assertEquals(count, tsc.count()); + assertEquals(count, tsc.timeSeries(now).twentyFourHours); + + long t = 0; + for (; t <= duration; t += resolution) { + assertEquals(count, tsc.timeSeries(start + t).twentyFourHours); + } + + TimeSeries series = tsc.timeSeries(start + t); + assertEquals(0, series.fiveMinutes); + assertEquals(0, series.fifteenMinutes); + assertEquals(0, series.twentyFourHours); + + series = tsc.timeSeries(start + duration + resolution); + assertEquals(0, series.fiveMinutes); + assertEquals(0, series.fifteenMinutes); + assertEquals(0, series.twentyFourHours); + assertEquals(count, tsc.timeSeries(start + duration + resolution - 1).twentyFourHours); + } + public void testCounterIncrementBucket() { long count = customCounterDuration / customCounterResolution; for (int i = 0; i < count; i++) { @@ -181,20 +210,23 @@ public void testFiveMinuteIncrementBucket() { for (int i = 0; i < count; i++) { inc(now + i * resolution); } - - TimeSeries ts = tsc.timeSeries(now + duration); + long t = now + duration; + TimeSeries ts = tsc.timeSeries(t); assertEquals(count, ts.fiveMinutes); assertEquals(count, ts.fifteenMinutes); assertEquals(count, ts.twentyFourHours); assertEquals(count, tsc.count()); - ts = tsc.timeSeries(now + duration + resolution); + t = now + duration + resolution; + ts = tsc.timeSeries(t); assertEquals(count - 1, ts.fiveMinutes); assertEquals(count, ts.fifteenMinutes); assertEquals(count, ts.twentyFourHours); - ts = tsc.timeSeries(now + duration + (2 * resolution)); - assertEquals(count - 2, ts.fiveMinutes); + long numRes = 2; + t = now + duration + (numRes * resolution); + ts = tsc.timeSeries(t); + assertEquals(count - numRes, ts.fiveMinutes); assertEquals(count, ts.fifteenMinutes); assertEquals(count, ts.twentyFourHours); @@ -226,10 +258,11 @@ public void testFifteenMinuteIncrementBucket() { assertEquals(count - 1, ts.fifteenMinutes); assertEquals(count, ts.twentyFourHours); - t = now + duration + (2 * resolution); + long numRes = 2; + t = now + duration + (numRes * resolution); ts = tsc.timeSeries(t); assertEquals(five(t), ts.fiveMinutes); - assertEquals(count - 2, ts.fifteenMinutes); + assertEquals(count - numRes, ts.fifteenMinutes); assertEquals(count, ts.twentyFourHours); inc(now + duration); @@ -241,6 +274,42 @@ public void testFifteenMinuteIncrementBucket() { assertEquals(count + 1, tsc.count()); } + public void testTwentyFourHourIncrementBucket() { + int count = tsc.twentyFourHours.buckets.length; + long resolution = tsc.twentyFourHours.resolution; + long duration = tsc.twentyFourHours.duration; + for (int i = 0; i < count; i++) { + long t = now + i * resolution; + inc(t); + } + long t = now + duration; + TimeSeries ts = tsc.timeSeries(t); + assertEquals(five(t), ts.fiveMinutes); + assertEquals(fifteen(t), ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + + t = now + duration + resolution; + ts = tsc.timeSeries(t); + assertEquals(five(t), ts.fiveMinutes); + assertEquals(0, ts.fifteenMinutes); + assertEquals(count - 1, ts.twentyFourHours); + + long numRes = 2; + t = now + duration + (numRes * resolution); + ts = tsc.timeSeries(t); + assertEquals(0, ts.fiveMinutes); + assertEquals(0, ts.fifteenMinutes); + assertEquals(count - numRes, ts.twentyFourHours); + + inc(now + duration); + t = now + duration + resolution; + ts = tsc.timeSeries(t); + assertEquals(0, ts.fiveMinutes); + assertEquals(1, ts.fifteenMinutes); + assertEquals(count, ts.twentyFourHours); + assertEquals(count + 1, tsc.count()); + } + public void testCounterSkipBuckets() { int count = (int) (customCounterDuration / customCounterResolution); for (int skip = 1; skip <= count; skip++) { @@ -291,6 +360,22 @@ public void testFifteenMinuteSkipBuckets() { } } + public void testTwentyFourHourSkipBuckets() { + int count = tsc.twentyFourHours.buckets.length; + long resolution = tsc.twentyFourHours.resolution; + long duration = tsc.twentyFourHours.duration; + for (int skip = 1; skip <= count; skip++) { + reset(); + for (int i = 0; (i * skip * resolution) < duration; i++) { + inc(now + (i * skip * resolution)); + } + TimeSeries ts = tsc.timeSeries(now + duration); + assertEquals(five(now + duration), ts.fiveMinutes); + assertEquals(events.size(), ts.twentyFourHours); + assertEquals(events.size(), tsc.count()); + } + } + public void testCounterReset() { long time = now; for (int i = 0; i < 20; i++) { @@ -344,6 +429,22 @@ public void testFifteenMinuteReset() { } } + public void testTwentyFourHourReset() { + long time = now; + long resolution = tsc.twentyFourHours.resolution; + long duration = tsc.twentyFourHours.duration; + for (int i = 0; i < 20; i++) { + long withinBucket = randomLongBetween(1, resolution); + time += resolution + (i * duration); + for (int j = 0; j < withinBucket; j++) { + inc(time + j); + } + TimeSeries ts = tsc.timeSeries(time); + assertThat(twentyFour(time) - ts.twentyFourHours, twentyFourDelta); + assertEquals(events.size(), tsc.count()); + } + } + // Count the last five minutes of events before t public long five(long t) { return countLast(t, tsc.fiveMinutes, events); From b2cce9ceaa8809719c3ba9a1432a3d5e9b1c2ff1 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 27 Oct 2021 14:25:19 -0500 Subject: [PATCH 22/26] Revert total --- .../action/admin/cluster/node/stats/NodeStatsTests.java | 2 +- .../test/java/org/elasticsearch/script/ScriptStatsTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index 581728e7350f2..706a3a261ebce 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -878,7 +878,7 @@ private static TimeSeries randomTimeSeries() { long day = randomLongBetween(0, 1024); long fifteen = day >= 1 ? randomLongBetween(0, day) : 0; long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; - return new TimeSeries(five, fifteen, day); + return new TimeSeries(five, fifteen, day, day); } else { return new TimeSeries(); } diff --git a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java index 4e250b6da18a8..6a5d50b63c03b 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java @@ -165,7 +165,7 @@ public ScriptContextStats randomStats() { long day = randomLongBetween(0, histStats[j]); long fifteen = day >= 1 ? randomLongBetween(0, day) : 0; long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; - timeSeries.add(new TimeSeries(five, fifteen, day, day)); + timeSeries.add(new TimeSeries(five, fifteen, day, 0)); } else { timeSeries.add(new TimeSeries()); } From d673c2008ceb209e6de6411160915884e556d00a Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 27 Oct 2021 14:30:05 -0500 Subject: [PATCH 23/26] align time period --- .../main/java/org/elasticsearch/script/TimeSeriesCounter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index 7c5c8405cd796..48db4d608596b 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -277,7 +277,7 @@ public static class Counter { * Future * [0] [_] [_] [_] [_] * |_____0_____|___________|___________|___________|___________| - * 400[c]-> 420[f]-> + * 400[c]-> 420[f]-> * */ protected final long resolution; From e9740d51926af1ec328ba4dc7a228a036753edf8 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 27 Oct 2021 17:04:11 -0500 Subject: [PATCH 24/26] total serialized in ScriptContextStats --- .../script/ScriptContextStats.java | 15 +++---- .../elasticsearch/script/ScriptMetrics.java | 9 +---- .../org/elasticsearch/script/TimeSeries.java | 16 +++++++- .../cluster/node/stats/NodeStatsTests.java | 9 ++--- .../script/ScriptStatsTests.java | 40 ++++++++----------- 5 files changed, 43 insertions(+), 46 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java b/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java index b6fa5e4a0ec79..7a6d4513ba9e4 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java @@ -28,15 +28,13 @@ public class ScriptContextStats implements Writeable, ToXContentFragment, Compar public ScriptContextStats( String context, - long compilations, - long cacheEvictions, long compilationLimitTriggered, TimeSeries compilationsHistory, TimeSeries cacheEvictionsHistory ) { this.context = Objects.requireNonNull(context); - this.compilations = compilations; - this.cacheEvictions = cacheEvictions; + this.compilations = compilationsHistory.total; + this.cacheEvictions = cacheEvictionsHistory.total; this.compilationLimitTriggered = compilationLimitTriggered; this.compilationsHistory = compilationsHistory; this.cacheEvictionsHistory = cacheEvictionsHistory; @@ -47,9 +45,12 @@ public ScriptContextStats(StreamInput in) throws IOException { compilations = in.readVLong(); cacheEvictions = in.readVLong(); compilationLimitTriggered = in.readVLong(); - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_8_1_0)) { compilationsHistory = new TimeSeries(in); cacheEvictionsHistory = new TimeSeries(in); + } else if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + compilationsHistory = new TimeSeries(in).withTotal(compilations); + cacheEvictionsHistory = new TimeSeries(in).withTotal(cacheEvictions); } else { compilationsHistory = null; cacheEvictionsHistory = null; @@ -99,7 +100,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.COMPILATIONS, getCompilations()); TimeSeries series = getCompilationsHistory(); - if (series != null && series.isEmpty() == false) { + if (series != null && series.areTimingsEmpty() == false) { builder.startObject(Fields.COMPILATIONS_HISTORY); series.toXContent(builder, params); builder.endObject(); @@ -107,7 +108,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.CACHE_EVICTIONS, getCacheEvictions()); series = getCacheEvictionsHistory(); - if (series != null && series.isEmpty() == false) { + if (series != null && series.areTimingsEmpty() == false) { builder.startObject(Fields.CACHE_EVICTIONS_HISTORY); series.toXContent(builder, params); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java index 0f18feb499d58..26fb3c48d0d66 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java @@ -46,13 +46,6 @@ public ScriptContextStats stats(String context) { long t = now(); TimeSeries compilationsTimeSeries = compilations.timeSeries(t); TimeSeries cacheEvictionsTimeSeries = cacheEvictions.timeSeries(t); - return new ScriptContextStats( - context, - compilationsTimeSeries.total, - cacheEvictionsTimeSeries.total, - compilationLimitTriggered.count(), - compilationsTimeSeries, - cacheEvictionsTimeSeries - ); + return new ScriptContextStats(context, compilationLimitTriggered.count(), compilationsTimeSeries, cacheEvictionsTimeSeries); } } diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeries.java b/server/src/main/java/org/elasticsearch/script/TimeSeries.java index 3007e74d5b959..43d7ef5706937 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeries.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeries.java @@ -31,6 +31,13 @@ public TimeSeries() { this.total = 0; } + public TimeSeries(long total) { + this.fiveMinutes = 0; + this.fifteenMinutes = 0; + this.twentyFourHours = 0; + this.total = total; + } + public TimeSeries(long fiveMinutes, long fifteenMinutes, long twentyFourHours, long total) { this.fiveMinutes = fiveMinutes; this.fifteenMinutes = fifteenMinutes; @@ -38,6 +45,10 @@ public TimeSeries(long fiveMinutes, long fifteenMinutes, long twentyFourHours, l this.total = total; } + TimeSeries withTotal(long total) { + return new TimeSeries(fiveMinutes, fifteenMinutes, twentyFourHours, total); + } + public TimeSeries(StreamInput in) throws IOException { fiveMinutes = in.readVLong(); fifteenMinutes = in.readVLong(); @@ -51,6 +62,7 @@ public TimeSeries(StreamInput in) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // total is omitted from toXContent as it's written at a higher level by ScriptContextStats builder.field(ScriptContextStats.Fields.FIVE_MINUTES, fiveMinutes); builder.field(ScriptContextStats.Fields.FIFTEEN_MINUTES, fifteenMinutes); builder.field(ScriptContextStats.Fields.TWENTY_FOUR_HOURS, twentyFourHours); @@ -67,8 +79,8 @@ public void writeTo(StreamOutput out) throws IOException { } } - public boolean isEmpty() { - return twentyFourHours == 0; + public boolean areTimingsEmpty() { + return fiveMinutes == 0 && fifteenMinutes == 0 && twentyFourHours == 0; } @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index 706a3a261ebce..45ea9165cc69f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -719,13 +719,9 @@ public static NodeStats createNodeStats() { List stats = new ArrayList<>(numContents); HashSet contexts = new HashSet<>(); for (int i = 0; i < numContents; i++) { - long compile = randomLongBetween(0, 1024); - long eviction = randomLongBetween(0, 1024); String context = randomValueOtherThanMany(contexts::contains, () -> randomAlphaOfLength(12)); contexts.add(context); - stats.add( - new ScriptContextStats(context, compile, eviction, randomLongBetween(0, 1024), randomTimeSeries(), randomTimeSeries()) - ); + stats.add(new ScriptContextStats(context, randomLongBetween(0, 1024), randomTimeSeries(), randomTimeSeries())); } scriptStats = new ScriptStats(stats); } @@ -875,7 +871,8 @@ public static NodeStats createNodeStats() { private static TimeSeries randomTimeSeries() { if (randomBoolean()) { - long day = randomLongBetween(0, 1024); + long total = randomLongBetween(0, 1024); + long day = total >= 1 ? randomLongBetween(0, total) : 0; long fifteen = day >= 1 ? randomLongBetween(0, day) : 0; long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; return new TimeSeries(five, fifteen, day, day); diff --git a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java index 6a5d50b63c03b..c229a1c9576e5 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java @@ -28,8 +28,8 @@ public class ScriptStatsTests extends ESTestCase { public void testXContent() throws IOException { List contextStats = List.of( - new ScriptContextStats("contextB", 100, 201, 302, new TimeSeries(1000, 1001, 1002, 100), new TimeSeries(2000, 2001, 2002, 201)), - new ScriptContextStats("contextA", 1000, 2010, 3020, null, new TimeSeries(0, 0, 0, 0)) + new ScriptContextStats("contextB", 302, new TimeSeries(1000, 1001, 1002, 100), new TimeSeries(2000, 2001, 2002, 201)), + new ScriptContextStats("contextA", 3020, new TimeSeries(1000), new TimeSeries(2010)) ); ScriptStats stats = new ScriptStats(contextStats); final XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); @@ -72,8 +72,7 @@ public void testXContent() throws IOException { } public void testSerializeEmptyTimeSeries() throws IOException { - TimeSeries empty = new TimeSeries(); - ScriptContextStats stats = new ScriptContextStats("c", 1111, 2222, 3333, null, empty); + ScriptContextStats stats = new ScriptContextStats("c", 3333, new TimeSeries(1111), new TimeSeries(2222)); XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); stats.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -89,13 +88,13 @@ public void testSerializeEmptyTimeSeries() throws IOException { } public void testSerializeTimeSeries() throws IOException { - Function mkContextStats = (ts) -> new ScriptContextStats("c", 1111, 2222, 3333, null, ts); + Function mkContextStats = (ts) -> new ScriptContextStats("c", 3333, new TimeSeries(1111), ts); TimeSeries series = new TimeSeries(0, 0, 5, 2222); String format = "{\n" + " \"context\" : \"c\",\n" + " \"compilations\" : 1111,\n" - + " \"cache_evictions\" : 2222,\n" + + " \"cache_evictions\" : %d,\n" + " \"cache_evictions_history\" : {\n" + " \"5m\" : %d,\n" + " \"15m\" : %d,\n" @@ -107,25 +106,27 @@ public void testSerializeTimeSeries() throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); mkContextStats.apply(series).toXContent(builder, ToXContent.EMPTY_PARAMS); - assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 0, 0, 5))); + assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 2222, 0, 0, 5))); - series = new TimeSeries(0, 7, 1234, 1234); + series = new TimeSeries(0, 7, 1234, 5678); builder = XContentFactory.jsonBuilder().prettyPrint(); mkContextStats.apply(series).toXContent(builder, ToXContent.EMPTY_PARAMS); - assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 0, 7, 1234))); + assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 5678, 0, 7, 1234))); - series = new TimeSeries(123, 456, 789, 1234); + series = new TimeSeries(123, 456, 789, 91011); builder = XContentFactory.jsonBuilder().prettyPrint(); mkContextStats.apply(series).toXContent(builder, ToXContent.EMPTY_PARAMS); - assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 123, 456, 789))); + assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 91011, 123, 456, 789))); } public void testTimeSeriesIsEmpty() { - assertTrue((new TimeSeries(0, 0, 0, 0)).isEmpty()); - long day = randomLongBetween(1, 1024); + assertTrue((new TimeSeries(0, 0, 0, 0)).areTimingsEmpty()); + assertTrue((new TimeSeries(0, 0, 0, 123)).areTimingsEmpty()); + long total = randomLongBetween(1, 1024); + long day = randomLongBetween(1, total); long fifteen = day >= 1 ? randomLongBetween(0, day) : 0; long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; - assertFalse((new TimeSeries(five, fifteen, day, 0)).isEmpty()); + assertFalse((new TimeSeries(five, fifteen, day, 0)).areTimingsEmpty()); } public void testTimeSeriesSerialization() throws IOException { @@ -165,18 +166,11 @@ public ScriptContextStats randomStats() { long day = randomLongBetween(0, histStats[j]); long fifteen = day >= 1 ? randomLongBetween(0, day) : 0; long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; - timeSeries.add(new TimeSeries(five, fifteen, day, 0)); + timeSeries.add(new TimeSeries(five, fifteen, day, histStats[j])); } else { timeSeries.add(new TimeSeries()); } } - return new ScriptContextStats( - randomAlphaOfLength(15), - histStats[0], - histStats[1], - randomLongBetween(0, 1024), - timeSeries.get(0), - timeSeries.get(1) - ); + return new ScriptContextStats(randomAlphaOfLength(15), randomLongBetween(0, 1024), timeSeries.get(0), timeSeries.get(1)); } } From 6b3f5d496c6e66001b86ac7339830125d7051b77 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 27 Oct 2021 17:48:21 -0500 Subject: [PATCH 25/26] diagram tweaks --- .../elasticsearch/script/TimeSeriesCounter.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index 48db4d608596b..d60fa59aa0620 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -82,17 +82,22 @@ public static class Counter { /* * In the following diagrams, we take a duration of 100 and resolution of 20. * - * |_______________________________________| + * |___________________________________________________________| * duration = 100 - * |_______|_______|_______|_______|_______| + * + * |___________|___________|___________|___________|___________| * buckets = 5 - * |_______| + * + * |___________| * resolution = 20 * - * Action: inc(235) + * Action: inc(235) - Increment the counter at time 235 seconds. + * + * While there is only one `buckets` array, it's useful to view the array as overlapping three + * epoch (time at bucket[0]), the last epoch, the present epoch and the future epoch. * * Past - * [_] [_] [_] [3] [4] + * [_] [_] [2] [3] [4] * |___________|___________|___________|___________|___________| * 140[e]-> 160-> 180-> 199 * @@ -240,7 +245,7 @@ public static class Counter { * * Future * [_] [_] [_] [_] [_] - * |_____0_____|___________|___________|___________|___________| + * |___________|___________|___________|___________|___________| * 400[c][f]-> * * ------------------------------------------------------------------------------------------------------------------ From 63a23dcc78deac49c26e874c400ca9193af12f3c Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 2 Nov 2021 18:13:08 -0500 Subject: [PATCH 26/26] time moved to TSC, add comments, docs --- docs/reference/cluster/nodes-stats.asciidoc | 37 ++++ .../modules/indices/circuit_breaker.asciidoc | 8 + .../script/ScriptCacheStats.java | 2 +- .../script/ScriptContextStats.java | 4 +- .../elasticsearch/script/ScriptMetrics.java | 31 +-- .../org/elasticsearch/script/ScriptStats.java | 55 +++++- .../org/elasticsearch/script/TimeSeries.java | 24 ++- .../script/TimeSeriesCounter.java | 181 +++++++++++++----- .../cluster/node/stats/NodeStatsTests.java | 2 +- .../script/ScriptStatsTests.java | 22 ++- .../script/TimeSeriesCounterTests.java | 155 ++++++++++----- 11 files changed, 392 insertions(+), 129 deletions(-) diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index 7e15acb99d4bd..253890cd2a175 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -2032,10 +2032,47 @@ Contains script statistics for the node. (integer) Total number of inline script compilations performed by the node. +`compilations_history`:: +(object) +Contains this recent history of script compilations + +.Properties of `compilations_history` +[%collapsible%open] +======= +`5m`:: +(long) +The number of script compilations in the last five minutes. +`15m`:: +(long) +The number of script compilations in the last fifteen minutes. +`24h`:: +(long) +The number of script compilations in the last twenty-four hours. +======= + `cache_evictions`:: (integer) Total number of times the script cache has evicted old data. + +`cache_evictions_history`:: +(object) +Contains this recent history of script cache evictions + +.Properties of `cache_evictions` +[%collapsible%open] +======= +`5m`:: +(long) +The number of script cache evictions in the last five minutes. +`15m`:: +(long) +The number of script cache evictions in the last fifteen minutes. +`24h`:: +(long) +The number of script cache evictions in the last twenty-four hours. +======= + `compilation_limit_triggered`:: (integer) Total number of times the <> to inspect +the number of recent cache evictions, `script.cache_evictions_history` and +compilations `script.compilations_history`. If there are a large +number of recent cache evictions or compilations, the script cache may be +undersized, consider doubling the size of the script cache via the setting +`script.cache.max_size`. + [[regex-circuit-breaker]] [discrete] ==== Regex circuit breaker diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java b/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java index 4efdaf81b9986..baf8384daf8d2 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java @@ -133,7 +133,7 @@ public ScriptStats sum() { cacheEvictions += stat.getCacheEvictions(); compilationLimitTriggered += stat.getCompilationLimitTriggered(); } - return new ScriptStats(compilations, cacheEvictions, compilationLimitTriggered); + return new ScriptStats(compilations, cacheEvictions, compilationLimitTriggered, null, null); } static final class Fields { diff --git a/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java b/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java index 7a6d4513ba9e4..acde189431c44 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java @@ -52,8 +52,8 @@ public ScriptContextStats(StreamInput in) throws IOException { compilationsHistory = new TimeSeries(in).withTotal(compilations); cacheEvictionsHistory = new TimeSeries(in).withTotal(cacheEvictions); } else { - compilationsHistory = null; - cacheEvictionsHistory = null; + compilationsHistory = new TimeSeries(compilations); + cacheEvictionsHistory = new TimeSeries(cacheEvictions); } } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java index 26fb3c48d0d66..c08d760108eb2 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java @@ -14,38 +14,41 @@ public class ScriptMetrics { final CounterMetric compilationLimitTriggered = new CounterMetric(); - final TimeSeriesCounter compilations = new TimeSeriesCounter(); - final TimeSeriesCounter cacheEvictions = new TimeSeriesCounter(); - final LongSupplier timeProvider; + final TimeSeriesCounter compilations; + final TimeSeriesCounter cacheEvictions; public ScriptMetrics(LongSupplier timeProvider) { - this.timeProvider = timeProvider; + compilations = new TimeSeriesCounter(timeProvider); + cacheEvictions = new TimeSeriesCounter(timeProvider); } public void onCompilation() { - compilations.inc(now()); + compilations.inc(); } public void onCacheEviction() { - cacheEvictions.inc(now()); + cacheEvictions.inc(); } public void onCompilationLimit() { compilationLimitTriggered.inc(); } - protected long now() { - return timeProvider.getAsLong() / 1000; - } - public ScriptStats stats() { - return new ScriptStats(compilations.count(), cacheEvictions.count(), compilationLimitTriggered.count()); + TimeSeries compilationsTimeSeries = compilations.timeSeries(); + TimeSeries cacheEvictionsTimeSeries = cacheEvictions.timeSeries(); + return new ScriptStats( + compilationsTimeSeries.total, + cacheEvictionsTimeSeries.total, + compilationLimitTriggered.count(), + compilationsTimeSeries, + cacheEvictionsTimeSeries + ); } public ScriptContextStats stats(String context) { - long t = now(); - TimeSeries compilationsTimeSeries = compilations.timeSeries(t); - TimeSeries cacheEvictionsTimeSeries = cacheEvictions.timeSeries(t); + TimeSeries compilationsTimeSeries = compilations.timeSeries(); + TimeSeries cacheEvictionsTimeSeries = cacheEvictions.timeSeries(); return new ScriptContextStats(context, compilationLimitTriggered.count(), compilationsTimeSeries, cacheEvictionsTimeSeries); } } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptStats.java b/server/src/main/java/org/elasticsearch/script/ScriptStats.java index c6e9bb908dd59..f9d365db126a7 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptStats.java @@ -8,6 +8,7 @@ package org.elasticsearch.script; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -26,6 +27,8 @@ public class ScriptStats implements Writeable, ToXContentFragment { private final long compilations; private final long cacheEvictions; private final long compilationLimitTriggered; + private final TimeSeries compilationsHistory; + private final TimeSeries cacheEvictionsHistory; public ScriptStats(List contextStats) { ArrayList ctxStats = new ArrayList<>(contextStats.size()); @@ -43,30 +46,60 @@ public ScriptStats(List contextStats) { this.compilations = compilations; this.cacheEvictions = cacheEvictions; this.compilationLimitTriggered = compilationLimitTriggered; + this.compilationsHistory = new TimeSeries(compilations); + this.cacheEvictionsHistory = new TimeSeries(cacheEvictions); } - public ScriptStats(long compilations, long cacheEvictions, long compilationLimitTriggered) { + public ScriptStats( + long compilations, + long cacheEvictions, + long compilationLimitTriggered, + TimeSeries compilationsHistory, + TimeSeries cacheEvictionsHistory + ) { this.contextStats = Collections.emptyList(); this.compilations = compilations; this.cacheEvictions = cacheEvictions; this.compilationLimitTriggered = compilationLimitTriggered; + this.compilationsHistory = compilationsHistory == null ? new TimeSeries(compilations) : compilationsHistory; + this.cacheEvictionsHistory = cacheEvictionsHistory == null ? new TimeSeries(cacheEvictions) : cacheEvictionsHistory; } public ScriptStats(ScriptContextStats context) { - this(context.getCompilations(), context.getCacheEvictions(), context.getCompilationLimitTriggered()); + this( + context.getCompilations(), + context.getCacheEvictions(), + context.getCompilationLimitTriggered(), + context.getCompilationsHistory(), + context.getCacheEvictionsHistory() + ); } public ScriptStats(StreamInput in) throws IOException { - compilations = in.readVLong(); - cacheEvictions = in.readVLong(); + if (in.getVersion().onOrAfter(Version.V_8_1_0)) { + compilationsHistory = new TimeSeries(in); + cacheEvictionsHistory = new TimeSeries(in); + compilations = compilationsHistory.total; + cacheEvictions = cacheEvictionsHistory.total; + } else { + compilations = in.readVLong(); + cacheEvictions = in.readVLong(); + compilationsHistory = new TimeSeries(compilations); + cacheEvictionsHistory = new TimeSeries(cacheEvictions); + } compilationLimitTriggered = in.readVLong(); contextStats = in.readList(ScriptContextStats::new); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeVLong(compilations); - out.writeVLong(cacheEvictions); + if (out.getVersion().onOrAfter(Version.V_8_1_0)) { + compilationsHistory.writeTo(out); + cacheEvictionsHistory.writeTo(out); + } else { + out.writeVLong(compilations); + out.writeVLong(cacheEvictions); + } out.writeVLong(compilationLimitTriggered); out.writeList(contextStats); } @@ -104,6 +137,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.COMPILATIONS, compilations); builder.field(Fields.CACHE_EVICTIONS, cacheEvictions); builder.field(Fields.COMPILATION_LIMIT_TRIGGERED, compilationLimitTriggered); + if (compilationsHistory != null && compilationsHistory.areTimingsEmpty() == false) { + builder.startObject(ScriptContextStats.Fields.COMPILATIONS_HISTORY); + compilationsHistory.toXContent(builder, params); + builder.endObject(); + } + if (cacheEvictionsHistory != null && cacheEvictionsHistory.areTimingsEmpty() == false) { + builder.startObject(ScriptContextStats.Fields.COMPILATIONS_HISTORY); + cacheEvictionsHistory.toXContent(builder, params); + builder.endObject(); + } builder.startArray(Fields.CONTEXTS); for (ScriptContextStats contextStats : contextStats) { contextStats.toXContent(builder, params); diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeries.java b/server/src/main/java/org/elasticsearch/script/TimeSeries.java index 43d7ef5706937..8399a65a57e08 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeries.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeries.java @@ -18,19 +18,15 @@ import java.io.IOException; import java.util.Objects; +/** + * A response class representing a snapshot of a {@link org.elasticsearch.script.TimeSeriesCounter} at a point in time. + */ public class TimeSeries implements Writeable, ToXContentFragment { public final long fiveMinutes; public final long fifteenMinutes; public final long twentyFourHours; public final long total; - public TimeSeries() { - this.fiveMinutes = 0; - this.fifteenMinutes = 0; - this.twentyFourHours = 0; - this.total = 0; - } - public TimeSeries(long total) { this.fiveMinutes = 0; this.fifteenMinutes = 0; @@ -98,4 +94,18 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(fiveMinutes, fifteenMinutes, twentyFourHours, total); } + + @Override + public String toString() { + return "TimeSeries{" + + "fiveMinutes=" + + fiveMinutes + + ", fifteenMinutes=" + + fifteenMinutes + + ", twentyFourHours=" + + twentyFourHours + + ", total=" + + total + + '}'; + } } diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java index d60fa59aa0620..642c6e6547abb 100644 --- a/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java +++ b/server/src/main/java/org/elasticsearch/script/TimeSeriesCounter.java @@ -8,15 +8,37 @@ package org.elasticsearch.script; +import org.elasticsearch.threadpool.ThreadPool; + import java.util.Arrays; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.LongSupplier; /** * Provides a counter with a history of 5m/15m/24h. * - * Callers increment the counter and query + * Callers increment the counter and query the current state of the TimeSeries. + * + * {@link TimeSeriesCounter#inc()} increments the counter to indicate an event happens "now", with metadata about "now" + * from the given {@link TimeSeriesCounter#timeProvider}. + * + * {@link TimeSeriesCounter#timeSeries()} provides a snapshot of the counters at the current time, given by + * {@link TimeSeriesCounter#timeProvider}. The snapshot includes the number of events in the last five minutes, the last fifteen + * minutes, the last twenty-four hours and an all-time count. + * + * Caveats: + * * If the timeProvider produces a time stamp value, {@code t[j]} that occurs _before_ an earlier invocation {@code t[j]} (where j is an + * invocation that *happens-before*, in the java memory model sense), the time stamp is treated as occurring at the latest time seen, + * {@code t[latest]}, EXCEPT if {@code t[latest] - t[j]} is earlier than any time covered by the twenty-four counter. If that occurs, + * time goes backwards more than 24 hours (between 24:00 and 23:45 depending on the circumstance) the history is reset but total is + * retained. + * * All counters have a resolution: + * - 5m resolution is 15s + * - 15m resolution is 90s + * - 24h resolution is 15m + * The counter will drop events between one second and the resolution minus one second during a bucket rollover. */ public class TimeSeriesCounter { public static final int SECOND = 1; @@ -29,23 +51,38 @@ public class TimeSeriesCounter { protected Counter fiveMinutes = new Counter(15 * SECOND, 5 * MINUTE); protected Counter fifteenMinutes = new Counter(90 * SECOND, 15 * MINUTE); protected Counter twentyFourHours = new Counter(15 * MINUTE, 24 * HOUR); + protected final LongSupplier timeProvider; + + /** + * Create a TimeSeriesCounter with the given {@code timeProvider}. + * + * The {@code timeProvider} must return positive values, in milliseconds. In live code, this is expected + * to be {@link System#currentTimeMillis()} or a proxy such as {@link ThreadPool#absoluteTimeInMillis()}. + */ + public TimeSeriesCounter(LongSupplier timeProvider) { + this.timeProvider = timeProvider; + } /** * Increment counters at timestamp t, any increment more than 24hours before the current time * series resets all historical counters, but the total counter is still increments. */ - public void inc(long t) { + public void inc() { + long now = now(); adder.increment(); lock.writeLock().lock(); try { - if (t < twentyFourHours.earliestTimeInCounter()) { - fiveMinutes.reset(t); - fifteenMinutes.reset(t); - twentyFourHours.reset(t); + // If time has skewed more than a day in the past, reset the histories, but not the adder. + // Counters clamp all times before the end of the current bucket as happening in the current + // bucket, so this reset avoids pegging counters to their current buckets for too long. + if (now < twentyFourHours.earliestTimeInCounter()) { + fiveMinutes.reset(now); + fifteenMinutes.reset(now); + twentyFourHours.reset(now); } else { - fiveMinutes.inc(t); - fifteenMinutes.inc(t); - twentyFourHours.inc(t); + fiveMinutes.inc(now); + fifteenMinutes.inc(now); + twentyFourHours.inc(now); } } finally { lock.writeLock().unlock(); @@ -56,17 +93,29 @@ public void inc(long t) { * Get the value of the counters for the last 5 minutes, last 15 minutes and the last 24 hours from * t. May include events up to resolution before those durations due to counter granularity. */ - public TimeSeries timeSeries(long t) { + public TimeSeries timeSeries() { + long now = now(); lock.readLock().lock(); try { - return new TimeSeries(fiveMinutes.sum(t), fifteenMinutes.sum(t), twentyFourHours.sum(t), count()); + return new TimeSeries(fiveMinutes.sum(now), fifteenMinutes.sum(now), twentyFourHours.sum(now), count()); } finally { lock.readLock().unlock(); } } + // The current time, in seconds, from the timeProvider, which emits milliseconds. Clamped to zero or positive numbers. + protected long now() { + long t = timeProvider.getAsLong(); + if (t < 0) { + t = 0; + } else { + t /= 1000; + } + return t; + } + /** - * The total number of events for all time covered gby the counters. + * The total number of events for all time covered by the counters. */ public long count() { long total = adder.sum(); @@ -112,43 +161,43 @@ public static class Counter { * 300[c]-> 320[f] * * [a] Beginning of the current epoch - * startOfEpoch = 200 = (t / duration) * duration = (235 / 100) * 100 - * Start time of bucket zero, this is used to anchor the bucket ring in time. Without `startOfEpoch`, + * startOfCurrentEpoch = 200 = (t / duration) * duration = (235 / 100) * 100 + * Start time of bucket zero, this is used to anchor the bucket ring in time. Without `startOfCurrentEpoch`, * it would be impossible to distinguish between two times that are `duration` away from each other. - * In this example, the first inc used time 235, since startOfEpoch is rounded down to the nearest + * In this example, the first inc used time 235, since startOfCurrentEpoch is rounded down to the nearest * duration (100), it is 200. * * [b] The current bucket - * curBucket = 1 = (t / resolution) % buckets.length = (235 / 20) % 5 + * curBucket = 1 = (t / resolution) % buckets.length = (235 / 20) % 5 * The latest active bucket in the bucket ring. The bucket of a timestamp is determined by the `resolution`. * In this case the `resolution` is 20, so each bucket covers 20 seconds, the first covers 200-219, the * second covers 220->239, the third 240->259 and so on. 235 is in the second bucket, at index 1. * * [c] Beginning of the next epoch - * nextEpoch() = 300 = startOfEpoch + duration = 200 + 100 - * The first time of the next epoch, this indicates when `startOfEpoch` should be updated. When `curBucket` - * advances to or past zero, `startOfEpoch` must be updated to `nextEpoch()` + * nextEpoch() = 300 = startOfCurrentEpoch + duration = 200 + 100 + * The first time of the next epoch, this indicates when `startOfCurrentEpoch` should be updated. When `curBucket` + * advances to or past zero, `startOfCurrentEpoch` must be updated to `nextEpoch()` * * [d] Beginning of the next bucket - * nextBucketStartTime() = 240 = startOfEpoch + ((curBucket + 1) * resolution) = 200 + ((1 + 1) * 20 + * nextBucketStartTime() = 240 = startOfCurrentEpoch + ((curBucket + 1) * resolution) = 200 + ((1 + 1) * 20 * The first time of the next bucket, when a timestamp is greater than or equal to this time, we must update - * the `curBucket` and potentially the `startOfEpoch`. + * the `curBucket` and potentially the `startOfCurrentEpoch`. * * [e] The earliest time to sum - * earliestTimeInCounter() = 140 = nextBucketStartTime() - duration = 240 - 100 + * earliestTimeInCounter() = 140 = nextBucketStartTime() - duration = 240 - 100 * `curBucket` covers the latest timestamp seen by the `Counter`. Since the counter keeps a history, when a * caller calls `sum(t)`, the `Counter` must clamp the range to the earliest time covered by its current state. * The times proceed backwards for `buckets.length - 1`. - * **Important** this is likely _before_ the `startOfEpoch`. `startOfEpoch` is the timestamp of bucket[0]. + * **Important** this is likely _before_ the `startOfCurrentEpoch`. `startOfCurrentEpoch` is the timestamp of bucket[0]. * * [f] The counter is no longer valid at this time - * counterExpired() = 320 = startOfEpoch + (curBucket * resolution) + duration = 200 + (1 * 20) + 100 + * counterExpired() = 320 = startOfCurrentEpoch + (curBucket * resolution) + duration = 200 + (1 * 20) + 100 * Where `earliestTimeInCounter()` is looking backwards, to the history covered by the counter, `counterExpired()` * looks forward to when current counter has expired. Since `curBucket` represents the latest time in this counter, * `counterExpired()` is `duration` from the start of the time covered from `curBucket` * * [g] The next bucket in the bucket ring - * nextBucket(curBucket) = 2 = (i + 1) % buckets.length = (1 + 1) % 5 + * nextBucket(curBucket) = 2 = (i + 1) % buckets.length = (1 + 1) % 5 * Since `buckets` is a ring, the next bucket may wrap around. * * ------------------------------------------------------------------------------------------------------------------ @@ -176,7 +225,7 @@ public static class Counter { * Action: inc(267) - 267 is in bucket 3, so bucket 2 is zeroed and skipped. Bucket 2 is zeroed because it may have * had contents that were relevant for timestamps 140 - 159. * - * The `startOfEpoch`[a], does not change while `curBucket`[b] is now bucket 3. + * The `startOfCurrentEpoch`[a], does not change while `curBucket`[b] is now bucket 3. * * `nextEpoch()`[c] does not change as there hasn't been a rollover. * @@ -213,11 +262,11 @@ public static class Counter { * Action: inc(310) - 310 is in bucket 0, so bucket 4 is zeroed and skipped, as it may have had contents * for timestamps 180-199. * - * The `startOfEpoch`[a], is now 300 as the `Counter` has rolled through bucket 0. + * The `startOfCurrentEpoch`[a], is now 300 as the `Counter` has rolled through bucket 0. * * `curBucket`[b] is now bucket 0. * - * `nextEpoch()`[c] is now 400 because `startOfEpoch`[a] has changed. + * `nextEpoch()`[c] is now 400 because `startOfCurrentEpoch`[a] has changed. * * `nextBucketStartTime()`[d] is now 320, the start time of bucket 1 in this new epoch. * @@ -252,7 +301,7 @@ public static class Counter { * * Action: inc(321) - 321 is in bucket 1, so the previous contents of bucket 1 is replaced with the value 1. * - * The `startOfEpoch`[a] remains 300. + * The `startOfCurrentEpoch`[a] remains 300. * * `curBucket`[b] is now bucket 1. * @@ -284,13 +333,39 @@ public static class Counter { * |_____0_____|___________|___________|___________|___________| * 400[c]-> 420[f]-> * + * + * ------------------------------------------------------------------------------------------------------------------ + * + * Action: sum(321) - This is a sum at the exact time of the last update, because of the `earliestTimeInCounter` check, + * it starts at bucket 2, which is after the current bucket index, 1, but the earliest time covered by + * the counter, 240 to 259. + * 1) calculate start = 321 - duration = 321 - 100 = 221 + * 2) start is before the nextBucketStartTime (340), so sum does not terminate early + * 3) start is before the earliestTimeInCounter (240) -> start = 240 + * 3) Iterate from bucket(start) = bucket(240) = 2 until curBucket 1, summing the following + * bucket 2 = 0, bucket 3 = 1, bucket 4 = 0, bucket 0 = 1 -> 1 + 1 = 2 + * 4) return that with the context of bucket 1 = 1 -> 2 + 1 = 3 + * + * Action: sum(465) - This sum is so far in the future, it does not overlap with any bucket in range + * 1) calculate start = 465 - duration = 465 - 100 = 365 + * 2) start is greater than or equal to the nextBucketStartTime (340), so we know the counter has no contexts + * -> return 0 + * + * Action: sum(439) - This sum starts _after_ the last update, which is at 321, but because of the bucket resolution + * sum still catches the value bucket 1, times 320 to 339. + * 1) calculate start = 439 - duration = 439 - 100 = 339 + * 2) start is before nextBucketStartTime(340), so sum does not terminate early + * 3) start is after earliestTimeInCounter (240), so it is now updated + * 4) bucket(start) = 1 which is curBucket, so the for loop falls through + * 5) return total = 0 + buckets[curBucket] = 0 + 1 = 1 */ protected final long resolution; protected final long duration; protected final long[] buckets; - // The start time of buckets[0]. bucket(t + (i * duration)) is the same for all i. startOfEpoch - protected long startOfEpoch; + // The start time of buckets[0]. bucket(t + (i * duration)) is the same for all i. startOfCurrentEpoch allows the counter + // to differentiate between those times. + protected long startOfCurrentEpoch; protected int curBucket = 0; /** @@ -307,36 +382,37 @@ public Counter(long resolution, long duration) { this.resolution = resolution; this.duration = duration; this.buckets = new long[(int) (duration / resolution)]; + this.startOfCurrentEpoch = 0; assert buckets.length > 0; } /** - * Increment the counter at time {@code t}, expressed in seconds. + * Increment the counter at time {@code now}, expressed in seconds. */ - public void inc(long t) { - if (t < nextBucketStartTime()) { + public void inc(long now) { + if (now < nextBucketStartTime()) { buckets[curBucket]++; - } else if (t >= counterExpired()) { - reset(t); + } else if (now >= counterExpired()) { + reset(now); } else { - int dstBucket = bucket(t); + int dstBucket = bucket(now); for (int i = nextBucket(curBucket); i != dstBucket; i = nextBucket(i)) { buckets[i] = 0; } curBucket = dstBucket; buckets[curBucket] = 1; - if (t >= nextEpoch()) { - startOfEpoch = epoch(t); + if (now >= nextEpoch()) { + startOfCurrentEpoch = epoch(now); } } } /** - * sum for the duration of the counter until {@code end}. + * sum for the duration of the counter until {@code now}. */ - public long sum(long end) { - long start = end - duration; - if (start >= nextBucketStartTime() || start < 0) { + public long sum(long now) { + long start = now - duration; + if (start >= nextBucketStartTime()) { return 0; } @@ -352,12 +428,12 @@ public long sum(long end) { } /** - * Reset the counter. Next counter begins at t. + * Reset the counter. Next counter begins at now. */ - void reset(long t) { + void reset(long now) { Arrays.fill(buckets, 0); - startOfEpoch = epoch(t); - curBucket = bucket(t); + startOfCurrentEpoch = epoch(now); + curBucket = bucket(now); buckets[curBucket] = 1; } @@ -368,17 +444,18 @@ long epoch(long t) { // What is the start time of the next epoch? long nextEpoch() { - return startOfEpoch + duration; + return startOfCurrentEpoch + duration; } - // What is the earliest time covered by this counter? + // What is the earliest time covered by this counter? Counters do not extend before zero. long earliestTimeInCounter() { - return nextBucketStartTime() - duration; + long time = nextBucketStartTime() - duration; + return time <= 0 ? 0 : time; } // When does this entire counter expire? long counterExpired() { - return startOfEpoch + (curBucket * resolution) + duration; + return startOfCurrentEpoch + (curBucket * resolution) + duration; } // bucket for the given time @@ -393,7 +470,7 @@ int nextBucket(int i) { // When does the next bucket start? long nextBucketStartTime() { - return startOfEpoch + ((curBucket + 1) * resolution); + return startOfCurrentEpoch + ((curBucket + 1) * resolution); } } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index 45ea9165cc69f..144c9f0843441 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -877,7 +877,7 @@ private static TimeSeries randomTimeSeries() { long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; return new TimeSeries(five, fifteen, day, day); } else { - return new TimeSeries(); + return new TimeSeries(randomLongBetween(0, 1024)); } } diff --git a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java index c229a1c9576e5..4bf3b74be7d49 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java @@ -136,8 +136,10 @@ public void testTimeSeriesSerialization() throws IOException { assertEquals(stats.getCompilations(), deserStats.getCompilations()); assertEquals(stats.getCacheEvictions(), deserStats.getCacheEvictions()); assertEquals(stats.getCompilationLimitTriggered(), deserStats.getCompilationLimitTriggered()); - assertNull(deserStats.getCompilationsHistory()); - assertNull(deserStats.getCacheEvictionsHistory()); + assertTrue(deserStats.getCompilationsHistory().areTimingsEmpty()); + assertEquals(stats.getCompilations(), deserStats.getCompilationsHistory().total); + assertTrue(deserStats.getCacheEvictionsHistory().areTimingsEmpty()); + assertEquals(stats.getCacheEvictions(), deserStats.getCacheEvictionsHistory().total); deserStats = serDeser(Version.V_8_0_0, Version.V_8_0_0, stats); assertEquals(stats.getCompilations(), deserStats.getCompilations()); @@ -145,6 +147,20 @@ public void testTimeSeriesSerialization() throws IOException { assertEquals(stats.getCompilationLimitTriggered(), deserStats.getCompilationLimitTriggered()); assertEquals(stats.getCompilationsHistory(), deserStats.getCompilationsHistory()); assertEquals(stats.getCacheEvictionsHistory(), deserStats.getCacheEvictionsHistory()); + + deserStats = serDeser(Version.V_8_1_0, Version.V_7_16_0, stats); + assertEquals(stats.getCompilations(), deserStats.getCompilations()); + assertEquals(stats.getCacheEvictions(), deserStats.getCacheEvictions()); + assertEquals(stats.getCompilationLimitTriggered(), deserStats.getCompilationLimitTriggered()); + assertEquals(new TimeSeries(stats.getCompilationsHistory().total), deserStats.getCompilationsHistory()); + assertEquals(new TimeSeries(stats.getCacheEvictionsHistory().total), deserStats.getCacheEvictionsHistory()); + + deserStats = serDeser(Version.V_8_1_0, Version.V_8_1_0, stats); + assertEquals(stats.getCompilations(), deserStats.getCompilations()); + assertEquals(stats.getCacheEvictions(), deserStats.getCacheEvictions()); + assertEquals(stats.getCompilationLimitTriggered(), deserStats.getCompilationLimitTriggered()); + assertEquals(stats.getCompilationsHistory(), deserStats.getCompilationsHistory()); + assertEquals(stats.getCacheEvictionsHistory(), deserStats.getCacheEvictionsHistory()); } public ScriptContextStats serDeser(Version outVersion, Version inVersion, ScriptContextStats stats) throws IOException { @@ -168,7 +184,7 @@ public ScriptContextStats randomStats() { long five = fifteen >= 1 ? randomLongBetween(0, fifteen) : 0; timeSeries.add(new TimeSeries(five, fifteen, day, histStats[j])); } else { - timeSeries.add(new TimeSeries()); + timeSeries.add(new TimeSeries(histStats[j])); } } return new ScriptContextStats(randomAlphaOfLength(15), randomLongBetween(0, 1024), timeSeries.get(0), timeSeries.get(1)); diff --git a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java index ca0124a8c3616..1eed55e5112d3 100644 --- a/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java +++ b/server/src/test/java/org/elasticsearch/script/TimeSeriesCounterTests.java @@ -14,6 +14,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.LongSupplier; import static org.elasticsearch.script.TimeSeriesCounter.Counter; import static org.elasticsearch.script.TimeSeriesCounter.HOUR; @@ -23,7 +24,8 @@ public class TimeSeriesCounterTests extends ESTestCase { protected long now; protected long customCounterResolution; protected long customCounterDuration; - protected TimeSeriesCounter tsc = new TimeSeriesCounter(); + protected TimeProvider timeProvider = new TimeProvider(); + protected TimeSeriesCounter tsc = new TimeSeriesCounter(timeProvider); protected final Matcher fiveDelta = lessThan(tsc.fiveMinutes.resolution); protected final Matcher fifteenDelta = lessThan(tsc.fifteenMinutes.resolution); protected final Matcher twentyFourDelta = lessThan(tsc.twentyFourHours.resolution); @@ -41,8 +43,9 @@ public void setUp() throws Exception { } protected void reset() { - tsc = new TimeSeriesCounter(); + timeProvider = new TimeProvider(); events = new ArrayList<>(); + tsc = new TimeSeriesCounter(timeProvider); counter = new Counter(customCounterResolution, customCounterDuration); } @@ -61,23 +64,47 @@ public void testCounterIndivisibleResolution() { assertEquals("duration [101] must divisible by resolution [3]", iae.getMessage()); } + public void testNegativeIncrement() { + inc(-100); + assertEquals(1, timeSeries(0).fiveMinutes); + } + + public void testNegativeSum() { + long t = 60; + // t += 24 * HOUR; + inc(t); + t += 2 * tsc.twentyFourHours.resolution; + inc(t); + TimeSeries ts = timeSeries(t); + assertEquals(2, ts.twentyFourHours); + } + + public void testNegativeStart() { + long t = -1 * 48 * HOUR; + inc(t); + t += 2 * tsc.twentyFourHours.resolution; + inc(t); + TimeSeries ts = timeSeries(t); + assertEquals(2, ts.twentyFourHours); + } + public void testOnePerSecond() { long time = now; long t; - long next = randomLongBetween(1, HOUR); + long nextAssertCheck = randomLongBetween(1, HOUR); long twentyFive = 25 * HOUR; for (int i = 0; i < twentyFive; i++) { t = time + i; inc(t); - if (i == next) { - TimeSeries ts = tsc.timeSeries(t); + if (i == nextAssertCheck) { + TimeSeries ts = timeSeries(t); assertThat(five(t) - ts.fiveMinutes, fiveDelta); assertThat(fifteen(t) - ts.fifteenMinutes, fifteenDelta); assertThat(twentyFour(t) - ts.twentyFourHours, twentyFourDelta); assertEquals(i + 1, tsc.count()); - next = Math.min(twentyFive, next + randomLongBetween(HOUR, 3 * HOUR)); + nextAssertCheck = Math.min(twentyFive, nextAssertCheck + randomLongBetween(HOUR, 3 * HOUR)); } } } @@ -87,6 +114,7 @@ public void testCounterIncrementSameBucket() { long duration = 900; counter.inc(now); long count = randomLongBetween(resolution / 2, resolution * 2); + // this is the beginning of the current epoch long start = (now / resolution) * resolution; for (int i = 1; i < count; i++) { counter.inc(start + randomLongBetween(0, resolution - 1)); @@ -96,11 +124,16 @@ public void testCounterIncrementSameBucket() { assertEquals(count, counter.sum(now)); long t = 0; + + // Since we only incremented the first bucket, we should have access to that throughout duration for (; t <= duration; t += resolution) { assertEquals(count, counter.sum(start + t)); } + + // Now we've gone past the end of the duration assertEquals(0, counter.sum(start + t)); assertEquals(0, counter.sum(start + duration + resolution)); + // The last second for which this counter is valid assertEquals(count, counter.sum(start + duration + resolution - 1)); } @@ -114,23 +147,23 @@ public void testFiveMinuteSameBucket() { inc(start + i); } assertEquals(count, tsc.count()); - assertEquals(count, tsc.timeSeries(now).fiveMinutes); + assertEquals(count, timeSeries(now).fiveMinutes); long t = 0; for (; t <= duration; t += resolution) { - assertEquals(count, tsc.timeSeries(start + t).fiveMinutes); + assertEquals(count, timeSeries(start + t).fiveMinutes); } - TimeSeries series = tsc.timeSeries(start + t); + TimeSeries series = timeSeries(start + t); assertEquals(0, series.fiveMinutes); assertEquals(count, series.fifteenMinutes); assertEquals(count, series.twentyFourHours); - series = tsc.timeSeries(start + duration + resolution); + series = timeSeries(start + duration + resolution); assertEquals(0, series.fiveMinutes); assertEquals(count, series.fifteenMinutes); assertEquals(count, series.twentyFourHours); - assertEquals(count, tsc.timeSeries(start + duration + resolution - 1).fiveMinutes); + assertEquals(count, timeSeries(start + duration + resolution - 1).fiveMinutes); } public void testFifteenMinuteSameBucket() { @@ -143,23 +176,23 @@ public void testFifteenMinuteSameBucket() { inc(start + i); } assertEquals(count, tsc.count()); - assertEquals(count, tsc.timeSeries(now).fifteenMinutes); + assertEquals(count, timeSeries(now).fifteenMinutes); long t = 0; for (; t <= duration; t += resolution) { - assertEquals(count, tsc.timeSeries(start + t).fifteenMinutes); + assertEquals(count, timeSeries(start + t).fifteenMinutes); } - TimeSeries series = tsc.timeSeries(start + t); + TimeSeries series = timeSeries(start + t); assertEquals(0, series.fiveMinutes); assertEquals(0, series.fifteenMinutes); assertEquals(count, series.twentyFourHours); - series = tsc.timeSeries(start + duration + resolution); + series = timeSeries(start + duration + resolution); assertEquals(0, series.fiveMinutes); assertEquals(0, series.fifteenMinutes); assertEquals(count, series.twentyFourHours); - assertEquals(count, tsc.timeSeries(start + duration + resolution - 1).fifteenMinutes); + assertEquals(count, timeSeries(start + duration + resolution - 1).fifteenMinutes); } public void testTwentyFourHourSameBucket() { @@ -172,23 +205,23 @@ public void testTwentyFourHourSameBucket() { inc(start + i); } assertEquals(count, tsc.count()); - assertEquals(count, tsc.timeSeries(now).twentyFourHours); + assertEquals(count, timeSeries(now).twentyFourHours); long t = 0; for (; t <= duration; t += resolution) { - assertEquals(count, tsc.timeSeries(start + t).twentyFourHours); + assertEquals(count, timeSeries(start + t).twentyFourHours); } - TimeSeries series = tsc.timeSeries(start + t); + TimeSeries series = timeSeries(start + t); assertEquals(0, series.fiveMinutes); assertEquals(0, series.fifteenMinutes); assertEquals(0, series.twentyFourHours); - series = tsc.timeSeries(start + duration + resolution); + series = timeSeries(start + duration + resolution); assertEquals(0, series.fiveMinutes); assertEquals(0, series.fifteenMinutes); assertEquals(0, series.twentyFourHours); - assertEquals(count, tsc.timeSeries(start + duration + resolution - 1).twentyFourHours); + assertEquals(count, timeSeries(start + duration + resolution - 1).twentyFourHours); } public void testCounterIncrementBucket() { @@ -211,27 +244,27 @@ public void testFiveMinuteIncrementBucket() { inc(now + i * resolution); } long t = now + duration; - TimeSeries ts = tsc.timeSeries(t); + TimeSeries ts = timeSeries(t); assertEquals(count, ts.fiveMinutes); assertEquals(count, ts.fifteenMinutes); assertEquals(count, ts.twentyFourHours); assertEquals(count, tsc.count()); t = now + duration + resolution; - ts = tsc.timeSeries(t); + ts = timeSeries(t); assertEquals(count - 1, ts.fiveMinutes); assertEquals(count, ts.fifteenMinutes); assertEquals(count, ts.twentyFourHours); long numRes = 2; t = now + duration + (numRes * resolution); - ts = tsc.timeSeries(t); + ts = timeSeries(t); assertEquals(count - numRes, ts.fiveMinutes); assertEquals(count, ts.fifteenMinutes); assertEquals(count, ts.twentyFourHours); inc(now + duration); - ts = tsc.timeSeries(now + duration + resolution); + ts = timeSeries(now + duration + resolution); assertEquals(count, ts.fiveMinutes); assertEquals(count + 1, ts.fifteenMinutes); assertEquals(count + 1, ts.twentyFourHours); @@ -247,27 +280,27 @@ public void testFifteenMinuteIncrementBucket() { inc(t); } long t = now + duration; - TimeSeries ts = tsc.timeSeries(t); + TimeSeries ts = timeSeries(t); assertEquals(five(t), ts.fiveMinutes); assertEquals(count, ts.fifteenMinutes); assertEquals(count, ts.twentyFourHours); t = now + duration + resolution; - ts = tsc.timeSeries(t); + ts = timeSeries(t); assertEquals(five(t), ts.fiveMinutes); assertEquals(count - 1, ts.fifteenMinutes); assertEquals(count, ts.twentyFourHours); long numRes = 2; t = now + duration + (numRes * resolution); - ts = tsc.timeSeries(t); + ts = timeSeries(t); assertEquals(five(t), ts.fiveMinutes); assertEquals(count - numRes, ts.fifteenMinutes); assertEquals(count, ts.twentyFourHours); inc(now + duration); t = now + duration + resolution; - ts = tsc.timeSeries(t); + ts = timeSeries(t); assertEquals(five(t), ts.fiveMinutes); assertEquals(count, ts.fifteenMinutes); assertEquals(count + 1, ts.twentyFourHours); @@ -283,27 +316,27 @@ public void testTwentyFourHourIncrementBucket() { inc(t); } long t = now + duration; - TimeSeries ts = tsc.timeSeries(t); + TimeSeries ts = timeSeries(t); assertEquals(five(t), ts.fiveMinutes); assertEquals(fifteen(t), ts.fifteenMinutes); assertEquals(count, ts.twentyFourHours); t = now + duration + resolution; - ts = tsc.timeSeries(t); + ts = timeSeries(t); assertEquals(five(t), ts.fiveMinutes); assertEquals(0, ts.fifteenMinutes); assertEquals(count - 1, ts.twentyFourHours); long numRes = 2; t = now + duration + (numRes * resolution); - ts = tsc.timeSeries(t); + ts = timeSeries(t); assertEquals(0, ts.fiveMinutes); assertEquals(0, ts.fifteenMinutes); assertEquals(count - numRes, ts.twentyFourHours); inc(now + duration); t = now + duration + resolution; - ts = tsc.timeSeries(t); + ts = timeSeries(t); assertEquals(0, ts.fiveMinutes); assertEquals(1, ts.fifteenMinutes); assertEquals(count, ts.twentyFourHours); @@ -328,14 +361,14 @@ public void testFiveMinuteSkipBucket() { long resolution = tsc.fiveMinutes.resolution; long duration = tsc.fiveMinutes.duration; for (int skip = 1; skip <= count; skip++) { - tsc = new TimeSeriesCounter(); + tsc = new TimeSeriesCounter(timeProvider); long increments = 0; for (int i = 0; (i * skip * resolution) < duration; i++) { inc(now + (i * skip * resolution)); increments++; } - TimeSeries series = tsc.timeSeries(now + duration); + TimeSeries series = timeSeries(now + duration); assertEquals(increments, series.fiveMinutes); assertEquals(increments, series.fifteenMinutes); assertEquals(increments, series.twentyFourHours); @@ -352,7 +385,7 @@ public void testFifteenMinuteSkipBuckets() { for (int i = 0; (i * skip * resolution) < duration; i++) { inc(now + (i * skip * resolution)); } - TimeSeries ts = tsc.timeSeries(now + duration); + TimeSeries ts = timeSeries(now + duration); assertEquals(five(now + duration), ts.fiveMinutes); assertEquals(events.size(), ts.fifteenMinutes); assertEquals(events.size(), ts.twentyFourHours); @@ -369,7 +402,7 @@ public void testTwentyFourHourSkipBuckets() { for (int i = 0; (i * skip * resolution) < duration; i++) { inc(now + (i * skip * resolution)); } - TimeSeries ts = tsc.timeSeries(now + duration); + TimeSeries ts = timeSeries(now + duration); assertEquals(five(now + duration), ts.fiveMinutes); assertEquals(events.size(), ts.twentyFourHours); assertEquals(events.size(), tsc.count()); @@ -403,7 +436,7 @@ public void testFiveMinuteReset() { for (int j = 0; j < withinBucket; j++) { inc(time + j); } - TimeSeries ts = tsc.timeSeries(time); + TimeSeries ts = timeSeries(time); assertThat(five(time) - ts.fiveMinutes, fiveDelta); assertThat(fifteen(time) - ts.fifteenMinutes, fifteenDelta); assertThat(twentyFour(time) - ts.twentyFourHours, twentyFourDelta); @@ -421,7 +454,7 @@ public void testFifteenMinuteReset() { for (int j = 0; j < withinBucket; j++) { inc(time + j); } - TimeSeries ts = tsc.timeSeries(time); + TimeSeries ts = timeSeries(time); assertThat(five(time) - ts.fiveMinutes, fiveDelta); assertThat(fifteen(time) - ts.fifteenMinutes, fifteenDelta); assertThat(twentyFour(time) - ts.twentyFourHours, twentyFourDelta); @@ -439,7 +472,7 @@ public void testTwentyFourHourReset() { for (int j = 0; j < withinBucket; j++) { inc(time + j); } - TimeSeries ts = tsc.timeSeries(time); + TimeSeries ts = timeSeries(time); assertThat(twentyFour(time) - ts.twentyFourHours, twentyFourDelta); assertEquals(events.size(), tsc.count()); } @@ -472,8 +505,44 @@ public long countLast(long t, Counter counter, List events) { return count; } - private void inc(long t) { - tsc.inc(t); - events.add(t); + protected void inc(long t) { + timeProvider.inc(t); + } + + protected TimeSeries timeSeries(long t) { + return timeProvider.timeSeries(t); + } + + class TimeProvider implements LongSupplier { + public int i = 0; + public boolean useTimeSeries = false; + public long timeSeriesT = 0; + + public void inc(long t) { + int last = i; + events.add(t); + tsc.inc(); + assert i == last + 1; + } + + public TimeSeries timeSeries(long t) { + int last = i; + useTimeSeries = true; + timeSeriesT = t; + TimeSeries ts = tsc.timeSeries(); + assert i == last; + return ts; + } + + @Override + public long getAsLong() { + if (useTimeSeries) { + useTimeSeries = false; + return timeSeriesT * 1000; + } + long event = events.get(i) * 1000; + i++; + return event; + } } }