Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Script: Time series compile and cache evict metrics #79078

Merged
Merged
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
a6c7a93
Script: Time series compile and cache evict metrics
stu-elastic Oct 13, 2021
829523e
checkstyle
stu-elastic Oct 13, 2021
610b95d
remove empty if
stu-elastic Oct 13, 2021
c36d060
Collectors import unused
stu-elastic Oct 13, 2021
795baaa
TimeSeriesCounterTest -> TimeSeriesCounterTests
stu-elastic Oct 13, 2021
937037d
correctly name seconds, clamp sub series to latest
stu-elastic Oct 14, 2021
ee1b1a7
Use threadpool for time and time ranges for counters
stu-elastic Oct 18, 2021
ee3e551
Merge branch 'master' into scripting/cache-metrics-collect-pr
elasticmachine Oct 18, 2021
dfe5b4c
Remove spurious newlines
stu-elastic Oct 18, 2021
67d3a43
Update comments
stu-elastic Oct 18, 2021
7c5af25
Test coverage
stu-elastic Oct 18, 2021
3d4d4a5
Rename internal snapshot to timeSuppliedSnapshot to avoid var args co…
stu-elastic Oct 18, 2021
3384062
Constructor test coverage
stu-elastic Oct 18, 2021
ad0e845
Merge branch 'master' into scripting/cache-metrics-collect-pr
elasticmachine Oct 18, 2021
bb6187e
External now, TimeSeries moved up, Remove authority verbage
stu-elastic Oct 18, 2021
7410f9a
update comments
stu-elastic Oct 18, 2021
a18de32
Merge branch 'scripting/cache-metrics-collect-pr' of github.com:stu-e…
stu-elastic Oct 18, 2021
82833ba
adder int instead of LongAdder
stu-elastic Oct 18, 2021
9a12c12
Merge branch 'master' into scripting/cache-metrics-collect-pr
elasticmachine Oct 18, 2021
046a105
fix time series tests
stu-elastic Oct 18, 2021
3117723
Merge branch 'master' into scripting/cache-metrics-collect-pr
elasticmachine Oct 20, 2021
40d2598
Merge branch 'master' of github.com:elastic/elasticsearch into script…
stu-elastic Oct 27, 2021
760b832
Simplify Counters, add example documentation
stu-elastic Oct 27, 2021
3c8ea05
Update merge from master
stu-elastic Oct 27, 2021
d078d19
Revert imports IngestServiceTests
stu-elastic Oct 27, 2021
333dcf3
Merge branch 'master' of github.com:elastic/elasticsearch into script…
stu-elastic Oct 27, 2021
916a4a1
spotless apply
stu-elastic Oct 27, 2021
44ebd20
Merge branch 'master' of github.com:elastic/elasticsearch into script…
stu-elastic Oct 27, 2021
2822f35
Tests and total
stu-elastic Oct 27, 2021
e2f1c6c
Merge branch 'master' of github.com:elastic/elasticsearch into script…
stu-elastic Oct 27, 2021
b2cce9c
Revert total
stu-elastic Oct 27, 2021
d673c20
align time period
stu-elastic Oct 27, 2021
e9740d5
total serialized in ScriptContextStats
stu-elastic Oct 27, 2021
6b3f5d4
diagram tweaks
stu-elastic Oct 27, 2021
8b8c3ae
Merge branch 'master' of github.com:elastic/elasticsearch into script…
stu-elastic Nov 2, 2021
63a23dc
time moved to TSC, add comments, docs
stu-elastic Nov 2, 2021
19a7516
Merge branch 'master' of github.com:elastic/elasticsearch into script…
stu-elastic Nov 3, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge branch 'master' of github.com:elastic/elasticsearch into script…
…ing/cache-metrics-collect-pr
  • Loading branch information
stu-elastic committed Oct 27, 2021
commit 333dcf350a45373765f5bd4ab8b9bd10f469dbbc
Original file line number Diff line number Diff line change
@@ -48,7 +48,7 @@ public boolean execute(Token token) {
};

@SuppressWarnings("unchecked")
ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L){
ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L) {
@Override
public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryType> context) {
assertEquals(context, AnalysisPredicateScript.CONTEXT);
Original file line number Diff line number Diff line change
@@ -48,7 +48,7 @@ public boolean execute(Token token) {
};

@SuppressWarnings("unchecked")
ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L){
ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L) {
@Override
public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryType> context) {
assertEquals(context, AnalysisPredicateScript.CONTEXT);
Original file line number Diff line number Diff line change
@@ -138,15 +138,15 @@ public void testInlineIsCompiled() throws Exception {
ScriptService scriptService = new ScriptService(
Settings.builder().build(),
Collections.singletonMap(
Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(
Script.DEFAULT_SCRIPT_LANG,
Collections.singletonMap(scriptName, ctx -> {
ctx.put("foo", "bar");
return null;
}),
Collections.emptyMap()
)
), new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L);
Script.DEFAULT_SCRIPT_LANG,
new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> {
ctx.put("foo", "bar");
return null;
}), Collections.emptyMap())
),
new HashMap<>(ScriptModule.CORE_CONTEXTS),
() -> 1L
);
factory = new ScriptProcessor.Factory(scriptService);

Map<String, Object> configMap = new HashMap<>();
Original file line number Diff line number Diff line change
@@ -714,8 +714,13 @@ CacheHolder generalCacheHolder(Settings settings) {
if (SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING.get(settings) || compilationLimitsEnabled() == false) {
rate = SCRIPT_COMPILATION_RATE_ZERO;
}
return new CacheHolder(SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(settings), SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(settings), rate,
SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), timeProvider);
return new CacheHolder(
SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(settings),
SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(settings),
rate,
SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(),
timeProvider
);
}

CacheHolder contextCacheHolder(Settings settings) {
Original file line number Diff line number Diff line change
@@ -140,21 +140,18 @@ public void testActsOnImmutableData() throws Exception {

public void testTypeDeprecation() throws Exception {

ScriptService scriptService = new ScriptService(Settings.builder().build(),
Collections.singletonMap(
Script.DEFAULT_SCRIPT_LANG,
new MockScriptEngine(
Script.DEFAULT_SCRIPT_LANG,
Collections.singletonMap(
scriptName, ctx -> {
ctx.get("_type");
return true;
}
),
Collections.emptyMap()
)
),
new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L);
ScriptService scriptService = new ScriptService(
Settings.builder().build(),
Collections.singletonMap(
Script.DEFAULT_SCRIPT_LANG,
new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> {
ctx.get("_type");
return true;
}), Collections.emptyMap())
),
new HashMap<>(ScriptModule.CORE_CONTEXTS),
() -> 1L
);

LongSupplier relativeTimeProvider = mock(LongSupplier.class);
when(relativeTimeProvider.getAsLong()).thenReturn(0L, TimeUnit.MILLISECONDS.toNanos(1), 0L, TimeUnit.MILLISECONDS.toNanos(2));
Original file line number Diff line number Diff line change
@@ -159,9 +159,15 @@ public void testActualCompoundProcessorWithOnFailure() throws Exception {

public void testActualCompoundProcessorWithOnFailureAndTrueCondition() throws Exception {
String scriptName = "conditionalScript";
ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG,
new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> true), Collections.emptyMap())),
new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L);
ScriptService scriptService = new ScriptService(
Settings.builder().build(),
Collections.singletonMap(
Script.DEFAULT_SCRIPT_LANG,
new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> true), Collections.emptyMap())
),
new HashMap<>(ScriptModule.CORE_CONTEXTS),
() -> 1L
);
RuntimeException exception = new RuntimeException("fail");
TestProcessor failProcessor = new TestProcessor("fail", "test", null, exception);
ConditionalProcessor conditionalProcessor = new ConditionalProcessor(
@@ -242,9 +248,15 @@ public void testActualCompoundProcessorWithFalseConditional() throws Exception {
String key3 = randomAlphaOfLength(10);

String scriptName = "conditionalScript";
ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG,
new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> false), Collections.emptyMap())),
new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L);
ScriptService scriptService = new ScriptService(
Settings.builder().build(),
Collections.singletonMap(
Script.DEFAULT_SCRIPT_LANG,
new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> false), Collections.emptyMap())
),
new HashMap<>(ScriptModule.CORE_CONTEXTS),
() -> 1L
);

CompoundProcessor compoundProcessor = new CompoundProcessor(
new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key1, randomInt()); }),
@@ -364,9 +376,15 @@ public void testActualPipelineProcessorWithTrueConditional() throws Exception {

String scriptName = "conditionalScript";

ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG,
new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> true), Collections.emptyMap())),
new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L);
ScriptService scriptService = new ScriptService(
Settings.builder().build(),
Collections.singletonMap(
Script.DEFAULT_SCRIPT_LANG,
new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> true), Collections.emptyMap())
),
new HashMap<>(ScriptModule.CORE_CONTEXTS),
() -> 1L
);

Pipeline pipeline1 = new Pipeline(
pipelineId1,
@@ -458,9 +476,15 @@ public void testActualPipelineProcessorWithFalseConditional() throws Exception {

String scriptName = "conditionalScript";

ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG,
new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> false), Collections.emptyMap())),
new HashMap<>(ScriptModule.CORE_CONTEXTS), () -> 1L);
ScriptService scriptService = new ScriptService(
Settings.builder().build(),
Collections.singletonMap(
Script.DEFAULT_SCRIPT_LANG,
new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> false), Collections.emptyMap())
),
new HashMap<>(ScriptModule.CORE_CONTEXTS),
() -> 1L
);

Pipeline pipeline1 = new Pipeline(
pipelineId1,
Original file line number Diff line number Diff line change
@@ -31,8 +31,13 @@ public void testCompilationCircuitBreaking() throws Exception {
ScriptCache.CompilationRate rate = ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context)
.get(Settings.EMPTY);
String rateSettingName = rateSetting.getKey();
ScriptCache cache = new ScriptCache(size, expire,
new ScriptCache.CompilationRate(1, TimeValue.timeValueMinutes(1)), rateSettingName, time);
ScriptCache cache = new ScriptCache(
size,
expire,
new ScriptCache.CompilationRate(1, TimeValue.timeValueMinutes(1)),
rateSettingName,
() -> 1L
);
cache.checkCompilationLimit(); // should pass
expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), rateSettingName,
@@ -49,9 +54,13 @@ public void testCompilationCircuitBreaking() throws Exception {
cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), rateSettingName,
time);
expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
cache = new ScriptCache(size, expire,
new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)), rateSettingName,
time);
cache = new ScriptCache(
size,
expire,
new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)),
rateSettingName,
() -> 1L
);
int largeLimit = randomIntBetween(1000, 10000);
for (int i = 0; i < largeLimit; i++) {
cache.checkCompilationLimit();
@@ -81,9 +90,13 @@ public void testGeneralCompilationCircuitBreaking() throws Exception {
cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), settingName,
() -> 1L);
expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
cache = new ScriptCache(size, expire,
new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)), settingName,
() -> 1L);
cache = new ScriptCache(
size,
expire,
new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)),
settingName,
() -> 1L
);
int largeLimit = randomIntBetween(1000, 10000);
for (int i = 0; i < largeLimit; i++) {
cache.checkCompilationLimit();
45 changes: 27 additions & 18 deletions server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java
Original file line number Diff line number Diff line change
@@ -28,9 +28,13 @@
public class ScriptStatsTests extends ESTestCase {
public void testXContent() throws IOException {
List<ScriptContextStats> contextStats = List.of(
new ScriptContextStats("contextB", 100, 201, 302,
new TimeSeries(1000, 1001, 1002),
new TimeSeries(2000, 2001, 2002)
new ScriptContextStats(
"contextB",
100,
201,
302,
new TimeSeries(1000, 1001, 1002),
new TimeSeries(2000, 2001, 2002)
),
new ScriptContextStats("contextA", 1000, 2010, 3020, null, new TimeSeries(0, 0, 0))
);
@@ -92,22 +96,27 @@ public void testSerializeEmptyTimeSeries() throws IOException {
}

public void testSerializeTimeSeries() throws IOException {
Function<TimeSeries, ScriptContextStats> mkContextStats =
(ts) -> new ScriptContextStats("c", 1111, 2222, 3333, null, ts);
Function<TimeSeries, ScriptContextStats> mkContextStats = (ts) -> new ScriptContextStats(
"c",
1111,
2222,
3333,
null,
ts
);

TimeSeries series = new TimeSeries(0, 0, 5);
String format =
"{\n" +
" \"context\" : \"c\",\n" +
" \"compilations\" : 1111,\n" +
" \"cache_evictions\" : 2222,\n" +
" \"cache_evictions_history\" : {\n" +
" \"5m\" : %d,\n" +
" \"15m\" : %d,\n" +
" \"24h\" : %d\n" +
" },\n" +
" \"compilation_limit_triggered\" : 3333\n" +
"}";
String format = "{\n"
+ " \"context\" : \"c\",\n"
+ " \"compilations\" : 1111,\n"
+ " \"cache_evictions\" : 2222,\n"
+ " \"cache_evictions_history\" : {\n"
+ " \"5m\" : %d,\n"
+ " \"15m\" : %d,\n"
+ " \"24h\" : %d\n"
+ " },\n"
+ " \"compilation_limit_triggered\" : 3333\n"
+ "}";

XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
mkContextStats.apply(series).toXContent(builder, ToXContent.EMPTY_PARAMS);
@@ -163,7 +172,7 @@ public ScriptContextStats serDeser(Version outVersion, Version inVersion, Script
}

public ScriptContextStats randomStats() {
long[] histStats = {randomLongBetween(0, 2048), randomLongBetween(0, 2048)};
long[] histStats = { randomLongBetween(0, 2048), randomLongBetween(0, 2048) };
List<TimeSeries> timeSeries = new ArrayList<>();
for (int j = 0; j < 2; j++) {
if (randomBoolean() && histStats[j] > 0) {
Loading
You are viewing a condensed version of this merge commit. You can view the full changes here.