From 388e1141bc63c51321f66636964bdb4c76c314b9 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 13 Jul 2020 14:30:44 -0600 Subject: [PATCH 1/4] Increase default write queue size This commit increases the default write queue size to 4000. This is to allow a greater number of pending indexing requests. This work is safe as we have added additional memory limits. --- docs/reference/release-notes/7.9.asciidoc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/reference/release-notes/7.9.asciidoc b/docs/reference/release-notes/7.9.asciidoc index 90722089440b7..38a52dd0fa158 100644 --- a/docs/reference/release-notes/7.9.asciidoc +++ b/docs/reference/release-notes/7.9.asciidoc @@ -22,3 +22,10 @@ Snapshot restore throttling:: default that's used for `indices.recovery.max_bytes_per_sec`. This means that no behavioral change will be observed by clusters where the recovery and restore settings had not been adapted from the defaults. {es-pull}58658[#58658] + +Thread pool write queue size:: +* The WRITE thread pool default queue size (`thread_pool.write.size`) has been + increased from 200 to 4000. Additional memory-oriented back pressure has been + introduced with the `indexing_limits.memory.limit` setting. This setting + configures a limit to the number of bytes allowed to be consumed by outstanding + indexing requests. {es-issue}59263[#59263] From 5442d2c1a558c19a676408b67e8088705037d211 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 14 Jul 2020 09:18:13 -0600 Subject: [PATCH 2/4] Changes --- docs/reference/release-notes/7.9.asciidoc | 2 +- .../src/main/java/org/elasticsearch/threadpool/ThreadPool.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/release-notes/7.9.asciidoc b/docs/reference/release-notes/7.9.asciidoc index 38a52dd0fa158..49348b5244970 100644 --- a/docs/reference/release-notes/7.9.asciidoc +++ b/docs/reference/release-notes/7.9.asciidoc @@ -26,6 +26,6 @@ Snapshot restore throttling:: Thread pool write queue size:: * The WRITE thread pool default queue size (`thread_pool.write.size`) has been increased from 200 to 4000. Additional memory-oriented back pressure has been - introduced with the `indexing_limits.memory.limit` setting. This setting + introduced with the `indexing_pressure.memory.limit` setting. This setting configures a limit to the number of bytes allowed to be consumed by outstanding indexing requests. {es-issue}59263[#59263] diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index 8bd44128a2eaf..6de0c553ceb45 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -172,7 +172,7 @@ public ThreadPool(final Settings settings, final ExecutorBuilder... customBui final int halfProcMaxAt10 = halfAllocatedProcessorsMaxTen(allocatedProcessors); final int genericThreadPoolMax = boundedBy(4 * allocatedProcessors, 128, 512); builders.put(Names.GENERIC, new ScalingExecutorBuilder(Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30))); - builders.put(Names.WRITE, new FixedExecutorBuilder(settings, Names.WRITE, allocatedProcessors, 200)); + builders.put(Names.WRITE, new FixedExecutorBuilder(settings, Names.WRITE, allocatedProcessors, 4000)); builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, allocatedProcessors, 1000)); builders.put(Names.ANALYZE, new FixedExecutorBuilder(settings, Names.ANALYZE, 1, 16)); builders.put(Names.SEARCH, new AutoQueueAdjustingExecutorBuilder(settings, From 11fa21497d40ae5b01a357019883452320c5308e Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 14 Jul 2020 09:37:06 -0600 Subject: [PATCH 3/4] 10000 --- docs/reference/release-notes/7.9.asciidoc | 2 +- .../src/main/java/org/elasticsearch/threadpool/ThreadPool.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/release-notes/7.9.asciidoc b/docs/reference/release-notes/7.9.asciidoc index 49348b5244970..c8946b2c07ad2 100644 --- a/docs/reference/release-notes/7.9.asciidoc +++ b/docs/reference/release-notes/7.9.asciidoc @@ -25,7 +25,7 @@ Snapshot restore throttling:: Thread pool write queue size:: * The WRITE thread pool default queue size (`thread_pool.write.size`) has been - increased from 200 to 4000. Additional memory-oriented back pressure has been + increased from 200 to 10000. Additional memory-oriented back pressure has been introduced with the `indexing_pressure.memory.limit` setting. This setting configures a limit to the number of bytes allowed to be consumed by outstanding indexing requests. {es-issue}59263[#59263] diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index 6de0c553ceb45..411bd9de9e8b5 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -172,7 +172,7 @@ public ThreadPool(final Settings settings, final ExecutorBuilder... customBui final int halfProcMaxAt10 = halfAllocatedProcessorsMaxTen(allocatedProcessors); final int genericThreadPoolMax = boundedBy(4 * allocatedProcessors, 128, 512); builders.put(Names.GENERIC, new ScalingExecutorBuilder(Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30))); - builders.put(Names.WRITE, new FixedExecutorBuilder(settings, Names.WRITE, allocatedProcessors, 4000)); + builders.put(Names.WRITE, new FixedExecutorBuilder(settings, Names.WRITE, allocatedProcessors, 10000)); builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, allocatedProcessors, 1000)); builders.put(Names.ANALYZE, new FixedExecutorBuilder(settings, Names.ANALYZE, 1, 16)); builders.put(Names.SEARCH, new AutoQueueAdjustingExecutorBuilder(settings, From 7adc265d064754bd24e718bb97279c2739280b53 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 14 Jul 2020 09:45:35 -0600 Subject: [PATCH 4/4] Coments --- docs/reference/release-notes/7.9.asciidoc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/reference/release-notes/7.9.asciidoc b/docs/reference/release-notes/7.9.asciidoc index c8946b2c07ad2..d29cd748f8a39 100644 --- a/docs/reference/release-notes/7.9.asciidoc +++ b/docs/reference/release-notes/7.9.asciidoc @@ -25,7 +25,9 @@ Snapshot restore throttling:: Thread pool write queue size:: * The WRITE thread pool default queue size (`thread_pool.write.size`) has been - increased from 200 to 10000. Additional memory-oriented back pressure has been - introduced with the `indexing_pressure.memory.limit` setting. This setting - configures a limit to the number of bytes allowed to be consumed by outstanding - indexing requests. {es-issue}59263[#59263] + increased from 200 to 10000. A small queue size (200) caused issues when users + wanted to send small indexing requests with a high client count. Additional + memory-oriented back pressure has been introduced with the + `indexing_pressure.memory.limit` setting. This setting configures a limit to + the number of bytes allowed to be consumed by outstanding indexing requests. + {es-issue}59263[#59263]