Cherry-pick #5150 to 6.0: Update queue.mem defaults (#5171)

* Update queue.mem defaults (#5150) * Update queue.mem defaults - Enable flush based memory queue by default: Increases chance of pushing bigger batches of events to the outputs. Especially if number of CPUs is limited to 1 or 2 this helps with throughput, as outputs and producers go-routines will interleave less. - Set flush.events=2048 and flush.timeout=1s * set queue flush parameters in system tests update queue flush parameters to min_events of 8 and a timeout of 0.1s. This hopefully stabilizes some time sensitive tests. * Fix typo in CHANGELOG (cherry picked from commit 052172f) * fix changelog
elastic · Sep 13, 2017 · 0c840b3 · 0c840b3
1 parent 14b7711
commit 0c840b3
Show file tree

Hide file tree

Showing 16 changed files with 65 additions and 16 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -64,6 +64,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 
 *Affecting all Beats*
 
+- Enable flush timeout by default. {pull}5150[5150]
 - Add @metadata.version to events send to Logstash. {pull}5166[5166]
 
 *Auditbeat*

diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
@@ -101,11 +101,11 @@ auditbeat.modules:
     # before providing a batch of events to the outputs.
     # A value of 0 (the default) ensures events are immediately available
     # to be sent to the outputs.
-    #flush.min_events: 0
+    #flush.min_events: 2048
 
     # Maximum duration after which events are available to the outputs,
     # if the number of events stored in the queue is < min_flush_events.
-    #flush.timeout: 0s
+    #flush.timeout: 1s
 
 # Sets the maximum number of CPUs that can be executing simultaneously. The
 # default is the number of logical CPUs available in the system.

diff --git a/auditbeat/tests/system/config/auditbeat.yml.j2 b/auditbeat/tests/system/config/auditbeat.yml.j2
@@ -17,6 +17,11 @@ auditbeat.modules:
   {% endif -%}
 {%- endfor %}
 
+queue.mem:
+  events: 4096
+  flush.min_events: 8
+  flush.timeout: 0.1s
+
 output.file:
   path: '{{ output_file_path|default(beat.working_dir + "/output") }}'
   filename: {{ output_file_filename|default("auditbeat") }}
diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -492,11 +492,11 @@ filebeat.prospectors:
     # before providing a batch of events to the outputs.
     # A value of 0 (the default) ensures events are immediately available
     # to be sent to the outputs.
-    #flush.min_events: 0
+    #flush.min_events: 2048
 
     # Maximum duration after which events are available to the outputs,
     # if the number of events stored in the queue is < min_flush_events.
-    #flush.timeout: 0s
+    #flush.timeout: 1s
 
 # Sets the maximum number of CPUs that can be executing simultaneously. The
 # default is the number of logical CPUs available in the system.

diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2
@@ -156,6 +156,13 @@ processors:
 
 {%- endif %}
 
+#================================ Queue =====================================
+
+queue.mem:
+  events: 4096
+  flush.min_events: 8
+  flush.timeout: 0.1s
+
 #================================ Outputs =====================================
 
 # Configure what outputs to use when sending the data collected by the beat.

diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
@@ -250,11 +250,11 @@ heartbeat.scheduler:
     # before providing a batch of events to the outputs.
     # A value of 0 (the default) ensures events are immediately available
     # to be sent to the outputs.
-    #flush.min_events: 0
+    #flush.min_events: 2048
 
     # Maximum duration after which events are available to the outputs,
     # if the number of events stored in the queue is < min_flush_events.
-    #flush.timeout: 0s
+    #flush.timeout: 1s
 
 # Sets the maximum number of CPUs that can be executing simultaneously. The
 # default is the number of logical CPUs available in the system.

diff --git a/heartbeat/tests/system/config/heartbeat.yml.j2 b/heartbeat/tests/system/config/heartbeat.yml.j2
@@ -56,6 +56,15 @@ fields:
   {% endfor -%}
 {% endif %}
 
+#================================ Queue =====================================
+
+queue.mem:
+  events: 4096
+  flush.min_events: 8
+  flush.timeout: 0.1s
+
+#================================ Outputs =====================================
+
 output.file:
   path: {{ output_file_path|default(beat.working_dir + "/output") }}
   filename: "{{ output_file_filename|default("heartbeat") }}"
diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml
@@ -36,11 +36,11 @@
     # before providing a batch of events to the outputs.
     # A value of 0 (the default) ensures events are immediately available
     # to be sent to the outputs.
-    #flush.min_events: 0
+    #flush.min_events: 2048
 
     # Maximum duration after which events are available to the outputs,
     # if the number of events stored in the queue is < min_flush_events.
-    #flush.timeout: 0s
+    #flush.timeout: 1s
 
 # Sets the maximum number of CPUs that can be executing simultaneously. The
 # default is the number of logical CPUs available in the system.

diff --git a/libbeat/publisher/queue/memqueue/config.go b/libbeat/publisher/queue/memqueue/config.go
@@ -13,8 +13,8 @@ type config struct {
 
 var defaultConfig = config{
 	Events:         4 * 1024,
-	FlushMinEvents: 0,
-	FlushTimeout:   0,
+	FlushMinEvents: 2 * 1024,
+	FlushTimeout:   1 * time.Second,
 }
 
 func (c *config) Validate() error {

diff --git a/libbeat/tests/system/config/mockbeat.yml.j2 b/libbeat/tests/system/config/mockbeat.yml.j2
@@ -18,6 +18,12 @@ tags: [
 	{%- endfor -%}
 {%- endif -%}]
 
+#================================ Queue =====================================
+
+queue.mem:
+  events: 4096
+  flush.min_events: 8
+  flush.timeout: 0.1s
 
 ############################# Output ############################################
 

diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
@@ -456,11 +456,11 @@ metricbeat.modules:
     # before providing a batch of events to the outputs.
     # A value of 0 (the default) ensures events are immediately available
     # to be sent to the outputs.
-    #flush.min_events: 0
+    #flush.min_events: 2048
 
     # Maximum duration after which events are available to the outputs,
     # if the number of events stored in the queue is < min_flush_events.
-    #flush.timeout: 0s
+    #flush.timeout: 1s
 
 # Sets the maximum number of CPUs that can be executing simultaneously. The
 # default is the number of logical CPUs available in the system.

diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2
@@ -139,6 +139,13 @@ processors:
 
 {%- endif %}
 
+#================================ Queue =====================================
+
+queue.mem:
+  events: 4096
+  flush.min_events: 8
+  flush.timeout: 0.1s
+
 #================================ Outputs =====================================
 
 # Configure what outputs to use when sending the data collected by the beat.

diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml
@@ -488,11 +488,11 @@ packetbeat.protocols:
     # before providing a batch of events to the outputs.
     # A value of 0 (the default) ensures events are immediately available
     # to be sent to the outputs.
-    #flush.min_events: 0
+    #flush.min_events: 2048
 
     # Maximum duration after which events are available to the outputs,
     # if the number of events stored in the queue is < min_flush_events.
-    #flush.timeout: 0s
+    #flush.timeout: 1s
 
 # Sets the maximum number of CPUs that can be executing simultaneously. The
 # default is the number of logical CPUs available in the system.

diff --git a/packetbeat/tests/system/config/packetbeat.yml.j2 b/packetbeat/tests/system/config/packetbeat.yml.j2
@@ -193,6 +193,13 @@ processors:
 
 {%- endif %}
 
+#================================ Queue =====================================
+
+queue.mem:
+  events: 4096
+  flush.min_events: 8
+  flush.timeout: 0.1s
+
 #================================ Outputs =====================================
 
 # Configure what outputs to use when sending the data collected by the beat.

diff --git a/winlogbeat/tests/system/config/winlogbeat.yml.j2 b/winlogbeat/tests/system/config/winlogbeat.yml.j2
@@ -77,6 +77,13 @@ fields:
   {% endfor -%}
 {% endif %}
 
+#================================ Queue =====================================
+
+queue.mem:
+  events: 4096
+  flush.min_events: 8
+  flush.timeout: 0.1s
+
 #================================ Outputs =====================================
 
 # Configure what outputs to use when sending the data collected by the beat.

diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml
@@ -65,11 +65,11 @@ winlogbeat.event_logs:
     # before providing a batch of events to the outputs.
     # A value of 0 (the default) ensures events are immediately available
     # to be sent to the outputs.
-    #flush.min_events: 0
+    #flush.min_events: 2048
 
     # Maximum duration after which events are available to the outputs,
     # if the number of events stored in the queue is < min_flush_events.
-    #flush.timeout: 0s
+    #flush.timeout: 1s
 
 # Sets the maximum number of CPUs that can be executing simultaneously. The
 # default is the number of logical CPUs available in the system.