Turned off data integrity check (#633)

* Turned off data integrity check when writing and reading logs from the filesystem for FluentBit * Replaced outdated comment * Addressed PR comments * Fixed typo in comment Co-authored-by: Ling Huang <[email protected]>
GoogleCloudPlatform · Jun 13, 2022 · 312285a · 312285a
1 parent 7d8d128
commit 312285a
Show file tree

Hide file tree

Showing 135 changed files with 137 additions and 136 deletions.
diff --git a/confgenerator/fluentbit/service.go b/confgenerator/fluentbit/service.go
@@ -36,8 +36,9 @@ func (s Service) Component() Component {
 			// https://docs.fluentbit.io/manual/administration/buffering-and-storage#service-section-configuration
 			// storage.path is set by Fluent Bit systemd unit (e.g. /var/lib/google-cloud-ops-agent/fluent-bit/buffers).
 			"storage.sync": "normal",
-			// Enable the data integrity check when writing and reading data from the filesystem.
-			"storage.checksum": "on",
+			// Disable data integrity check to avoid spurious corruption errors due to empty spaces.
+			// Most users do not enforce checksum.
+			"storage.checksum": "off",
 			// The maximum amount of data to load into the memory when processing old chunks from the backlog that is from previous Fluent Bit processes (e.g. Fluent Bit may have crashed or restarted).
 			"storage.backlog.mem_limit": "50M",
 			// Enable storage metrics in the built-in HTTP server.

diff --git a/...a/valid/linux/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf b/...a/valid/linux/all-backward_compatible_with_explicit_exporters/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/all-built_in_config/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/all-built_in_config/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/all-user_config_file_deleted/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/all-user_config_file_deleted/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-custom_log_level/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-custom_log_level/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 warn
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...or/testdata/valid/linux/logging-default_overrides_disable_all/golden_fluent_bit_main.conf b/...or/testdata/valid/linux/logging-default_overrides_disable_all/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...ator/testdata/valid/linux/logging-pipeline_multiple_pipelines/golden_fluent_bit_main.conf b/...ator/testdata/valid/linux/logging-pipeline_multiple_pipelines/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...generator/testdata/valid/linux/logging-processor_exclude_logs/golden_fluent_bit_main.conf b/...generator/testdata/valid/linux/logging-processor_exclude_logs/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...enerator/testdata/valid/linux/logging-processor_modify_fields/golden_fluent_bit_main.conf b/...enerator/testdata/valid/linux/logging-processor_modify_fields/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-processor_order/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-processor_order/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...alid/linux/logging-processor_parse_json_and_parse_regex_types/golden_fluent_bit_main.conf b/...alid/linux/logging-processor_parse_json_and_parse_regex_types/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...erator/testdata/valid/linux/logging-processor_parse_multiline/golden_fluent_bit_main.conf b/...erator/testdata/valid/linux/logging-processor_parse_multiline/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/.../linux/logging-processor_parse_regex_type_on_default_pipeline/golden_fluent_bit_main.conf b/.../linux/logging-processor_parse_regex_type_on_default_pipeline/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_apache/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_apache/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...generator/testdata/valid/linux/logging-receiver_apache_custom/golden_fluent_bit_main.conf b/...generator/testdata/valid/linux/logging-receiver_apache_custom/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_cassandra/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_cassandra/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...erator/testdata/valid/linux/logging-receiver_cassandra_custom/golden_fluent_bit_main.conf b/...erator/testdata/valid/linux/logging-receiver_cassandra_custom/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_couchdb/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_couchdb/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...generator/testdata/valid/linux/logging-receiver_elasticsearch/golden_fluent_bit_main.conf b/...generator/testdata/valid/linux/logging-receiver_elasticsearch/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...or/testdata/valid/linux/logging-receiver_elasticsearch_custom/golden_fluent_bit_main.conf b/...or/testdata/valid/linux/logging-receiver_elasticsearch_custom/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/.../testdata/valid/linux/logging-receiver_files_refresh_interval/golden_fluent_bit_main.conf b/.../testdata/valid/linux/logging-receiver_files_refresh_interval/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...ta/valid/linux/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf b/...ta/valid/linux/logging-receiver_files_type_multiple_receivers/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_forward/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_forward/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...ux/logging-receiver_forward_multiple_receivers_conflicting_id/golden_fluent_bit_main.conf b/...ux/logging-receiver_forward_multiple_receivers_conflicting_id/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...id/linux/logging-receiver_forward_multiple_receivers_with_dot/golden_fluent_bit_main.conf b/...id/linux/logging-receiver_forward_multiple_receivers_with_dot/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...d/linux/logging-receiver_forward_omitting_optional_parameters/golden_fluent_bit_main.conf b/...d/linux/logging-receiver_forward_omitting_optional_parameters/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_hadoop/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_hadoop/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...testdata/valid/linux/logging-receiver_hadoop_refresh_interval/golden_fluent_bit_main.conf b/...testdata/valid/linux/logging-receiver_hadoop_refresh_interval/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_hbase/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_hbase/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_jetty/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_jetty/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_kafka/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_kafka/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_kafka_custom/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_kafka_custom/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_mongodb/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_mongodb/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_mysql/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_mysql/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_mysql_custom/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_mysql_custom/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_nginx/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_nginx/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_nginx_custom/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_nginx_custom/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_postgresql/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_postgresql/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...rator/testdata/valid/linux/logging-receiver_postgresql_custom/golden_fluent_bit_main.conf b/...rator/testdata/valid/linux/logging-receiver_postgresql_custom/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_rabbitmq/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_rabbitmq/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_redis/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_redis/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_redis_custom/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_redis_custom/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/confgenerator/testdata/valid/linux/logging-receiver_solr/golden_fluent_bit_main.conf b/confgenerator/testdata/valid/linux/logging-receiver_solr/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal

diff --git a/...a/valid/linux/logging-receiver_syslog_type_multiple_receivers/golden_fluent_bit_main.conf b/...a/valid/linux/logging-receiver_syslog_type_multiple_receivers/golden_fluent_bit_main.conf
@@ -7,7 +7,7 @@
     Log_Level                 info
     dns.resolver              legacy
     storage.backlog.mem_limit 50M
-    storage.checksum          on
+    storage.checksum          off
     storage.max_chunks_up     128
     storage.metrics           on
     storage.sync              normal