Merge pull request #1678 from kube-logging/aslafy-z-patch-2

feat(fluentbit): add storage.delete_irrecoverable_chunks option
kube-logging · Feb 28, 2024 · 7f1e67b · 7f1e67b
2 parents bf88984 + b4bf4da
commit 7f1e67b
Show file tree

Hide file tree

Showing 8 changed files with 24 additions and 0 deletions.
diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_fluentbitagents.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_fluentbitagents.yaml
@@ -436,6 +436,8 @@ spec:
                     type: string
                   storage.checksum:
                     type: string
+                  storage.delete_irrecoverable_chunks:
+                    type: string
                   storage.metrics:
                     type: string
                   storage.path:

diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml
@@ -1453,6 +1453,8 @@ spec:
                         type: string
                       storage.checksum:
                         type: string
+                      storage.delete_irrecoverable_chunks:
+                        type: string
                       storage.metrics:
                         type: string
                       storage.path:
@@ -7183,6 +7185,8 @@ spec:
                               type: string
                             storage.checksum:
                               type: string
+                            storage.delete_irrecoverable_chunks:
+                              type: string
                             storage.metrics:
                               type: string
                             storage.path:

diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_nodeagents.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_nodeagents.yaml
@@ -51,6 +51,8 @@ spec:
                         type: string
                       storage.checksum:
                         type: string
+                      storage.delete_irrecoverable_chunks:
+                        type: string
                       storage.metrics:
                         type: string
                       storage.path:

diff --git a/config/crd/bases/logging.banzaicloud.io_fluentbitagents.yaml b/config/crd/bases/logging.banzaicloud.io_fluentbitagents.yaml
@@ -436,6 +436,8 @@ spec:
                     type: string
                   storage.checksum:
                     type: string
+                  storage.delete_irrecoverable_chunks:
+                    type: string
                   storage.metrics:
                     type: string
                   storage.path:

diff --git a/config/crd/bases/logging.banzaicloud.io_loggings.yaml b/config/crd/bases/logging.banzaicloud.io_loggings.yaml
@@ -1453,6 +1453,8 @@ spec:
                         type: string
                       storage.checksum:
                         type: string
+                      storage.delete_irrecoverable_chunks:
+                        type: string
                       storage.metrics:
                         type: string
                       storage.path:
@@ -7183,6 +7185,8 @@ spec:
                               type: string
                             storage.checksum:
                               type: string
+                            storage.delete_irrecoverable_chunks:
+                              type: string
                             storage.metrics:
                               type: string
                             storage.path:

diff --git a/config/crd/bases/logging.banzaicloud.io_nodeagents.yaml b/config/crd/bases/logging.banzaicloud.io_nodeagents.yaml
@@ -51,6 +51,8 @@ spec:
                         type: string
                       storage.checksum:
                         type: string
+                      storage.delete_irrecoverable_chunks:
+                        type: string
                       storage.metrics:
                         type: string
                       storage.path:

diff --git a/docs/configuration/crds/v1beta1/fluentbit_types.md b/docs/configuration/crds/v1beta1/fluentbit_types.md
@@ -336,6 +336,12 @@ Enable the data integrity check when writing and reading data from the filesyste
 
 Default: Off
 
+### storage.delete_irrecoverable_chunks (string, optional) {#bufferstorage-storage.delete_irrecoverable_chunks}
+
+When enabled, irrecoverable chunks will be deleted during runtime, and any other irrecoverable chunk located in the configured storage path directory will be deleted when Fluent-Bit starts.
+
+Default: Off
+
 ### storage.metrics (string, optional) {#bufferstorage-storage.metrics}
 
 Available in Logging operator version 4.4 and later. If the `http_server` option has been enabled in the main Service configuration section, this option registers a new endpoint where internal metrics of the storage layer can be consumed.

diff --git a/pkg/sdk/logging/api/v1beta1/fluentbit_types.go b/pkg/sdk/logging/api/v1beta1/fluentbit_types.go
@@ -198,6 +198,8 @@ type BufferStorage struct {
 	StorageSync string `json:"storage.sync,omitempty"`
 	// Enable the data integrity check when writing and reading data from the filesystem. The storage layer uses the CRC32 algorithm. (default:Off)
 	StorageChecksum string `json:"storage.checksum,omitempty"`
+	// When enabled, irrecoverable chunks will be deleted during runtime, and any other irrecoverable chunk located in the configured storage path directory will be deleted when Fluent-Bit starts. (default:Off)
+	StorageDeleteIrrecoverableChunks string `json:"storage.delete_irrecoverable_chunks,omitempty"`
 	// If storage.path is set, Fluent Bit will look for data chunks that were not delivered and are still in the storage layer, these are called backlog data. This option configure a hint of maximum value of memory to use when processing these records. (default:5M)
 	StorageBacklogMemLimit string `json:"storage.backlog.mem_limit,omitempty"`
 	// Available in Logging operator version 4.4 and later. If the `http_server` option has been enabled in the main Service configuration section, this option registers a new endpoint where internal metrics of the storage layer can be consumed. (default:Off)