Use max_log_size when recombining log entries (#713)

signalfx · Mar 30, 2023 · d29b28e · d29b28e
1 parent cc8b2f9
commit d29b28e
Show file tree

Hide file tree

Showing 5 changed files with 13 additions and 1 deletion.
diff --git a/examples/only-logs-otel/rendered_manifests/configmap-agent.yaml b/examples/only-logs-otel/rendered_manifests/configmap-agent.yaml
@@ -163,6 +163,7 @@ data:
           combine_with: ""
           id: crio-recombine
           is_last_entry: attributes.logtag == 'F'
+          max_log_size: 1048576
           output: handle_empty_log
           source_identifier: attributes["log.file.path"]
           type: recombine
@@ -176,6 +177,7 @@ data:
           combine_with: ""
           id: containerd-recombine
           is_last_entry: attributes.logtag == 'F'
+          max_log_size: 1048576
           output: handle_empty_log
           source_identifier: attributes["log.file.path"]
           type: recombine
@@ -188,6 +190,7 @@ data:
           combine_with: ""
           id: docker-recombine
           is_last_entry: attributes.log endsWith "\n"
+          max_log_size: 1048576
           output: handle_empty_log
           source_identifier: attributes["log.file.path"]
           type: recombine

diff --git a/examples/only-logs-otel/rendered_manifests/daemonset.yaml b/examples/only-logs-otel/rendered_manifests/daemonset.yaml
@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 664949a4d833c1a4b2a30964878c4e57c7029b9af1375b61097ecea125658b19
+        checksum/config: 08d4c631b80ddfd23d85c6b7fbe51d23e7fa1b701107bb41fe8fb8da0b1caeb4
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl
@@ -288,6 +288,7 @@ receivers:
         source_identifier: attributes["log.file.path"]
         is_last_entry: "attributes.logtag == 'F'"
         combine_with: ""
+        max_log_size: {{ $.Values.logsCollection.containers.maxRecombineLogSize }}
       {{- end }}
       {{- if or (not .Values.logsCollection.containers.containerRuntime) (eq .Values.logsCollection.containers.containerRuntime "containerd") }}
       # Parse CRI-Containerd format
@@ -304,6 +305,7 @@ receivers:
         source_identifier: attributes["log.file.path"]
         is_last_entry: "attributes.logtag == 'F'"
         combine_with: ""
+        max_log_size: {{ $.Values.logsCollection.containers.maxRecombineLogSize }}
       {{- end }}
       {{- if or (not .Values.logsCollection.containers.containerRuntime) (eq .Values.logsCollection.containers.containerRuntime "docker") }}
       # Parse Docker format
@@ -319,6 +321,7 @@ receivers:
         source_identifier: attributes["log.file.path"]
         is_last_entry: attributes.log endsWith "\n"
         combine_with: ""
+        max_log_size: {{ $.Values.logsCollection.containers.maxRecombineLogSize }}
       {{- end }}
       - type: add
         id: handle_empty_log

diff --git a/helm-charts/splunk-otel-collector/values.schema.json b/helm-charts/splunk-otel-collector/values.schema.json
@@ -759,6 +759,9 @@
               "items": {
                 "type": "object"
               }
+            },
+            "maxRecombineLogSize": {
+              "type": "integer"
             }
           }
         },

diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml
@@ -566,6 +566,9 @@ logsCollection:
     # Set useSplunkIncludeAnnotation flag to `true` to collect logs from pods with `splunk.com/include: true` annotation and ignore others.
     # All other logs will be ignored.
     useSplunkIncludeAnnotation: false
+    # maxRecombineLogsSize sets the maximum size in bytes of a message recombined from cri-o, containerd and docker log entries.
+    # Set to 0 to remove any size limit.
+    maxRecombineLogSize: 1048576
 
 # Configuration for collecting journald logs using otel collector
   journald: