From d29b28e0ea4e8f878b8fe1d113001d4940a1f77d Mon Sep 17 00:00:00 2001
From: Antoine Toulme <antoine@lunar-ocean.com>
Date: Thu, 30 Mar 2023 08:44:02 -0700
Subject: [PATCH] Use max_log_size when recombining log entries (#713)

---
 .../only-logs-otel/rendered_manifests/configmap-agent.yaml     | 3 +++
 examples/only-logs-otel/rendered_manifests/daemonset.yaml      | 2 +-
 .../splunk-otel-collector/templates/config/_otel-agent.tpl     | 3 +++
 helm-charts/splunk-otel-collector/values.schema.json           | 3 +++
 helm-charts/splunk-otel-collector/values.yaml                  | 3 +++
 5 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/examples/only-logs-otel/rendered_manifests/configmap-agent.yaml b/examples/only-logs-otel/rendered_manifests/configmap-agent.yaml
index 34a7a07060..bcbd4cf97b 100644
--- a/examples/only-logs-otel/rendered_manifests/configmap-agent.yaml
+++ b/examples/only-logs-otel/rendered_manifests/configmap-agent.yaml
@@ -163,6 +163,7 @@ data:
           combine_with: ""
           id: crio-recombine
           is_last_entry: attributes.logtag == 'F'
+          max_log_size: 1048576
           output: handle_empty_log
           source_identifier: attributes["log.file.path"]
           type: recombine
@@ -176,6 +177,7 @@ data:
           combine_with: ""
           id: containerd-recombine
           is_last_entry: attributes.logtag == 'F'
+          max_log_size: 1048576
           output: handle_empty_log
           source_identifier: attributes["log.file.path"]
           type: recombine
@@ -188,6 +190,7 @@ data:
           combine_with: ""
           id: docker-recombine
           is_last_entry: attributes.log endsWith "\n"
+          max_log_size: 1048576
           output: handle_empty_log
           source_identifier: attributes["log.file.path"]
           type: recombine
diff --git a/examples/only-logs-otel/rendered_manifests/daemonset.yaml b/examples/only-logs-otel/rendered_manifests/daemonset.yaml
index c848712459..097e2b883f 100644
--- a/examples/only-logs-otel/rendered_manifests/daemonset.yaml
+++ b/examples/only-logs-otel/rendered_manifests/daemonset.yaml
@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 664949a4d833c1a4b2a30964878c4e57c7029b9af1375b61097ecea125658b19
+        checksum/config: 08d4c631b80ddfd23d85c6b7fbe51d23e7fa1b701107bb41fe8fb8da0b1caeb4
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true
diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl
index 80f174ea62..242ab17de9 100644
--- a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl
+++ b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl
@@ -288,6 +288,7 @@ receivers:
         source_identifier: attributes["log.file.path"]
         is_last_entry: "attributes.logtag == 'F'"
         combine_with: ""
+        max_log_size: {{ $.Values.logsCollection.containers.maxRecombineLogSize }}
       {{- end }}
       {{- if or (not .Values.logsCollection.containers.containerRuntime) (eq .Values.logsCollection.containers.containerRuntime "containerd") }}
       # Parse CRI-Containerd format
@@ -304,6 +305,7 @@ receivers:
         source_identifier: attributes["log.file.path"]
         is_last_entry: "attributes.logtag == 'F'"
         combine_with: ""
+        max_log_size: {{ $.Values.logsCollection.containers.maxRecombineLogSize }}
       {{- end }}
       {{- if or (not .Values.logsCollection.containers.containerRuntime) (eq .Values.logsCollection.containers.containerRuntime "docker") }}
       # Parse Docker format
@@ -319,6 +321,7 @@ receivers:
         source_identifier: attributes["log.file.path"]
         is_last_entry: attributes.log endsWith "\n"
         combine_with: ""
+        max_log_size: {{ $.Values.logsCollection.containers.maxRecombineLogSize }}
       {{- end }}
       - type: add
         id: handle_empty_log
diff --git a/helm-charts/splunk-otel-collector/values.schema.json b/helm-charts/splunk-otel-collector/values.schema.json
index 06aa2fdefb..7410bdc2ae 100644
--- a/helm-charts/splunk-otel-collector/values.schema.json
+++ b/helm-charts/splunk-otel-collector/values.schema.json
@@ -759,6 +759,9 @@
               "items": {
                 "type": "object"
               }
+            },
+            "maxRecombineLogSize": {
+              "type": "integer"
             }
           }
         },
diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml
index 9d50543b39..5e6be4e44e 100644
--- a/helm-charts/splunk-otel-collector/values.yaml
+++ b/helm-charts/splunk-otel-collector/values.yaml
@@ -566,6 +566,9 @@ logsCollection:
     # Set useSplunkIncludeAnnotation flag to `true` to collect logs from pods with `splunk.com/include: true` annotation and ignore others.
     # All other logs will be ignored.
     useSplunkIncludeAnnotation: false
+    # maxRecombineLogsSize sets the maximum size in bytes of a message recombined from cri-o, containerd and docker log entries.
+    # Set to 0 to remove any size limit.
+    maxRecombineLogSize: 1048576
 
 # Configuration for collecting journald logs using otel collector
   journald: