From f6f3b87efe67b5f2a5004b928a1a2f358a76fac7 Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Tue, 9 Mar 2021 13:01:45 -0800
Subject: [PATCH 01/24] initial test

---
 .../templates/_config.tpl                     | 79 +++++++++++++++++++
 charts/opentelemetry-collector/values.yaml    | 18 ++++-
 2 files changed, 96 insertions(+), 1 deletion(-)

diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index 7937f4b64..0f128c19a 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -32,6 +32,7 @@ Build config file for agent OpenTelemetry Collector
 {{- $values := deepCopy .Values.agentCollector | mustMergeOverwrite (deepCopy .Values)  }}
 {{- $data := dict "Values" $values | mustMergeOverwrite (deepCopy .) }}
 {{- $config := include "opentelemetry-collector.baseConfig" $data | fromYaml }}
+{{- $config := include "opentelemetry-collector.agent.containerLogsConfig" $data | fromYaml | mustMergeOverwrite $config }}
 {{- $config := include "opentelemetry-collector.agentConfigOverride" $data | fromYaml | mustMergeOverwrite $config }}
 {{- .Values.agentCollector.configOverride | mustMergeOverwrite $config | toYaml }}
 {{- end }}
@@ -121,3 +122,81 @@ service:
       exporters: [otlp]
 {{- end }}
 {{- end }}
+
+{{- define "opentelemetry-collector.agent.containerLogsConfig" -}}
+{{- if .Values.agentCollector.containerLogs.enabled }}
+receivers:
+  filelog:
+    include: [ /var/log/pods/*/*/*.log ]
+    start_at: beginning
+    include_file_path: true
+    include_file_name: false
+    operators:
+      # # Find out which format is used by kubernetes
+      # - type: router
+      #   id: get-format
+      #   routes:
+      #     - output: parser-docker
+      #       expr: '$$record matches "^\\{"'
+      #     - output: parser-crio
+      #       expr: '$$record matches "^[^ Z]+ "'
+      #     - output: parser-containerd
+      #       expr: '$$record matches "^[^ Z]+Z"'
+      # # Parse CRI-O format
+      # - type: regex_parser
+      #   id: parser-crio
+      #   regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
+      #   output: extract_metadata_from_filepath
+      #   timestamp:
+      #     parse_from: time
+      #     layout_type: gotime
+      #     layout: '2006-01-02T15:04:05.000000000-07:00'
+      # Parse CRI-Containerd format
+      - type: regex_parser
+        id: parser-containerd
+        regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: time
+          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+      # # Parse Docker format
+      # - type: json_parser
+      #   id: parser-docker
+      #   output: extract_metadata_from_filepath
+      #   timestamp:
+      #     parse_from: time
+      #     layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+      # Extract metadata from file path
+      - type: regex_parser
+        id: extract_metadata_from_filepath
+        regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{36})\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$'
+        parse_from: $$labels.file_path
+      # Move out attributes to Attributes
+      - type: metadata
+        labels:
+          stream: 'EXPR($.stream)'
+          k8s.container.name: 'EXPR($.container_name)'
+          k8s.namespace.name: 'EXPR($.namespace)'
+          k8s.pod.name: 'EXPR($.pod_name)'
+          run_id: 'EXPR($.run_id)'
+          k8s.pod.uid: 'EXPR($.uid)'
+      # Clean up log record
+      - type: restructure
+        id: clean-up-log-record
+        ops:
+          - remove: logtag
+          - remove: stream
+          - remove: container_name
+          - remove: namespace
+          - remove: pod_name
+          - remove: run_id
+          - remove: uid
+service:
+  pipelines:
+    logs:
+      receivers:
+        - filelog
+      exporters:
+        - logging
+{{- end }}
+{{- end }}
diff --git a/charts/opentelemetry-collector/values.yaml b/charts/opentelemetry-collector/values.yaml
index 8b487087c..1326d59bd 100644
--- a/charts/opentelemetry-collector/values.yaml
+++ b/charts/opentelemetry-collector/values.yaml
@@ -39,6 +39,11 @@ config:
     extensions:
       - health_check
     pipelines:
+      logs:
+        exporters:
+          - logging
+        receivers:
+          - filelog
       metrics:
         exporters:
           - logging
@@ -93,7 +98,15 @@ tolerations: []
 affinity: {}
 
 extraEnvs: []
-extraHostPathMounts: []
+extraHostPathMounts:
+  - name: varlog
+    hostPath: /var/log
+    mountPath: /var/log
+    readOnly: true
+  - name: varlibdockercontainers
+    hostPath: /var/lib/docker/containers
+    mountPath: /var/lib/docker/containers
+    readOnly: true
 
 # Configuration for ports, shared between agentCollector, standaloneCollector and service.
 # Can be overridden here or for agentCollector and standaloneCollector independently.
@@ -127,6 +140,9 @@ ports:
 agentCollector:
   enabled: true
 
+  containerLogs:
+    enabled: true
+
   resources:
     limits:
       cpu: 256m

From 2eadd83a15249eaad8a01facabe48c2a77aeaae8 Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Tue, 9 Mar 2021 17:13:05 -0800
Subject: [PATCH 02/24] test values

---
 .../log_test_values.yaml                      | 246 ++++++++++++++++++
 1 file changed, 246 insertions(+)
 create mode 100644 charts/opentelemetry-collector/log_test_values.yaml

diff --git a/charts/opentelemetry-collector/log_test_values.yaml b/charts/opentelemetry-collector/log_test_values.yaml
new file mode 100644
index 000000000..10d811fd1
--- /dev/null
+++ b/charts/opentelemetry-collector/log_test_values.yaml
@@ -0,0 +1,246 @@
+# Default values for opentelemetry-collector.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+nameOverride: ""
+fullnameOverride: ""
+
+config:
+  exporters:
+    logging: {}
+  extensions:
+    health_check: {}
+  processors:
+    batch: {}
+    # If set to null, will be overridden with values based on k8s resource limits
+    memory_limiter: null
+  receivers:
+    jaeger:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:14250
+        thrift_http:
+          endpoint: 0.0.0.0:14268
+    #otlp:
+    #  protocols:
+    #    grpc: null
+    #    http: null
+    prometheus:
+      config:
+        scrape_configs:
+          - job_name: opentelemetry-collector
+            scrape_interval: 10s
+            static_configs:
+              - targets:
+                  - ${MY_POD_IP}:8888
+    zipkin:
+      endpoint: 0.0.0.0:9411
+  service:
+    extensions:
+      - health_check
+    pipelines:
+      logs:
+        exporters:
+          - logging
+        receivers:
+          - filelog
+      metrics:
+        exporters:
+          - logging
+        processors:
+          - memory_limiter
+          - batch
+        receivers:
+          #- otlp
+          - prometheus
+      traces:
+        exporters:
+          - logging
+        processors:
+          - memory_limiter
+          - batch
+        receivers:
+          #- otlp
+          - jaeger
+          - zipkin
+
+# Shared params for agentCollector daemonset and standaloneCollector deployment pods.
+# Can be overridden here or for any component independently using the same keys.
+
+image:
+  repository: sumodrosiek/otelcontribcol
+  # repository: otel/opentelemetry-collector-contrib
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: k8s-1613747494
+imagePullSecrets: []
+
+# OpenTelemetry Collector executable
+command:
+  #name: otelcol
+  name: otelcontribcol
+  extraArgs: []
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+podSecurityContext: {}
+securityContext: {}
+
+nodeSelector: {}
+tolerations: []
+affinity: {}
+
+extraEnvs: []
+extraHostPathMounts:
+  - name: varlog
+    hostPath: /var/log
+    mountPath: /var/log
+    readOnly: true
+  - name: varlibdockercontainers
+    hostPath: /var/lib/docker/containers
+    mountPath: /var/lib/docker/containers
+    readOnly: true
+
+# Configuration for ports, shared between agentCollector, standaloneCollector and service.
+# Can be overridden here or for agentCollector and standaloneCollector independently.
+ports:
+  otlp:
+    enabled: false
+    containerPort: 4317
+    servicePort: 4317
+    hostPort: 4317
+    protocol: TCP
+  jaeger-thrift:
+    enabled: false
+    containerPort: 14268
+    servicePort: 14268
+    hostPort: 14268
+    protocol: TCP
+  jaeger-grpc:
+    enabled: false
+    containerPort: 14250
+    servicePort: 14250
+    hostPort: 14250
+    protocol: TCP
+  zipkin:
+    enabled: false
+    containerPort: 9411
+    servicePort: 9411
+    hostPort: 9411
+    protocol: TCP
+
+# Configuration for agent OpenTelemetry Collector daemonset, enabled by default
+agentCollector:
+  enabled: true
+
+  containerLogs:
+    enabled: true
+
+  resources:
+    limits:
+      cpu: 256m
+      memory: 512Mi
+
+  podAnnotations: {}
+
+  # Configuration override that will be merged into the agent's default config
+  configOverride: {}
+  # The following config override can be used to enable host receiver
+  #   receivers:
+  #     hostmetrics:
+  #       scrapers:
+  #         cpu:
+  #         disk:
+  #         filesystem:
+  #   service:
+  #     pipelines:
+  #       metrics:
+  #         receivers: [prometheus, hostmetrics]
+
+  # Any of the top level options can be overridden specifically for agent
+  # image: {}
+  # imagePullSecrets: []
+  # command: {}
+  # serviceAccount: {}
+  # podSecurityContext: {}
+  # securityContext: {}
+  # nodeSelector: {}
+  # tolerations: []
+  # affinity: {}
+
+  # The following option overrides can be used with host receiver
+  # extraEnvs:
+  # - name: HOST_PROC
+  #   value: /hostfs/proc
+  # - name: HOST_SYS
+  #   value: /hostfs/sys
+  # - name: HOST_ETC
+  #   value: /hostfs/etc
+  # - name: HOST_VAR
+  #   value: /hostfs/var
+  # - name: HOST_RUN
+  #   value: /hostfs/run
+  # - name: HOST_DEV
+  #   value: /hostfs/dev
+  # extraHostPathMounts:
+  # - name: hostfs
+  #   hostPath: /
+  #   mountPath: /hostfs
+  #   readOnly: true
+  #   mountPropagation: HostToContainer
+
+  # The following example disables all jaeger ports and zipkin hostPort on the agent
+  # ports:
+  #   jaeger-thrift:
+  #     enabled: false
+  #   jaeger-grpc:
+  #     enabled: false
+  #   zipkin:
+  #     hostPort: ""
+
+# Configuration for standalone OpenTelemetry Collector deployment, disabled by default
+standaloneCollector:
+  enabled: false
+
+  replicaCount: 1
+
+  resources:
+    limits:
+      cpu: 1
+      memory: 2Gi
+
+  podAnnotations: {}
+
+  # Configuration override that will be merged into the standalone collector default config
+  configOverride: {}
+
+  # Any of the top level options can be overridden specifically for standalone collector
+  # image: {}
+  # imagePullSecrets: []
+  # command: {}
+  # serviceAccount: {}
+  # podSecurityContext: {}
+  # securityContext: {}
+  # nodeSelector: {}
+  # tolerations: []
+  # affinity: {}
+  # ports: {}
+
+service:
+  type: ClusterIP
+  annotations: {}
+
+# autoscaling is used only if standaloneCollector enabled
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 10
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80

From 21461dfc4713d4ec46f655e0167b13a58be7e622 Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Wed, 10 Mar 2021 09:08:39 -0800
Subject: [PATCH 03/24] enrichment

---
 .../log_test_values.yaml                      |  2 +
 .../templates/_config.tpl                     | 60 +++++++++++--------
 charts/opentelemetry-collector/values.yaml    |  9 ++-
 3 files changed, 44 insertions(+), 27 deletions(-)

diff --git a/charts/opentelemetry-collector/log_test_values.yaml b/charts/opentelemetry-collector/log_test_values.yaml
index 10d811fd1..57a092560 100644
--- a/charts/opentelemetry-collector/log_test_values.yaml
+++ b/charts/opentelemetry-collector/log_test_values.yaml
@@ -142,6 +142,8 @@ agentCollector:
 
   containerLogs:
     enabled: true
+    # Container runtime. One of `docker`, `cri-o`, or `containerd`
+    containerRunTime: docker
 
   resources:
     limits:
diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index 0f128c19a..ebcceedc0 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -128,29 +128,25 @@ service:
 receivers:
   filelog:
     include: [ /var/log/pods/*/*/*.log ]
+    {{- if .Values.agentCollector.containerLogs.includeAgentLogs }}
+    exclude: [ /var/log/pods/{{ .Release.Namespace }}_{{ include "opentelemetry-collector.fullname" . }}-agent-*_*/{{ .Chart.Name }}/*.log ]
+    {{- end }}
     start_at: beginning
     include_file_path: true
     include_file_name: false
     operators:
-      # # Find out which format is used by kubernetes
-      # - type: router
-      #   id: get-format
-      #   routes:
-      #     - output: parser-docker
-      #       expr: '$$record matches "^\\{"'
-      #     - output: parser-crio
-      #       expr: '$$record matches "^[^ Z]+ "'
-      #     - output: parser-containerd
-      #       expr: '$$record matches "^[^ Z]+Z"'
-      # # Parse CRI-O format
-      # - type: regex_parser
-      #   id: parser-crio
-      #   regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
-      #   output: extract_metadata_from_filepath
-      #   timestamp:
-      #     parse_from: time
-      #     layout_type: gotime
-      #     layout: '2006-01-02T15:04:05.000000000-07:00'
+      {{- if eq .Values.agentCollector.containerLogs.containerRunTime "cri-o" }}
+      # Parse CRI-O format
+      - type: regex_parser
+        id: parser-crio
+        regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: time
+          layout_type: gotime
+          layout: '2006-01-02T15:04:05.000000000-07:00'
+      {{- end }}
+      {{- if eq .Values.agentCollector.containerLogs.containerRunTime "containerd" }}
       # Parse CRI-Containerd format
       - type: regex_parser
         id: parser-containerd
@@ -159,13 +155,16 @@ receivers:
         timestamp:
           parse_from: time
           layout: '%Y-%m-%dT%H:%M:%S.%LZ'
-      # # Parse Docker format
-      # - type: json_parser
-      #   id: parser-docker
-      #   output: extract_metadata_from_filepath
-      #   timestamp:
-      #     parse_from: time
-      #     layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+      {{- end }}
+      # Parse Docker format
+      {{- if eq .Values.agentCollector.containerLogs.containerRunTime "docker" }}
+      - type: json_parser
+        id: parser-docker
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: time
+          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+      {{- end }}
       # Extract metadata from file path
       - type: regex_parser
         id: extract_metadata_from_filepath
@@ -191,6 +190,15 @@ receivers:
           - remove: pod_name
           - remove: run_id
           - remove: uid
+      # Enrich log with k8s metadata
+      - type: k8s_metadata_decorator
+        id: k8s-metadata-enrichment
+        namespace_field: k8s.namespace.name
+        pod_name_field: k8s.pod.name
+        cache_ttl: 10m
+        timeout: 10s
+      # TODO: multiline concatenate per container
+      #- type: file
 service:
   pipelines:
     logs:
diff --git a/charts/opentelemetry-collector/values.yaml b/charts/opentelemetry-collector/values.yaml
index 1326d59bd..8de5fb28c 100644
--- a/charts/opentelemetry-collector/values.yaml
+++ b/charts/opentelemetry-collector/values.yaml
@@ -142,7 +142,14 @@ agentCollector:
 
   containerLogs:
     enabled: true
-
+    # Container runtime. One of `docker`, `cri-o`, or `containerd`
+    containerRunTime: ""
+    includeAgentLogs: false
+    enrichK8sMetadata: true
+
+    output:
+      splunk:
+        
   resources:
     limits:
       cpu: 256m

From 79fbef8d525a617f0a4a9981d25bbdbd1c575175 Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Wed, 10 Mar 2021 09:19:20 -0800
Subject: [PATCH 04/24] disable enrichment

---
 charts/opentelemetry-collector/templates/_config.tpl | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index ebcceedc0..14be31a24 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -191,12 +191,12 @@ receivers:
           - remove: run_id
           - remove: uid
       # Enrich log with k8s metadata
-      - type: k8s_metadata_decorator
-        id: k8s-metadata-enrichment
-        namespace_field: k8s.namespace.name
-        pod_name_field: k8s.pod.name
-        cache_ttl: 10m
-        timeout: 10s
+      # - type: k8s_metadata_decorator
+      #   id: k8s-metadata-enrichment
+      #   namespace_field: k8s.namespace.name
+      #   pod_name_field: k8s.pod.name
+      #   cache_ttl: 10m
+      #   timeout: 10s
       # TODO: multiline concatenate per container
       #- type: file
 service:

From 24c3338d7041143decafe1643fe232aa1c93086c Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Wed, 10 Mar 2021 10:28:28 -0800
Subject: [PATCH 05/24] uid 32 from 36

---
 charts/opentelemetry-collector/templates/_config.tpl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index 14be31a24..cff7c2237 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -163,12 +163,12 @@ receivers:
         output: extract_metadata_from_filepath
         timestamp:
           parse_from: time
-          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+          layout: '%Y-%m-%dT%H:%M:%S.%NZ'
       {{- end }}
       # Extract metadata from file path
       - type: regex_parser
         id: extract_metadata_from_filepath
-        regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{36})\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$'
+        regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{32})\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$'
         parse_from: $$labels.file_path
       # Move out attributes to Attributes
       - type: metadata

From f0bbd8aedebd971402e41d07560f93b02d6b5dbd Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Wed, 10 Mar 2021 10:30:40 -0800
Subject: [PATCH 06/24] N to L

---
 charts/opentelemetry-collector/templates/_config.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index cff7c2237..198829269 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -163,7 +163,7 @@ receivers:
         output: extract_metadata_from_filepath
         timestamp:
           parse_from: time
-          layout: '%Y-%m-%dT%H:%M:%S.%NZ'
+          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
       {{- end }}
       # Extract metadata from file path
       - type: regex_parser

From 3714321b1326ee4ed38e5f5e719da8488be3074c Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Wed, 10 Mar 2021 10:56:53 -0800
Subject: [PATCH 07/24] regex fix

---
 charts/opentelemetry-collector/templates/_config.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index 198829269..26d1d3fac 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -168,7 +168,7 @@ receivers:
       # Extract metadata from file path
       - type: regex_parser
         id: extract_metadata_from_filepath
-        regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{32})\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$'
+        regex: '^\/var\/log\/pods\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[^\/]+)\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$'
         parse_from: $$labels.file_path
       # Move out attributes to Attributes
       - type: metadata

From 198ac05bf399b2fb7155f079634aa9eeb3ff18bb Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Wed, 10 Mar 2021 11:17:51 -0800
Subject: [PATCH 08/24] health port

---
 charts/opentelemetry-collector/templates/_pod.tpl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/charts/opentelemetry-collector/templates/_pod.tpl b/charts/opentelemetry-collector/templates/_pod.tpl
index c19a676ca..05de168d3 100644
--- a/charts/opentelemetry-collector/templates/_pod.tpl
+++ b/charts/opentelemetry-collector/templates/_pod.tpl
@@ -21,6 +21,9 @@ containers:
     image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
     imagePullPolicy: {{ .Values.image.pullPolicy }}
     ports:
+      - name: liveness
+        containerPort: 13133
+        protocal: TCP 
       {{- range $key, $port := .Values.ports }}
       {{- if $port.enabled }}
       - name: {{ $key }}

From 996f0918e52ff98cc557da04858c87183ba5306b Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Wed, 10 Mar 2021 11:17:58 -0800
Subject: [PATCH 09/24] health port

---
 charts/opentelemetry-collector/templates/_pod.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/opentelemetry-collector/templates/_pod.tpl b/charts/opentelemetry-collector/templates/_pod.tpl
index 05de168d3..fad53e51d 100644
--- a/charts/opentelemetry-collector/templates/_pod.tpl
+++ b/charts/opentelemetry-collector/templates/_pod.tpl
@@ -21,7 +21,7 @@ containers:
     image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
     imagePullPolicy: {{ .Values.image.pullPolicy }}
     ports:
-      - name: liveness
+      - name: health
         containerPort: 13133
         protocal: TCP 
       {{- range $key, $port := .Values.ports }}

From 1d75a09e704e92e0dff0f21c9df4bff8ae794854 Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Wed, 10 Mar 2021 11:18:16 -0800
Subject: [PATCH 10/24] health port

---
 charts/opentelemetry-collector/templates/_pod.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/opentelemetry-collector/templates/_pod.tpl b/charts/opentelemetry-collector/templates/_pod.tpl
index fad53e51d..e07517ab9 100644
--- a/charts/opentelemetry-collector/templates/_pod.tpl
+++ b/charts/opentelemetry-collector/templates/_pod.tpl
@@ -23,7 +23,7 @@ containers:
     ports:
       - name: health
         containerPort: 13133
-        protocal: TCP 
+        protocol: TCP 
       {{- range $key, $port := .Values.ports }}
       {{- if $port.enabled }}
       - name: {{ $key }}

From 2338113bdf69207e0714e3bafd75a6b5bf497679 Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Wed, 10 Mar 2021 11:58:07 -0800
Subject: [PATCH 11/24] remove health probes

---
 .../log_test_values.yaml                      |  3 +++
 .../templates/_config.tpl                     |  2 ++
 .../templates/_pod.tpl                        | 21 ++++++++++++-------
 charts/opentelemetry-collector/values.yaml    |  3 +++
 4 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/charts/opentelemetry-collector/log_test_values.yaml b/charts/opentelemetry-collector/log_test_values.yaml
index 57a092560..356cd06c0 100644
--- a/charts/opentelemetry-collector/log_test_values.yaml
+++ b/charts/opentelemetry-collector/log_test_values.yaml
@@ -42,6 +42,9 @@ config:
       logs:
         exporters:
           - logging
+        processors:
+          - memory_limiter
+          - batch
         receivers:
           - filelog
       metrics:
diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index 26d1d3fac..739ad8f7a 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -199,6 +199,8 @@ receivers:
       #   timeout: 10s
       # TODO: multiline concatenate per container
       #- type: file
+processors:
+
 service:
   pipelines:
     logs:
diff --git a/charts/opentelemetry-collector/templates/_pod.tpl b/charts/opentelemetry-collector/templates/_pod.tpl
index e07517ab9..ea63768de 100644
--- a/charts/opentelemetry-collector/templates/_pod.tpl
+++ b/charts/opentelemetry-collector/templates/_pod.tpl
@@ -40,17 +40,22 @@ containers:
           fieldRef:
             apiVersion: v1
             fieldPath: status.podIP
+    - name: KUBE_NODE_NAME
+      valueFrom:
+        fieldRef:
+          apiVersion: v1
+          fieldPath: spec.nodeName
       {{- with .Values.extraEnvs }}
       {{- . | toYaml | nindent 6 }}
       {{- end }}
-    livenessProbe:
-      httpGet:
-        path: /
-        port: 13133
-    readinessProbe:
-      httpGet:
-        path: /
-        port: 13133
+    # livenessProbe:
+    #   httpGet:
+    #     path: /
+    #     port: 13133
+    # readinessProbe:
+    #   httpGet:
+    #     path: /
+    #     port: 13133
     resources:
       {{- toYaml .Values.resources | nindent 6 }}
     volumeMounts:
diff --git a/charts/opentelemetry-collector/values.yaml b/charts/opentelemetry-collector/values.yaml
index 8de5fb28c..c90d0d838 100644
--- a/charts/opentelemetry-collector/values.yaml
+++ b/charts/opentelemetry-collector/values.yaml
@@ -42,6 +42,9 @@ config:
       logs:
         exporters:
           - logging
+        processors:
+          - memory_limiter
+          - batch
         receivers:
           - filelog
       metrics:

From 86b706652f794665c222589b850ad71ff3fb1bac Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Wed, 10 Mar 2021 12:00:09 -0800
Subject: [PATCH 12/24] indent

---
 charts/opentelemetry-collector/templates/_pod.tpl | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/charts/opentelemetry-collector/templates/_pod.tpl b/charts/opentelemetry-collector/templates/_pod.tpl
index ea63768de..a9f151b6e 100644
--- a/charts/opentelemetry-collector/templates/_pod.tpl
+++ b/charts/opentelemetry-collector/templates/_pod.tpl
@@ -40,11 +40,11 @@ containers:
           fieldRef:
             apiVersion: v1
             fieldPath: status.podIP
-    - name: KUBE_NODE_NAME
-      valueFrom:
-        fieldRef:
-          apiVersion: v1
-          fieldPath: spec.nodeName
+      - name: KUBE_NODE_NAME
+        valueFrom:
+          fieldRef:
+            apiVersion: v1
+            fieldPath: spec.nodeName
       {{- with .Values.extraEnvs }}
       {{- . | toYaml | nindent 6 }}
       {{- end }}

From bbc8c4ef8261774c4799f2e490066309ea1f64e0 Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Wed, 10 Mar 2021 12:23:02 -0800
Subject: [PATCH 13/24] exclude

---
 charts/opentelemetry-collector/templates/_config.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index 739ad8f7a..f699f31e9 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -128,7 +128,7 @@ service:
 receivers:
   filelog:
     include: [ /var/log/pods/*/*/*.log ]
-    {{- if .Values.agentCollector.containerLogs.includeAgentLogs }}
+    {{- if not .Values.agentCollector.containerLogs.includeAgentLogs }}
     exclude: [ /var/log/pods/{{ .Release.Namespace }}_{{ include "opentelemetry-collector.fullname" . }}-agent-*_*/{{ .Chart.Name }}/*.log ]
     {{- end }}
     start_at: beginning

From 4ae1d8db625424d08fbe85f6dbb99b274d568757 Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Wed, 10 Mar 2021 16:14:13 -0800
Subject: [PATCH 14/24] k8s meta

---
 .../templates/_config.tpl                      | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index f699f31e9..7d410f68d 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -200,12 +200,28 @@ receivers:
       # TODO: multiline concatenate per container
       #- type: file
 processors:
-
+  k8s_tagger:
+    passthrough: false
+    auth_type: "kubeConfig"
+    extract:
+      metadata:
+        # extract the following well-known metadata fields
+        - podName
+        - podUID
+        - deployment
+        - cluster
+        - namespace
+        - node
+        - startTime
+    filter:
+      node_from_env_var: KUBE_NODE_NAME
 service:
   pipelines:
     logs:
       receivers:
         - filelog
+      processors:
+        - k8s_tagger
       exporters:
         - logging
 {{- end }}

From 0f9df780655b2402b6f65d90ed099a2129bb0ced Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Thu, 11 Mar 2021 10:47:22 -0800
Subject: [PATCH 15/24] rbac

---
 .../templates/_config.tpl                     | 22 +++++++++++++++++
 .../templates/_pod.tpl                        |  3 ---
 .../templates/clusterRole.yaml                | 24 +++++++++++++++++++
 .../templates/clusterRoleBinding.yaml         | 18 ++++++++++++++
 charts/opentelemetry-collector/values.yaml    |  7 ------
 5 files changed, 64 insertions(+), 10 deletions(-)
 create mode 100644 charts/opentelemetry-collector/templates/clusterRole.yaml
 create mode 100644 charts/opentelemetry-collector/templates/clusterRoleBinding.yaml

diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index 7d410f68d..9a2927134 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -215,6 +215,28 @@ processors:
         - startTime
     filter:
       node_from_env_var: KUBE_NODE_NAME
+
+exporters:
+  splunk_hec:
+    # Splunk HTTP Event Collector token.
+    token: "00000000-0000-0000-0000-0000000000000"
+    # URL to a Splunk instance to send data to.
+    endpoint: "https://splunk:8088/services/collector"
+    # Optional Splunk source: https://docs.splunk.com/Splexicon:Source
+    source: "otel"
+    # Optional Splunk source type: https://docs.splunk.com/Splexicon:Sourcetype
+    sourcetype: "otel"
+    # Splunk index, optional name of the Splunk index targeted.
+    index: "metrics"
+    # Maximum HTTP connections to use simultaneously when sending data. Defaults to 100.
+    max_connections: 20
+    # Whether to disable gzip compression over HTTP. Defaults to false.
+    disable_compression: false
+    # HTTP timeout when sending data. Defaults to 10s.
+    timeout: 10s
+    # Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false.
+    # For this demo, we use a self-signed certificate on the Splunk docker instance, so this flag is set to true.
+    insecure_skip_verify: true
 service:
   pipelines:
     logs:
diff --git a/charts/opentelemetry-collector/templates/_pod.tpl b/charts/opentelemetry-collector/templates/_pod.tpl
index a9f151b6e..9b657ab76 100644
--- a/charts/opentelemetry-collector/templates/_pod.tpl
+++ b/charts/opentelemetry-collector/templates/_pod.tpl
@@ -29,9 +29,6 @@ containers:
       - name: {{ $key }}
         containerPort: {{ $port.containerPort }}
         protocol: {{ $port.protocol }}
-        {{- if and $.isAgent $port.hostPort }}
-        hostPort: {{ $port.hostPort }}
-        {{- end }}
       {{- end }}
       {{- end }}
     env:
diff --git a/charts/opentelemetry-collector/templates/clusterRole.yaml b/charts/opentelemetry-collector/templates/clusterRole.yaml
new file mode 100644
index 000000000..f337b861c
--- /dev/null
+++ b/charts/opentelemetry-collector/templates/clusterRole.yaml
@@ -0,0 +1,24 @@
+{{- if eq .Values.agentCollector.enabled true -}}
+{{- if eq .Values.agentCollector.containerLogs.enrichK8sMetadata true -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ template "opentelemetry-collector.fullname" . }}
+  labels:
+    {{- include "opentelemetry-collector.labels" . | nindent 4 }}
+rules:
+- apiGroups:      ['extensions']
+  resources:      ['podsecuritypolicies']
+  verbs:          ['use']
+  resourceNames:  [{{ template "opentelemetry-collector.fullname" . }}]
+- apiGroups:
+    - ""
+  resources:
+    - "namespaces"
+    - "pods"
+  verbs:
+    - "list"
+    - "get"
+    - "watch"
+{{- end -}}
+{{- end -}}
\ No newline at end of file
diff --git a/charts/opentelemetry-collector/templates/clusterRoleBinding.yaml b/charts/opentelemetry-collector/templates/clusterRoleBinding.yaml
new file mode 100644
index 000000000..00455891c
--- /dev/null
+++ b/charts/opentelemetry-collector/templates/clusterRoleBinding.yaml
@@ -0,0 +1,18 @@
+{{- if eq .Values.agentCollector.enabled true -}}
+{{- if eq .Values.agentCollector.containerLogs.enrichK8sMetadata true -}}
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "opentelemetry-collector.fullname" . }}
+  labels:
+    {{- include "opentelemetry-collector.labels" . | nindent 4 }}
+roleRef:
+  kind: ClusterRole
+  name: {{ template "opentelemetry-collector.fullname" . }}
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+  name: {{ include "opentelemetry-collector.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+{{- end -}}
+{{- end -}}
\ No newline at end of file
diff --git a/charts/opentelemetry-collector/values.yaml b/charts/opentelemetry-collector/values.yaml
index c90d0d838..86c8d9521 100644
--- a/charts/opentelemetry-collector/values.yaml
+++ b/charts/opentelemetry-collector/values.yaml
@@ -118,25 +118,21 @@ ports:
     enabled: true
     containerPort: 4317
     servicePort: 4317
-    hostPort: 4317
     protocol: TCP
   jaeger-thrift:
     enabled: true
     containerPort: 14268
     servicePort: 14268
-    hostPort: 14268
     protocol: TCP
   jaeger-grpc:
     enabled: true
     containerPort: 14250
     servicePort: 14250
-    hostPort: 14250
     protocol: TCP
   zipkin:
     enabled: true
     containerPort: 9411
     servicePort: 9411
-    hostPort: 9411
     protocol: TCP
 
 # Configuration for agent OpenTelemetry Collector daemonset, enabled by default
@@ -149,9 +145,6 @@ agentCollector:
     containerRunTime: ""
     includeAgentLogs: false
     enrichK8sMetadata: true
-
-    output:
-      splunk:
         
   resources:
     limits:

From 95d31fbc01422e412f17f5a072cc0f3a79d79a68 Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Thu, 11 Mar 2021 13:01:00 -0800
Subject: [PATCH 16/24] batch

---
 .../log_test_values.yaml                      | 29 ++++++++++++++++++-
 .../templates/_config.tpl                     | 27 ++++-------------
 charts/opentelemetry-collector/values.yaml    | 26 ++++++++++++++++-
 3 files changed, 59 insertions(+), 23 deletions(-)

diff --git a/charts/opentelemetry-collector/log_test_values.yaml b/charts/opentelemetry-collector/log_test_values.yaml
index 356cd06c0..8a12abe6e 100644
--- a/charts/opentelemetry-collector/log_test_values.yaml
+++ b/charts/opentelemetry-collector/log_test_values.yaml
@@ -147,13 +147,40 @@ agentCollector:
     enabled: true
     # Container runtime. One of `docker`, `cri-o`, or `containerd`
     containerRunTime: docker
+    includeAgentLogs: false
+    enrichK8sMetadata: true
+
+    # Exporters for container logs
+    exporters:
+      # Example to send logs to splunk
+      splunk_hec:
+        # Splunk HTTP Event Collector token.
+        token: ""
+        # URL to a Splunk instance to send data to.
+        endpoint: "https://172.31.18.227:8088/services/collector"
+        # Optional Splunk source: https://docs.splunk.com/Splexicon:Source
+        source: "otel"
+        # Optional Splunk source type: https://docs.splunk.com/Splexicon:Sourcetype
+        sourcetype: "otel"
+        # Splunk index, optional name of the Splunk index targeted.
+        index: "k8s_log"
+        # Maximum HTTP connections to use simultaneously when sending data. Defaults to 100.
+        max_connections: 2000
+        # Whether to disable gzip compression over HTTP. Defaults to false.
+        disable_compression: false
+        # HTTP timeout when sending data. Defaults to 10s.
+        timeout: 10s
+        # Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false.
+        # For this demo, we use a self-signed certificate on the Splunk docker instance, so this flag is set to true.
+        insecure_skip_verify: true
 
   resources:
     limits:
       cpu: 256m
       memory: 512Mi
 
-  podAnnotations: {}
+  podAnnotations: 
+    hello: log
 
   # Configuration override that will be merged into the agent's default config
   configOverride: {}
diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index 9a2927134..f6d0dca75 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -217,34 +217,19 @@ processors:
       node_from_env_var: KUBE_NODE_NAME
 
 exporters:
-  splunk_hec:
-    # Splunk HTTP Event Collector token.
-    token: "00000000-0000-0000-0000-0000000000000"
-    # URL to a Splunk instance to send data to.
-    endpoint: "https://splunk:8088/services/collector"
-    # Optional Splunk source: https://docs.splunk.com/Splexicon:Source
-    source: "otel"
-    # Optional Splunk source type: https://docs.splunk.com/Splexicon:Sourcetype
-    sourcetype: "otel"
-    # Splunk index, optional name of the Splunk index targeted.
-    index: "metrics"
-    # Maximum HTTP connections to use simultaneously when sending data. Defaults to 100.
-    max_connections: 20
-    # Whether to disable gzip compression over HTTP. Defaults to false.
-    disable_compression: false
-    # HTTP timeout when sending data. Defaults to 10s.
-    timeout: 10s
-    # Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false.
-    # For this demo, we use a self-signed certificate on the Splunk docker instance, so this flag is set to true.
-    insecure_skip_verify: true
+  {{- toYaml .Values.agentCollector.containerLogs.exporters | nindent 2 }}
+
 service:
   pipelines:
     logs:
       receivers:
         - filelog
       processors:
+        - batch
         - k8s_tagger
       exporters:
-        - logging
+        {{- range $key, $exporterData := .Values.agentCollector.containerLogs.exporters }}
+        - {{ $key }}
+        {{- end }}
 {{- end }}
 {{- end }}
diff --git a/charts/opentelemetry-collector/values.yaml b/charts/opentelemetry-collector/values.yaml
index 86c8d9521..48fa4b4b8 100644
--- a/charts/opentelemetry-collector/values.yaml
+++ b/charts/opentelemetry-collector/values.yaml
@@ -145,7 +145,31 @@ agentCollector:
     containerRunTime: ""
     includeAgentLogs: false
     enrichK8sMetadata: true
-        
+
+    # Exporters for container logs 
+    exporters:
+      # # Example to send logs to splunk
+      # splunk_hec:
+      #   # Splunk HTTP Event Collector token.
+      #   token: "00000000-0000-0000-0000-0000000000000"
+      #   # URL to a Splunk instance to send data to.
+      #   endpoint: "https://splunk:8088/services/collector"
+      #   # Optional Splunk source: https://docs.splunk.com/Splexicon:Source
+      #   source: "otel"
+      #   # Optional Splunk source type: https://docs.splunk.com/Splexicon:Sourcetype
+      #   sourcetype: "otel"
+      #   # Splunk index, optional name of the Splunk index targeted.
+      #   index: "metrics"
+      #   # Maximum HTTP connections to use simultaneously when sending data. Defaults to 100.
+      #   max_connections: 20
+      #   # Whether to disable gzip compression over HTTP. Defaults to false.
+      #   disable_compression: false
+      #   # HTTP timeout when sending data. Defaults to 10s.
+      #   timeout: 10s
+      #   # Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false.
+      #   # For this demo, we use a self-signed certificate on the Splunk docker instance, so this flag is set to true.
+      #   insecure_skip_verify: true
+
   resources:
     limits:
       cpu: 256m

From d7596322d4df4b027b33826ca3d6e84e7aeaf75c Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Thu, 11 Mar 2021 14:08:58 -0800
Subject: [PATCH 17/24] anno label

---
 .gitignore                                    |  1 +
 .../templates/_config.tpl                     | 19 +++++++++----------
 charts/opentelemetry-collector/values.yaml    |  3 +++
 3 files changed, 13 insertions(+), 10 deletions(-)
 create mode 100644 .gitignore

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 000000000..3931c8054
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+*/.tmp/
diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index f6d0dca75..0e17fe7c0 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -190,15 +190,7 @@ receivers:
           - remove: pod_name
           - remove: run_id
           - remove: uid
-      # Enrich log with k8s metadata
-      # - type: k8s_metadata_decorator
-      #   id: k8s-metadata-enrichment
-      #   namespace_field: k8s.namespace.name
-      #   pod_name_field: k8s.pod.name
-      #   cache_ttl: 10m
-      #   timeout: 10s
-      # TODO: multiline concatenate per container
-      #- type: file
+
 processors:
   k8s_tagger:
     passthrough: false
@@ -213,9 +205,16 @@ processors:
         - namespace
         - node
         - startTime
+      annotations:
+        {{- toYaml .Values.agentCollector.containerLogs.listOfAnnotations | nindent 8 }}
+      labels:
+        {{- toYaml .Values.agentCollector.containerLogs.listOfLabels | nindent 8 }}
+        
+    pod_association:
+      - from: resource_attribute
+        name: k8s.pod.uid
     filter:
       node_from_env_var: KUBE_NODE_NAME
-
 exporters:
   {{- toYaml .Values.agentCollector.containerLogs.exporters | nindent 2 }}
 
diff --git a/charts/opentelemetry-collector/values.yaml b/charts/opentelemetry-collector/values.yaml
index 48fa4b4b8..cb91a8e03 100644
--- a/charts/opentelemetry-collector/values.yaml
+++ b/charts/opentelemetry-collector/values.yaml
@@ -145,6 +145,9 @@ agentCollector:
     containerRunTime: ""
     includeAgentLogs: false
     enrichK8sMetadata: true
+    listOfAnnotations: 
+      - key: splunk.com/index
+    listOfLabels: []
 
     # Exporters for container logs 
     exporters:

From 9de5087f13a2cb18117492ad12334b4dc23531a6 Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Thu, 11 Mar 2021 19:22:56 -0800
Subject: [PATCH 18/24] test

---
 .gitignore                                    |   2 +-
 .../log_test_values.yaml                      | 278 ------------------
 .../templates/_config.tpl                     |  51 ++--
 3 files changed, 27 insertions(+), 304 deletions(-)
 delete mode 100644 charts/opentelemetry-collector/log_test_values.yaml

diff --git a/.gitignore b/.gitignore
index 3931c8054..3018b3a68 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1 @@
-*/.tmp/
+.tmp/
diff --git a/charts/opentelemetry-collector/log_test_values.yaml b/charts/opentelemetry-collector/log_test_values.yaml
deleted file mode 100644
index 8a12abe6e..000000000
--- a/charts/opentelemetry-collector/log_test_values.yaml
+++ /dev/null
@@ -1,278 +0,0 @@
-# Default values for opentelemetry-collector.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-nameOverride: ""
-fullnameOverride: ""
-
-config:
-  exporters:
-    logging: {}
-  extensions:
-    health_check: {}
-  processors:
-    batch: {}
-    # If set to null, will be overridden with values based on k8s resource limits
-    memory_limiter: null
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: 0.0.0.0:14250
-        thrift_http:
-          endpoint: 0.0.0.0:14268
-    #otlp:
-    #  protocols:
-    #    grpc: null
-    #    http: null
-    prometheus:
-      config:
-        scrape_configs:
-          - job_name: opentelemetry-collector
-            scrape_interval: 10s
-            static_configs:
-              - targets:
-                  - ${MY_POD_IP}:8888
-    zipkin:
-      endpoint: 0.0.0.0:9411
-  service:
-    extensions:
-      - health_check
-    pipelines:
-      logs:
-        exporters:
-          - logging
-        processors:
-          - memory_limiter
-          - batch
-        receivers:
-          - filelog
-      metrics:
-        exporters:
-          - logging
-        processors:
-          - memory_limiter
-          - batch
-        receivers:
-          #- otlp
-          - prometheus
-      traces:
-        exporters:
-          - logging
-        processors:
-          - memory_limiter
-          - batch
-        receivers:
-          #- otlp
-          - jaeger
-          - zipkin
-
-# Shared params for agentCollector daemonset and standaloneCollector deployment pods.
-# Can be overridden here or for any component independently using the same keys.
-
-image:
-  repository: sumodrosiek/otelcontribcol
-  # repository: otel/opentelemetry-collector-contrib
-  pullPolicy: IfNotPresent
-  # Overrides the image tag whose default is the chart appVersion.
-  tag: k8s-1613747494
-imagePullSecrets: []
-
-# OpenTelemetry Collector executable
-command:
-  #name: otelcol
-  name: otelcontribcol
-  extraArgs: []
-
-serviceAccount:
-  # Specifies whether a service account should be created
-  create: true
-  # Annotations to add to the service account
-  annotations: {}
-  # The name of the service account to use.
-  # If not set and create is true, a name is generated using the fullname template
-  name: ""
-
-podSecurityContext: {}
-securityContext: {}
-
-nodeSelector: {}
-tolerations: []
-affinity: {}
-
-extraEnvs: []
-extraHostPathMounts:
-  - name: varlog
-    hostPath: /var/log
-    mountPath: /var/log
-    readOnly: true
-  - name: varlibdockercontainers
-    hostPath: /var/lib/docker/containers
-    mountPath: /var/lib/docker/containers
-    readOnly: true
-
-# Configuration for ports, shared between agentCollector, standaloneCollector and service.
-# Can be overridden here or for agentCollector and standaloneCollector independently.
-ports:
-  otlp:
-    enabled: false
-    containerPort: 4317
-    servicePort: 4317
-    hostPort: 4317
-    protocol: TCP
-  jaeger-thrift:
-    enabled: false
-    containerPort: 14268
-    servicePort: 14268
-    hostPort: 14268
-    protocol: TCP
-  jaeger-grpc:
-    enabled: false
-    containerPort: 14250
-    servicePort: 14250
-    hostPort: 14250
-    protocol: TCP
-  zipkin:
-    enabled: false
-    containerPort: 9411
-    servicePort: 9411
-    hostPort: 9411
-    protocol: TCP
-
-# Configuration for agent OpenTelemetry Collector daemonset, enabled by default
-agentCollector:
-  enabled: true
-
-  containerLogs:
-    enabled: true
-    # Container runtime. One of `docker`, `cri-o`, or `containerd`
-    containerRunTime: docker
-    includeAgentLogs: false
-    enrichK8sMetadata: true
-
-    # Exporters for container logs
-    exporters:
-      # Example to send logs to splunk
-      splunk_hec:
-        # Splunk HTTP Event Collector token.
-        token: ""
-        # URL to a Splunk instance to send data to.
-        endpoint: "https://172.31.18.227:8088/services/collector"
-        # Optional Splunk source: https://docs.splunk.com/Splexicon:Source
-        source: "otel"
-        # Optional Splunk source type: https://docs.splunk.com/Splexicon:Sourcetype
-        sourcetype: "otel"
-        # Splunk index, optional name of the Splunk index targeted.
-        index: "k8s_log"
-        # Maximum HTTP connections to use simultaneously when sending data. Defaults to 100.
-        max_connections: 2000
-        # Whether to disable gzip compression over HTTP. Defaults to false.
-        disable_compression: false
-        # HTTP timeout when sending data. Defaults to 10s.
-        timeout: 10s
-        # Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false.
-        # For this demo, we use a self-signed certificate on the Splunk docker instance, so this flag is set to true.
-        insecure_skip_verify: true
-
-  resources:
-    limits:
-      cpu: 256m
-      memory: 512Mi
-
-  podAnnotations: 
-    hello: log
-
-  # Configuration override that will be merged into the agent's default config
-  configOverride: {}
-  # The following config override can be used to enable host receiver
-  #   receivers:
-  #     hostmetrics:
-  #       scrapers:
-  #         cpu:
-  #         disk:
-  #         filesystem:
-  #   service:
-  #     pipelines:
-  #       metrics:
-  #         receivers: [prometheus, hostmetrics]
-
-  # Any of the top level options can be overridden specifically for agent
-  # image: {}
-  # imagePullSecrets: []
-  # command: {}
-  # serviceAccount: {}
-  # podSecurityContext: {}
-  # securityContext: {}
-  # nodeSelector: {}
-  # tolerations: []
-  # affinity: {}
-
-  # The following option overrides can be used with host receiver
-  # extraEnvs:
-  # - name: HOST_PROC
-  #   value: /hostfs/proc
-  # - name: HOST_SYS
-  #   value: /hostfs/sys
-  # - name: HOST_ETC
-  #   value: /hostfs/etc
-  # - name: HOST_VAR
-  #   value: /hostfs/var
-  # - name: HOST_RUN
-  #   value: /hostfs/run
-  # - name: HOST_DEV
-  #   value: /hostfs/dev
-  # extraHostPathMounts:
-  # - name: hostfs
-  #   hostPath: /
-  #   mountPath: /hostfs
-  #   readOnly: true
-  #   mountPropagation: HostToContainer
-
-  # The following example disables all jaeger ports and zipkin hostPort on the agent
-  # ports:
-  #   jaeger-thrift:
-  #     enabled: false
-  #   jaeger-grpc:
-  #     enabled: false
-  #   zipkin:
-  #     hostPort: ""
-
-# Configuration for standalone OpenTelemetry Collector deployment, disabled by default
-standaloneCollector:
-  enabled: false
-
-  replicaCount: 1
-
-  resources:
-    limits:
-      cpu: 1
-      memory: 2Gi
-
-  podAnnotations: {}
-
-  # Configuration override that will be merged into the standalone collector default config
-  configOverride: {}
-
-  # Any of the top level options can be overridden specifically for standalone collector
-  # image: {}
-  # imagePullSecrets: []
-  # command: {}
-  # serviceAccount: {}
-  # podSecurityContext: {}
-  # securityContext: {}
-  # nodeSelector: {}
-  # tolerations: []
-  # affinity: {}
-  # ports: {}
-
-service:
-  type: ClusterIP
-  annotations: {}
-
-# autoscaling is used only if standaloneCollector enabled
-autoscaling:
-  enabled: false
-  minReplicas: 1
-  maxReplicas: 10
-  targetCPUUtilizationPercentage: 80
-  # targetMemoryUtilizationPercentage: 80
diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index 0e17fe7c0..f847bdaa2 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -179,6 +179,8 @@ receivers:
           k8s.pod.name: 'EXPR($.pod_name)'
           run_id: 'EXPR($.run_id)'
           k8s.pod.uid: 'EXPR($.uid)'
+        resource:
+          k8s.pod.uid: 'EXPR($.uid)'
       # Clean up log record
       - type: restructure
         id: clean-up-log-record
@@ -191,30 +193,29 @@ receivers:
           - remove: run_id
           - remove: uid
 
-processors:
-  k8s_tagger:
-    passthrough: false
-    auth_type: "kubeConfig"
-    extract:
-      metadata:
-        # extract the following well-known metadata fields
-        - podName
-        - podUID
-        - deployment
-        - cluster
-        - namespace
-        - node
-        - startTime
-      annotations:
-        {{- toYaml .Values.agentCollector.containerLogs.listOfAnnotations | nindent 8 }}
-      labels:
-        {{- toYaml .Values.agentCollector.containerLogs.listOfLabels | nindent 8 }}
-        
-    pod_association:
-      - from: resource_attribute
-        name: k8s.pod.uid
-    filter:
-      node_from_env_var: KUBE_NODE_NAME
+# processors:
+#   k8s_tagger:
+#     passthrough: false
+#     auth_type: "kubeConfig"
+#     pod_association:
+#       - from: resource_attribute
+#         name: k8s.pod.uid
+#     extract:
+#       metadata:
+#         # extract the following well-known metadata fields
+#         - podName
+#         - podUID
+#         - deployment
+#         - cluster
+#         - namespace
+#         - node
+#         - startTime
+#       annotations:
+#         {{- toYaml .Values.agentCollector.containerLogs.listOfAnnotations | nindent 8 }}
+#       labels:
+#         {{- toYaml .Values.agentCollector.containerLogs.listOfLabels | nindent 8 }}
+#     filter:
+#       node_from_env_var: KUBE_NODE_NAME
 exporters:
   {{- toYaml .Values.agentCollector.containerLogs.exporters | nindent 2 }}
 
@@ -225,7 +226,7 @@ service:
         - filelog
       processors:
         - batch
-        - k8s_tagger
+        # - k8s_tagger
       exporters:
         {{- range $key, $exporterData := .Values.agentCollector.containerLogs.exporters }}
         - {{ $key }}

From e126857ef16787c1ccab3e4b808358db5a675fd2 Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Thu, 11 Mar 2021 19:29:50 -0800
Subject: [PATCH 19/24] fix error

---
 .../templates/_config.tpl                     | 50 +++++++++----------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index f847bdaa2..43283017a 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -172,7 +172,7 @@ receivers:
         parse_from: $$labels.file_path
       # Move out attributes to Attributes
       - type: metadata
-        labels:
+        attributes:
           stream: 'EXPR($.stream)'
           k8s.container.name: 'EXPR($.container_name)'
           k8s.namespace.name: 'EXPR($.namespace)'
@@ -193,29 +193,29 @@ receivers:
           - remove: run_id
           - remove: uid
 
-# processors:
-#   k8s_tagger:
-#     passthrough: false
-#     auth_type: "kubeConfig"
-#     pod_association:
-#       - from: resource_attribute
-#         name: k8s.pod.uid
-#     extract:
-#       metadata:
-#         # extract the following well-known metadata fields
-#         - podName
-#         - podUID
-#         - deployment
-#         - cluster
-#         - namespace
-#         - node
-#         - startTime
-#       annotations:
-#         {{- toYaml .Values.agentCollector.containerLogs.listOfAnnotations | nindent 8 }}
-#       labels:
-#         {{- toYaml .Values.agentCollector.containerLogs.listOfLabels | nindent 8 }}
-#     filter:
-#       node_from_env_var: KUBE_NODE_NAME
+processors:
+  k8s_tagger:
+    passthrough: false
+    auth_type: "kubeConfig"
+    # pod_association:
+    #   - from: resource_attribute
+    #     name: k8s.pod.uid
+    extract:
+      metadata:
+        # extract the following well-known metadata fields
+        - podName
+        - podUID
+        - deployment
+        - cluster
+        - namespace
+        - node
+        - startTime
+      annotations:
+        {{- toYaml .Values.agentCollector.containerLogs.listOfAnnotations | nindent 8 }}
+      labels:
+        {{- toYaml .Values.agentCollector.containerLogs.listOfLabels | nindent 8 }}
+    filter:
+      node_from_env_var: KUBE_NODE_NAME
 exporters:
   {{- toYaml .Values.agentCollector.containerLogs.exporters | nindent 2 }}
 
@@ -226,7 +226,7 @@ service:
         - filelog
       processors:
         - batch
-        # - k8s_tagger
+        #- k8s_tagger
       exporters:
         {{- range $key, $exporterData := .Values.agentCollector.containerLogs.exporters }}
         - {{ $key }}

From a755dd7e7c7a3ef26943c30b6c74c3844c6da915 Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Thu, 11 Mar 2021 19:39:36 -0800
Subject: [PATCH 20/24] configurable enrichment

---
 .../opentelemetry-collector/templates/_config.tpl  | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index 43283017a..ff15038e7 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -192,14 +192,14 @@ receivers:
           - remove: pod_name
           - remove: run_id
           - remove: uid
-
+{{- if .Values.agentCollector.containerLogs.enrichK8sMetadata }}
 processors:
   k8s_tagger:
     passthrough: false
     auth_type: "kubeConfig"
-    # pod_association:
-    #   - from: resource_attribute
-    #     name: k8s.pod.uid
+    pod_association:
+      - from: resource_attribute
+        name: k8s.pod.uid
     extract:
       metadata:
         # extract the following well-known metadata fields
@@ -216,9 +216,9 @@ processors:
         {{- toYaml .Values.agentCollector.containerLogs.listOfLabels | nindent 8 }}
     filter:
       node_from_env_var: KUBE_NODE_NAME
+{{- end }}
 exporters:
   {{- toYaml .Values.agentCollector.containerLogs.exporters | nindent 2 }}
-
 service:
   pipelines:
     logs:
@@ -226,7 +226,9 @@ service:
         - filelog
       processors:
         - batch
-        #- k8s_tagger
+        {{- if .Values.agentCollector.containerLogs.enrichK8sMetadata }}
+        - k8s_tagger
+        {{- end }}
       exporters:
         {{- range $key, $exporterData := .Values.agentCollector.containerLogs.exporters }}
         - {{ $key }}

From e00a5bda3b0c14f696b9abbefb9488d743d3f29f Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Thu, 11 Mar 2021 20:37:49 -0800
Subject: [PATCH 21/24] attributes

---
 .../templates/_config.tpl                     |  8 +++---
 charts/opentelemetry-collector/values.yaml    | 27 ++++++++-----------
 2 files changed, 15 insertions(+), 20 deletions(-)

diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
index ff15038e7..042f04545 100644
--- a/charts/opentelemetry-collector/templates/_config.tpl
+++ b/charts/opentelemetry-collector/templates/_config.tpl
@@ -169,7 +169,7 @@ receivers:
       - type: regex_parser
         id: extract_metadata_from_filepath
         regex: '^\/var\/log\/pods\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[^\/]+)\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$'
-        parse_from: $$labels.file_path
+        parse_from: $$attributes.file_path
       # Move out attributes to Attributes
       - type: metadata
         attributes:
@@ -197,9 +197,9 @@ processors:
   k8s_tagger:
     passthrough: false
     auth_type: "kubeConfig"
-    pod_association:
-      - from: resource_attribute
-        name: k8s.pod.uid
+    # pod_association: # TODO: it is not working. 
+    #   - from: resource_attribute
+    #     name: k8s.pod.uid
     extract:
       metadata:
         # extract the following well-known metadata fields
diff --git a/charts/opentelemetry-collector/values.yaml b/charts/opentelemetry-collector/values.yaml
index cb91a8e03..a3663b7f8 100644
--- a/charts/opentelemetry-collector/values.yaml
+++ b/charts/opentelemetry-collector/values.yaml
@@ -145,33 +145,28 @@ agentCollector:
     containerRunTime: ""
     includeAgentLogs: false
     enrichK8sMetadata: true
-    listOfAnnotations: 
-      - key: splunk.com/index
+    listOfAnnotations: []
     listOfLabels: []
 
-    # Exporters for container logs 
+    # Exporters for container logs. 
+    # https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter
+    # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter
     exporters:
-      # # Example to send logs to splunk
+      # # Example of sending logs to splunk
       # splunk_hec:
-      #   # Splunk HTTP Event Collector token.
       #   token: "00000000-0000-0000-0000-0000000000000"
-      #   # URL to a Splunk instance to send data to.
       #   endpoint: "https://splunk:8088/services/collector"
-      #   # Optional Splunk source: https://docs.splunk.com/Splexicon:Source
       #   source: "otel"
-      #   # Optional Splunk source type: https://docs.splunk.com/Splexicon:Sourcetype
       #   sourcetype: "otel"
-      #   # Splunk index, optional name of the Splunk index targeted.
-      #   index: "metrics"
-      #   # Maximum HTTP connections to use simultaneously when sending data. Defaults to 100.
-      #   max_connections: 20
-      #   # Whether to disable gzip compression over HTTP. Defaults to false.
+      #   index: "k8s_logs"
+      #   max_connections: 100
       #   disable_compression: false
-      #   # HTTP timeout when sending data. Defaults to 10s.
       #   timeout: 10s
-      #   # Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false.
-      #   # For this demo, we use a self-signed certificate on the Splunk docker instance, so this flag is set to true.
       #   insecure_skip_verify: true
+      #   sending_queue:
+      #     enabled: true
+      #     num_consumers: 2
+      #     queue_size: 10
 
   resources:
     limits:

From f616ad163e6482a69d78346a6d19b62942acd1b1 Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Thu, 11 Mar 2021 21:59:27 -0800
Subject: [PATCH 22/24] version bump

---
 charts/opentelemetry-collector/Chart.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/charts/opentelemetry-collector/Chart.yaml b/charts/opentelemetry-collector/Chart.yaml
index c5521213e..d3b600887 100644
--- a/charts/opentelemetry-collector/Chart.yaml
+++ b/charts/opentelemetry-collector/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: opentelemetry-collector
-version: 0.4.3
+version: 0.5.0
 description: OpenTelemetry Collector Helm chart for Kubernetes
 type: application
 home: https://opentelemetry.io/
@@ -13,4 +13,4 @@ maintainers:
   - name: pjanotti
   - name: tigrannajaryan
 icon: https://opentelemetry.io/img/logos/opentelemetry-logo-nav.png
-appVersion: 0.20.0
+appVersion: 0.22.0

From 49128fb862bb9ef5575c266759ae9ce28f2aef9f Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Thu, 11 Mar 2021 22:07:56 -0800
Subject: [PATCH 23/24] lint fix

---
 charts/opentelemetry-collector/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/opentelemetry-collector/values.yaml b/charts/opentelemetry-collector/values.yaml
index a3663b7f8..60b5c5b17 100644
--- a/charts/opentelemetry-collector/values.yaml
+++ b/charts/opentelemetry-collector/values.yaml
@@ -148,7 +148,7 @@ agentCollector:
     listOfAnnotations: []
     listOfLabels: []
 
-    # Exporters for container logs. 
+    # Exporters for container logs.
     # https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter
     # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter
     exporters:

From 90c86cdaabe15833efb796bc3495cf02e2d5be45 Mon Sep 17 00:00:00 2001
From: Rock Baek <rbaek@splunk.com>
Date: Thu, 11 Mar 2021 22:11:13 -0800
Subject: [PATCH 24/24] memo

---
 charts/opentelemetry-collector/templates/_pod.tpl | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/charts/opentelemetry-collector/templates/_pod.tpl b/charts/opentelemetry-collector/templates/_pod.tpl
index 9b657ab76..51efa5eb4 100644
--- a/charts/opentelemetry-collector/templates/_pod.tpl
+++ b/charts/opentelemetry-collector/templates/_pod.tpl
@@ -21,9 +21,6 @@ containers:
     image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
     imagePullPolicy: {{ .Values.image.pullPolicy }}
     ports:
-      - name: health
-        containerPort: 13133
-        protocol: TCP 
       {{- range $key, $port := .Values.ports }}
       {{- if $port.enabled }}
       - name: {{ $key }}
@@ -45,7 +42,7 @@ containers:
       {{- with .Values.extraEnvs }}
       {{- . | toYaml | nindent 6 }}
       {{- end }}
-    # livenessProbe:
+    # livenessProbe: # TODO: does not work. it keeps killing pod. 
     #   httpGet:
     #     path: /
     #     port: 13133