Helm logging support (#1)

* enrichment * regex fix * health port * remove health probes * rbac * batch * anno label * attributes * version bump
rockb1017 · Mar 12, 2021 · 30094a5 · 30094a5
1 parent 4e74903
commit 30094a5
Show file tree

Hide file tree

Showing 7 changed files with 218 additions and 18 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+.tmp/
diff --git a/charts/opentelemetry-collector/Chart.yaml b/charts/opentelemetry-collector/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: opentelemetry-collector
-version: 0.4.3
+version: 0.5.0
 description: OpenTelemetry Collector Helm chart for Kubernetes
 type: application
 home: https://opentelemetry.io/
@@ -13,4 +13,4 @@ maintainers:
   - name: pjanotti
   - name: tigrannajaryan
 icon: https://opentelemetry.io/img/logos/opentelemetry-logo-nav.png
-appVersion: 0.20.0
+appVersion: 0.22.0
diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl
@@ -32,6 +32,7 @@ Build config file for agent OpenTelemetry Collector
 {{- $values := deepCopy .Values.agentCollector | mustMergeOverwrite (deepCopy .Values)  }}
 {{- $data := dict "Values" $values | mustMergeOverwrite (deepCopy .) }}
 {{- $config := include "opentelemetry-collector.baseConfig" $data | fromYaml }}
+{{- $config := include "opentelemetry-collector.agent.containerLogsConfig" $data | fromYaml | mustMergeOverwrite $config }}
 {{- $config := include "opentelemetry-collector.agentConfigOverride" $data | fromYaml | mustMergeOverwrite $config }}
 {{- .Values.agentCollector.configOverride | mustMergeOverwrite $config | toYaml }}
 {{- end }}
@@ -121,3 +122,116 @@ service:
       exporters: [otlp]
 {{- end }}
 {{- end }}
+
+{{- define "opentelemetry-collector.agent.containerLogsConfig" -}}
+{{- if .Values.agentCollector.containerLogs.enabled }}
+receivers:
+  filelog:
+    include: [ /var/log/pods/*/*/*.log ]
+    {{- if not .Values.agentCollector.containerLogs.includeAgentLogs }}
+    exclude: [ /var/log/pods/{{ .Release.Namespace }}_{{ include "opentelemetry-collector.fullname" . }}-agent-*_*/{{ .Chart.Name }}/*.log ]
+    {{- end }}
+    start_at: beginning
+    include_file_path: true
+    include_file_name: false
+    operators:
+      {{- if eq .Values.agentCollector.containerLogs.containerRunTime "cri-o" }}
+      # Parse CRI-O format
+      - type: regex_parser
+        id: parser-crio
+        regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: time
+          layout_type: gotime
+          layout: '2006-01-02T15:04:05.000000000-07:00'
+      {{- end }}
+      {{- if eq .Values.agentCollector.containerLogs.containerRunTime "containerd" }}
+      # Parse CRI-Containerd format
+      - type: regex_parser
+        id: parser-containerd
+        regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: time
+          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+      {{- end }}
+      # Parse Docker format
+      {{- if eq .Values.agentCollector.containerLogs.containerRunTime "docker" }}
+      - type: json_parser
+        id: parser-docker
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: time
+          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+      {{- end }}
+      # Extract metadata from file path
+      - type: regex_parser
+        id: extract_metadata_from_filepath
+        regex: '^\/var\/log\/pods\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[^\/]+)\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$'
+        parse_from: $$attributes.file_path
+      # Move out attributes to Attributes
+      - type: metadata
+        attributes:
+          stream: 'EXPR($.stream)'
+          k8s.container.name: 'EXPR($.container_name)'
+          k8s.namespace.name: 'EXPR($.namespace)'
+          k8s.pod.name: 'EXPR($.pod_name)'
+          run_id: 'EXPR($.run_id)'
+          k8s.pod.uid: 'EXPR($.uid)'
+        resource:
+          k8s.pod.uid: 'EXPR($.uid)'
+      # Clean up log record
+      - type: restructure
+        id: clean-up-log-record
+        ops:
+          - remove: logtag
+          - remove: stream
+          - remove: container_name
+          - remove: namespace
+          - remove: pod_name
+          - remove: run_id
+          - remove: uid
+{{- if .Values.agentCollector.containerLogs.enrichK8sMetadata }}
+processors:
+  k8s_tagger:
+    passthrough: false
+    auth_type: "kubeConfig"
+    # pod_association: # TODO: it is not working. 
+    #   - from: resource_attribute
+    #     name: k8s.pod.uid
+    extract:
+      metadata:
+        # extract the following well-known metadata fields
+        - podName
+        - podUID
+        - deployment
+        - cluster
+        - namespace
+        - node
+        - startTime
+      annotations:
+        {{- toYaml .Values.agentCollector.containerLogs.listOfAnnotations | nindent 8 }}
+      labels:
+        {{- toYaml .Values.agentCollector.containerLogs.listOfLabels | nindent 8 }}
+    filter:
+      node_from_env_var: KUBE_NODE_NAME
+{{- end }}
+exporters:
+  {{- toYaml .Values.agentCollector.containerLogs.exporters | nindent 2 }}
+service:
+  pipelines:
+    logs:
+      receivers:
+        - filelog
+      processors:
+        - batch
+        {{- if .Values.agentCollector.containerLogs.enrichK8sMetadata }}
+        - k8s_tagger
+        {{- end }}
+      exporters:
+        {{- range $key, $exporterData := .Values.agentCollector.containerLogs.exporters }}
+        - {{ $key }}
+        {{- end }}
+{{- end }}
+{{- end }}
diff --git a/charts/opentelemetry-collector/templates/_pod.tpl b/charts/opentelemetry-collector/templates/_pod.tpl
@@ -26,9 +26,6 @@ containers:
       - name: {{ $key }}
         containerPort: {{ $port.containerPort }}
         protocol: {{ $port.protocol }}
-        {{- if and $.isAgent $port.hostPort }}
-        hostPort: {{ $port.hostPort }}
-        {{- end }}
       {{- end }}
       {{- end }}
     env:
@@ -37,17 +34,22 @@ containers:
           fieldRef:
             apiVersion: v1
             fieldPath: status.podIP
+      - name: KUBE_NODE_NAME
+        valueFrom:
+          fieldRef:
+            apiVersion: v1
+            fieldPath: spec.nodeName
       {{- with .Values.extraEnvs }}
       {{- . | toYaml | nindent 6 }}
       {{- end }}
-    livenessProbe:
-      httpGet:
-        path: /
-        port: 13133
-    readinessProbe:
-      httpGet:
-        path: /
-        port: 13133
+    # livenessProbe: # TODO: does not work. it keeps killing pod. 
+    #   httpGet:
+    #     path: /
+    #     port: 13133
+    # readinessProbe:
+    #   httpGet:
+    #     path: /
+    #     port: 13133
     resources:
       {{- toYaml .Values.resources | nindent 6 }}
     volumeMounts:

diff --git a/charts/opentelemetry-collector/templates/clusterRole.yaml b/charts/opentelemetry-collector/templates/clusterRole.yaml
@@ -0,0 +1,24 @@
+{{- if eq .Values.agentCollector.enabled true -}}
+{{- if eq .Values.agentCollector.containerLogs.enrichK8sMetadata true -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ template "opentelemetry-collector.fullname" . }}
+  labels:
+    {{- include "opentelemetry-collector.labels" . | nindent 4 }}
+rules:
+- apiGroups:      ['extensions']
+  resources:      ['podsecuritypolicies']
+  verbs:          ['use']
+  resourceNames:  [{{ template "opentelemetry-collector.fullname" . }}]
+- apiGroups:
+    - ""
+  resources:
+    - "namespaces"
+    - "pods"
+  verbs:
+    - "list"
+    - "get"
+    - "watch"
+{{- end -}}
+{{- end -}}
diff --git a/charts/opentelemetry-collector/templates/clusterRoleBinding.yaml b/charts/opentelemetry-collector/templates/clusterRoleBinding.yaml
@@ -0,0 +1,18 @@
+{{- if eq .Values.agentCollector.enabled true -}}
+{{- if eq .Values.agentCollector.containerLogs.enrichK8sMetadata true -}}
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "opentelemetry-collector.fullname" . }}
+  labels:
+    {{- include "opentelemetry-collector.labels" . | nindent 4 }}
+roleRef:
+  kind: ClusterRole
+  name: {{ template "opentelemetry-collector.fullname" . }}
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+  name: {{ include "opentelemetry-collector.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+{{- end -}}
+{{- end -}}
diff --git a/charts/opentelemetry-collector/values.yaml b/charts/opentelemetry-collector/values.yaml
@@ -39,6 +39,14 @@ config:
     extensions:
       - health_check
     pipelines:
+      logs:
+        exporters:
+          - logging
+        processors:
+          - memory_limiter
+          - batch
+        receivers:
+          - filelog
       metrics:
         exporters:
           - logging
@@ -93,7 +101,15 @@ tolerations: []
 affinity: {}
 
 extraEnvs: []
-extraHostPathMounts: []
+extraHostPathMounts:
+  - name: varlog
+    hostPath: /var/log
+    mountPath: /var/log
+    readOnly: true
+  - name: varlibdockercontainers
+    hostPath: /var/lib/docker/containers
+    mountPath: /var/lib/docker/containers
+    readOnly: true
 
 # Configuration for ports, shared between agentCollector, standaloneCollector and service.
 # Can be overridden here or for agentCollector and standaloneCollector independently.
@@ -102,31 +118,56 @@ ports:
     enabled: true
     containerPort: 4317
     servicePort: 4317
-    hostPort: 4317
     protocol: TCP
   jaeger-thrift:
     enabled: true
     containerPort: 14268
     servicePort: 14268
-    hostPort: 14268
     protocol: TCP
   jaeger-grpc:
     enabled: true
     containerPort: 14250
     servicePort: 14250
-    hostPort: 14250
     protocol: TCP
   zipkin:
     enabled: true
     containerPort: 9411
     servicePort: 9411
-    hostPort: 9411
     protocol: TCP
 
 # Configuration for agent OpenTelemetry Collector daemonset, enabled by default
 agentCollector:
   enabled: true
 
+  containerLogs:
+    enabled: true
+    # Container runtime. One of `docker`, `cri-o`, or `containerd`
+    containerRunTime: ""
+    includeAgentLogs: false
+    enrichK8sMetadata: true
+    listOfAnnotations: []
+    listOfLabels: []
+
+    # Exporters for container logs.
+    # https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter
+    # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter
+    exporters:
+      # # Example of sending logs to splunk
+      # splunk_hec:
+      #   token: "00000000-0000-0000-0000-0000000000000"
+      #   endpoint: "https://splunk:8088/services/collector"
+      #   source: "otel"
+      #   sourcetype: "otel"
+      #   index: "k8s_logs"
+      #   max_connections: 100
+      #   disable_compression: false
+      #   timeout: 10s
+      #   insecure_skip_verify: true
+      #   sending_queue:
+      #     enabled: true
+      #     num_consumers: 2
+      #     queue_size: 10
+
   resources:
     limits:
       cpu: 256m