feat: Update fluent bit config (#42)

* feat: Update fluent bit config * Parse json logs
cloudquery · Jun 24, 2022 · 532fd27 · 532fd27
1 parent da03901
commit 532fd27
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 56 deletions.
diff --git a/charts/cloudquery/Chart.yaml b/charts/cloudquery/Chart.yaml
@@ -17,7 +17,7 @@ maintainers:
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.6
+version: 0.2.7
 
 # -- This is the version number of the application being deployed.This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/cloudquery/README.md b/charts/cloudquery/README.md
@@ -1,6 +1,6 @@
 # cloudquery
 
-![Version: 0.2.6](https://img.shields.io/badge/Version-0.2.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.24](https://img.shields.io/badge/AppVersion-0.24-informational?style=flat-square)
+![Version: 0.2.7](https://img.shields.io/badge/Version-0.2.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.24](https://img.shields.io/badge/AppVersion-0.24-informational?style=flat-square)
 
 The open-source cloud asset inventory powered by SQL.
 

diff --git a/charts/cloudquery/templates/aws-cloudwatch.yaml b/charts/cloudquery/templates/aws-cloudwatch.yaml
@@ -6,4 +6,4 @@ metadata:
   name: amazon-cloudwatch
   labels:
     name: amazon-cloudwatch
-{{- end }}
+{{- end }}
diff --git a/charts/cloudquery/templates/configmap-fluentbit.yaml b/charts/cloudquery/templates/configmap-fluentbit.yaml
@@ -13,4 +13,4 @@ data:
   read.head: "Off"
   read.tail: "On"
   logs.region: {{ .Values.cloudwatch.region }}
-{{- end }}
+{{- end }}
diff --git a/charts/cloudquery/templates/cronjob.yaml b/charts/cloudquery/templates/cronjob.yaml
@@ -25,7 +25,7 @@ spec:
                   name: {{ include "cloudquery.fullname" . }}-secret
               image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
               imagePullPolicy: Always
-              args: ["fetch", "--config", "/app/config/config.hcl", "--enable-console-log"]
+              args: ["fetch", "--config", "/app/config/config.hcl", "--enable-console-log", "--encode-json"]
               resources:
               {{- toYaml .Values.resources | nindent 16 }}
               volumeMounts:

diff --git a/...query/templates/daemonset-fluentdbit.yaml → ...dquery/templates/daemonset-fluentbit.yaml b/...query/templates/daemonset-fluentdbit.yaml → ...dquery/templates/daemonset-fluentbit.yaml
@@ -5,7 +5,7 @@ metadata:
   name: fluent-bit
   namespace: amazon-cloudwatch
   annotations:
-    "eks.amazonaws.com/role-arn": {{ .Values.cloudwatch.role_arn }}
+    eks.amazonaws.com/role-arn: {{ .Values.cloudwatch.role_arn }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -21,6 +21,8 @@ rules:
       - namespaces
       - pods
       - pods/logs
+      - nodes
+      - nodes/proxy
     verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -57,11 +59,11 @@ data:
         storage.sync              normal
         storage.checksum          off
         storage.backlog.mem_limit 5M
-        
+
     @INCLUDE application-log.conf
     @INCLUDE dataplane-log.conf
     @INCLUDE host-log.conf
-  
+
   application-log.conf: |
     [INPUT]
         Name                tail
@@ -71,7 +73,7 @@ data:
         Docker_Mode         On
         Docker_Mode_Flush   5
         Docker_Mode_Parser  container_firstline
-        Parser              docker
+        Parser              crio
         DB                  /var/fluent-bit/state/flb_container.db
         Mem_Buf_Limit       50MB
         Skip_Long_Lines     On
@@ -116,6 +118,9 @@ data:
         K8S-Logging.Exclude Off
         Labels              Off
         Annotations         Off
+        Use_Kubelet         On
+        Kubelet_Port        10250
+        Buffer_Size         0
 
     [OUTPUT]
         Name                cloudwatch_logs
@@ -173,7 +178,7 @@ data:
         log_stream_prefix   ${HOST_NAME}-
         auto_create_group   true
         extra_user_agent    container-insights
-    
+
   host-log.conf: |
     [INPUT]
         Name                tail
@@ -223,6 +228,13 @@ data:
         extra_user_agent    container-insights
 
   parsers.conf: |
+    [PARSER]
+        Name                crio
+        Format              Regex
+        Regex               ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>P|F) (?<log>.*)$
+        Time_Key            time
+        Time_Format         %Y-%m-%dT%H:%M:%S.%L%z
+
     [PARSER]
         Name                docker
         Format              json
@@ -271,10 +283,10 @@ spec:
         kubernetes.io/cluster-service: "true"
     spec:
       containers:
-      - name: fluent-bit
-        image: amazon/aws-for-fluent-bit:2.10.0
-        imagePullPolicy: Always
-        env:
+        - name: fluent-bit
+          image: public.ecr.aws/aws-observability/aws-for-fluent-bit:stable
+          imagePullPolicy: Always
+          env:
             - name: AWS_REGION
               valueFrom:
                 configMapKeyRef:
@@ -309,59 +321,66 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            - name: HOSTNAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
             - name: CI_VERSION
-              value: "k8s/1.3.9"
-        resources:
+              value: "k8s/1.3.10"
+          resources:
             limits:
               memory: 200Mi
             requests:
               cpu: 500m
               memory: 100Mi
-        volumeMounts:
-        # Please don't change below read-only permissions
+          volumeMounts:
+            # Please don't change below read-only permissions
+            - name: fluentbitstate
+              mountPath: /var/fluent-bit/state
+            - name: varlog
+              mountPath: /var/log
+              readOnly: true
+            - name: varlibdockercontainers
+              mountPath: /var/lib/docker/containers
+              readOnly: true
+            - name: fluent-bit-config
+              mountPath: /fluent-bit/etc/
+            - name: runlogjournal
+              mountPath: /run/log/journal
+              readOnly: true
+            - name: dmesg
+              mountPath: /var/log/dmesg
+              readOnly: true
+      terminationGracePeriodSeconds: 10
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      volumes:
         - name: fluentbitstate
-          mountPath: /var/fluent-bit/state
+          hostPath:
+            path: /var/fluent-bit/state
         - name: varlog
-          mountPath: /var/log
-          readOnly: true
+          hostPath:
+            path: /var/log
         - name: varlibdockercontainers
-          mountPath: /var/lib/docker/containers
-          readOnly: true
+          hostPath:
+            path: /var/lib/docker/containers
         - name: fluent-bit-config
-          mountPath: /fluent-bit/etc/
+          configMap:
+            name: fluent-bit-config
         - name: runlogjournal
-          mountPath: /run/log/journal
-          readOnly: true
+          hostPath:
+            path: /run/log/journal
         - name: dmesg
-          mountPath: /var/log/dmesg
-          readOnly: true
-      terminationGracePeriodSeconds: 10
-      volumes:
-      - name: fluentbitstate
-        hostPath:
-          path: /var/fluent-bit/state
-      - name: varlog
-        hostPath:
-          path: /var/log
-      - name: varlibdockercontainers
-        hostPath:
-          path: /var/lib/docker/containers
-      - name: fluent-bit-config
-        configMap:
-          name: fluent-bit-config
-      - name: runlogjournal
-        hostPath:
-          path: /run/log/journal
-      - name: dmesg
-        hostPath:
-          path: /var/log/dmesg
+          hostPath:
+            path: /var/log/dmesg
       serviceAccountName: fluent-bit
       tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      - operator: "Exists"
-        effect: "NoExecute"
-      - operator: "Exists"
-        effect: "NoSchedule"
-{{- end }}
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+        - operator: "Exists"
+          effect: "NoExecute"
+        - operator: "Exists"
+          effect: "NoSchedule"
+{{- end }}