Merge pull request #15 from observIQ/file-fields

Remove file_name_field and file_path_field

plugins/kafka.yaml

@@ -40,7 +40,6 @@ pipeline:
 {{ if .log_cleaner_log_path }}      - {{ .log_cleaner_log_path}}{{ end }}
     multiline:
       line_start_pattern: '\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\]'
-    file_name_field: log_file_name
     write_to: log_entry
     start_at: {{ or .start_at "end" }}
 
@@ -59,19 +58,19 @@ pipeline:
   - id: filename_router
     type: router
     routes:
-      - expr: '$record.log_file_name != nil and $record.log_file_name matches "^server"'
+      - expr: '$labels.file_name != nil and $labels.file_name matches "^server"'
         output: timestamp_restructurer
         labels:
           log_type: 'kafka.server'
-      - expr: '$record.log_file_name != nil and $record.log_file_name matches "^controller"'
+      - expr: '$labels.file_name != nil and $labels.file_name matches "^controller"'
         output: timestamp_restructurer
         labels:
           log_type: 'kafka.controller'
-      - expr: '$record.log_file_name != nil and $record.log_file_name matches "^state-change"'
+      - expr: '$labels.file_name != nil and $labels.file_name matches "^state-change"'
         output: timestamp_restructurer
         labels:
           log_type: 'kafka.state_change'
-      - expr: '$record.log_file_name != nil and $record.log_file_name matches "^log-cleaner"'
+      - expr: '$labels.file_name != nil and $labels.file_name matches "^log-cleaner"'
         output: timestamp_restructurer
         labels:
           log_type: 'kafka.log_cleaner'

plugins/kubernetes.yaml

@@ -26,27 +26,26 @@ pipeline:
     include:
       - {{ .container_log_path }}
     start_at: {{ or .start_at "end" }}
-    file_path_field: log_type
-    file_name_field: file_name
+    include_file_path: true
     write_to: log
 
   # Filter carbon logs. Check if file_name field starts with carbon. if it does drop the log entry otherwise continue down pipeline
   - id: filename_router
     type: router
     routes:
-      - expr: '$record.file_name != nil and $record.file_name matches "^carbon"'
+      - expr: '$labels.file_name != nil and $labels.file_name matches "^carbon"'
         output: drop_output
       - expr: true
         output: remove_file_name
 
   # Drop unwanted logs
   - type: "drop_output"
 
-  # Remove file_name field. We have filtered carbon logs and no longer need file_name field
+  # Remove file_name label. We have filtered carbon logs and no longer need the file name
   - id: remove_file_name
     type: restructure
     ops:
-      - remove: file_name
+      - remove: $labels.file_name
 
   # Initial log entry should be safe to parse as JSON
   - id: container_json_parser
@@ -67,10 +66,10 @@ pipeline:
       - output: nested_json_parser
         expr: $record.log matches '^{.*}$'
       # If log field doesn't appear to be JSON then, skip nested JSON parsers
-      - output: container_regex_parser 
+      - output: container_regex_parser
         expr: true
 
-  # Remove new line from end of jsonUnable to parse json if it has a newline. 
+  # Remove new line from end of jsonUnable to parse json if it has a newline.
   - id: remove_new_line
     type: regex_parser
     parse_from: $record.log
@@ -92,7 +91,7 @@ pipeline:
   # Log field has been parsed if possible and now we can parse log_type field for container information.
   - id: container_regex_parser
     type: regex_parser
-    parse_from: log_type
+    parse_from: $labels.file_path
     regex: '\/var\/log\/containers\/(?P<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?P<namespace>[^_]+)_(?P<container_name>.+)-(?P<container_id>[a-z0-9]{64})\.log'
     severity:
       parse_from: stream