diff --git a/plugins/ibm_db2.yaml b/plugins/ibm_db2.yaml index 8e63ecd6..1d772187 100644 --- a/plugins/ibm_db2.yaml +++ b/plugins/ibm_db2.yaml @@ -6,10 +6,15 @@ parameters: label: Log Path description: Path to the log file type: string + required: true start_at: label: Start At description: "Start reading file from 'beginning' or 'end'" - type: string + type: enum + valid_values: + - beginning + - end + default: end pipeline: - id: ibm_db2_reader type: file_input diff --git a/plugins/kubernetes.yaml b/plugins/kubernetes.yaml index 7fb1010c..f5642f80 100644 --- a/plugins/kubernetes.yaml +++ b/plugins/kubernetes.yaml @@ -1,186 +1,186 @@ - version: 0.0.5 - title: Kubernetes - description: Log parser for Kubernetes - parameters: - container_log_path: - label: Containers Log Path - description: Kubernetes Containers Log Path - type: string - kubelet_journald_log_path: - label: Kublet Journald Log Path - description: 'Kubernetes Kublet Journald Log path. It will read from /run/journal or /var/log/journal if this parameter is omitted' - type: string - start_at: - label: Start At - description: "Start reading file from 'beginning' or 'end'" - type: string - pipeline: - # {{ if .container_log_path }} - - id: container_reader - type: file_input - include: - - {{ .container_log_path }} - # {{ if .start_at }} - start_at: {{ .start_at }} - # {{ end }} - file_path_field: log_name - write_to: log - output: container_json_parser - - - id: container_json_parser - type: json_parser - parse_from: log - output: nested_json_router - - - id: nested_json_router - type: router - routes: - - output: nested_json_parser - expr: $record.log matches '^{.*}$' - - output: container_regex_parser - expr: true - - - id: nested_json_parser - type: json_parser - parse_from: $.log - output: container_regex_parser - - - id: container_regex_parser - type: regex_parser - parse_from: log_name - regex: '\/var\/log\/containers\/(?P[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?P[^_]+)_(?P.+)-(?P[a-z0-9]{64})\.log' - severity: - parse_from: stream - preserve: true - mapping: - error: - - stderr - info: - - stdout - timestamp: - parse_from: time - layout: '%Y-%m-%dT%H:%M:%S.%sZ' - output: log_parse_router - - - id: log_parse_router - type: router - routes: - - output: standard_regex_parser - expr: '$record.log matches "^\\w\\d{4}"' - - output: add_kubernetes_metadata - expr: true - - - id: standard_regex_parser - type: regex_parser - parse_from: log - regex: '(?P\w)(?P\d{4} \d{2}:\d{2}:\d{2}.\d+)\s+(?P\d+)\s+(?P[^ \]]+)\] (?P.*)' - severity: - parse_from: severity - mapping: - debug: - - d - info: - - i - warning: - - w - error: - - e - critical: - - c - timestamp: - parse_from: timestamp - layout: '%m%d %H:%M:%S.%s' - output: add_kubernetes_metadata - - - id: add_kubernetes_metadata - type: k8s_metadata_decorator - output: add_labels_router - - - id: add_labels_router - type: router - routes: - - output: add_kube_controller_metadata - expr: '$labels["k8s_pod_label/component"] == "kube-controller-manager"' - - output: add_kube_scheduler_metadata - expr: '$labels["k8s_pod_label/component"] == "kube-scheduler"' - - output: add_kube_apiserver_metadata - expr: '$labels["k8s_pod_label/component"] == "kube-apiserver"' - - output: add_kube_proxy_metadata - expr: '$labels["k8s_pod_label/component"] startsWith "kube-proxy"' - - output: add_container_metadata - expr: true - - - id: add_kube_controller_metadata - type: metadata - labels: - log_name: 'kubernetes.controller' - output: {{ .output }} - - - id: add_kube_scheduler_metadata - type: metadata - labels: - log_name: 'kubernetes.scheduler' - output: {{ .output }} - - - id: add_kube_apiserver_metadata - type: metadata - labels: - log_name: 'kubernetes.apiserver' - output: {{ .output }} - - - id: add_kube_proxy_metadata - type: metadata - labels: - log_name: 'kubernetes.proxy' - output: {{ .output }} - - - id: add_container_metadata - type: metadata - labels: - log_name: 'kubernetes.container' - output: {{ .output }} - # {{ end }} - - - id: kubelet_reader - type: journald_input - # {{ if .kubelet_journald_log_path }} - directory: {{ .kubelet_journald_log_path }} - # {{ end }} - output: kubelet_filter_router - - - id: kubelet_filter_router - type: router - routes: - - output: kubelet_message_parser_router - expr: '$record._SYSTEMD_UNIT == "kubelet.service"' - - - id: kubelet_message_parser_router - type: router - routes: - - output: message_regex_parser - expr: '$record.MESSAGE matches "^\\w\\d{4}"' - - output: add_kublet_metadata - expr: true - - - id: message_regex_parser - type: regex_parser - parse_from: MESSAGE - regex: '(?P\w)(?P\d{4} \d{2}:\d{2}:\d{2}.\d+)\s+(?P\d+)\s+(?P[^ \]]+)\] (?P.*)' - severity: - parse_from: severity - mapping: - debug: d - info: i - warning: w - error: e - critical: c - timestamp: - parse_from: timestamp - layout: '%m%d %H:%M:%S.%s' - output: add_kublet_metadata - - - id: add_kublet_metadata - type: metadata - labels: - log_name: 'kubernetes.kubelet' - output: {{ .output }} +version: 0.0.5 +title: Kubernetes +description: Log parser for Kubernetes +parameters: + container_log_path: + label: Containers Log Path + description: Kubernetes Containers Log Path + type: string + required: true + kubelet_journald_log_path: + label: Kublet Journald Log Path + description: 'Kubernetes Kublet Journald Log path. It will read from /run/journal or /var/log/journal if this parameter is omitted' + type: string + start_at: + label: Start At + description: "Start reading file from 'beginning' or 'end'" + type: enum + valid_values: + - beginning + - end + default: end +pipeline: + # {{ if .container_log_path }} + - id: container_reader + type: file_input + include: + - {{ .container_log_path }} + # {{ if .start_at }} + start_at: {{ .start_at }} + # {{ end }} + file_path_field: log_name + write_to: log + output: container_json_parser + + - id: container_json_parser + type: json_parser + parse_from: log + output: nested_json_router + + - id: nested_json_router + type: router + routes: + - output: nested_json_parser + expr: $record.log matches '^{.*}$' + - output: container_regex_parser + expr: true + + - id: nested_json_parser + type: json_parser + parse_from: $.log + output: container_regex_parser + + - id: container_regex_parser + type: regex_parser + parse_from: log_name + regex: '\/var\/log\/containers\/(?P[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?P[^_]+)_(?P.+)-(?P[a-z0-9]{64})\.log' + severity: + parse_from: stream + preserve: true + mapping: + error: + - stderr + info: + - stdout + timestamp: + parse_from: time + layout: '%Y-%m-%dT%H:%M:%S.%sZ' + output: log_parse_router + + - id: log_parse_router + type: router + routes: + - output: standard_regex_parser + expr: '$record.log matches "^\\w\\d{4}"' + - output: add_kubernetes_metadata + expr: true + + - id: standard_regex_parser + type: regex_parser + parse_from: log + regex: '(?P\w)(?P\d{4} \d{2}:\d{2}:\d{2}.\d+)\s+(?P\d+)\s+(?P[^ \]]+)\] (?P.*)' + severity: + parse_from: severity + mapping: + debug: d + info: i + warning: w + error: e + critical: c + timestamp: + parse_from: timestamp + layout: '%m%d %H:%M:%S.%s' + output: add_kubernetes_metadata + + - id: add_kubernetes_metadata + type: k8s_metadata_decorator + output: add_labels_router + + - id: add_labels_router + type: router + routes: + - output: add_kube_controller_metadata + expr: '$labels["k8s_pod_label/component"] == "kube-controller-manager"' + - output: add_kube_scheduler_metadata + expr: '$labels["k8s_pod_label/component"] == "kube-scheduler"' + - output: add_kube_apiserver_metadata + expr: '$labels["k8s_pod_label/component"] == "kube-apiserver"' + - output: add_kube_proxy_metadata + expr: '$labels["k8s_pod_label/component"] startsWith "kube-proxy"' + - output: add_container_metadata + expr: true + + - id: add_kube_controller_metadata + type: metadata + labels: + log_name: 'kubernetes.controller' + output: {{ .output }} + + - id: add_kube_scheduler_metadata + type: metadata + labels: + log_name: 'kubernetes.scheduler' + output: {{ .output }} + + - id: add_kube_apiserver_metadata + type: metadata + labels: + log_name: 'kubernetes.apiserver' + output: {{ .output }} + + - id: add_kube_proxy_metadata + type: metadata + labels: + log_name: 'kubernetes.proxy' + output: {{ .output }} + + - id: add_container_metadata + type: metadata + labels: + log_name: 'kubernetes.container' + output: {{ .output }} + # {{ end }} + + - id: kubelet_reader + type: journald_input + # {{ if .kubelet_journald_log_path }} + directory: {{ .kubelet_journald_log_path }} + # {{ end }} + output: kubelet_filter_router + + - id: kubelet_filter_router + type: router + routes: + - output: kubelet_message_parser_router + expr: '$record._SYSTEMD_UNIT == "kubelet.service"' + + - id: kubelet_message_parser_router + type: router + routes: + - output: message_regex_parser + expr: '$record.MESSAGE matches "^\\w\\d{4}"' + - output: add_kublet_metadata + expr: true + + - id: message_regex_parser + type: regex_parser + parse_from: MESSAGE + regex: '(?P\w)(?P\d{4} \d{2}:\d{2}:\d{2}.\d+)\s+(?P\d+)\s+(?P[^ \]]+)\] (?P.*)' + severity: + parse_from: severity + mapping: + debug: d + info: i + warning: w + error: e + critical: c + timestamp: + parse_from: timestamp + layout: '%m%d %H:%M:%S.%s' + output: add_kublet_metadata + + - id: add_kublet_metadata + type: metadata + labels: + log_name: 'kubernetes.kubelet' + output: {{ .output }} diff --git a/plugins/mongodb.yaml b/plugins/mongodb.yaml index fd2579d6..7eb7b99f 100644 --- a/plugins/mongodb.yaml +++ b/plugins/mongodb.yaml @@ -6,10 +6,15 @@ parameters: label: Path description: The path of the log file type: string + required: true start_at: label: Start At description: Start reading file from 'beginning' or 'end' - type: string + type: enum + valid_values: + - beginning + - end + default: end pipeline: - id: file_input type: file_input @@ -30,14 +35,10 @@ pipeline: severity: parse_from: severity mapping: - critical: - - F - error: - - E - warning: - - W - info: - - I + critical: F + error: E + warning: W + info: I debug: - D - D1 diff --git a/plugins/mysql.yaml b/plugins/mysql.yaml index 49c53ad3..fc5a1bf2 100644 --- a/plugins/mysql.yaml +++ b/plugins/mysql.yaml @@ -17,7 +17,11 @@ parameters: start_at: label: Start At description: Start reading file from 'beginning' or 'end' - type: string + type: enum + valid_values: + - beginning + - end + default: end pipeline: # {{ if .slow_query_log_path }} - id: slow_query_reader diff --git a/plugins/openshift.yaml b/plugins/openshift.yaml index 3988aef9..c813d4a0 100644 --- a/plugins/openshift.yaml +++ b/plugins/openshift.yaml @@ -6,10 +6,15 @@ parameters: label: Containers Log Path description: The directory where the journald logs are located. Defaults to '/var/log/journal'. type: string + default: /var/log/journal start_at: label: Start At description: Start reading file from 'beginning' or 'end'. Defaults to 'end'. - type: string + type: enum + valid_values: + - beginning + - end + default: end container_log: label: Enable Container Logs description: Enable collection of container logs diff --git a/plugins/oracledb.yaml b/plugins/oracledb.yaml index d30ba44e..547e2329 100644 --- a/plugins/oracledb.yaml +++ b/plugins/oracledb.yaml @@ -9,7 +9,7 @@ parameters: truncate_audit_action: label: Truncate Audit Action description: Whether or not to truncate the audit log action field - type: boolean + type: bool alert_log_path: label: Alert Log Path description: Path to the alert log file @@ -21,7 +21,11 @@ parameters: start_at: label: Start At description: Start reading file from 'beginning' or 'end' - type: string + type: enum + valid_values: + - beginning + - end + default: end pipeline: # {{ if .audit_log_path }} - id: audit_reader diff --git a/plugins/postgresql.yaml b/plugins/postgresql.yaml index 5c30b2e2..eb47f4cc 100644 --- a/plugins/postgresql.yaml +++ b/plugins/postgresql.yaml @@ -19,10 +19,15 @@ parameters: label: PostgreSQL Log Path description: Path to the PostgreSQL log file type: string + required: true start_at: label: Start At description: Start reading file from 'beginning' or 'end' - type: string + type: enum + valid_values: + - beginning + - end + default: end pipeline: - type: file_input include: diff --git a/plugins/tomcat.yaml b/plugins/tomcat.yaml index dfe51b73..813934a9 100644 --- a/plugins/tomcat.yaml +++ b/plugins/tomcat.yaml @@ -13,7 +13,11 @@ parameters: start_at: label: Start At description: Start reading file from 'beginning' or 'end' - type: string + type: enum + valid_values: + - beginning + - end + default: end pipeline: #{{ if .access_log_path }} - id: tomcat_access_reader