From fbfe8b085b6218fa3d78ed1a382b5f5e1cf1f0bb Mon Sep 17 00:00:00 2001 From: Joshua Williams Date: Mon, 10 Aug 2020 12:05:37 -0400 Subject: [PATCH] Added gitlab --- plugins/gitlab.yaml | 191 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 plugins/gitlab.yaml diff --git a/plugins/gitlab.yaml b/plugins/gitlab.yaml new file mode 100644 index 00000000..a2bd4a64 --- /dev/null +++ b/plugins/gitlab.yaml @@ -0,0 +1,191 @@ +# Plugin Info +version: 0.0.1 +title: GitLab +description: Log parser for GitLab +parameters: + enable_rails_application_log: + label: Rails Application Logs + description: Enable to collect GitLab rails application logs + type: bool + default: true + rails_application_log_path: + label: Rails Application Logs Path + description: The absolute path to the GitLab rails application logs + type: string + default: "/var/log/gitlab/gitlab-rails/application.log" + enable_rails_production_log: + label: Rails Production Logs + description: Enable to collect GitLab rails production logs + type: bool + default: true + rails_production_log_path: + label: Rails Production Logs Path + description: The absolute path to the GitLab rails production logs + type: string + default: "/var/log/gitlab/gitlab-rails/production_json.log" + enable_rails_api_request_log: + label: Rails API Request Logs + description: Enable to collect GitLab rails API request logs + type: bool + default: true + rails_api_request_log_path: + label: Rails API Request Logs Path + description: The absolute path to the GitLab rails API Request logs + type: string + default: "/var/log/gitlab/gitlab-rails/api_json.log" + enable_rails_sidekiq_log: + label: Rails Sidekiq Logs + description: Enable to collect GitLab rails Sidekiq logs + type: bool + default: true + rails_sidekiq_log_path: + label: Rails Sidekiq Logs Path + description: The absolute path to the GitLab rails Sidekiq logs + type: string + default: "/var/log/gitlab/gitlab-rails/sidekiq.log" + enable_shell_log: + label: Shell Logs + description: Enable to collect GitLab shell logs + type: bool + default: true + rails_shell_path: + label: Shell Logs Path + description: The absolute path to the GitLab shell logs + type: string + default: "/var/log/gitlab/gitlab-shell/gitlab-shell.log" + start_at: + label: Start At + description: Start reading file from 'beginning' or 'end' + type: enum + valid_values: + - beginning + - end + default: end + +# Set Defaults +{{$enable_rails_application_log := default true .enable_rails_application_log}} +{{$rails_application_log_path := default "/var/log/gitlab/gitlab-rails/application.log" .rails_application_log_path}} +{{$enable_rails_production_log := default true .enable_rails_production_log}} +{{$rails_production_log_path := default "/var/log/gitlab/gitlab-rails/production_json.log" .rails_production_log_path}} +{{$enable_rails_api_request_log := default true .enable_rails_api_request_log}} +{{$rails_api_request_log_path := default "/var/log/gitlab/gitlab-rails/api_json.log" .rails_api_request_log_path}} +{{$enable_rails_sidekiq_log := default true .enable_rails_sidekiq_log}} +{{$rails_sidekiq_log_path := default "/var/log/gitlab/gitlab-rails/sidekiq.log" .rails_sidekiq_log_path}} +{{$enable_shell_log := default true .enable_shell_log}} +{{$rails_shell_path := default "/var/log/gitlab/gitlab-shell/gitlab-shell.log" .rails_shell_path}} +{{$start_at := default "end" .start_at}} + +# Pipeline Template +pipeline: +{{ if $enable_rails_application_log }} + - id: gitlab_rails_application_input + type: file_input + include: + - {{ $rails_application_log_path }} + start_at: {{ $start_at }} + labels: + log_type: gitlab.rails.application + output: gitlab_rails_application_parser + + - id: gitlab_rails_application_parser + type: regex_parser + regex: '^(?P\w*\s\d*,\s\d*\s\d*:\d*)\:\s?(?P.*)' + timestamp: + parse_from: timestamp + layout: '%B %d, %Y %H:%M' + output: {{.output}} +{{ end }} + +{{ if $enable_rails_production_log }} + - id: gitlab_rails_production_input + type: file_input + include: + - {{ $rails_production_log_path }} + start_at: {{ $start_at }} + labels: + log_type: gitlab.rails.production + output: gitlab_rails_production_parser + + - id: gitlab_rails_production_parser + type: json_parser + timestamp: + parse_from: time + layout: '%Y-%m-%dT%H:%M:%S.%sZ' + output: {{.output}} +{{ end }} + +{{ if $enable_rails_api_request_log }} + - id: gitlab_rails_api_request_input + type: file_input + include: + - {{ $rails_api_request_log_path }} + start_at: {{ $start_at }} + labels: + log_type: gitlab.rails.api_request + output: gitlab_rails_api_request_parser + + - id: gitlab_rails_api_request_parser + type: json_parser + timestamp: + parse_from: time + layout: '%Y-%m-%dT%H:%M:%S.%LZ' + severity: + parse_from: severity + preserve: true + mapping: + emergency: emerg + error: err + critical: crit + warning: warn + output: {{.output}} +{{ end }} + +{{ if $enable_rails_sidekiq_log }} + - id: gitlab_rails_sidekiq_input + type: file_input + include: + - {{ $rails_sidekiq_log_path }} + start_at: {{ $start_at }} + labels: + log_type: gitlab.rails.sidekiq + output: gitlab_rails_sidekiq_parser + + - id: gitlab_rails_sidekiq_parser + type: json_parser + timestamp: + parse_from: time + layout: '%Y-%m-%dT%H:%M:%S.%sZ' + severity: + parse_from: severity + preserve: true + mapping: + emergency: emerg + error: err + critical: crit + warning: warn + output: {{.output}} +{{ end }} + +{{ if $enable_shell_log }} + - id: gitlab_shell_input + type: file_input + include: + - {{ $rails_shell_path }} + start_at: {{ $start_at }} + labels: + log_type: gitlab.shell + output: gitlab_shell_parser + + - id: gitlab_shell_parser + type: json_parser + timestamp: + parse_from: time + layout: '%Y-%m-%dT%H:%M:%S%j' + severity: + parse_from: level + preserve: true + mapping: + error: err + warning: warn + output: {{.output}} +{{ end }}