diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 55d0a8b412c1..65dea2eeaccf 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -138,6 +138,7 @@ https://github.com/elastic/beats/compare/v6.6.0...6.x[Check the HEAD diff] - Add support for ssl_request_log in apache2 module. {issue}8088[8088] {pull}9833[9833] - Add support for iis 7.5 log format. {issue}9753[9753] {pull}9967[9967] - Add support for MariaDB in the `slowlog` fileset of `mysql` module. {pull}9731[9731] +- Add convert_timezone to nginx module. {issue}9839[9839] {pull}10148[10148] *Heartbeat* - Made monitors.d configuration part of the default config. {pull}9004[9004] diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 2b54cf209b0c..49a98e0f8ea9 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -304,6 +304,9 @@ filebeat.modules: # can be added under this section. #input: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + # Error logs #error: #enabled: true @@ -316,6 +319,9 @@ filebeat.modules: # can be added under this section. #input: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + #------------------------------- Osquery Module ------------------------------ - module: osquery result: diff --git a/filebeat/module/nginx/_meta/config.reference.yml b/filebeat/module/nginx/_meta/config.reference.yml index 572341217e65..94a8d84b2c74 100644 --- a/filebeat/module/nginx/_meta/config.reference.yml +++ b/filebeat/module/nginx/_meta/config.reference.yml @@ -11,6 +11,9 @@ # can be added under this section. #input: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + # Error logs #error: #enabled: true @@ -22,3 +25,6 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false diff --git a/filebeat/module/nginx/_meta/config.yml b/filebeat/module/nginx/_meta/config.yml index 22fe90e486e4..e117f33ecb50 100644 --- a/filebeat/module/nginx/_meta/config.yml +++ b/filebeat/module/nginx/_meta/config.yml @@ -7,6 +7,9 @@ # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: true + # Error logs error: enabled: true @@ -14,3 +17,6 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: + + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: true diff --git a/filebeat/module/nginx/access/config/nginx-access.yml b/filebeat/module/nginx/access/config/nginx-access.yml index 0afd17317d4f..4e6434053401 100644 --- a/filebeat/module/nginx/access/config/nginx-access.yml +++ b/filebeat/module/nginx/access/config/nginx-access.yml @@ -4,3 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] + +{{ if .convert_timezone }} +processors: +- add_locale: ~ +{{ end }} diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json index ce0a9b7de954..37588178fabf 100644 --- a/filebeat/module/nginx/access/ingest/default.json +++ b/filebeat/module/nginx/access/ingest/default.json @@ -60,7 +60,9 @@ "target_field": "@timestamp", "formats": [ "dd/MMM/YYYY:H:m:s Z" - ] + ], + {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >} + "ignore_failure": true } }, { diff --git a/filebeat/module/nginx/access/manifest.yml b/filebeat/module/nginx/access/manifest.yml index a0fede4ed51d..6ff64f6496f9 100644 --- a/filebeat/module/nginx/access/manifest.yml +++ b/filebeat/module/nginx/access/manifest.yml @@ -8,6 +8,13 @@ var: - /usr/local/var/log/nginx/access.log* os.windows: - c:/programdata/nginx/logs/*access.log* + - name: convert_timezone + default: false + # if ES < 6.1.0, this flag switches to false automatically when evaluating the + # pipeline + min_elasticsearch_version: + version: 6.1.0 + value: false ingest_pipeline: ingest/default.json input: config/nginx-access.yml diff --git a/filebeat/module/nginx/error/config/nginx-error.yml b/filebeat/module/nginx/error/config/nginx-error.yml index 0afd17317d4f..4e6434053401 100644 --- a/filebeat/module/nginx/error/config/nginx-error.yml +++ b/filebeat/module/nginx/error/config/nginx-error.yml @@ -4,3 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] + +{{ if .convert_timezone }} +processors: +- add_locale: ~ +{{ end }} diff --git a/filebeat/module/nginx/error/ingest/pipeline.json b/filebeat/module/nginx/error/ingest/pipeline.json index f8231e046077..c7ec636d9e15 100644 --- a/filebeat/module/nginx/error/ingest/pipeline.json +++ b/filebeat/module/nginx/error/ingest/pipeline.json @@ -21,7 +21,9 @@ "date": { "field": "nginx.error.time", "target_field": "@timestamp", - "formats": ["YYYY/MM/dd H:m:s"] + "formats": ["YYYY/MM/dd H:m:s"], + {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >} + "ignore_failure": true } }, { "remove": { diff --git a/filebeat/module/nginx/error/manifest.yml b/filebeat/module/nginx/error/manifest.yml index 641ec771bbb3..f050bfaad9a3 100644 --- a/filebeat/module/nginx/error/manifest.yml +++ b/filebeat/module/nginx/error/manifest.yml @@ -8,6 +8,13 @@ var: - /usr/local/var/log/nginx/error.log* os.windows: - c:/programdata/nginx/logs/error.log* + - name: convert_timezone + default: false + # if ES < 6.1.0, this flag switches to false automatically when evaluating the + # pipeline + min_elasticsearch_version: + version: 6.1.0 + value: false ingest_pipeline: ingest/pipeline.json input: config/nginx-error.yml diff --git a/filebeat/modules.d/nginx.yml.disabled b/filebeat/modules.d/nginx.yml.disabled index 22fe90e486e4..e117f33ecb50 100644 --- a/filebeat/modules.d/nginx.yml.disabled +++ b/filebeat/modules.d/nginx.yml.disabled @@ -7,6 +7,9 @@ # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: true + # Error logs error: enabled: true @@ -14,3 +17,6 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: + + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: true diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 3b4741a89d05..bb06ef6e6d07 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -304,6 +304,9 @@ filebeat.modules: # can be added under this section. #input: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + # Error logs #error: #enabled: true @@ -316,6 +319,9 @@ filebeat.modules: # can be added under this section. #input: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + #-------------------------------- Osquery Module -------------------------------- - module: osquery result: