From 74fa8e39ac6e4b0cb615c4f90f9d57bcc5925478 Mon Sep 17 00:00:00 2001 From: beats-jenkins Date: Mon, 28 Jan 2019 14:49:41 +0100 Subject: [PATCH 1/5] [Filebeat] List of text fields to be converted This PR is for discussing the current text fields in Filebeat and if the type should be changed. --- filebeat/module/elasticsearch/slowlog/_meta/fields.yml | 6 +++++- filebeat/module/haproxy/_meta/fields.yml | 7 +++++-- filebeat/module/haproxy/log/_meta/fields.yml | 3 +++ filebeat/module/kafka/log/_meta/fields.yml | 5 ++++- filebeat/module/logstash/log/_meta/fields.yml | 2 +- filebeat/module/logstash/slowlog/_meta/fields.yml | 4 ++++ filebeat/module/traefik/access/_meta/fields.yml | 2 ++ 7 files changed, 24 insertions(+), 5 deletions(-) diff --git a/filebeat/module/elasticsearch/slowlog/_meta/fields.yml b/filebeat/module/elasticsearch/slowlog/_meta/fields.yml index 77a3f9e0ffd0..b0931cdf61b4 100644 --- a/filebeat/module/elasticsearch/slowlog/_meta/fields.yml +++ b/filebeat/module/elasticsearch/slowlog/_meta/fields.yml @@ -10,7 +10,8 @@ - name: took description: "Time it took to execute the query" example: "300ms" - type: text + # Should we extrac that nicer? + type: keyword - name: types description: "Types" example: "" @@ -18,6 +19,7 @@ - name: stats description: "Statistics" example: "" + # What does this exactly contain, do we need text? type: text - name: search_type description: "Search type" @@ -26,10 +28,12 @@ - name: source_query description: "Slow query" example: "{\"query\":{\"match_all\":{\"boost\":1.0}}}" + # Do we need text? type: text - name: extra_source description: "Extra source information" example: "" + # Do we need text? type: text - name: total_hits description: "Total hits" diff --git a/filebeat/module/haproxy/_meta/fields.yml b/filebeat/module/haproxy/_meta/fields.yml index 674d20f29be8..a2930c623bf3 100644 --- a/filebeat/module/haproxy/_meta/fields.yml +++ b/filebeat/module/haproxy/_meta/fields.yml @@ -50,17 +50,20 @@ - name: error_message description: Error message logged by HAProxy in case of error. + # Text needed? + # Should it map to ECS? type: text - name: source - type: text + # What kind of source is this? + type: keyword description: The HAProxy source of the log - name: termination_state description: Condition the session was in when the session ended. - name: mode - type: text + type: keyword description: mode that the frontend is operating (TCP or HTTP) - name: connections diff --git a/filebeat/module/haproxy/log/_meta/fields.yml b/filebeat/module/haproxy/log/_meta/fields.yml index 9176a8ae7282..7e9b5596ca9f 100644 --- a/filebeat/module/haproxy/log/_meta/fields.yml +++ b/filebeat/module/haproxy/log/_meta/fields.yml @@ -14,6 +14,7 @@ - name: captured_headers description: > List of headers captured in the response due to the presence of the "capture response header" statement in the frontend. + # Do we need text? type: text - name: status_code @@ -32,10 +33,12 @@ - name: captured_headers description: > List of headers captured in the request due to the presence of the "capture request header" statement in the frontend. + # Do we need text? does this even have be be indexed? type: text - name: raw_request_line description: Complete HTTP request line, including the method, request and HTTP version string. + # keyword? sounds like a .original field but none is matching type: text - name: time_wait_without_data_ms diff --git a/filebeat/module/kafka/log/_meta/fields.yml b/filebeat/module/kafka/log/_meta/fields.yml index 76d9467bb044..b0bfbc944984 100644 --- a/filebeat/module/kafka/log/_meta/fields.yml +++ b/filebeat/module/kafka/log/_meta/fields.yml @@ -17,7 +17,8 @@ description: > Component the log is coming from. - name: class - type: text + # Is there a reaons this would not be an exact search? + type: keyword description: > Java class the log is coming from. - name: trace @@ -30,10 +31,12 @@ description: > Java class the trace is coming from. - name: message + # Should this be mapped to `message` instead`? How is it different from the above message? type: text description: > Message part of the trace. - name: full + # Sounds like potentially worth as text, not sure where this fits type: text description: > The full trace in the log line. diff --git a/filebeat/module/logstash/log/_meta/fields.yml b/filebeat/module/logstash/log/_meta/fields.yml index dd46dcbef949..ac25a118248d 100644 --- a/filebeat/module/logstash/log/_meta/fields.yml +++ b/filebeat/module/logstash/log/_meta/fields.yml @@ -9,7 +9,7 @@ description: > The module or class where the event originate. - name: thread - type: text + type: keyword description: > Information about the running thread where the log originate. - name: log_event diff --git a/filebeat/module/logstash/slowlog/_meta/fields.yml b/filebeat/module/logstash/slowlog/_meta/fields.yml index f10626d7c125..c0918f3e9b71 100644 --- a/filebeat/module/logstash/slowlog/_meta/fields.yml +++ b/filebeat/module/logstash/slowlog/_meta/fields.yml @@ -4,6 +4,7 @@ slowlog fields: - name: message + # This should go into log.original? Or is it `message`? type: text description: > Contains the un-parsed log message @@ -12,10 +13,12 @@ description: > The module or class where the event originate. - name: thread + # Keyword? type: text description: > Information about the running thread where the log originate. - name: event + # event.original? or log.original? type: text description: > Raw dump of the original event @@ -32,6 +35,7 @@ description: > Execution time for the plugin in milliseconds. - name: plugin_params + # Keyword? type: text description: > String value of the plugin configuration diff --git a/filebeat/module/traefik/access/_meta/fields.yml b/filebeat/module/traefik/access/_meta/fields.yml index f9cc5a263ba9..87b80b4f9527 100644 --- a/filebeat/module/traefik/access/_meta/fields.yml +++ b/filebeat/module/traefik/access/_meta/fields.yml @@ -12,10 +12,12 @@ description: > The number of requests - name: frontend_name + # Keyword? type: text description: > The name of the frontend used - name: backend_url + # Keyword? Or map to url.original type: text description: The url of the backend where request is forwarded From 94d27f800139757d001526663218d51c768c375b Mon Sep 17 00:00:00 2001 From: beats-jenkins Date: Tue, 29 Jan 2019 15:08:09 +0100 Subject: [PATCH 2/5] skip some haproxy fields --- filebeat/module/haproxy/_meta/fields.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/filebeat/module/haproxy/_meta/fields.yml b/filebeat/module/haproxy/_meta/fields.yml index a2930c623bf3..7a4954be2096 100644 --- a/filebeat/module/haproxy/_meta/fields.yml +++ b/filebeat/module/haproxy/_meta/fields.yml @@ -50,12 +50,9 @@ - name: error_message description: Error message logged by HAProxy in case of error. - # Text needed? - # Should it map to ECS? type: text - name: source - # What kind of source is this? type: keyword description: The HAProxy source of the log From 283b9638c265bee0eb533d04f3c0b295a6187a2d Mon Sep 17 00:00:00 2001 From: beats-jenkins Date: Tue, 29 Jan 2019 15:13:10 +0100 Subject: [PATCH 3/5] remove haproxy changes --- filebeat/module/haproxy/_meta/fields.yml | 4 ++-- filebeat/module/haproxy/log/_meta/fields.yml | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/filebeat/module/haproxy/_meta/fields.yml b/filebeat/module/haproxy/_meta/fields.yml index 7a4954be2096..674d20f29be8 100644 --- a/filebeat/module/haproxy/_meta/fields.yml +++ b/filebeat/module/haproxy/_meta/fields.yml @@ -53,14 +53,14 @@ type: text - name: source - type: keyword + type: text description: The HAProxy source of the log - name: termination_state description: Condition the session was in when the session ended. - name: mode - type: keyword + type: text description: mode that the frontend is operating (TCP or HTTP) - name: connections diff --git a/filebeat/module/haproxy/log/_meta/fields.yml b/filebeat/module/haproxy/log/_meta/fields.yml index 7e9b5596ca9f..9176a8ae7282 100644 --- a/filebeat/module/haproxy/log/_meta/fields.yml +++ b/filebeat/module/haproxy/log/_meta/fields.yml @@ -14,7 +14,6 @@ - name: captured_headers description: > List of headers captured in the response due to the presence of the "capture response header" statement in the frontend. - # Do we need text? type: text - name: status_code @@ -33,12 +32,10 @@ - name: captured_headers description: > List of headers captured in the request due to the presence of the "capture request header" statement in the frontend. - # Do we need text? does this even have be be indexed? type: text - name: raw_request_line description: Complete HTTP request line, including the method, request and HTTP version string. - # keyword? sounds like a .original field but none is matching type: text - name: time_wait_without_data_ms From 1b8151f78b68a0793ad0412f4dcd030478f2635e Mon Sep 17 00:00:00 2001 From: beats-jenkins Date: Tue, 29 Jan 2019 15:44:25 +0100 Subject: [PATCH 4/5] remove kafka changes --- filebeat/module/kafka/log/_meta/fields.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/filebeat/module/kafka/log/_meta/fields.yml b/filebeat/module/kafka/log/_meta/fields.yml index b0bfbc944984..76d9467bb044 100644 --- a/filebeat/module/kafka/log/_meta/fields.yml +++ b/filebeat/module/kafka/log/_meta/fields.yml @@ -17,8 +17,7 @@ description: > Component the log is coming from. - name: class - # Is there a reaons this would not be an exact search? - type: keyword + type: text description: > Java class the log is coming from. - name: trace @@ -31,12 +30,10 @@ description: > Java class the trace is coming from. - name: message - # Should this be mapped to `message` instead`? How is it different from the above message? type: text description: > Message part of the trace. - name: full - # Sounds like potentially worth as text, not sure where this fits type: text description: > The full trace in the log line. From 411f474dee538d63220bc3136d08f7cc052d52c6 Mon Sep 17 00:00:00 2001 From: beats-jenkins Date: Tue, 29 Jan 2019 15:51:36 +0100 Subject: [PATCH 5/5] remove traefik changes --- filebeat/module/traefik/access/_meta/fields.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/filebeat/module/traefik/access/_meta/fields.yml b/filebeat/module/traefik/access/_meta/fields.yml index 87b80b4f9527..f9cc5a263ba9 100644 --- a/filebeat/module/traefik/access/_meta/fields.yml +++ b/filebeat/module/traefik/access/_meta/fields.yml @@ -12,12 +12,10 @@ description: > The number of requests - name: frontend_name - # Keyword? type: text description: > The name of the frontend used - name: backend_url - # Keyword? Or map to url.original type: text description: The url of the backend where request is forwarded