[Filebeat] Changes to text fields in elasticsearch module (#10414)

This PR is an offshoot of conversations and decisions made in #10372 w.r.t `text` fields, but scoped to the `elasticsearch` module.
elastic · Jan 31, 2019 · 33b789c · 33b789c
1 parent ae83e46
commit 33b789c
Show file tree

Hide file tree

Showing 7 changed files with 36 additions and 22 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -69,6 +69,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Ingesting Elasticsearch audit logs is only supported with Elasticsearch 6.5.0 and above {pull}10352[10352]
 - Migrate Elasticsearch audit logs fields to ECS {pull}10352[10352]
 - Several text fields in the Logstash module are now indexed as `keyword` fields with `text` multi-fields (ECS). {pull}10417[10417]
+- Several text fields in the Elasticsearch module are now indexed as `keyword` fields with `text` multi-fields (ECS). {pull}10414[10414]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -4907,7 +4907,7 @@ Logger name
 *`elasticsearch.slowlog.took`*::
 +
 --
-type: text
+type: keyword
 
 example: 300ms
 
@@ -4929,11 +4929,11 @@ Types
 *`elasticsearch.slowlog.stats`*::
 +
 --
-type: text
+type: keyword
 
-example: 
+example: group1
 
-Statistics
+Stats groups
 
 --
 
@@ -4951,7 +4951,7 @@ Search type
 *`elasticsearch.slowlog.source_query`*::
 +
 --
-type: text
+type: keyword
 
 example: {"query":{"match_all":{"boost":1.0}}}
 
@@ -4962,7 +4962,7 @@ Slow query
 *`elasticsearch.slowlog.extra_source`*::
 +
 --
-type: text
+type: keyword
 
 example: 
 

diff --git a/filebeat/module/elasticsearch/fields.go b/filebeat/module/elasticsearch/fields.go
diff --git a/filebeat/module/elasticsearch/slowlog/_meta/fields.yml b/filebeat/module/elasticsearch/slowlog/_meta/fields.yml
@@ -10,27 +10,27 @@
   - name: took
     description: "Time it took to execute the query"
     example: "300ms"
-    type: text
+    type: keyword
   - name: types
     description: "Types"
     example: ""
     type: keyword
   - name: stats
-    description: "Statistics"
-    example: ""
-    type: text
+    description: "Stats groups"
+    example: "group1"
+    type: keyword
   - name: search_type
     description: "Search type"
     example: "QUERY_THEN_FETCH"
     type: keyword
   - name: source_query
     description: "Slow query"
     example: "{\"query\":{\"match_all\":{\"boost\":1.0}}}"
-    type: text
+    type: keyword
   - name: extra_source
     description: "Extra source information"
     example: ""
-    type: text
+    type: keyword
   - name: total_hits
     description: "Total hits"
     example: 42

diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json
@@ -19,6 +19,14 @@
               ]
           }
       },
+      {
+          "split": {
+              "if": "ctx.elasticsearch.slowlog?.stats != ''",
+              "field": "elasticsearch.slowlog.stats",
+              "separator": ",",
+              "ignore_missing": true
+          }
+      },
       {
           "date": {
               "field": "elasticsearch.slowlog.timestamp",
@@ -32,7 +40,9 @@
       },
       {
           "remove": {
-              "field": "elasticsearch.slowlog.timestamp"
+              "field": [
+                    "elasticsearch.slowlog.timestamp"
+              ]
           }
       },
 

diff --git a/filebeat/module/elasticsearch/slowlog/test/test.log b/filebeat/module/elasticsearch/slowlog/test/test.log
@@ -1,4 +1,4 @@
-[2018-06-29T10:06:14,933][INFO ][index.search.slowlog.query] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[4.5ms], took_millis[4], total_hits[19435], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{"query":{"match_all":{"boost":1.0}}}],
+[2018-06-29T10:06:14,933][INFO ][index.search.slowlog.query] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[4.5ms], took_millis[4], total_hits[19435], types[], stats[group1,group2], search_type[QUERY_THEN_FETCH], total_shards[1], source[{"query":{"match_all":{"boost":1.0}}}],
 [2018-06-29T10:06:14,943][INFO ][index.search.slowlog.fetch] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[10.8ms], took_millis[10], total_hits[19435], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{"query":{"match_all":{"boost":1.0}}}],
 [2018-06-29T09:01:01,821][INFO ][index.search.slowlog.query] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[124.3ms], took_millis[124], total_hits[0], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{"size":500,"query":{"match_none":{"boost":1.0}},"version":true,"_source":{"includes":[],"excludes":[]},"stored_fields":"*","docvalue_fields":["@timestamp","ceph.monitor_health.last_updated","docker.container.created","docker.healthcheck.event.end_date","docker.healthcheck.event.start_date","docker.image.created","kubernetes.container.start_time","kubernetes.event.metadata.timestamp.created","kubernetes.node.start_time","kubernetes.pod.start_time","kubernetes.system.start_time","mongodb.status.background_flushing.last_finished","mongodb.status.local_time","php_fpm.pool.start_time","postgresql.activity.backend_start","postgresql.activity.query_start","postgresql.activity.state_change","postgresql.activity.transaction_start","postgresql.bgwriter.stats_reset","postgresql.database.stats_reset","system.process.cpu.start_time"],"script_fields":{},"sort":[{"@timestamp":{"order":"desc","unmapped_type":"boolean"}}],"aggregations":{"2":{"date_histogram":{"field":"@timestamp","time_zone":"Europe/Berlin","interval":"30s","offset":0,"order":{"_key":"asc"},"keyed":false,"min_doc_count":1}}},"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fragment_size":2147483647,"fields":{"*":{}}}}],
 [2018-06-29T09:01:01,827][INFO ][index.search.slowlog.fetch] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[7.2ms], took_millis[7], total_hits[0], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{"size":500,"query":{"match_none":{"boost":1.0}},"version":true,"_source":{"includes":[],"excludes":[]},"stored_fields":"*","docvalue_fields":["@timestamp","ceph.monitor_health.last_updated","docker.container.created","docker.healthcheck.event.end_date","docker.healthcheck.event.start_date","docker.image.created","kubernetes.container.start_time","kubernetes.event.metadata.timestamp.created","kubernetes.node.start_time","kubernetes.pod.start_time","kubernetes.system.start_time","mongodb.status.background_flushing.last_finished","mongodb.status.local_time","php_fpm.pool.start_time","postgresql.activity.backend_start","postgresql.activity.query_start","postgresql.activity.state_change","postgresql.activity.transaction_start","postgresql.bgwriter.stats_reset","postgresql.database.stats_reset","system.process.cpu.start_time"],"script_fields":{},"sort":[{"@timestamp":{"order":"desc","unmapped_type":"boolean"}}],"aggregations":{"2":{"date_histogram":{"field":"@timestamp","time_zone":"Europe/Berlin","interval":"30s","offset":0,"order":{"_key":"asc"},"keyed":false,"min_doc_count":1}}},"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fragment_size":2147483647,"fields":{"*":{}}}}],