From 04b46c65de4016a08d49add0c4a6c096e2c4ca0f Mon Sep 17 00:00:00 2001 From: Johannes Freden Jansson Date: Wed, 13 Mar 2024 11:12:48 +0100 Subject: [PATCH] Add new slowlog fields to filebeat --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 55 ++++++++ filebeat/module/elasticsearch/fields.go | 2 +- .../elasticsearch/slowlog/_meta/fields.yml | 20 +++ .../slowlog/ingest/pipeline-json.yml | 20 +++ .../es814_index_indexing_slowlog-json.log | 4 + ...ex_indexing_slowlog-json.log-expected.json | 130 ++++++++++++++++++ .../test/es814_index_search_slowlog-json.log | 3 + ...ndex_search_slowlog-json.log-expected.json | 104 ++++++++++++++ 9 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log create mode 100644 filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log-expected.json create mode 100644 filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log create mode 100644 filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d06ffb5f7057..bd27df020b40 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -204,6 +204,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Improve rate limit handling by HTTPJSON {issue}36207[36207] {pull}38161[38161] {pull}38237[38237] - Add parseDateInTZ value template for the HTTPJSON input. {pull}37738[37738] - Add support for complex event objects in the HTTP Endpoint input. {issue}37910[37910] {pull}38193[38193] +- Parse more fields from Elasticsearch slowlogs {pull}38295[38295] *Auditbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index ddc887d246f2..98d2e70e2dfb 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -50730,6 +50730,61 @@ type: keyword -- +*`elasticsearch.slowlog.user.realm`*:: ++ +-- +The authentication realm the user was authenticated against + +type: keyword + +example: default_file + +-- + +*`elasticsearch.slowlog.user.effective.realm`*:: ++ +-- +The authentication realm the effective user was authenticated against + +type: keyword + +example: default_file + +-- + +*`elasticsearch.slowlog.auth.type`*:: ++ +-- +The authentication type used to authenticate the user. One of TOKEN | REALM | API_KEY + +type: keyword + +example: REALM + +-- + +*`elasticsearch.slowlog.apikey.id`*:: ++ +-- +The id of the API key used + +type: keyword + +example: WzL_kb6VSvOhAq0twPvHOQ + +-- + +*`elasticsearch.slowlog.apikey.name`*:: ++ +-- +The name of the API key used + +type: keyword + +example: my-api-key + +-- + [[exported-fields-envoyproxy]] == Envoyproxy fields diff --git a/filebeat/module/elasticsearch/fields.go b/filebeat/module/elasticsearch/fields.go index 525d0c50eac7..4f27bd426ffb 100644 --- a/filebeat/module/elasticsearch/fields.go +++ b/filebeat/module/elasticsearch/fields.go @@ -32,5 +32,5 @@ func init() { // AssetElasticsearch returns asset data. // This is the base64 encoded zlib format compressed contents of module/elasticsearch. func AssetElasticsearch() string { - return "eJzUWltz2zb2f8+nOKOXfztj8y9f6taa2Z1plcRxprk0sp1tFQ/nCDyiEIEADYCS1U6++w5AShYpkrpsm+3qxSZxOb9zPwfEMUxp0QMSaCxnhlCzyTMAy62gHnRerL/vPAPQJAgN9SDGZwARGaZ5armSPfjnMwAo7wRvVJQJegYw5iQi0/NTjkFiQptE3c8uUre5VllavKmhUd5ufUumklRJknY1UtmgzNHTfBhrlcB8QprATgiEioFmbkBpHnOJlqLO2qb0iEnqRaQCCliQBG/I4nO02NeElq5lRI8D0jPOaH1dzt+UFnOlo034IjOWdJBlPGrk4Pb2+jmosYdZLKhHdpXM9OiVeHvDB3e/8vfjH6aP8WW8Pxr31IjmLSa0E5pIsSnp45o57SikiihoEceTMNzMetrPB/wju1nQzeSjvf3Xzz9dvu7+9Ga+J4adxdCMY/bx7Wvz29nuhLkzo3bK3tL89HqaYy5oRGiPLRl7zGWa2X3pt0nfU+cNvoHvruLn89Hth3H/7rvvfxywh1E/3kPuZoI6aiUfLYXup9aj6O5OsAhJYapVlDEb5s6/dfmGixqKYLSAItiAscimYBXwiKTl4wXMJ7wUfpZc+InGvXGPmh4yMraerSkfocTOBg8Ta9OgWBk8hirFh4zCtoBSRluERqvATrSyVhCgjCCiKEsFZ2gJIko1MXTrYY5achmbBov/HtN0D/m7kBs4GrHSi0bE/WLCUmzrePwW9WCcuNHykaAQU747KswibjdmrycpqMlJ6zsIXJAujVRZunFJx81aZiJnH3bCzUYa6oEmY4/AapQmVdqNAU/DMReVwFrmXlcsqc2cKxoJ3byt8N0kp5EcsJ2gBcVYprXDjFLJRaIyEyJjZEwYkeQUHQFmduKcIldfOEYu/OvKrPwx1iite2ZKSmJ+Rd275TKLSUqaorDwhyPQmQxxbaPiOV/QLLwy/f3FmKsv2CrHj6sKpEC8oXj4ZnMktxmEDy8GN/Dj++vl4m/XrWS1bo4GNDHiM4pASU/taRqboJQkvj0CoRiK0GUz+CaviRgKn92AG5NRtI7z22bZPe2zv9w0oUi2Wl7ZhvJFHlxlwHE+Q8EjLzSMkctNnyiAd1zNQmPMhHWudQD2zJAOdmPATf0/U8vHEfDx+kCjlXa8mVo+ozDimphVenEoaCXItIL+4Ga4HLEMVASp5pLxFAWMSKhKRihZxHCZvEKMEi47R9BxFYopHuH+QNTelavV0v4b1Kls+w51rcduK9HHrK02ItdKzHxJHmOdUdMjsazZNFzuy+vuXqIkt0r/f4JcHmAdWgQpaky2WIeLQ7cfrsHPJUu62Rg6fzjZu+3/8RnZVHI2Of3SqaXOZcTZFsO8zucUCSMvbHJptZnjWKnj0+7JZdA9CbrnziBLb8423lwcYqXLooxHrSzcSv6QEeSNTU0JWBbfx99/Dqeji7vB7N3kx4eunb+fvXr3yyGRNgdX4z7NqX6ZUvYwxL4g1AOmlRAf6nnbGWs4UtGidjEKjlU7SdFOKtWxWx8wJe2m2yY81phzbHVGLUk9xCjSZKrktgExKtOMAp4eQDjTfE9qznGLekEcQHAV2/clazbPD3almZAxGNeHckuPtiFELPN7gCmf0sIEai4pCkeLsJREQwetdu+RUoJQ1ofqUoKuK+d2KAVrGrIGzpZL1vqbrW1I7XkZtJyZxezwPQGu+uCqI0O2IBDs2BKlEzT1wqtS34LA/V56QmBSYnzMmStMrvo5iaAyuQ7TOq4aa4VWve4E0P3Wz4iu+sCUEHm7Ug90Tf1Z7iyhIdYIbSwUVoPYjsD6FSQrgi6LKh1xGefHAQSvcYYw49pmKCBBNuGyBbhhOhuFZpGMlAgtusbb8oT+Kj7gPWaGwJEALsEQUzIywJw7Ox6yFHIs4LGYrcCt5jL+CsB3wO2hbMU9J5yGmsYmTLVyBZDH/xciv3GYTeoa/ieKHgZoGpMm6YqxJ6aaobtSUQgSoSbDUH4t1GvyTlBPHXrBZwRq9JmYNa6/EQSY5kdfzie4AWNVmlLUzAwTaEyYSaEw+lqc5NS8vcjMVb4exI7SZ2nmcTZirAvKO2J8nxsG9N/f5jZe2AvpsdKJA/wUCmsgNodsqDRwDUKGrYLekRH3qzChMmt4lJ/YTElLEnUMrAWWhfkvoOSyChJaUboO+GvAvFEWBZDA1NlrBbRVvqcWZHPka/nSH3cZi9rPGnPJzSSorTI+z5JQZ7LBBZsZ2cKA74EcVI/k9d2bAk2WrnnbEaABzLd3Vp4qLi3ILBmRrkdrJ5owMqF1cgldlGkKHgcjv0I9wrgkzYIqeKo+thVqqAsaK0N2IdBnlyXmP1vEDoJVyn83yUEVOFtxWYzrO7H60m2btPogVBznqTduIDkhrEbGgwvZV4QpoBCqSDYoo6Ve+O9717JuTTgdNQZ1Li3FG23QDjBh5byOeU/HGf6UCzVa2LYKxWWmvwyS/5LlETWDWbVhIgpjqh67Hay4dyKCmCQVhbNiLEtRssXfX4NeeWrsBLLOwd9AnY0y3a7dhcpk/Gfq91e34f+4hhdVHv4GOm6Raz26ldxIz0pEy6eUAz/s7/FUv97s+vn2KdUhm1qNrFwdr9Hr9GCQf+J3sxxw5tpoNQbSWulyQvJ3KnowRlE6/6g9jqlyleej8oFpk0m3Hb54S2hzgE6ul6t+80Fu/bFtnWvVu8AqEMvNrqOMpUqpDcUSh1AbDK5KhLn6GgRX/M1ITwij0NBDq8gH9JC5frkoERslf3Z+fnl5eVor/kYUT/VeuDzdCbZ8xSl3yVf9I/cn4ULwogJrRHhy0e3uWAeupDRyDo37AfTRzdeqTsirSzWrynaOptiYoj3Q/7AT+lV4EGouVNwcifLx/FqEyTuGjXuVGyA6w9PuyQ/H3Yvj08ubk26ve9E7OT+6PDu7H16/ffkO7of53ax8i6AAETxkpBf3MJyFd68nn+/uYZiQ1Zz5G2AXwVnQPXb7Bt2L4PTifti99yX28Dz4LjH3R/4hzIU0PPfPrhGZcGuGJ5fnZ9+5V4uUzPD+yIVFm//jIfg7IsNfbl98+DW8efXibfjyxU3/1WoPfz/LDE/cfP/lY/jHp45H+6nT++NTJ0HLJiEKkT+OlDL2U6d3EnS/fPlyf/SfxG9XwVfSU1lDP/sJG3fo1rVRK+wx2bL2mnuNVexRatqCxLsct6u+p/ic5vtfL6wmfGfdbmL2hOIU2YbFjTfR24+UN5UWUgM3nmu0kaIfPdmT7pNltlEvLrwt0kb1V816Txje4EOvwDYcQs3btbyHy+yHkB6txjDH2YLwhZtWsANcjpVOcPPT+qFW8hRs2qwy7zq5bTKU89MDiObRaStZJ3xOUX7ZtAnA6X4AtMosryTt6l0bP6NJyKZ78uq3019+ml5+np/HNsaXVu4n+MqnyBL16+jP0W27C960+F6k2CHu1kxtkNuvGkOkWJasLiu6asHHeYpa6P07AAD//wFdWko=" + return "eJzUmt1z2zYSwN/zV+zo5doZmyd/1K09czeTKo7jNB9ubCeXKh7OClxRiECABkDJapv//QYgJYsUSX20zfX8klAEsL9dLHYXIPZhTLMzIIHGcmYINRs9AbDcCjqDzvny750nAJoEoaEziPEJQESGaZ5aruQZ/PsJAJRHgtcqygQ9ARhyEpE58032QWJCq0Ldn52lbnCtsrT4pUZGebjlIZlKUiVJ2sWbygBljR7bw1CrBKYj0gR2RCBUDDRxL5TmMZdoKeosDUoPmKTeRCqggAVJ8JosPkOLPU1o6VJG9HBNesIZLffL9RvTbKp0tIovMmNJB1nGo0YNbm8vn4EaesyiQz3ZRTLRgxfizQ2/fv+RXw1/GD/Ep/H2NO6pkeYNJrQRTaTYmPR+TZt2CqkiClrM8WgM17Je9rNr/oHdzOhm9MHe/ufVj6cvuz++nm7JsLEZmjkmH968NL8cbS6YOzdql+w9zTevlznkggaEdt+SsftcppndVn6b9b103rA28O1F/Gw6uH037L3/7vun1+x+0Iu3sLsZoY5axUdzo/um9RTdzQUWISlMtYoyZsN88a/tvrJEDUUwmEERbMBYZGOwCnhE0vLhDKYjXgo/cy18Q+N+cY+a7jMytl6tMR+gxM6KDiNr06DoGTyEKsX7jMK2gFKmLUKjVWBHWlkrCFBGEFGUpYIztAQRpZoYuv4wRS25jE2Dx3+PabqF/V3IDZyMWOlZI3GvaDA32zKPH6IexpkbLR8ICjHlm1NhFnG70no5SUFNTloeQeCMdOlNVaUbl3Rcq3kmcv5hR9yspKEz0GTsHliN0qRKu3fA03DIRSWwlrXXFU9qc+fKjISu3Vp818jNSA5sR2hBMZZp7ZhRKjlLVGZCZIyMCSOSnKI9wMyO3KLIpy8cIhf+50qr/DHWKK17ZkpKYr5H3W/zbhaTlDRFYbEe9kBnMsSlgYrnvEOz8crytzdjPn3BWjt+WFQgBfHKxMM3q29yn0F4d359A0+vLuedv132kkW/KRrQxIhPKAIlvbTHZmyEUpL4dg+EYihCl83gm7wmYih8dgNuTEbRMue3zbZ7HGd7u2lCkaz1vLIP5Z08XOWF03yCgkfeaBgjl6trogDvuJqFhpgJ65bWDuyZIR1spoBr+g9Tq8ce8OHyi0Yv7Xg3tXxCYcQ1Mav0bFdoJci0Qr9zLVyOmAcqglRzyXiKAgYkVCUjlDyiP09eIUYJl5096LgKxRSPcLcjtV/K1Wpp+wHqpmz9CHVbj816oo9Za31ELpWYeZc8xjqnpgdiWbNruNyX191niZLcKv3PBLncwTu0CFLUmKzxDheHbt9dgm9LlnSzM3R+c7Z3w//rM7Kx5Gx0+KVTK53LiLM1jnmZtykSRl7Y5NZqc8ehUvuH3YPToHsQdI+dQ5Z+OVr55WQXL50XZTxqVeFW8vuMIN/Y1JSAZfN9+PVVOB6cvL+evB09ve/a6dXkxdufd4m0OVzN8mlO9fOUsoUj9gShvmZaCfGuXreNWcOBima1nVFwrPpJinZUqY5d/4ApaVeXbcJjjbnGVmfUktRDjCJNpipuHYhRmWYU8HQHwZnmW0pzC7eoF8QOAhexfVuxZvX8YFOZCRmDcX0ot/RgG0LEPL8HmPIxzUygppKicDALS0k0dGi1Yw+UEoSyPlSXEnRdObdBKVizIWvQbN5laX+zdhtSe14GLWdmMdt9TICLHrjqyJAtBAQbbonSEZp641WlryFwf8+9IDApMT7kzBUmF71cRFBpXMe0zFXjrdA6rxsBur/lM6KLHjAlRL5dqQddmv4sXyyhIdaINhQKq0FsQ7BehWQh0GVRpSMu4/w4gOAlThAmXNsMBSTIRly2gBums0FoZslAidCi23hbntBfpQdcYWYInAjgEgwxJSMDzC1np0OWQs4CnsWsBbeay/grgG/A7VHWck8Jx6GmoQlTrVwB5Pn/QvIbx2xSt+F/lOgxQNOQNElXjD0q1YzuSkUhSISaDEP5taiX7J2gHjt6wScEavCZmDVufyMIMM2Pvtya4AaMVWlKUbMyTKAxYSaFwuhraZJL8/4iM1f5eogNrc/SzHM2MtYF5Q0Zr3LHgN7Vbe7jhb+QHiqdOODHUFiD2ByyobKBazAyrDX0hoq4v4oSKrOGR/mJzZi0JFGnwFJgmZn/ASWXVUhopXQ74K+BeaMsCiCBqfPXCrRVfk8tyObkS/nSH3cZi9q3GnLJzSiorTI+T5JQZ7JhCTYrskYBvwdyqJ7k5fvXBU2WLq22PUADmA/vvDxVXFqQWTIgXU9rR5owMqF1dgldlGkKHjuTX6AeYFyyZiEVvFQf24ppqAsaC0d2IdBnlznzn21ih2CV8t9NcqiCs5XLYly/E6sv3dZZqwdCxXGeeuMGkSPCamTcuZB9QZgCCqGKZIMyms8L/3XrWtb1CceDxqDOpaV4ZRu0ASYsFq9T3stxjj/mQg1mtq1CcZnpL0PyX7I8UTPMYhsmojCm6rHbzhP3VkQQk6SicFaMZSlKNvv7z6CfPDV0BlnW4G8wnY02XT+7M5XJ+M+c349uwP/zGZ5VdfgbzHGLXevpFnYjPSkJLZ9SXvvX/h5P9evNpp9vH1MdsrHVyMrV8ZK8zhlc55/4XSsHztw2Wg2BtFa6nJD8nYozGKIonX/UHsdUtcrzUfnAtMml2w5fvCe0LYBOPi8XveaD3Ppj27qlVb8EFoFYru46yixVSW0Ucw6hVhRclAhT9TUELvSbkB4RRqGh+1aTX9N95vbLRYnYaPmj4+PT09PDWvM3UjzWe+H8dCdY8xWnvEu+6O25fxIuBC8qsEbCg5Nud8M6cGGlgVvQuB2gj26+VnVGXlyqWVS2UzTFwBRtQf/DRvSL8CDUVKi4ORLl7/NrESbfMazcq1yB6PQPuwc/7HdP9g9Pbw66Z92Ts4PjvdOjo7v+5Zvnb+Gun9/NyocICojgPiM9u4P+JHz/cvT5/R30E7KaM38D7CQ4Crr7btygexIcntz1u3e+xO4fB98l5m7PP4S5kfrH/tltREbcmv7B6fHRd+6nWUqmf7fnwqLN/+MR/B2R/s+35+8+hjcvzt+Ez89vei8WY/j7WaZ/4Nr7Lx/93z51PO2nztlvnzoJWjYKUYj8caCUsZ86ZwdB98uXL3d7fyR+uwq+kp7KM/TKN1i5Q7c8G7XGHpItz17zXmMRe5Qat5D4JcftYt9TfE7z+19vrCa+o243MVuiuIlsY3Hvm+RtJ8q7Souoa/c+n9FGif7twZZyHz2zTXpx4W2WNk5/1a23xPAOH/oJbOMQato+y1ssme0I6cFqDHPOFsJz16xQB7gcKp3g6qf1Xb3kMdi0eWW+6+S2yVGOD3cQmkentWKd8TlF+WXTJoDD7QC0yiyvJO3qXRvfosnIpnvw4pfDn38cn36eHsc2xudWbmf4yqfIkvTL6M+Z2/YleNOy9iLFdlluzdKuc/9VQ4gUy5LFZUVXLfg4T9FW8hrueZVktt5ScwN46aUv1PP7aU1mabybthEvDYfkL4v9IfLFKF9bBydl9fP7OnB/bcV/G7GqBLqYhgDeSu8aN29/On8Dv8O786evXsPv8PTqMvzp/GO9Ir7Rdvz+ckL1JtAKP1/cb396demG8/QN1tzkJtCGXCsfwVfIlm+jbcCWzPYx5ftjmrXx/DcAAP//7NJVlQ==" } diff --git a/filebeat/module/elasticsearch/slowlog/_meta/fields.yml b/filebeat/module/elasticsearch/slowlog/_meta/fields.yml index fa251b39789c..0055a7df3649 100644 --- a/filebeat/module/elasticsearch/slowlog/_meta/fields.yml +++ b/filebeat/module/elasticsearch/slowlog/_meta/fields.yml @@ -54,3 +54,23 @@ - name: source description: Source of document that was indexed type: keyword + - name: user.realm + description: The authentication realm the user was authenticated against + example: "default_file" + type: keyword + - name: user.effective.realm + description: The authentication realm the effective user was authenticated against + example: "default_file" + type: keyword + - name: auth.type + description: The authentication type used to authenticate the user. One of TOKEN | REALM | API_KEY + example: REALM + type: keyword + - name: apikey.id + description: The id of the API key used + example: "WzL_kb6VSvOhAq0twPvHOQ" + type: keyword + - name: apikey.name + description: The name of the API key used + example: "my-api-key" + type: keyword diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml index 614c9f7aa439..8a3c8e4f6f00 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml @@ -10,3 +10,23 @@ processors: - pipeline: if: 'ctx.message.contains("ecs.version")' name: '{< IngestPipeline "pipeline-json-8" >}' + - rename: + field: auth.type + target_field: elasticsearch.slowlog.auth.type + ignore_missing: true + - rename: + field: user.realm + target_field: elasticsearch.slowlog.user.realm + ignore_missing: true + - rename: + field: user.effective.realm + target_field: elasticsearch.slowlog.user.effective.realm + ignore_missing: true + - rename: + field: apikey.id + target_field: elasticsearch.slowlog.user.apikey.id + ignore_missing: true + - rename: + field: apikey.name + target_field: elasticsearch.slowlog.user.apikey.name + ignore_missing: true diff --git a/filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log b/filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log new file mode 100644 index 000000000000..920951b8caf9 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log @@ -0,0 +1,4 @@ +{"@timestamp":"2024-03-13T10:34:33.289Z", "log.level": "WARN", "auth.type":"REALM","elasticsearch.slowlog.id":"2","elasticsearch.slowlog.message":"[my-index/stZSoQ12R56VZORRItBKjA]","elasticsearch.slowlog.source":"{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}","elasticsearch.slowlog.took":"12.3ms","elasticsearch.slowlog.took_millis":"12","user.name":"elastic","user.realm":"reserved" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#7]","log.logger":"index.indexing.slowlog.index","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2024-03-13T10:34:36.139Z", "log.level": "WARN", "auth.type":"REALM","elasticsearch.slowlog.id":"3","elasticsearch.slowlog.message":"[my-index/stZSoQ12R56VZORRItBKjA]","elasticsearch.slowlog.source":"{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}","elasticsearch.slowlog.took":"5.9ms","elasticsearch.slowlog.took_millis":"5","user.name":"elastic","user.realm":"reserved" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#9]","log.logger":"index.indexing.slowlog.index","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2024-03-13T10:34:37.257Z", "log.level": "WARN", "auth.type":"REALM","elasticsearch.slowlog.id":"4","elasticsearch.slowlog.message":"[my-index/stZSoQ12R56VZORRItBKjA]","elasticsearch.slowlog.source":"{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}","elasticsearch.slowlog.took":"2.5ms","elasticsearch.slowlog.took_millis":"2","user.name":"elastic","user.realm":"reserved" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#12]","log.logger":"index.indexing.slowlog.index","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2024-03-13T10:34:38.373Z", "log.level": "WARN", "auth.type":"REALM","elasticsearch.slowlog.id":"5","elasticsearch.slowlog.message":"[my-index/stZSoQ12R56VZORRItBKjA]","elasticsearch.slowlog.source":"{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}","elasticsearch.slowlog.took":"2.2ms","elasticsearch.slowlog.took_millis":"2","user.name":"elastic","user.realm":"reserved" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#3]","log.logger":"index.indexing.slowlog.index","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} diff --git a/filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log-expected.json new file mode 100644 index 000000000000..b24a197b41c7 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es814_index_indexing_slowlog-json.log-expected.json @@ -0,0 +1,130 @@ +[ + { + "@timestamp": "2024-03-13T10:34:33.289Z", + "log.level": "WARN", + "log.offset": 0, + "event.type": "info", + "event.kind": "event", + "fileset.name": "slowlog", + "elasticsearch.slowlog.auth.type": "REALM", + "elasticsearch.slowlog.id": "2", + "elasticsearch.index.id": "stZSoQ12R56VZORRItBKjA", + "elasticsearch.index.name": "my-index", + "message": "[my-index/stZSoQ12R56VZORRItBKjA]", + "elasticsearch.slowlog.source": "{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}", + "elasticsearch.slowlog.took": "12.3ms", + "host.id": "a8BUD2RfQSu4aqtpePX7BA", + "input.type": "log", + "event.category": "database", + "user.name": "elastic", + "elasticsearch.slowlog.user.realm": "reserved", + "ecs.version": "1.2.0", + "service.name": "ES_ECS", + "event.dataset": "elasticsearch.index_indexing_slowlog", + "event.duration": 12000000, + "event.module": "elasticsearch", + "process.thread.name": "elasticsearch[runTask-0][write][T#7]", + "log.logger": "index.indexing.slowlog.index", + "elasticsearch.cluster.uuid": "0d2MZYNKR7Wqr2U6Cvpp7g", + "elasticsearch.node.id": "a8BUD2RfQSu4aqtpePX7BA", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.cluster.name": "runTask", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2024-03-13T10:34:36.139Z", + "log.level": "WARN", + "log.offset": 980, + "event.type": "info", + "event.kind": "event", + "fileset.name": "slowlog", + "elasticsearch.slowlog.auth.type": "REALM", + "elasticsearch.slowlog.id": "3", + "elasticsearch.index.id": "stZSoQ12R56VZORRItBKjA", + "elasticsearch.index.name": "my-index", + "message": "[my-index/stZSoQ12R56VZORRItBKjA]", + "elasticsearch.slowlog.source": "{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}", + "elasticsearch.slowlog.took": "5.9ms", + "host.id": "a8BUD2RfQSu4aqtpePX7BA", + "input.type": "log", + "event.category": "database", + "user.name": "elastic", + "elasticsearch.slowlog.user.realm": "reserved", + "ecs.version": "1.2.0", + "service.name": "ES_ECS", + "event.dataset": "elasticsearch.index_indexing_slowlog", + "event.duration": 5000000, + "event.module": "elasticsearch", + "process.thread.name": "elasticsearch[runTask-0][write][T#9]", + "log.logger": "index.indexing.slowlog.index", + "elasticsearch.cluster.uuid": "0d2MZYNKR7Wqr2U6Cvpp7g", + "elasticsearch.node.id": "a8BUD2RfQSu4aqtpePX7BA", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.cluster.name": "runTask", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2024-03-13T10:34:37.257Z", + "log.level": "WARN", + "log.offset": 1958, + "event.type": "info", + "event.kind": "event", + "fileset.name": "slowlog", + "elasticsearch.slowlog.auth.type": "REALM", + "elasticsearch.slowlog.id": "4", + "elasticsearch.index.id": "stZSoQ12R56VZORRItBKjA", + "elasticsearch.index.name": "my-index", + "message": "[my-index/stZSoQ12R56VZORRItBKjA]", + "elasticsearch.slowlog.source": "{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}", + "elasticsearch.slowlog.took": "2.5ms", + "host.id": "a8BUD2RfQSu4aqtpePX7BA", + "input.type": "log", + "event.category": "database", + "user.name": "elastic", + "elasticsearch.slowlog.user.realm": "reserved", + "ecs.version": "1.2.0", + "service.name": "ES_ECS", + "event.dataset": "elasticsearch.index_indexing_slowlog", + "event.duration": 2000000, + "event.module": "elasticsearch", + "process.thread.name": "elasticsearch[runTask-0][write][T#12]", + "log.logger": "index.indexing.slowlog.index", + "elasticsearch.cluster.uuid": "0d2MZYNKR7Wqr2U6Cvpp7g", + "elasticsearch.node.id": "a8BUD2RfQSu4aqtpePX7BA", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.cluster.name": "runTask", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2024-03-13T10:34:38.373Z", + "log.level": "WARN", + "log.offset": 2937, + "event.type": "info", + "event.kind": "event", + "fileset.name": "slowlog", + "elasticsearch.slowlog.auth.type": "REALM", + "elasticsearch.slowlog.id": "5", + "elasticsearch.index.id": "stZSoQ12R56VZORRItBKjA", + "elasticsearch.index.name": "my-index", + "message": "[my-index/stZSoQ12R56VZORRItBKjA]", + "elasticsearch.slowlog.source": "{\\\"indices\\\":{\\\"field_security\\\":{\\\"grant\\\":\\\"read\\\",\\\"except\\\":\\\"confidential\\\"},\\\"names\\\":[\\\"foo\\\",\\\"bar\\\"],\\\"privileges\\\":\\\"admin\\\",\\\"query\\\":\\\"example\\\",\\\"allow_restricted_indices\\\":true}}", + "elasticsearch.slowlog.took": "2.2ms", + "host.id": "a8BUD2RfQSu4aqtpePX7BA", + "input.type": "log", + "event.category": "database", + "user.name": "elastic", + "elasticsearch.slowlog.user.realm": "reserved", + "ecs.version": "1.2.0", + "service.name": "ES_ECS", + "event.dataset": "elasticsearch.index_indexing_slowlog", + "event.duration": 2000000, + "event.module": "elasticsearch", + "process.thread.name": "elasticsearch[runTask-0][write][T#3]", + "log.logger": "index.indexing.slowlog.index", + "elasticsearch.cluster.uuid": "0d2MZYNKR7Wqr2U6Cvpp7g", + "elasticsearch.node.id": "a8BUD2RfQSu4aqtpePX7BA", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.cluster.name": "runTask", + "service.type": "elasticsearch" + } +] diff --git a/filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log b/filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log new file mode 100644 index 000000000000..40e1a31906f9 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log @@ -0,0 +1,3 @@ +{"@timestamp":"2024-03-13T09:42:41.350Z", "log.level": "WARN", "elasticsearch.slowlog.id":null,"elasticsearch.slowlog.message":"[my-index][0]","elasticsearch.slowlog.search_type":"QUERY_THEN_FETCH","elasticsearch.slowlog.source":"{\\\"query\\\":{\\\"match_none\\\":{\\\"boost\\\":1.0}}}","elasticsearch.slowlog.stats":"[]","elasticsearch.slowlog.took":"7.7ms","elasticsearch.slowlog.took_millis":7,"elasticsearch.slowlog.total_hits":"0 hits","elasticsearch.slowlog.total_shards":1 , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_search_slowlog","process.thread.name":"elasticsearch[runTask-0][search][T#3]","log.logger":"index.search.slowlog.query","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2024-03-13T09:43:56.663Z", "log.level": "WARN", "elasticsearch.slowlog.id":null,"elasticsearch.slowlog.message":"[my-index][0]","elasticsearch.slowlog.search_type":"QUERY_THEN_FETCH","elasticsearch.slowlog.source":"{\\\"query\\\":{\\\"match_none\\\":{\\\"boost\\\":1.0}}}","elasticsearch.slowlog.stats":"[]","elasticsearch.slowlog.took":"946.6micros","elasticsearch.slowlog.took_millis":0,"elasticsearch.slowlog.total_hits":"0 hits","elasticsearch.slowlog.total_shards":1 , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_search_slowlog","process.thread.name":"elasticsearch[runTask-0][search][T#6]","log.logger":"index.search.slowlog.query","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2024-03-13T09:44:20.724Z", "log.level": "WARN", "auth.type":"REALM","elasticsearch.slowlog.id":null,"elasticsearch.slowlog.message":"[my-index][0]","elasticsearch.slowlog.search_type":"QUERY_THEN_FETCH","elasticsearch.slowlog.source":"{\\\"query\\\":{\\\"match_none\\\":{\\\"boost\\\":1.0}}}","elasticsearch.slowlog.stats":"[]","elasticsearch.slowlog.took":"509.5micros","elasticsearch.slowlog.took_millis":0,"elasticsearch.slowlog.total_hits":"0 hits","elasticsearch.slowlog.total_shards":1,"user.name":"elastic","user.realm":"reserved" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_search_slowlog","process.thread.name":"elasticsearch[runTask-0][search][T#8]","log.logger":"index.search.slowlog.query","elasticsearch.cluster.uuid":"0d2MZYNKR7Wqr2U6Cvpp7g","elasticsearch.node.id":"a8BUD2RfQSu4aqtpePX7BA","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} diff --git a/filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log-expected.json new file mode 100644 index 000000000000..651f6ce267f3 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es814_index_search_slowlog-json.log-expected.json @@ -0,0 +1,104 @@ +[ + { + "@timestamp": "2024-03-13T09:42:41.350Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "0d2MZYNKR7Wqr2U6Cvpp7g", + "elasticsearch.index.name": "my-index", + "elasticsearch.node.id": "a8BUD2RfQSu4aqtpePX7BA", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.slowlog.id": null, + "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", + "elasticsearch.slowlog.source": "{\\\"query\\\":{\\\"match_none\\\":{\\\"boost\\\":1.0}}}", + "elasticsearch.slowlog.took": "7.7ms", + "elasticsearch.slowlog.total_hits": "0 hits", + "elasticsearch.shard.id": "0", + "elasticsearch.slowlog.stats": "[]", + "elasticsearch.slowlog.total_shards": 1, + "event.dataset": "elasticsearch.index_search_slowlog", + "event.type": "info", + "event.kind": "event", + "fileset.name": "slowlog", + "host.id": "a8BUD2RfQSu4aqtpePX7BA", + "input.type": "log", + "log.level": "WARN", + "log.offset": 0, + "message": "[my-index][0]", + "service.type": "elasticsearch", + "event.category": "database", + "ecs.version": "1.2.0", + "service.name": "ES_ECS", + "event.duration": 7000000, + "event.module": "elasticsearch", + "process.thread.name": "elasticsearch[runTask-0][search][T#3]", + "log.logger": "index.search.slowlog.query" + }, + { + "@timestamp": "2024-03-13T09:43:56.663Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "0d2MZYNKR7Wqr2U6Cvpp7g", + "elasticsearch.index.name": "my-index", + "elasticsearch.node.id": "a8BUD2RfQSu4aqtpePX7BA", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.slowlog.id": null, + "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", + "elasticsearch.slowlog.source": "{\\\"query\\\":{\\\"match_none\\\":{\\\"boost\\\":1.0}}}", + "elasticsearch.slowlog.took": "946.6micros", + "elasticsearch.slowlog.total_hits": "0 hits", + "elasticsearch.shard.id": "0", + "elasticsearch.slowlog.total_shards": 1, + "elasticsearch.slowlog.stats": "[]", + "event.dataset": "elasticsearch.index_search_slowlog", + "event.type": "info", + "event.kind": "event", + "fileset.name": "slowlog", + "host.id": "a8BUD2RfQSu4aqtpePX7BA", + "input.type": "log", + "log.level": "WARN", + "log.offset": 869, + "message": "[my-index][0]", + "service.type": "elasticsearch", + "event.category": "database", + "ecs.version": "1.2.0", + "service.name": "ES_ECS", + "event.duration": 0, + "event.module": "elasticsearch", + "process.thread.name": "elasticsearch[runTask-0][search][T#6]", + "log.logger": "index.search.slowlog.query" + }, + { + "@timestamp": "2024-03-13T09:44:20.724Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "0d2MZYNKR7Wqr2U6Cvpp7g", + "elasticsearch.index.name": "my-index", + "elasticsearch.node.id": "a8BUD2RfQSu4aqtpePX7BA", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.slowlog.id": null, + "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", + "elasticsearch.slowlog.source": "{\\\"query\\\":{\\\"match_none\\\":{\\\"boost\\\":1.0}}}", + "elasticsearch.slowlog.took": "509.5micros", + "elasticsearch.slowlog.total_hits": "0 hits", + "elasticsearch.shard.id": "0", + "elasticsearch.slowlog.stats": "[]", + "elasticsearch.slowlog.total_shards": 1, + "event.type": "info", + "event.kind": "event", + "event.dataset": "elasticsearch.index_search_slowlog", + "fileset.name": "slowlog", + "host.id": "a8BUD2RfQSu4aqtpePX7BA", + "input.type": "log", + "log.level": "WARN", + "log.offset": 1744, + "message": "[my-index][0]", + "service.type": "elasticsearch", + "elasticsearch.slowlog.auth.type": "REALM", + "event.category": "database", + "user.name": "elastic", + "elasticsearch.slowlog.user.realm": "reserved", + "ecs.version": "1.2.0", + "service.name": "ES_ECS", + "event.duration": 0, + "event.module": "elasticsearch", + "process.thread.name": "elasticsearch[runTask-0][search][T#8]", + "log.logger": "index.search.slowlog.query" + } +]