From 4a9faf4045ebe8a2f5286f7c56072021e94de77c Mon Sep 17 00:00:00 2001 From: Andrew Kroh <andrew.kroh@elastic.co> Date: Tue, 3 Aug 2021 08:47:50 -0400 Subject: [PATCH] [Fleet] Remove subseconds from event.ingested (#104044) The `event.ingested` field is added to all documents ingested via Fleet plus Agent. By removing the subseconds we can be better compression of the values in Elasticsearch. The primary user of `event.ingested` today is the the Security Detection Engine as a tie-breaker in search_after, but once it moves to the using the point-in-time API the need for precision will be lessened because PIT has an implicit tie-breaker. Relates #103944 Relates https://github.com/elastic/beats/issues/22388 Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../fleet/server/constants/fleet_es_assets.ts | 8 +++++++- .../apis/epm/final_pipeline.ts | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts index 365c2d1d8a902..eb70aff3238a0 100644 --- a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts +++ b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts @@ -23,6 +23,7 @@ export const FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT = { properties: { ingested: { type: 'date', + format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', }, agent_id_status: { ignore_above: 1024, @@ -42,7 +43,12 @@ processors: - set: description: Add time when event was ingested. field: event.ingested - value: '{{{_ingest.timestamp}}}' + copy_from: _ingest.timestamp + - script: + description: Remove sub-seconds from event.ingested to improve storage efficiency. + tag: truncate-subseconds-event-ingested + source: ctx.event.ingested = ctx.event.ingested.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); + ignore_failure: true - remove: description: Remove any pre-existing untrusted values. field: diff --git a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts index 68a78dd842c4b..073d08c5b1d8d 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts @@ -91,6 +91,26 @@ export default function (providerContext: FtrProviderContext) { ); }); + it('all docs should contain event.ingested without sub-seconds', async () => { + const res = await es.index({ + index: 'logs-log.log-test', + body: { + '@timestamp': '2020-01-01T09:09:00', + message: 'hello', + }, + }); + + const { body: doc } = await es.get({ + id: res.body._id, + index: res.body._index, + }); + // @ts-expect-error + const ingestTimestamp = doc._source.event.ingested; + + // 2021-06-30T12:06:28Z + expect(ingestTimestamp).to.match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/); + }); + it('For a doc written without api key should write the correct api key status', async () => { const res = await es.index({ index: 'logs-log.log-test',