From a58e3bdbbf7a0e1e22c5084c3fa5186766919671 Mon Sep 17 00:00:00 2001 From: Peter Pisljar Date: Thu, 4 Mar 2021 21:59:25 +0100 Subject: [PATCH 1/7] fix agg config sub agg dsl generation (#93276) --- api_docs/data.json | 100 +++++++++--------- api_docs/data_search.json | 100 +++++++++--------- .../common/search/aggs/agg_config.test.ts | 45 ++++++++ .../data/common/search/aggs/agg_config.ts | 6 +- 4 files changed, 150 insertions(+), 101 deletions(-) diff --git a/api_docs/data.json b/api_docs/data.json index ffcaad34c7261..82ab1accac717 100644 --- a/api_docs/data.json +++ b/api_docs/data.json @@ -5434,8 +5434,8 @@ ], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 259, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L259" + "lineNumber": 263, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L263" } }, { @@ -5461,8 +5461,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 290, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L290" + "lineNumber": 294, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L294" } }, { @@ -5482,8 +5482,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 301, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L301" + "lineNumber": 305, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L305" } }, { @@ -5509,8 +5509,8 @@ ], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 310, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L310" + "lineNumber": 314, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L314" } }, { @@ -5542,8 +5542,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 365, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L365" + "lineNumber": 369, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L369" } }, { @@ -5567,8 +5567,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 369, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L369" + "lineNumber": 373, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L373" } }, { @@ -5592,8 +5592,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 373, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L373" + "lineNumber": 377, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L377" } }, { @@ -5615,8 +5615,8 @@ "description": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 377, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L377" + "lineNumber": 381, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L381" } } ], @@ -5624,8 +5624,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 377, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L377" + "lineNumber": 381, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L381" } }, { @@ -5647,8 +5647,8 @@ "description": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 381, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L381" + "lineNumber": 385, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L385" } }, { @@ -5661,8 +5661,8 @@ "description": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 381, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L381" + "lineNumber": 385, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L385" } } ], @@ -5670,8 +5670,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 381, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L381" + "lineNumber": 385, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L385" } }, { @@ -5687,8 +5687,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 389, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L389" + "lineNumber": 393, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L393" } }, { @@ -5704,8 +5704,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 395, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L395" + "lineNumber": 399, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L399" } }, { @@ -5723,8 +5723,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 404, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L404" + "lineNumber": 408, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L408" } }, { @@ -5746,8 +5746,8 @@ "description": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 408, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L408" + "lineNumber": 412, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L412" } } ], @@ -5755,8 +5755,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 408, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L408" + "lineNumber": 412, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L412" } }, { @@ -5779,8 +5779,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 422, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L422" + "lineNumber": 426, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L426" } }, { @@ -5804,8 +5804,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 426, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L426" + "lineNumber": 430, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L430" } }, { @@ -5821,8 +5821,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 430, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L430" + "lineNumber": 434, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L434" } }, { @@ -5838,8 +5838,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 435, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L435" + "lineNumber": 439, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L439" } }, { @@ -5850,8 +5850,8 @@ "description": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 442, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L442" + "lineNumber": 446, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L446" }, "signature": [ { @@ -5871,8 +5871,8 @@ "description": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 446, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L446" + "lineNumber": 450, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L450" }, "signature": [ { @@ -5917,8 +5917,8 @@ "description": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 476, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L476" + "lineNumber": 480, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L480" } } ], @@ -5926,8 +5926,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 476, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L476" + "lineNumber": 480, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L480" } } ], diff --git a/api_docs/data_search.json b/api_docs/data_search.json index 4c0829dd181ce..f060343ecef7c 100644 --- a/api_docs/data_search.json +++ b/api_docs/data_search.json @@ -2876,8 +2876,8 @@ ], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 259, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L259" + "lineNumber": 263, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L263" } }, { @@ -2903,8 +2903,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 290, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L290" + "lineNumber": 294, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L294" } }, { @@ -2924,8 +2924,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 301, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L301" + "lineNumber": 305, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L305" } }, { @@ -2951,8 +2951,8 @@ ], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 310, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L310" + "lineNumber": 314, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L314" } }, { @@ -2984,8 +2984,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 365, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L365" + "lineNumber": 369, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L369" } }, { @@ -3009,8 +3009,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 369, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L369" + "lineNumber": 373, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L373" } }, { @@ -3034,8 +3034,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 373, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L373" + "lineNumber": 377, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L377" } }, { @@ -3057,8 +3057,8 @@ "description": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 377, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L377" + "lineNumber": 381, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L381" } } ], @@ -3066,8 +3066,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 377, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L377" + "lineNumber": 381, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L381" } }, { @@ -3089,8 +3089,8 @@ "description": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 381, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L381" + "lineNumber": 385, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L385" } }, { @@ -3103,8 +3103,8 @@ "description": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 381, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L381" + "lineNumber": 385, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L385" } } ], @@ -3112,8 +3112,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 381, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L381" + "lineNumber": 385, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L385" } }, { @@ -3129,8 +3129,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 389, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L389" + "lineNumber": 393, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L393" } }, { @@ -3146,8 +3146,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 395, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L395" + "lineNumber": 399, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L399" } }, { @@ -3165,8 +3165,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 404, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L404" + "lineNumber": 408, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L408" } }, { @@ -3188,8 +3188,8 @@ "description": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 408, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L408" + "lineNumber": 412, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L412" } } ], @@ -3197,8 +3197,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 408, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L408" + "lineNumber": 412, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L412" } }, { @@ -3221,8 +3221,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 422, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L422" + "lineNumber": 426, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L426" } }, { @@ -3246,8 +3246,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 426, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L426" + "lineNumber": 430, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L430" } }, { @@ -3263,8 +3263,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 430, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L430" + "lineNumber": 434, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L434" } }, { @@ -3280,8 +3280,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 435, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L435" + "lineNumber": 439, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L439" } }, { @@ -3292,8 +3292,8 @@ "description": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 442, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L442" + "lineNumber": 446, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L446" }, "signature": [ { @@ -3313,8 +3313,8 @@ "description": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 446, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L446" + "lineNumber": 450, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L450" }, "signature": [ { @@ -3359,8 +3359,8 @@ "description": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 476, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L476" + "lineNumber": 480, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L480" } } ], @@ -3368,8 +3368,8 @@ "returnComment": [], "source": { "path": "src/plugins/data/common/search/aggs/agg_config.ts", - "lineNumber": 476, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L476" + "lineNumber": 480, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/data/common/search/aggs/agg_config.ts#L480" } } ], diff --git a/src/plugins/data/common/search/aggs/agg_config.test.ts b/src/plugins/data/common/search/aggs/agg_config.test.ts index 5e52779ffa218..818255f6c8bcc 100644 --- a/src/plugins/data/common/search/aggs/agg_config.test.ts +++ b/src/plugins/data/common/search/aggs/agg_config.test.ts @@ -141,6 +141,51 @@ describe('AggConfig', () => { expect(dsl.aggs[avgConfig.id]).toHaveProperty('avg'); expect(dsl.aggs[avgConfig.id].avg).toBe(football); }); + + it('merges subAggs from #write() output to the current subaggs', () => { + const configStates = [ + { + enabled: true, + type: 'avg', + schema: 'metric', + params: {}, + }, + { + enabled: true, + type: 'median', + schema: 'metric', + params: {}, + }, + { + enabled: true, + type: 'date_histogram', + schema: 'segment', + params: {}, + }, + ]; + const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); + + const histoConfig = ac.byName('date_histogram')[0]; + const avgConfig = ac.byName('avg')[0]; + const medianConfig = ac.byName('median')[0]; + const football = {}; + + jest + .spyOn(histoConfig, 'write') + .mockImplementation(() => ({ params: {}, subAggs: [avgConfig] })); + jest.spyOn(avgConfig, 'write').mockImplementation(() => ({ params: football })); + jest.spyOn(medianConfig, 'write').mockImplementation(() => ({ params: football })); + + (histoConfig as any).subAggs = [medianConfig]; + const dsl = histoConfig.toDsl(); + expect(dsl).toHaveProperty('aggs'); + expect(dsl.aggs).toHaveProperty(avgConfig.id); + expect(dsl.aggs[avgConfig.id]).toHaveProperty('avg'); + expect(dsl.aggs[avgConfig.id].avg).toBe(football); + expect(dsl.aggs).toHaveProperty(medianConfig.id); + expect(dsl.aggs[medianConfig.id]).toHaveProperty('percentiles'); + expect(dsl.aggs[medianConfig.id].percentiles).toBe(football); + }); }); describe('::ensureIds', () => { diff --git a/src/plugins/data/common/search/aggs/agg_config.ts b/src/plugins/data/common/search/aggs/agg_config.ts index 6a5bc64cbdc55..f62fedc13b32a 100644 --- a/src/plugins/data/common/search/aggs/agg_config.ts +++ b/src/plugins/data/common/search/aggs/agg_config.ts @@ -235,7 +235,11 @@ export class AggConfig { configDsl[this.type.dslName || this.type.name] = output.params; // if the config requires subAggs, write them to the dsl as well - if (this.subAggs.length && !output.subAggs) output.subAggs = this.subAggs; + if (this.subAggs.length) { + if (!output.subAggs) output.subAggs = this.subAggs; + else output.subAggs.push(...this.subAggs); + } + if (output.subAggs) { const subDslLvl = configDsl.aggs || (configDsl.aggs = {}); output.subAggs.forEach(function nestAdhocSubAggs(subAggConfig: any) { From 28750e913b7ec4bcaf09646ad1b7c439ce23d607 Mon Sep 17 00:00:00 2001 From: Candace Park <56409205+parkiino@users.noreply.github.com> Date: Thu, 4 Mar 2021 16:05:46 -0500 Subject: [PATCH 2/7] [Security Solution][Endpoint][Admin][Policy] Register as AV os restrictions tooltip note (#93306) --- .../antivirus_registration_form/index.tsx | 9 ++++- .../view/components/config_form/index.tsx | 33 ++++++++++++++++--- .../pages/policy/view/policy_details.test.tsx | 2 +- .../view/policy_forms/protections/malware.tsx | 4 +-- .../policy_forms/protections/ransomware.tsx | 4 +-- 5 files changed, 42 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/components/antivirus_registration_form/index.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/components/antivirus_registration_form/index.tsx index 7be2c9870b2f7..51edf1a200c53 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/components/antivirus_registration_form/index.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/components/antivirus_registration_form/index.tsx @@ -54,7 +54,14 @@ export const AntivirusRegistrationForm = memo(() => { ); return ( - + {TRANSLATIONS.description} ( ConfigFormHeading.displayName = 'ConfigFormHeading'; export const ConfigForm: FC = memo( - ({ type, supportedOss, dataTestSubj, rightCorner, children }) => ( + ({ type, supportedOss, osRestriction, dataTestSubj, rightCorner, children }) => ( - + {TITLES.type} {type} {TITLES.os} - {supportedOss.map((os) => OS_TITLES[os]).join(', ')} + + + {supportedOss.map((os) => OS_TITLES[os]).join(', ')} + + {osRestriction && ( + + + + + + + + + + + + + )} + - + {rightCorner} diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.test.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.test.tsx index 1ae4144a26835..ff5f410611099 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_details.test.tsx @@ -326,7 +326,7 @@ describe('Policy Details', () => { const userNotificationCustomMessageTextArea = policyView.find( 'EuiTextArea[data-test-subj="malwareUserNotificationCustomMessage"]' ); - const tooltip = policyView.find('EuiIconTip'); + const tooltip = policyView.find('EuiIconTip[data-test-subj="malwareTooltip"]'); expect(userNotificationCheckbox).toHaveLength(0); expect(userNotificationCustomMessageTextArea).toHaveLength(0); expect(tooltip).toHaveLength(0); diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx index ec0b563421537..a5be095abfc59 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx @@ -211,14 +211,14 @@ export const MalwareProtections = React.memo(() => { - + - + { - + - + Date: Thu, 4 Mar 2021 14:22:37 -0700 Subject: [PATCH 3/7] [Security Solutions] Sets our default date time to be "today" instead of "Last 24 hours" to enable cachability and fixes one date math bug in the URL (#93548) ## Summary Enables caching in our application by setting the default date time of our application to be `from: now/d` and `to: now/d`. When users go to the advanced settings they will see this now: Screen Shot 2021-03-04 at 11 53 08 AM In their date time bars on page loads they will see today instead of 24 hours: Screen Shot 2021-03-04 at 11 50 18 AM When before they used to have `from: now-24` and `to: now`. This new date time frame plays well with Elastic caches and no longer "busts" them for users on each page request. Now users will send the same date time frame on each query which will cache the views as the default. This also fixes a small bug with the URL's where the "to" was not being rounded up when it was a dynamic date time on first load. For example if you went to the URL, `/app/security/hosts/allHosts` with no additional state information but have a default of `from: now/d` and `to: now/d` it would not round up the date time. Now it rounds it up through the date math utilities which only rounds when it sees that it is a dynamic date math. When requests are being sent, expect to see this where you have `from` rounded down and `to rounded up. This should be a consistent non-sliding date time math for caching to operate: Screen Shot 2021-03-04 at 11 33 11 AM If you change the `to` to be another date math such as `now+1d/d` expect to see it also rounded up. This behavior mirrors that of discover: Screen Shot 2021-03-04 at 11 33 11 AM You can manually verify this behavior by setting the same now dates in discover as well as security solutions and both should work as is even when you remove the URL state from the right side of a `?` ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../security_solution/common/constants.ts | 4 ++-- .../components/query_bar/index.test.tsx | 4 ++-- .../super_date_picker/index.test.tsx | 10 ++++----- .../common/store/inputs/helpers.test.ts | 8 +++---- .../utils/default_date_settings.test.ts | 21 +++++++++++++++++++ .../common/utils/default_date_settings.ts | 14 +++++++++---- .../recent_cases/no_cases/index.test.tsx | 2 +- .../timeline/query_bar/index.test.tsx | 4 ++-- 8 files changed, 47 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index c277ce369dca0..994e420260b25 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -28,8 +28,8 @@ export const DEFAULT_SEARCH_AFTER_PAGE_SIZE = 100; export const DEFAULT_ANOMALY_SCORE = 'securitySolution:defaultAnomalyScore'; export const DEFAULT_MAX_TABLE_QUERY_SIZE = 10000; export const DEFAULT_SCALE_DATE_FORMAT = 'dateFormat:scaled'; -export const DEFAULT_FROM = 'now-24h'; -export const DEFAULT_TO = 'now'; +export const DEFAULT_FROM = 'now/d'; +export const DEFAULT_TO = 'now/d'; export const DEFAULT_INTERVAL_PAUSE = true; export const DEFAULT_INTERVAL_TYPE = 'manual'; export const DEFAULT_INTERVAL_VALUE = 300000; // ms diff --git a/x-pack/plugins/security_solution/public/common/components/query_bar/index.test.tsx b/x-pack/plugins/security_solution/public/common/components/query_bar/index.test.tsx index 1e998f9798e97..ca1a93e25534a 100644 --- a/x-pack/plugins/security_solution/public/common/components/query_bar/index.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/query_bar/index.test.tsx @@ -82,8 +82,8 @@ describe('QueryBar ', () => { expect(searchBarProps).toEqual({ dataTestSubj: undefined, - dateRangeFrom: 'now-24h', - dateRangeTo: 'now', + dateRangeFrom: 'now/d', + dateRangeTo: 'now/d', filters: [], indexPatterns: [ { diff --git a/x-pack/plugins/security_solution/public/common/components/super_date_picker/index.test.tsx b/x-pack/plugins/security_solution/public/common/components/super_date_picker/index.test.tsx index 6d52ea8c011c5..d10f0eb28fadc 100644 --- a/x-pack/plugins/security_solution/public/common/components/super_date_picker/index.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/super_date_picker/index.test.tsx @@ -135,8 +135,8 @@ describe('SIEM Super Date Picker', () => { expect(store.getState().inputs.global.timerange.kind).toBe('relative'); }); - test('Make Sure it is last 24 hours date', () => { - expect(store.getState().inputs.global.timerange.fromStr).toBe('now-24h'); + test('Make Sure it is last "now-${x}h" where ${x} is in hours date', () => { + expect(store.getState().inputs.global.timerange.fromStr).toMatch(/^now-[0-9]+h/); expect(store.getState().inputs.global.timerange.toStr).toBe('now'); }); @@ -206,7 +206,7 @@ describe('SIEM Super Date Picker', () => { expect(wrapper.find('div.euiQuickSelectPopover__section').at(1).text()).toBe('Today'); }); - test('Today and Last 24 hours are in Recently used date ranges', () => { + test('Today and "Last ${x} hours" where ${x} is in hours are in Recently used date ranges', () => { wrapper .find('[data-test-subj="superDatePickerToggleQuickMenuButton"]') .first() @@ -216,8 +216,8 @@ describe('SIEM Super Date Picker', () => { wrapper.find('button.euiQuickSelect__applyButton').first().simulate('click'); wrapper.update(); - expect(wrapper.find('div.euiQuickSelectPopover__section').at(1).text()).toBe( - 'Last 24 hoursToday' + expect(wrapper.find('div.euiQuickSelectPopover__section').at(1).text()).toMatch( + /^Last\s[0-9]+\shoursToday/ ); }); diff --git a/x-pack/plugins/security_solution/public/common/store/inputs/helpers.test.ts b/x-pack/plugins/security_solution/public/common/store/inputs/helpers.test.ts index 6dac1a20fefc0..c8848e6f7ffc7 100644 --- a/x-pack/plugins/security_solution/public/common/store/inputs/helpers.test.ts +++ b/x-pack/plugins/security_solution/public/common/store/inputs/helpers.test.ts @@ -276,10 +276,10 @@ describe('Inputs', () => { ], timerange: { from: '2020-07-07T08:20:18.966Z', - fromStr: 'now-24h', + fromStr: 'now/d', kind: 'relative', to: '2020-07-08T08:20:18.966Z', - toStr: 'now', + toStr: 'now/d', }, query: { query: '', language: 'kuery' }, filters: [], @@ -293,10 +293,10 @@ describe('Inputs', () => { queries: [], timerange: { from: '2020-07-07T08:20:18.966Z', - fromStr: 'now-24h', + fromStr: 'now/d', kind: 'relative', to: '2020-07-08T08:20:18.966Z', - toStr: 'now', + toStr: 'now/d', }, query: { query: '', language: 'kuery' }, filters: [], diff --git a/x-pack/plugins/security_solution/public/common/utils/default_date_settings.test.ts b/x-pack/plugins/security_solution/public/common/utils/default_date_settings.test.ts index 26d25a49f43cf..2c5f9d2dda471 100644 --- a/x-pack/plugins/security_solution/public/common/utils/default_date_settings.test.ts +++ b/x-pack/plugins/security_solution/public/common/utils/default_date_settings.test.ts @@ -315,6 +315,22 @@ describe('getTimeRangeSettings', () => { expect(to).toBe(new Date(DEFAULT_TO_DATE).toISOString()); }); + test('should round up "to" when from and to are both "now/d"', () => { + const mockTo = 'now/d'; + const mockFrom = 'now/d'; + mockTimeRange({ from: mockFrom, to: mockTo }); + const { to } = getTimeRangeSettings(); + expect(to).toContain('59:59.999Z'); + }); + + test('should round up "to" when from and to are different date maths', () => { + const mockTo = 'now/d'; + const mockFrom = 'now/d+1'; + mockTimeRange({ from: mockFrom, to: mockTo }); + const { to } = getTimeRangeSettings(); + expect(to).toContain('59:59.999Z'); + }); + test('should return the DEFAULT_TO_DATE when the from value is malformed', () => { const malformedTimeRange = { from: true }; if (isMalformedTimeRange(malformedTimeRange)) { @@ -506,5 +522,10 @@ describe('getIntervalSettings', () => { const value = parseDateWithDefault('trashed string', moment('1950-05-31T13:03:54.234Z')); expect(value.toISOString()).toBe(new Date('1950-05-31T13:03:54.234Z').toISOString()); }); + + test('should round up a valid date string and end with 59:59.999Z', () => { + const value = parseDateWithDefault('now/d', moment('1950-05-31T13:03:54.234Z'), true); + expect(value.toISOString()).toContain('59:59.999Z'); + }); }); }); diff --git a/x-pack/plugins/security_solution/public/common/utils/default_date_settings.ts b/x-pack/plugins/security_solution/public/common/utils/default_date_settings.ts index 7a8645cd0eb10..ed747dbcf22bb 100644 --- a/x-pack/plugins/security_solution/public/common/utils/default_date_settings.ts +++ b/x-pack/plugins/security_solution/public/common/utils/default_date_settings.ts @@ -51,8 +51,7 @@ export const getTimeRangeSettings = (uiSettings = true) => { const fromStr = (isString(timeRange?.from) && timeRange?.from) || DEFAULT_FROM; const toStr = (isString(timeRange?.to) && timeRange?.to) || DEFAULT_TO; const from = parseDateWithDefault(fromStr, DEFAULT_FROM_MOMENT).toISOString(); - const to = parseDateWithDefault(toStr, DEFAULT_TO_MOMENT).toISOString(); - + const to = parseDateWithDefault(toStr, DEFAULT_TO_MOMENT, true).toISOString(); return { from, fromStr, to, toStr }; }; @@ -72,11 +71,18 @@ export const getIntervalSettings = (uiSettings = true): Policy => { return { kind, duration }; }; +/** + * Parses a date and returns the default if the date string is not valid. + * @param dateString The date string to parse + * @param defaultDate The defaultDate if we cannot parse the dateMath + * @returns The moment of the date time parsed + */ export const parseDateWithDefault = ( dateString: string, - defaultDate: moment.Moment + defaultDate: moment.Moment, + roundUp: boolean = false ): moment.Moment => { - const date = dateMath.parse(dateString); + const date = dateMath.parse(dateString, { roundUp }); if (date != null && date.isValid()) { return date; } else { diff --git a/x-pack/plugins/security_solution/public/overview/components/recent_cases/no_cases/index.test.tsx b/x-pack/plugins/security_solution/public/overview/components/recent_cases/no_cases/index.test.tsx index beb55b3833ad4..ccb2d776f6e61 100644 --- a/x-pack/plugins/security_solution/public/overview/components/recent_cases/no_cases/index.test.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/recent_cases/no_cases/index.test.tsx @@ -34,7 +34,7 @@ describe('RecentCases', () => { wrapper.find(`[data-test-subj="no-cases-create-case"]`).first().simulate('click'); expect(navigateToApp).toHaveBeenCalledWith('securitySolution:case', { path: - "/create?sourcerer=(default:!('apm-*-transaction*','auditbeat-*','endgame-*','filebeat-*','logs-*','packetbeat-*','winlogbeat-*'))&timerange=(global:(linkTo:!(timeline),timerange:(from:'2020-07-07T08:20:18.966Z',fromStr:now-24h,kind:relative,to:'2020-07-08T08:20:18.966Z',toStr:now)),timeline:(linkTo:!(global),timerange:(from:'2020-07-07T08:20:18.966Z',fromStr:now-24h,kind:relative,to:'2020-07-08T08:20:18.966Z',toStr:now)))", + "/create?sourcerer=(default:!('apm-*-transaction*','auditbeat-*','endgame-*','filebeat-*','logs-*','packetbeat-*','winlogbeat-*'))&timerange=(global:(linkTo:!(timeline),timerange:(from:'2020-07-07T08:20:18.966Z',fromStr:now%2Fd,kind:relative,to:'2020-07-08T08:20:18.966Z',toStr:now%2Fd)),timeline:(linkTo:!(global),timerange:(from:'2020-07-07T08:20:18.966Z',fromStr:now%2Fd,kind:relative,to:'2020-07-08T08:20:18.966Z',toStr:now%2Fd)))", }); }); }); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/index.test.tsx index fef579d78b620..0f0f2a8641ac0 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/index.test.tsx @@ -96,8 +96,8 @@ describe('Timeline QueryBar ', () => { ); const queryBarProps = wrapper.find(QueryBar).props(); - expect(queryBarProps.dateRangeFrom).toEqual('now-24h'); - expect(queryBarProps.dateRangeTo).toEqual('now'); + expect(queryBarProps.dateRangeFrom).toEqual('now/d'); + expect(queryBarProps.dateRangeTo).toEqual('now/d'); expect(queryBarProps.filterQuery).toEqual({ query: 'here: query', language: 'kuery' }); expect(queryBarProps.savedQuery).toEqual(undefined); expect(queryBarProps.filters).toHaveLength(1); From abb71b471fd791c3bd61109b3d07cf438d37f438 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Thu, 4 Mar 2021 16:22:58 -0500 Subject: [PATCH 4/7] can not query the world (#93556) --- .../components/timeline/pinned_tab_content/index.tsx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/pinned_tab_content/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/pinned_tab_content/index.tsx index 68461a7234d09..62aa4f0c2456d 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/pinned_tab_content/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/pinned_tab_content/index.tsx @@ -21,6 +21,7 @@ import { StatefulBody } from '../body'; import { Footer, footerHeight } from '../footer'; import { requiredFieldsForActions } from '../../../../detections/components/alerts_table/default_config'; import { EventDetailsWidthProvider } from '../../../../common/components/events_viewer/event_details_width_context'; +import { sourcererSelectors } from '../../../../common/store/sourcerer'; import { SourcererScopeName } from '../../../../common/store/sourcerer/model'; import { timelineDefaults } from '../../../store/timeline/defaults'; import { useSourcererScope } from '../../../../common/containers/sourcerer'; @@ -30,6 +31,7 @@ import { State } from '../../../../common/store'; import { calculateTotalPages } from '../helpers'; import { TimelineTabs } from '../../../../../common/types/timeline'; import { DetailsPanel } from '../../side_panel'; +import { useDeepEqualSelector } from '../../../../common/hooks/use_selector'; const StyledEuiFlyoutBody = styled(EuiFlyoutBody)` overflow-y: hidden; @@ -97,6 +99,12 @@ export const PinnedTabContentComponent: React.FC = ({ SourcererScopeName.timeline ); + const existingIndexNamesSelector = useMemo( + () => sourcererSelectors.getAllExistingIndexNamesSelector(), + [] + ); + const existingIndexNames = useDeepEqualSelector(existingIndexNamesSelector); + const filterQuery = useMemo(() => { if (isEmpty(pinnedEventIds)) { return ''; @@ -159,7 +167,7 @@ export const PinnedTabContentComponent: React.FC = ({ docValueFields, endDate: '', id: `pinned-${timelineId}`, - indexNames: [''], + indexNames: existingIndexNames, fields: timelineQueryFields, limit: itemsPerPage, filterQuery, From 0d4ca46cf067e67ca8a2d8a04d4a2dbc65d40048 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Thu, 4 Mar 2021 13:34:47 -0800 Subject: [PATCH 5/7] [Fleet] Set all keyword and text fields for `index.query.default_field` index template setting (#91791) * Set all keyword and text fields for `index.query.default_field` setting * Update tests and snapshots * Fix test * Add default field limit safeguard * Add logging when beyond limit * Update tests to mock app context (because I added logger usage) * Update api integration test * Rename consts --- .../fleet/server/{mocks.ts => mocks/index.ts} | 20 +- .../__snapshots__/template.test.ts.snap | 62 +- .../elasticsearch/template/install.test.ts | 352 ++++---- .../epm/elasticsearch/template/install.ts | 4 +- .../elasticsearch/template/template.test.ts | 839 +++++++++--------- .../epm/elasticsearch/template/template.ts | 58 +- .../apis/epm/template.ts | 20 + 7 files changed, 744 insertions(+), 611 deletions(-) rename x-pack/plugins/fleet/server/{mocks.ts => mocks/index.ts} (82%) diff --git a/x-pack/plugins/fleet/server/mocks.ts b/x-pack/plugins/fleet/server/mocks/index.ts similarity index 82% rename from x-pack/plugins/fleet/server/mocks.ts rename to x-pack/plugins/fleet/server/mocks/index.ts index ea3ba20c59e9c..cff80f533d5e3 100644 --- a/x-pack/plugins/fleet/server/mocks.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -4,22 +4,18 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import { elasticsearchServiceMock, loggingSystemMock, savedObjectsServiceMock, -} from 'src/core/server/mocks'; - -import { coreMock } from '../../../../src/core/server/mocks'; -import { licensingMock } from '../../../plugins/licensing/server/mocks'; - -import { encryptedSavedObjectsMock } from '../../encrypted_saved_objects/server/mocks'; -import { securityMock } from '../../security/server/mocks'; - -import type { FleetAppContext } from './plugin'; -import type { PackagePolicyServiceInterface } from './services/package_policy'; -import type { AgentPolicyServiceInterface, AgentService } from './services'; + coreMock, +} from '../../../../../src/core/server/mocks'; +import { licensingMock } from '../../../../plugins/licensing/server/mocks'; +import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks'; +import { securityMock } from '../../../security/server/mocks'; +import type { PackagePolicyServiceInterface } from '../services/package_policy'; +import type { AgentPolicyServiceInterface, AgentService } from '../services'; +import type { FleetAppContext } from '../plugin'; export const createAppContextStartContractMock = (): FleetAppContext => { return { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index 2d2478843c454..65eec939d5850 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`tests loading base.yml: base.yml 1`] = ` +exports[`EPM template tests loading base.yml: base.yml 1`] = ` { "priority": 200, "index_patterns": [ @@ -20,12 +20,12 @@ exports[`tests loading base.yml: base.yml 1`] = ` }, "refresh_interval": "5s", "number_of_shards": "1", + "number_of_routing_shards": "30", "query": { "default_field": [ - "message" + "long.nested.foo" ] - }, - "number_of_routing_shards": "30" + } } }, "mappings": { @@ -109,7 +109,7 @@ exports[`tests loading base.yml: base.yml 1`] = ` } `; -exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` +exports[`EPM template tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` { "priority": 200, "index_patterns": [ @@ -129,12 +129,17 @@ exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` }, "refresh_interval": "5s", "number_of_shards": "1", + "number_of_routing_shards": "30", "query": { "default_field": [ - "message" + "coredns.id", + "coredns.query.class", + "coredns.query.name", + "coredns.query.type", + "coredns.response.code", + "coredns.response.flags" ] - }, - "number_of_routing_shards": "30" + } } }, "mappings": { @@ -218,7 +223,7 @@ exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` } `; -exports[`tests loading system.yml: system.yml 1`] = ` +exports[`EPM template tests loading system.yml: system.yml 1`] = ` { "priority": 200, "index_patterns": [ @@ -238,12 +243,45 @@ exports[`tests loading system.yml: system.yml 1`] = ` }, "refresh_interval": "5s", "number_of_shards": "1", + "number_of_routing_shards": "30", "query": { "default_field": [ - "message" + "system.diskio.name", + "system.diskio.serial_number", + "system.filesystem.device_name", + "system.filesystem.type", + "system.filesystem.mount_point", + "system.network.name", + "system.process.state", + "system.process.cmdline", + "system.process.cgroup.id", + "system.process.cgroup.path", + "system.process.cgroup.cpu.id", + "system.process.cgroup.cpu.path", + "system.process.cgroup.cpuacct.id", + "system.process.cgroup.cpuacct.path", + "system.process.cgroup.memory.id", + "system.process.cgroup.memory.path", + "system.process.cgroup.blkio.id", + "system.process.cgroup.blkio.path", + "system.raid.name", + "system.raid.status", + "system.raid.level", + "system.raid.sync_action", + "system.socket.remote.host", + "system.socket.remote.etld_plus_one", + "system.socket.remote.host_error", + "system.socket.process.cmdline", + "system.users.id", + "system.users.seat", + "system.users.path", + "system.users.type", + "system.users.service", + "system.users.state", + "system.users.scope", + "system.users.remote_host" ] - }, - "number_of_routing_shards": "30" + } } }, "mappings": { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index a580f58d4fed1..d68d7715436a3 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -4,194 +4,206 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import { elasticsearchServiceMock } from 'src/core/server/mocks'; +import { createAppContextStartContractMock } from '../../../../mocks'; +import { appContextService } from '../../../../services'; + import type { RegistryDataStream } from '../../../../types'; import type { Field } from '../../fields/field'; import { installTemplate } from './install'; -test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { index_templates: [] }; - } +describe('EPM install', () => { + beforeEach(async () => { + appContextService.start(createAppContextStartContractMock()); }); - const fields: Field[] = []; - const dataStreamDatasetIsPrefixUnset = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - } as RegistryDataStream; - const pkg = { - name: 'package', - version: '0.0.1', - }; - const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixUnset = 200; - await installTemplate({ - callCluster, - fields, - dataStream: dataStreamDatasetIsPrefixUnset, - packageVersion: pkg.version, - packageName: pkg.name, - }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; - expect(sentTemplate).toBeDefined(); - expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); - expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); -}); + it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() + .callAsCurrentUser; + callCluster.mockImplementation(async (_, params) => { + if ( + params && + params.method === 'GET' && + params.path === '/_index_template/metrics-package.dataset' + ) { + return { index_templates: [] }; + } + }); -test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { index_templates: [] }; - } + const fields: Field[] = []; + const dataStreamDatasetIsPrefixUnset = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixUnset = 200; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixUnset, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[1][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); }); - const fields: Field[] = []; - const dataStreamDatasetIsPrefixFalse = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - dataset_is_prefix: false, - } as RegistryDataStream; - const pkg = { - name: 'package', - version: '0.0.1', - }; - const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixFalse = 200; - await installTemplate({ - callCluster, - fields, - dataStream: dataStreamDatasetIsPrefixFalse, - packageVersion: pkg.version, - packageName: pkg.name, - }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; - expect(sentTemplate).toBeDefined(); - expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse); - expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]); -}); + it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() + .callAsCurrentUser; + callCluster.mockImplementation(async (_, params) => { + if ( + params && + params.method === 'GET' && + params.path === '/_index_template/metrics-package.dataset' + ) { + return { index_templates: [] }; + } + }); -test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { index_templates: [] }; - } + const fields: Field[] = []; + const dataStreamDatasetIsPrefixFalse = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: false, + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixFalse = 200; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixFalse, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[1][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]); }); - const fields: Field[] = []; - const dataStreamDatasetIsPrefixTrue = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - dataset_is_prefix: true, - } as RegistryDataStream; - const pkg = { - name: 'package', - version: '0.0.1', - }; - const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; - const templatePriorityDatasetIsPrefixTrue = 150; - await installTemplate({ - callCluster, - fields, - dataStream: dataStreamDatasetIsPrefixTrue, - packageVersion: pkg.version, - packageName: pkg.name, + it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() + .callAsCurrentUser; + callCluster.mockImplementation(async (_, params) => { + if ( + params && + params.method === 'GET' && + params.path === '/_index_template/metrics-package.dataset' + ) { + return { index_templates: [] }; + } + }); + + const fields: Field[] = []; + const dataStreamDatasetIsPrefixTrue = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: true, + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; + const templatePriorityDatasetIsPrefixTrue = 150; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixTrue, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[1][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]); }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; - expect(sentTemplate).toBeDefined(); - expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue); - expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]); -}); -test('tests installPackage remove the aliases property if the property existed', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { - index_templates: [ - { - name: 'metrics-package.dataset', - index_template: { - index_patterns: ['metrics-package.dataset-*'], - template: { aliases: {} }, + it('tests installPackage remove the aliases property if the property existed', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() + .callAsCurrentUser; + callCluster.mockImplementation(async (_, params) => { + if ( + params && + params.method === 'GET' && + params.path === '/_index_template/metrics-package.dataset' + ) { + return { + index_templates: [ + { + name: 'metrics-package.dataset', + index_template: { + index_patterns: ['metrics-package.dataset-*'], + template: { aliases: {} }, + }, }, - }, - ], - }; - } - }); + ], + }; + } + }); - const fields: Field[] = []; - const dataStreamDatasetIsPrefixUnset = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - } as RegistryDataStream; - const pkg = { - name: 'package', - version: '0.0.1', - }; - const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixUnset = 200; - await installTemplate({ - callCluster, - fields, - dataStream: dataStreamDatasetIsPrefixUnset, - packageVersion: pkg.version, - packageName: pkg.name, - }); + const fields: Field[] = []; + const dataStreamDatasetIsPrefixUnset = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixUnset = 200; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixUnset, + packageVersion: pkg.version, + packageName: pkg.name, + }); - // @ts-ignore - const removeAliases = callCluster.mock.calls[1][1].body; - expect(removeAliases.template.aliases).not.toBeDefined(); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[2][1].body; - expect(sentTemplate).toBeDefined(); - expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); - expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); + // @ts-ignore + const removeAliases = callCluster.mock.calls[1][1].body; + expect(removeAliases.template.aliases).not.toBeDefined(); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[2][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 28cfda3fd4189..2769b97fe48a1 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -300,7 +300,8 @@ export async function installTemplate({ packageVersion: string; packageName: string; }): Promise { - const mappings = generateMappings(processFields(fields)); + const validFields = processFields(fields); + const mappings = generateMappings(validFields); const templateName = generateTemplateName(dataStream); const templateIndexPattern = generateTemplateIndexPattern(dataStream); const templatePriority = getTemplatePriority(dataStream); @@ -362,6 +363,7 @@ export async function installTemplate({ const template = getTemplate({ type: dataStream.type, templateIndexPattern, + fields: validFields, mappings, pipelineName, packageName, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 0f53d260592e7..df82aa90b5a13 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -10,6 +10,9 @@ import path from 'path'; import { safeLoad } from 'js-yaml'; +import { createAppContextStartContractMock } from '../../../../mocks'; +import { appContextService } from '../../../../services'; + import type { RegistryDataStream } from '../../../../types'; import { processFields } from '../../fields/field'; import type { Field } from '../../fields/field'; @@ -32,132 +35,145 @@ expect.addSnapshotSerializer({ }, }); -test('get template', () => { - const templateIndexPattern = 'logs-nginx.access-abcd-*'; - - const template = getTemplate({ - type: 'logs', - templateIndexPattern, - packageName: 'nginx', - mappings: { properties: {} }, - composedOfTemplates: [], - templatePriority: 200, +describe('EPM template', () => { + beforeEach(async () => { + appContextService.start(createAppContextStartContractMock()); }); - expect(template.index_patterns).toStrictEqual([templateIndexPattern]); -}); - -test('adds composed_of correctly', () => { - const composedOfTemplates = ['component1', 'component2']; - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'name-*', - packageName: 'nginx', - mappings: { properties: {} }, - composedOfTemplates, - templatePriority: 200, + it('get template', () => { + const templateIndexPattern = 'logs-nginx.access-abcd-*'; + + const template = getTemplate({ + type: 'logs', + templateIndexPattern, + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates: [], + templatePriority: 200, + }); + expect(template.index_patterns).toStrictEqual([templateIndexPattern]); }); - expect(template.composed_of).toStrictEqual(composedOfTemplates); -}); - -test('adds empty composed_of correctly', () => { - const composedOfTemplates: string[] = []; - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'name-*', - packageName: 'nginx', - mappings: { properties: {} }, - composedOfTemplates, - templatePriority: 200, + it('adds composed_of correctly', () => { + const composedOfTemplates = ['component1', 'component2']; + + const template = getTemplate({ + type: 'logs', + templateIndexPattern: 'name-*', + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates, + templatePriority: 200, + }); + expect(template.composed_of).toStrictEqual(composedOfTemplates); }); - expect(template.composed_of).toStrictEqual(composedOfTemplates); -}); -test('adds hidden field correctly', () => { - const templateIndexPattern = 'logs-nginx.access-abcd-*'; - - const templateWithHidden = getTemplate({ - type: 'logs', - templateIndexPattern, - packageName: 'nginx', - mappings: { properties: {} }, - composedOfTemplates: [], - templatePriority: 200, - hidden: true, - }); - expect(templateWithHidden.data_stream.hidden).toEqual(true); - - const templateWithoutHidden = getTemplate({ - type: 'logs', - templateIndexPattern, - packageName: 'nginx', - mappings: { properties: {} }, - composedOfTemplates: [], - templatePriority: 200, + it('adds empty composed_of correctly', () => { + const composedOfTemplates: string[] = []; + + const template = getTemplate({ + type: 'logs', + templateIndexPattern: 'name-*', + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates, + templatePriority: 200, + }); + expect(template.composed_of).toStrictEqual(composedOfTemplates); }); - expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined); -}); -test('tests loading base.yml', () => { - const ymlPath = path.join(__dirname, '../../fields/tests/base.yml'); - const fieldsYML = readFileSync(ymlPath, 'utf-8'); - const fields: Field[] = safeLoad(fieldsYML); - - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'foo-*', - packageName: 'nginx', - mappings, - composedOfTemplates: [], - templatePriority: 200, + it('adds hidden field correctly', () => { + const templateIndexPattern = 'logs-nginx.access-abcd-*'; + + const templateWithHidden = getTemplate({ + type: 'logs', + templateIndexPattern, + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates: [], + templatePriority: 200, + hidden: true, + }); + expect(templateWithHidden.data_stream.hidden).toEqual(true); + + const templateWithoutHidden = getTemplate({ + type: 'logs', + templateIndexPattern, + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates: [], + templatePriority: 200, + }); + expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined); }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); -}); - -test('tests loading coredns.logs.yml', () => { - const ymlPath = path.join(__dirname, '../../fields/tests/coredns.logs.yml'); - const fieldsYML = readFileSync(ymlPath, 'utf-8'); - const fields: Field[] = safeLoad(fieldsYML); - - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'foo-*', - packageName: 'coredns', - mappings, - composedOfTemplates: [], - templatePriority: 200, + it('tests loading base.yml', () => { + const ymlPath = path.join(__dirname, '../../fields/tests/base.yml'); + const fieldsYML = readFileSync(ymlPath, 'utf-8'); + const fields: Field[] = safeLoad(fieldsYML); + + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + const template = getTemplate({ + type: 'logs', + templateIndexPattern: 'foo-*', + packageName: 'nginx', + fields: processedFields, + mappings, + composedOfTemplates: [], + templatePriority: 200, + }); + + expect(template).toMatchSnapshot(path.basename(ymlPath)); }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); -}); - -test('tests loading system.yml', () => { - const ymlPath = path.join(__dirname, '../../fields/tests/system.yml'); - const fieldsYML = readFileSync(ymlPath, 'utf-8'); - const fields: Field[] = safeLoad(fieldsYML); - - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'metrics', - templateIndexPattern: 'whatsthis-*', - packageName: 'system', - mappings, - composedOfTemplates: [], - templatePriority: 200, + it('tests loading coredns.logs.yml', () => { + const ymlPath = path.join(__dirname, '../../fields/tests/coredns.logs.yml'); + const fieldsYML = readFileSync(ymlPath, 'utf-8'); + const fields: Field[] = safeLoad(fieldsYML); + + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + const template = getTemplate({ + type: 'logs', + templateIndexPattern: 'foo-*', + packageName: 'coredns', + fields: processedFields, + mappings, + composedOfTemplates: [], + templatePriority: 200, + }); + + expect(template).toMatchSnapshot(path.basename(ymlPath)); }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); -}); + it('tests loading system.yml', () => { + const ymlPath = path.join(__dirname, '../../fields/tests/system.yml'); + const fieldsYML = readFileSync(ymlPath, 'utf-8'); + const fields: Field[] = safeLoad(fieldsYML); + + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + const template = getTemplate({ + type: 'metrics', + templateIndexPattern: 'whatsthis-*', + packageName: 'system', + fields: processedFields, + mappings, + composedOfTemplates: [], + templatePriority: 200, + }); + + expect(template).toMatchSnapshot(path.basename(ymlPath)); + }); -test('tests processing text field with multi fields', () => { - const textWithMultiFieldsLiteralYml = ` + it('tests processing text field with multi fields', () => { + const textWithMultiFieldsLiteralYml = ` - name: textWithMultiFields type: text multi_fields: @@ -166,30 +182,30 @@ test('tests processing text field with multi fields', () => { - name: indexed type: text `; - const textWithMultiFieldsMapping = { - properties: { - textWithMultiFields: { - type: 'text', - fields: { - raw: { - ignore_above: 1024, - type: 'keyword', - }, - indexed: { - type: 'text', + const textWithMultiFieldsMapping = { + properties: { + textWithMultiFields: { + type: 'text', + fields: { + raw: { + ignore_above: 1024, + type: 'keyword', + }, + indexed: { + type: 'text', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(textWithMultiFieldsLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(textWithMultiFieldsMapping); -}); + }; + const fields: Field[] = safeLoad(textWithMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(textWithMultiFieldsMapping); + }); -test('tests processing keyword field with multi fields', () => { - const keywordWithMultiFieldsLiteralYml = ` + it('tests processing keyword field with multi fields', () => { + const keywordWithMultiFieldsLiteralYml = ` - name: keywordWithMultiFields type: keyword multi_fields: @@ -199,31 +215,31 @@ test('tests processing keyword field with multi fields', () => { type: text `; - const keywordWithMultiFieldsMapping = { - properties: { - keywordWithMultiFields: { - ignore_above: 1024, - type: 'keyword', - fields: { - raw: { - ignore_above: 1024, - type: 'keyword', - }, - indexed: { - type: 'text', + const keywordWithMultiFieldsMapping = { + properties: { + keywordWithMultiFields: { + ignore_above: 1024, + type: 'keyword', + fields: { + raw: { + ignore_above: 1024, + type: 'keyword', + }, + indexed: { + type: 'text', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(keywordWithMultiFieldsLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(keywordWithMultiFieldsMapping); -}); + }; + const fields: Field[] = safeLoad(keywordWithMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(keywordWithMultiFieldsMapping); + }); -test('tests processing keyword field with multi fields with analyzed text field', () => { - const keywordWithAnalyzedMultiFieldsLiteralYml = ` + it('tests processing keyword field with multi fields with analyzed text field', () => { + const keywordWithAnalyzedMultiFieldsLiteralYml = ` - name: keywordWithAnalyzedMultiField type: keyword multi_fields: @@ -233,29 +249,29 @@ test('tests processing keyword field with multi fields with analyzed text field' search_analyzer: standard `; - const keywordWithAnalyzedMultiFieldsMapping = { - properties: { - keywordWithAnalyzedMultiField: { - ignore_above: 1024, - type: 'keyword', - fields: { - analyzed: { - analyzer: 'autocomplete', - search_analyzer: 'standard', - type: 'text', + const keywordWithAnalyzedMultiFieldsMapping = { + properties: { + keywordWithAnalyzedMultiField: { + ignore_above: 1024, + type: 'keyword', + fields: { + analyzed: { + analyzer: 'autocomplete', + search_analyzer: 'standard', + type: 'text', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(keywordWithAnalyzedMultiFieldsLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping); -}); + }; + const fields: Field[] = safeLoad(keywordWithAnalyzedMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping); + }); -test('tests processing keyword field with multi fields with normalized keyword field', () => { - const keywordWithNormalizedMultiFieldsLiteralYml = ` + it('tests processing keyword field with multi fields with normalized keyword field', () => { + const keywordWithNormalizedMultiFieldsLiteralYml = ` - name: keywordWithNormalizedMultiField type: keyword multi_fields: @@ -264,235 +280,235 @@ test('tests processing keyword field with multi fields with normalized keyword f normalizer: lowercase `; - const keywordWithNormalizedMultiFieldsMapping = { - properties: { - keywordWithNormalizedMultiField: { - ignore_above: 1024, - type: 'keyword', - fields: { - normalized: { - type: 'keyword', - ignore_above: 1024, - normalizer: 'lowercase', + const keywordWithNormalizedMultiFieldsMapping = { + properties: { + keywordWithNormalizedMultiField: { + ignore_above: 1024, + type: 'keyword', + fields: { + normalized: { + type: 'keyword', + ignore_above: 1024, + normalizer: 'lowercase', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(keywordWithNormalizedMultiFieldsLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(keywordWithNormalizedMultiFieldsMapping); -}); + }; + const fields: Field[] = safeLoad(keywordWithNormalizedMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(keywordWithNormalizedMultiFieldsMapping); + }); -test('tests processing object field with no other attributes', () => { - const objectFieldLiteralYml = ` + it('tests processing object field with no other attributes', () => { + const objectFieldLiteralYml = ` - name: objectField type: object `; - const objectFieldMapping = { - properties: { - objectField: { - type: 'object', + const objectFieldMapping = { + properties: { + objectField: { + type: 'object', + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldMapping); + }); -test('tests processing object field with enabled set to false', () => { - const objectFieldEnabledFalseLiteralYml = ` + it('tests processing object field with enabled set to false', () => { + const objectFieldEnabledFalseLiteralYml = ` - name: objectField type: object enabled: false `; - const objectFieldEnabledFalseMapping = { - properties: { - objectField: { - type: 'object', - enabled: false, + const objectFieldEnabledFalseMapping = { + properties: { + objectField: { + type: 'object', + enabled: false, + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldEnabledFalseLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldEnabledFalseMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldEnabledFalseLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldEnabledFalseMapping); + }); -test('tests processing object field with dynamic set to false', () => { - const objectFieldDynamicFalseLiteralYml = ` + it('tests processing object field with dynamic set to false', () => { + const objectFieldDynamicFalseLiteralYml = ` - name: objectField type: object dynamic: false `; - const objectFieldDynamicFalseMapping = { - properties: { - objectField: { - type: 'object', - dynamic: false, + const objectFieldDynamicFalseMapping = { + properties: { + objectField: { + type: 'object', + dynamic: false, + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldDynamicFalseLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldDynamicFalseMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldDynamicFalseLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldDynamicFalseMapping); + }); -test('tests processing object field with dynamic set to true', () => { - const objectFieldDynamicTrueLiteralYml = ` + it('tests processing object field with dynamic set to true', () => { + const objectFieldDynamicTrueLiteralYml = ` - name: objectField type: object dynamic: true `; - const objectFieldDynamicTrueMapping = { - properties: { - objectField: { - type: 'object', - dynamic: true, + const objectFieldDynamicTrueMapping = { + properties: { + objectField: { + type: 'object', + dynamic: true, + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldDynamicTrueLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldDynamicTrueMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldDynamicTrueLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldDynamicTrueMapping); + }); -test('tests processing object field with dynamic set to strict', () => { - const objectFieldDynamicStrictLiteralYml = ` + it('tests processing object field with dynamic set to strict', () => { + const objectFieldDynamicStrictLiteralYml = ` - name: objectField type: object dynamic: strict `; - const objectFieldDynamicStrictMapping = { - properties: { - objectField: { - type: 'object', - dynamic: 'strict', + const objectFieldDynamicStrictMapping = { + properties: { + objectField: { + type: 'object', + dynamic: 'strict', + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldDynamicStrictLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldDynamicStrictMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldDynamicStrictLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldDynamicStrictMapping); + }); -test('tests processing object field with property', () => { - const objectFieldWithPropertyLiteralYml = ` + it('tests processing object field with property', () => { + const objectFieldWithPropertyLiteralYml = ` - name: a type: object - name: a.b type: keyword `; - const objectFieldWithPropertyMapping = { - properties: { - a: { - properties: { - b: { - ignore_above: 1024, - type: 'keyword', + const objectFieldWithPropertyMapping = { + properties: { + a: { + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldWithPropertyLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldWithPropertyMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldWithPropertyLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldWithPropertyMapping); + }); -test('tests processing object field with property, reverse order', () => { - const objectFieldWithPropertyReversedLiteralYml = ` + it('tests processing object field with property, reverse order', () => { + const objectFieldWithPropertyReversedLiteralYml = ` - name: a.b type: keyword - name: a type: object dynamic: false `; - const objectFieldWithPropertyReversedMapping = { - properties: { - a: { - dynamic: false, - properties: { - b: { - ignore_above: 1024, - type: 'keyword', + const objectFieldWithPropertyReversedMapping = { + properties: { + a: { + dynamic: false, + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); + }); -test('tests processing nested field with property', () => { - const nestedYaml = ` + it('tests processing nested field with property', () => { + const nestedYaml = ` - name: a.b type: keyword - name: a type: nested dynamic: false `; - const expectedMapping = { - properties: { - a: { - dynamic: false, - type: 'nested', - properties: { - b: { - ignore_above: 1024, - type: 'keyword', + const expectedMapping = { + properties: { + a: { + dynamic: false, + type: 'nested', + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(nestedYaml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(expectedMapping); -}); + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); + }); -test('tests processing nested field with property, nested field first', () => { - const nestedYaml = ` + it('tests processing nested field with property, nested field first', () => { + const nestedYaml = ` - name: a type: nested include_in_parent: true - name: a.b type: keyword `; - const expectedMapping = { - properties: { - a: { - include_in_parent: true, - type: 'nested', - properties: { - b: { - ignore_above: 1024, - type: 'keyword', + const expectedMapping = { + properties: { + a: { + include_in_parent: true, + type: 'nested', + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(nestedYaml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(expectedMapping); -}); + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); + }); -test('tests processing nested leaf field with properties', () => { - const nestedYaml = ` + it('tests processing nested leaf field with properties', () => { + const nestedYaml = ` - name: a type: object dynamic: false @@ -500,98 +516,99 @@ test('tests processing nested leaf field with properties', () => { type: nested enabled: false `; - const expectedMapping = { - properties: { - a: { - dynamic: false, - properties: { - b: { - enabled: false, - type: 'nested', + const expectedMapping = { + properties: { + a: { + dynamic: false, + properties: { + b: { + enabled: false, + type: 'nested', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(nestedYaml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(expectedMapping); -}); + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); + }); -test('tests constant_keyword field type handling', () => { - const constantKeywordLiteralYaml = ` + it('tests constant_keyword field type handling', () => { + const constantKeywordLiteralYaml = ` - name: constantKeyword type: constant_keyword `; - const constantKeywordMapping = { - properties: { - constantKeyword: { - type: 'constant_keyword', + const constantKeywordMapping = { + properties: { + constantKeyword: { + type: 'constant_keyword', + }, }, - }, - }; - const fields: Field[] = safeLoad(constantKeywordLiteralYaml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(JSON.stringify(mappings)).toEqual(JSON.stringify(constantKeywordMapping)); -}); + }; + const fields: Field[] = safeLoad(constantKeywordLiteralYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(JSON.stringify(mappings)).toEqual(JSON.stringify(constantKeywordMapping)); + }); -test('tests priority and index pattern for data stream without dataset_is_prefix', () => { - const dataStreamDatasetIsPrefixUnset = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - } as RegistryDataStream; - const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixUnset = 200; - const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixUnset); - const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixUnset); - - expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixUnset); - expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixUnset); -}); + it('tests priority and index pattern for data stream without dataset_is_prefix', () => { + const dataStreamDatasetIsPrefixUnset = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixUnset = 200; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixUnset); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixUnset); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixUnset); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixUnset); + }); -test('tests priority and index pattern for data stream with dataset_is_prefix set to false', () => { - const dataStreamDatasetIsPrefixFalse = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - dataset_is_prefix: false, - } as RegistryDataStream; - const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixFalse = 200; - const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixFalse); - const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixFalse); - - expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixFalse); - expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixFalse); -}); + it('tests priority and index pattern for data stream with dataset_is_prefix set to false', () => { + const dataStreamDatasetIsPrefixFalse = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: false, + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixFalse = 200; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixFalse); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixFalse); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixFalse); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixFalse); + }); -test('tests priority and index pattern for data stream with dataset_is_prefix set to true', () => { - const dataStreamDatasetIsPrefixTrue = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - dataset_is_prefix: true, - } as RegistryDataStream; - const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; - const templatePriorityDatasetIsPrefixTrue = 150; - const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixTrue); - const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixTrue); - - expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixTrue); - expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixTrue); + it('tests priority and index pattern for data stream with dataset_is_prefix set to true', () => { + const dataStreamDatasetIsPrefixTrue = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: true, + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; + const templatePriorityDatasetIsPrefixTrue = 150; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixTrue); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixTrue); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixTrue); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixTrue); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index a13ad007663d8..01b9a92045b29 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -13,6 +13,7 @@ import type { IndexTemplate, IndexTemplateMappings, } from '../../../../types'; +import { appContextService } from '../../../'; import { getRegistryDataStreamAssetBaseName } from '../index'; interface Properties { @@ -37,6 +38,9 @@ const DEFAULT_IGNORE_ABOVE = 1024; const DEFAULT_TEMPLATE_PRIORITY = 200; const DATASET_IS_PREFIX_TEMPLATE_PRIORITY = 150; +const QUERY_DEFAULT_FIELD_TYPES = ['keyword', 'text']; +const QUERY_DEFAULT_FIELD_LIMIT = 1024; + /** * getTemplate retrieves the default template but overwrites the index pattern with the given value. * @@ -45,6 +49,7 @@ const DATASET_IS_PREFIX_TEMPLATE_PRIORITY = 150; export function getTemplate({ type, templateIndexPattern, + fields, mappings, pipelineName, packageName, @@ -55,6 +60,7 @@ export function getTemplate({ }: { type: string; templateIndexPattern: string; + fields: Fields; mappings: IndexTemplateMappings; pipelineName?: string | undefined; packageName: string; @@ -66,6 +72,7 @@ export function getTemplate({ const template = getBaseTemplate( type, templateIndexPattern, + fields, mappings, packageName, composedOfTemplates, @@ -296,9 +303,28 @@ export function generateESIndexPatterns( return patterns; } +const flattenFieldsToNameAndType = ( + fields: Fields, + path: string = '' +): Array> => { + let newFields: Array> = []; + fields.forEach((field) => { + const fieldName = path ? `${path}.${field.name}` : field.name; + newFields.push({ + name: fieldName, + type: field.type, + }); + if (field.fields && field.fields.length) { + newFields = newFields.concat(flattenFieldsToNameAndType(field.fields, fieldName)); + } + }); + return newFields; +}; + function getBaseTemplate( type: string, templateIndexPattern: string, + fields: Fields, mappings: IndexTemplateMappings, packageName: string, composedOfTemplates: string[], @@ -306,6 +332,8 @@ function getBaseTemplate( ilmPolicy?: string | undefined, hidden?: boolean ): IndexTemplate { + const logger = appContextService.getLogger(); + // Meta information to identify Ingest Manager's managed templates and indices const _meta = { package: { @@ -315,6 +343,21 @@ function getBaseTemplate( managed: true, }; + // Find all field names to set `index.query.default_field` to, which will be + // the first 1024 keyword or text fields + const defaultFields = flattenFieldsToNameAndType(fields).filter( + (field) => field.type && QUERY_DEFAULT_FIELD_TYPES.includes(field.type) + ); + if (defaultFields.length > QUERY_DEFAULT_FIELD_LIMIT) { + logger.warn( + `large amount of default fields detected for index template ${templateIndexPattern} in package ${packageName}, applying the first ${QUERY_DEFAULT_FIELD_LIMIT} fields` + ); + } + const defaultFieldNames = (defaultFields.length > QUERY_DEFAULT_FIELD_LIMIT + ? defaultFields.slice(0, QUERY_DEFAULT_FIELD_LIMIT) + : defaultFields + ).map((field) => field.name); + return { priority: templatePriority, // To be completed with the correct index patterns @@ -338,13 +381,18 @@ function getBaseTemplate( refresh_interval: '5s', // Default in the stack now, still good to have it in number_of_shards: '1', - // All the default fields which should be queried have to be added here. - // So far we add all keyword and text fields here. - query: { - default_field: ['message'], - }, // We are setting 30 because it can be devided by several numbers. Useful when shrinking. number_of_routing_shards: '30', + // All the default fields which should be queried have to be added here. + // So far we add all keyword and text fields here if there are any, otherwise + // this setting is skipped. + ...(defaultFieldNames.length + ? { + query: { + default_field: defaultFieldNames, + }, + } + : {}), }, }, mappings: { diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index d79452ca0eb6f..517d2c77d430d 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -7,6 +7,7 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { appContextService } from '../../../../plugins/fleet/server/services'; import { getTemplate } from '../../../../plugins/fleet/server/services/epm/elasticsearch/template/template'; export default function ({ getService }: FtrProviderContext) { @@ -20,12 +21,31 @@ export default function ({ getService }: FtrProviderContext) { }, }, }; + const fields = [ + { + name: 'foo', + type: 'keyword', + }, + ]; + // This test was inspired by https://github.com/elastic/kibana/blob/master/x-pack/test/api_integration/apis/monitoring/common/mappings_exist.js describe('EPM - template', async () => { + beforeEach(async () => { + appContextService.start({ + // @ts-ignore + elasticsearch: { client: {} }, + // @ts-ignore + logger: { + warn: () => {}, + }, + }); + }); + it('can be loaded', async () => { const template = getTemplate({ type: 'logs', templateIndexPattern, + fields, mappings, packageName: 'system', composedOfTemplates: [], From 8381fdd60ebc950f7d34ac9af7ada8b60551c7c9 Mon Sep 17 00:00:00 2001 From: Jason Stoltzfus Date: Thu, 4 Mar 2021 17:06:03 -0500 Subject: [PATCH 6/7] Hide Value and Funtional boost for geolocation (#93683) --- .../relevance_tuning/boosts/boosts.test.tsx | 14 ++++++++++++++ .../components/relevance_tuning/boosts/boosts.tsx | 4 +++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boosts.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boosts.test.tsx index 75c22d2ae9473..8a355b97e7b9f 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boosts.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boosts.test.tsx @@ -64,6 +64,20 @@ describe('Boosts', () => { expect(select.prop('options').map((o: any) => o.value)).toEqual(['add-boost', 'value']); }); + it('will not render functional or value options if "type" prop is "geolocation"', () => { + const wrapper = shallow( + + ); + + const select = wrapper.find(EuiSuperSelect); + expect(select.prop('options').map((o: any) => o.value)).toEqual(['add-boost', 'proximity']); + }); + it('will add a boost of the selected type when a selection is made', () => { const wrapper = shallow(); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boosts.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boosts.tsx index d6d43ea7beab0..4268e21110277 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boosts.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boosts.tsx @@ -13,7 +13,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiTitle, EuiSuperSelect } from '@ import { i18n } from '@kbn/i18n'; -import { TEXT } from '../../../../shared/constants/field_types'; +import { GEOLOCATION, TEXT } from '../../../../shared/constants/field_types'; import { SchemaTypes } from '../../../../shared/types'; import { BoostIcon } from '../boost_icon'; @@ -68,6 +68,8 @@ const BASE_OPTIONS = [ const filterInvalidOptions = (value: BoostType, type: SchemaTypes) => { // Proximity and Functional boost types are not valid for text fields if (type === TEXT && [BoostType.Proximity, BoostType.Functional].includes(value)) return false; + // Value and Functional boost types are not valid for geolocation fields + if (type === GEOLOCATION && [BoostType.Functional, BoostType.Value].includes(value)) return false; return true; }; From a21f48a9f2fe2a4777f38aa86f115717f305cba4 Mon Sep 17 00:00:00 2001 From: Devon Thomson Date: Thu, 4 Mar 2021 17:06:58 -0500 Subject: [PATCH 7/7] Make Dashboard Unsaved Changes Space Specific (#92680) * Made dashboard unsaved changes space specific --- api_docs/dashboard.json | 36 +++++++++---------- src/plugins/dashboard/kibana.json | 1 + .../public/application/dashboard_router.tsx | 10 +++++- .../lib/dashboard_panel_storage.ts | 20 ++++++----- src/plugins/dashboard/public/plugin.tsx | 2 ++ .../dashboard/public/services/spaces.ts | 9 +++++ src/plugins/dashboard/tsconfig.json | 1 + 7 files changed, 52 insertions(+), 27 deletions(-) create mode 100644 src/plugins/dashboard/public/services/spaces.ts diff --git a/api_docs/dashboard.json b/api_docs/dashboard.json index 0df54e577ede6..778bd8b6da64a 100644 --- a/api_docs/dashboard.json +++ b/api_docs/dashboard.json @@ -1389,15 +1389,15 @@ "description": [], "source": { "path": "src/plugins/dashboard/public/plugin.tsx", - "lineNumber": 89, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L89" + "lineNumber": 90, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L90" } } ], "source": { "path": "src/plugins/dashboard/public/plugin.tsx", - "lineNumber": 88, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L88" + "lineNumber": 89, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L89" }, "initialIsOpen": false }, @@ -1939,8 +1939,8 @@ "description": [], "source": { "path": "src/plugins/dashboard/public/plugin.tsx", - "lineNumber": 86, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L86" + "lineNumber": 87, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L87" }, "signature": [ "UrlGeneratorContract<\"DASHBOARD_APP_URL_GENERATOR\">" @@ -2103,8 +2103,8 @@ "description": [], "source": { "path": "src/plugins/dashboard/public/plugin.tsx", - "lineNumber": 118, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L118" + "lineNumber": 120, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L120" }, "signature": [ "void" @@ -2127,8 +2127,8 @@ "description": [], "source": { "path": "src/plugins/dashboard/public/plugin.tsx", - "lineNumber": 121, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L121" + "lineNumber": 123, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L123" }, "signature": [ "() => ", @@ -2149,8 +2149,8 @@ "description": [], "source": { "path": "src/plugins/dashboard/public/plugin.tsx", - "lineNumber": 122, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L122" + "lineNumber": 124, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L124" }, "signature": [ { @@ -2171,8 +2171,8 @@ "description": [], "source": { "path": "src/plugins/dashboard/public/plugin.tsx", - "lineNumber": 123, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L123" + "lineNumber": 125, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L125" }, "signature": [ { @@ -2192,8 +2192,8 @@ "description": [], "source": { "path": "src/plugins/dashboard/public/plugin.tsx", - "lineNumber": 124, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L124" + "lineNumber": 126, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L126" }, "signature": [ "React.FC" @@ -2202,8 +2202,8 @@ ], "source": { "path": "src/plugins/dashboard/public/plugin.tsx", - "lineNumber": 120, - "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L120" + "lineNumber": 122, + "link": "https://github.com/elastic/kibana/tree/mastersrc/plugins/dashboard/public/plugin.tsx#L122" }, "lifecycle": "start", "initialIsOpen": true diff --git a/src/plugins/dashboard/kibana.json b/src/plugins/dashboard/kibana.json index 8286a4badcbe5..32507cbc5e5f4 100644 --- a/src/plugins/dashboard/kibana.json +++ b/src/plugins/dashboard/kibana.json @@ -15,6 +15,7 @@ ], "optionalPlugins": [ "home", + "spacesOss", "savedObjectsTaggingOss", "usageCollection"], "server": true, diff --git a/src/plugins/dashboard/public/application/dashboard_router.tsx b/src/plugins/dashboard/public/application/dashboard_router.tsx index 5c849aa960a4c..f981b135c4359 100644 --- a/src/plugins/dashboard/public/application/dashboard_router.tsx +++ b/src/plugins/dashboard/public/application/dashboard_router.tsx @@ -13,6 +13,7 @@ import { parse, ParsedQuery } from 'query-string'; import { render, unmountComponentAtNode } from 'react-dom'; import { Switch, Route, RouteComponentProps, HashRouter, Redirect } from 'react-router-dom'; +import { first } from 'rxjs/operators'; import { DashboardListing } from './listing'; import { DashboardApp } from './dashboard_app'; import { addHelpMenuToAppChrome, DashboardPanelStorage } from './lib'; @@ -47,6 +48,7 @@ export const dashboardUrlParams = { export interface DashboardMountProps { appUnMounted: () => void; restorePreviousUrl: () => void; + scopedHistory: ScopedHistory; element: AppMountParameters['element']; initializerContext: PluginInitializerContext; @@ -80,6 +82,9 @@ export async function mountApp({ savedObjectsTaggingOss, } = pluginsStart; + const spacesApi = pluginsStart.spacesOss?.isSpacesAvailable ? pluginsStart.spacesOss : undefined; + const activeSpaceId = spacesApi && (await spacesApi.activeSpace$.pipe(first()).toPromise())?.id; + const dashboardServices: DashboardAppServices = { navigation, onAppLeave, @@ -99,7 +104,10 @@ export async function mountApp({ indexPatterns: dataStart.indexPatterns, savedQueryService: dataStart.query.savedQueries, savedObjectsClient: coreStart.savedObjects.client, - dashboardPanelStorage: new DashboardPanelStorage(core.notifications.toasts), + dashboardPanelStorage: new DashboardPanelStorage( + core.notifications.toasts, + activeSpaceId || 'default' + ), savedDashboards: dashboardStart.getSavedDashboardLoader(), savedObjectsTagging: savedObjectsTaggingOss?.getTaggingApi(), allowByValueEmbeddables: initializerContext.config.get() diff --git a/src/plugins/dashboard/public/application/lib/dashboard_panel_storage.ts b/src/plugins/dashboard/public/application/lib/dashboard_panel_storage.ts index fcd3a09dc6b4e..02890f6aaa790 100644 --- a/src/plugins/dashboard/public/application/lib/dashboard_panel_storage.ts +++ b/src/plugins/dashboard/public/application/lib/dashboard_panel_storage.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { set } from '@elastic/safer-lodash-set'; import { Storage } from '../../services/kibana_utils'; import { NotificationsStart } from '../../services/core'; import { panelStorageErrorStrings } from '../../dashboard_strings'; @@ -17,16 +18,17 @@ const DASHBOARD_PANELS_SESSION_KEY = 'dashboardStateManagerPanels'; export class DashboardPanelStorage { private sessionStorage: Storage; - constructor(private toasts: NotificationsStart['toasts']) { + constructor(private toasts: NotificationsStart['toasts'], private activeSpaceId: string) { this.sessionStorage = new Storage(sessionStorage); } public clearPanels(id = DASHBOARD_PANELS_UNSAVED_ID) { try { - const sessionStoragePanels = this.sessionStorage.get(DASHBOARD_PANELS_SESSION_KEY) || {}; - if (sessionStoragePanels[id]) { - delete sessionStoragePanels[id]; - this.sessionStorage.set(DASHBOARD_PANELS_SESSION_KEY, sessionStoragePanels); + const sessionStorage = this.sessionStorage.get(DASHBOARD_PANELS_SESSION_KEY); + const sessionStorageForSpace = sessionStorage?.[this.activeSpaceId] || {}; + if (sessionStorageForSpace[id]) { + delete sessionStorageForSpace[id]; + this.sessionStorage.set(DASHBOARD_PANELS_SESSION_KEY, sessionStorage); } } catch (e) { this.toasts.addDanger({ @@ -38,7 +40,7 @@ export class DashboardPanelStorage { public getPanels(id = DASHBOARD_PANELS_UNSAVED_ID): SavedDashboardPanel[] | undefined { try { - return this.sessionStorage.get(DASHBOARD_PANELS_SESSION_KEY)?.[id]; + return this.sessionStorage.get(DASHBOARD_PANELS_SESSION_KEY)?.[this.activeSpaceId]?.[id]; } catch (e) { this.toasts.addDanger({ title: panelStorageErrorStrings.getPanelsGetError(e.message), @@ -50,7 +52,7 @@ export class DashboardPanelStorage { public setPanels(id = DASHBOARD_PANELS_UNSAVED_ID, newPanels: SavedDashboardPanel[]) { try { const sessionStoragePanels = this.sessionStorage.get(DASHBOARD_PANELS_SESSION_KEY) || {}; - sessionStoragePanels[id] = newPanels; + set(sessionStoragePanels, [this.activeSpaceId, id], newPanels); this.sessionStorage.set(DASHBOARD_PANELS_SESSION_KEY, sessionStoragePanels); } catch (e) { this.toasts.addDanger({ @@ -62,7 +64,9 @@ export class DashboardPanelStorage { public getDashboardIdsWithUnsavedChanges() { try { - return Object.keys(this.sessionStorage.get(DASHBOARD_PANELS_SESSION_KEY) || {}); + return Object.keys( + this.sessionStorage.get(DASHBOARD_PANELS_SESSION_KEY)?.[this.activeSpaceId] || {} + ); } catch (e) { this.toasts.addDanger({ title: panelStorageErrorStrings.getPanelsGetError(e.message), diff --git a/src/plugins/dashboard/public/plugin.tsx b/src/plugins/dashboard/public/plugin.tsx index 4385e3e8744c2..ae2d2b5f237c9 100644 --- a/src/plugins/dashboard/public/plugin.tsx +++ b/src/plugins/dashboard/public/plugin.tsx @@ -76,6 +76,7 @@ import { UrlGeneratorState } from '../../share/public'; import { ExportCSVAction } from './application/actions/export_csv_action'; import { dashboardFeatureCatalog } from './dashboard_strings'; import { replaceUrlHashQuery } from '../../kibana_utils/public'; +import { SpacesOssPluginStart } from './services/spaces'; declare module '../../share/public' { export interface UrlGeneratorStateMapping { @@ -113,6 +114,7 @@ export interface DashboardStartDependencies { savedObjects: SavedObjectsStart; presentationUtil: PresentationUtilPluginStart; savedObjectsTaggingOss?: SavedObjectTaggingOssPluginStart; + spacesOss?: SpacesOssPluginStart; } export type DashboardSetup = void; diff --git a/src/plugins/dashboard/public/services/spaces.ts b/src/plugins/dashboard/public/services/spaces.ts new file mode 100644 index 0000000000000..e6d2c6400818f --- /dev/null +++ b/src/plugins/dashboard/public/services/spaces.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { SpacesOssPluginStart } from '../../../spaces_oss/public'; diff --git a/src/plugins/dashboard/tsconfig.json b/src/plugins/dashboard/tsconfig.json index ddda2f81d1f62..dd99119cfb457 100644 --- a/src/plugins/dashboard/tsconfig.json +++ b/src/plugins/dashboard/tsconfig.json @@ -31,5 +31,6 @@ { "path": "../saved_objects_tagging_oss/tsconfig.json" }, { "path": "../saved_objects/tsconfig.json" }, { "path": "../ui_actions/tsconfig.json" }, + { "path": "../spaces_oss/tsconfig.json" }, ] }