From bc363181cf15a4e9462552d784bbfa131a1f3580 Mon Sep 17 00:00:00 2001 From: igoristic Date: Fri, 4 Jun 2021 11:52:27 -0400 Subject: [PATCH] Allow . system indices in regex (#100831) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../common/es_glob_patterns.test.ts | 120 ++++++++++++++++++ .../server/alerts/large_shard_size_alert.ts | 2 +- .../lib/alerts/fetch_index_shard_size.ts | 6 +- 3 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 x-pack/plugins/monitoring/common/es_glob_patterns.test.ts diff --git a/x-pack/plugins/monitoring/common/es_glob_patterns.test.ts b/x-pack/plugins/monitoring/common/es_glob_patterns.test.ts new file mode 100644 index 0000000000000..64250d0b3c5ae --- /dev/null +++ b/x-pack/plugins/monitoring/common/es_glob_patterns.test.ts @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ESGlobPatterns } from './es_glob_patterns'; + +const testIndices = [ + '.kibana_task_manager_inifc_1', + '.kibana_shahzad_4', + '.kibana_shahzad_3', + '.kibana_shahzad_2', + '.kibana_shahzad_1', + '.kibana_task_manager_cmarcondes-24_8.0.0_001', + '.kibana_task_manager_custom_kbn-pr93025_8.0.0_001', + '.kibana_task_manager_spong_8.0.0_001', + '.ds-metrics-system.process.summary-default-2021.05.25-00000', + '.kibana_shahzad_9', + '.kibana-felix-log-stream_8.0.0_001', + '.kibana_smith_alerts-observability-apm-000001', + '.ds-logs-endpoint.events.process-default-2021.05.26-000001', + '.kibana_dominiqueclarke54_8.0.0_001', + '.kibana-cmarcondes-19_8.0.0_001', + '.kibana_task_manager_cmarcondes-17_8.0.0_001', + '.kibana_task_manager_jrhodes_8.0.0_001', + '.kibana_task_manager_dominiqueclarke7_8', + 'data_prod_0', + 'data_prod_1', + 'data_prod_2', + 'data_prod_3', + 'filebeat-8.0.0-2021.04.13-000001', + '.kibana_dominiqueclarke55-alerts-8.0.0-000001', + '.ds-metrics-system.socket_summary-default-2021.05.12-000001', + '.kibana_task_manager_dominiqueclarke24_8.0.0_001', + '.kibana_custom_kbn-pr94906_8.0.0_001', + '.kibana_task_manager_cmarcondes-22_8.0.0_001', + '.kibana_dominiqueclarke49-event-log-8.0.0-000001', + 'data_stage_2', + 'data_stage_3', +].sort(); + +const noSystemIndices = [ + 'data_prod_0', + 'data_prod_1', + 'data_prod_2', + 'data_prod_3', + 'filebeat-8.0.0-2021.04.13-000001', + 'data_stage_2', + 'data_stage_3', +].sort(); + +const onlySystemIndices = [ + '.kibana_task_manager_inifc_1', + '.kibana_shahzad_4', + '.kibana_shahzad_3', + '.kibana_shahzad_2', + '.kibana_shahzad_1', + '.kibana_task_manager_cmarcondes-24_8.0.0_001', + '.kibana_task_manager_custom_kbn-pr93025_8.0.0_001', + '.kibana_task_manager_spong_8.0.0_001', + '.ds-metrics-system.process.summary-default-2021.05.25-00000', + '.kibana_shahzad_9', + '.kibana-felix-log-stream_8.0.0_001', + '.kibana_smith_alerts-observability-apm-000001', + '.ds-logs-endpoint.events.process-default-2021.05.26-000001', + '.kibana_dominiqueclarke54_8.0.0_001', + '.kibana-cmarcondes-19_8.0.0_001', + '.kibana_task_manager_cmarcondes-17_8.0.0_001', + '.kibana_task_manager_jrhodes_8.0.0_001', + '.kibana_task_manager_dominiqueclarke7_8', + '.kibana_dominiqueclarke55-alerts-8.0.0-000001', + '.ds-metrics-system.socket_summary-default-2021.05.12-000001', + '.kibana_task_manager_dominiqueclarke24_8.0.0_001', + '.kibana_custom_kbn-pr94906_8.0.0_001', + '.kibana_task_manager_cmarcondes-22_8.0.0_001', + '.kibana_dominiqueclarke49-event-log-8.0.0-000001', +].sort(); + +const kibanaNoTaskIndices = [ + '.kibana_shahzad_4', + '.kibana_shahzad_3', + '.kibana_shahzad_2', + '.kibana_shahzad_1', + '.kibana_shahzad_9', + '.kibana-felix-log-stream_8.0.0_001', + '.kibana_smith_alerts-observability-apm-000001', + '.kibana_dominiqueclarke54_8.0.0_001', + '.kibana-cmarcondes-19_8.0.0_001', + '.kibana_dominiqueclarke55-alerts-8.0.0-000001', + '.kibana_custom_kbn-pr94906_8.0.0_001', + '.kibana_dominiqueclarke49-event-log-8.0.0-000001', +].sort(); + +describe('ES glob index patterns', () => { + it('should exclude system/internal indices', () => { + const validIndexPatterns = ESGlobPatterns.createRegExPatterns('-.*'); + const validIndices = testIndices.filter((index) => + ESGlobPatterns.isValid(index, validIndexPatterns) + ); + expect(validIndices.sort()).toEqual(noSystemIndices); + }); + + it('should only show ".index" system indices', () => { + const validIndexPatterns = ESGlobPatterns.createRegExPatterns('.*'); + const validIndices = testIndices.filter((index) => + ESGlobPatterns.isValid(index, validIndexPatterns) + ); + expect(validIndices.sort()).toEqual(onlySystemIndices); + }); + + it('should only show ".kibana*" indices without _task_', () => { + const validIndexPatterns = ESGlobPatterns.createRegExPatterns('.kibana*,-*_task_*'); + const validIndices = testIndices.filter((index) => + ESGlobPatterns.isValid(index, validIndexPatterns) + ); + expect(validIndices.sort()).toEqual(kibanaNoTaskIndices); + }); +}); diff --git a/x-pack/plugins/monitoring/server/alerts/large_shard_size_alert.ts b/x-pack/plugins/monitoring/server/alerts/large_shard_size_alert.ts index db318d7962beb..a6a101bc42afa 100644 --- a/x-pack/plugins/monitoring/server/alerts/large_shard_size_alert.ts +++ b/x-pack/plugins/monitoring/server/alerts/large_shard_size_alert.ts @@ -42,7 +42,7 @@ export class LargeShardSizeAlert extends BaseAlert { id: ALERT_LARGE_SHARD_SIZE, name: ALERT_DETAILS[ALERT_LARGE_SHARD_SIZE].label, throttle: '12h', - defaultParams: { indexPattern: '*', threshold: 55 }, + defaultParams: { indexPattern: '-.*', threshold: 55 }, actionVariables: [ { name: 'shardIndices', diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts index e1da45ab7d991..aab3f0101ef83 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_index_shard_size.ts @@ -120,11 +120,7 @@ export async function fetchIndexShardSize( for (const indexBucket of indexBuckets) { const shardIndex = indexBucket.key; const topHit = indexBucket.hits?.hits?.hits[0] as TopHitType; - if ( - !topHit || - shardIndex.charAt() === '.' || - !ESGlobPatterns.isValid(shardIndex, validIndexPatterns) - ) { + if (!topHit || !ESGlobPatterns.isValid(shardIndex, validIndexPatterns)) { continue; } const {