[Stack Monitoring] support custom ccs remote prefixes

docs/settings/monitoring-settings.asciidoc

@@ -40,6 +40,9 @@ When enabled, specifies the email address where you want to receive cluster aler
 `monitoring.ui.ccs.enabled`::
 Set to `true` (default) to enable {ref}/modules-cross-cluster-search.html[cross-cluster search] of your monitoring data. The {ref}/modules-remote-clusters.html#remote-cluster-settings[`remote_cluster_client`] role must exist on each node.
 
+`monitoring.ui.ccs.remotePatterns`::
+Set to `*` (default) to perform a {ref}/modules-cross-cluster-search.html[cross-cluster search] for all available clusters. Accepts a string or array of custom remote names to only search. The {ref}/modules-remote-clusters.html#remote-cluster-settings[`remote_cluster_client`] role must exist on each node. `monitoring.ui.ccs.enabled` must be enabled.
+
 `monitoring.ui.elasticsearch.hosts`::
 Specifies the location of the {es} cluster where your monitoring data is stored.
 +

x-pack/plugins/monitoring/common/ccs_utils.test.js

@@ -6,21 +6,21 @@
  */
 
 import expect from '@kbn/expect';
-import { parseCrossClusterPrefix, prefixIndexPattern } from './ccs_utils';
+import { parseCrossClusterPrefix, prefixIndexPatternWithCcs } from './ccs_utils';
 
 // TODO: tests were not running and are not updated.
 // They need to be changed to run.
 describe.skip('ccs_utils', () => {
-  describe('prefixIndexPattern', () => {
+  describe('prefixIndexPatternWithCcs', () => {
     const indexPattern = '.monitoring-xyz-1-*,.monitoring-xyz-2-*';
 
     it('returns the index pattern if ccs is not enabled', () => {
       // TODO apply as MonitoringConfig during typescript conversion
       const config = { ui: { css: { enabled: false } } };
 
       // falsy string values should be ignored
-      const allPattern = prefixIndexPattern(config, indexPattern, '*');
-      const onePattern = prefixIndexPattern(config, indexPattern, 'do_not_use_me');
+      const allPattern = prefixIndexPatternWithCcs(config, indexPattern, '*');
+      const onePattern = prefixIndexPatternWithCcs(config, indexPattern, 'do_not_use_me');
 
       expect(allPattern).to.be(indexPattern);
       expect(onePattern).to.be(indexPattern);
@@ -31,9 +31,9 @@ describe.skip('ccs_utils', () => {
       const config = { ui: { css: { enabled: true } } };
 
       // falsy string values should be ignored
-      const undefinedPattern = prefixIndexPattern(config, indexPattern);
-      const nullPattern = prefixIndexPattern(config, indexPattern, null);
-      const blankPattern = prefixIndexPattern(config, indexPattern, '');
+      const undefinedPattern = prefixIndexPatternWithCcs(config, indexPattern);
+      const nullPattern = prefixIndexPatternWithCcs(config, indexPattern, null);
+      const blankPattern = prefixIndexPatternWithCcs(config, indexPattern, '');
 
       expect(undefinedPattern).to.be(indexPattern);
       expect(nullPattern).to.be(indexPattern);
@@ -44,8 +44,8 @@ describe.skip('ccs_utils', () => {
       // TODO apply as MonitoringConfig during typescript conversion
       const config = { ui: { css: { enabled: true } } };
 
-      const abcPattern = prefixIndexPattern(config, indexPattern, 'aBc');
-      const underscorePattern = prefixIndexPattern(config, indexPattern, 'cluster_one');
+      const abcPattern = prefixIndexPatternWithCcs(config, indexPattern, 'aBc');
+      const underscorePattern = prefixIndexPatternWithCcs(config, indexPattern, 'cluster_one');
 
       expect(abcPattern).to.eql(
         'aBc:.monitoring-xyz-1-*,aBc:.monitoring-xyz-2-*,aBc:monitoring-xyz-1-*,aBc:monitoring-xyz-2-*'
@@ -59,7 +59,7 @@ describe.skip('ccs_utils', () => {
       // TODO apply as MonitoringConfig during typescript conversion
       const config = { ui: { css: { enabled: true } } };
 
-      const pattern = prefixIndexPattern(config, indexPattern, '*');
+      const pattern = prefixIndexPatternWithCcs(config, indexPattern, '*');
 
       // it should have BOTH patterns so that it searches all CCS clusters and the local cluster
       expect(pattern).to.eql(

x-pack/plugins/monitoring/common/ccs_utils.ts

@@ -6,12 +6,8 @@
  */
 
 // eslint-disable-next-line @kbn/eslint/no-restricted-paths
-import type { MonitoringConfig } from '../server/config';
+import { MonitoringConfig } from '../server/config';
 
-export function getConfigCcs(config: MonitoringConfig): boolean {
-  // TODO: (Mat) this function can probably be removed in favor of direct config access where it's used.
-  return config.ui.ccs.enabled;
-}
 /**
  * Prefix all comma separated index patterns within the original {@code indexPattern}.
  *
@@ -20,20 +16,32 @@ export function getConfigCcs(config: MonitoringConfig): boolean {
  *
  * @param  {Object} config The Kibana configuration object.
  * @param  {String} indexPattern The index pattern name
- * @param  {String} ccs The optional cluster-prefix to prepend.
+ * @param  {Array|String} ccs The optional cluster-prefixes to prepend. An array when passed from config
+ * and a string when passed from a request.  This is optional because the request could be empty if its not a remote cluster
  * @return {String} The index pattern with the {@code cluster} prefix appropriately prepended.
  */
-export function prefixIndexPattern(config: MonitoringConfig, indexPattern: string, ccs?: string) {
-  const ccsEnabled = getConfigCcs(config);
+export function prefixIndexPatternWithCcs(
+  config: MonitoringConfig,
+  indexPattern: string,
+  ccs?: string[] | string
+) {
+  const ccsEnabled = config.ui.ccs.enabled;
   if (!ccsEnabled || !ccs) {
     return indexPattern;
   }
 
   const patterns = indexPattern.split(',');
-  const prefixedPattern = patterns.map((pattern) => `${ccs}:${pattern}`).join(',');
+  // for each index pattern prefix with each remote ccs pattern
+  const prefixedPattern = patterns
+    .map((pattern) =>
+      Array.isArray(ccs) ? ccs.map((ccsValue) => `${ccsValue}:${pattern}`) : `${ccs}:${pattern}`
+    )
+    .join(',');
 
-  // if a wildcard is used, then we also want to search the local indices
-  if (ccs === '*') {
+  // if a wildcard is used, then we also want to search the local indices.
+  if (ccs.includes('*')) {
+    // this case is met when user does not set monitoring.ui.ccs.remotePatterns and uses default ([*])
+    // or user explicitly sets monitoring.ui.ccs to '*'
     return `${prefixedPattern},${indexPattern}`;
   }
   return prefixedPattern;

x-pack/plugins/monitoring/public/alerts/components/param_details_form/use_derived_index_pattern.tsx

@@ -7,7 +7,7 @@
 
 import { useEffect, useState } from 'react';
 import { DataViewsPublicPluginStart, DataView } from 'src/plugins/data_views/public';
-import { prefixIndexPattern } from '../../../../common/ccs_utils';
+import { prefixIndexPatternWithCcs } from '../../../../common/ccs_utils';
 import {
   INDEX_PATTERN_BEATS,
   INDEX_PATTERN_ELASTICSEARCH,
@@ -22,7 +22,11 @@ export const useDerivedIndexPattern = (
   dataViews: DataViewsPublicPluginStart,
   config?: MonitoringConfig
 ): { loading: boolean; derivedIndexPattern?: DataView } => {
-  const indexPattern = prefixIndexPattern(config || ({} as MonitoringConfig), INDEX_PATTERNS, '*');
+  const indexPattern = prefixIndexPatternWithCcs(
+    config || ({} as MonitoringConfig),
+    INDEX_PATTERNS,
+    config?.ui.ccs.remotePatterns
+  );
   const [loading, setLoading] = useState<boolean>(true);
   const [dataView, setDataView] = useState<DataView>();
   useEffect(() => {

x-pack/plugins/monitoring/server/config.test.ts

@@ -56,6 +56,7 @@ describe('config schema', () => {
         "ui": Object {
           "ccs": Object {
             "enabled": true,
+            "remotePatterns": "*",
           },
           "container": Object {
             "apm": Object {
@@ -150,4 +151,59 @@ describe('createConfig()', () => {
     });
     expect(config.ui.elasticsearch.ssl).toEqual(expected);
   });
+  it('accepts both string and array of strings for ui.ccs.remotePatterns', () => {
+    let configValue = createConfig(
+      configSchema.validate({
+        ui: {
+          ccs: {
+            remotePatterns: 'remote1',
+          },
+        },
+      })
+    );
+    expect(configValue.ui.ccs.remotePatterns).toEqual(['remote1']);
+    configValue = createConfig(
+      configSchema.validate({
+        ui: {
+          ccs: {
+            remotePatterns: ['remote1'],
+          },
+        },
+      })
+    );
+    expect(configValue.ui.ccs.remotePatterns).toEqual(['remote1']);
+    configValue = createConfig(
+      configSchema.validate({
+        ui: {
+          ccs: {
+            remotePatterns: ['remote1', 'remote2'],
+          },
+        },
+      })
+    );
+    expect(configValue.ui.ccs.remotePatterns).toEqual(['remote1', 'remote2']);
+  });
+});
+
+describe('throws when config is invalid', () => {
+  describe('ui.ccs.remotePattern errors', () => {
+    it('throws error with a space', () => {
+      expect(() => {
+        configSchema.validate({ ui: { ccs: { remotePatterns: 'my remote' } } });
+      }).toThrowError();
+    });
+    it('throws error when wildcard (*) pattern is used in an array', () => {
+      expect(() => {
+        configSchema.validate({ ui: { ccs: { remotePatterns: ['remote1', '*'] } } });
+      }).toThrowError();
+    });
+    it('throws error with an invalid remote name', () => {
+      expect(() => {
+        configSchema.validate({ ui: { ccs: { remotePatterns: 'remote-*' } } });
+      }).toThrowError();
+      expect(() => {
+        configSchema.validate({ ui: { ccs: { remotePatterns: 'remote1, remote2' } } });
+      }).toThrowError();
+    });
+  });
 });

x-pack/plugins/monitoring/server/config.ts

@@ -22,13 +22,43 @@ export const monitoringElasticsearchConfigSchema = elasticsearchConfigSchema.ext
   hosts: schema.maybe(schema.oneOf([hostURISchema, schema.arrayOf(hostURISchema, { minSize: 1 })])),
 });
 
+const validateSingleCcsRemotePattern = (value: string) => {
+  if (value.includes(' ')) {
+    return 'Spaces are not allowed in a remote name.';
+  }
+  if (value.match(/[^a-zA-Z\d\-_]/g)) {
+    return 'Remote names contain only letters, numbers, underscores, and dashes.';
+  }
+};
+const ccsSingleRemotePatternsSchema = schema.string({
+  validate(value) {
+    return validateSingleCcsRemotePattern(value);
+  },
+});
+const ccsMultiRemotePatternsSchema = schema.string({
+  validate(value) {
+    if (value === '*') {
+      return 'Cannot use the default wildcard (*) value in an array.';
+    }
+    return validateSingleCcsRemotePattern(value);
+  },
+});
 export const configSchema = schema.object({
   ui: schema.object({
     enabled: schema.boolean({ defaultValue: true }),
     debug_mode: schema.boolean({ defaultValue: false }),
     debug_log_path: schema.string({ defaultValue: '' }),
     ccs: schema.object({
       enabled: schema.boolean({ defaultValue: true }),
+      remotePatterns: schema.oneOf(
+        [
+          ccsSingleRemotePatternsSchema,
+          schema.arrayOf(ccsMultiRemotePatternsSchema, { minSize: 1, maxSize: 10 }),
+        ],
+        {
+          defaultValue: '*',
+        }
+      ),
     }),
     logs: schema.object({
       index: schema.string({ defaultValue: 'filebeat-*' }),
@@ -97,6 +127,10 @@ type MonitoringConfigTypeOverriddenUI = Omit<MonitoringConfigSchema, 'ui'>;
 interface MonitoringConfigTypeOverriddenUIElasticsearch
   extends Omit<MonitoringConfigSchema['ui'], 'elasticsearch'> {
   elasticsearch: MonitoringElasticsearchConfig;
+  ccs: {
+    enabled: MonitoringConfigSchema['ui']['ccs']['enabled'];
+    remotePatterns: string[];
+  };
 }
 
 /**
@@ -114,6 +148,12 @@ export function createConfig(config: MonitoringConfigSchema): MonitoringConfig {
     ...config,
     ui: {
       ...config.ui,
+      ccs: {
+        ...config.ui.ccs,
+        remotePatterns: Array.isArray(config.ui.ccs.remotePatterns)
+          ? config.ui.ccs.remotePatterns
+          : [config.ui.ccs.remotePatterns],
+      },
       elasticsearch: new MonitoringElasticsearchConfig(config.ui.elasticsearch),
     },
   };

x-pack/plugins/monitoring/server/index.ts

@@ -27,6 +27,7 @@ export const config: PluginConfigDescriptor<TypeOf<typeof configSchema>> = {
       container: true,
       ccs: {
         enabled: true,
+        remotePatterns: true,
       },
     },
     kibana: true,

x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/fetch_stack_product_usage.ts

@@ -10,7 +10,7 @@ import { ElasticsearchClient } from 'src/core/server';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { MonitoringConfig } from '../../../config';
 // @ts-ignore
-import { prefixIndexPattern } from '../../../../common/ccs_utils';
+import { prefixIndexPatternWithCcs } from '../../../../common/ccs_utils';
 import { StackProductUsage } from '../types';
 
 interface ESResponse {

x-pack/plugins/monitoring/server/kibana_monitoring/collectors/lib/get_stack_products_usage.ts

@@ -12,7 +12,7 @@ import { MonitoringConfig } from '../../../config';
 // @ts-ignore
 import { getIndexPatterns } from '../../../lib/cluster/get_index_patterns';
 // @ts-ignore
-import { prefixIndexPattern } from '../../../../common/ccs_utils';
+import { prefixIndexPatternWithCcs } from '../../../../common/ccs_utils';
 import {
   INDEX_PATTERN_ELASTICSEARCH,
   INDEX_PATTERN_KIBANA,

x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.test.ts

@@ -14,7 +14,7 @@ jest.mock('../../static_globals', () => ({
     app: {
       config: {
         ui: {
-          ccs: { enabled: true },
+          ccs: { enabled: true, remotePatterns: '*' },
         },
       },
     },
@@ -140,4 +140,21 @@ describe('fetchCCReadExceptions', () => {
     // @ts-ignore
     expect(params.index).toBe('.monitoring-es-*,metrics-elasticsearch.ccr-*');
   });
+  it('should call ES with correct query when ccs enabled and monitoring.ui.ccs.remotePatterns has array value', async () => {
+    // @ts-ignore
+    Globals.app.config.ui.ccs.enabled = true;
+    Globals.app.config.ui.ccs.remotePatterns = ['remote1', 'remote2'];
+    let params = null;
+    esClient.search.mockImplementation((...args) => {
+      params = args[0];
+      return Promise.resolve(esRes as any);
+    });
+
+    await fetchCCRReadExceptions(esClient, 1643306331418, 1643309869056, 10000);
+
+    // @ts-ignore
+    expect(params.index).toBe(
+      'remote1:.monitoring-es-*,remote2:.monitoring-es-*,remote1:metrics-elasticsearch.ccr-*,remote2:metrics-elasticsearch.ccr-*'
+    );
+  });
 });

x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.ts

@@ -11,7 +11,6 @@ import { CCRReadExceptionsStats } from '../../../common/types/alerts';
 import { getNewIndexPatterns } from '../cluster/get_index_patterns';
 import { createDatasetFilter } from './create_dataset_query_filter';
 import { Globals } from '../../static_globals';
-import { getConfigCcs } from '../../../common/ccs_utils';
 
 export async function fetchCCRReadExceptions(
   esClient: ElasticsearchClient,
@@ -24,7 +23,7 @@ export async function fetchCCRReadExceptions(
     config: Globals.app.config,
     moduleType: 'elasticsearch',
     dataset: 'ccr',
-    ccs: getConfigCcs(Globals.app.config) ? '*' : undefined,
+    ccs: Globals.app.config.ui.ccs.remotePatterns,
   });
   const params = {
     index: indexPatterns,

x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.test.ts

@@ -15,7 +15,7 @@ jest.mock('../../static_globals', () => ({
     app: {
       config: {
         ui: {
-          ccs: { enabled: true },
+          ccs: { enabled: true, remotePatterns: '*' },
         },
       },
     },
@@ -111,4 +111,23 @@ describe('fetchClusterHealth', () => {
     // @ts-ignore
     expect(params.index).toBe('.monitoring-es-*,metrics-elasticsearch.cluster_stats-*');
   });
+
+  it('should call ES with correct query when ccs enabled and monitoring.ui.ccs.remotePatterns has array value', async () => {
+    // @ts-ignore
+    Globals.app.config.ui.ccs.enabled = true;
+    Globals.app.config.ui.ccs.remotePatterns = ['remote1', 'remote2'];
+    const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
+    let params = null;
+    esClient.search.mockImplementation((...args) => {
+      params = args[0];
+      return Promise.resolve({} as any);
+    });
+
+    await fetchClusterHealth(esClient, [{ clusterUuid: '1', clusterName: 'foo1' }]);
+
+    // @ts-ignore
+    expect(params.index).toBe(
+      'remote1:.monitoring-es-*,remote2:.monitoring-es-*,remote1:metrics-elasticsearch.cluster_stats-*,remote2:metrics-elasticsearch.cluster_stats-*'
+    );
+  });
 });