[data views] Provide method of excluding data tiers when getting field list

src/plugins/data_views/common/constants.ts

@@ -69,3 +69,5 @@ export const FIELDS_PATH = '/internal/data_views/fields';
  * @public
  */
 export const EXISTING_INDICES_PATH = '/internal/data_views/_existing_indices';
+
+export const DATA_VIEWS_FIELDS_EXCLUDED_TIERS = 'data_views:fields_excluded_data_tiers';

src/plugins/data_views/common/types.ts

@@ -544,5 +544,6 @@ export interface HasDataService {
 
 export interface ClientConfigType {
   scriptedFieldsEnabled?: boolean;
+  dataTiersExcludedForFields?: string;
   fieldListCachingEnabled?: boolean;
 }

src/plugins/data_views/server/index.ts

@@ -46,6 +46,13 @@ const configSchema = schema.object({
     schema.boolean({ defaultValue: false }),
     schema.never()
   ),
+
+  dataTiersExcludedForFields: schema.conditional(
+    schema.contextRef('serverless'),
+    true,
+    schema.never(),
+    schema.boolean({ defaultValue: true })
+  ),
   fieldListCachingEnabled: schema.conditional(
     schema.contextRef('serverless'),
     true,

src/plugins/data_views/server/plugin.ts

@@ -16,6 +16,7 @@ import { registerIndexPatternsUsageCollector } from './register_index_pattern_us
 import { createScriptedFieldsDeprecationsConfig } from './deprecations';
 import { DATA_VIEW_SAVED_OBJECT_TYPE, LATEST_VERSION } from '../common';
 import type { ClientConfigType } from '../common/types';
+import { dataTiersUiSettingsConfig } from './ui_settings';
 import {
   DataViewsServerPluginSetup,
   DataViewsServerPluginStart,
@@ -50,6 +51,9 @@ export class DataViewsServerPlugin
 
     const config = this.initializerContext.config.get<ClientConfigType>();
 
+    if (config.dataTiersExcludedForFields) {
+      core.uiSettings.register(dataTiersUiSettingsConfig);
+    }
     if (config.fieldListCachingEnabled) {
       core.uiSettings.register(cacheMaxAge);
     }

src/plugins/data_views/server/rest_api_routes/internal/fields.ts

@@ -15,9 +15,10 @@ import type {
   DataViewsServerPluginStartDependencies,
 } from '../../types';
 import type { FieldDescriptorRestResponse } from '../route_types';
-import { FIELDS_PATH as path } from '../../../common/constants';
+import { FIELDS_PATH as path, DATA_VIEWS_FIELDS_EXCLUDED_TIERS } from '../../../common/constants';
 import { parseFields, IBody, IQuery, querySchema, validate } from './fields_for';
 import { DEFAULT_FIELD_CACHE_FRESHNESS } from '../../constants';
+import { getIndexFilterDsl } from './utils';
 
 export function calculateHash(srcBuffer: Buffer) {
   const hash = createHash('sha1');
@@ -31,6 +32,9 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
     const uiSettings = core.uiSettings.client;
     const { asCurrentUser } = core.elasticsearch.client;
     const indexPatterns = new IndexPatternsFetcher(asCurrentUser, undefined, isRollupsEnabled());
+    const excludedTiers = await core.uiSettings.client.get<string>(
+      DATA_VIEWS_FIELDS_EXCLUDED_TIERS
+    );
     const {
       pattern,
       meta_fields: metaFields,
@@ -59,6 +63,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
           allow_no_indices: allowNoIndex || false,
           includeUnmapped,
         },
+        indexFilter: getIndexFilterDsl({ excludedTiers }),
         ...(parsedFields.length > 0 ? { fields: parsedFields } : {}),
       });
 

src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts

@@ -17,7 +17,11 @@ import type {
   DataViewsServerPluginStartDependencies,
 } from '../../types';
 import type { FieldDescriptorRestResponse } from '../route_types';
-import { FIELDS_FOR_WILDCARD_PATH as path } from '../../../common/constants';
+import {
+  FIELDS_FOR_WILDCARD_PATH as path,
+  DATA_VIEWS_FIELDS_EXCLUDED_TIERS,
+} from '../../../common/constants';
+import { getIndexFilterDsl } from './utils';
 
 /**
  * Accepts one of the following:
@@ -115,8 +119,13 @@ export const validate: FullValidationConfig<any, any, any> = {
 
 const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, IBody> =
   (isRollupsEnabled) => async (context, request, response) => {
-    const { asCurrentUser } = (await context.core).elasticsearch.client;
+    const core = await context.core;
+    const { asCurrentUser } = core.elasticsearch.client;
+    const excludedTiers = await core.uiSettings.client.get<string>(
+      DATA_VIEWS_FIELDS_EXCLUDED_TIERS
+    );
     const indexPatterns = new IndexPatternsFetcher(asCurrentUser, undefined, isRollupsEnabled());
+
     const {
       pattern,
       meta_fields: metaFields,
@@ -149,7 +158,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
           allow_no_indices: allowNoIndex || false,
           includeUnmapped,
         },
-        indexFilter,
+        indexFilter: getIndexFilterDsl({ indexFilter, excludedTiers }),
         allowHidden,
         ...(parsedFields.length > 0 ? { fields: parsedFields } : {}),
       });

src/plugins/data_views/server/rest_api_routes/internal/utils.test.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { getIndexFilterDsl } from './utils';
+
+describe('getIndexFilterDsl', () => {
+  const indexFilter = { term: { _id: '1' } };
+  const tiersQuery = {
+    bool: {
+      must_not: [
+        {
+          terms: {
+            _tier: ['data_cold', 'data_frozen'],
+          },
+        },
+      ],
+    },
+  };
+  const excludedTiers = 'data_cold, data_frozen';
+
+  it('no indexFilter, no excluded tiers', () => {
+    expect(getIndexFilterDsl({})).toBeUndefined();
+  });
+
+  it('indexFilter, no excluded tiers', () => {
+    expect(getIndexFilterDsl({ indexFilter })).toEqual(indexFilter);
+  });
+
+  it('excluded tiers, no indexFilter', () => {
+    expect(getIndexFilterDsl({ excludedTiers })).toEqual(tiersQuery);
+  });
+
+  it('indexFilter and excluded tiers', () => {
+    const combinedQuery = {
+      bool: {
+        must: [indexFilter, tiersQuery],
+      },
+    };
+
+    expect(getIndexFilterDsl({ indexFilter, excludedTiers })).toEqual(combinedQuery);
+  });
+});

src/plugins/data_views/server/rest_api_routes/internal/utils.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { QueryDslQueryContainer } from '../../../common/types';
+
+const excludeTiersDsl = (excludedTiers: string) => {
+  const _tier = excludedTiers.split(',').map((tier) => tier.trim());
+  return {
+    bool: {
+      must_not: [
+        {
+          terms: {
+            _tier,
+          },
+        },
+      ],
+    },
+  };
+};
+
+interface GetIndexFilterDslOptions {
+  indexFilter?: QueryDslQueryContainer;
+  excludedTiers?: string;
+}
+
+export const getIndexFilterDsl = ({
+  indexFilter,
+  excludedTiers,
+}: GetIndexFilterDslOptions): QueryDslQueryContainer | undefined => {
+  if (!indexFilter) {
+    return excludedTiers ? excludeTiersDsl(excludedTiers) : undefined;
+  }
+
+  return !excludedTiers
+    ? indexFilter
+    : {
+        bool: {
+          must: [indexFilter, excludeTiersDsl(excludedTiers)],
+        },
+      };
+};

src/plugins/data_views/server/ui_settings.ts

@@ -8,8 +8,25 @@
 
 import { i18n } from '@kbn/i18n';
 import { schema } from '@kbn/config-schema';
+import type { UiSettingsParams } from '@kbn/core/server';
+import { DATA_VIEWS_FIELDS_EXCLUDED_TIERS } from '../common/constants';
 import { DEFAULT_FIELD_CACHE_FRESHNESS } from './constants';
 
+export const dataTiersUiSettingsConfig: Record<string, UiSettingsParams> = {
+  [DATA_VIEWS_FIELDS_EXCLUDED_TIERS]: {
+    name: i18n.translate('dataViews.advancedSettings.dataTiersName', {
+      defaultMessage: 'Data tiers excluded from field requests',
+    }),
+    value: '',
+    type: 'string',
+    description: i18n.translate('dataViews.advancedSettings.dataTiersText', {
+      defaultMessage:
+        'Exclude fields from specified tiers (such as data_frozen) for faster performance. Comma delimit to exclude multiple tiers - data_warm,data_cold',
+    }),
+    schema: schema.string(),
+  },
+};
+
 export const cacheMaxAge = {
   'data_views:cache_max_age': {
     name: i18n.translate('dataViews.advancedSettings.cacheMaxAgeTitle', {

src/plugins/kibana_usage_collection/server/collectors/management/schema.ts

@@ -620,4 +620,8 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
     type: 'integer',
     _meta: { description: 'Non-default value of setting.' },
   },
+  'data_views:fields_excluded_data_tiers': {
+    type: 'keyword',
+    _meta: { description: 'Non-default value of setting.' },
+  },
 };

src/plugins/kibana_usage_collection/server/collectors/management/types.ts

@@ -164,4 +164,5 @@ export interface UsageStats {
   'observability:profilingDatacenterPUE': number;
   'observability:profilingCostPervCPUPerHour': number;
   'observability:profilingAWSCostDiscountRate': number;
+  'data_views:fields_excluded_data_tiers': string;
 }

src/plugins/telemetry/schema/oss_plugins.json

@@ -10114,6 +10114,12 @@
           "_meta": {
             "description": "Non-default value of setting."
           }
+        },
+        "data_views:fields_excluded_data_tiers": {
+          "type": "keyword",
+          "_meta": {
+            "description": "Non-default value of setting."
+          }
         }
       }
     },