[Lens] Supports include and exclude terms (#136179)

* [Lens] Supports include and exclude terms

* Change type

* Fix pattern logic

* Adds unit tests

* Implement it with 2 extra properties on the expression

* Distinguish the regex and exact match bejhavior

* Enables transferring of include/exclude from TSVB

* Fix tests

* Fix tests

* Update all snapshots

* Update texts

* Revers screenshots

* Use debounce value instead

* Placeholder change

Co-authored-by: Joe Reuter <[email protected]>

src/plugins/data/common/search/aggs/agg_config.test.ts

@@ -563,12 +563,18 @@ describe('AggConfig', () => {
                 "enabled": Array [
                   true,
                 ],
+                "excludeIsRegex": Array [
+                  true,
+                ],
                 "field": Array [
                   "machine.os.keyword",
                 ],
                 "id": Array [
                   "1",
                 ],
+                "includeIsRegex": Array [
+                  true,
+                ],
                 "missingBucket": Array [
                   false,
                 ],
@@ -629,12 +635,18 @@ describe('AggConfig', () => {
                   "enabled": Array [
                     true,
                   ],
+                  "excludeIsRegex": Array [
+                    true,
+                  ],
                   "field": Array [
                     "bytes",
                   ],
                   "id": Array [
                     "1-orderAgg",
                   ],
+                  "includeIsRegex": Array [
+                    true,
+                  ],
                   "missingBucket": Array [
                     false,
                   ],

src/plugins/data/common/search/aggs/buckets/migrate_include_exclude_format.ts

@@ -40,6 +40,13 @@ export const migrateIncludeExcludeFormat = {
       if (parsedValue.length) {
         output.params[this.name] = parsedValue;
       }
+    } else if (Array.isArray(value) && value.length > 0 && isStringType(aggConfig)) {
+      // check if is regex
+      const filtered = value.filter((val) => val !== '');
+      if (filtered.length) {
+        const isRegularExpression = aggConfig.getParam(`${this.name}IsRegex`);
+        output.params[this.name] = isRegularExpression ? filtered[0] : filtered;
+      }
     } else if (isObject(value)) {
       output.params[this.name] = (value as any).pattern;
     } else if (value && isStringType(aggConfig)) {

src/plugins/data/common/search/aggs/buckets/terms.test.ts

@@ -94,12 +94,18 @@ describe('Terms Agg', () => {
                 "enabled": Array [
                   true,
                 ],
+                "excludeIsRegex": Array [
+                  true,
+                ],
                 "field": Array [
                   "field",
                 ],
                 "id": Array [
                   "test",
                 ],
+                "includeIsRegex": Array [
+                  true,
+                ],
                 "missingBucket": Array [
                   false,
                 ],
@@ -196,6 +202,70 @@ describe('Terms Agg', () => {
       expect(params.exclude).toBe('exclude value');
     });
 
+    test('accepts array of strings from string field type and writes this value', () => {
+      const aggConfigs = getAggConfigs({
+        include: ['include1', 'include2'],
+        exclude: ['exclude1'],
+        includeIsRegex: false,
+        excludeIsRegex: false,
+        field: {
+          name: 'string_field',
+          type: 'string',
+        },
+        orderAgg: {
+          type: 'count',
+        },
+      });
+
+      const { [BUCKET_TYPES.TERMS]: params } = aggConfigs.aggs[0].toDsl();
+
+      expect(params.field).toBe('string_field');
+      expect(params.include).toStrictEqual(['include1', 'include2']);
+      expect(params.exclude).toStrictEqual(['exclude1']);
+    });
+
+    test('accepts array of string with regex and returns the pattern', () => {
+      const aggConfigs = getAggConfigs({
+        include: ['include.*'],
+        exclude: ['exclude.*'],
+        includeIsRegex: true,
+        excludeIsRegex: true,
+        field: {
+          name: 'string_field',
+          type: 'string',
+        },
+        orderAgg: {
+          type: 'count',
+        },
+      });
+
+      const { [BUCKET_TYPES.TERMS]: params } = aggConfigs.aggs[0].toDsl();
+
+      expect(params.field).toBe('string_field');
+      expect(params.include).toBe('include.*');
+      expect(params.exclude).toBe('exclude.*');
+    });
+
+    test('accepts array of empty strings and does not write a value', () => {
+      const aggConfigs = getAggConfigs({
+        include: [''],
+        exclude: [''],
+        field: {
+          name: 'string_field',
+          type: 'string',
+        },
+        orderAgg: {
+          type: 'count',
+        },
+      });
+
+      const { [BUCKET_TYPES.TERMS]: params } = aggConfigs.aggs[0].toDsl();
+
+      expect(params.field).toBe('string_field');
+      expect(params.include).toBe(undefined);
+      expect(params.exclude).toBe(undefined);
+    });
+
     test('accepts empty array from number field type and does not write a value', () => {
       const aggConfigs = getAggConfigs({
         include: [],

src/plugins/data/common/search/aggs/buckets/terms.ts

@@ -45,8 +45,10 @@ export interface AggParamsTerms extends BaseAggParams {
   otherBucket?: boolean;
   otherBucketLabel?: string;
   // advanced
-  exclude?: string;
-  include?: string;
+  exclude?: string[] | number[];
+  include?: string[] | number[];
+  includeIsRegex?: boolean;
+  excludeIsRegex?: boolean;
 }
 
 export const getTermsBucketAgg = () =>
@@ -178,5 +180,15 @@ export const getTermsBucketAgg = () =>
         shouldShow: isStringOrNumberType,
         ...migrateIncludeExcludeFormat,
       },
+      {
+        name: 'includeIsRegex',
+        default: true,
+        write: noop,
+      },
+      {
+        name: 'excludeIsRegex',
+        default: true,
+        write: noop,
+      },
     ],
   });

src/plugins/data/common/search/aggs/buckets/terms_fn.test.ts

@@ -27,8 +27,10 @@ describe('agg_expression_functions', () => {
             "params": Object {
               "customLabel": undefined,
               "exclude": undefined,
+              "excludeIsRegex": undefined,
               "field": "machine.os.keyword",
               "include": undefined,
+              "includeIsRegex": undefined,
               "json": undefined,
               "missingBucket": undefined,
               "missingBucketLabel": undefined,
@@ -61,8 +63,8 @@ describe('agg_expression_functions', () => {
         missingBucketLabel: 'missing',
         otherBucket: true,
         otherBucketLabel: 'other',
-        include: 'win',
-        exclude: 'ios',
+        include: ['win'],
+        exclude: ['ios'],
       });
 
       expect(actual.value).toMatchInlineSnapshot(`
@@ -71,9 +73,15 @@ describe('agg_expression_functions', () => {
           "id": "1",
           "params": Object {
             "customLabel": undefined,
-            "exclude": "ios",
+            "exclude": Array [
+              "ios",
+            ],
+            "excludeIsRegex": undefined,
             "field": "machine.os.keyword",
-            "include": "win",
+            "include": Array [
+              "win",
+            ],
+            "includeIsRegex": undefined,
             "json": undefined,
             "missingBucket": true,
             "missingBucketLabel": "missing",
@@ -103,8 +111,10 @@ describe('agg_expression_functions', () => {
         Object {
           "customLabel": undefined,
           "exclude": undefined,
+          "excludeIsRegex": undefined,
           "field": "machine.os.keyword",
           "include": undefined,
+          "includeIsRegex": undefined,
           "json": undefined,
           "missingBucket": undefined,
           "missingBucketLabel": undefined,
@@ -115,8 +125,10 @@ describe('agg_expression_functions', () => {
             "params": Object {
               "customLabel": undefined,
               "exclude": undefined,
+              "excludeIsRegex": undefined,
               "field": "name",
               "include": undefined,
+              "includeIsRegex": undefined,
               "json": undefined,
               "missingBucket": undefined,
               "missingBucketLabel": undefined,

src/plugins/data/common/search/aggs/buckets/terms_fn.ts

@@ -117,16 +117,30 @@ export const aggTerms = (): FunctionDefinition => ({
       }),
     },
     exclude: {
-      types: ['string'],
+      types: ['string', 'number'],
       help: i18n.translate('data.search.aggs.buckets.terms.exclude.help', {
         defaultMessage: 'Specific bucket values to exclude from results',
       }),
+      multi: true,
     },
     include: {
-      types: ['string'],
+      types: ['string', 'number'],
       help: i18n.translate('data.search.aggs.buckets.terms.include.help', {
         defaultMessage: 'Specific bucket values to include in results',
       }),
+      multi: true,
+    },
+    includeIsRegex: {
+      types: ['boolean'],
+      help: i18n.translate('data.search.aggs.buckets.terms.includeIsRegex.help', {
+        defaultMessage: 'When set to true, the include property is handled as regex',
+      }),
+    },
+    excludeIsRegex: {
+      types: ['boolean'],
+      help: i18n.translate('data.search.aggs.buckets.terms.excludeIsRegex.help', {
+        defaultMessage: 'When set to true, the exclude property is handled as regex',
+      }),
     },
     json: {
       types: ['string'],

src/plugins/data/common/search/tabify/response_writer.test.ts

@@ -180,6 +180,8 @@ describe('TabbedAggResponseWriter class', () => {
               order: 'desc',
               otherBucket: false,
               otherBucketLabel: 'Other',
+              includeIsRegex: true,
+              excludeIsRegex: true,
               size: 5,
             },
             type: 'terms',
@@ -245,6 +247,8 @@ describe('TabbedAggResponseWriter class', () => {
               otherBucket: false,
               otherBucketLabel: 'Other',
               size: 5,
+              includeIsRegex: true,
+              excludeIsRegex: true,
             },
             type: 'terms',
           },

src/plugins/vis_types/timeseries/public/trigger_action/index.test.ts

@@ -247,6 +247,37 @@ describe('triggerTSVBtoLensConfiguration', () => {
       otherBucket: false,
       orderDirection: 'desc',
       orderBy: { type: 'alphabetical' },
+      includeIsRegex: false,
+      excludeIsRegex: false,
+      parentFormat: {
+        id: 'terms',
+      },
+    });
+  });
+
+  test('should return include exclude information if the chart is broken down by terms', async () => {
+    const modelWithTerms = {
+      ...model,
+      series: [
+        {
+          ...model.series[0],
+          split_mode: 'terms',
+          terms_size: 6,
+          terms_direction: 'desc',
+          terms_order_by: '_key',
+          terms_include: 't.*',
+        },
+      ] as unknown as Series[],
+    };
+    const triggerOptions = await triggerTSVBtoLensConfiguration(modelWithTerms);
+    expect(triggerOptions?.layers[0]?.termsParams).toStrictEqual({
+      size: 6,
+      otherBucket: false,
+      orderDirection: 'desc',
+      orderBy: { type: 'alphabetical' },
+      includeIsRegex: true,
+      include: ['t.*'],
+      excludeIsRegex: false,
       parentFormat: {
         id: 'terms',
       },

src/plugins/vis_types/timeseries/public/trigger_action/index.ts

@@ -141,6 +141,10 @@ export const triggerTSVBtoLensConfiguration = async (
       ...(layer.split_mode === 'terms' && {
         termsParams: {
           size: layer.terms_size ?? 10,
+          ...(layer.terms_include && { include: [layer.terms_include] }),
+          includeIsRegex: Boolean(layer.terms_include),
+          ...(layer.terms_exclude && { exclude: [layer.terms_exclude] }),
+          excludeIsRegex: Boolean(layer.terms_exclude),
           otherBucket: false,
           orderDirection: layer.terms_direction ?? 'desc',
           orderBy: layer.terms_order_by === '_key' ? { type: 'alphabetical' } : { type: 'column' },

test/interpreter_functional/snapshots/baseline/combined_test2.json

@@ -1 +1 @@
-{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"1","indexPatternId":"logstash-*","params":{"emptyAsNull":false},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"meta":{"source":"logstash-*","statistics":{"totalCount":14004},"type":"esaggs"},"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"}
+{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"2","indexPatternId":"logstash-*","params":{"excludeIsRegex":true,"field":"response.raw","includeIsRegex":true,"missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"1","indexPatternId":"logstash-*","params":{"emptyAsNull":false},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"meta":{"source":"logstash-*","statistics":{"totalCount":14004},"type":"esaggs"},"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"}

test/interpreter_functional/snapshots/baseline/combined_test3.json

@@ -1 +1 @@
-{"as":"legacyMetricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"autoScale":null,"colorFullBackground":false,"labels":{"position":"bottom","show":true,"style":{"css":"font-family:'Open Sans', Helvetica, Arial, sans-serif;font-weight:normal;font-style:normal;text-decoration:none;text-align:center;font-size:24px;line-height:1","spec":{"fontFamily":"'Open Sans', Helvetica, Arial, sans-serif","fontSize":"24px","fontStyle":"normal","fontWeight":"normal","lineHeight":"1","textAlign":"center","textDecoration":"none"},"type":"style"}},"metricColorMode":"None","palette":null,"percentageMode":false,"style":{"bgColor":false,"css":"font-family:'Open Sans', Helvetica, Arial, sans-serif;font-weight:normal;font-style:normal;text-decoration:none;text-align:center;font-size:60px;line-height:1","labelColor":false,"spec":{"fontFamily":"'Open Sans', Helvetica, Arial, sans-serif","fontSize":"60px","fontStyle":"normal","fontWeight":"normal","lineHeight":"1","textAlign":"center","textDecoration":"none"},"type":"style"}}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"1","indexPatternId":"logstash-*","params":{"emptyAsNull":false},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"meta":{"source":"logstash-*","statistics":{"totalCount":14004},"type":"esaggs"},"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}
+{"as":"legacyMetricVis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"autoScale":null,"colorFullBackground":false,"labels":{"position":"bottom","show":true,"style":{"css":"font-family:'Open Sans', Helvetica, Arial, sans-serif;font-weight:normal;font-style:normal;text-decoration:none;text-align:center;font-size:24px;line-height:1","spec":{"fontFamily":"'Open Sans', Helvetica, Arial, sans-serif","fontSize":"24px","fontStyle":"normal","fontWeight":"normal","lineHeight":"1","textAlign":"center","textDecoration":"none"},"type":"style"}},"metricColorMode":"None","palette":null,"percentageMode":false,"style":{"bgColor":false,"css":"font-family:'Open Sans', Helvetica, Arial, sans-serif;font-weight:normal;font-style:normal;text-decoration:none;text-align:center;font-size:60px;line-height:1","labelColor":false,"spec":{"fontFamily":"'Open Sans', Helvetica, Arial, sans-serif","fontSize":"60px","fontStyle":"normal","fontWeight":"normal","lineHeight":"1","textAlign":"center","textDecoration":"none"},"type":"style"}}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"2","indexPatternId":"logstash-*","params":{"excludeIsRegex":true,"field":"response.raw","includeIsRegex":true,"missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"hasPrecisionError":false,"id":"1","indexPatternId":"logstash-*","params":{"emptyAsNull":false},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"meta":{"source":"logstash-*","statistics":{"totalCount":14004},"type":"esaggs"},"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}}