[BUG][Data] Support for custom filters with heterogeneous data fields

When enabling the advanced setting `courier:ignoreFilterIfFieldNotInIndex`
Custom OpenSearch Query DSL filters could technically be applied to index
patterns that map to indices that are not exactly the same. Since the
custom query filter is a user input then users can really type anything
that they need. Or any field that they know is present but we do not know
for sure.

Therefore, we can check if the id which is the index pattern title to check
if we should apply the filter or not.

Issue resolved:
https://github.com/opensearch-project/dashboards-visualizations/issues/281

I believe issue:
opensearch-project#5423

Should closed as that is expected functionality.

Signed-off-by: Kawika Avilla <[email protected]>

CHANGELOG.md

@@ -31,7 +31,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [BUG] Add platform "darwin-arm64" to unit test ([#5290](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5290))
 - [BUG][Dev Tool] Add dev tool documentation link to dev tool's help menu [#5166](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5166)
 - Fix missing border for header navigation control on right ([#5450](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5450))
-- [BUG] Fix filtering issue in data source selector ([5484](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5484))
+- [BUG][Data] Support for custom filters with heterogeneous data fields ([5577](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5577))
 
 ### 🚞 Infrastructure
 

src/plugins/data/common/opensearch_query/opensearch_query/custom_filter_matches_index.test.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { Filter } from '../filters';
+import { customFilterMatchesIndex } from './custom_filter_matches_index';
+import { IIndexPattern } from '../../index_patterns';
+
+describe('customFilterMatchesIndex', () => {
+  it('should return true if the custom filter has no meta', () => {
+    const filter = {} as Filter;
+    const indexPattern = { id: 'foo', fields: [{ name: 'bar' }] } as IIndexPattern;
+
+    expect(customFilterMatchesIndex(filter, indexPattern)).toBe(true);
+  });
+
+  it('should return true if no index pattern is passed', () => {
+    const filter = { meta: { index: 'foo', key: 'bar', type: 'custom' } } as Filter;
+
+    expect(customFilterMatchesIndex(filter, undefined)).toBe(true);
+  });
+
+  it('should return true if the custom filter has meta without a key', () => {
+    const filter = { meta: { index: 'foo', type: 'custom' } } as Filter;
+    const indexPattern = { id: 'foo', fields: [{ name: 'bar' }] } as IIndexPattern;
+
+    expect(customFilterMatchesIndex(filter, indexPattern)).toBe(true);
+  });
+
+  it('should return false if the filter is not custom', () => {
+    const filter = { meta: { index: 'foo', key: 'bar', type: 'match_all' } } as Filter;
+    const indexPattern = { id: 'foo', fields: [{ name: 'bar' }] } as IIndexPattern;
+
+    expect(customFilterMatchesIndex(filter, indexPattern)).toBe(false);
+  });
+
+  it('should return false if the custom filter is a different index id', () => {
+    const filter = { meta: { index: 'foo', key: 'bar', type: 'custom' } } as Filter;
+    const indexPattern = { id: 'bar', fields: [{ name: 'foo' }] } as IIndexPattern;
+
+    expect(customFilterMatchesIndex(filter, indexPattern)).toBe(false);
+  });
+
+  it('should return true if the custom filter is the same index id', () => {
+    const filter = { meta: { index: 'foo', key: 'bar', type: 'custom' } } as Filter;
+    const indexPattern = { id: 'foo', fields: [{ name: 'barf' }] } as IIndexPattern;
+
+    expect(customFilterMatchesIndex(filter, indexPattern)).toBe(true);
+  });
+});

src/plugins/data/common/opensearch_query/opensearch_query/custom_filter_matches_index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { IIndexPattern } from '../../index_patterns';
+import { Filter } from '../filters';
+
+export function customFilterMatchesIndex(filter: Filter, indexPattern?: IIndexPattern | null) {
+  if (!filter.meta?.key || !indexPattern) {
+    return true;
+  }
+  if (filter.meta?.type !== 'custom') {
+    return false;
+  }
+  return filter.meta.index === indexPattern.id;
+}

src/plugins/data/common/opensearch_query/opensearch_query/from_filters.ts

@@ -34,6 +34,7 @@ import { filterMatchesIndex } from './filter_matches_index';
 import { Filter, cleanFilter, isFilterDisabled } from '../filters';
 import { IIndexPattern } from '../../index_patterns';
 import { handleNestedFilter } from './handle_nested_filter';
+import { customFilterMatchesIndex } from './custom_filter_matches_index';
 
 /**
  * Create a filter that can be reversed for filters with negate set
@@ -76,7 +77,10 @@ export const buildQueryFromFilters = (
     return filters
       .filter(filterNegate(negate))
       .filter(
-        (filter) => !ignoreFilterIfFieldNotInIndex || filterMatchesIndex(filter, indexPattern)
+        (filter) =>
+          !ignoreFilterIfFieldNotInIndex ||
+          filterMatchesIndex(filter, indexPattern) ||
+          customFilterMatchesIndex(filter, indexPattern)
       )
       .map((filter) => {
         return migrateFilter(filter, indexPattern);