[BUG][Data] Support for custom filters with heterogeneous data fields (…

…#5577) * [BUG][Data] Support for custom filters with heterogeneous data fields When enabling the advanced setting `courier:ignoreFilterIfFieldNotInIndex` Custom OpenSearch Query DSL filters could technically be applied to index patterns that map to indices that are not exactly the same. Since the custom query filter is a user input then users can really type anything that they need. Or any field that they know is present but we do not know for sure. Therefore, we can check if the id which is the index pattern title to check if we should apply the filter or not. Issue resolved: https://github.com/opensearch-project/dashboards-visualizations/issues/281 I believe issue: #5423 Should closed as that is expected functionality. Signed-off-by: Kawika Avilla <[email protected]> * [Cleanup] utilize the same helper function Originally when implementing the fix the historical comment caused concern about potential breaking changes. But after discussion, we decided it is more clear to consolidate the helper functions. Signed-off-by: Kawika Avilla <[email protected]>
opensearch-project · Jan 8, 2024 · e566ce0 · e566ce0
1 parent c8f6db9
commit e566ce0
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 20 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -32,6 +32,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [BUG][Dev Tool] Add dev tool documentation link to dev tool's help menu [#5166](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5166)
 - Fix missing border for header navigation control on right ([#5450](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5450))
 - [BUG] Fix filtering issue in data source selector ([5484](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5484))
+- [BUG][Data] Support for custom filters with heterogeneous data fields ([5577](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5577))
+- [BUG] Remove duplicate sample data 90943e30-9a47-11e8-b64d-95841ca0b247 ([5591](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5591))
 
 ### 🚞 Infrastructure
 

diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/filter_matches_index.test.ts b/src/plugins/data/common/opensearch_query/opensearch_query/filter_matches_index.test.ts
@@ -66,4 +66,18 @@ describe('filterMatchesIndex', () => {
 
     expect(filterMatchesIndex(filter, indexPattern)).toBe(true);
   });
+
+  it('should return false if the custom filter is a different index id', () => {
+    const filter = { meta: { index: 'foo', key: 'bar', type: 'custom' } } as Filter;
+    const indexPattern = { id: 'bar', fields: [{ name: 'foo' }] } as IIndexPattern;
+
+    expect(filterMatchesIndex(filter, indexPattern)).toBe(false);
+  });
+
+  it('should return true if the custom filter is the same index id', () => {
+    const filter = { meta: { index: 'foo', key: 'bar', type: 'custom' } } as Filter;
+    const indexPattern = { id: 'foo', fields: [{ name: 'barf' }] } as IIndexPattern;
+
+    expect(filterMatchesIndex(filter, indexPattern)).toBe(true);
+  });
 });
diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/filter_matches_index.ts b/src/plugins/data/common/opensearch_query/opensearch_query/filter_matches_index.ts
@@ -31,14 +31,14 @@
 import { IIndexPattern, IFieldType } from '../../index_patterns';
 import { Filter } from '../filters';
 
-/*
- * TODO: We should base this on something better than `filter.meta.key`. We should probably modify
- * this to check if `filter.meta.index` matches `indexPattern.id` instead, but that's a breaking
- * change.
- */
 export function filterMatchesIndex(filter: Filter, indexPattern?: IIndexPattern | null) {
   if (!filter.meta?.key || !indexPattern) {
     return true;
   }
+
+  if (filter.meta?.type === 'custom') {
+    return filter.meta.index === indexPattern.id;
+  }
+
   return indexPattern.fields.some((field: IFieldType) => field.name === filter.meta.key);
 }
diff --git a/src/plugins/home/server/services/sample_data/data_sets/logs/saved_objects.ts b/src/plugins/home/server/services/sample_data/data_sets/logs/saved_objects.ts
@@ -269,21 +269,6 @@ export const getSavedObjects = (): SavedObject[] => [
     },
     references: [],
   },
-  {
-    id: '90943e30-9a47-11e8-b64d-95841ca0b247',
-    type: 'index-pattern',
-    updated_at: '2018-08-29T13:22:17.617Z',
-    version: '1',
-    migrationVersion: {},
-    attributes: {
-      title: 'opensearch_dashboards_sample_data_logs',
-      timeFieldName: 'timestamp',
-      fields:
-        '[{"name":"@timestamp","type":"date","esTypes":["date"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"_id","type":"string","esTypes":["_id"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":false},{"name":"_index","type":"string","esTypes":["_index"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":false},{"name":"_score","type":"number","count":0,"scripted":false,"searchable":false,"aggregatable":false,"readFromDocValues":false},{"name":"_source","type":"_source","esTypes":["_source"],"count":0,"scripted":false,"searchable":false,"aggregatable":false,"readFromDocValues":false},{"name":"_type","type":"string","esTypes":["_type"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":false},{"name":"agent","type":"string","esTypes":["text"],"count":0,"scripted":false,"searchable":true,"aggregatable":false,"readFromDocValues":false},{"name":"agent.keyword","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true,"subType":{"multi":{"parent": "agent"}}},{"name":"bytes","type":"number","esTypes":["long"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"clientip","type":"ip","esTypes":["ip"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"event.dataset","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"extension","type":"string","esTypes":["text"],"count":0,"scripted":false,"searchable":true,"aggregatable":false,"readFromDocValues":false},{"name":"extension.keyword","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true,"subType":{"multi":{"parent": "extension"}}},{"name":"geo.coordinates","type":"geo_point","esTypes":["geo_point"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"geo.dest","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"geo.src","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"geo.srcdest","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"host","type":"string","esTypes":["text"],"count":0,"scripted":false,"searchable":true,"aggregatable":false,"readFromDocValues":false},{"name":"host.keyword","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true,"subType":{"multi":{"parent": "host"}}},{"name":"index","type":"string","esTypes":["text"],"count":0,"scripted":false,"searchable":true,"aggregatable":false,"readFromDocValues":false},{"name":"index.keyword","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true,"subType":{"multi":{"parent": "index"}}},{"name":"ip","type":"ip","esTypes":["ip"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"machine.os","type":"string","esTypes":["text"],"count":0,"scripted":false,"searchable":true,"aggregatable":false,"readFromDocValues":false},{"name":"machine.os.keyword","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true,"subType":{"multi":{"parent": "machine.os"}}},{"name":"machine.ram","type":"number","esTypes":["long"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"memory","type":"number","esTypes":["double"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"message","type":"string","esTypes":["text"],"count":0,"scripted":false,"searchable":true,"aggregatable":false,"readFromDocValues":false},{"name":"message.keyword","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true,"subType":{"multi":{"parent": "message"}}},{"name":"phpmemory","type":"number","esTypes":["long"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"referer","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"request","type":"string","esTypes":["text"],"count":0,"scripted":false,"searchable":true,"aggregatable":false,"readFromDocValues":false},{"name":"request.keyword","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true,"subType":{"multi":{"parent": "request"}}},{"name":"response","type":"string","esTypes":["text"],"count":0,"scripted":false,"searchable":true,"aggregatable":false,"readFromDocValues":false},{"name":"response.keyword","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true,"subType":{"multi":{"parent": "response"}}},{"name":"tags","type":"string","esTypes":["text"],"count":0,"scripted":false,"searchable":true,"aggregatable":false,"readFromDocValues":false},{"name":"tags.keyword","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true,"subType":{"multi":{"parent": "tags"}}},{"name":"timestamp","type":"date","esTypes":["date"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"url","type":"string","esTypes":["text"],"count":0,"scripted":false,"searchable":true,"aggregatable":false,"readFromDocValues":false},{"name":"url.keyword","type":"string","esTypes":["keyword"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true,"subType":{"multi":{"parent": "url"}}},{"name":"utc_time","type":"date","esTypes":["date"],"count":0,"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"name":"hour_of_day","type":"number","count":0,"scripted":true,"script":"doc[\'timestamp\'].value.getHour()","lang":"painless","searchable":true,"aggregatable":true,"readFromDocValues":false}]',
-      fieldFormatMap: '{"hour_of_day":{}}',
-    },
-    references: [],
-  },
   {
     id: 'edf84fe0-e1a0-11e7-b6d5-4dc382ef7f5b',
     type: 'dashboard',