[Visualizations] Expensive queries are causing unnecessary load and d…

…elays on Elasticsearch (#99031) (#110447) * [Visualizations] Expensive queries are causing unnecessary load and delays on Elasticsearch Part of: #93770 * fix CI * fix typo * fix namespaces issue * fix tests Co-authored-by: Kibana Machine <[email protected]> Co-authored-by: Kibana Machine <[email protected]>
elastic · Aug 30, 2021 · acc8bc6 · acc8bc6
1 parent 949e123
commit acc8bc6
Show file tree

Hide file tree

Showing 5 changed files with 128 additions and 177 deletions.
diff --git a/src/plugins/visualizations/server/plugin.ts b/src/plugins/visualizations/server/plugin.ts
@@ -8,33 +8,29 @@
 
 import { i18n } from '@kbn/i18n';
 import { schema } from '@kbn/config-schema';
-import { Observable } from 'rxjs';
-import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
-import {
+
+import { VISUALIZE_ENABLE_LABS_SETTING } from '../common/constants';
+import { visualizationSavedObjectType } from './saved_objects';
+import { registerVisualizationsCollector } from './usage_collector';
+
+import type { VisualizationsPluginSetup, VisualizationsPluginStart } from './types';
+import type {
   PluginInitializerContext,
   CoreSetup,
   CoreStart,
   Plugin,
   Logger,
 } from '../../../core/server';
-
-import { VISUALIZE_ENABLE_LABS_SETTING } from '../common/constants';
-
-import { visualizationSavedObjectType } from './saved_objects';
-
-import { VisualizationsPluginSetup, VisualizationsPluginStart } from './types';
-import { registerVisualizationsCollector } from './usage_collector';
-import { EmbeddableSetup } from '../../embeddable/server';
+import type { UsageCollectionSetup } from '../../usage_collection/server';
+import type { EmbeddableSetup } from '../../embeddable/server';
 import { visualizeEmbeddableFactory } from './embeddable/visualize_embeddable_factory';
 
 export class VisualizationsPlugin
   implements Plugin<VisualizationsPluginSetup, VisualizationsPluginStart> {
   private readonly logger: Logger;
-  private readonly config: Observable<{ kibana: { index: string } }>;
 
   constructor(initializerContext: PluginInitializerContext) {
     this.logger = initializerContext.logger.get();
-    this.config = initializerContext.config.legacy.globalConfig$;
   }
 
   public setup(
@@ -61,7 +57,7 @@ export class VisualizationsPlugin
     });
 
     if (plugins.usageCollection) {
-      registerVisualizationsCollector(plugins.usageCollection, this.config);
+      registerVisualizationsCollector(plugins.usageCollection);
     }
 
     plugins.embeddable.registerEmbeddableFactory(visualizeEmbeddableFactory());

diff --git a/src/plugins/visualizations/server/usage_collector/get_usage_collector.test.ts b/src/plugins/visualizations/server/usage_collector/get_usage_collector.test.ts
@@ -7,126 +7,106 @@
  */
 
 import moment from 'moment';
-import { ElasticsearchClient } from 'src/core/server';
 import { getStats } from './get_usage_collector';
+import type { SavedObjectsClientContract } from '../../../../core/server';
 
 const defaultMockSavedObjects = [
   {
-    _id: 'visualization:coolviz-123',
-    _source: {
-      type: 'visualization',
-      visualization: { visState: '{"type": "shell_beads"}' },
-      updated_at: moment().subtract(7, 'days').startOf('day').toString(),
-    },
+    id: 'visualization:coolviz-123',
+    attributes: { visState: '{"type": "shell_beads"}' },
+    updated_at: moment().subtract(7, 'days').startOf('day').toString(),
   },
 ];
 
 const enlargedMockSavedObjects = [
   // default space
   {
-    _id: 'visualization:coolviz-123',
-    _source: {
-      type: 'visualization',
-      visualization: { visState: '{"type": "cave_painting"}' },
-      updated_at: moment().subtract(7, 'days').startOf('day').toString(),
-    },
+    id: 'visualization:coolviz-123',
+    namespaces: ['default'],
+    attributes: { visState: '{"type": "cave_painting"}' },
+    updated_at: moment().subtract(7, 'days').startOf('day').toString(),
   },
   {
-    _id: 'visualization:coolviz-456',
-    _source: {
-      type: 'visualization',
-      visualization: { visState: '{"type": "printing_press"}' },
-      updated_at: moment().subtract(20, 'days').startOf('day').toString(),
-    },
+    id: 'visualization:coolviz-456',
+    namespaces: ['default'],
+    attributes: { visState: '{"type": "printing_press"}' },
+    updated_at: moment().subtract(20, 'days').startOf('day').toString(),
   },
   {
-    _id: 'meat:visualization:coolviz-789',
-    _source: {
-      type: 'visualization',
-      visualization: { visState: '{"type": "floppy_disk"}' },
-      updated_at: moment().subtract(2, 'months').startOf('day').toString(),
-    },
+    id: 'meat:visualization:coolviz-789',
+    namespaces: ['default'],
+    attributes: { visState: '{"type": "floppy_disk"}' },
+    updated_at: moment().subtract(2, 'months').startOf('day').toString(),
   },
   // meat space
   {
-    _id: 'meat:visualization:coolviz-789',
-    _source: {
-      type: 'visualization',
-      visualization: { visState: '{"type": "cave_painting"}' },
-      updated_at: moment().subtract(89, 'days').startOf('day').toString(),
-    },
+    id: 'meat:visualization:coolviz-789',
+    namespaces: ['meat'],
+    attributes: { visState: '{"type": "cave_painting"}' },
+    updated_at: moment().subtract(89, 'days').startOf('day').toString(),
   },
   {
-    _id: 'meat:visualization:coolviz-789',
-    _source: {
-      type: 'visualization',
-      visualization: { visState: '{"type": "cuneiform"}' },
-      updated_at: moment().subtract(5, 'months').startOf('day').toString(),
-    },
+    id: 'meat:visualization:coolviz-789',
+    namespaces: ['meat'],
+    attributes: { visState: '{"type": "cuneiform"}' },
+    updated_at: moment().subtract(5, 'months').startOf('day').toString(),
   },
   {
-    _id: 'meat:visualization:coolviz-789',
-    _source: {
-      type: 'visualization',
-      visualization: { visState: '{"type": "cuneiform"}' },
-      updated_at: moment().subtract(2, 'days').startOf('day').toString(),
-    },
+    id: 'meat:visualization:coolviz-789',
+    namespaces: ['meat'],
+    attributes: { visState: '{"type": "cuneiform"}' },
+    updated_at: moment().subtract(2, 'days').startOf('day').toString(),
   },
   {
-    _id: 'meat:visualization:coolviz-789',
-    _source: {
-      type: 'visualization',
-      visualization: { visState: '{"type": "floppy_disk"}' },
-      updated_at: moment().subtract(7, 'days').startOf('day').toString(),
-    },
+    id: 'meat:visualization:coolviz-789',
+    attributes: { visState: '{"type": "floppy_disk"}' },
+    updated_at: moment().subtract(7, 'days').startOf('day').toString(),
   },
   // cyber space
   {
-    _id: 'cyber:visualization:coolviz-789',
-    _source: {
-      type: 'visualization',
-      visualization: { visState: '{"type": "floppy_disk"}' },
-      updated_at: moment().subtract(7, 'months').startOf('day').toString(),
-    },
+    id: 'cyber:visualization:coolviz-789',
+    namespaces: ['cyber'],
+    attributes: { visState: '{"type": "floppy_disk"}' },
+    updated_at: moment().subtract(7, 'months').startOf('day').toString(),
   },
   {
-    _id: 'cyber:visualization:coolviz-789',
-    _source: {
-      type: 'visualization',
-      visualization: { visState: '{"type": "floppy_disk"}' },
-      updated_at: moment().subtract(3, 'days').startOf('day').toString(),
-    },
+    id: 'cyber:visualization:coolviz-789',
+    namespaces: ['cyber'],
+    attributes: { visState: '{"type": "floppy_disk"}' },
+    updated_at: moment().subtract(3, 'days').startOf('day').toString(),
   },
   {
-    _id: 'cyber:visualization:coolviz-123',
-    _source: {
-      type: 'visualization',
-      visualization: { visState: '{"type": "cave_painting"}' },
-      updated_at: moment().subtract(15, 'days').startOf('day').toString(),
-    },
+    id: 'cyber:visualization:coolviz-123',
+    namespaces: ['cyber'],
+    attributes: { visState: '{"type": "cave_painting"}' },
+    updated_at: moment().subtract(15, 'days').startOf('day').toString(),
   },
 ];
 
 describe('Visualizations usage collector', () => {
-  const mockIndex = '';
-
-  const getMockCallCluster = (hits: unknown[]) =>
-    ({
-      search: () => Promise.resolve({ body: { hits: { hits } } }) as unknown,
-    } as ElasticsearchClient);
+  const getMockCallCluster = (savedObjects: unknown[]) =>
+    (({
+      createPointInTimeFinder: jest.fn().mockResolvedValue({
+        close: jest.fn(),
+        find: function* asyncGenerator() {
+          yield { saved_objects: savedObjects };
+        },
+      }),
+    } as unknown) as SavedObjectsClientContract);
 
   test('Returns undefined when no results found (undefined)', async () => {
-    const result = await getStats(getMockCallCluster(undefined as any), mockIndex);
+    const result = await getStats(getMockCallCluster(undefined as any));
+
     expect(result).toBeUndefined();
   });
 
   test('Returns undefined when no results found (0 results)', async () => {
-    const result = await getStats(getMockCallCluster([]), mockIndex);
+    const result = await getStats(getMockCallCluster([]));
     expect(result).toBeUndefined();
   });
 
   test('Summarizes visualizations response data', async () => {
-    const result = await getStats(getMockCallCluster(defaultMockSavedObjects), mockIndex);
+    const result = await getStats(getMockCallCluster(defaultMockSavedObjects));
 
     expect(result).toMatchObject({
       shell_beads: {
@@ -181,7 +161,7 @@ describe('Visualizations usage collector', () => {
       },
     };
 
-    const result = await getStats(getMockCallCluster(enlargedMockSavedObjects), mockIndex);
+    const result = await getStats(getMockCallCluster(enlargedMockSavedObjects));
 
     expect(result).toMatchObject(expectedStats);
   });

diff --git a/src/plugins/visualizations/server/usage_collector/get_usage_collector.ts b/src/plugins/visualizations/server/usage_collector/get_usage_collector.ts
@@ -6,12 +6,11 @@
  * Side Public License, v 1.
  */
 
-import { countBy, get, groupBy, mapValues, max, min, values } from 'lodash';
-import { ElasticsearchClient } from 'kibana/server';
-import type { estypes } from '@elastic/elasticsearch';
-
+import { countBy, groupBy, mapValues, max, min, values } from 'lodash';
 import { getPastDays } from './get_past_days';
-type ESResponse = estypes.SearchResponse<{ visualization: { visState: string } }>;
+
+import type { SavedObjectsClientContract, SavedObjectsFindResult } from '../../../../core/server';
+import type { SavedVisState } from '../../../visualizations/common';
 
 interface VisSummary {
   type: string;
@@ -35,61 +34,50 @@ export interface VisualizationUsage {
  * Parse the response data into telemetry payload
  */
 export async function getStats(
-  esClient: ElasticsearchClient,
-  index: string
+  soClient: SavedObjectsClientContract
 ): Promise<VisualizationUsage | undefined> {
-  const searchParams = {
-    size: 10000, // elasticsearch index.max_result_window default value
-    index,
-    ignoreUnavailable: true,
-    filterPath: [
-      'hits.hits._id',
-      'hits.hits._source.visualization',
-      'hits.hits._source.updated_at',
-    ],
-    body: {
-      query: {
-        bool: { filter: { term: { type: 'visualization' } } },
-      },
-    },
-  };
-  const { body: esResponse } = await esClient.search<ESResponse>(searchParams);
-  const size = get(esResponse, 'hits.hits.length', 0);
-  if (size < 1) {
-    return;
-  }
-
-  // `map` to get the raw types
-  const visSummaries: VisSummary[] = esResponse.hits.hits.map((hit) => {
-    const spacePhrases = hit._id.toString().split(':');
-    const lastUpdated: string = get(hit, '_source.updated_at');
-    const space = spacePhrases.length === 3 ? spacePhrases[0] : 'default'; // if in a custom space, the format of a saved object ID is space:type:id
-    const visualization = get(hit, '_source.visualization', { visState: '{}' });
-    const visState: { type?: string } = JSON.parse(visualization.visState);
-    return {
-      type: visState.type || '_na_',
-      space,
-      past_days: getPastDays(lastUpdated),
-    };
+  const finder = await soClient.createPointInTimeFinder({
+    type: 'visualization',
+    perPage: 1000,
+    namespaces: ['*'],
   });
 
-  // organize stats per type
-  const visTypes = groupBy(visSummaries, 'type');
+  const visSummaries: VisSummary[] = [];
 
-  // get the final result
-  return mapValues(visTypes, (curr) => {
-    const total = curr.length;
-    const spacesBreakdown = countBy(curr, 'space');
-    const spaceCounts: number[] = values(spacesBreakdown);
+  for await (const response of finder.find()) {
+    (response.saved_objects || []).forEach((so: SavedObjectsFindResult<any>) => {
+      if (so.attributes?.visState) {
+        const visState: SavedVisState = JSON.parse(so.attributes.visState);
 
-    return {
-      total,
-      spaces_min: min(spaceCounts),
-      spaces_max: max(spaceCounts),
-      spaces_avg: total / spaceCounts.length,
-      saved_7_days_total: curr.filter((c) => c.past_days <= 7).length,
-      saved_30_days_total: curr.filter((c) => c.past_days <= 30).length,
-      saved_90_days_total: curr.filter((c) => c.past_days <= 90).length,
-    };
-  });
+        visSummaries.push({
+          type: visState.type ?? '_na_',
+          space: so.namespaces?.[0] ?? 'default',
+          past_days: getPastDays(so.updated_at!),
+        });
+      }
+    });
+  }
+  await finder.close();
+
+  if (visSummaries.length) {
+    // organize stats per type
+    const visTypes = groupBy(visSummaries, 'type');
+
+    // get the final result
+    return mapValues(visTypes, (curr) => {
+      const total = curr.length;
+      const spacesBreakdown = countBy(curr, 'space');
+      const spaceCounts: number[] = values(spacesBreakdown);
+
+      return {
+        total,
+        spaces_min: min(spaceCounts),
+        spaces_max: max(spaceCounts),
+        spaces_avg: total / spaceCounts.length,
+        saved_7_days_total: curr.filter((c) => c.past_days <= 7).length,
+        saved_30_days_total: curr.filter((c) => c.past_days <= 30).length,
+        saved_90_days_total: curr.filter((c) => c.past_days <= 90).length,
+      };
+    });
+  }
 }