Merge remote-tracking branch 'upstream/master' into kbn-xxx-legacy-es…

…-plugin-pre-removal

.telemetryrc.json

@@ -7,7 +7,6 @@
       "src/plugins/testbed/",
       "src/plugins/kibana_utils/",
       "src/plugins/kibana_usage_collection/server/collectors/kibana/kibana_usage_collector.ts",
-      "src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.ts",
       "src/plugins/kibana_usage_collection/server/collectors/management/telemetry_management_collector.ts",
       "src/plugins/kibana_usage_collection/server/collectors/ui_metric/telemetry_ui_metric_collector.ts",
       "src/plugins/telemetry/server/collectors/usage/telemetry_usage_collector.ts"

docs/developer/best-practices/index.asciidoc

@@ -48,7 +48,7 @@ guidelines]
 * Write all new code on
 {kib-repo}blob/{branch}/src/core/README.md[the
 platform], and following
-{kib-repo}blob/{branch}/src/core/CONVENTIONS.md[conventions]
+{kib-repo}blob/{branch}/src/core/CONVENTIONS.md[conventions].
 * _Always_ use the `SavedObjectClient` for reading and writing Saved
 Objects.
 * Add `README`s to all your plugins and services.

docs/developer/best-practices/stability.asciidoc

@@ -52,15 +52,15 @@ storeinSessions?)
 [discrete]
 === Browser coverage
 
-Refer to the list of browsers and OS {kib} supports
+Refer to the list of browsers and OS {kib} supports:
 https://www.elastic.co/support/matrix
 
 Does the feature work efficiently on the list of supported browsers? 
 
 [discrete]
-=== Upgrade Scenarios - Migration scenarios- 
+=== Upgrade and Migration scenarios
 
-Does the feature affect old
-indices, saved objects ? - Has the feature been tested with {kib}
-aliases - Read/Write privileges of the indices before and after the
+* Does the feature affect old indices or saved objects?
+* Has the feature been tested with {kib} aliases?
+* Read/Write privileges of the indices before and after the
 upgrade?

docs/developer/getting-started/building-kibana.asciidoc

@@ -1,7 +1,7 @@
 [[building-kibana]]
 == Building a {kib} distributable
 
-The following commands will build a {kib} production distributable.
+The following command will build a {kib} production distributable:
 
 [source,bash]
 ----
@@ -36,4 +36,4 @@ To specify a package to build you can add `rpm` or `deb` as an argument.
 yarn build --rpm
 ----
 
-Distributable packages can be found in `target/` after the build completes.
+Distributable packages can be found in `target/` after the build completes.

docs/developer/getting-started/index.asciidoc

@@ -49,7 +49,7 @@ ____
 
 (You can also run `yarn kbn` to see the other available commands. For
 more info about this tool, see
-{kib-repo}tree/{branch}/packages/kbn-pm[{kib-repo}tree/{branch}packages/kbn-pm].)
+{kib-repo}tree/{branch}/packages/kbn-pm[{kib-repo}tree/{branch}/packages/kbn-pm].)
 
 When switching branches which use different versions of npm packages you
 may need to run:
@@ -137,4 +137,4 @@ include::debugging.asciidoc[leveloffset=+1]
 
 include::building-kibana.asciidoc[leveloffset=+1]
 
-include::development-plugin-resources.asciidoc[leveloffset=+1]
+include::development-plugin-resources.asciidoc[leveloffset=+1]

docs/developer/getting-started/running-kibana-advanced.asciidoc

@@ -73,8 +73,8 @@ settings].
 [discrete]
 === Potential Optimization Pitfalls
 
-* Webpack is trying to include a file in the bundle that I deleted and
-is now complaining about it is missing
+* Webpack is trying to include a file in the bundle that was deleted and
+is now complaining about it being missing
 * A module id that used to resolve to a single file now resolves to a
 directory, but webpack isn’t adapting
 * (if you discover other scenarios, please send a PR!)
@@ -84,4 +84,4 @@ directory, but webpack isn’t adapting
 
 {kib} includes self-signed certificates that can be used for
 development purposes in the browser and for communicating with
-{es}: `yarn start --ssl` & `yarn es snapshot --ssl`.
+{es}: `yarn start --ssl` & `yarn es snapshot --ssl`.

packages/kbn-telemetry-tools/src/tools/__fixture__/mock_schema.json

@@ -5,6 +5,22 @@
         "flat": {
           "type": "keyword"
         },
+        "my_index_signature_prop": {
+          "properties": {
+            "avg": {
+              "type": "number"
+            },
+            "count": {
+              "type": "number"
+            },
+            "max": {
+              "type": "number"
+            },
+            "min": {
+              "type": "number"
+            }
+          }
+        },
         "my_str": {
           "type": "text"
         },

packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_indexed_interface_with_not_matching_schema.ts

@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { SyntaxKind } from 'typescript';
+import { ParsedUsageCollection } from '../ts_parser';
+
+export const parsedIndexedInterfaceWithNoMatchingSchema: ParsedUsageCollection = [
+  'src/fixtures/telemetry_collectors/indexed_interface_with_not_matching_schema.ts',
+  {
+    collectorName: 'indexed_interface_with_not_matching_schema',
+    schema: {
+      value: {
+        something: {
+          count_1: {
+            type: 'number',
+          },
+        },
+      },
+    },
+    fetch: {
+      typeName: 'Usage',
+      typeDescriptor: {
+        '': {
+          '@@INDEX@@': {
+            count_1: {
+              kind: SyntaxKind.NumberKeyword,
+              type: 'NumberKeyword',
+            },
+            count_2: {
+              kind: SyntaxKind.NumberKeyword,
+              type: 'NumberKeyword',
+            },
+          },
+        },
+      },
+    },
+  },
+];

packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_working_collector.ts

@@ -32,6 +32,20 @@ export const parsedWorkingCollector: ParsedUsageCollection = [
         my_str: {
           type: 'text',
         },
+        my_index_signature_prop: {
+          avg: {
+            type: 'number',
+          },
+          count: {
+            type: 'number',
+          },
+          max: {
+            type: 'number',
+          },
+          min: {
+            type: 'number',
+          },
+        },
         my_objects: {
           total: {
             type: 'number',
@@ -60,6 +74,14 @@ export const parsedWorkingCollector: ParsedUsageCollection = [
           kind: SyntaxKind.StringKeyword,
           type: 'StringKeyword',
         },
+        my_index_signature_prop: {
+          '': {
+            '@@INDEX@@': {
+              kind: SyntaxKind.NumberKeyword,
+              type: 'NumberKeyword',
+            },
+          },
+        },
         my_objects: {
           total: {
             kind: SyntaxKind.NumberKeyword,

packages/kbn-telemetry-tools/src/tools/check_collector__integrity.test.ts

@@ -20,6 +20,7 @@
 import { cloneDeep } from 'lodash';
 import * as ts from 'typescript';
 import { parsedWorkingCollector } from './__fixture__/parsed_working_collector';
+import { parsedIndexedInterfaceWithNoMatchingSchema } from './__fixture__/parsed_indexed_interface_with_not_matching_schema';
 import { checkCompatibleTypeDescriptor, checkMatchingMapping } from './check_collector_integrity';
 import * as path from 'path';
 import { readFile } from 'fs';
@@ -82,6 +83,20 @@ describe('checkCompatibleTypeDescriptor', () => {
     expect(incompatibles).toHaveLength(0);
   });
 
+  it('returns diff on indexed interface with no matching schema', () => {
+    const incompatibles = checkCompatibleTypeDescriptor([
+      parsedIndexedInterfaceWithNoMatchingSchema,
+    ]);
+    expect(incompatibles).toHaveLength(1);
+    const { diff, message } = incompatibles[0];
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    expect(diff).toEqual({ '.@@INDEX@@.count_2.kind': 'number' });
+    expect(message).toHaveLength(1);
+    expect(message).toEqual([
+      'incompatible Type key (Usage..@@INDEX@@.count_2): expected (undefined) got ("number").',
+    ]);
+  });
+
   describe('Interface Change', () => {
     it('returns diff on incompatible type descriptor with mapping', () => {
       const malformedParsedCollector = cloneDeep(parsedWorkingCollector);

packages/kbn-telemetry-tools/src/tools/extract_collectors.test.ts

@@ -34,7 +34,7 @@ describe('extractCollectors', () => {
     const programPaths = await getProgramPaths(configs[0]);
 
     const results = [...extractCollectors(programPaths, tsConfig)];
-    expect(results).toHaveLength(6);
+    expect(results).toHaveLength(7);
     expect(results).toMatchSnapshot();
   });
 });

packages/kbn-telemetry-tools/src/tools/serializer.ts

@@ -84,6 +84,11 @@ export function getDescriptor(node: ts.Node, program: ts.Program): Descriptor |
     }, {} as any);
   }
 
+  // If it's defined as signature { [key: string]: OtherInterface }
+  if (ts.isIndexSignatureDeclaration(node) && node.type) {
+    return { '@@INDEX@@': getDescriptor(node.type, program) };
+  }
+
   if (ts.SyntaxKind.FirstNode === node.kind) {
     return getDescriptor((node as any).right, program);
   }

packages/kbn-telemetry-tools/src/tools/utils.ts

@@ -98,6 +98,14 @@ export function getVariableValue(node: ts.Node): string | Record<string, any> {
     return serializeObject(node);
   }
 
+  if (ts.isIdentifier(node)) {
+    const declaration = getIdentifierDeclaration(node);
+    if (ts.isVariableDeclaration(declaration) && declaration.initializer) {
+      return getVariableValue(declaration.initializer);
+    }
+    // TODO: If this is another imported value from another file, we'll need to go fetch it like in getPropertyValue
+  }
+
   throw Error(`Unsuppored Node: cannot get value of node (${node.getText()}) of kind ${node.kind}`);
 }
 
@@ -112,10 +120,11 @@ export function serializeObject(node: ts.Node) {
     if (typeof propertyName === 'undefined') {
       throw new Error(`Unable to get property name ${property.getText()}`);
     }
+    const cleanPropertyName = propertyName.replace(/["']/g, '');
     if (ts.isPropertyAssignment(property)) {
-      value[propertyName] = getVariableValue(property.initializer);
+      value[cleanPropertyName] = getVariableValue(property.initializer);
     } else {
-      value[propertyName] = getVariableValue(property);
+      value[cleanPropertyName] = getVariableValue(property);
     }
   }
 
@@ -222,9 +231,29 @@ export const flattenKeys = (obj: any, keyPath: any[] = []): any => {
 };
 
 export function difference(actual: any, expected: any) {
-  function changes(obj: any, base: any) {
+  function changes(obj: { [key: string]: any }, base: { [key: string]: any }) {
     return transform(obj, function (result, value, key) {
-      if (key && !isEqual(value, base[key])) {
+      if (key && /@@INDEX@@/.test(`${key}`)) {
+        // The type definition is an Index Signature, fuzzy searching for similar keys
+        const regexp = new RegExp(`${key}`.replace(/@@INDEX@@/g, '(.+)?'));
+        const keysInBase = Object.keys(base)
+          .map((k) => {
+            const match = k.match(regexp);
+            return match && match[0];
+          })
+          .filter((s): s is string => !!s);
+
+        if (keysInBase.length === 0) {
+          // Mark this key as wrong because we couldn't find any matching keys
+          result[key] = value;
+        }
+
+        keysInBase.forEach((k) => {
+          if (!isEqual(value, base[k])) {
+            result[k] = isObject(value) && isObject(base[k]) ? changes(value, base[k]) : value;
+          }
+        });
+      } else if (key && !isEqual(value, base[key])) {
         result[key] = isObject(value) && isObject(base[key]) ? changes(value, base[key]) : value;
       }
     });