Merge branch 'master' into improves-detections

.github/CODEOWNERS

@@ -25,6 +25,7 @@
 /src/plugins/vis_type_vislib/ @elastic/kibana-app
 /src/plugins/vis_type_xy/ @elastic/kibana-app
 /src/plugins/visualize/ @elastic/kibana-app
+/src/plugins/visualizations/ @elastic/kibana-app
 
 # App Architecture
 /examples/bfetch_explorer/ @elastic/kibana-app-arch
@@ -51,7 +52,6 @@
 /src/plugins/navigation/ @elastic/kibana-app-arch
 /src/plugins/share/ @elastic/kibana-app-arch
 /src/plugins/ui_actions/ @elastic/kibana-app-arch
-/src/plugins/visualizations/ @elastic/kibana-app-arch
 /x-pack/examples/ui_actions_enhanced_examples/ @elastic/kibana-app-arch
 /x-pack/plugins/data_enhanced/ @elastic/kibana-app-arch
 /x-pack/plugins/embeddable_enhanced/ @elastic/kibana-app-arch
@@ -246,6 +246,8 @@ x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @elastic/kib
 /x-pack/plugins/upgrade_assistant/  @elastic/es-ui
 /x-pack/plugins/watcher/  @elastic/es-ui
 /x-pack/plugins/ingest_pipelines/  @elastic/es-ui
+/packages/kbn-ace/ @elastic/es-ui
+/packages/kbn-monaco/ @elastic/es-ui
 
 # Endpoint
 /x-pack/plugins/endpoint/ @elastic/endpoint-app-team @elastic/siem

.telemetryrc.json

@@ -6,7 +6,6 @@
       "src/plugins/kibana_react/",
       "src/plugins/testbed/",
       "src/plugins/kibana_utils/",
-      "src/plugins/kibana_usage_collection/server/collectors/management/telemetry_management_collector.ts",
       "src/plugins/kibana_usage_collection/server/collectors/ui_metric/telemetry_ui_metric_collector.ts"
     ]
   }

package.json

@@ -138,6 +138,8 @@
     "@kbn/telemetry-tools": "1.0.0",
     "@kbn/test-subj-selector": "0.2.1",
     "@kbn/ui-framework": "1.0.0",
+    "@kbn/ace": "1.0.0",
+    "@kbn/monaco": "1.0.0",
     "@kbn/ui-shared-deps": "1.0.0",
     "@types/yauzl": "^2.9.1",
     "JSONStream": "1.3.5",
@@ -193,7 +195,7 @@
     "p-map": "^4.0.0",
     "pegjs": "0.10.0",
     "proxy-from-env": "1.0.0",
-    "query-string": "5.1.1",
+    "query-string": "^6.13.2",
     "re2": "^1.15.4",
     "react": "^16.12.0",
     "react-color": "^2.13.8",
@@ -248,8 +250,11 @@
     "@microsoft/api-documenter": "7.7.2",
     "@microsoft/api-extractor": "7.7.0",
     "@percy/agent": "^0.26.0",
-    "@testing-library/react": "^9.3.2",
-    "@testing-library/react-hooks": "^3.2.1",
+    "@testing-library/dom": "^7.24.2",
+    "@testing-library/jest-dom": "^5.11.4",
+    "@testing-library/react": "^11.0.4",
+    "@testing-library/react-hooks": "^3.4.1",
+    "@testing-library/user-event": "^12.1.6",
     "@types/accept": "3.1.1",
     "@types/angular": "^1.6.56",
     "@types/angular-mocks": "^1.7.0",
@@ -329,10 +334,8 @@
     "@types/supertest": "^2.0.5",
     "@types/supertest-as-promised": "^2.0.38",
     "@types/tar": "^4.0.3",
-    "@types/testing-library__dom": "^6.10.0",
-    "@types/testing-library__jest-dom": "^5.7.0",
-    "@types/testing-library__react": "^9.1.2",
-    "@types/testing-library__react-hooks": "^3.1.0",
+    "@types/testing-library__jest-dom": "^5.9.2",
+    "@types/testing-library__react-hooks": "^3.4.0",
     "@types/type-detect": "^4.0.1",
     "@types/uuid": "^3.4.4",
     "@types/vinyl": "^2.0.4",

packages/kbn-ace/README.md

@@ -0,0 +1,5 @@
+# @kbn/ace
+
+Contains all Kibana-specific brace related code. Excluding the code that still inside of Console because that code is only used inside of console at the moment.
+
+This package enables plugins to use this functionality and import it as needed -- behind an async import so that brace does not bloat the JS code needed for first page load of Kibana.

packages/kbn-ace/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "@kbn/ace",
+  "version": "1.0.0",
+  "private": true,
+  "main": "./target/index.js",
+  "license": "Apache-2.0",
+  "scripts": {
+    "build": "node ./scripts/build.js",
+    "kbn:bootstrap": "yarn build --dev"
+  },
+  "dependencies": {
+    "brace": "0.11.1"
+  },
+  "devDependencies": {
+    "@kbn/dev-utils": "1.0.0",
+    "@kbn/babel-preset": "1.0.0",
+    "raw-loader": "3.1.0",
+    "typescript": "4.0.2"
+  }
+}

packages/kbn-ace/scripts/build.js

@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const path = require('path');
+const del = require('del');
+const fs = require('fs');
+const supportsColor = require('supports-color');
+const { run } = require('@kbn/dev-utils');
+
+const TARGET_BUILD_DIR = path.resolve(__dirname, '../target');
+const ROOT_DIR = path.resolve(__dirname, '../');
+const WORKER_PATH_SECTION = 'ace/modes/x_json/worker/x_json.ace.worker.js';
+
+run(
+  async ({ procRunner, log }) => {
+    log.info('Deleting old output');
+
+    await del(TARGET_BUILD_DIR);
+
+    const cwd = ROOT_DIR;
+    const env = { ...process.env };
+
+    if (supportsColor.stdout) {
+      env.FORCE_COLOR = 'true';
+    }
+
+    await procRunner.run('tsc   ', {
+      cmd: 'tsc',
+      args: [],
+      wait: true,
+      env,
+      cwd,
+    });
+
+    log.success('Copying worker file to target.');
+
+    fs.copyFileSync(
+      path.resolve(__dirname, '..', 'src', WORKER_PATH_SECTION),
+      path.resolve(__dirname, '..', 'target', WORKER_PATH_SECTION)
+    );
+
+    log.success('Complete');
+  },
+  {
+    flags: {
+      boolean: ['dev'],
+    },
+  }
+);

src/plugins/es_ui_shared/public/monaco/index.ts → packages/kbn-ace/src/index.ts

@@ -17,4 +17,11 @@
  * under the License.
  */
 
-export { useXJsonMode } from '../../__packages_do_not_import__/monaco';
+export {
+  ElasticsearchSqlHighlightRules,
+  ScriptHighlightRules,
+  XJsonHighlightRules,
+  addXJsonToRules,
+  XJsonMode,
+  installXJsonMode,
+} from './ace/modes';

packages/kbn-ace/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./target",
+    "declaration": true,
+    "sourceMap": true,
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "src/**/*"
+  ]
+}

packages/kbn-ace/yarn.lock

@@ -0,0 +1 @@
+../../yarn.lock

packages/kbn-std/src/index.ts

@@ -24,6 +24,6 @@ export { mapToObject } from './map_to_object';
 export { merge } from './merge';
 export { pick } from './pick';
 export { withTimeout } from './promise';
-export { isRelativeUrl, modifyUrl, URLMeaningfulParts, ParsedQuery } from './url';
+export { isRelativeUrl, modifyUrl, URLMeaningfulParts } from './url';
 export { unset } from './unset';
 export { getFlattenedObject } from './get_flattened_object';

packages/kbn-std/src/url.ts

@@ -18,11 +18,7 @@
  */
 
 import { format as formatUrl, parse as parseUrl, UrlObject } from 'url';
-
-// duplicate type from 'query-string' to avoid adding the d.ts file to all packages depending on kbn-std
-export interface ParsedQuery<T = string> {
-  [key: string]: T | T[] | null | undefined;
-}
+import type { ParsedQuery } from 'query-string';
 
 /**
  * We define our own typings because the current version of @types/node

packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_working_collector.ts

@@ -75,11 +75,9 @@ export const parsedWorkingCollector: ParsedUsageCollection = [
           type: 'StringKeyword',
         },
         my_index_signature_prop: {
-          '': {
-            '@@INDEX@@': {
-              kind: SyntaxKind.NumberKeyword,
-              type: 'NumberKeyword',
-            },
+          '@@INDEX@@': {
+            kind: SyntaxKind.NumberKeyword,
+            type: 'NumberKeyword',
           },
         },
         my_objects: {

packages/kbn-telemetry-tools/src/tools/check_collector_integrity.ts

@@ -47,6 +47,7 @@ export function checkCompatibleTypeDescriptor(
       const typeDescriptorKinds = reduce(
         typeDescriptorTypes,
         (acc: any, type: number, key: string) => {
+          key = key.replace(/'/g, '');
           try {
             acc[key] = kindToDescriptorName(type);
           } catch (err) {
@@ -61,6 +62,7 @@ export function checkCompatibleTypeDescriptor(
       const transformedMappingKinds = reduce(
         schemaTypes,
         (acc: any, type: string, key: string) => {
+          key = key.replace(/'/g, '');
           try {
             acc[key.replace(/.type$/, '.kind')] = compatibleSchemaTypes(type as any);
           } catch (err) {

packages/kbn-telemetry-tools/src/tools/serializer.test.ts

@@ -44,13 +44,13 @@ export function loadFixtureProgram(fixtureName: string) {
 }
 
 describe('getDescriptor', () => {
-  const usageInterfaces = new Map<string, ts.InterfaceDeclaration>();
+  const usageInterfaces = new Map<string, ts.InterfaceDeclaration | ts.TypeAliasDeclaration>();
   let tsProgram: ts.Program;
   beforeAll(() => {
     const { program, sourceFile } = loadFixtureProgram('constants');
     tsProgram = program;
     for (const node of traverseNodes(sourceFile)) {
-      if (ts.isInterfaceDeclaration(node)) {
+      if (ts.isInterfaceDeclaration(node) || ts.isTypeAliasDeclaration(node)) {
         const interfaceName = node.name.getText();
         usageInterfaces.set(interfaceName, node);
       }
@@ -102,4 +102,26 @@ describe('getDescriptor', () => {
       'Mapping does not support conflicting union types.'
     );
   });
+
+  it('serializes TypeAliasDeclaration', () => {
+    const usageInterface = usageInterfaces.get('TypeAliasWithUnion')!;
+    const descriptor = getDescriptor(usageInterface, tsProgram);
+    expect(descriptor).toEqual({
+      locale: { kind: ts.SyntaxKind.StringKeyword, type: 'StringKeyword' },
+      prop1: { kind: ts.SyntaxKind.StringKeyword, type: 'StringKeyword' },
+      prop2: { kind: ts.SyntaxKind.StringKeyword, type: 'StringKeyword' },
+      prop3: { kind: ts.SyntaxKind.StringKeyword, type: 'StringKeyword' },
+      prop4: { kind: ts.SyntaxKind.StringLiteral, type: 'StringLiteral' },
+      prop5: { kind: ts.SyntaxKind.FirstLiteralToken, type: 'FirstLiteralToken' },
+    });
+  });
+
+  it('serializes Record entries', () => {
+    const usageInterface = usageInterfaces.get('TypeAliasWithRecord')!;
+    const descriptor = getDescriptor(usageInterface, tsProgram);
+    expect(descriptor).toEqual({
+      locale: { kind: ts.SyntaxKind.StringKeyword, type: 'StringKeyword' },
+      '@@INDEX@@': { kind: ts.SyntaxKind.NumberKeyword, type: 'NumberKeyword' },
+    });
+  });
 });

packages/kbn-telemetry-tools/src/tools/serializer.ts

@@ -79,9 +79,13 @@ export function getDescriptor(node: ts.Node, program: ts.Program): Descriptor |
   }
   if (ts.isTypeLiteralNode(node) || ts.isInterfaceDeclaration(node)) {
     return node.members.reduce((acc, m) => {
-      acc[m.name?.getText() || ''] = getDescriptor(m, program);
-      return acc;
-    }, {} as any);
+      const key = m.name?.getText();
+      if (key) {
+        return { ...acc, [key]: getDescriptor(m, program) };
+      } else {
+        return { ...acc, ...getDescriptor(m, program) };
+      }
+    }, {});
   }
 
   // If it's defined as signature { [key: string]: OtherInterface }
@@ -114,6 +118,10 @@ export function getDescriptor(node: ts.Node, program: ts.Program): Descriptor |
     if (symbolName === 'Date') {
       return { kind: TelemetryKinds.Date, type: 'Date' };
     }
+    // Support `Record<string, SOMETHING>`
+    if (symbolName === 'Record' && node.typeArguments![0].kind === ts.SyntaxKind.StringKeyword) {
+      return { '@@INDEX@@': getDescriptor(node.typeArguments![1], program) };
+    }
     const declaration = (symbol?.getDeclarations() || [])[0];
     if (declaration) {
       return getDescriptor(declaration, program);
@@ -157,6 +165,19 @@ export function getDescriptor(node: ts.Node, program: ts.Program): Descriptor |
     return uniqueKinds[0];
   }
 
+  // Support `type MyUsageType = SomethingElse`
+  if (ts.isTypeAliasDeclaration(node)) {
+    return getDescriptor(node.type, program);
+  }
+
+  // Support `&` unions
+  if (ts.isIntersectionTypeNode(node)) {
+    return node.types.reduce(
+      (acc, unionNode) => ({ ...acc, ...getDescriptor(unionNode, program) }),
+      {}
+    );
+  }
+
   switch (node.kind) {
     case ts.SyntaxKind.NumberKeyword:
     case ts.SyntaxKind.BooleanKeyword:

packages/kbn-telemetry-tools/src/tools/utils.ts

@@ -249,7 +249,7 @@ export function difference(actual: any, expected: any) {
       function (result, value, key) {
         if (key && /@@INDEX@@/.test(`${key}`)) {
           // The type definition is an Index Signature, fuzzy searching for similar keys
-          const regexp = new RegExp(`${key}`.replace(/@@INDEX@@/g, '(.+)?'));
+          const regexp = new RegExp(`^${key}`.replace(/@@INDEX@@/g, '(.+)?'));
           const keysInBase = Object.keys(base)
             .map((k) => {
               const match = k.match(regexp);