Merge branch 'master' of https://github.com/elastic/kibana into inter…

…nal_check_fix

package.json

@@ -115,8 +115,6 @@
     ]
   },
   "dependencies": {
-    "@babel/core": "^7.11.1",
-    "@babel/register": "^7.10.5",
     "@elastic/datemath": "5.0.3",
     "@elastic/elasticsearch": "7.9.1",
     "@elastic/eui": "29.0.0",
@@ -128,7 +126,6 @@
     "@hapi/wreck": "^15.0.2",
     "@kbn/analytics": "1.0.0",
     "@kbn/apm-config-loader": "1.0.0",
-    "@kbn/babel-preset": "1.0.0",
     "@kbn/config": "1.0.0",
     "@kbn/config-schema": "1.0.0",
     "@kbn/i18n": "1.0.0",
@@ -214,6 +211,7 @@
     "rxjs": "^6.5.5",
     "seedrandom": "^3.0.5",
     "semver": "^5.7.0",
+    "source-map-support": "^0.5.19",
     "style-it": "^2.1.3",
     "symbol-observable": "^1.2.0",
     "tar": "4.4.13",
@@ -227,7 +225,9 @@
     "yauzl": "^2.10.0"
   },
   "devDependencies": {
+    "@babel/core": "^7.11.1",
     "@babel/parser": "^7.11.2",
+    "@babel/register": "^7.10.5",
     "@babel/types": "^7.11.0",
     "@elastic/apm-rum": "^5.6.1",
     "@elastic/charts": "23.0.0",
@@ -238,6 +238,7 @@
     "@elastic/github-checks-reporter": "0.0.20b3",
     "@elastic/makelogs": "^6.0.0",
     "@elastic/ui-ace": "0.2.3",
+    "@kbn/babel-preset": "1.0.0",
     "@kbn/dev-utils": "1.0.0",
     "@kbn/es": "1.0.0",
     "@kbn/es-archiver": "1.0.0",

packages/kbn-babel-preset/node_preset.js

@@ -49,13 +49,5 @@ module.exports = (_, options = {}) => {
       ],
       require('./common_preset'),
     ],
-    plugins: [
-      [
-        require.resolve('babel-plugin-transform-define'),
-        {
-          'global.__BUILT_WITH_BABEL__': 'true',
-        },
-      ],
-    ],
   };
 };

packages/kbn-babel-preset/package.json

@@ -14,7 +14,6 @@
     "@babel/preset-typescript": "^7.10.4",
     "babel-plugin-add-module-exports": "^1.0.2",
     "babel-plugin-styled-components": "^1.10.7",
-    "babel-plugin-transform-define": "^1.3.1",
     "babel-plugin-transform-react-remove-prop-types": "^0.4.24",
     "react-is": "^16.8.0",
     "styled-components": "^5.1.0"

packages/kbn-test/src/functional_tests/lib/babel_register_for_test_plugins.js

@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const Path = require('path');
+
+const { REPO_ROOT } = require('@kbn/dev-utils');
+
+// modifies all future calls to require() to automatically
+// compile the required source with babel
+require('@babel/register')({
+  ignore: [/[\/\\](node_modules|target|dist)[\/\\]/],
+  only: [
+    Path.resolve(REPO_ROOT, 'test'),
+    Path.resolve(REPO_ROOT, 'x-pack/test'),
+    Path.resolve(REPO_ROOT, 'examples'),
+    Path.resolve(REPO_ROOT, 'x-pack/examples'),
+    // TODO: should should probably remove this link back to the source
+    Path.resolve(REPO_ROOT, 'x-pack/plugins/task_manager/server/config.ts'),
+  ],
+  babelrc: false,
+  presets: [require.resolve('@kbn/babel-preset/node_preset')],
+  extensions: ['.js', '.ts', '.tsx'],
+});

packages/kbn-test/src/functional_tests/lib/run_kibana_server.js

@@ -17,9 +17,26 @@
  * under the License.
  */
 
-import { resolve } from 'path';
+import { resolve, relative } from 'path';
 import { KIBANA_ROOT, KIBANA_EXEC, KIBANA_EXEC_PATH } from './paths';
 
+function extendNodeOptions(installDir) {
+  if (!installDir) {
+    return {};
+  }
+
+  const testOnlyRegisterPath = relative(
+    installDir,
+    require.resolve('./babel_register_for_test_plugins')
+  );
+
+  return {
+    NODE_OPTIONS: `--require=${testOnlyRegisterPath}${
+      process.env.NODE_OPTIONS ? ` ${process.env.NODE_OPTIONS}` : ''
+    }`,
+  };
+}
+
 export async function runKibanaServer({ procs, config, options }) {
   const { installDir } = options;
 
@@ -29,6 +46,7 @@ export async function runKibanaServer({ procs, config, options }) {
     env: {
       FORCE_COLOR: 1,
       ...process.env,
+      ...extendNodeOptions(installDir),
     },
     cwd: installDir || KIBANA_ROOT,
     wait: /http server running/,

scripts/build_plugin_list_docs.js

@@ -17,5 +17,5 @@
  * under the License.
  */
 
-require('../src/setup_node_env/prebuilt_dev_only_entry');
+require('../src/setup_node_env/no_transpilation');
 require('@kbn/dev-utils').runPluginListCli();

scripts/es.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-require('../src/setup_node_env/prebuilt_dev_only_entry');
+require('../src/setup_node_env/no_transpilation');
 
 var resolve = require('path').resolve;
 var pkg = require('../package.json');

scripts/generate_plugin.js

@@ -17,5 +17,5 @@
  * under the License.
  */
 
-require('../src/setup_node_env/prebuilt_dev_only_entry');
+require('../src/setup_node_env/no_transpilation');
 require('@kbn/plugin-generator').runCli();

scripts/plugin_helpers.js

@@ -17,5 +17,5 @@
  * under the License.
  */
 
-require('../src/setup_node_env/prebuilt_dev_only_entry');
+require('../src/setup_node_env/no_transpilation');
 require('@kbn/plugin-helpers').runCli();

scripts/register_git_hook.js

@@ -17,5 +17,5 @@
  * under the License.
  */
 
-require('../src/setup_node_env/prebuilt_dev_only_entry');
+require('../src/setup_node_env/no_transpilation');
 require('@kbn/dev-utils/target/precommit_hook/cli');

scripts/release_notes.js

@@ -17,5 +17,5 @@
  * under the License.
  */
 
-require('../src/setup_node_env/prebuilt_dev_only_entry');
+require('../src/setup_node_env/no_transpilation');
 require('@kbn/release-notes').runReleaseNotesCli();

scripts/telemetry_check.js

@@ -17,5 +17,5 @@
  * under the License.
  */
 
-require('../src/setup_node_env/prebuilt_dev_only_entry');
+require('../src/setup_node_env/no_transpilation');
 require('@kbn/telemetry-tools').runTelemetryCheck();

scripts/telemetry_extract.js

@@ -17,5 +17,5 @@
  * under the License.
  */
 
-require('../src/setup_node_env/prebuilt_dev_only_entry');
+require('../src/setup_node_env/no_transpilation');
 require('@kbn/telemetry-tools').runTelemetryExtract();

src/cli/index.js

@@ -18,5 +18,6 @@
  */
 
 require('../apm')();
-require('../setup_node_env');
+require('../setup_node_env/no_transpilation');
+require('../setup_node_env/babel_register/polyfill');
 require('./cli');

src/cli/serve/integration_tests/invalid_config.test.ts

@@ -18,10 +18,10 @@
  */
 
 import { spawnSync } from 'child_process';
-import { resolve } from 'path';
 
-const ROOT_DIR = resolve(__dirname, '../../../../');
-const INVALID_CONFIG_PATH = resolve(__dirname, '__fixtures__/invalid_config.yml');
+import { REPO_ROOT } from '@kbn/dev-utils';
+
+const INVALID_CONFIG_PATH = require.resolve('./__fixtures__/invalid_config.yml');
 
 interface LogEntry {
   message: string;
@@ -35,11 +35,11 @@ describe('cli invalid config support', function () {
     function () {
       // Unused keys only throw once LegacyService starts, so disable migrations so that Core
       // will finish the start lifecycle without a running Elasticsearch instance.
-      const { error, status, stdout } = spawnSync(
+      const { error, status, stdout, stderr } = spawnSync(
         process.execPath,
-        ['src/cli', '--config', INVALID_CONFIG_PATH, '--migrations.skip=true'],
+        ['scripts/kibana', '--config', INVALID_CONFIG_PATH, '--migrations.skip=true'],
         {
-          cwd: ROOT_DIR,
+          cwd: REPO_ROOT,
         }
       );
 
@@ -57,13 +57,21 @@ describe('cli invalid config support', function () {
         }));
 
       expect(error).toBe(undefined);
-      expect(status).toBe(64);
+
+      if (!fatalLogLine) {
+        throw new Error(
+          `cli did not log the expected fatal error message:\n\nstdout: \n${stdout}\n\nstderr:\n${stderr}`
+        );
+      }
+
       expect(fatalLogLine.message).toContain(
         'Error: Unknown configuration key(s): "unknown.key", "other.unknown.key", "other.third", "some.flat.key", ' +
           '"some.array". Check for spelling errors and ensure that expected plugins are installed.'
       );
       expect(fatalLogLine.tags).toEqual(['fatal', 'root']);
       expect(fatalLogLine.type).toEqual('log');
+
+      expect(status).toBe(64);
     },
     20 * 1000
   );

src/dev/build/tasks/copy_source_task.ts

@@ -37,6 +37,8 @@ export const CopySource: Task = {
         '!src/cli/repl/**',
         '!src/functional_test_runner/**',
         '!src/dev/**',
+        '!src/setup_node_env/babel_register/index.js',
+        '!src/setup_node_env/babel_register/register.js',
         '!**/public/**',
         'typings/**',
         'config/kibana.yml',

src/setup_node_env/babel_register/register.js

@@ -46,26 +46,12 @@ var ignore = [
 
   // ignore paths matching `/canvas/canvas_plugin/`
   /[\/\\]canvas[\/\\]canvas_plugin[\/\\]/,
-];
 
-if (global.__BUILT_WITH_BABEL__) {
-  // when building the Kibana source we replace the statement
-  // `global.__BUILT_WITH_BABEL__` with the value `true` so that
-  // when @babel/register is required for the first time by users
-  // it will exclude kibana's `src` directory.
-  //
-  // We still need @babel/register for plugins though, we've been
-  // building their server code at require-time since version 4.2
-  // TODO: the plugin install process could transpile plugin server code...
-  ignore.push(resolve(__dirname, '../../../src'));
-} else {
-  ignore.push(
-    // ignore any path in the packages, unless it is in the package's
-    // root `src` directory, in any test or __tests__ directory, or it
-    // ends with .test.js, .test.ts, or .test.tsx
-    /[\/\\]packages[\/\\](eslint-|kbn-)[^\/\\]+[\/\\](?!src[\/\\].*|(.+[\/\\])?(test|__tests__)[\/\\].+|.+\.test\.(js|ts|tsx)$)(.+$)/
-  );
-}
+  // ignore any path in the packages, unless it is in the package's
+  // root `src` directory, in any test or __tests__ directory, or it
+  // ends with .test.js, .test.ts, or .test.tsx
+  /[\/\\]packages[\/\\](eslint-|kbn-)[^\/\\]+[\/\\](?!src[\/\\].*|(.+[\/\\])?(test|__tests__)[\/\\].+|.+\.test\.(js|ts|tsx)$)(.+$)/,
+];
 
 // modifies all future calls to require() to automatically
 // compile the required source with babel

src/setup_node_env/index.js

@@ -17,5 +17,5 @@
  * under the License.
  */
 
-require('./prebuilt_dev_only_entry');
+require('./no_transpilation');
 require('./babel_register');

x-pack/package.json

@@ -32,6 +32,7 @@
     "@cypress/webpack-preprocessor": "^5.4.1",
     "@elastic/apm-rum-react": "^1.2.5",
     "@elastic/maki": "6.3.0",
+    "@kbn/babel-preset": "1.0.0",
     "@kbn/dev-utils": "1.0.0",
     "@kbn/es": "1.0.0",
     "@kbn/expect": "1.0.0",
@@ -280,7 +281,6 @@
     "@elastic/node-crypto": "1.2.1",
     "@elastic/numeral": "^2.5.0",
     "@elastic/safer-lodash-set": "0.0.0",
-    "@kbn/babel-preset": "1.0.0",
     "@kbn/config-schema": "1.0.0",
     "@kbn/i18n": "1.0.0",
     "@kbn/interpreter": "1.0.0",

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts

@@ -212,6 +212,37 @@ test('tests processing keyword field with multi fields with analyzed text field'
   expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping);
 });
 
+test('tests processing keyword field with multi fields with normalized keyword field', () => {
+  const keywordWithNormalizedMultiFieldsLiteralYml = `
+  - name: keywordWithNormalizedMultiField
+    type: keyword
+    multi_fields:
+      - name: normalized
+        type: keyword
+        normalizer: lowercase
+  `;
+
+  const keywordWithNormalizedMultiFieldsMapping = {
+    properties: {
+      keywordWithNormalizedMultiField: {
+        ignore_above: 1024,
+        type: 'keyword',
+        fields: {
+          normalized: {
+            type: 'keyword',
+            ignore_above: 1024,
+            normalizer: 'lowercase',
+          },
+        },
+      },
+    },
+  };
+  const fields: Field[] = safeLoad(keywordWithNormalizedMultiFieldsLiteralYml);
+  const processedFields = processFields(fields);
+  const mappings = generateMappings(processedFields);
+  expect(mappings).toEqual(keywordWithNormalizedMultiFieldsMapping);
+});
+
 test('tests processing object field with no other attributes', () => {
   const objectFieldLiteralYml = `
 - name: objectField

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts

@@ -189,6 +189,9 @@ function generateKeywordMapping(field: Field): IndexTemplateMapping {
   if (field.ignore_above) {
     mapping.ignore_above = field.ignore_above;
   }
+  if (field.normalizer) {
+    mapping.normalizer = field.normalizer;
+  }
   return mapping;
 }
 

x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts

@@ -20,6 +20,7 @@ export interface Field {
   index?: boolean;
   required?: boolean;
   multi_fields?: Fields;
+  normalizer?: string;
   doc_values?: boolean;
   copy_to?: string;
   analyzer?: string;

x-pack/plugins/lens/public/async_services.ts

@@ -21,4 +21,5 @@ export * from './xy_visualization/xy_visualization';
 export * from './indexpattern_datasource/indexpattern';
 
 export * from './editor_frame_service/editor_frame';
+export * from './editor_frame_service/embeddable';
 export * from './app_plugin/mounter';

x-pack/plugins/lens/public/editor_frame_service/editor_frame/index.ts

@@ -5,3 +5,5 @@
  */
 
 export * from './editor_frame';
+export * from './state_helpers';
+export * from './state_management';