Merge remote-tracking branch 'upstream/7.x' into backport/7.x/pr-62819

.i18nrc.json

@@ -54,7 +54,8 @@
     "visTypeVega": "src/legacy/core_plugins/vis_type_vega",
     "visTypeVislib": "src/legacy/core_plugins/vis_type_vislib",
     "visTypeXy": "src/legacy/core_plugins/vis_type_xy",
-    "visualizations": "src/plugins/visualizations"
+    "visualizations": "src/plugins/visualizations",
+    "visualize": "src/plugins/visualize"
   },
   "exclude": [
     "src/legacy/ui/ui_render/ui_render_mixin.js"

docs/api/saved-objects/create.asciidoc

@@ -59,12 +59,12 @@ any data that you send to the API is properly formed.
 
 [source,sh]
 --------------------------------------------------
-$ curl -X POST "localhost:5601/api/saved_objects/index-pattern/my-pattern"
+$ curl -X POST "localhost:5601/api/saved_objects/index-pattern/my-pattern"  -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '
 {
   "attributes": {
     "title": "my-pattern-*"
   }
-}
+}'
 --------------------------------------------------
 // KIBANA
 

docs/api/saved-objects/export.asciidoc

@@ -68,46 +68,46 @@ Export all index pattern saved objects:
 
 [source,sh]
 --------------------------------------------------
-$ curl -X POST "localhost:5601/api/saved_objects/_export"
+$ curl -X POST "localhost:5601/api/saved_objects/_export" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '
 {
   "type": "index-pattern"
-}
+}'
 --------------------------------------------------
 // KIBANA
 
 Export all index pattern saved objects and exclude the export summary from the stream:
 
 [source,sh]
 --------------------------------------------------
-$ curl -X POST "localhost:5601/api/saved_objects/_export"
+$ curl -X POST "localhost:5601/api/saved_objects/_export" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '
 {
   "type": "index-pattern",
   "excludeExportDetails": true
-}
+}'
 --------------------------------------------------
 // KIBANA
 
 Export a specific saved object:
 
 [source,sh]
 --------------------------------------------------
-$ curl -X POST "localhost:5601/api/saved_objects/_export"
+$ curl -X POST "localhost:5601/api/saved_objects/_export" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '
 {
   "objects": [
     {
       "type": "dashboard",
       "id": "be3733a0-9efe-11e7-acb3-3dab96693fab"
     }
   ]
-}
+}'
 --------------------------------------------------
 // KIBANA
 
 Export a specific saved object and it's related objects :
 
 [source,sh]
 --------------------------------------------------
-$ curl -X POST "localhost:5601/api/saved_objects/_export"
+$ curl -X POST "localhost:5601/api/saved_objects/_export" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '
 {
   "objects": [
     {
@@ -116,6 +116,6 @@ $ curl -X POST "localhost:5601/api/saved_objects/_export"
     }
   ],
   "includeReferencesDeep": true
-}
+}'
 --------------------------------------------------
 // KIBANA

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternfield.indexpattern.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPatternField](./kibana-plugin-plugins-data-public.indexpatternfield.md) &gt; [indexPattern](./kibana-plugin-plugins-data-public.indexpatternfield.indexpattern.md)
+
+## IndexPatternField.indexPattern property
+
+<b>Signature:</b>
+
+```typescript
+indexPattern?: IndexPattern;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternfield.md

@@ -27,6 +27,7 @@ export declare class Field implements IFieldType
 |  [esTypes](./kibana-plugin-plugins-data-public.indexpatternfield.estypes.md) |  | <code>string[]</code> |  |
 |  [filterable](./kibana-plugin-plugins-data-public.indexpatternfield.filterable.md) |  | <code>boolean</code> |  |
 |  [format](./kibana-plugin-plugins-data-public.indexpatternfield.format.md) |  | <code>any</code> |  |
+|  [indexPattern](./kibana-plugin-plugins-data-public.indexpatternfield.indexpattern.md) |  | <code>IndexPattern</code> |  |
 |  [lang](./kibana-plugin-plugins-data-public.indexpatternfield.lang.md) |  | <code>string</code> |  |
 |  [name](./kibana-plugin-plugins-data-public.indexpatternfield.name.md) |  | <code>string</code> |  |
 |  [routes](./kibana-plugin-plugins-data-public.indexpatternfield.routes.md) |  | <code>Record&lt;string, string&gt;</code> |  |

docs/ingest_manager/index-templates.asciidoc

@@ -0,0 +1,7 @@
+# Elasticsearch Index Templates
+
+## Generation
+
+* Index templates are generated from `YAML` files contained in the package.
+* There is one index template per dataset.
+* For the generation of an index template, all `yml` files contained in the package subdirectory `dataset/DATASET_NAME/fields/` are used.

docs/ingest_manager/index.asciidoc

@@ -197,3 +197,10 @@ The new ingest pipeline is expected to still work with the data coming from olde
 In case of a breaking change in the data structure, the new ingest pipeline is also expected to deal with this change. In case there are breaking changes which cannot be dealt with in an ingest pipeline, a new package has to be created.
 
 Each package lists its minimal required agent version. In case there are agents enrolled with an older version, the user is notified to upgrade these agents as otherwise the new configs cannot be rolled out.
+
+=== Generated assets
+
+When a package is installed or upgraded, certain Kibana and Elasticsearch assets are generated from . These follow the naming conventions explained above (see "indexing strategy") and contain configuration for the elastic stack that makes ingesting and displaying data work with as little user interaction as possible.
+
+* link:index-templates.asciidoc[Elasticsearch Index Templates]
+* Kibana Index Patterns

package.json

@@ -118,7 +118,7 @@
     "@babel/core": "^7.9.0",
     "@babel/register": "^7.9.0",
     "@elastic/apm-rum": "^4.6.0",
-    "@elastic/charts": "18.2.2",
+    "@elastic/charts": "18.3.0",
     "@elastic/datemath": "5.0.2",
     "@elastic/ems-client": "7.8.0",
     "@elastic/eui": "21.0.1",

packages/kbn-ui-shared-deps/package.json

@@ -9,7 +9,7 @@
     "kbn:watch": "node scripts/build --watch"
   },
   "dependencies": {
-    "@elastic/charts": "18.2.2",
+    "@elastic/charts": "18.3.0",
     "@elastic/eui": "21.0.1",
     "@kbn/i18n": "1.0.0",
     "abortcontroller-polyfill": "^1.4.0",

src/core/server/legacy/legacy_service.test.mocks.ts

@@ -30,6 +30,11 @@ export const findLegacyPluginSpecsMock = jest.fn().mockImplementation((settings:
   uiExports: {},
   navLinks: [],
 }));
-jest.doMock('./plugins/find_legacy_plugin_specs.ts', () => ({
+jest.doMock('./plugins/find_legacy_plugin_specs', () => ({
   findLegacyPluginSpecs: findLegacyPluginSpecsMock,
 }));
+
+export const logLegacyThirdPartyPluginDeprecationWarningMock = jest.fn();
+jest.doMock('./plugins/log_legacy_plugins_warning', () => ({
+  logLegacyThirdPartyPluginDeprecationWarning: logLegacyThirdPartyPluginDeprecationWarningMock,
+}));

src/core/server/legacy/legacy_service.test.ts

@@ -22,7 +22,10 @@ jest.mock('../../../cli/cluster/cluster_manager');
 jest.mock('./config/legacy_deprecation_adapters', () => ({
   convertLegacyDeprecationProvider: (provider: any) => Promise.resolve(provider),
 }));
-import { findLegacyPluginSpecsMock } from './legacy_service.test.mocks';
+import {
+  findLegacyPluginSpecsMock,
+  logLegacyThirdPartyPluginDeprecationWarningMock,
+} from './legacy_service.test.mocks';
 
 import { BehaviorSubject, throwError } from 'rxjs';
 
@@ -476,6 +479,38 @@ describe('#discoverPlugins()', () => {
     expect(configService.addDeprecationProvider).toHaveBeenCalledWith('', 'providerA');
     expect(configService.addDeprecationProvider).toHaveBeenCalledWith('', 'providerB');
   });
+
+  it(`logs deprecations for legacy third party plugins`, async () => {
+    const pluginSpecs = [
+      { getId: () => 'pluginA', getDeprecationsProvider: () => undefined },
+      { getId: () => 'pluginB', getDeprecationsProvider: () => undefined },
+    ];
+    findLegacyPluginSpecsMock.mockImplementation(
+      settings =>
+        Promise.resolve({
+          pluginSpecs,
+          pluginExtendedConfig: settings,
+          disabledPluginSpecs: [],
+          uiExports: {},
+          navLinks: [],
+        }) as any
+    );
+
+    const legacyService = new LegacyService({
+      coreId,
+      env,
+      logger,
+      configService: configService as any,
+    });
+
+    await legacyService.discoverPlugins();
+
+    expect(logLegacyThirdPartyPluginDeprecationWarningMock).toHaveBeenCalledTimes(1);
+    expect(logLegacyThirdPartyPluginDeprecationWarningMock).toHaveBeenCalledWith({
+      specs: pluginSpecs,
+      log: expect.any(Object),
+    });
+  });
 });
 
 test('Sets the server.uuid property on the legacy configuration', async () => {

src/core/server/legacy/legacy_service.ts

@@ -28,7 +28,7 @@ import { DevConfig, DevConfigType, config as devConfig } from '../dev';
 import { BasePathProxyServer, HttpConfig, HttpConfigType, config as httpConfig } from '../http';
 import { Logger } from '../logging';
 import { PathConfigType } from '../path';
-import { findLegacyPluginSpecs } from './plugins';
+import { findLegacyPluginSpecs, logLegacyThirdPartyPluginDeprecationWarning } from './plugins';
 import { convertLegacyDeprecationProvider } from './config';
 import {
   ILegacyInternals,
@@ -133,6 +133,11 @@ export class LegacyService implements CoreService {
       this.coreContext.env.packageInfo
     );
 
+    logLegacyThirdPartyPluginDeprecationWarning({
+      specs: pluginSpecs,
+      log: this.log,
+    });
+
     this.legacyPlugins = {
       pluginSpecs,
       disabledPluginSpecs,

src/core/server/legacy/plugins/index.ts

@@ -18,3 +18,4 @@
  */
 
 export { findLegacyPluginSpecs } from './find_legacy_plugin_specs';
+export { logLegacyThirdPartyPluginDeprecationWarning } from './log_legacy_plugins_warning';

src/core/server/legacy/plugins/log_legacy_plugins_warning.test.ts

@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { loggerMock } from '../../logging/logger.mock';
+import { logLegacyThirdPartyPluginDeprecationWarning } from './log_legacy_plugins_warning';
+import { LegacyPluginSpec } from '../types';
+
+const createPluginSpec = ({ id, path }: { id: string; path: string }): LegacyPluginSpec => {
+  return {
+    getId: () => id,
+    getExpectedKibanaVersion: () => 'kibana',
+    getConfigPrefix: () => 'plugin.config',
+    getDeprecationsProvider: () => undefined,
+    getPack: () => ({
+      getPath: () => path,
+    }),
+  };
+};
+
+describe('logLegacyThirdPartyPluginDeprecationWarning', () => {
+  let log: ReturnType<typeof loggerMock.create>;
+
+  beforeEach(() => {
+    log = loggerMock.create();
+  });
+
+  it('logs warning for third party plugins', () => {
+    logLegacyThirdPartyPluginDeprecationWarning({
+      specs: [createPluginSpec({ id: 'plugin', path: '/some-external-path' })],
+      log,
+    });
+    expect(log.warn).toHaveBeenCalledTimes(1);
+    expect(log.warn.mock.calls[0]).toMatchInlineSnapshot(`
+      Array [
+        "Some installed third party plugin(s) [plugin] are using the legacy plugin format and will no longer work in a future Kibana release. Please refer to https://www.elastic.co/guide/en/kibana/master/breaking-changes-8.0.html for a list of breaking changes and https://github.com/elastic/kibana/blob/master/src/core/MIGRATION.md for documentation on how to migrate legacy plugins.",
+      ]
+    `);
+  });
+
+  it('lists all the deprecated plugins and only log once', () => {
+    logLegacyThirdPartyPluginDeprecationWarning({
+      specs: [
+        createPluginSpec({ id: 'pluginA', path: '/abs/path/to/pluginA' }),
+        createPluginSpec({ id: 'pluginB', path: '/abs/path/to/pluginB' }),
+        createPluginSpec({ id: 'pluginC', path: '/abs/path/to/pluginC' }),
+      ],
+      log,
+    });
+    expect(log.warn).toHaveBeenCalledTimes(1);
+    expect(log.warn.mock.calls[0]).toMatchInlineSnapshot(`
+      Array [
+        "Some installed third party plugin(s) [pluginA, pluginB, pluginC] are using the legacy plugin format and will no longer work in a future Kibana release. Please refer to https://www.elastic.co/guide/en/kibana/master/breaking-changes-8.0.html for a list of breaking changes and https://github.com/elastic/kibana/blob/master/src/core/MIGRATION.md for documentation on how to migrate legacy plugins.",
+      ]
+    `);
+  });
+
+  it('does not log warning for internal legacy plugins', () => {
+    logLegacyThirdPartyPluginDeprecationWarning({
+      specs: [
+        createPluginSpec({
+          id: 'plugin',
+          path: '/absolute/path/to/kibana/src/legacy/core_plugins',
+        }),
+        createPluginSpec({
+          id: 'plugin',
+          path: '/absolute/path/to/kibana/x-pack',
+        }),
+      ],
+      log,
+    });
+
+    expect(log.warn).not.toHaveBeenCalled();
+  });
+});

src/core/server/legacy/plugins/log_legacy_plugins_warning.ts

@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Logger } from '../../logging';
+import { LegacyPluginSpec } from '../types';
+
+const internalPaths = ['/src/legacy/core_plugins', '/x-pack'];
+
+const breakingChangesUrl =
+  'https://www.elastic.co/guide/en/kibana/master/breaking-changes-8.0.html';
+const migrationGuideUrl = 'https://github.com/elastic/kibana/blob/master/src/core/MIGRATION.md';
+
+export const logLegacyThirdPartyPluginDeprecationWarning = ({
+  specs,
+  log,
+}: {
+  specs: LegacyPluginSpec[];
+  log: Logger;
+}) => {
+  const thirdPartySpecs = specs.filter(isThirdPartyPluginSpec);
+  if (thirdPartySpecs.length > 0) {
+    const pluginIds = thirdPartySpecs.map(spec => spec.getId());
+    log.warn(
+      `Some installed third party plugin(s) [${pluginIds.join(
+        ', '
+      )}] are using the legacy plugin format and will no longer work in a future Kibana release. ` +
+        `Please refer to ${breakingChangesUrl} for a list of breaking changes ` +
+        `and ${migrationGuideUrl} for documentation on how to migrate legacy plugins.`
+    );
+  }
+};
+
+const isThirdPartyPluginSpec = (spec: LegacyPluginSpec): boolean => {
+  const pluginPath = spec.getPack().getPath();
+  return !internalPaths.some(internalPath => pluginPath.indexOf(internalPath) > -1);
+};

src/core/server/legacy/types.ts

@@ -98,6 +98,7 @@ export interface LegacyPluginSpec {
   getExpectedKibanaVersion: () => string;
   getConfigPrefix: () => string;
   getDeprecationsProvider: () => LegacyConfigDeprecationProvider | undefined;
+  getPack: () => LegacyPluginPack;
 }
 
 /**