Merge branch 'master' of https://github.com/elastic/kibana into alert…

…ing/default-es-index-schema

.github/workflows/project-assigner.yml

@@ -11,7 +11,5 @@ jobs:
         uses: elastic/github-actions/[email protected]
         id: project_assigner
         with:
-          issue-mappings: '[{"label": "Feature:Lens", "projectNumber": 32, "columnName": "Long-term goals"}, {"label": "Feature:Canvas", "projectNumber": 38, "columnName": "Inbox"}, {"label": "Feature:Dashboard", "projectNumber": 68, "columnName": "Inbox"}, {"label": "Feature:Drilldowns", "projectNumber": 68, "columnName": "Inbox"}], {"label": "Feature:Input Controls", "projectNumber": 72, "columnName": "Inbox"}]'
+          issue-mappings: '[{"label": "Feature:Lens", "projectNumber": 32, "columnName": "Long-term goals"}, {"label": "Feature:Canvas", "projectNumber": 38, "columnName": "Inbox"}, {"label": "Feature:Dashboard", "projectNumber": 68, "columnName": "Inbox"}, {"label": "Feature:Drilldowns", "projectNumber": 68, "columnName": "Inbox"}, {"label": "Feature:Input Controls", "projectNumber": 72, "columnName": "Inbox"}]'
           ghToken: ${{ secrets.PROJECT_ASSIGNER_TOKEN }}
-
-

docs/setup/upgrade/upgrade-migrations.asciidoc

@@ -50,6 +50,16 @@ For large deployments with more than 10 {kib} instances and more than 10 000 sav
 ==== Preventing migration failures
 This section highlights common causes of {kib} upgrade failures and how to prevent them.
 
+[float]
+===== timeout_exception or receive_timeout_transport_exception
+There is a known issue in v7.12.0 for users who tried the fleet beta. Upgrade migrations fail because of a large number of documents in the `.kibana` index.
+
+This can cause Kibana to log errors like:
+> Error: Unable to complete saved object migrations for the [.kibana] index. Please check the health of your Elasticsearch cluster and try again. Error: [receive_timeout_transport_exception]: [instance-0000000002][10.32.1.112:19541][cluster:monitor/task/get] request_id [2648] timed out after [59940ms]
+> Error: Unable to complete saved object migrations for the [.kibana] index. Please check the health of your Elasticsearch cluster and try again. Error: [timeout_exception]: Timed out waiting for completion of [org.elasticsearch.index.reindex.BulkByScrollTask@6a74c54]
+
+See https://github.com/elastic/kibana/issues/95321 for instructions to work around this issue.
+
 [float]
 ===== Corrupt saved objects
 We highly recommend testing your {kib} upgrade in a development cluster to discover and remedy problems caused by corrupt documents, especially when there are custom integrations creating saved objects in your environment. Saved objects that were corrupted through manual editing or integrations will cause migration failures with a log message like `Failed to transform document. Transform: index-pattern:7.0.0\n Doc: {...}` or `Unable to migrate the corrupt Saved Object document ...`. Corrupt documents will have to be fixed or deleted before an upgrade migration can succeed.

src/core/server/config/deprecation/core_deprecations.test.ts

@@ -6,27 +6,12 @@
  * Side Public License, v 1.
  */
 
-import { configDeprecationFactory, applyDeprecations } from '@kbn/config';
+import { getDeprecationsForGlobalSettings } from '../test_utils';
 import { coreDeprecationProvider } from './core_deprecations';
-
 const initialEnv = { ...process.env };
 
-const applyCoreDeprecations = (settings: Record<string, any> = {}) => {
-  const deprecations = coreDeprecationProvider(configDeprecationFactory);
-  const deprecationMessages: string[] = [];
-  const migrated = applyDeprecations(
-    settings,
-    deprecations.map((deprecation) => ({
-      deprecation,
-      path: '',
-    })),
-    () => ({ message }) => deprecationMessages.push(message)
-  );
-  return {
-    messages: deprecationMessages,
-    migrated,
-  };
-};
+const applyCoreDeprecations = (settings?: Record<string, any>) =>
+  getDeprecationsForGlobalSettings({ provider: coreDeprecationProvider, settings });
 
 describe('core deprecations', () => {
   beforeEach(() => {

src/core/server/config/test_utils.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import type { ConfigDeprecationProvider } from '@kbn/config';
+import { configDeprecationFactory, applyDeprecations } from '@kbn/config';
+
+function collectDeprecations(
+  provider: ConfigDeprecationProvider,
+  settings: Record<string, any>,
+  path: string
+) {
+  const deprecations = provider(configDeprecationFactory);
+  const deprecationMessages: string[] = [];
+  const migrated = applyDeprecations(
+    settings,
+    deprecations.map((deprecation) => ({
+      deprecation,
+      path,
+    })),
+    () => ({ message }) => deprecationMessages.push(message)
+  );
+  return {
+    messages: deprecationMessages,
+    migrated,
+  };
+}
+
+export const getDeprecationsFor = ({
+  provider,
+  settings = {},
+  path,
+}: {
+  provider: ConfigDeprecationProvider;
+  settings?: Record<string, any>;
+  path: string;
+}) => {
+  return collectDeprecations(provider, { [path]: settings }, path);
+};
+
+export const getDeprecationsForGlobalSettings = ({
+  provider,
+  settings = {},
+}: {
+  provider: ConfigDeprecationProvider;
+  settings?: Record<string, any>;
+}) => {
+  return collectDeprecations(provider, settings, '');
+};

src/core/server/elasticsearch/elasticsearch_config.test.ts

@@ -12,29 +12,17 @@ import {
   mockReadPkcs12Truststore,
 } from './elasticsearch_config.test.mocks';
 
-import { applyDeprecations, configDeprecationFactory } from '@kbn/config';
 import { ElasticsearchConfig, config } from './elasticsearch_config';
+import { getDeprecationsFor } from '../config/test_utils';
 
 const CONFIG_PATH = 'elasticsearch';
 
-const applyElasticsearchDeprecations = (settings: Record<string, any> = {}) => {
-  const deprecations = config.deprecations!(configDeprecationFactory);
-  const deprecationMessages: string[] = [];
-  const _config: any = {};
-  _config[CONFIG_PATH] = settings;
-  const migrated = applyDeprecations(
-    _config,
-    deprecations.map((deprecation) => ({
-      deprecation,
-      path: CONFIG_PATH,
-    })),
-    () => ({ message }) => deprecationMessages.push(message)
-  );
-  return {
-    messages: deprecationMessages,
-    migrated,
-  };
-};
+const applyElasticsearchDeprecations = (settings: Record<string, any> = {}) =>
+  getDeprecationsFor({
+    provider: config.deprecations!,
+    settings,
+    path: CONFIG_PATH,
+  });
 
 test('set correct defaults', () => {
   const configValue = new ElasticsearchConfig(config.schema.validate({}));

src/core/server/kibana_config.test.ts

@@ -7,28 +7,16 @@
  */
 
 import { config } from './kibana_config';
-import { applyDeprecations, configDeprecationFactory } from '@kbn/config';
+import { getDeprecationsFor } from './config/test_utils';
 
 const CONFIG_PATH = 'kibana';
 
-const applyKibanaDeprecations = (settings: Record<string, any> = {}) => {
-  const deprecations = config.deprecations!(configDeprecationFactory);
-  const deprecationMessages: string[] = [];
-  const _config: any = {};
-  _config[CONFIG_PATH] = settings;
-  const migrated = applyDeprecations(
-    _config,
-    deprecations.map((deprecation) => ({
-      deprecation,
-      path: CONFIG_PATH,
-    })),
-    () => ({ message }) => deprecationMessages.push(message)
-  );
-  return {
-    messages: deprecationMessages,
-    migrated,
-  };
-};
+const applyKibanaDeprecations = (settings: Record<string, any> = {}) =>
+  getDeprecationsFor({
+    provider: config.deprecations!,
+    settings,
+    path: CONFIG_PATH,
+  });
 
 it('set correct defaults ', () => {
   const configValue = config.schema.validate({});

src/core/server/saved_objects/migrationsv2/integration_tests/migration.test.ts

@@ -19,7 +19,8 @@ import { Root } from '../../../root';
 
 const kibanaVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version;
 
-describe('migration v2', () => {
+// FLAKY: https://github.com/elastic/kibana/issues/91107
+describe.skip('migration v2', () => {
   let esServer: kbnTestServer.TestElasticsearchUtils;
   let root: Root;
   let coreStart: InternalCoreStart;

src/core/server/saved_objects/saved_objects_config.test.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { savedObjectsMigrationConfig } from './saved_objects_config';
+import { getDeprecationsFor } from '../config/test_utils';
+
+const applyMigrationsDeprecations = (settings: Record<string, any> = {}) =>
+  getDeprecationsFor({
+    provider: savedObjectsMigrationConfig.deprecations!,
+    settings,
+    path: 'migrations',
+  });
+
+describe('migrations config', function () {
+  describe('deprecations', () => {
+    it('logs a warning if migrations.enableV2 is set: true', () => {
+      const { messages } = applyMigrationsDeprecations({ enableV2: true });
+      expect(messages).toMatchInlineSnapshot(`
+        Array [
+          "\\"migrations.enableV2\\" is deprecated and will be removed in an upcoming release without any further notice.",
+        ]
+      `);
+    });
+
+    it('logs a warning if migrations.enableV2 is set: false', () => {
+      const { messages } = applyMigrationsDeprecations({ enableV2: false });
+      expect(messages).toMatchInlineSnapshot(`
+        Array [
+          "\\"migrations.enableV2\\" is deprecated and will be removed in an upcoming release without any further notice.",
+        ]
+      `);
+    });
+  });
+
+  it('does not log a warning if migrations.enableV2 is not set', () => {
+    const { messages } = applyMigrationsDeprecations({ batchSize: 1_000 });
+    expect(messages).toMatchInlineSnapshot(`Array []`);
+  });
+});

src/core/server/saved_objects/saved_objects_config.ts

@@ -7,31 +7,50 @@
  */
 
 import { schema, TypeOf } from '@kbn/config-schema';
+import type { ServiceConfigDescriptor } from '../internal_types';
+import type { ConfigDeprecationProvider } from '../config';
 
-export type SavedObjectsMigrationConfigType = TypeOf<typeof savedObjectsMigrationConfig.schema>;
+const migrationSchema = schema.object({
+  batchSize: schema.number({ defaultValue: 1_000 }),
+  scrollDuration: schema.string({ defaultValue: '15m' }),
+  pollInterval: schema.number({ defaultValue: 1_500 }),
+  skip: schema.boolean({ defaultValue: false }),
+  enableV2: schema.boolean({ defaultValue: true }),
+  retryAttempts: schema.number({ defaultValue: 15 }),
+});
 
-export const savedObjectsMigrationConfig = {
+export type SavedObjectsMigrationConfigType = TypeOf<typeof migrationSchema>;
+
+const migrationDeprecations: ConfigDeprecationProvider = () => [
+  (settings, fromPath, addDeprecation) => {
+    const migrationsConfig = settings[fromPath];
+    if (migrationsConfig?.enableV2 !== undefined) {
+      addDeprecation({
+        message:
+          '"migrations.enableV2" is deprecated and will be removed in an upcoming release without any further notice.',
+        documentationUrl: 'https://ela.st/kbn-so-migration-v2',
+      });
+    }
+    return settings;
+  },
+];
+
+export const savedObjectsMigrationConfig: ServiceConfigDescriptor<SavedObjectsMigrationConfigType> = {
   path: 'migrations',
-  schema: schema.object({
-    batchSize: schema.number({ defaultValue: 1000 }),
-    scrollDuration: schema.string({ defaultValue: '15m' }),
-    pollInterval: schema.number({ defaultValue: 1500 }),
-    skip: schema.boolean({ defaultValue: false }),
-    // TODO migrationsV2: remove/deprecate once we release migrations v2
-    enableV2: schema.boolean({ defaultValue: true }),
-    /** the number of times v2 migrations will retry temporary failures such as a timeout, 503 status code or snapshot_in_progress_exception */
-    retryAttempts: schema.number({ defaultValue: 15 }),
-  }),
+  schema: migrationSchema,
+  deprecations: migrationDeprecations,
 };
 
-export type SavedObjectsConfigType = TypeOf<typeof savedObjectsConfig.schema>;
+const soSchema = schema.object({
+  maxImportPayloadBytes: schema.byteSize({ defaultValue: 26_214_400 }),
+  maxImportExportSize: schema.number({ defaultValue: 10_000 }),
+});
+
+export type SavedObjectsConfigType = TypeOf<typeof soSchema>;
 
-export const savedObjectsConfig = {
+export const savedObjectsConfig: ServiceConfigDescriptor<SavedObjectsConfigType> = {
   path: 'savedObjects',
-  schema: schema.object({
-    maxImportPayloadBytes: schema.byteSize({ defaultValue: 26_214_400 }),
-    maxImportExportSize: schema.number({ defaultValue: 10_000 }),
-  }),
+  schema: soSchema,
 };
 
 export class SavedObjectConfig {

src/core/server/test_utils.ts

@@ -9,3 +9,4 @@
 export { createHttpServer } from './http/test_utils';
 export { ServiceStatusLevelSnapshotSerializer } from './status/test_utils';
 export { setupServer } from './saved_objects/routes/test_utils';
+export { getDeprecationsFor, getDeprecationsForGlobalSettings } from './config/test_utils';

x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx

@@ -80,6 +80,7 @@ export const DataGrid: FC<Props> = memo(
       baseline,
       chartsVisible,
       chartsButtonVisible,
+      ccsWarning,
       columnsWithCharts,
       dataTestSubj,
       errorMessage,
@@ -291,6 +292,24 @@ export const DataGrid: FC<Props> = memo(
             <EuiSpacer size="m" />
           </div>
         )}
+        {ccsWarning && (
+          <div data-test-subj={`${dataTestSubj} ccsWarning`}>
+            <EuiCallOut
+              title={i18n.translate('xpack.ml.dataGrid.CcsWarningCalloutTitle', {
+                defaultMessage: 'Cross-cluster search returned no fields data.',
+              })}
+              color="warning"
+            >
+              <p>
+                {i18n.translate('xpack.ml.dataGrid.CcsWarningCalloutBody', {
+                  defaultMessage:
+                    'There was an issue retrieving data for the index pattern. Source preview in combination with cross-cluster search is only supported for versions 7.10 and above. You may still configure and create the transform.',
+                })}
+              </p>
+            </EuiCallOut>
+            <EuiSpacer size="m" />
+          </div>
+        )}
         <div className="mlDataGrid">
           <EuiDataGrid
             aria-label={isWithHeader(props) ? props.title : ''}

x-pack/plugins/ml/public/application/components/data_grid/types.ts

@@ -59,6 +59,7 @@ export interface UseIndexDataReturnType
     UseDataGridReturnType,
     | 'chartsVisible'
     | 'chartsButtonVisible'
+    | 'ccsWarning'
     | 'columnsWithCharts'
     | 'errorMessage'
     | 'invalidSortingColumnns'
@@ -84,6 +85,7 @@ export interface UseIndexDataReturnType
 }
 
 export interface UseDataGridReturnType {
+  ccsWarning: boolean;
   chartsVisible: ChartsVisible;
   chartsButtonVisible: boolean;
   columnsWithCharts: EuiDataGridColumn[];
@@ -97,6 +99,7 @@ export interface UseDataGridReturnType {
   resetPagination: () => void;
   rowCount: number;
   rowCountRelation: RowCountRelation;
+  setCcsWarning: Dispatch<SetStateAction<boolean>>;
   setColumnCharts: Dispatch<SetStateAction<ChartData[]>>;
   setErrorMessage: Dispatch<SetStateAction<string>>;
   setNoDataMessage: Dispatch<SetStateAction<string>>;

x-pack/plugins/ml/public/application/components/data_grid/use_data_grid.tsx

@@ -36,6 +36,7 @@ export const useDataGrid = (
 ): UseDataGridReturnType => {
   const defaultPagination: IndexPagination = { pageIndex: 0, pageSize: defaultPageSize };
 
+  const [ccsWarning, setCcsWarning] = useState(false);
   const [noDataMessage, setNoDataMessage] = useState('');
   const [errorMessage, setErrorMessage] = useState('');
   const [status, setStatus] = useState(INDEX_STATUS.UNUSED);
@@ -152,6 +153,7 @@ export const useDataGrid = (
   }, [chartsVisible, rowCount, rowCountRelation]);
 
   return {
+    ccsWarning,
     chartsVisible,
     chartsButtonVisible: true,
     columnsWithCharts,
@@ -166,6 +168,7 @@ export const useDataGrid = (
     rowCount,
     rowCountRelation,
     setColumnCharts,
+    setCcsWarning,
     setErrorMessage,
     setNoDataMessage,
     setPagination,