Merge branch 'master' into fix-blowup

.github/CODEOWNERS

@@ -87,6 +87,7 @@
 /src/dev/ @elastic/kibana-operations
 /src/setup_node_env/ @elastic/kibana-operations
 /src/optimize/ @elastic/kibana-operations
+/src/es_archiver/ @elastic/kibana-operations
 /packages/*eslint*/ @elastic/kibana-operations
 /packages/*babel*/ @elastic/kibana-operations
 /packages/kbn-dev-utils*/ @elastic/kibana-operations
@@ -112,6 +113,7 @@
 /src/legacy/server/logging/ @elastic/kibana-platform
 /src/legacy/server/saved_objects/  @elastic/kibana-platform
 /src/legacy/server/status/ @elastic/kibana-platform
+/src/dev/run_check_core_api_changes.ts @elastic/kibana-platform
 
 # Security
 /src/core/server/csp/  @elastic/kibana-security @elastic/kibana-platform

docs/migration/migrate_8_0.asciidoc

@@ -80,4 +80,15 @@ specified explicitly.
 
 *Impact:* Any workflow that involved manually clearing generated bundles will have to be updated with the new path.
 
+
+[float]
+[[breaking_80_reporting_changes]]
+=== Reporting changes
+
+[float]
+==== Legacy job parameters are no longer supported
+*Details:* POST URL snippets that were copied in Kibana 6.2 or below are no longer supported. These logs have
+been deprecated with warnings that have been logged throughout 7.x. Please use Kibana UI to re-generate the
+POST URL snippets if you depend on these for automated PDF reports.
+
 // end::notable-breaking-changes[]

src/core/server/config/deprecation/core_deprecations.ts

@@ -119,56 +119,6 @@ export const coreDeprecationProvider: ConfigDeprecationProvider = ({
   renameFromRoot('xpack.telemetry.config', 'telemetry.config'),
   renameFromRoot('xpack.telemetry.banner', 'telemetry.banner'),
   renameFromRoot('xpack.telemetry.url', 'telemetry.url'),
-  // Monitoring renames
-  // TODO: Remove these from here once the monitoring plugin is migrated to NP
-  renameFromRoot('xpack.monitoring.enabled', 'monitoring.enabled'),
-  renameFromRoot('xpack.monitoring.ui.enabled', 'monitoring.ui.enabled'),
-  renameFromRoot(
-    'xpack.monitoring.kibana.collection.enabled',
-    'monitoring.kibana.collection.enabled'
-  ),
-  renameFromRoot('xpack.monitoring.max_bucket_size', 'monitoring.ui.max_bucket_size'),
-  renameFromRoot('xpack.monitoring.min_interval_seconds', 'monitoring.ui.min_interval_seconds'),
-  renameFromRoot(
-    'xpack.monitoring.show_license_expiration',
-    'monitoring.ui.show_license_expiration'
-  ),
-  renameFromRoot(
-    'xpack.monitoring.ui.container.elasticsearch.enabled',
-    'monitoring.ui.container.elasticsearch.enabled'
-  ),
-  renameFromRoot(
-    'xpack.monitoring.ui.container.logstash.enabled',
-    'monitoring.ui.container.logstash.enabled'
-  ),
-  renameFromRoot(
-    'xpack.monitoring.tests.cloud_detector.enabled',
-    'monitoring.tests.cloud_detector.enabled'
-  ),
-  renameFromRoot(
-    'xpack.monitoring.kibana.collection.interval',
-    'monitoring.kibana.collection.interval'
-  ),
-  renameFromRoot('xpack.monitoring.elasticsearch.hosts', 'monitoring.ui.elasticsearch.hosts'),
-  renameFromRoot('xpack.monitoring.elasticsearch.username', 'monitoring.ui.elasticsearch.username'),
-  renameFromRoot('xpack.monitoring.elasticsearch.password', 'monitoring.ui.elasticsearch.password'),
-  renameFromRoot(
-    'xpack.monitoring.xpack_api_polling_frequency_millis',
-    'monitoring.xpack_api_polling_frequency_millis'
-  ),
-  renameFromRoot(
-    'xpack.monitoring.cluster_alerts.email_notifications.enabled',
-    'monitoring.cluster_alerts.email_notifications.enabled'
-  ),
-  renameFromRoot(
-    'xpack.monitoring.cluster_alerts.email_notifications.email_address',
-    'monitoring.cluster_alerts.email_notifications.email_address'
-  ),
-  renameFromRoot('xpack.monitoring.ccs.enabled', 'monitoring.ui.ccs.enabled'),
-  renameFromRoot(
-    'xpack.monitoring.elasticsearch.logFetchCount',
-    'monitoring.ui.elasticsearch.logFetchCount'
-  ),
   configPathDeprecation,
   dataPathDeprecation,
   rewriteBasePathDeprecation,

src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts

@@ -108,8 +108,6 @@ describe('getSortedObjectsForExport()', () => {
               "namespace": undefined,
               "perPage": 500,
               "search": undefined,
-              "sortField": "_id",
-              "sortOrder": "asc",
               "type": Array [
                 "index-pattern",
                 "search",
@@ -256,8 +254,6 @@ describe('getSortedObjectsForExport()', () => {
               "namespace": undefined,
               "perPage": 500,
               "search": "foo",
-              "sortField": "_id",
-              "sortOrder": "asc",
               "type": Array [
                 "index-pattern",
                 "search",
@@ -345,8 +341,6 @@ describe('getSortedObjectsForExport()', () => {
               "namespace": "foo",
               "perPage": 500,
               "search": undefined,
-              "sortField": "_id",
-              "sortOrder": "asc",
               "type": Array [
                 "index-pattern",
                 "search",
@@ -399,6 +393,79 @@ describe('getSortedObjectsForExport()', () => {
     ).rejects.toThrowErrorMatchingInlineSnapshot(`"Can't export more than 1 objects"`);
   });
 
+  test('sorts objects within type', async () => {
+    savedObjectsClient.find.mockResolvedValueOnce({
+      total: 3,
+      per_page: 10000,
+      page: 1,
+      saved_objects: [
+        {
+          id: '3',
+          type: 'index-pattern',
+          attributes: {
+            name: 'baz',
+          },
+          references: [],
+        },
+        {
+          id: '1',
+          type: 'index-pattern',
+          attributes: {
+            name: 'foo',
+          },
+          references: [],
+        },
+        {
+          id: '2',
+          type: 'index-pattern',
+          attributes: {
+            name: 'bar',
+          },
+          references: [],
+        },
+      ],
+    });
+    const exportStream = await getSortedObjectsForExport({
+      exportSizeLimit: 10000,
+      savedObjectsClient,
+      types: ['index-pattern'],
+    });
+    const response = await readStreamToCompletion(exportStream);
+    expect(response).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "attributes": Object {
+            "name": "foo",
+          },
+          "id": "1",
+          "references": Array [],
+          "type": "index-pattern",
+        },
+        Object {
+          "attributes": Object {
+            "name": "bar",
+          },
+          "id": "2",
+          "references": Array [],
+          "type": "index-pattern",
+        },
+        Object {
+          "attributes": Object {
+            "name": "baz",
+          },
+          "id": "3",
+          "references": Array [],
+          "type": "index-pattern",
+        },
+        Object {
+          "exportedCount": 3,
+          "missingRefCount": 0,
+          "missingReferences": Array [],
+        },
+      ]
+    `);
+  });
+
   test('exports selected objects and sorts them', async () => {
     savedObjectsClient.bulkGet.mockResolvedValueOnce({
       saved_objects: [

src/core/server/saved_objects/export/get_sorted_objects_for_export.ts

@@ -19,7 +19,7 @@
 
 import Boom from 'boom';
 import { createListStream } from '../../../../legacy/utils/streams';
-import { SavedObjectsClientContract } from '../types';
+import { SavedObjectsClientContract, SavedObject } from '../types';
 import { fetchNestedDependencies } from './inject_nested_depdendencies';
 import { sortObjects } from './sort_objects';
 
@@ -105,15 +105,17 @@ async function fetchObjectsToExport({
     const findResponse = await savedObjectsClient.find({
       type: types,
       search,
-      sortField: '_id',
-      sortOrder: 'asc',
       perPage: exportSizeLimit,
       namespace,
     });
     if (findResponse.total > exportSizeLimit) {
       throw Boom.badRequest(`Can't export more than ${exportSizeLimit} objects`);
     }
-    return findResponse.saved_objects;
+
+    // sorts server-side by _id, since it's only available in fielddata
+    return findResponse.saved_objects.sort((a: SavedObject, b: SavedObject) =>
+      a.id > b.id ? 1 : -1
+    );
   } else {
     throw Boom.badRequest('Either `type` or `objects` are required.');
   }
@@ -137,14 +139,17 @@ export async function getSortedObjectsForExport({
     exportSizeLimit,
     namespace,
   });
-  let exportedObjects = [...rootObjects];
+  let exportedObjects = [];
   let missingReferences: SavedObjectsExportResultDetails['missingReferences'] = [];
+
   if (includeReferencesDeep) {
     const fetchResult = await fetchNestedDependencies(rootObjects, savedObjectsClient, namespace);
-    exportedObjects = fetchResult.objects;
+    exportedObjects = sortObjects(fetchResult.objects);
     missingReferences = fetchResult.missingRefs;
+  } else {
+    exportedObjects = sortObjects(rootObjects);
   }
-  exportedObjects = sortObjects(exportedObjects);
+
   const exportDetails: SavedObjectsExportResultDetails = {
     exportedCount: exportedObjects.length,
     missingRefCount: missingReferences.length,

src/dev/run_check_lockfile_symlinks.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { existsSync, lstatSync, readFileSync } from 'fs';
+import { existsSync, lstatSync, readFileSync, readlinkSync } from 'fs';
 import globby from 'globby';
 import { dirname } from 'path';
 
@@ -63,6 +63,7 @@ async function checkLockfileSymlinks(log, files) {
   await checkOnlyLockfileAtProjectRoot(filtered);
   await checkSuperfluousSymlinks(log, filtered);
   await checkMissingSymlinks(log, filtered);
+  await checkIncorrectSymlinks(log, filtered);
 }
 
 async function checkOnlyLockfileAtProjectRoot(files) {
@@ -157,8 +158,9 @@ async function checkMissingSymlinks(log, files) {
         try {
           const json = JSON.parse(manifest);
           if (json.dependencies && Object.keys(json.dependencies).length) {
+            const correctSymlink = getCorrectSymlink(lockfilePath);
             log.warning(
-              `Manifest at '${path}' has dependencies, but did not find an adjacent 'yarn.lock' symlink.`
+              `Manifest at '${path}' has dependencies, but did not find an adjacent 'yarn.lock' symlink to '${correctSymlink}'.`
             );
             errorPaths.push(`${parent}/yarn.lock`);
           }
@@ -177,6 +179,42 @@ async function checkMissingSymlinks(log, files) {
   }
 }
 
+async function checkIncorrectSymlinks(log, files) {
+  const errorPaths = [];
+
+  files
+    .filter(file => matchesAnyGlob(file.getRelativePath(), LOCKFILE_GLOBS))
+    .forEach(file => {
+      const path = file.getRelativePath();
+      const stats = lstatSync(path);
+      if (!stats.isSymbolicLink()) {
+        return;
+      }
+
+      const symlink = readlinkSync(path);
+      const correctSymlink = getCorrectSymlink(path);
+      if (symlink !== correctSymlink) {
+        log.warning(
+          `Symlink at '${path}' points to '${symlink}', but it should point to '${correctSymlink}'.`
+        );
+        errorPaths.push(path);
+      }
+    });
+
+  if (errorPaths.length) {
+    throw createFailError(
+      `These symlinks do NOT point to the 'yarn.lock' file in the project root:\n${listPaths(
+        errorPaths
+      )}`
+    );
+  }
+}
+
+function getCorrectSymlink(path) {
+  const count = path.split('/').length - 1;
+  return `${'../'.repeat(count)}yarn.lock`;
+}
+
 function listPaths(paths) {
   return paths.map(path => ` - ${path}`).join('\n');
 }

test/functional/apps/context/_date_nanos_custom_timestamp.js

@@ -28,8 +28,9 @@ export default function({ getService, getPageObjects }) {
   const docTable = getService('docTable');
   const PageObjects = getPageObjects(['common', 'context', 'timePicker', 'discover']);
   const esArchiver = getService('esArchiver');
-
-  describe('context view for date_nanos with custom timestamp', () => {
+  // skipped due to a recent change in ES that caused search_after queries with data containing
+  // custom timestamp formats like in the testdata to fail
+  describe.skip('context view for date_nanos with custom timestamp', () => {
     before(async function() {
       await esArchiver.loadIfNeeded('date_nanos_custom');
       await kibanaServer.uiSettings.replace({ defaultIndex: TEST_INDEX_PATTERN });

x-pack/legacy/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration/exploration.tsx

@@ -453,7 +453,7 @@ export const Exploration: FC<Props> = React.memo(({ jobId, jobStatus }) => {
   const MlInMemoryTableBasic = mlInMemoryTableBasicFactory<TableItem>();
 
   return (
-    <EuiPanel grow={false}>
+    <EuiPanel grow={false} data-test-subj="mlDFAnalyticsOutlierExplorationTablePanel">
       <EuiFlexGroup alignItems="center" justifyContent="spaceBetween" responsive={false}>
         <EuiFlexItem grow={false}>
           <EuiFlexGroup gutterSize="s">

x-pack/legacy/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/evaluate_panel.tsx

@@ -229,7 +229,7 @@ export const EvaluatePanel: FC<Props> = ({ jobConfig, jobStatus, searchQuery })
   }, [JSON.stringify(searchQuery)]);
 
   return (
-    <EuiPanel>
+    <EuiPanel data-test-subj="mlDFAnalyticsRegressionExplorationEvaluatePanel">
       <EuiFlexGroup alignItems="center" justifyContent="spaceBetween">
         <EuiFlexItem grow={false}>
           <EuiTitle size="xs">
@@ -296,13 +296,15 @@ export const EvaluatePanel: FC<Props> = ({ jobConfig, jobStatus, searchQuery })
               <Fragment>
                 <EuiFlexItem>
                   <EvaluateStat
+                    dataTestSubj={'mlDFAnalyticsRegressionGenMSEstat'}
                     isLoading={isLoadingGeneralization}
                     title={generalizationEval.meanSquaredError}
                     isMSE
                   />
                 </EuiFlexItem>
                 <EuiFlexItem>
                   <EvaluateStat
+                    dataTestSubj={'mlDFAnalyticsRegressionGenRSquaredStat'}
                     isLoading={isLoadingGeneralization}
                     title={generalizationEval.rSquared}
                     isMSE={false}
@@ -339,13 +341,15 @@ export const EvaluatePanel: FC<Props> = ({ jobConfig, jobStatus, searchQuery })
               <Fragment>
                 <EuiFlexItem>
                   <EvaluateStat
+                    dataTestSubj={'mlDFAnalyticsRegressionTrainingMSEstat'}
                     isLoading={isLoadingTraining}
                     title={trainingEval.meanSquaredError}
                     isMSE
                   />
                 </EuiFlexItem>
                 <EuiFlexItem>
                   <EvaluateStat
+                    dataTestSubj={'mlDFAnalyticsRegressionTrainingRSquaredStat'}
                     isLoading={isLoadingTraining}
                     title={trainingEval.rSquared}
                     isMSE={false}

x-pack/legacy/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/evaluate_stat.tsx

@@ -12,6 +12,7 @@ interface Props {
   isLoading: boolean;
   title: number | string;
   isMSE: boolean;
+  dataTestSubj: string;
 }
 
 const meanSquaredErrorText = i18n.translate(
@@ -41,8 +42,8 @@ const rSquaredTooltipContent = i18n.translate(
   }
 );
 
-export const EvaluateStat: FC<Props> = ({ isLoading, isMSE, title }) => (
-  <EuiFlexGroup gutterSize="xs">
+export const EvaluateStat: FC<Props> = ({ isLoading, isMSE, title, dataTestSubj }) => (
+  <EuiFlexGroup gutterSize="xs" data-test-subj={dataTestSubj}>
     <EuiFlexItem grow={false}>
       <EuiStat
         reverse