Merge branch 'master' into telemetry/fix-search-in-advanced-settings

.eslintrc.js

@@ -562,7 +562,7 @@ module.exports = {
      */
     {
       // front end typescript and javascript files only
-      files: ['x-pack/legacy/plugins/siem/public/**/*.{js,ts,tsx}'],
+      files: ['x-pack/plugins/siem/public/**/*.{js,ts,tsx}'],
       rules: {
         'import/no-nodejs-modules': 'error',
         'no-restricted-imports': [
@@ -611,7 +611,7 @@ module.exports = {
     // {
     //   // will introduced after the other warns are fixed
     //   // typescript and javascript for front end react performance
-    //   files: ['x-pack/legacy/plugins/siem/public/**/!(*.test).{js,ts,tsx}'],
+    //   files: ['x-pack/plugins/siem/public/**/!(*.test).{js,ts,tsx}'],
     //   plugins: ['react-perf'],
     //   rules: {
     //     // 'react-perf/jsx-no-new-object-as-prop': 'error',

Jenkinsfile

@@ -4,54 +4,56 @@ library 'kibana-pipeline-library'
 kibanaLibrary.load()
 
 kibanaPipeline(timeoutMinutes: 135, checkPrChanges: true) {
-  githubPr.withDefaultPrComments {
-    catchError {
-      retryable.enable()
-      parallel([
-        'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
-        'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
-        'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
-          'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
-          'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
-          'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
-          'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
-          'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
-          'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
-          'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
-          'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
-          'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
-          'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
-          'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
-          'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
-          'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
-          'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'),
-          // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
-        ]),
-        'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
-          'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
-          'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
-          'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
-          'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
-          'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
-          'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
-          'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
-          'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
-          'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
-          'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
-          'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
-          'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
-          'xpack-siemCypress': { processNumber ->
-            whenChanged(['x-pack/plugins/siem/', 'x-pack/legacy/plugins/siem/', 'x-pack/test/siem_cypress/']) {
-              kibanaPipeline.functionalTestProcess('xpack-siemCypress', './test/scripts/jenkins_siem_cypress.sh')(processNumber)
-            }
-          },
+  ciStats.trackBuild {
+    githubPr.withDefaultPrComments {
+      catchError {
+        retryable.enable()
+        parallel([
+          'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
+          'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
+          'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
+            'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
+            'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
+            'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
+            'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
+            'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
+            'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
+            'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
+            'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
+            'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
+            'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
+            'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
+            'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
+            'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
+            'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'),
+            // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
+          ]),
+          'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
+            'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
+            'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
+            'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
+            'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
+            'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
+            'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
+            'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
+            'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
+            'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
+            'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
+            'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
+            'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
+            'xpack-siemCypress': { processNumber ->
+              whenChanged(['x-pack/plugins/siem/', 'x-pack/test/siem_cypress/']) {
+                kibanaPipeline.functionalTestProcess('xpack-siemCypress', './test/scripts/jenkins_siem_cypress.sh')(processNumber)
+              }
+            },
 
-          // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'),
-        ]),
-      ])
+            // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'),
+          ]),
+        ])
+      }
     }
-  }
 
-  retryable.printFlakyFailures()
-  kibanaPipeline.sendMail()
+    retryable.printFlakyFailures()
+    kibanaPipeline.sendMail()
+  }
 }

docs/developer/core-development.asciidoc

@@ -7,6 +7,7 @@
 * <<development-elasticsearch>>
 * <<development-unit-tests>>
 * <<development-functional-tests>>
+* <<development-es-snapshots>>
 
 include::core/development-basepath.asciidoc[]
 
@@ -19,3 +20,5 @@ include::core/development-elasticsearch.asciidoc[]
 include::core/development-unit-tests.asciidoc[]
 
 include::core/development-functional-tests.asciidoc[]
+
+include::core/development-es-snapshots.asciidoc[]

docs/developer/core/development-es-snapshots.asciidoc

@@ -0,0 +1,113 @@
+[[development-es-snapshots]]
+=== Daily Elasticsearch Snapshots
+
+For local development and CI, Kibana, by default, uses Elasticsearch snapshots that are built daily when running tasks that require Elasticsearch (e.g. functional tests).
+
+A snapshot is just a group of tarballs, one for each supported distribution/architecture/os of Elasticsearch, and a JSON-based manifest file containing metadata about the distributions.
+
+https://ci.kibana.dev/es-snapshots[A dashboard] is available that shows the current status and compatibility of the latest Elasticsearch snapshots.
+
+==== Process Overview
+
+1. Elasticsearch snapshots are built for each current tracked branch of Kibana.
+2. Each snapshot is uploaded to a public Google Cloud Storage bucket, `kibana-ci-es-snapshots-daily`.
+** At this point, the snapshot is not automatically used in CI or local development. It needs to be tested/verified first.
+3. Each snapshot is tested with the latest commit of the corresponding Kibana branch, using the full CI suite.
+4. After CI
+** If the snapshot passes, it is promoted and automatically used in CI and local development.
+** If the snapshot fails, the issue must be investigated and resolved. A new incompatibility may exist between Elasticsearch and Kibana.
+
+==== Using the latest snapshot
+
+When developing locally, you may wish to use the most recent Elasticsearch snapshot, even if it's failing CI. To do so, prefix your commands with the follow environment variable:
+
+["source","bash"]
+-----------
+KBN_ES_SNAPSHOT_USE_UNVERIFIED=true
+-----------
+
+You can use this flag with any command that downloads and runs Elasticsearch snapshots, such as `scripts/es` or the FTR.
+
+For example, to run functional tests with the latest snapshot:
+
+["source","bash"]
+-----------
+KBN_ES_SNAPSHOT_USE_UNVERIFIED=true node scripts/functional_tests_server
+-----------
+
+===== For Pull Requests
+
+Currently, there is not a way to run your pull request with the latest unverified snapshot without a code change. You can, however, do it with a small code change.
+
+1. Edit `Jenkinsfile` in the root of the Kibana repo
+2. Add `env.KBN_ES_SNAPSHOT_USE_UNVERIFIED = 'true'` at the top of the file.
+3. Commit the change
+
+Your pull request should then use the latest snapshot the next time that it runs. Just don't merge the change to `Jenkinsfile`!
+
+==== Google Cloud Storage buckets
+
+===== kibana-ci-es-snapshots-daily
+
+This bucket stores snapshots that are created on a daily basis, and is the primary location used by `kbn-es` to download snapshots.
+
+Snapshots are automatically deleted after 10 days.
+
+The file structure for this bucket looks like this:
+
+* `<version>/manifest-latest.json`
+* `<version>/manifest-latest-verified.json`
+* `<version>/archives/<unique id>/*.tar.gz`
+* `<version>/archives/<unique id>/*.tar.gz.sha512`
+* `<version>/archives/<unique id>/manifest.json`
+
+===== kibana-ci-es-snapshots-permanent
+
+This bucket stores only the most recently promoted snapshot for each version. Old snapshots are only deleted when new ones are uploaded.
+
+This bucket serves as permanent snapshot storage for old branches/versions that are no longer being built. `kbn-es` checks the daily bucket first, followed by this one if no snapshots were found.
+
+The file structure for this bucket looks like this:
+
+* `<version>/*.tar.gz`
+* `<version>/*.tar.gz.sha512`
+* `<version>/manifest.json`
+
+==== How snapshots are built, tested, and promoted
+
+Each day, a https://kibana-ci.elastic.co/job/elasticsearch+snapshots+trigger/[Jenkins job] runs that triggers Elasticsearch builds for each currently tracked branch/version. This job is automatically updated with the correct branches whenever we release new versions of Kibana.
+
+===== Build
+
+https://kibana-ci.elastic.co/job/elasticsearch+snapshots+build/[This Jenkins job] builds the Elasticsearch snapshots and uploads them to GCS.
+
+The Jenkins job pipeline definition is https://github.com/elastic/kibana/blob/master/.ci/es-snapshots/Jenkinsfile_build_es[in the kibana repo].
+
+1. Checkout Elasticsearch repo for the given branch/version.
+2. Run `./gradlew -p distribution/archives assemble --parallel` to create all of the Elasticsearch distributions.
+3. Create a tarball for each distribution.
+4. Create a manifest JSON file containing info about the distribution, as well as its download URL.
+5. Upload the tarballs and manifest to a unique location in the GCS bucket `kibana-ci-es-snapshots-daily`.
+** e.g. `<version>/archives/<unique id>`
+6. Replace `<version>/manifest-latest.json` in GCS with this newest manifest.
+** This allows the `KBN_ES_SNAPSHOT_USE_UNVERIFIED` flag to work.
+7. Trigger the verification job, to run the full Kibana CI test suite with this snapshot.
+
+===== Verification and Promotion
+
+https://kibana-ci.elastic.co/job/elasticsearch+snapshots+verify/[This Jenkins job] tests the latest Elasticsearch snapshot with the full Kibana CI pipeline, and promotes if it there are no test failures.
+
+The Jenkins job pipeline definition is https://github.com/elastic/kibana/blob/master/.ci/es-snapshots/Jenkinsfile_verify_es[in the kibana repo].
+
+1. Checkout Kibana and set up CI environment as normal.
+2. Set the `ES_SNAPSHOT_MANIFEST` env var to point to the latest snapshot manifest.
+3. Run CI (functional tests, integration tests, etc).
+4. After CI
+** If there was a test failure or other build error, send out an e-mail notification and stop.
+** If there were no errors, promote the snapshot.
+
+Promotion is done as part of the same pipeline:
+
+1. Replace the manifest at `kibana-ci-es-snapshots-daily/<version>/manifest-latest-verified.json` with the manifest from the tested snapshot.
+** At this point, the snapshot has been promoted and will automatically be used in CI and in local development.
+2. Replace the snapshot at `kibana-ci-es-snapshots-permanent/<version>/` with the tested snapshot by copying all of the tarballs and the manifest file.

docs/development/core/server/kibana-plugin-core-server.md

@@ -263,13 +263,14 @@ The plugin integrates with the core system via lifecycle events: `setup`<!-- -->
 |  [SavedObjectAttribute](./kibana-plugin-core-server.savedobjectattribute.md) | Type definition for a Saved Object attribute value |
 |  [SavedObjectAttributeSingle](./kibana-plugin-core-server.savedobjectattributesingle.md) | Don't use this type, it's simply a helper type for [SavedObjectAttribute](./kibana-plugin-core-server.savedobjectattribute.md) |
 |  [SavedObjectMigrationFn](./kibana-plugin-core-server.savedobjectmigrationfn.md) | A migration function for a [saved object type](./kibana-plugin-core-server.savedobjectstype.md) used to migrate it to a given version |
-|  [SavedObjectSanitizedDoc](./kibana-plugin-core-server.savedobjectsanitizeddoc.md) |  |
+|  [SavedObjectSanitizedDoc](./kibana-plugin-core-server.savedobjectsanitizeddoc.md) | Describes Saved Object documents that have passed through the migration framework and are guaranteed to have a <code>references</code> root property. |
 |  [SavedObjectsClientContract](./kibana-plugin-core-server.savedobjectsclientcontract.md) | Saved Objects is Kibana's data persisentence mechanism allowing plugins to use Elasticsearch for storing plugin state.<!-- -->\#\# SavedObjectsClient errors<!-- -->Since the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to to application code. Ideally, all errors will be either:<!-- -->1. Caused by bad implementation (ie. undefined is not a function) and as such unpredictable 2. An error that has been classified and decorated appropriately by the decorators in [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md)<!-- -->Type 1 errors are inevitable, but since all expected/handle-able errors should be Type 2 the <code>isXYZError()</code> helpers exposed at <code>SavedObjectsErrorHelpers</code> should be used to understand and manage error responses from the <code>SavedObjectsClient</code>.<!-- -->Type 2 errors are decorated versions of the source error, so if the elasticsearch client threw an error it will be decorated based on its type. That means that rather than looking for <code>error.body.error.type</code> or doing substring checks on <code>error.body.error.reason</code>, just use the helpers to understand the meaning of the error:<!-- -->\`\`\`<!-- -->js if (SavedObjectsErrorHelpers.isNotFoundError(error)) { // handle 404 }<!-- -->if (SavedObjectsErrorHelpers.isNotAuthorizedError(error)) { // 401 handling should be automatic, but in case you wanted to know }<!-- -->// always rethrow the error unless you handle it throw error; \`\`\`<!-- -->\#\#\# 404s from missing index<!-- -->From the perspective of application code and APIs the SavedObjectsClient is a black box that persists objects. One of the internal details that users have no control over is that we use an elasticsearch index for persistance and that index might be missing.<!-- -->At the time of writing we are in the process of transitioning away from the operating assumption that the SavedObjects index is always available. Part of this transition is handling errors resulting from an index missing. These used to trigger a 500 error in most cases, and in others cause 404s with different error messages.<!-- -->From my (Spencer) perspective, a 404 from the SavedObjectsApi is a 404; The object the request/call was targeting could not be found. This is why \#14141 takes special care to ensure that 404 errors are generic and don't distinguish between index missing or document missing.<!-- -->See [SavedObjectsClient](./kibana-plugin-core-server.savedobjectsclient.md) See [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md) |
 |  [SavedObjectsClientFactory](./kibana-plugin-core-server.savedobjectsclientfactory.md) | Describes the factory used to create instances of the Saved Objects Client. |
 |  [SavedObjectsClientFactoryProvider](./kibana-plugin-core-server.savedobjectsclientfactoryprovider.md) | Provider to invoke to retrieve a [SavedObjectsClientFactory](./kibana-plugin-core-server.savedobjectsclientfactory.md)<!-- -->. |
 |  [SavedObjectsClientWrapperFactory](./kibana-plugin-core-server.savedobjectsclientwrapperfactory.md) | Describes the factory used to create instances of Saved Objects Client Wrappers. |
 |  [SavedObjectsFieldMapping](./kibana-plugin-core-server.savedobjectsfieldmapping.md) | Describe a [saved object type mapping](./kibana-plugin-core-server.savedobjectstypemappingdefinition.md) field.<!-- -->Please refer to [elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) For the mapping documentation |
 |  [SavedObjectsNamespaceType](./kibana-plugin-core-server.savedobjectsnamespacetype.md) | The namespace type dictates how a saved object can be interacted in relation to namespaces. Each type is mutually exclusive: \* single (default): this type of saved object is namespace-isolated, e.g., it exists in only one namespace. \* multiple: this type of saved object is shareable, e.g., it can exist in one or more namespaces. \* agnostic: this type of saved object is global.<!-- -->Note: do not write logic that uses this value directly; instead, use the appropriate accessors in the [type registry](./kibana-plugin-core-server.savedobjecttyperegistry.md)<!-- -->. |
+|  [SavedObjectUnsanitizedDoc](./kibana-plugin-core-server.savedobjectunsanitizeddoc.md) | Describes Saved Object documents from Kibana &lt; 7.0.0 which don't have a <code>references</code> root property defined. This type should only be used in migrations. |
 |  [ScopeableRequest](./kibana-plugin-core-server.scopeablerequest.md) | A user credentials container. It accommodates the necessary auth credentials to impersonate the current user.<!-- -->See [KibanaRequest](./kibana-plugin-core-server.kibanarequest.md)<!-- -->. |
 |  [ServiceStatusLevel](./kibana-plugin-core-server.servicestatuslevel.md) | A convenience type that represents the union of each value in [ServiceStatusLevels](./kibana-plugin-core-server.servicestatuslevels.md)<!-- -->. |
 |  [SharedGlobalConfig](./kibana-plugin-core-server.sharedglobalconfig.md) |  |

docs/development/core/server/kibana-plugin-core-server.savedobjectsanitizeddoc.md

@@ -4,6 +4,7 @@
 
 ## SavedObjectSanitizedDoc type
 
+Describes Saved Object documents that have passed through the migration framework and are guaranteed to have a `references` root property.
 
 <b>Signature:</b>