Merge remote-tracking branch 'upstream/main' into migrate-add-from-lib

.buildkite/scripts/bootstrap.sh

@@ -19,7 +19,7 @@ if ! yarn kbn bootstrap; then
 fi
 
 if [[ "$DISABLE_BOOTSTRAP_VALIDATION" != "true" ]]; then
-  verify_no_git_changes 'yarn kbn bootstrap'
+  check_for_changed_files 'yarn kbn bootstrap'
 fi
 
 ###

.buildkite/scripts/common/util.sh

@@ -14,17 +14,46 @@ is_pr() {
   false
 }
 
-verify_no_git_changes() {
+check_for_changed_files() {
   RED='\033[0;31m'
   YELLOW='\033[0;33m'
   C_RESET='\033[0m' # Reset color
 
+  SHOULD_AUTO_COMMIT_CHANGES="${2:-}"
   GIT_CHANGES="$(git ls-files --modified -- . ':!:.bazelrc')"
+
   if [ "$GIT_CHANGES" ]; then
-    echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
-    echo -e "$GIT_CHANGES\n"
-    echo -e "\n${YELLOW}TO FIX: Run '$1' locally, commit the changes and push to your branch${C_RESET}\n"
-    exit 1
+    if [[ "$SHOULD_AUTO_COMMIT_CHANGES" == "true" && "${BUILDKITE_PULL_REQUEST:-}" ]]; then
+      NEW_COMMIT_MESSAGE="[CI] Auto-commit changed files from '$1'"
+      PREVIOUS_COMMIT_MESSAGE="$(git log -1 --pretty=%B)"
+
+      if [[ "$NEW_COMMIT_MESSAGE" == "$PREVIOUS_COMMIT_MESSAGE" ]]; then
+        echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
+        echo -e "$GIT_CHANGES\n"
+        echo -e "CI already attempted to commit these changes, but the file(s) seem to have changed again."
+        echo -e "Please review and fix manually."
+        exit 1
+      fi
+
+      echo "'$1' caused changes to the following files:"
+      echo "$GIT_CHANGES"
+      echo ""
+      echo "Auto-committing these changes now. A new build should start soon if successful."
+
+      git config --global user.name kibanamachine
+      git config --global user.email '[email protected]'
+      gh pr checkout "${BUILDKITE_PULL_REQUEST}"
+      git add -u -- . ':!.bazelrc'
+
+      git commit -m "$NEW_COMMIT_MESSAGE"
+      git push
+      exit 1
+    else
+      echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
+      echo -e "$GIT_CHANGES\n"
+      echo -e "\n${YELLOW}TO FIX: Run '$1' locally, commit the changes and push to your branch${C_RESET}\n"
+      exit 1
+    fi
   fi
 }
 

.buildkite/scripts/steps/checks/bazel_packages.sh

@@ -5,4 +5,6 @@ set -euo pipefail
 source .buildkite/scripts/common/util.sh
 
 echo --- Check Bazel Packages Manifest
-node scripts/generate packages_build_manifest --validate
+node scripts/generate packages_build_manifest
+
+check_for_changed_files 'node scripts/generate packages_build_manifest' true

.buildkite/scripts/steps/checks/kbn_pm_dist.sh

@@ -7,4 +7,4 @@ source .buildkite/scripts/common/util.sh
 echo "--- Building kbn-pm distributable"
 yarn kbn run build -i @kbn/pm
 
-verify_no_git_changes 'yarn kbn run build -i @kbn/pm'
+check_for_changed_files 'yarn kbn run build -i @kbn/pm' true

.buildkite/scripts/steps/checks/plugin_list_docs.sh

@@ -7,4 +7,4 @@ source .buildkite/scripts/common/util.sh
 echo "--- Building plugin list docs"
 node scripts/build_plugin_list_docs
 
-verify_no_git_changes 'node scripts/build_plugin_list_docs'
+check_for_changed_files 'node scripts/build_plugin_list_docs' true

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -32,6 +32,10 @@ node scripts/build \
   --skip-docker-ubuntu \
   --skip-docker-contexts
 
+ELASTICSEARCH_MANIFEST_URL="https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(jq -r '.version' package.json)/manifest-latest-verified.json"
+ELASTICSEARCH_SHA=$(curl -s $ELASTICSEARCH_MANIFEST_URL | jq -r '.sha')
+ELASTICSEARCH_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$VERSION-$ELASTICSEARCH_SHA"
+
 CLOUD_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" docker.elastic.co/kibana-ci/kibana-cloud)
 CLOUD_DEPLOYMENT_NAME="kibana-pr-$BUILDKITE_PULL_REQUEST"
 
@@ -40,6 +44,7 @@ JSON_FILE=$(mktemp --suffix ".json")
 if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
   jq '
     .resources.kibana[0].plan.kibana.docker_image = "'$CLOUD_IMAGE'" |
+    .resources.elasticsearch[0].plan.elasticsearch.docker_image = "'$ELASTICSEARCH_CLOUD_IMAGE'" |
     .name = "'$CLOUD_DEPLOYMENT_NAME'" |
     .resources.kibana[0].plan.kibana.version = "'$VERSION'" |
     .resources.elasticsearch[0].plan.elasticsearch.version = "'$VERSION'" |
@@ -70,6 +75,7 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
 else
 ecctl deployment show "$CLOUD_DEPLOYMENT_ID" --generate-update-payload | jq '
   .resources.kibana[0].plan.kibana.docker_image = "'$CLOUD_IMAGE'" |
+  .resources.elasticsearch[0].plan.elasticsearch.docker_image = "'$ELASTICSEARCH_CLOUD_IMAGE'" |
   (.. | select(.version? != null).version) = "'$VERSION'"
   ' > /tmp/deploy.json
   ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json &> "$JSON_FILE"
@@ -87,7 +93,9 @@ cat << EOF | buildkite-agent annotate --style "info" --context cloud
 
   Credentials: \`vault read secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
 
-  Image: $CLOUD_IMAGE
+  Kibana image: \`$CLOUD_IMAGE\`
+
+  Elasticsearch image: \`$ELASTICSEARCH_CLOUD_IMAGE\`
 EOF
 
 buildkite-agent meta-data set pr_comment:deploy_cloud:head "* [Cloud Deployment](${CLOUD_DEPLOYMENT_KIBANA_URL})"

.github/CODEOWNERS

@@ -118,7 +118,11 @@
 /x-pack/test/api_integration/apis/monitoring @elastic/infra-monitoring-ui
 
 # Fleet
+/fleet_packages.json @elastic/fleet
 /x-pack/plugins/fleet/ @elastic/fleet
+/x-pack/test/fleet_api_integration @elastic/fleet
+/x-pack/test/fleet_cypress @elastic/fleet
+/x-pack/test/fleet_functional @elastic/fleet
 
 # APM
 /x-pack/plugins/apm/  @elastic/apm-ui

.i18nrc.json

@@ -60,6 +60,7 @@
     "server": "src/legacy/server",
     "share": "src/plugins/share",
     "sharedUX": "src/plugins/shared_ux",
+    "sharedUXComponents": "packages/kbn-shared-ux-components/src",
     "statusPage": "src/legacy/core_plugins/status_page",
     "telemetry": ["src/plugins/telemetry", "src/plugins/telemetry_management_section"],
     "timelion": ["src/plugins/vis_types/timelion"],

config/kibana.yml

@@ -62,6 +62,10 @@
 # must be a positive integer.
 #elasticsearch.requestTimeout: 30000
 
+# The maximum number of sockets that can be used for communications with elasticsearch.
+# Defaults to `Infinity`.
+#elasticsearch.maxSockets: 1024
+
 # Specifies whether Kibana should use compression for communications with elasticsearch
 # Defaults to `false`.
 #elasticsearch.compression: false

docs/development/core/server/kibana-plugin-core-server.elasticsearchclientconfig.md

@@ -9,7 +9,7 @@ Configuration options to be used to create a [cluster client](./kibana-plugin-co
 <b>Signature:</b>
 
 ```typescript
-export declare type ElasticsearchClientConfig = Pick<ElasticsearchConfig, 'customHeaders' | 'compression' | 'sniffOnStart' | 'sniffOnConnectionFault' | 'requestHeadersWhitelist' | 'sniffInterval' | 'hosts' | 'username' | 'password' | 'serviceAccountToken'> & {
+export declare type ElasticsearchClientConfig = Pick<ElasticsearchConfig, 'customHeaders' | 'maxSockets' | 'compression' | 'sniffOnStart' | 'sniffOnConnectionFault' | 'requestHeadersWhitelist' | 'sniffInterval' | 'hosts' | 'username' | 'password' | 'serviceAccountToken'> & {
     pingTimeout?: ElasticsearchConfig['pingTimeout'] | ClientOptions['pingTimeout'];
     requestTimeout?: ElasticsearchConfig['requestTimeout'] | ClientOptions['requestTimeout'];
     ssl?: Partial<ElasticsearchConfig['ssl']>;

docs/development/core/server/kibana-plugin-core-server.elasticsearchconfig.maxsockets.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [ElasticsearchConfig](./kibana-plugin-core-server.elasticsearchconfig.md) &gt; [maxSockets](./kibana-plugin-core-server.elasticsearchconfig.maxsockets.md)
+
+## ElasticsearchConfig.maxSockets property
+
+The maximum number of sockets that can be used for communications with elasticsearch.
+
+<b>Signature:</b>
+
+```typescript
+readonly maxSockets: number;
+```

docs/development/core/server/kibana-plugin-core-server.elasticsearchconfig.md

@@ -28,6 +28,7 @@ export declare class ElasticsearchConfig
 |  [healthCheckDelay](./kibana-plugin-core-server.elasticsearchconfig.healthcheckdelay.md) |  | Duration | The interval between health check requests Kibana sends to the Elasticsearch. |
 |  [hosts](./kibana-plugin-core-server.elasticsearchconfig.hosts.md) |  | string\[\] | Hosts that the client will connect to. If sniffing is enabled, this list will be used as seeds to discover the rest of your cluster. |
 |  [ignoreVersionMismatch](./kibana-plugin-core-server.elasticsearchconfig.ignoreversionmismatch.md) |  | boolean | Whether to allow kibana to connect to a non-compatible elasticsearch node. |
+|  [maxSockets](./kibana-plugin-core-server.elasticsearchconfig.maxsockets.md) |  | number | The maximum number of sockets that can be used for communications with elasticsearch. |
 |  [password?](./kibana-plugin-core-server.elasticsearchconfig.password.md) |  | string | <i>(Optional)</i> If Elasticsearch is protected with basic authentication, this setting provides the password that the Kibana server uses to perform its administrative functions. |
 |  [pingTimeout](./kibana-plugin-core-server.elasticsearchconfig.pingtimeout.md) |  | Duration | Timeout after which PING HTTP request will be aborted and retried. |
 |  [requestHeadersWhitelist](./kibana-plugin-core-server.elasticsearchconfig.requestheaderswhitelist.md) |  | string\[\] | List of Kibana client-side headers to send to Elasticsearch when request scoped cluster client is used. If this is an empty array then \*no\* client-side will be sent. |

docs/maps/import-geospatial-data.asciidoc

@@ -6,9 +6,6 @@ To import geospatical data into the Elastic Stack, the data must be indexed as {
 Geospatial data comes in many formats.
 Choose an import tool based on the format of your geospatial data.
 
-TIP: When you upload files in {kib}, there is a file size
-limit, which is configurable in <<fileupload-maxfilesize,Advanced Settings>>.
-
 [discrete]
 [[import-geospatial-privileges]]
 === Security privileges

docs/setup/settings.asciidoc

@@ -90,6 +90,10 @@ enforce even rudimentary CSP rules, though {kib} is still accessible. This
 configuration is effectively ignored when <<csp-strict, `csp.strict`>> is enabled.
 *Default: `true`*
 
+|[[elasticsearch-maxSockets]] `elasticsearch.maxSockets`
+ | The maximum number of sockets that can be used for communications with elasticsearch.
+*Default: `Infinity`*
+
 | `elasticsearch.customHeaders:`
  | Header names and values to send to {es}. Any custom headers cannot be
 overwritten by client-side headers, regardless of the

package.json

@@ -167,6 +167,10 @@
     "@kbn/securitysolution-utils": "link:bazel-bin/packages/kbn-securitysolution-utils",
     "@kbn/server-http-tools": "link:bazel-bin/packages/kbn-server-http-tools",
     "@kbn/server-route-repository": "link:bazel-bin/packages/kbn-server-route-repository",
+    "@kbn/shared-ux-components": "link:bazel-bin/packages/kbn-shared-ux-components",
+    "@kbn/shared-ux-services": "link:bazel-bin/packages/kbn-shared-ux-services",
+    "@kbn/shared-ux-storybook": "link:bazel-bin/packages/kbn-shared-ux-storybook",
+    "@kbn/shared-ux-utility": "link:bazel-bin/packages/kbn-shared-ux-utility",
     "@kbn/std": "link:bazel-bin/packages/kbn-std",
     "@kbn/timelion-grammar": "link:bazel-bin/packages/kbn-timelion-grammar",
     "@kbn/tinymath": "link:bazel-bin/packages/kbn-tinymath",
@@ -196,6 +200,10 @@
     "@turf/helpers": "6.0.1",
     "@turf/length": "^6.0.2",
     "@types/jsonwebtoken": "^8.5.6",
+    "@types/kbn__shared-ux-components": "link:bazel-bin/packages/kbn-shared-ux-components/npm_module_types",
+    "@types/kbn__shared-ux-services": "link:bazel-bin/packages/kbn-shared-ux-services/npm_module_types",
+    "@types/kbn__shared-ux-storybook": "link:bazel-bin/packages/kbn-shared-ux-storybook/npm_module_types",
+    "@types/kbn__shared-ux-utility": "link:bazel-bin/packages/kbn-shared-ux-utility/npm_module_types",
     "@types/moment-duration-format": "^2.2.3",
     "JSONStream": "1.3.5",
     "abort-controller": "^3.0.0",
@@ -410,7 +418,7 @@
     "usng.js": "^0.4.5",
     "utility-types": "^3.10.0",
     "uuid": "3.3.2",
-    "vega": "^5.21.0",
+    "vega": "^5.22.0",
     "vega-interpreter": "^1.0.4",
     "vega-lite": "^5.2.0",
     "vega-schema-url-parser": "^2.2.0",

packages/BUILD.bazel

@@ -66,6 +66,10 @@ filegroup(
     "//packages/kbn-securitysolution-utils:build",
     "//packages/kbn-server-http-tools:build",
     "//packages/kbn-server-route-repository:build",
+    "//packages/kbn-shared-ux-components:build",
+    "//packages/kbn-shared-ux-services:build",
+    "//packages/kbn-shared-ux-storybook:build",
+    "//packages/kbn-shared-ux-utility:build",
     "//packages/kbn-spec-to-console:build",
     "//packages/kbn-std:build",
     "//packages/kbn-storybook:build",
@@ -139,6 +143,10 @@ filegroup(
     "//packages/kbn-securitysolution-utils:build_types",
     "//packages/kbn-server-http-tools:build_types",
     "//packages/kbn-server-route-repository:build_types",
+    "//packages/kbn-shared-ux-components:build_types",
+    "//packages/kbn-shared-ux-services:build_types",
+    "//packages/kbn-shared-ux-storybook:build_types",
+    "//packages/kbn-shared-ux-utility:build_types",
     "//packages/kbn-std:build_types",
     "//packages/kbn-storybook:build_types",
     "//packages/kbn-telemetry-tools:build_types",

packages/kbn-optimizer/src/worker/webpack.config.ts

@@ -236,6 +236,7 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker:
       mainFields: ['browser', 'main'],
       alias: {
         core_app_image_assets: Path.resolve(worker.repoRoot, 'src/core/public/core_app/images'),
+        vega: Path.resolve(worker.repoRoot, 'node_modules/vega/build-es5/vega.js'),
       },
       symlinks: false,
     },