Merge branch 'main' into migrate-add-from-lib

.buildkite/scripts/bootstrap.sh

@@ -19,7 +19,7 @@ if ! yarn kbn bootstrap; then
 fi
 
 if [[ "$DISABLE_BOOTSTRAP_VALIDATION" != "true" ]]; then
-  verify_no_git_changes 'yarn kbn bootstrap'
+  check_for_changed_files 'yarn kbn bootstrap'
 fi
 
 ###

.buildkite/scripts/common/util.sh

@@ -14,17 +14,46 @@ is_pr() {
   false
 }
 
-verify_no_git_changes() {
+check_for_changed_files() {
   RED='\033[0;31m'
   YELLOW='\033[0;33m'
   C_RESET='\033[0m' # Reset color
 
+  SHOULD_AUTO_COMMIT_CHANGES="${2:-}"
   GIT_CHANGES="$(git ls-files --modified -- . ':!:.bazelrc')"
+
   if [ "$GIT_CHANGES" ]; then
-    echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
-    echo -e "$GIT_CHANGES\n"
-    echo -e "\n${YELLOW}TO FIX: Run '$1' locally, commit the changes and push to your branch${C_RESET}\n"
-    exit 1
+    if [[ "$SHOULD_AUTO_COMMIT_CHANGES" == "true" && "${BUILDKITE_PULL_REQUEST:-}" ]]; then
+      NEW_COMMIT_MESSAGE="[CI] Auto-commit changed files from '$1'"
+      PREVIOUS_COMMIT_MESSAGE="$(git log -1 --pretty=%B)"
+
+      if [[ "$NEW_COMMIT_MESSAGE" == "$PREVIOUS_COMMIT_MESSAGE" ]]; then
+        echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
+        echo -e "$GIT_CHANGES\n"
+        echo -e "CI already attempted to commit these changes, but the file(s) seem to have changed again."
+        echo -e "Please review and fix manually."
+        exit 1
+      fi
+
+      echo "'$1' caused changes to the following files:"
+      echo "$GIT_CHANGES"
+      echo ""
+      echo "Auto-committing these changes now. A new build should start soon if successful."
+
+      git config --global user.name kibanamachine
+      git config --global user.email '[email protected]'
+      gh pr checkout "${BUILDKITE_PULL_REQUEST}"
+      git add -u -- . ':!.bazelrc'
+
+      git commit -m "$NEW_COMMIT_MESSAGE"
+      git push
+      exit 1
+    else
+      echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
+      echo -e "$GIT_CHANGES\n"
+      echo -e "\n${YELLOW}TO FIX: Run '$1' locally, commit the changes and push to your branch${C_RESET}\n"
+      exit 1
+    fi
   fi
 }
 

.buildkite/scripts/steps/checks/bazel_packages.sh

@@ -5,4 +5,6 @@ set -euo pipefail
 source .buildkite/scripts/common/util.sh
 
 echo --- Check Bazel Packages Manifest
-node scripts/generate packages_build_manifest --validate
+node scripts/generate packages_build_manifest
+
+check_for_changed_files 'node scripts/generate packages_build_manifest' true

.buildkite/scripts/steps/checks/kbn_pm_dist.sh

@@ -7,4 +7,4 @@ source .buildkite/scripts/common/util.sh
 echo "--- Building kbn-pm distributable"
 yarn kbn run build -i @kbn/pm
 
-verify_no_git_changes 'yarn kbn run build -i @kbn/pm'
+check_for_changed_files 'yarn kbn run build -i @kbn/pm' true

.buildkite/scripts/steps/checks/plugin_list_docs.sh

@@ -7,4 +7,4 @@ source .buildkite/scripts/common/util.sh
 echo "--- Building plugin list docs"
 node scripts/build_plugin_list_docs
 
-verify_no_git_changes 'node scripts/build_plugin_list_docs'
+check_for_changed_files 'node scripts/build_plugin_list_docs' true

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -32,6 +32,10 @@ node scripts/build \
   --skip-docker-ubuntu \
   --skip-docker-contexts
 
+ELASTICSEARCH_MANIFEST_URL="https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(jq -r '.version' package.json)/manifest-latest-verified.json"
+ELASTICSEARCH_SHA=$(curl -s $ELASTICSEARCH_MANIFEST_URL | jq -r '.sha')
+ELASTICSEARCH_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$VERSION-$ELASTICSEARCH_SHA"
+
 CLOUD_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" docker.elastic.co/kibana-ci/kibana-cloud)
 CLOUD_DEPLOYMENT_NAME="kibana-pr-$BUILDKITE_PULL_REQUEST"
 
@@ -40,6 +44,7 @@ JSON_FILE=$(mktemp --suffix ".json")
 if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
   jq '
     .resources.kibana[0].plan.kibana.docker_image = "'$CLOUD_IMAGE'" |
+    .resources.elasticsearch[0].plan.elasticsearch.docker_image = "'$ELASTICSEARCH_CLOUD_IMAGE'" |
     .name = "'$CLOUD_DEPLOYMENT_NAME'" |
     .resources.kibana[0].plan.kibana.version = "'$VERSION'" |
     .resources.elasticsearch[0].plan.elasticsearch.version = "'$VERSION'" |
@@ -70,6 +75,7 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
 else
 ecctl deployment show "$CLOUD_DEPLOYMENT_ID" --generate-update-payload | jq '
   .resources.kibana[0].plan.kibana.docker_image = "'$CLOUD_IMAGE'" |
+  .resources.elasticsearch[0].plan.elasticsearch.docker_image = "'$ELASTICSEARCH_CLOUD_IMAGE'" |
   (.. | select(.version? != null).version) = "'$VERSION'"
   ' > /tmp/deploy.json
   ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json &> "$JSON_FILE"
@@ -87,7 +93,9 @@ cat << EOF | buildkite-agent annotate --style "info" --context cloud
 
   Credentials: \`vault read secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
 
-  Image: $CLOUD_IMAGE
+  Kibana image: \`$CLOUD_IMAGE\`
+
+  Elasticsearch image: \`$ELASTICSEARCH_CLOUD_IMAGE\`
 EOF
 
 buildkite-agent meta-data set pr_comment:deploy_cloud:head "* [Cloud Deployment](${CLOUD_DEPLOYMENT_KIBANA_URL})"

.github/CODEOWNERS

@@ -118,7 +118,11 @@
 /x-pack/test/api_integration/apis/monitoring @elastic/infra-monitoring-ui
 
 # Fleet
+/fleet_packages.json @elastic/fleet
 /x-pack/plugins/fleet/ @elastic/fleet
+/x-pack/test/fleet_api_integration @elastic/fleet
+/x-pack/test/fleet_cypress @elastic/fleet
+/x-pack/test/fleet_functional @elastic/fleet
 
 # APM
 /x-pack/plugins/apm/  @elastic/apm-ui

config/kibana.yml

@@ -62,6 +62,10 @@
 # must be a positive integer.
 #elasticsearch.requestTimeout: 30000
 
+# The maximum number of sockets that can be used for communications with elasticsearch.
+# Defaults to `Infinity`.
+#elasticsearch.maxSockets: 1024
+
 # Specifies whether Kibana should use compression for communications with elasticsearch
 # Defaults to `false`.
 #elasticsearch.compression: false

docs/development/core/server/kibana-plugin-core-server.elasticsearchclientconfig.md

@@ -9,7 +9,7 @@ Configuration options to be used to create a [cluster client](./kibana-plugin-co
 <b>Signature:</b>
 
 ```typescript
-export declare type ElasticsearchClientConfig = Pick<ElasticsearchConfig, 'customHeaders' | 'compression' | 'sniffOnStart' | 'sniffOnConnectionFault' | 'requestHeadersWhitelist' | 'sniffInterval' | 'hosts' | 'username' | 'password' | 'serviceAccountToken'> & {
+export declare type ElasticsearchClientConfig = Pick<ElasticsearchConfig, 'customHeaders' | 'maxSockets' | 'compression' | 'sniffOnStart' | 'sniffOnConnectionFault' | 'requestHeadersWhitelist' | 'sniffInterval' | 'hosts' | 'username' | 'password' | 'serviceAccountToken'> & {
     pingTimeout?: ElasticsearchConfig['pingTimeout'] | ClientOptions['pingTimeout'];
     requestTimeout?: ElasticsearchConfig['requestTimeout'] | ClientOptions['requestTimeout'];
     ssl?: Partial<ElasticsearchConfig['ssl']>;

docs/development/core/server/kibana-plugin-core-server.elasticsearchconfig.maxsockets.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [ElasticsearchConfig](./kibana-plugin-core-server.elasticsearchconfig.md) &gt; [maxSockets](./kibana-plugin-core-server.elasticsearchconfig.maxsockets.md)
+
+## ElasticsearchConfig.maxSockets property
+
+The maximum number of sockets that can be used for communications with elasticsearch.
+
+<b>Signature:</b>
+
+```typescript
+readonly maxSockets: number;
+```

docs/development/core/server/kibana-plugin-core-server.elasticsearchconfig.md

@@ -28,6 +28,7 @@ export declare class ElasticsearchConfig
 |  [healthCheckDelay](./kibana-plugin-core-server.elasticsearchconfig.healthcheckdelay.md) |  | Duration | The interval between health check requests Kibana sends to the Elasticsearch. |
 |  [hosts](./kibana-plugin-core-server.elasticsearchconfig.hosts.md) |  | string\[\] | Hosts that the client will connect to. If sniffing is enabled, this list will be used as seeds to discover the rest of your cluster. |
 |  [ignoreVersionMismatch](./kibana-plugin-core-server.elasticsearchconfig.ignoreversionmismatch.md) |  | boolean | Whether to allow kibana to connect to a non-compatible elasticsearch node. |
+|  [maxSockets](./kibana-plugin-core-server.elasticsearchconfig.maxsockets.md) |  | number | The maximum number of sockets that can be used for communications with elasticsearch. |
 |  [password?](./kibana-plugin-core-server.elasticsearchconfig.password.md) |  | string | <i>(Optional)</i> If Elasticsearch is protected with basic authentication, this setting provides the password that the Kibana server uses to perform its administrative functions. |
 |  [pingTimeout](./kibana-plugin-core-server.elasticsearchconfig.pingtimeout.md) |  | Duration | Timeout after which PING HTTP request will be aborted and retried. |
 |  [requestHeadersWhitelist](./kibana-plugin-core-server.elasticsearchconfig.requestheaderswhitelist.md) |  | string\[\] | List of Kibana client-side headers to send to Elasticsearch when request scoped cluster client is used. If this is an empty array then \*no\* client-side will be sent. |

docs/setup/settings.asciidoc

@@ -90,6 +90,10 @@ enforce even rudimentary CSP rules, though {kib} is still accessible. This
 configuration is effectively ignored when <<csp-strict, `csp.strict`>> is enabled.
 *Default: `true`*
 
+|[[elasticsearch-maxSockets]] `elasticsearch.maxSockets`
+ | The maximum number of sockets that can be used for communications with elasticsearch.
+*Default: `Infinity`*
+
 | `elasticsearch.customHeaders:`
  | Header names and values to send to {es}. Any custom headers cannot be
 overwritten by client-side headers, regardless of the

package.json

@@ -410,7 +410,7 @@
     "usng.js": "^0.4.5",
     "utility-types": "^3.10.0",
     "uuid": "3.3.2",
-    "vega": "^5.21.0",
+    "vega": "^5.22.0",
     "vega-interpreter": "^1.0.4",
     "vega-lite": "^5.2.0",
     "vega-schema-url-parser": "^2.2.0",

packages/kbn-optimizer/src/worker/webpack.config.ts

@@ -236,6 +236,7 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker:
       mainFields: ['browser', 'main'],
       alias: {
         core_app_image_assets: Path.resolve(worker.repoRoot, 'src/core/public/core_app/images'),
+        vega: Path.resolve(worker.repoRoot, 'node_modules/vega/build-es5/vega.js'),
       },
       symlinks: false,
     },

src/core/server/elasticsearch/client/client_config.test.ts

@@ -16,6 +16,7 @@ const createConfig = (
   return {
     customHeaders: {},
     compression: false,
+    maxSockets: Infinity,
     sniffOnStart: false,
     sniffOnConnectionFault: false,
     sniffInterval: false,
@@ -107,6 +108,18 @@ describe('parseClientOptions', () => {
       });
     });
 
+    describe('`maxSockets` option', () => {
+      it('uses the specified config value', () => {
+        const options = parseClientOptions(createConfig({ maxSockets: 1024 }), false);
+        expect(options.agent).toHaveProperty('maxSockets', 1024);
+      });
+
+      it('defaults to `Infinity` if not specified by the config', () => {
+        const options = parseClientOptions(createConfig({}), false);
+        expect(options.agent).toHaveProperty('maxSockets', Infinity);
+      });
+    });
+
     describe('`compression` option', () => {
       it('`compression` is true', () => {
         const options = parseClientOptions(createConfig({ compression: true }), false);

src/core/server/elasticsearch/client/client_config.ts

@@ -22,6 +22,7 @@ import { DEFAULT_HEADERS } from '../default_headers';
 export type ElasticsearchClientConfig = Pick<
   ElasticsearchConfig,
   | 'customHeaders'
+  | 'maxSockets'
   | 'compression'
   | 'sniffOnStart'
   | 'sniffOnConnectionFault'
@@ -61,7 +62,7 @@ export function parseClientOptions(
     // fixes https://github.com/elastic/kibana/issues/101944
     disablePrototypePoisoningProtection: true,
     agent: {
-      maxSockets: Infinity,
+      maxSockets: config.maxSockets,
       keepAlive: config.keepAlive ?? true,
     },
     compression: config.compression,

src/core/server/elasticsearch/client/cluster_client.test.ts

@@ -26,6 +26,7 @@ const createConfig = (
     sniffOnStart: false,
     sniffOnConnectionFault: false,
     sniffInterval: false,
+    maxSockets: Infinity,
     compression: false,
     requestHeadersWhitelist: ['authorization'],
     customHeaders: {},

src/core/server/elasticsearch/elasticsearch_config.test.ts

@@ -36,6 +36,7 @@ test('set correct defaults', () => {
         "http://localhost:9200",
       ],
       "ignoreVersionMismatch": false,
+      "maxSockets": Infinity,
       "password": undefined,
       "pingTimeout": "PT30S",
       "requestHeadersWhitelist": Array [
@@ -80,6 +81,45 @@ test('#hosts accepts both string and array of strings', () => {
   expect(configValue.hosts).toEqual(['http://some.host:1234', 'https://some.another.host']);
 });
 
+describe('#maxSockets', () => {
+  test('accepts positive numeric values', () => {
+    const configValue = new ElasticsearchConfig(config.schema.validate({ maxSockets: 512 }));
+    expect(configValue.maxSockets).toEqual(512);
+  });
+
+  test('throws if it does not contain a numeric value', () => {
+    expect(() => {
+      config.schema.validate({ maxSockets: 'foo' });
+    }).toThrowErrorMatchingInlineSnapshot(
+      `"[maxSockets]: expected value of type [number] but got [string]"`
+    );
+
+    expect(() => {
+      config.schema.validate({ maxSockets: true });
+    }).toThrowErrorMatchingInlineSnapshot(
+      `"[maxSockets]: expected value of type [number] but got [boolean]"`
+    );
+  });
+
+  test('throws if it does not contain a valid numeric value', () => {
+    expect(() => {
+      config.schema.validate({ maxSockets: -1 });
+    }).toThrowErrorMatchingInlineSnapshot(
+      '"[maxSockets]: Value must be equal to or greater than [1]."'
+    );
+
+    expect(() => {
+      config.schema.validate({ maxSockets: 0 });
+    }).toThrowErrorMatchingInlineSnapshot(
+      '"[maxSockets]: Value must be equal to or greater than [1]."'
+    );
+
+    expect(() => {
+      config.schema.validate({ maxSockets: Infinity });
+    }).toThrowErrorMatchingInlineSnapshot('"[maxSockets]: \\"maxSockets\\" cannot be infinity"');
+  });
+});
+
 test('#requestHeadersWhitelist accepts both string and array of strings', () => {
   let configValue = new ElasticsearchConfig(
     config.schema.validate({ requestHeadersWhitelist: 'token' })

src/core/server/elasticsearch/elasticsearch_config.ts

@@ -36,6 +36,7 @@ export const configSchema = schema.object({
   hosts: schema.oneOf([hostURISchema, schema.arrayOf(hostURISchema, { minSize: 1 })], {
     defaultValue: 'http://localhost:9200',
   }),
+  maxSockets: schema.number({ defaultValue: Infinity, min: 1 }),
   compression: schema.boolean({ defaultValue: false }),
   username: schema.maybe(
     schema.string({
@@ -298,6 +299,11 @@ export class ElasticsearchConfig {
    */
   public readonly apiVersion: string;
 
+  /**
+   * The maximum number of sockets that can be used for communications with elasticsearch.
+   */
+  public readonly maxSockets: number;
+
   /**
    * Whether to use compression for communications with elasticsearch.
    */
@@ -405,6 +411,7 @@ export class ElasticsearchConfig {
     this.password = rawConfig.password;
     this.serviceAccountToken = rawConfig.serviceAccountToken;
     this.customHeaders = rawConfig.customHeaders;
+    this.maxSockets = rawConfig.maxSockets;
     this.compression = rawConfig.compression;
     this.skipStartupConnectionCheck = rawConfig.skipStartupConnectionCheck;
 

src/core/server/server.api.md

@@ -231,6 +231,7 @@ export const config: {
         sniffInterval: Type<false | Duration>;
         sniffOnConnectionFault: Type<boolean>;
         hosts: Type<string | string[]>;
+        maxSockets: Type<number>;
         compression: Type<boolean>;
         username: Type<string | undefined>;
         password: Type<string | undefined>;
@@ -889,7 +890,7 @@ export { EcsEventType }
 export type ElasticsearchClient = Omit<Client, 'connectionPool' | 'serializer' | 'extend' | 'close' | 'diagnostic'>;
 
 // @public
-export type ElasticsearchClientConfig = Pick<ElasticsearchConfig, 'customHeaders' | 'compression' | 'sniffOnStart' | 'sniffOnConnectionFault' | 'requestHeadersWhitelist' | 'sniffInterval' | 'hosts' | 'username' | 'password' | 'serviceAccountToken'> & {
+export type ElasticsearchClientConfig = Pick<ElasticsearchConfig, 'customHeaders' | 'maxSockets' | 'compression' | 'sniffOnStart' | 'sniffOnConnectionFault' | 'requestHeadersWhitelist' | 'sniffInterval' | 'hosts' | 'username' | 'password' | 'serviceAccountToken'> & {
     pingTimeout?: ElasticsearchConfig['pingTimeout'] | ClientOptions['pingTimeout'];
     requestTimeout?: ElasticsearchConfig['requestTimeout'] | ClientOptions['requestTimeout'];
     ssl?: Partial<ElasticsearchConfig['ssl']>;
@@ -907,6 +908,7 @@ export class ElasticsearchConfig {
     readonly healthCheckDelay: Duration;
     readonly hosts: string[];
     readonly ignoreVersionMismatch: boolean;
+    readonly maxSockets: number;
     readonly password?: string;
     readonly pingTimeout: Duration;
     readonly requestHeadersWhitelist: string[];

x-pack/plugins/alerting/server/rules_client/lib/mapped_params_utils.test.ts

@@ -128,4 +128,23 @@ describe('modifyFilterKueryNode', () => {
       value: '40-medium',
     });
   });
+
+  it('do NOT modify the resulting kuery node AST filter for alert params when they are not part of mapped params', () => {
+    // Make sure it works for both camel and snake case params
+    const astFilter = fromKueryExpression(
+      'alert.attributes.name: "Rule I" and alert.attributes.tags: "fast" and alert.attributes.params.threat.tactic.name: Exfiltration'
+    );
+
+    modifyFilterKueryNode({ astFilter });
+
+    expect(astFilter.arguments[2].arguments[0]).toEqual({
+      type: 'literal',
+      value: 'alert.attributes.params.threat.tactic.name',
+    });
+
+    expect(astFilter.arguments[2].arguments[1]).toEqual({
+      type: 'literal',
+      value: 'Exfiltration',
+    });
+  });
 });