Add support for 'size' in EQL Sample queries

.ci/bwcVersions

@@ -55,6 +55,7 @@ BWC_VERSION:
   - "7.17.4"
   - "7.17.5"
   - "7.17.6"
+  - "7.17.7"
   - "8.0.0"
   - "8.0.1"
   - "8.1.0"
@@ -69,6 +70,6 @@ BWC_VERSION:
   - "8.3.1"
   - "8.3.2"
   - "8.3.3"
-  - "8.3.4"
   - "8.4.0"
+  - "8.4.1"
   - "8.5.0"

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml

@@ -14,7 +14,8 @@
           name: os
           values:
             - "centos-7&&immutable"
-            - "amazon&&immutable"
+            - "amazon-2&&immutable"
+            - "amazon-2022&&immutable"
             - "debian-10&&immutable"
             - "debian-11&&immutable"
             - "opensuse-15-1&&immutable"

.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml

@@ -1,7 +1,6 @@
 ---
 - job:
     name: elastic+elasticsearch+%BRANCH%+periodic+ear
-    workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+ear
     display-name: "elastic / elasticsearch # %BRANCH% - encryption at rest"
     description: "The Elasticsearch %BRANCH% branch encryption at rest compatibility tests.\n\n"
     node: packaging-large

.ci/jobs.t/elastic+elasticsearch+periodic+single-processor-node-tests.yml

@@ -0,0 +1,16 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+periodic+single-processor-node-tests
+    display-name: "elastic / elasticsearch # %BRANCH% - single processor node tests"
+    description: "Testing with node.processors set to '1' for the Elasticsearch %BRANCH% branch.\n"
+    node: "general-purpose && docker"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA11_HOME=$HOME/.java/java11
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true check

.ci/jobs.t/elastic+elasticsearch+periodic+single-processor-tests-trigger.yml

@@ -0,0 +1,6 @@
+---
+jjbb-template: periodic-trigger-lgc.yml
+vars:
+  - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+single-processor-node-tests
+  - lgc-job: elastic+elasticsearch+%BRANCH%+intake
+  - cron: "H H/12 * * *"

.ci/scripts/packaging-test.sh

@@ -43,6 +43,13 @@ if [ -f "/etc/os-release" ] ; then
             sudo apt-get install -y --allow-downgrades lintian=2.15.0
         fi
     fi
+    if [[ "$ID" == "rhel" ]] ; then
+      # Downgrade containerd if necessary to work around runc bug
+      # See: https://github.com/opencontainers/runc/issues/3551
+      if containerd -version | grep -sF 1.6.7; then
+        sudo yum downgrade -y containerd.io
+      fi
+    fi
 else
     cat /etc/issue || true
 fi

.ci/snapshotBwcVersions

@@ -1,5 +1,4 @@
 BWC_VERSION:
-  - "7.17.6"
-  - "8.3.4"
-  - "8.4.0"
+  - "7.17.7"
+  - "8.4.1"
   - "8.5.0"

.github/CODEOWNERS

@@ -15,3 +15,7 @@ x-pack/plugin/core/src/main/resources/monitoring-logstash-mb.json @elastic/infra
 x-pack/plugin/core/src/main/resources/monitoring-logstash.json @elastic/infra-monitoring-ui
 x-pack/plugin/core/src/main/resources/monitoring-mb-ilm-policy.json @elastic/infra-monitoring-ui
 x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java @elastic/infra-monitoring-ui
+
+# Elastic Agent
+x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet @elastic/elastic-agent-control-plane
+x-pack/plugin/core/src/main/resources/fleet-* @elastic/elastic-agent-control-plane

.github/PULL_REQUEST_TEMPLATE.md

@@ -7,8 +7,8 @@ attention.
 -->
 
 - Have you signed the [contributor license agreement](https://www.elastic.co/contributor-agreement)?
-- Have you followed the [contributor guidelines](https://github.com/elastic/elasticsearch/blob/master/CONTRIBUTING.md)?
+- Have you followed the [contributor guidelines](https://github.com/elastic/elasticsearch/blob/main/CONTRIBUTING.md)?
 - If submitting code, have you built your formula locally prior to submission with `gradle check`?
-- If submitting code, is your pull request against master? Unless there is a good reason otherwise, we prefer pull requests against master and will backport as needed.
+- If submitting code, is your pull request against main? Unless there is a good reason otherwise, we prefer pull requests against main and will backport as needed.
 - If submitting code, have you checked that your submission is for an [OS and architecture that we support](https://www.elastic.co/support/matrix#show_os)?
-- If you are submitting this code for a class then read our [policy](https://github.com/elastic/elasticsearch/blob/master/CONTRIBUTING.md#contributing-as-part-of-a-class) for that.
+- If you are submitting this code for a class then read our [policy](https://github.com/elastic/elasticsearch/blob/main/CONTRIBUTING.md#contributing-as-part-of-a-class) for that.

TRACING.md

@@ -45,7 +45,7 @@ For context, the APM agent pulls configuration from [multiple
 sources][agent-config], with a hierarchy that means, for example, that options
 set in the config file cannot be overridden via system properties.
 
-Now, in order to send tracing data to the APM server, ES needs to configured with
+Now, in order to send tracing data to the APM server, ES needs to be configured with
 either a `secret_key` or an `api_key`. We could configure these in the agent via
 system properties, but then their values would be available to any Java code in
 Elasticsearch that can read system properties.

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=97a52d145762adc241bad7fd18289bf7f6801e08ece6badf80402fe2b9f250b1
+distributionSha256Sum=db9c8211ed63f61f60292c69e80d89196f9eb36665e369e7f00ac4cc841c2219

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -39,7 +39,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
         """
 
         when:
-        def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build()
+        def result = gradleRunner("setupDistro", '-g', gradleUserHome).build()
 
         then:
         result.task(":distribution:archives:${testArchiveProjectName}:buildExpanded").outcome == TaskOutcome.SUCCESS

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.gradle.internal
 
+import spock.lang.TempDir
 import spock.lang.Unroll
 import com.github.tomakehurst.wiremock.WireMockServer
 
@@ -126,7 +127,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         when:
         def result = WiremockFixture.withWireMock(mockRepoUrl, mockedContent) { server ->
             buildFile << repositoryMockSetup(server, jdkVendor, jdkVersion)
-            gradleRunner('getJdk', '-i', '-g', testProjectDir.newFolder().toString()).build()
+            gradleRunner('getJdk', '-i', '-g', gradleUserHome).build()
         }
 
         then:
@@ -179,13 +180,12 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         def result = WiremockFixture.withWireMock(mockRepoUrl, mockedContent) { server ->
             buildFile << repositoryMockSetup(server, VENDOR_ADOPTIUM, ADOPT_JDK_VERSION)
 
-            def commonGradleUserHome = testProjectDir.newFolder().toString()
             // initial run
-            def firstResult = gradleRunner('clean', 'getJdk', '-i', '--warning-mode', 'all', '-g', commonGradleUserHome).build()
+            def firstResult = gradleRunner('clean', 'getJdk', '-i', '--warning-mode', 'all', '-g', gradleUserHome).build()
             // assert the output of an executed transform is shown
             assertOutputContains(firstResult.output, "Unpacking $expectedArchiveName using $transformType")
             // run against up-to-date transformations
-            gradleRunner('clean', 'getJdk', '-i', '--warning-mode', 'all', '-g', commonGradleUserHome).build()
+            gradleRunner('clean', 'getJdk', '-i', '--warning-mode', 'all', '-g', gradleUserHome).build()
         }
 
         then:

build-tools-internal/src/main/groovy/elasticsearch.build-complete.gradle

@@ -49,7 +49,7 @@ if (buildNumber && performanceTest == null) {
 
         fileset(dir: "${gradle.gradleUserHomeDir}/workers", followsymlinks: false)
 
-        fileset(dir: "${project.projectDir}/.gradle/reaper", followsymlinks: false)
+        fileset(dir: "${project.projectDir}/.gradle/reaper", followsymlinks: false, erroronmissingdir: false)
       }
     } catch (Exception e) {
       logger.lifecycle("Failed to archive additional logs", e)

build-tools-internal/src/main/groovy/elasticsearch.run-ccs.gradle

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask
+import org.elasticsearch.gradle.testclusters.RunTask
+
+boolean proxyMode = true;
+
+def fulfillingCluster = testClusters.register('fulfilling-cluster') {
+  setting 'xpack.watcher.enabled', 'false'
+  setting 'xpack.ml.enabled', 'false'
+  setting 'xpack.license.self_generated.type', 'trial'
+
+  user username: 'elastic-admin', password: 'elastic-password', role: '_es_test_root'
+}
+
+def queryingCluster = testClusters.register('querying-cluster') {
+  setting 'xpack.watcher.enabled', 'false'
+  setting 'xpack.ml.enabled', 'false'
+  setting 'xpack.license.self_generated.type', 'trial'
+  if (proxyMode) {
+    setting 'cluster.remote.my_remote_cluster.mode', 'proxy'
+    setting 'cluster.remote.my_remote_cluster.proxy_address', {
+      "\"${fulfillingCluster.get().getAllTransportPortURI().get(0)}\""
+    }
+  } else {
+    setting 'cluster.remote.my_remote_cluster.seeds', {
+      fulfillingCluster.get().getAllTransportPortURI().collect { "\"$it\"" }.toString()
+    }
+  }
+  setting 'cluster.remote.connections_per_cluster', "1"
+
+  user username: 'elastic-admin', password: 'elastic-password', role: '_es_test_root'
+}
+
+// the following task is needed to make sure the fulfilling cluster is fully configured before starting both clusters
+// this allows the quering cluster to use configuration from the fulfilling cluster while honoring the RunTasks configuration (such as use port 9200)
+tasks.register('initfulfillingCluster', RunTask) {
+  useCluster testClusters.named("fulfilling-cluster")
+  initOnly = true //only initialize the testCluster, don't start it
+  portOffset = 1 //when only initializing, instruct to use one above the normal ports to avoid collisions when other cluster also initializes
+  //debug = true //this task doesn't honor the command line options for run-ccs, so need to statically configure debug
+}
+
+tasks.register("run-ccs", RunTask) {
+  dependsOn initfulfillingCluster
+  useCluster testClusters.named("fulfilling-cluster")
+  useCluster testClusters.named("querying-cluster")
+  doFirst {
+    println "** Querying cluster HTTP endpoints are: ${-> queryingCluster.get().allHttpSocketURI.join(",")}"
+    println "** Querying cluster transport endpoints are: ${-> queryingCluster.get().getAllTransportPortURI().join(",")}"
+    println "** Fulfilling cluster HTTP endpoints are: ${-> fulfillingCluster.get().allHttpSocketURI.join(",")}"
+    println "** Fulfilling cluster transport endpoints are: ${-> fulfillingCluster.get().getAllTransportPortURI().join(",")}"
+  }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java

@@ -24,6 +24,7 @@
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
 
 import java.io.File;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
@@ -249,9 +250,10 @@ static void createBuildBwcTask(
                 @Override
                 public void execute(Task task) {
                     if (expectedOutputFile.exists() == false) {
-                        throw new InvalidUserDataException(
-                            "Building " + bwcVersion.get() + " didn't generate expected artifact " + expectedOutputFile
-                        );
+                        Path relativeOutputPath = project.getRootDir().toPath().relativize(expectedOutputFile.toPath());
+                        final String message = "Building %s didn't generate expected artifact [%s]. The working branch may be "
+                            + "out-of-date - try merging in the latest upstream changes to the branch.";
+                        throw new InvalidUserDataException(message.formatted(bwcVersion.get(), relativeOutputPath));
                     }
                 }
             });

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java

@@ -10,7 +10,9 @@
 
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.info.BuildParams;
+import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
+import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.provider.ProviderFactory;
@@ -36,6 +38,16 @@ public void apply(Project project) {
             version -> (version.equals(VersionProperties.getElasticsearchVersion()) && BuildParams.isSnapshotBuild() == false)
                 || BuildParams.getBwcVersions().unreleasedInfo(version) == null
         );
+
+        if (shouldConfigureTestClustersWithOneProcessor()) {
+            NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
+                .getExtensions()
+                .getByName(TestClustersPlugin.EXTENSION_NAME);
+            testClusters.configureEach(elasticsearchCluster -> elasticsearchCluster.setting("node.processors", "1"));
+        }
     }
 
+    private boolean shouldConfigureTestClustersWithOneProcessor() {
+        return Boolean.parseBoolean(System.getProperty("tests.configure_test_clusters_with_one_processor", "false"));
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonAgainstSchemaTask.java

@@ -118,11 +118,16 @@ public void validate(InputChanges inputChanges) throws IOException {
                 errors.values().stream().flatMap(Collection::stream).forEach(printWriter::println);
             }
             StringBuilder sb = new StringBuilder();
-            sb.append("Error validating JSON. See the report at: ");
+            sb.append("Verification failed. See the report at: ");
             sb.append(getReport().toURI().toASCIIString());
             sb.append(System.lineSeparator());
             sb.append(
-                String.format("Verification failed: %d files contained %d violations", errors.keySet().size(), errors.values().size())
+                String.format(
+                    "Error validating %s: %d files contained %d violations",
+                    getFileType(),
+                    errors.keySet().size(),
+                    errors.values().size()
+                )
             );
             throw new JsonSchemaException(sb.toString());
         }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java

@@ -21,8 +21,7 @@
 import java.util.stream.Collectors;
 
 /**
- * This class models the contents of a changelog YAML file. We validate it using a
- * JSON Schema, as well as some programmatic checks in {@link ValidateChangelogEntryTask}.
+ * This class models the contents of a changelog YAML file. We validate it using a JSON Schema.
  * <ul>
  *   <li><code>buildSrc/src/main/resources/changelog-schema.json</code></li>
  *   <li><a href="https://json-schema.org/understanding-json-schema/">Understanding JSON Schema</a></li>

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java

@@ -19,7 +19,6 @@
 import org.gradle.api.file.FileTree;
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.provider.Provider;
-import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.util.PatternSet;
 
 import java.io.File;
@@ -52,23 +51,15 @@ public void apply(Project project) {
             .getAsFileTree()
             .matching(new PatternSet().include("**/*.yml", "**/*.yaml"));
 
-        final Provider<ValidateYamlAgainstSchemaTask> validateChangelogsAgainstYamlTask = project.getTasks()
-            .register("validateChangelogsAgainstSchema", ValidateYamlAgainstSchemaTask.class, task -> {
+        final Provider<ValidateYamlAgainstSchemaTask> validateChangelogsTask = project.getTasks()
+            .register("validateChangelogs", ValidateYamlAgainstSchemaTask.class, task -> {
                 task.setGroup("Documentation");
                 task.setDescription("Validate that the changelog YAML files comply with the changelog schema");
                 task.setInputFiles(yamlFiles);
                 task.setJsonSchema(new File(project.getRootDir(), RESOURCES + "changelog-schema.json"));
                 task.setReport(new File(project.getBuildDir(), "reports/validateYaml.txt"));
             });
 
-        final TaskProvider<ValidateChangelogEntryTask> validateChangelogsTask = project.getTasks()
-            .register("validateChangelogs", ValidateChangelogEntryTask.class, task -> {
-                task.setGroup("Documentation");
-                task.setDescription("Validate that all changelog YAML files are well-formed");
-                task.setChangelogs(yamlFiles);
-                task.dependsOn(validateChangelogsAgainstYamlTask);
-            });
-
         final Function<Boolean, Action<GenerateReleaseNotesTask>> configureGenerateTask = shouldConfigureYamlFiles -> task -> {
             task.setGroup("Documentation");
             if (shouldConfigureYamlFiles) {