Merge branch 'upstream/master' into number-support-single-type

* upstream/master: (40 commits)
  Fix CI job naming
  [ML] disallow autoscaling downscaling in two trained model assignment scenarios (elastic#88623)
  Add "Vector Search" area to changelog schema
  [DOCS] Update API key API (elastic#88499)
  Enable the pipeline on the feature branch (elastic#88672)
  Adding the ability to register a PeerFinderListener to Coordinator (elastic#88626)
  [DOCS] Fix transform painless example syntax (elastic#88364)
  [ML] Muting InternalCategorizationAggregationTests testReduceRandom (elastic#88685)
  Fix double rounding errors for disk usage (elastic#88683)
  Replace health request with a state observer. (elastic#88641)
  [ML] Fail model deployment if all allocations cannot be provided (elastic#88656)
  Upgrade to OpenJDK 18.0.2+9 (elastic#88675)
  [ML] make bucket_correlation aggregation generally available (elastic#88655)
  Adding cardinality support for random_sampler agg (elastic#86838)
  Use custom task instead of generic AckedClusterStateUpdateTask (elastic#88643)
  Reinstate test cluster throttling behavior (elastic#88664)
  Mute testReadBlobWithPrematureConnectionClose
  Simplify plugin descriptor tests (elastic#88659)
  Add CI job for testing more job parallelism
  [ML] make deployment infer requests fully cancellable (elastic#88649)
  ...

.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots.yml

@@ -25,8 +25,6 @@
           black-list-labels:
             - '>test-mute'
             - 'test-full-bwc'
-          black-list-target-branches:
-            - feature/desired-balance-allocator
     axes:
       - axis:
           type: slave

.ci/jobs.t/elastic+elasticsearch+pull-request+docs-check.yml

@@ -22,8 +22,6 @@
             - ^docs/.*
           black-list-labels:
             - '>test-mute'
-          black-list-target-branches:
-            - feature/desired-balance-allocator
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.ci/jobs.t/elastic+elasticsearch+pull-request+eql-correctness.yml

@@ -20,7 +20,6 @@
           cancel-builds-on-update: true
           black-list-target-branches:
             - 6.8
-            - feature/desired-balance-allocator
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+example-plugins.yml

@@ -23,8 +23,6 @@
             - build-tools/.*
             - build-tools-internal/.*
             - plugins/examples/.*
-          black-list-target-branches:
-            - feature/desired-balance-allocator
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix-sample.yml

@@ -21,7 +21,6 @@
           cancel-builds-on-update: true
           black-list-target-branches:
             - 6.8
-            - feature/desired-balance-allocator
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample.yml

@@ -24,7 +24,6 @@
           black-list-target-branches:
             - 7.17
             - 7.16
-            - feature/desired-balance-allocator
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-upgrade-tests.yml

@@ -22,7 +22,6 @@
           cancel-builds-on-update: true
           black-list-target-branches:
             - 6.8
-            - feature/desired-balance-allocator
           excluded-regions:
             - ^docs/.*
           white-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+part-3.yml

@@ -0,0 +1,33 @@
+---
+- job:
+    name: "elastic+elasticsearch+pull-request+part-3"
+    display-name: "elastic / elasticsearch - pull request part-3"
+    description: "Testing of Elasticsearch pull requests - part 3"
+    workspace: "/dev/shm/elastic+elasticsearch+pull-request+part-3"
+    scm:
+      - git:
+          refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+          branches:
+            - "${ghprbActualCommit}"
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+elasticsearch-ci/part-3.*'
+          github-hooks: true
+          status-context: elasticsearch-ci/part-3
+          cancel-builds-on-update: true
+          white-list-labels:
+            - 'test-part-3'
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA8_HOME=$HOME/.java/java8
+            JAVA11_HOME=$HOME/.java/java11
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart3

.ci/jobs.t/elastic+elasticsearch+pull-request+precommit.yml

@@ -20,8 +20,6 @@
           cancel-builds-on-update: true
           white-list-labels:
             - '>test-mute'
-          black-list-target-branches:
-            - feature/desired-balance-allocator
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.ci/jobs.t/elastic+elasticsearch+pull-request+rest-compatibility.yml

@@ -23,7 +23,6 @@
             - 7.16
             - 7.15
             - 6.8
-            - feature/desired-balance-allocator
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/templates.t/pull-request-gradle-unix.yml

@@ -22,8 +22,6 @@
             - ^docs/.*
           black-list-labels:
             - '>test-mute'
-          black-list-target-branches:
-            - feature/desired-balance-allocator
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

TESTING.asciidoc

@@ -108,6 +108,40 @@ password: `elastic-password`.
 - In order to set an Elasticsearch setting, provide a setting with the following prefix: `-Dtests.es.`
 - In order to pass a JVM setting, e.g. to disable assertions: `-Dtests.jvm.argline="-da"`
 
+==== Customizing the test cluster for ./gradlew run
+
+You may need to customize the cluster configuration for the ./gradlew run task.
+You can simply find the task in the source code and configure it there.
+(The task is currently defined in build-tools-internal/src/main/groovy/elasticsearch.run.gradle)
+However, this requires modifying a source controlled file and subject to accidental commits.
+Alternatively, you can use a Gradle init script to inject custom build logic with the -I flag to configure this task locally.
+
+For example:
+
+To enable HTTPS for use with ./gradlew run, an extraConfigFile is needed to be added to the cluster configuration.
+Create a file (for example ~/custom-run.gradle) with the following contents:
+-------------------------------------
+rootProject {
+  if(project.name == 'elasticsearch') {
+    afterEvaluate {
+        testClusters.matching { it.name == "runTask"}.configureEach {
+          extraConfigFile 'http.p12', file("<path/to/file>/http.p12")
+        }
+    }
+  }
+}
+-------------------------------------
+Now tell Gradle to use this init script:
+-------------------------------------
+./gradlew run -I ~/custom-run.gradle \
+-Dtests.es.xpack.security.http.ssl.enabled=true \
+-Dtests.es.xpack.security.http.ssl.keystore.path=http.p12
+-------------------------------------
+
+Now the http.p12 file will be placed in the config directory of the running cluster and available for use.
+Assuming you have the http.ssl.keystore setup correctly, you can now use HTTPS with ./gradlew run without the risk
+of accidentally committing your local configurations.
+
 === Test case filtering.
 
 You can run a single test, provided that you specify the Gradle project. See the documentation on

build-tools-internal/src/main/resources/changelog-schema.json

@@ -83,6 +83,7 @@
             "TLS",
             "Transform",
             "TSDB",
+            "Vector Search",
             "Watcher"
           ]
         },

build-tools-internal/src/test/java/org/elasticsearch/gradle/AbstractDistributionDownloadPluginTests.java

@@ -10,15 +10,14 @@
 
 import org.elasticsearch.gradle.internal.BwcVersions;
 import org.elasticsearch.gradle.internal.BwcVersions.VersionPair;
-import org.elasticsearch.gradle.internal.test.GradleUnitTestCase;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Project;
 import org.gradle.testfixtures.ProjectBuilder;
 
 import java.io.File;
 import java.util.Arrays;
 
-public class AbstractDistributionDownloadPluginTests extends GradleUnitTestCase {
+public class AbstractDistributionDownloadPluginTests {
     protected static Project rootProject;
     protected static Project archivesProject;
     protected static Project packagesProject;

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/ConcatFilesTaskTests.java

@@ -7,18 +7,21 @@
  */
 package org.elasticsearch.gradle.internal;
 
-import org.elasticsearch.gradle.internal.test.GradleUnitTestCase;
 import org.gradle.api.Project;
 import org.gradle.testfixtures.ProjectBuilder;
+import org.junit.Test;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.Arrays;
 
-public class ConcatFilesTaskTests extends GradleUnitTestCase {
+import static org.junit.Assert.assertEquals;
 
+public class ConcatFilesTaskTests {
+
+    @Test
     public void testHeaderAdded() throws IOException {
 
         Project project = createProject();
@@ -39,6 +42,7 @@ public void testHeaderAdded() throws IOException {
         file.delete();
     }
 
+    @Test
     public void testConcatenationWithUnique() throws IOException {
 
         Project project = createProject();

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/EmptyDirTaskTests.java

@@ -7,18 +7,22 @@
  */
 package org.elasticsearch.gradle.internal;
 
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-
 import org.apache.tools.ant.taskdefs.condition.Os;
-import org.elasticsearch.gradle.internal.test.GradleUnitTestCase;
 import org.gradle.api.Project;
 import org.gradle.testfixtures.ProjectBuilder;
+import org.junit.Test;
 
 import java.io.File;
 import java.io.IOException;
 
-public class EmptyDirTaskTests extends GradleUnitTestCase {
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeFalse;
+
+public class EmptyDirTaskTests {
 
+    @Test
     public void testCreateEmptyDir() throws Exception {
         Project project = ProjectBuilder.builder().build();
         EmptyDirTask emptyDirTask = project.getTasks().create("emptyDirTask", EmptyDirTask.class);
@@ -40,8 +44,9 @@ public void testCreateEmptyDir() throws Exception {
         newEmptyFolder.delete();
     }
 
+    @Test
     public void testCreateEmptyDirNoPermissions() throws Exception {
-        RandomizedTest.assumeFalse("Functionality is Unix specific", Os.isFamily(Os.FAMILY_WINDOWS));
+        assumeFalse("Functionality is Unix specific", Os.isFamily(Os.FAMILY_WINDOWS));
 
         Project project = ProjectBuilder.builder().build();
         EmptyDirTask emptyDirTask = project.getTasks().create("emptyDirTask", EmptyDirTask.class);

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginTests.java

@@ -14,11 +14,13 @@
 import org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes;
 import org.gradle.api.Project;
 import org.gradle.testfixtures.ProjectBuilder;
+import org.junit.Test;
 
 import java.io.File;
 
 public class InternalDistributionDownloadPluginTests extends AbstractDistributionDownloadPluginTests {
 
+    @Test
     public void testLocalCurrentVersionPackages() {
         ElasticsearchDistributionType[] types = { InternalElasticsearchDistributionTypes.RPM, InternalElasticsearchDistributionTypes.DEB };
         for (ElasticsearchDistributionType packageType : types) {
@@ -33,6 +35,7 @@ public void testLocalCurrentVersionPackages() {
         }
     }
 
+    @Test
     public void testLocalBwcPackages() {
         ElasticsearchDistributionType[] types = { InternalElasticsearchDistributionTypes.RPM, InternalElasticsearchDistributionTypes.DEB };
         for (ElasticsearchDistributionType packageType : types) {

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/JdkDownloadPluginTests.java

@@ -8,26 +8,30 @@
 
 package org.elasticsearch.gradle.internal;
 
-import org.elasticsearch.gradle.internal.test.GradleUnitTestCase;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Project;
 import org.gradle.testfixtures.ProjectBuilder;
 import org.junit.BeforeClass;
+import org.junit.Test;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertThrows;
 
-public class JdkDownloadPluginTests extends GradleUnitTestCase {
+public class JdkDownloadPluginTests {
     private static Project rootProject;
 
     @BeforeClass
     public static void setupRoot() {
         rootProject = ProjectBuilder.builder().build();
     }
 
+    @Test
     public void testMissingVendor() {
         assertJdkError(createProject(), "testjdk", null, "11.0.2+33", "linux", "x64", "vendor not specified for jdk [testjdk]");
     }
 
+    @Test
     public void testUnknownVendor() {
         assertJdkError(
             createProject(),
@@ -40,10 +44,12 @@ public void testUnknownVendor() {
         );
     }
 
+    @Test
     public void testMissingVersion() {
         assertJdkError(createProject(), "testjdk", "openjdk", null, "linux", "x64", "version not specified for jdk [testjdk]");
     }
 
+    @Test
     public void testBadVersionFormat() {
         assertJdkError(
             createProject(),
@@ -56,10 +62,12 @@ public void testBadVersionFormat() {
         );
     }
 
+    @Test
     public void testMissingPlatform() {
         assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", null, "x64", "platform not specified for jdk [testjdk]");
     }
 
+    @Test
     public void testUnknownPlatform() {
         assertJdkError(
             createProject(),
@@ -72,10 +80,12 @@ public void testUnknownPlatform() {
         );
     }
 
+    @Test
     public void testMissingArchitecture() {
         assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", "linux", null, "architecture not specified for jdk [testjdk]");
     }
 
+    @Test
     public void testUnknownArchitecture() {
         assertJdkError(
             createProject(),
@@ -97,7 +107,7 @@ private void assertJdkError(
         final String architecture,
         final String message
     ) {
-        IllegalArgumentException e = expectThrows(
+        IllegalArgumentException e = assertThrows(
             IllegalArgumentException.class,
             () -> createJdk(project, name, vendor, version, platform, architecture)
         );