Merge branch 'main' into rollup-support-cancel

* main: (163 commits)
  [DOCS] Edits frequent items aggregation (elastic#91564)
  Handle providers of optional services in ubermodule classloader (elastic#91217)
  Add `exportDockerImages` lifecycle task for exporting docker tarballs (elastic#91571)
  Fix CSV dependency report output file location in DRA CI job
  Fix variable placeholder for Strings.format calls (elastic#91531)
  Fix output dir creation in ConcatFileTask (elastic#91568)
  Fix declaration of dependencies in DRA snapshots CI job (elastic#91569)
  Upgrade Gradle Enterprise plugin to 3.11.4 (elastic#91435)
  Ingest DateProcessor (small) speedup, optimize collections code in DateFormatter.forPattern (elastic#91521)
  Fix inter project handling of generateDependenciesReport (elastic#91555)
  [Synthetics] Add synthetics-* read to fleet-server (elastic#91391)
  [ML] Copy more settings when creating DF analytics destination index (elastic#91546)
  Reduce CartesianCentroidIT flakiness (elastic#91553)
  Propagate last node to reinitialized routing tables (elastic#91549)
  Forecast write load during rollovers (elastic#91425)
  [DOCS] Warn about potential overhead of named queries (elastic#91512)
  Datastream unavailable exception metadata (elastic#91461)
  Generate docker images and dependency report in DRA ci job (elastic#91545)
  Support cartesian_bounds aggregation on point and shape (elastic#91298)
  Add support for EQL samples queries (elastic#91312)
  ...

# Conflicts:
#	x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupShardIndexer.java

.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots-trigger.yml

@@ -0,0 +1,6 @@
+---
+jjbb-template: periodic-trigger-lgc.yml
+vars:
+  - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+dra-snapshot
+  - lgc-job: elastic+elasticsearch+%BRANCH%+intake
+  - cron: "H H/12 * * *"

.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots.yml

@@ -0,0 +1,65 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+periodic+dra-snapshot
+    workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+dra-snapshot
+    display-name: "elastic / elasticsearch # %BRANCH% - DRA snapshot"
+    description: "Publishing Daily Releasable Artifacts (DRAs) of Elasticsearch %BRANCH% snapshots.\n"
+    node: "ubuntu-20.04"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+      - shell: |
+            #!/usr/local/bin/runbld --redirect-stderr
+            ES_VERSION=$(cat build-tools-internal/version.properties \
+                  | grep elasticsearch \
+                  | sed "s/elasticsearch//g" \
+                  | sed "s/ //g" \
+                  | sed "s/=//")
+            BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats master)"
+            ML_CPP_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh ml-cpp master)"
+            set -euo pipefail
+            set +x
+            VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID)
+            export VAULT_TOKEN
+            $WORKSPACE/.ci/scripts/run-gradle.sh -Ddra.artifacts=true \
+                -Ddra.artifacts.dependency.beats=${BEATS_BUILD_ID} \
+                -Ddra.artifacts.dependency.ml-cpp=${ML_CPP_BUILD_ID} \
+                -Dcsv=$WORKSPACE/build/distributions/dependencies-${ES_VERSION}-SNAPSHOT.csv \
+                buildReleaseArtifacts \
+                exportDockerImages \
+                :distribution:generateDependenciesReport
+
+            unset VAULT_TOKEN
+            set -x
+            $WORKSPACE/x-pack/plugin/sql/connectors/tableau/package.sh asm qualifier=-SNAPSHOT
+
+            # we regenerate this file as part of the release manager invocation
+            rm $WORKSPACE/build/distributions/elasticsearch-jdbc-${ES_VERSION}-SNAPSHOT.taco.sha512
+
+            # Allow other users access to read the artifacts so they are readable in the
+            # container
+            find $WORKSPACE -type f -path "*/build/distributions/*" -exec chmod a+r {} \;
+
+            # Allow other users write access to create checksum files
+            find $WORKSPACE -type d -path "*/build/distributions" -exec chmod a+w {} \;
+
+            # Artifacts should be generated
+            docker run --rm \
+                   --name release-manager \
+                   -e VAULT_ADDR \
+                   -e VAULT_ROLE_ID \
+                   -e VAULT_SECRET_ID \
+                   --mount type=bind,readonly=false,src="$PWD",target=/artifacts \
+                   docker.elastic.co/infra/release-manager:latest \
+                   cli collect \
+                   --project elasticsearch \
+                   --branch "master" \
+                   --commit "$GIT_COMMIT" \
+                   --workflow "snapshot" \
+                   --version "$ES_VERSION" \
+                   --artifact-set main \
+                   --dependency beats:https://artifacts-snapshot.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}-SNAPSHOT.json \
+                   --dependency ml-cpp:https://artifacts-snapshot.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}-SNAPSHOT.json

.ci/scripts/resolve-dra-manifest.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+ARTIFACT="${ARTIFACT:-$1}"
+BRANCH="${BRANCH:-$2}"
+
+curl -sS https://artifacts-snapshot.elastic.co/$ARTIFACT/latest/$BRANCH.json \
+  | jq -r '.build_id'

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -8,6 +8,7 @@
 package org.elasticsearch.benchmark.routing.allocation;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -150,6 +151,9 @@ private int toInt(String v) {
         return Integer.valueOf(v.trim());
     }
 
+    /**
+     * Once we use DesiredBalanceShardsAllocator this only measures reconciliation, not the balance calculation
+     */
     @Benchmark
     public ClusterState measureAllocation() {
         ClusterState clusterState = initialClusterState;
@@ -162,7 +166,7 @@ public ClusterState measureAllocation() {
                     .filter(ShardRouting::initializing)
                     .collect(Collectors.toList())
             );
-            clusterState = strategy.reroute(clusterState, "reroute");
+            clusterState = strategy.reroute(clusterState, "reroute", ActionListener.noop());
         }
         return clusterState;
     }

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@@ -32,6 +33,8 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision.NOT_TAKEN;
+
 public final class Allocators {
     private static class NoopGatewayAllocator extends GatewayAllocator {
         public static final NoopGatewayAllocator INSTANCE = new NoopGatewayAllocator();
@@ -54,6 +57,14 @@ public void allocateUnassigned(
         ) {
             // noop
         }
+
+        @Override
+        public AllocateUnassignedDecision explainUnassignedShardAllocation(
+            ShardRouting unassignedShard,
+            RoutingAllocation routingAllocation
+        ) {
+            return NOT_TAKEN;
+        }
     }
 
     private Allocators() {

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java

@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.benchmark.routing.allocation;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.DataTier;
+import org.elasticsearch.cluster.routing.allocation.ShardsAvailabilityHealthIndicatorService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.health.HealthIndicatorResult;
+import org.elasticsearch.health.node.HealthInfo;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.tasks.TaskManager;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING;
+
+@Fork(3)
+@Warmup(iterations = 10)
+@Measurement(iterations = 10)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@SuppressWarnings("unused") // invoked by benchmarking framework
+public class ShardsAvailabilityHealthIndicatorBenchmark {
+
+    @Param(
+        {
+            // indices| shards| replicas| nodes
+            "     10000|        1|        0|    3",
+            "     20000|        1|        0|    3",
+            "     50000|        1|        0|    3",
+            "     100000|       1|        0|    3",
+            "     200000|       1|        0|    3",
+
+            "     10000|        1|        1|    3",
+            "     20000|        1|        1|    3",
+            "     50000|        1|        1|    3",
+            "     100000|       1|        1|    3",
+            "     200000|       1|        1|    3" }
+    )
+    public String indicesShardsReplicasNodes = "10|1|0|1";
+
+    private ShardsAvailabilityHealthIndicatorService indicatorService;
+
+    @Setup
+    public void setUp() throws Exception {
+        final String[] params = indicesShardsReplicasNodes.split("\\|");
+
+        int numIndices = toInt(params[0]);
+        int numShards = toInt(params[1]);
+        int numReplicas = toInt(params[2]);
+        int numNodes = toInt(params[3]);
+
+        AllocationService allocationService = Allocators.createAllocationService(Settings.EMPTY);
+
+        Metadata.Builder mb = Metadata.builder();
+        RoutingTable.Builder rb = RoutingTable.builder();
+
+        DiscoveryNodes.Builder nb = DiscoveryNodes.builder();
+        Set<String> failedNodeIds = new HashSet<>();
+        for (int i = 1; i <= numNodes; i++) {
+            String nodeId = "node" + i;
+            nb.add(Allocators.newNode(nodeId, Map.of()));
+            failedNodeIds.add(nodeId);
+        }
+
+        UnassignedInfo decidersNoUnassignedInfo = new UnassignedInfo(
+            UnassignedInfo.Reason.ALLOCATION_FAILED,
+            null,
+            null,
+            failedNodeIds.size(),
+            System.nanoTime(),
+            System.currentTimeMillis(),
+            false,
+            UnassignedInfo.AllocationStatus.DECIDERS_NO,
+            failedNodeIds,
+            null
+        );
+
+        RoutingTable.Builder routingTable = RoutingTable.builder();
+        for (int i = 1; i <= numIndices; i++) {
+            IndexMetadata indexMetadata = IndexMetadata.builder("test_" + i)
+                .settings(
+                    Settings.builder()
+                        .put("index.version.created", Version.CURRENT)
+                        .put(DataTier.TIER_PREFERENCE_SETTING.getKey(), "data_warm")
+                        .put(INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "data", "warm")
+                )
+                .numberOfShards(numShards)
+                .numberOfReplicas(numReplicas)
+                .build();
+
+            final IndexRoutingTable.Builder indexRountingTableBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex());
+            for (int shardIdNumber = 0; shardIdNumber < numShards; shardIdNumber++) {
+                ShardId shardId = new ShardId(indexMetadata.getIndex(), shardIdNumber);
+                final IndexShardRoutingTable.Builder shardBuilder = new IndexShardRoutingTable.Builder(shardId);
+                ShardRouting shardRouting = ShardRouting.newUnassigned(
+                    shardId,
+                    true,
+                    RecoverySource.ExistingStoreRecoverySource.INSTANCE,
+                    decidersNoUnassignedInfo
+                );
+                shardBuilder.addShard(shardRouting);
+                if (shardIdNumber < numReplicas) {
+                    shardBuilder.addShard(
+                        ShardRouting.newUnassigned(
+                            shardId,
+                            false,
+                            RecoverySource.EmptyStoreRecoverySource.INSTANCE,
+                            decidersNoUnassignedInfo
+                        )
+                    );
+                }
+                indexRountingTableBuilder.addIndexShard(shardBuilder);
+            }
+
+            routingTable.add(indexRountingTableBuilder);
+            mb.put(indexMetadata, false);
+        }
+
+        ClusterState initialClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metadata(mb)
+            .routingTable(routingTable)
+            .nodes(nb)
+            .build();
+
+        Settings settings = Settings.builder().put("node.name", ShardsAvailabilityHealthIndicatorBenchmark.class.getSimpleName()).build();
+        ThreadPool threadPool = new ThreadPool(settings);
+
+        ClusterService clusterService = new ClusterService(
+            Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+            threadPool,
+            new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet())
+        );
+        clusterService.getClusterApplierService().setInitialState(initialClusterState);
+        indicatorService = new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService);
+    }
+
+    private int toInt(String v) {
+        return Integer.parseInt(v.trim());
+    }
+
+    @Benchmark
+    public HealthIndicatorResult measureCalculate() {
+        return indicatorService.calculate(true, HealthInfo.EMPTY_HEALTH_INFO);
+    }
+
+}

build-tools-internal/build.gradle

@@ -51,6 +51,10 @@ gradlePlugin {
       id = 'elasticsearch.docs-test'
       implementationClass = 'org.elasticsearch.gradle.internal.doc.DocsTestPlugin'
     }
+    draArtifacts {
+      id = 'elasticsearch.dra-artifacts'
+      implementationClass = 'org.elasticsearch.gradle.internal.dra.DraResolvePlugin'
+    }
     globalBuildInfo {
       id = 'elasticsearch.global-build-info'
       implementationClass = 'org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin'

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy

@@ -17,7 +17,7 @@ class LocalRepositoryFixture extends ExternalResource {
 
     private TemporaryFolder temporaryFolder
 
-    LocalRepositoryFixture(){
+    LocalRepositoryFixture() {
         this.temporaryFolder = new TemporaryFolder()
     }
 
@@ -33,22 +33,27 @@ class LocalRepositoryFixture extends ExternalResource {
         temporaryFolder.after()
     }
 
-    void generateJar(String group, String module, String version, String... clazzNames){
+    void generateJar(String group, String module, String version, String... clazzNames) {
         def baseGroupFolderPath = group.replace('.', '/')
         def targetFolder = new File(repoDir, "${baseGroupFolderPath}/$module/$version")
         targetFolder.mkdirs()
 
         def jarFile = new File(targetFolder, "${module}-${version}.jar")
-        clazzNames.each {clazzName ->
-            DynamicType.Unloaded<?> dynamicType = new ByteBuddy().subclass(Object.class)
-                    .name(clazzName)
-                    .make()
-            if(jarFile.exists()) {
-                dynamicType.inject(jarFile);
-            }else {
-                dynamicType.toJar(jarFile);
+        if (clazzNames.size() == 0) {
+            jarFile.write("blubb")
+        } else {
+            clazzNames.each { clazzName ->
+                DynamicType.Unloaded<?> dynamicType = new ByteBuddy().subclass(Object.class)
+                        .name(clazzName)
+                        .make()
+                if (jarFile.exists()) {
+                    dynamicType.inject(jarFile);
+                } else {
+                    dynamicType.toJar(jarFile);
+                }
             }
         }
+
     }
 
     void configureBuild(File buildFile) {

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy

@@ -52,6 +52,7 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest {
         THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.""".stripIndent()
 
     def setup() {
+        configurationCacheCompatible = false
         buildFile << """
         plugins {
           id 'java'