Merge remote-tracking branch 'wjp/feature/data_stream_support_routing…

…' into feature/data_stream_support_routing

* wjp/feature/data_stream_support_routing: (44 commits)
  Revert "Adjust /_cat/templates not to request all metadata (elastic#78812)"
  Allow indices lookup to be built lazily (elastic#78745)
  [DOCS] Document default security in alpha2 (elastic#78227)
  Add cluster applier stats (elastic#77552)
  Fix failing URLDecodeProcessorTests::testProcessor test (elastic#78690)
  Upgrade to lucene snapshot ba75dc5e6bf (elastic#78817)
  Adjust /_cat/templates not to request all metadata (elastic#78812)
  Simplify build plugin license handling (elastic#77009)
  Fix SearchableSnapshotsBlobStoreCacheIntegTests.testBlobStoreCache (elastic#78616)
  Improve Docker image caching and testing (elastic#78552)
  Load knn vectors format with mmapfs (elastic#78724)
  Fix date math zone test to use negative minutes (elastic#78796)
  Changing name of shards field in node/stats api to shard_stats (elastic#78531)
  [DOCS] Fix system index refs in restore tutorial (elastic#78582)
  Add previously removed settings back for 8.0 (elastic#78784)
  TSDB: Fix template name in test
  Add a system property to forcibly format everything (elastic#78768)
  Revert "Adding config so that some tests will break if over-the-wire encryption fails (elastic#78409)" (elastic#78787)
  Must date math test failure
  Adding config so that some tests will break if over-the-wire encryption fails (elastic#78409)
  ...

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java

@@ -15,7 +15,6 @@
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
-import org.gradle.initialization.layout.BuildLayout;
 
 import javax.inject.Inject;
 import java.io.File;
@@ -29,7 +28,7 @@ class GitInfoPlugin implements Plugin<Project> {
     private Property<GitInfo> gitInfo;
 
     @Inject
-    public GitInfoPlugin(ProviderFactory factory, ObjectFactory objectFactory) {
+    GitInfoPlugin(ProviderFactory factory, ObjectFactory objectFactory) {
         this.factory = factory;
         this.objectFactory = objectFactory;
     }

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java

@@ -11,17 +11,15 @@
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.provider.MapProperty;
-import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 
 import javax.inject.Inject;
 import java.util.Map;
-import java.util.concurrent.Callable;
 
 public class LicensingPlugin implements Plugin<Project> {
-    final static String ELASTIC_LICENSE_URL_PREFIX = "https://raw.githubusercontent.com/elastic/elasticsearch/";
-    final static String ELASTIC_LICENSE_URL_POSTFIX = "/licenses/ELASTIC-LICENSE-2.0.txt";
+    static final String ELASTIC_LICENSE_URL_PREFIX = "https://raw.githubusercontent.com/elastic/elasticsearch/";
+    static final String ELASTIC_LICENSE_URL_POSTFIX = "/licenses/ELASTIC-LICENSE-2.0.txt";
 
     private ProviderFactory providerFactory;
 
@@ -34,24 +32,23 @@ public LicensingPlugin(ProviderFactory providerFactory) {
     public void apply(Project project) {
         Provider<String> revision = project.getRootProject().getPlugins().apply(GitInfoPlugin.class).getRevision();
         Provider<String> licenseCommitProvider = providerFactory.provider(() ->
-             isSnapshotVersion(project) ? revision.get() : "v" + project.getVersion().toString()
+             isSnapshotVersion(project) ? revision.get() : "v" + project.getVersion()
         );
 
-        MapProperty<String, String> licensesProperty = project.getObjects().mapProperty(String.class, String.class);
         Provider<String> projectLicenseURL = licenseCommitProvider.map(licenseCommit -> ELASTIC_LICENSE_URL_PREFIX +
                 licenseCommit + ELASTIC_LICENSE_URL_POSTFIX);
         // But stick the Elastic license url in project.ext so we can get it if we need to switch to it
         project.getExtensions().getExtraProperties().set("elasticLicenseUrl", projectLicenseURL);
 
-        MapProperty<String, String> convention = licensesProperty.convention(
-                providerFactory.provider((Callable<Map<? extends String, ? extends String>>) () -> Map.of(
+        MapProperty<String, String> licensesProperty = project.getObjects().mapProperty(String.class, String.class).convention(
+                providerFactory.provider(() -> Map.of(
                         "Server Side Public License, v 1", "https://www.mongodb.com/licensing/server-side-public-license",
                         "Elastic License 2.0", projectLicenseURL.get())
                 )
         );
+
         // Default to the SSPL+Elastic dual license
-        project.getExtensions().getExtraProperties().set("licenseCommit", licenseCommitProvider);
-        project.getExtensions().getExtraProperties().set("projectLicenses", convention);
+        project.getExtensions().getExtraProperties().set("projectLicenses", licensesProperty);
     }
 
     private boolean isSnapshotVersion(Project project) {

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy

@@ -277,8 +277,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
                     
             // requires elasticsearch artifact available
             tasks.named('bundlePlugin').configure { enabled = false }
-            licenseFile = file('license.txt')
-            noticeFile = file('notice.txt')
+            licenseFile.set(file('license.txt'))
+            noticeFile.set(file('notice.txt'))
             version = "1.0"
             group = 'org.acme'        
         """
@@ -352,8 +352,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
            
             // requires elasticsearch artifact available
             tasks.named('bundlePlugin').configure { enabled = false }
-            licenseFile = file('license.txt')
-            noticeFile = file('notice.txt')
+            licenseFile.set(file('license.txt'))
+            noticeFile.set(file('notice.txt'))
             version = "2.0"
             group = 'org.acme'        
         """

build-tools-internal/src/integTest/java/org/elasticsearch/gradle/internal/BuildPluginIT.java

@@ -8,6 +8,7 @@
 package org.elasticsearch.gradle.internal;
 
 import org.apache.commons.io.IOUtils;
+import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.test.GradleIntegrationTestCase;
 import org.gradle.testkit.runner.BuildResult;
 import org.junit.Rule;
@@ -47,8 +48,9 @@ public void testCheckTask() {
     public void testLicenseAndNotice() throws IOException {
         BuildResult result = getGradleRunner().withArguments("clean", "assemble").build();
         assertTaskSuccessful(result, ":assemble");
-        assertBuildFileExists(result, projectName(), "distributions/elasticsearch.build.jar");
-        try (ZipFile zipFile = new ZipFile(new File(getBuildDir(projectName()), "distributions/elasticsearch.build.jar"))) {
+        String expectedFilePath = "distributions/elasticsearch.build.jar";
+        assertBuildFileExists(result, projectName(), expectedFilePath);
+        try (ZipFile zipFile = new ZipFile(new File(getBuildDir(projectName()), expectedFilePath))) {
             ZipEntry licenseEntry = zipFile.getEntry("META-INF/LICENSE.txt");
             ZipEntry noticeEntry = zipFile.getEntry("META-INF/NOTICE.txt");
             assertNotNull("Jar does not have META-INF/LICENSE.txt", licenseEntry);

build-tools-internal/src/integTest/java/org/elasticsearch/gradle/internal/test/TestClasspathUtils.java

@@ -10,7 +10,6 @@
 
 import java.io.File;
 import java.io.IOException;
-
 import net.bytebuddy.ByteBuddy;
 import net.bytebuddy.description.modifier.Ownership;
 import net.bytebuddy.description.modifier.Visibility;

build-tools-internal/src/main/groovy/elasticsearch.base.gradle

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import org.elasticsearch.gradle.VersionProperties
+
+project.setDescription("Elasticsearch subproject " + project.getPath());
+// common maven publishing configuration
+project.setGroup("org.elasticsearch");
+project.setVersion(VersionProperties.getElasticsearch())

build-tools-internal/src/main/groovy/elasticsearch.formatting.gradle

@@ -196,7 +196,8 @@ def projectPathsToExclude = [
 
 subprojects {
   plugins.withType(ElasticsearchJavaPlugin).whenPluginAdded {
-    if (projectPathsToExclude.contains(project.path) == false) {
+    if (projectPathsToExclude.contains(project.path) == false ||
+        providers.systemProperty("es.format.everything").forUseAtConfigurationTime().isPresent()) {
       project.apply plugin: "com.diffplug.spotless"
 
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java

@@ -8,58 +8,81 @@
 
 package org.elasticsearch.gradle.internal;
 
-import org.codehaus.groovy.runtime.DefaultGroovyMethods;
 import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
 import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks;
-import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin;
-import org.elasticsearch.gradle.internal.precommit.SplitPackagesAuditPrecommitPlugin;
-import org.gradle.api.GradleException;
 import org.gradle.api.InvalidUserDataException;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.file.ProjectLayout;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
+import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.bundling.Jar;
+import org.gradle.initialization.layout.BuildLayout;
 
+import javax.inject.Inject;
 import java.io.File;
 
 /**
  * Encapsulates build configuration for elasticsearch projects.
  */
 public class BuildPlugin implements Plugin<Project> {
+
+    public static final String SSPL_LICENSE_PATH = "licenses/SSPL-1.0+ELASTIC-LICENSE-2.0.txt";
+
+    private final BuildLayout buildLayout;
+    private final ObjectFactory objectFactory;
+    private final ProviderFactory providerFactory;
+    private final ProjectLayout projectLayout;
+
+    @Inject
+    BuildPlugin(BuildLayout buildLayout, ObjectFactory objectFactory, ProviderFactory providerFactory, ProjectLayout projectLayout){
+        this.buildLayout = buildLayout;
+        this.objectFactory = objectFactory;
+        this.providerFactory = providerFactory;
+        this.projectLayout = projectLayout;
+    }
+
     @Override
     public void apply(final Project project) {
         // make sure the global build info plugin is applied to the root project
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
-
         if (project.getPluginManager().hasPlugin("elasticsearch.standalone-rest-test")) {
             throw new InvalidUserDataException(
                 "elasticsearch.standalone-test, " + "elasticsearch.standalone-rest-test, and elasticsearch.build are mutually exclusive"
             );
         }
 
         project.getPluginManager().apply("elasticsearch.java");
-        configureLicenseAndNotice(project);
         project.getPluginManager().apply("elasticsearch.publish");
         project.getPluginManager().apply(DependenciesInfoPlugin.class);
         project.getPluginManager().apply(DependenciesGraphPlugin.class);
-
         InternalPrecommitTasks.create(project, true);
+        configureLicenseAndNotice(project);
     }
 
-    public static void configureLicenseAndNotice(final Project project) {
+
+    public void configureLicenseAndNotice(final Project project) {
         final ExtraPropertiesExtension ext = project.getExtensions().getByType(ExtraPropertiesExtension.class);
-        ext.set("licenseFile", null);
-        ext.set("noticeFile", null);
-        // add license/notice files
-        project.afterEvaluate(p -> p.getTasks().withType(Jar.class).configureEach(jar -> {
-            if (ext.has("licenseFile") == false
-                || ext.get("licenseFile") == null
-                || ext.has("noticeFile") == false
-                || ext.get("noticeFile") == null) {
-                throw new GradleException("Must specify license and notice file for project " + p.getPath());
-            }
-            final File licenseFile = DefaultGroovyMethods.asType(ext.get("licenseFile"), File.class);
-            final File noticeFile = DefaultGroovyMethods.asType(ext.get("noticeFile"), File.class);
+        RegularFileProperty licenseFileProperty = objectFactory.fileProperty();
+        RegularFileProperty noticeFileProperty = objectFactory.fileProperty();
+        ext.set("licenseFile", licenseFileProperty);
+        ext.set("noticeFile", noticeFileProperty);
+
+        configureLicenseDefaultConvention(licenseFileProperty);
+        configureNoticeDefaultConvention(noticeFileProperty);
+
+        updateJarTasksMetaInf(project);
+    }
+
+    private void updateJarTasksMetaInf(Project project) {
+        final ExtraPropertiesExtension ext = project.getExtensions().getByType(ExtraPropertiesExtension.class);
+        project.getTasks().withType(Jar.class).configureEach(jar -> {
+            final RegularFileProperty licenseFileExtProperty = (RegularFileProperty) ext.get("licenseFile");
+            final RegularFileProperty noticeFileExtProperty = (RegularFileProperty) ext.get("noticeFile");
+            File licenseFile = licenseFileExtProperty.getAsFile().get();
+            File noticeFile = noticeFileExtProperty.getAsFile().get();
             jar.metaInf(spec -> {
                 spec.from(licenseFile.getParent(), from -> {
                     from.include(licenseFile.getName());
@@ -70,6 +93,16 @@ public static void configureLicenseAndNotice(final Project project) {
                     from.rename(s -> "NOTICE.txt");
                 });
             });
-        }));
+        });
+    }
+
+    private void configureLicenseDefaultConvention(RegularFileProperty licenseFileProperty) {
+        File licenseFileDefault = new File(buildLayout.getRootDirectory(), SSPL_LICENSE_PATH);
+        licenseFileProperty.convention(projectLayout.file(providerFactory.provider(() -> licenseFileDefault)));
+    }
+
+    private void configureNoticeDefaultConvention(RegularFileProperty noticeFileProperty) {
+        File noticeFileDefault = new File(buildLayout.getRootDirectory(), "NOTICE.txt");
+        noticeFileProperty.convention(projectLayout.file(providerFactory.provider(() -> noticeFileDefault)));
     }
 }