Merge branch 'main' of https://github.com/opensearch-project/OpenSearch…

… into replace-whitelist-blacklist Signed-off-by: Andreas <[email protected]>
opensearch-project · Mar 4, 2022 · fdd8a6e · fdd8a6e
2 parents 097ca57 + 0cc2c9b
commit fdd8a6e
Show file tree

Hide file tree

Showing 923 changed files with 7,167 additions and 12,712 deletions.
diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -33,8 +33,6 @@
       - [runtimeOnly](#runtimeonly)
       - [compileOnly](#compileonly)
       - [testImplementation](#testimplementation)
-    - [Gradle Plugins](#gradle-plugins)
-      - [Distribution Download Plugin](#distribution-download-plugin)
   - [Misc](#misc)
     - [git-secrets](#git-secrets)
       - [Installation](#installation)
@@ -342,15 +340,6 @@ somehow. OpenSearch plugins use this configuration to include dependencies that
 Code that is on the classpath for compiling tests that are part of this project but not production code. The canonical example
 of this is `junit`.
 
-### Gradle Plugins
-
-#### Distribution Download Plugin
-
-The Distribution Download plugin downloads the latest version of OpenSearch by default, and supports overriding this behavior by setting `customDistributionUrl`.
-```
-./gradlew integTest -DcustomDistributionUrl="https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/1127/linux/x64/dist/opensearch-1.2.0-linux-x64.tar.gz"
-```
-
 ## Misc
 
 ### git-secrets

diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
@@ -48,8 +48,8 @@ dependencies {
   api "org.openjdk.jmh:jmh-core:$versions.jmh"
   annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
   // Dependencies of JMH
-  runtimeOnly 'net.sf.jopt-simple:jopt-simple:4.6'
-  runtimeOnly 'org.apache.commons:commons-math3:3.2'
+  runtimeOnly 'net.sf.jopt-simple:jopt-simple:5.0.4'
+  runtimeOnly 'org.apache.commons:commons-math3:3.6.1'
 }
 
 // enable the JMH's BenchmarkProcessor to generate the final benchmark classes

diff --git a/build.gradle b/build.gradle
@@ -48,7 +48,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.2.0" apply false
+  id "com.diffplug.spotless" version "6.3.0" apply false
 }
 
 apply from: 'gradle/build-complete.gradle'

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -102,7 +102,7 @@ dependencies {
 
   api localGroovy()
 
-  api 'commons-codec:commons-codec:1.13'
+  api 'commons-codec:commons-codec:1.15'
   api 'org.apache.commons:commons-compress:1.21'
   api 'org.apache.ant:ant:1.10.12'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
@@ -124,7 +124,7 @@ dependencies {
   testFixturesApi gradleTestKit()
   testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
   testImplementation "org.mockito:mockito-core:${props.getProperty('mockito')}"
-  integTestImplementation('org.spockframework:spock-core:2.0-groovy-3.0') {
+  integTestImplementation('org.spockframework:spock-core:2.1-groovy-3.0') {
     exclude module: "groovy"
   }
 }
@@ -158,8 +158,8 @@ if (project != rootProject) {
   apply plugin: 'opensearch.publish'
 
   allprojects {
-    targetCompatibility = 11
-    sourceCompatibility = 11
+    targetCompatibility = 8
+    sourceCompatibility = 8
   }
 
   // groovydoc succeeds, but has some weird internal exception...

diff --git a/buildSrc/src/integTest/java/org/opensearch/gradle/precommit/ThirdPartyAuditTaskIT.java b/buildSrc/src/integTest/java/org/opensearch/gradle/precommit/ThirdPartyAuditTaskIT.java
@@ -71,7 +71,7 @@ public void testWithEmptyRules() {
             "-PcompileOnlyVersion=0.0.1",
             "-PcompileGroup=other.gradle:dummy-io",
             "-PcompileVersion=0.0.1"
-        ).buildAndFail();
+        ).build();
     }
 
     public void testViolationFoundAndCompileOnlyIgnored() {

diff --git a/buildSrc/src/main/groovy/org/opensearch/gradle/ResolveAllDependencies.java b/buildSrc/src/main/groovy/org/opensearch/gradle/ResolveAllDependencies.java
@@ -55,7 +55,7 @@ static boolean canBeResolved(Configuration configuration) {
             return false;
         }
         if (configuration instanceof org.gradle.internal.deprecation.DeprecatableConfiguration) {
-            var deprecatableConfiguration = (DeprecatableConfiguration) configuration;
+            DeprecatableConfiguration deprecatableConfiguration = (DeprecatableConfiguration) configuration;
             if (deprecatableConfiguration.canSafelyBeResolved() == false) {
                 return false;
             }

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java
@@ -105,6 +105,7 @@ public void apply(Project project) {
 
         setupResolutionsContainer(project);
         setupDistributionContainer(project, dockerSupport);
+        setupDownloadServiceRepo(project);
         project.afterEvaluate(this::setupDistributions);
     }
 
@@ -152,7 +153,6 @@ void setupDistributions(Project project) {
                 dependencies.add(distribution.getExtracted().getName(), distributionDependency.getExtractedNotation());
             }
         }
-        setupDownloadServiceRepo(project);
     }
 
     private DistributionDependency resolveDependencyNotation(Project p, OpenSearchDistribution distribution) {
@@ -195,22 +195,15 @@ private static void setupDownloadServiceRepo(Project project) {
         if (project.getRepositories().findByName(DOWNLOAD_REPO_NAME) != null) {
             return;
         }
-        Object customDistributionUrl = project.findProperty("customDistributionUrl");
-        // checks if custom Distribution Url has been passed by user from plugins
-        if (customDistributionUrl != null) {
-            addIvyRepo(project, DOWNLOAD_REPO_NAME, customDistributionUrl.toString(), FAKE_IVY_GROUP, "");
-            addIvyRepo(project, SNAPSHOT_REPO_NAME, customDistributionUrl.toString(), FAKE_SNAPSHOT_IVY_GROUP, "");
-        } else {
-            addIvyRepo(
-                project,
-                DOWNLOAD_REPO_NAME,
-                "https://artifacts.opensearch.org",
-                FAKE_IVY_GROUP,
-                "/releases" + RELEASE_PATTERN_LAYOUT,
-                "/release-candidates" + RELEASE_PATTERN_LAYOUT
-            );
-            addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://artifacts.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, SNAPSHOT_PATTERN_LAYOUT);
-        }
+        addIvyRepo(
+            project,
+            DOWNLOAD_REPO_NAME,
+            "https://artifacts.opensearch.org",
+            FAKE_IVY_GROUP,
+            "/releases" + RELEASE_PATTERN_LAYOUT,
+            "/release-candidates" + RELEASE_PATTERN_LAYOUT
+        );
+        addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://artifacts.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, SNAPSHOT_PATTERN_LAYOUT);
 
         addIvyRepo2(project, DOWNLOAD_REPO_NAME_ES, "https://artifacts-no-kpi.elastic.co", FAKE_IVY_GROUP_ES);
         addIvyRepo2(project, SNAPSHOT_REPO_NAME_ES, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP_ES);

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/LoggedExec.java b/buildSrc/src/main/java/org/opensearch/gradle/LoggedExec.java
@@ -64,7 +64,6 @@
 /**
  * A wrapper around gradle's Exec task to capture output and log on error.
  */
-@SuppressWarnings("unchecked")
 public class LoggedExec extends Exec implements FileSystemOperationsAware {
 
     private static final Logger LOGGER = Logging.getLogger(LoggedExec.class);

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchJavaPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchJavaPlugin.java
@@ -63,8 +63,8 @@
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
 
 import java.io.File;
+import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.function.Consumer;
 
 import static org.opensearch.gradle.util.Util.toStringable;
@@ -212,10 +212,12 @@ static void configureJars(Project project) {
                 public void execute(Task task) {
                     // this doFirst is added before the info plugin, therefore it will run
                     // after the doFirst added by the info plugin, and we can override attributes
-                    jarTask.getManifest()
-                        .attributes(
-                            Map.of("Build-Date", BuildParams.getBuildDate(), "Build-Java-Version", BuildParams.getGradleJavaVersion())
-                        );
+                    jarTask.getManifest().attributes(new HashMap<String, Object>() {
+                        {
+                            put("Build-Date", BuildParams.getBuildDate());
+                            put("Build-Java-Version", BuildParams.getGradleJavaVersion());
+                        }
+                    });
                 }
             });
         });

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java
@@ -49,6 +49,7 @@
 import org.gradle.api.tasks.testing.Test;
 
 import java.io.File;
+import java.util.HashMap;
 import java.util.Map;
 
 import static org.opensearch.gradle.util.FileUtils.mkdirs;
@@ -95,7 +96,7 @@ public void apply(Project project) {
 
             // We specifically use an anonymous inner class here because lambda task actions break Gradle cacheability
             // See: https://docs.gradle.org/current/userguide/more_about_tasks.html#sec:how_does_it_work
-            test.doFirst(new Action<>() {
+            test.doFirst(new Action<Task>() {
                 @Override
                 public void execute(Task t) {
                     mkdirs(testOutputDir);
@@ -137,20 +138,16 @@ public void execute(Task t) {
                 test.jvmArgs("-ea", "-esa");
             }
 
-            Map<String, String> sysprops = Map.of(
-                "java.awt.headless",
-                "true",
-                "tests.gradle",
-                "true",
-                "tests.artifact",
-                project.getName(),
-                "tests.task",
-                test.getPath(),
-                "tests.security.manager",
-                "true",
-                "jna.nosys",
-                "true"
-            );
+            Map<String, String> sysprops = new HashMap<String, String>() {
+                {
+                    put("java.awt.headless", "true");
+                    put("tests.gradle", "true");
+                    put("tests.artifact", project.getName());
+                    put("tests.task", test.getPath());
+                    put("tests.security.manager", "true");
+                    put("jna.nosys", "true");
+                }
+            };
             test.systemProperties(sysprops);
 
             // ignore changing test seed when build is passed -Dignore.tests.seed for cacheability experimentation

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/ReaperPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/ReaperPlugin.java
@@ -36,6 +36,7 @@
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 
+import java.lang.management.ManagementFactory;
 import java.nio.file.Path;
 
 /**
@@ -51,14 +52,31 @@ public void apply(Project project) {
 
         project.getPlugins().apply(GlobalBuildInfoPlugin.class);
 
-        Path inputDir = project.getRootDir()
-            .toPath()
-            .resolve(".gradle")
-            .resolve("reaper")
-            .resolve("build-" + ProcessHandle.current().pid());
+        Path inputDir = project.getRootDir().toPath().resolve(".gradle").resolve("reaper").resolve("build-" + getProcessId("xx"));
         ReaperService service = project.getExtensions()
             .create("reaper", ReaperService.class, project, project.getBuildDir().toPath(), inputDir);
 
         project.getGradle().buildFinished(result -> service.shutdown());
     }
+
+    private static String getProcessId(final String fallback) {
+        // Note: may fail in some JVM implementations
+        // therefore fallback has to be provided
+
+        // something like '<pid>@<hostname>', at least in SUN / Oracle JVMs
+        final String jvmName = ManagementFactory.getRuntimeMXBean().getName();
+        final int index = jvmName.indexOf('@');
+
+        if (index < 1) {
+            // part before '@' empty (index = 0) / '@' not found (index = -1)
+            return fallback;
+        }
+
+        try {
+            return Long.toString(Long.parseLong(jvmName.substring(0, index)));
+        } catch (NumberFormatException e) {
+            // ignore
+        }
+        return fallback;
+    }
 }
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/ReaperService.java b/buildSrc/src/main/java/org/opensearch/gradle/ReaperService.java
@@ -179,7 +179,11 @@ private Path locateReaperJar() {
                 InputStream jarInput = this.getClass().getResourceAsStream("/META-INF/reaper.jar");
             ) {
                 logger.info("Copying reaper.jar...");
-                jarInput.transferTo(out);
+                byte[] buffer = new byte[4096];
+                int len;
+                while ((len = jarInput.read(buffer)) > 0) {
+                    out.write(buffer, 0, len);
+                }
             } catch (IOException e) {
                 throw new UncheckedIOException(e);
             }

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java b/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java
@@ -49,6 +49,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -280,7 +281,7 @@ static Map<String, String> parseOsRelease(final List<String> osReleaseLines) {
      */
     private Optional<String> getDockerPath() {
         // Check if the Docker binary exists
-        return List.of(DOCKER_BINARIES).stream().filter(path -> new File(path).exists()).findFirst();
+        return Arrays.asList(DOCKER_BINARIES).stream().filter(path -> new File(path).exists()).findFirst();
     }
 
     /**
@@ -291,7 +292,7 @@ private Optional<String> getDockerPath() {
      */
     private Optional<String> getDockerComposePath() {
         // Check if the Docker binary exists
-        return List.of(DOCKER_COMPOSE_BINARIES).stream().filter(path -> new File(path).exists()).findFirst();
+        return Arrays.asList(DOCKER_COMPOSE_BINARIES).stream().filter(path -> new File(path).exists()).findFirst();
     }
 
     private void throwDockerRequiredException(final String message) {

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalBwcGitPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalBwcGitPlugin.java
@@ -109,7 +109,7 @@ public void apply(Project project) {
         });
 
         TaskProvider<LoggedExec> fetchLatestTaskProvider = tasks.register("fetchLatest", LoggedExec.class, fetchLatest -> {
-            var gitFetchLatest = project.getProviders()
+            Provider<Object> gitFetchLatest = project.getProviders()
                 .systemProperty("tests.bwc.git_fetch_latest")
                 .forUseAtConfigurationTime()
                 .orElse("true")
@@ -122,7 +122,7 @@ public void apply(Project project) {
                     }
                     throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + fetchProp + "]");
                 });
-            fetchLatest.onlyIf(t -> project.getGradle().getStartParameter().isOffline() == false && gitFetchLatest.get());
+            fetchLatest.onlyIf(t -> project.getGradle().getStartParameter().isOffline() == false && gitFetchLatest.get() != null);
             fetchLatest.dependsOn(addRemoteTaskProvider);
             fetchLatest.setWorkingDir(gitExtension.getCheckoutDir().get());
             fetchLatest.setCommandLine(asList("git", "fetch", "--all"));

diff --git a/.../src/main/java/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java b/.../src/main/java/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java
@@ -39,13 +39,15 @@
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
+import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
 import org.gradle.api.plugins.BasePlugin;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.api.tasks.AbstractCopyTask;
 import org.gradle.api.tasks.Sync;
 import org.gradle.api.tasks.TaskContainer;
+import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.bundling.AbstractArchiveTask;
 import org.gradle.api.tasks.bundling.Compression;
 import org.gradle.api.tasks.bundling.Zip;
@@ -105,14 +107,18 @@ private Action<Task> configure(String name) {
 
     private void registerAndConfigureDistributionArchivesExtension(Project project) {
         container = project.container(DistributionArchive.class, name -> {
-            var subProjectDir = archiveToSubprojectName(name);
-            var copyDistributionTaskName = "build" + capitalize(name.substring(0, name.length() - 3));
+            String subProjectDir = archiveToSubprojectName(name);
+            String copyDistributionTaskName = "build" + capitalize(name.substring(0, name.length() - 3));
             TaskContainer tasks = project.getTasks();
-            var explodedDist = tasks.register(copyDistributionTaskName, Sync.class, sync -> sync.into(subProjectDir + "/build/install/"));
+            TaskProvider<Sync> explodedDist = tasks.register(
+                copyDistributionTaskName,
+                Sync.class,
+                sync -> sync.into(subProjectDir + "/build/install/")
+            );
             explodedDist.configure(configure(name));
-            var archiveTaskName = "build" + capitalize(name);
+            String archiveTaskName = "build" + capitalize(name);
 
-            var archiveTask = name.endsWith("Tar")
+            TaskProvider<? extends AbstractArchiveTask> archiveTask = name.endsWith("Tar")
                 ? tasks.register(archiveTaskName, SymbolicLinkPreservingTar.class)
                 : tasks.register(archiveTaskName, Zip.class);
             archiveTask.configure(configure(name));
@@ -122,11 +128,11 @@ private void registerAndConfigureDistributionArchivesExtension(Project project)
         // Each defined distribution archive is linked to a subproject.
         // A distribution archive definition not matching a sub project will result in build failure.
         container.whenObjectAdded(distributionArchive -> {
-            var subProjectName = archiveToSubprojectName(distributionArchive.getName());
+            String subProjectName = archiveToSubprojectName(distributionArchive.getName());
             project.project(subProjectName, sub -> {
                 sub.getPlugins().apply(BasePlugin.class);
                 sub.getArtifacts().add(DEFAULT_CONFIGURATION_NAME, distributionArchive.getArchiveTask());
-                var extractedConfiguration = sub.getConfigurations().create("extracted");
+                Configuration extractedConfiguration = sub.getConfigurations().create("extracted");
                 extractedConfiguration.setCanBeResolved(false);
                 extractedConfiguration.getAttributes().attribute(ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
                 sub.getArtifacts().add(EXTRACTED_CONFIGURATION_NAME, distributionArchive.getExpandedDistTask());