Merge branch 'main' of https://github.com/opensearch-project/OpenSearch…

… into createpit

buildSrc/build.gradle

@@ -115,7 +115,7 @@ dependencies {
   api 'de.thetaphi:forbiddenapis:3.3'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.15.2'
   api 'org.apache.maven:maven-model:3.6.2'
-  api 'com.networknt:json-schema-validator:1.0.68'
+  api 'com.networknt:json-schema-validator:1.0.69'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
 
   testFixturesApi "junit:junit:${props.getProperty('junit')}"

buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java

@@ -125,18 +125,21 @@ public String call() throws Exception {
             // Add git origin info to generated POM files
             publication.getPom().withXml(PublishPlugin::addScmInfo);
 
-            // have to defer this until archivesBaseName is set
-            project.afterEvaluate(p -> publication.setArtifactId(getArchivesBaseName(project)));
+            if (!publication.getName().toLowerCase().contains("zip")) {
 
-            // publish sources and javadoc for Java projects.
-            if (project.getPluginManager().hasPlugin("opensearch.java")) {
-                publication.artifact(project.getTasks().getByName("sourcesJar"));
-                publication.artifact(project.getTasks().getByName("javadocJar"));
-            }
+                // have to defer this until archivesBaseName is set
+                project.afterEvaluate(p -> publication.setArtifactId(getArchivesBaseName(project)));
+
+                // publish sources and javadoc for Java projects.
+                if (project.getPluginManager().hasPlugin("opensearch.java")) {
+                    publication.artifact(project.getTasks().getByName("sourcesJar"));
+                    publication.artifact(project.getTasks().getByName("javadocJar"));
+                }
 
-            generatePomTask.configure(
-                t -> t.dependsOn(String.format("generatePomFileFor%sPublication", Util.capitalize(publication.getName())))
-            );
+                generatePomTask.configure(
+                    t -> t.dependsOn(String.format("generatePomFileFor%sPublication", Util.capitalize(publication.getName())))
+                );
+            }
         });
 
     }

buildSrc/src/main/java/org/opensearch/gradle/pluginzip/Publish.java

@@ -0,0 +1,74 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.gradle.pluginzip;
+
+import java.util.*;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.publish.PublishingExtension;
+import org.gradle.api.publish.maven.MavenPublication;
+import org.gradle.api.publish.maven.plugins.MavenPublishPlugin;
+import java.nio.file.Path;
+
+public class Publish implements Plugin<Project> {
+    private Project project;
+
+    public final static String EXTENSION_NAME = "zipmavensettings";
+    public final static String PUBLICATION_NAME = "pluginZip";
+    public final static String STAGING_REPO = "zipStaging";
+    public final static String PLUGIN_ZIP_PUBLISH_POM_TASK = "generatePomFileForPluginZipPublication";
+    public final static String LOCALMAVEN = "publishToMavenLocal";
+    public final static String LOCAL_STAGING_REPO_PATH = "/build/local-staging-repo";
+    public String zipDistributionLocation = "/build/distributions/";
+
+    public static void configMaven(Project project) {
+        final Path buildDirectory = project.getRootDir().toPath();
+        project.getPluginManager().apply(MavenPublishPlugin.class);
+        project.getExtensions().configure(PublishingExtension.class, publishing -> {
+            publishing.repositories(repositories -> {
+                repositories.maven(maven -> {
+                    maven.setName(STAGING_REPO);
+                    maven.setUrl(buildDirectory.toString() + LOCAL_STAGING_REPO_PATH);
+                });
+            });
+            publishing.publications(publications -> {
+                publications.create(PUBLICATION_NAME, MavenPublication.class, mavenZip -> {
+                    String zipGroup = "org.opensearch.plugin";
+                    String zipArtifact = project.getName();
+                    String zipVersion = getProperty("version", project);
+                    mavenZip.artifact(project.getTasks().named("bundlePlugin"));
+                    mavenZip.setGroupId(zipGroup);
+                    mavenZip.setArtifactId(zipArtifact);
+                    mavenZip.setVersion(zipVersion);
+                });
+            });
+        });
+    }
+
+    static String getProperty(String name, Project project) {
+        if (project.hasProperty(name)) {
+            Object property = project.property(name);
+            if (property != null) {
+                return property.toString();
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public void apply(Project project) {
+        this.project = project;
+        project.afterEvaluate(evaluatedProject -> { configMaven(project); });
+        project.getGradle().getTaskGraph().whenReady(graph -> {
+            if (graph.hasTask(LOCALMAVEN)) {
+                project.getTasks().getByName(PLUGIN_ZIP_PUBLISH_POM_TASK).setEnabled(false);
+            }
+
+        });
+    }
+}

buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.pluginzip.properties

@@ -0,0 +1 @@
+implementation-class=org.opensearch.gradle.pluginzip.Publish

buildSrc/src/main/resources/forbidden/opensearch-test-signatures.txt

@@ -26,4 +26,4 @@ com.carrotsearch.randomizedtesting.annotations.Nightly @ We don't run nightly te
 
 org.junit.Test @defaultMessage Just name your test method testFooBar
 
-java.lang.Math#random() @ Use one of the various randomization methods from LuceneTestCase or ESTestCase for reproducibility
+java.lang.Math#random() @ Use one of the various randomization methods from LuceneTestCase or OpenSearchTestCase for reproducibility

buildSrc/src/test/java/org/opensearch/gradle/pluginzip/PublishTests.java

@@ -0,0 +1,104 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gradle.pluginzip;
+
+import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.GradleRunner;
+import org.gradle.testfixtures.ProjectBuilder;
+import org.gradle.api.Project;
+import org.opensearch.gradle.test.GradleUnitTestCase;
+import org.junit.Test;
+import java.io.IOException;
+import org.gradle.api.publish.maven.tasks.PublishToMavenRepository;
+import java.io.File;
+import org.gradle.testkit.runner.BuildResult;
+import java.io.FileWriter;
+import java.io.Writer;
+import static org.gradle.testkit.runner.TaskOutcome.SUCCESS;
+import static org.junit.Assert.assertEquals;
+import java.nio.file.Files;
+import org.apache.maven.model.Model;
+import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
+import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
+import java.io.FileReader;
+import org.gradle.api.tasks.bundling.Zip;
+
+public class PublishTests extends GradleUnitTestCase {
+
+    @Test
+    public void testZipPublish() throws IOException, XmlPullParserException {
+        Project project = ProjectBuilder.builder().build();
+        String zipPublishTask = "publishPluginZipPublicationToZipStagingRepository";
+        // Apply the opensearch.pluginzip plugin
+        project.getPluginManager().apply("opensearch.pluginzip");
+        // Check if the plugin has been applied to the project
+        assertTrue(project.getPluginManager().hasPlugin("opensearch.pluginzip"));
+        // Check if the project has the task from class PublishToMavenRepository after plugin apply
+        assertNotNull(project.getTasks().withType(PublishToMavenRepository.class));
+        // Create a mock bundlePlugin task
+        Zip task = project.getTasks().create("bundlePlugin", Zip.class);
+        Publish.configMaven(project);
+        // Check if the main task publishPluginZipPublicationToZipStagingRepository exists after plugin apply
+        assertTrue(project.getTasks().getNames().contains(zipPublishTask));
+        assertNotNull("Task to generate: ", project.getTasks().getByName(zipPublishTask));
+        // Run Gradle functional tests, but calling a build.gradle file, that resembles the plugin publish behavior
+        File projectDir = new File("build/functionalTest");
+        // Create a sample plugin zip file
+        File sampleZip = new File("build/functionalTest/sample-plugin.zip");
+        Files.createDirectories(projectDir.toPath());
+        Files.createFile(sampleZip.toPath());
+        writeString(new File(projectDir, "settings.gradle"), "");
+        // Generate the build.gradle file
+        String buildFileContent = "apply plugin: 'maven-publish' \n"
+            + "publishing {\n"
+            + "  repositories {\n"
+            + "       maven {\n"
+            + "          url = 'local-staging-repo/'\n"
+            + "          name = 'zipStaging'\n"
+            + "        }\n"
+            + "  }\n"
+            + "   publications {\n"
+            + "         pluginZip(MavenPublication) {\n"
+            + "             groupId = 'org.opensearch.plugin' \n"
+            + "             artifactId = 'sample-plugin' \n"
+            + "             version = '2.0.0.0' \n"
+            + "             artifact('sample-plugin.zip') \n"
+            + "         }\n"
+            + "   }\n"
+            + "}";
+        writeString(new File(projectDir, "build.gradle"), buildFileContent);
+        // Execute the task publishPluginZipPublicationToZipStagingRepository
+        GradleRunner runner = GradleRunner.create();
+        runner.forwardOutput();
+        runner.withPluginClasspath();
+        runner.withArguments(zipPublishTask);
+        runner.withProjectDir(projectDir);
+        BuildResult result = runner.build();
+        // Check if task publishMavenzipPublicationToZipstagingRepository has ran well
+        assertEquals(SUCCESS, result.task(":" + zipPublishTask).getOutcome());
+        // check if the zip has been published to local staging repo
+        assertTrue(
+            new File("build/functionalTest/local-staging-repo/org/opensearch/plugin/sample-plugin/2.0.0.0/sample-plugin-2.0.0.0.zip")
+                .exists()
+        );
+        // Parse the maven file and validate the groupID to org.opensearch.plugin
+        MavenXpp3Reader reader = new MavenXpp3Reader();
+        Model model = reader.read(
+            new FileReader("build/functionalTest/local-staging-repo/org/opensearch/plugin/sample-plugin/2.0.0.0/sample-plugin-2.0.0.0.pom")
+        );
+        assertEquals(model.getGroupId(), "org.opensearch.plugin");
+    }
+
+    private void writeString(File file, String string) throws IOException {
+        try (Writer writer = new FileWriter(file)) {
+            writer.write(string);
+        }
+    }
+
+}

client/benchmark/README.md

@@ -29,7 +29,7 @@ Example invocation:
 wget http://benchmarks.elasticsearch.org.s3.amazonaws.com/corpora/geonames/documents-2.json.bz2
 bzip2 -d documents-2.json.bz2
 mv documents-2.json client/benchmark/build
-gradlew -p client/benchmark run --args ' rest bulk localhost build/documents-2.json geonames type 8647880 5000'
+gradlew -p client/benchmark run --args ' rest bulk localhost build/documents-2.json geonames 8647880 5000'
 ```
 
 The parameters are all in the `'`s and are in order:
@@ -39,7 +39,6 @@ The parameters are all in the `'`s and are in order:
 * Benchmark target host IP (the host where OpenSearch is running)
 * full path to the file that should be bulk indexed
 * name of the index
-* name of the (sole) type in the index
 * number of documents in the file
 * bulk size
 

client/benchmark/src/main/java/org/opensearch/client/benchmark/AbstractBenchmark.java

@@ -49,7 +49,7 @@ public abstract class AbstractBenchmark<T extends Closeable> {
 
     protected abstract T client(String benchmarkTargetHost) throws Exception;
 
-    protected abstract BulkRequestExecutor bulkRequestExecutor(T client, String indexName, String typeName);
+    protected abstract BulkRequestExecutor bulkRequestExecutor(T client, String indexName);
 
     protected abstract SearchRequestExecutor searchRequestExecutor(T client, String indexName);
 
@@ -76,16 +76,15 @@ public final void run(String[] args) throws Exception {
 
     @SuppressForbidden(reason = "system out is ok for a command line tool")
     private void runBulkIndexBenchmark(String[] args) throws Exception {
-        if (args.length != 7) {
-            System.err.println("usage: 'bulk' benchmarkTargetHostIp indexFilePath indexName typeName numberOfDocuments bulkSize");
+        if (args.length != 6) {
+            System.err.println("usage: 'bulk' benchmarkTargetHostIp indexFilePath indexName numberOfDocuments bulkSize");
             System.exit(1);
         }
         String benchmarkTargetHost = args[1];
         String indexFilePath = args[2];
         String indexName = args[3];
-        String typeName = args[4];
-        int totalDocs = Integer.valueOf(args[5]);
-        int bulkSize = Integer.valueOf(args[6]);
+        int totalDocs = Integer.valueOf(args[4]);
+        int bulkSize = Integer.valueOf(args[5]);
 
         int totalIterationCount = (int) Math.floor(totalDocs / bulkSize);
         // consider 40% of all iterations as warmup iterations
@@ -97,7 +96,7 @@ private void runBulkIndexBenchmark(String[] args) throws Exception {
         BenchmarkRunner benchmark = new BenchmarkRunner(
             warmupIterations,
             iterations,
-            new BulkBenchmarkTask(bulkRequestExecutor(client, indexName, typeName), indexFilePath, warmupIterations, iterations, bulkSize)
+            new BulkBenchmarkTask(bulkRequestExecutor(client, indexName), indexFilePath, warmupIterations, iterations, bulkSize)
         );
 
         try {

client/benchmark/src/main/java/org/opensearch/client/benchmark/rest/RestClientBenchmark.java

@@ -65,8 +65,8 @@ protected RestClient client(String benchmarkTargetHost) {
     }
 
     @Override
-    protected BulkRequestExecutor bulkRequestExecutor(RestClient client, String indexName, String typeName) {
-        return new RestBulkRequestExecutor(client, indexName, typeName);
+    protected BulkRequestExecutor bulkRequestExecutor(RestClient client, String indexName) {
+        return new RestBulkRequestExecutor(client, indexName);
     }
 
     @Override
@@ -78,9 +78,9 @@ private static final class RestBulkRequestExecutor implements BulkRequestExecuto
         private final RestClient client;
         private final String actionMetadata;
 
-        RestBulkRequestExecutor(RestClient client, String index, String type) {
+        RestBulkRequestExecutor(RestClient client, String index) {
             this.client = client;
-            this.actionMetadata = String.format(Locale.ROOT, "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }%n", index, type);
+            this.actionMetadata = String.format(Locale.ROOT, "{ \"index\" : { \"_index\" : \"%s\" } }%n", index);
         }
 
         @Override
@@ -91,7 +91,7 @@ public boolean bulkIndex(List<String> bulkData) {
                 bulkRequestBody.append(bulkItem);
                 bulkRequestBody.append("\n");
             }
-            Request request = new Request("POST", "/geonames/type/_noop_bulk");
+            Request request = new Request("POST", "/geonames/_noop_bulk");
             request.setJsonEntity(bulkRequestBody.toString());
             try {
                 Response response = client.performRequest(request);

client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java

@@ -895,8 +895,8 @@ Params withFields(String[] fields) {
          * @deprecated As of 2.0, because supporting inclusive language, replaced by {@link #withClusterManagerTimeout(TimeValue)}
          */
         @Deprecated
-        Params withMasterTimeout(TimeValue masterTimeout) {
-            return putParam("master_timeout", masterTimeout);
+        Params withMasterTimeout(TimeValue clusterManagerTimeout) {
+            return putParam("master_timeout", clusterManagerTimeout);
         }
 
         Params withClusterManagerTimeout(TimeValue clusterManagerTimeout) {

client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java

@@ -1917,6 +1917,10 @@ private <Req, Resp> Cancellable internalPerformRequestAsync(
         ActionListener<Resp> listener,
         Set<Integer> ignores
     ) {
+        if (listener == null) {
+            throw new IllegalArgumentException("The listener is required and cannot be null");
+        }
+
         Request req;
         try {
             req = requestConverter.apply(request);
@@ -2069,7 +2073,7 @@ protected final <Resp> Resp parseEntity(final HttpEntity entity, final CheckedFu
         if (entity.getContentType() == null) {
             throw new IllegalStateException("OpenSearch didn't return the [Content-Type] header, unable to parse response body");
         }
-        XContentType xContentType = XContentType.fromMediaTypeOrFormat(entity.getContentType().getValue());
+        XContentType xContentType = XContentType.fromMediaType(entity.getContentType().getValue());
         if (xContentType == null) {
             throw new IllegalStateException("Unsupported Content-Type: " + entity.getContentType().getValue());
         }

client/rest-high-level/src/main/java/org/opensearch/client/TimedRequest.java

@@ -47,7 +47,7 @@ public abstract class TimedRequest implements Validatable {
     public static final TimeValue DEFAULT_MASTER_NODE_TIMEOUT = TimeValue.timeValueSeconds(30);
 
     private TimeValue timeout = DEFAULT_ACK_TIMEOUT;
-    private TimeValue masterTimeout = DEFAULT_MASTER_NODE_TIMEOUT;
+    private TimeValue clusterManagerTimeout = DEFAULT_MASTER_NODE_TIMEOUT;
 
     /**
      * Sets the timeout to wait for the all the nodes to acknowledge
@@ -58,11 +58,11 @@ public void setTimeout(TimeValue timeout) {
     }
 
     /**
-     * Sets the timeout to connect to the master node
-     * @param masterTimeout timeout as a {@link TimeValue}
+     * Sets the timeout to connect to the cluster-manager node
+     * @param clusterManagerTimeout timeout as a {@link TimeValue}
      */
-    public void setMasterTimeout(TimeValue masterTimeout) {
-        this.masterTimeout = masterTimeout;
+    public void setMasterTimeout(TimeValue clusterManagerTimeout) {
+        this.clusterManagerTimeout = clusterManagerTimeout;
     }
 
     /**
@@ -73,9 +73,9 @@ public TimeValue timeout() {
     }
 
     /**
-     * Returns the timeout for the request to be completed on the master node
+     * Returns the timeout for the request to be completed on the cluster-manager node
      */
     public TimeValue masterNodeTimeout() {
-        return masterTimeout;
+        return clusterManagerTimeout;
     }
 }