Merge remote-tracking branch 'upstream/main' into fix_pipeline

opensearch-project · Mar 25, 2024 · a26af14 · a26af14
2 parents 481070f + 5b4b4aa
commit a26af14
Show file tree

Hide file tree

Showing 110 changed files with 395 additions and 117 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remote reindex: Add support for configurable retry mechanism ([#12561](https://github.com/opensearch-project/OpenSearch/pull/12561))
 - [Admission Control] Integrate IO Usage Tracker to the Resource Usage Collector Service and Emit IO Usage Stats ([#11880](https://github.com/opensearch-project/OpenSearch/pull/11880))
 - Tracing for deep search path ([#12103](https://github.com/opensearch-project/OpenSearch/pull/12103))
+- Add explicit dependency to validatePom and generatePom tasks ([#12103](https://github.com/opensearch-project/OpenSearch/pull/12807))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -105,6 +106,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Dependencies
 
 ### Changed
+- [BWC and API enforcement] Enforcing the presence of API annotations at build time ([#12872](https://github.com/opensearch-project/OpenSearch/pull/12872))
 
 ### Deprecated
 

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/pluginzip/Publish.java b/buildSrc/src/main/java/org/opensearch/gradle/pluginzip/Publish.java
@@ -65,9 +65,6 @@ public void apply(Project project) {
                 addLocalMavenRepo(project);
                 addZipArtifact(project);
                 Task validatePluginZipPom = project.getTasks().findByName("validatePluginZipPom");
-                if (validatePluginZipPom != null) {
-                    validatePluginZipPom.dependsOn("generatePomFileForNebulaPublication");
-                }
 
                 // There are number of tasks prefixed by 'publishPluginZipPublication', f.e.:
                 // publishPluginZipPublicationToZipStagingRepository, publishPluginZipPublicationToMavenLocal
@@ -76,7 +73,11 @@ public void apply(Project project) {
                     .filter(t -> t.getName().startsWith("publishPluginZipPublicationTo"))
                     .collect(Collectors.toSet());
                 if (!publishPluginZipPublicationToTasks.isEmpty()) {
-                    publishPluginZipPublicationToTasks.forEach(t -> t.dependsOn("generatePomFileForNebulaPublication"));
+                    if (validatePluginZipPom != null) {
+                        publishPluginZipPublicationToTasks.forEach(t -> t.dependsOn(validatePluginZipPom));
+                    } else {
+                        publishPluginZipPublicationToTasks.forEach(t -> t.dependsOn("generatePomFileForNebulaPublication"));
+                    }
                 }
             } else {
                 project.getLogger()

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationPrecommitPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationPrecommitPlugin.java
@@ -48,17 +48,24 @@ public class PomValidationPrecommitPlugin extends PrecommitPlugin {
     public TaskProvider<? extends Task> createTask(Project project) {
         TaskProvider<Task> validatePom = project.getTasks().register("validatePom");
         PublishingExtension publishing = project.getExtensions().getByType(PublishingExtension.class);
-        publishing.getPublications().all(publication -> {
+        publishing.getPublications().configureEach(publication -> {
             String publicationName = Util.capitalize(publication.getName());
             TaskProvider<PomValidationTask> validateTask = project.getTasks()
                 .register("validate" + publicationName + "Pom", PomValidationTask.class);
             validatePom.configure(t -> t.dependsOn(validateTask));
+            TaskProvider<GenerateMavenPom> generateMavenPom = project.getTasks()
+                .withType(GenerateMavenPom.class)
+                .named("generatePomFileFor" + publicationName + "Publication");
             validateTask.configure(task -> {
-                GenerateMavenPom generateMavenPom = project.getTasks()
-                    .withType(GenerateMavenPom.class)
-                    .getByName("generatePomFileFor" + publicationName + "Publication");
                 task.dependsOn(generateMavenPom);
-                task.getPomFile().fileValue(generateMavenPom.getDestination());
+                task.getPomFile().fileProvider(generateMavenPom.map(GenerateMavenPom::getDestination));
+                publishing.getPublications().configureEach(publicationForPomGen -> {
+                    task.mustRunAfter(
+                        project.getTasks()
+                            .withType(GenerateMavenPom.class)
+                            .getByName("generatePomFileFor" + Util.capitalize(publicationForPomGen.getName()) + "Publication")
+                    );
+                });
             });
         });
 

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationTask.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationTask.java
@@ -106,6 +106,7 @@ private <T> void validateNonNull(String element, T value, Runnable validator) {
 
     private void validateString(String element, String value) {
         validateNonNull(element, value, () -> validateNonEmpty(element, value, s -> s.trim().isEmpty()));
+        getLogger().info(element + " with value " + value + " is validated.");
     }
 
     private <T> void validateCollection(String element, Collection<T> value, Consumer<T> validator) {

diff --git a/libs/core/src/main/java/org/opensearch/OpenSearchException.java b/libs/core/src/main/java/org/opensearch/OpenSearchException.java
@@ -33,6 +33,7 @@
 
 import org.opensearch.common.CheckedFunction;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
@@ -69,8 +70,9 @@
 /**
  * A core library base class for all opensearch exceptions.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class OpenSearchException extends RuntimeException implements Writeable, ToXContentFragment {
 
     protected static final Version UNKNOWN_VERSION_ADDED = Version.fromId(0);

diff --git a/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java
@@ -32,14 +32,17 @@
 
 package org.opensearch.core.common.breaker;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.Locale;
 
 /**
  * Interface for an object that can be incremented, breaking after some
  * configured limit has been reached.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface CircuitBreaker {
 
     /**
@@ -72,8 +75,10 @@ public interface CircuitBreaker {
     /**
      * The type of breaker
      * can be {@link #MEMORY}, {@link #PARENT}, or {@link #NOOP}
-     * @opensearch.internal
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     enum Type {
         /** A regular or ChildMemoryCircuitBreaker */
         MEMORY,

diff --git a/libs/core/src/main/java/org/opensearch/core/common/transport/BoundTransportAddress.java b/libs/core/src/main/java/org/opensearch/core/common/transport/BoundTransportAddress.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.core.common.transport;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.network.InetAddresses;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -44,8 +45,9 @@
  * the addresses the transport is bound to, and the other is the published one that represents the address clients
  * should communicate on.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class BoundTransportAddress implements Writeable {
 
     private TransportAddress[] boundAddresses;

diff --git a/libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java b/libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.core.transport;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -40,8 +41,9 @@
 /**
  * Response over the transport interface
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class TransportResponse extends TransportMessage {
 
     /**

diff --git a/libs/core/src/main/java/org/opensearch/semver/SemverRange.java b/libs/core/src/main/java/org/opensearch/semver/SemverRange.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.semver.expr.Caret;
@@ -31,7 +32,10 @@
  *     <li>'~' Allows for patch version variability starting from the range version. For example, ~1.2.3 range would match versions greater than or equal to 1.2.3 but less than 1.3.0</li>
  *     <li>'^' Allows for patch and minor version variability starting from the range version. For example, ^1.2.3 range would match versions greater than or equal to 1.2.3 but less than 2.0.0</li>
  * </ul>
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.13.0")
 public class SemverRange implements ToXContentFragment {
 
     private final Version rangeVersion;

diff --git a/release-notes/opensearch.release-notes-2.13.0.md b/release-notes/opensearch.release-notes-2.13.0.md
@@ -26,6 +26,7 @@
 - Add Remote Store Migration Experimental flag and allow mixed mode clusters under same ([#11986](https://github.com/opensearch-project/OpenSearch/pull/11986))
 - Built-in secure transports support ([#12435](https://github.com/opensearch-project/OpenSearch/pull/12435))
 - Lightweight Transport action to verify local term before fetching cluster-state from remote ([#12252](https://github.com/opensearch-project/OpenSearch/pull/12252/))
+- Integrate with admission controller for cluster-manager Read API. ([#12496](https://github.com/opensearch-project/OpenSearch/pull/12496))
 
 ### Dependencies
 - Bump `com.squareup.okio:okio` from 3.7.0 to 3.8.0 ([#12290](https://github.com/opensearch-project/OpenSearch/pull/12290))

diff --git a/server/build.gradle b/server/build.gradle
@@ -141,7 +141,7 @@ tasks.withType(JavaCompile).configureEach {
 
 compileJava {
   options.compilerArgs += ['-processor', ['org.apache.logging.log4j.core.config.plugins.processor.PluginProcessor',
-    'org.opensearch.common.annotation.processor.ApiAnnotationProcessor'].join(','), '-AcontinueOnFailingChecks']
+    'org.opensearch.common.annotation.processor.ApiAnnotationProcessor'].join(',')]
 }
 
 tasks.named("internalClusterTest").configure {

diff --git a/...h/cluster/state/FetchByTermVersionIT.java → ...termanager/term/FetchByTermVersionIT.java b/...h/cluster/state/FetchByTermVersionIT.java → ...termanager/term/FetchByTermVersionIT.java
@@ -6,12 +6,10 @@
  * compatible open source license.
  */
 
-package org.opensearch.cluster.state;
+package org.opensearch.action.support.clustermanager.term;
 
 import org.opensearch.action.admin.cluster.state.ClusterStateRequest;
 import org.opensearch.action.admin.cluster.state.ClusterStateResponse;
-import org.opensearch.action.admin.cluster.state.term.GetTermVersionAction;
-import org.opensearch.action.admin.cluster.state.term.GetTermVersionResponse;
 import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.coordination.ClusterStateTermVersion;

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedExplainIT.java b/server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedExplainIT.java
@@ -30,6 +30,11 @@
  */
 public class SimpleNestedExplainIT extends OpenSearchIntegTestCase {
 
+    @Override
+    protected int numberOfShards() {
+        return 1;
+    }
+
     /*
      * Tests the explain output for multiple docs. Concurrent search with multiple slices is tested
      * here as call to indexRandomForMultipleSlices is made and compared with explain output for
@@ -70,7 +75,23 @@ public void testExplainMultipleDocs() throws Exception {
             .setRefreshPolicy(IMMEDIATE)
             .get();
 
-        indexRandomForMultipleSlices("test");
+        client().prepareIndex("test")
+            .setId("2")
+            .setSource(
+                jsonBuilder().startObject()
+                    .field("field1", "value2")
+                    .startArray("nested1")
+                    .startObject()
+                    .field("n_field1", "n_value2")
+                    .endObject()
+                    .startObject()
+                    .field("n_field1", "n_value2")
+                    .endObject()
+                    .endArray()
+                    .endObject()
+            )
+            .setRefreshPolicy(IMMEDIATE)
+            .get();
 
         // Turn off the concurrent search setting to test search with non-concurrent search
         client().admin()

diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java
@@ -107,8 +107,6 @@
 import org.opensearch.action.admin.cluster.snapshots.status.TransportSnapshotsStatusAction;
 import org.opensearch.action.admin.cluster.state.ClusterStateAction;
 import org.opensearch.action.admin.cluster.state.TransportClusterStateAction;
-import org.opensearch.action.admin.cluster.state.term.GetTermVersionAction;
-import org.opensearch.action.admin.cluster.state.term.TransportGetTermVersionAction;
 import org.opensearch.action.admin.cluster.stats.ClusterStatsAction;
 import org.opensearch.action.admin.cluster.stats.TransportClusterStatsAction;
 import org.opensearch.action.admin.cluster.storedscripts.DeleteStoredScriptAction;
@@ -283,6 +281,8 @@
 import org.opensearch.action.support.AutoCreateIndex;
 import org.opensearch.action.support.DestructiveOperations;
 import org.opensearch.action.support.TransportAction;
+import org.opensearch.action.support.clustermanager.term.GetTermVersionAction;
+import org.opensearch.action.support.clustermanager.term.TransportGetTermVersionAction;
 import org.opensearch.action.termvectors.MultiTermVectorsAction;
 import org.opensearch.action.termvectors.TermVectorsAction;
 import org.opensearch.action.termvectors.TransportMultiTermVectorsAction;
@@ -294,6 +294,7 @@
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.common.NamedRegistry;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.AbstractModule;
 import org.opensearch.common.inject.TypeLiteral;
 import org.opensearch.common.inject.multibindings.MapBinder;
@@ -1052,8 +1053,9 @@ public RestController getRestController() {
      * <p>
      * This class is modeled after {@link NamedRegistry} but provides both register and unregister capabilities.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "2.7.0")
     public static class DynamicActionRegistry {
         // This is the unmodifiable actions map created during node bootstrap, which
         // will continue to link ActionType and TransportAction pairs from core and plugin

diff --git a/server/src/main/java/org/opensearch/action/ActionRequest.java b/server/src/main/java/org/opensearch/action/ActionRequest.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.transport.TransportRequest;
@@ -43,6 +44,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class ActionRequest extends TransportRequest {
 
     public ActionRequest() {

diff --git a/server/src/main/java/org/opensearch/action/ActionType.java b/server/src/main/java/org/opensearch/action/ActionType.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -43,6 +44,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ActionType<Response extends ActionResponse> {
 
     private final String name;

diff --git a/server/src/main/java/org/opensearch/action/support/TransportAction.java b/server/src/main/java/org/opensearch/action/support/TransportAction.java
@@ -36,6 +36,7 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lease.Releasables;
 import org.opensearch.common.util.concurrent.ThreadContext;
@@ -52,8 +53,9 @@
 /**
  * Base class for a transport action
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class TransportAction<Request extends ActionRequest, Response extends ActionResponse> {
 
     public final String actionName;

diff --git a/...src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeRequest.java b/...src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.support.clustermanager;
 
 import org.opensearch.action.ActionRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -42,8 +43,9 @@
 /**
  * A based request for cluster-manager based operation.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class ClusterManagerNodeRequest<Request extends ClusterManagerNodeRequest<Request>> extends ActionRequest {
 
     public static final TimeValue DEFAULT_CLUSTER_MANAGER_NODE_TIMEOUT = TimeValue.timeValueSeconds(30);

diff --git a/.../java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java b/.../java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java
@@ -37,13 +37,13 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.action.ActionListenerResponseHandler;
 import org.opensearch.action.ActionRunnable;
-import org.opensearch.action.admin.cluster.state.term.GetTermVersionAction;
-import org.opensearch.action.admin.cluster.state.term.GetTermVersionRequest;
-import org.opensearch.action.admin.cluster.state.term.GetTermVersionResponse;
 import org.opensearch.action.bulk.BackoffPolicy;
 import org.opensearch.action.support.ActionFilters;
 import org.opensearch.action.support.HandledTransportAction;
 import org.opensearch.action.support.RetryableAction;
+import org.opensearch.action.support.clustermanager.term.GetTermVersionAction;
+import org.opensearch.action.support.clustermanager.term.GetTermVersionRequest;
+import org.opensearch.action.support.clustermanager.term.GetTermVersionResponse;
 import org.opensearch.cluster.ClusterManagerNodeChangePredicate;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateObserver;

diff --git a/...ster/state/term/GetTermVersionAction.java → ...termanager/term/GetTermVersionAction.java b/...ster/state/term/GetTermVersionAction.java → ...termanager/term/GetTermVersionAction.java
@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.action.admin.cluster.state.term;
+package org.opensearch.action.support.clustermanager.term;
 
 import org.opensearch.action.ActionType;
 
@@ -18,7 +18,7 @@
 public class GetTermVersionAction extends ActionType<GetTermVersionResponse> {
 
     public static final GetTermVersionAction INSTANCE = new GetTermVersionAction();
-    public static final String NAME = "cluster:monitor/term";
+    public static final String NAME = "internal:monitor/term";
 
     private GetTermVersionAction() {
         super(NAME, GetTermVersionResponse::new);