Merge branch 'main' into unset-keystore-password

Signed-off-by: Craig Perkins <[email protected]>

CHANGELOG.md

@@ -19,6 +19,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remote reindex: Add support for configurable retry mechanism ([#12561](https://github.com/opensearch-project/OpenSearch/pull/12561))
 - [Admission Control] Integrate IO Usage Tracker to the Resource Usage Collector Service and Emit IO Usage Stats ([#11880](https://github.com/opensearch-project/OpenSearch/pull/11880))
 - Tracing for deep search path ([#12103](https://github.com/opensearch-project/OpenSearch/pull/12103))
+- Add explicit dependency to validatePom and generatePom tasks ([#12807](https://github.com/opensearch-project/OpenSearch/pull/12807))
+- Replace configureEach with all for publication iteration ([#12876](https://github.com/opensearch-project/OpenSearch/pull/12876))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -101,19 +103,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- Add a counter to node stat api to track shard going from idle to non-idle ([#12768](https://github.com/opensearch-project/OpenSearch/pull/12768))
 - Allow setting KEYSTORE_PASSWORD through env variable ([#12865](https://github.com/opensearch-project/OpenSearch/pull/12865))
 
 ### Dependencies
+- Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896))
+- Bump `asm` from 9.6 to 9.7 ([#12908](https://github.com/opensearch-project/OpenSearch/pull/12908))
+- Bump `net.minidev:json-smart` from 2.5.0 to 2.5.1 ([#12893](https://github.com/opensearch-project/OpenSearch/pull/12893))
+- Bump `netty` from 4.1.107.Final to 4.1.108.Final ([#12924](https://github.com/opensearch-project/OpenSearch/pull/12924))
 
 ### Changed
+- [BWC and API enforcement] Enforcing the presence of API annotations at build time ([#12872](https://github.com/opensearch-project/OpenSearch/pull/12872))
 
 ### Deprecated
 
 ### Removed
 
 ### Fixed
+- Fix issue with feature flags where default value may not be honored ([#12849](https://github.com/opensearch-project/OpenSearch/pull/12849))
 
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.13...2.x

README.md

@@ -7,8 +7,8 @@
 [![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability")
 [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues)
 [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls)
-[![2.10 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.10.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.10.0")
-[![3.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
+[![2.14.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.14.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.14.0")
+[![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
 [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml)
 [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml)
 [![GHA precommit](https://github.com/opensearch-project/OpenSearch/actions/workflows/precommit.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/precommit.yml)

buildSrc/src/main/java/org/opensearch/gradle/pluginzip/Publish.java

@@ -65,9 +65,6 @@ public void apply(Project project) {
                 addLocalMavenRepo(project);
                 addZipArtifact(project);
                 Task validatePluginZipPom = project.getTasks().findByName("validatePluginZipPom");
-                if (validatePluginZipPom != null) {
-                    validatePluginZipPom.dependsOn("generatePomFileForNebulaPublication");
-                }
 
                 // There are number of tasks prefixed by 'publishPluginZipPublication', f.e.:
                 // publishPluginZipPublicationToZipStagingRepository, publishPluginZipPublicationToMavenLocal
@@ -76,7 +73,11 @@ public void apply(Project project) {
                     .filter(t -> t.getName().startsWith("publishPluginZipPublicationTo"))
                     .collect(Collectors.toSet());
                 if (!publishPluginZipPublicationToTasks.isEmpty()) {
-                    publishPluginZipPublicationToTasks.forEach(t -> t.dependsOn("generatePomFileForNebulaPublication"));
+                    if (validatePluginZipPom != null) {
+                        publishPluginZipPublicationToTasks.forEach(t -> t.dependsOn(validatePluginZipPom));
+                    } else {
+                        publishPluginZipPublicationToTasks.forEach(t -> t.dependsOn("generatePomFileForNebulaPublication"));
+                    }
                 }
             } else {
                 project.getLogger()

buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationPrecommitPlugin.java

@@ -53,12 +53,19 @@ public TaskProvider<? extends Task> createTask(Project project) {
             TaskProvider<PomValidationTask> validateTask = project.getTasks()
                 .register("validate" + publicationName + "Pom", PomValidationTask.class);
             validatePom.configure(t -> t.dependsOn(validateTask));
+            TaskProvider<GenerateMavenPom> generateMavenPom = project.getTasks()
+                .withType(GenerateMavenPom.class)
+                .named("generatePomFileFor" + publicationName + "Publication");
             validateTask.configure(task -> {
-                GenerateMavenPom generateMavenPom = project.getTasks()
-                    .withType(GenerateMavenPom.class)
-                    .getByName("generatePomFileFor" + publicationName + "Publication");
                 task.dependsOn(generateMavenPom);
-                task.getPomFile().fileValue(generateMavenPom.getDestination());
+                task.getPomFile().fileProvider(generateMavenPom.map(GenerateMavenPom::getDestination));
+                publishing.getPublications().all(publicationForPomGen -> {
+                    task.mustRunAfter(
+                        project.getTasks()
+                            .withType(GenerateMavenPom.class)
+                            .getByName("generatePomFileFor" + Util.capitalize(publicationForPomGen.getName()) + "Publication")
+                    );
+                });
             });
         });
 

buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationTask.java

@@ -106,6 +106,7 @@ private <T> void validateNonNull(String element, T value, Runnable validator) {
 
     private void validateString(String element, String value) {
         validateNonNull(element, value, () -> validateNonEmpty(element, value, s -> s.trim().isEmpty()));
+        getLogger().info(element + " with value " + value + " is validated.");
     }
 
     private <T> void validateCollection(String element, Collection<T> value, Consumer<T> validator) {

buildSrc/src/testFixtures/java/org/opensearch/gradle/test/GradleThreadsFilter.java

@@ -45,6 +45,8 @@ public class GradleThreadsFilter implements ThreadFilter {
     public boolean reject(Thread t) {
         return t.getName().startsWith("Exec process")
             || t.getName().startsWith("Memory manager")
-            || t.getName().startsWith("File watcher consumer");
+            || t.getName().startsWith("File watcher consumer")
+            || t.getName().startsWith("sshd-SshClient") /* Started by SshClient (sshd-core), part of SftpFileSystemProvider */
+            || t.getName().startsWith("Thread-"); /* Started by AbstractFactoryManager (sshd-core), part of SftpFileSystemProvider */
     }
 }

buildSrc/version.properties

@@ -14,7 +14,7 @@ icu4j             = 70.1
 supercsv          = 2.4.0
 log4j             = 2.21.0
 slf4j             = 1.7.36
-asm               = 9.6
+asm               = 9.7
 jettison          = 1.5.4
 woodstox          = 6.4.0
 kotlin            = 1.7.10
@@ -26,7 +26,7 @@ jakarta_annotation = 1.3.5
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.13.0
 
-netty             = 4.1.107.Final
+netty             = 4.1.108.Final
 joda              = 2.12.2
 
 # project reactor

gradle/wrapper/gradle-wrapper.properties

@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=85719317abd2112f021d4f41f09ec370534ba288432065f4b477b6a3b652910d
+distributionSha256Sum=194717442575a6f96e1c1befa2c30e9a4fc90f701d7aee33eb879b79e7ff05c0

libs/core/src/main/java/org/opensearch/OpenSearchException.java

@@ -33,6 +33,7 @@
 
 import org.opensearch.common.CheckedFunction;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
@@ -69,8 +70,9 @@
 /**
  * A core library base class for all opensearch exceptions.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class OpenSearchException extends RuntimeException implements Writeable, ToXContentFragment {
 
     protected static final Version UNKNOWN_VERSION_ADDED = Version.fromId(0);

libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java

@@ -32,14 +32,17 @@
 
 package org.opensearch.core.common.breaker;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.Locale;
 
 /**
  * Interface for an object that can be incremented, breaking after some
  * configured limit has been reached.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface CircuitBreaker {
 
     /**
@@ -72,8 +75,10 @@ public interface CircuitBreaker {
     /**
      * The type of breaker
      * can be {@link #MEMORY}, {@link #PARENT}, or {@link #NOOP}
-     * @opensearch.internal
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     enum Type {
         /** A regular or ChildMemoryCircuitBreaker */
         MEMORY,

libs/core/src/main/java/org/opensearch/core/common/transport/BoundTransportAddress.java

@@ -32,6 +32,7 @@
 
 package org.opensearch.core.common.transport;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.network.InetAddresses;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -44,8 +45,9 @@
  * the addresses the transport is bound to, and the other is the published one that represents the address clients
  * should communicate on.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class BoundTransportAddress implements Writeable {
 
     private TransportAddress[] boundAddresses;

libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java

@@ -32,6 +32,7 @@
 
 package org.opensearch.core.transport;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -40,8 +41,9 @@
 /**
  * Response over the transport interface
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class TransportResponse extends TransportMessage {
 
     /**

libs/core/src/main/java/org/opensearch/semver/SemverRange.java

@@ -10,6 +10,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.semver.expr.Caret;
@@ -31,7 +32,10 @@
  *     <li>'~' Allows for patch version variability starting from the range version. For example, ~1.2.3 range would match versions greater than or equal to 1.2.3 but less than 1.3.0</li>
  *     <li>'^' Allows for patch and minor version variability starting from the range version. For example, ^1.2.3 range would match versions greater than or equal to 1.2.3 but less than 2.0.0</li>
  * </ul>
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.13.0")
 public class SemverRange implements ToXContentFragment {
 
     private final Version rangeVersion;

modules/ingest-common/src/test/java/org/opensearch/ingest/common/CopyProcessorTests.java

@@ -24,7 +24,7 @@ public class CopyProcessorTests extends OpenSearchTestCase {
     public void testCopyExistingField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String sourceFieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
-        String targetFieldName = RandomDocumentPicks.randomFieldName(random());
+        String targetFieldName = RandomDocumentPicks.randomNonExistingFieldName(random(), ingestDocument);
         Processor processor = createCopyProcessor(sourceFieldName, targetFieldName, false, false, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(targetFieldName), equalTo(true));
@@ -71,7 +71,8 @@ public void testCopyWithIgnoreMissing() throws Exception {
     public void testCopyWithRemoveSource() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String sourceFieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
-        String targetFieldName = RandomDocumentPicks.randomFieldName(random());
+        String targetFieldName = RandomDocumentPicks.randomNonExistingFieldName(random(), ingestDocument);
+
         Object sourceValue = ingestDocument.getFieldValue(sourceFieldName, Object.class);
 
         Processor processor = createCopyProcessor(sourceFieldName, targetFieldName, false, true, false);

modules/lang-expression/licenses/asm-9.7.jar.sha1

@@ -0,0 +1 @@
+073d7b3086e14beb604ced229c302feff6449723

modules/lang-expression/licenses/asm-commons-9.7.jar.sha1

@@ -0,0 +1 @@
+e86dda4696d3c185fcc95d8d311904e7ce38a53f

modules/lang-expression/licenses/asm-tree-9.7.jar.sha1

@@ -0,0 +1 @@
+e446a17b175bfb733b87c5c2560ccb4e57d69f1a

modules/lang-painless/licenses/asm-9.7.jar.sha1

@@ -0,0 +1 @@
+073d7b3086e14beb604ced229c302feff6449723

modules/lang-painless/licenses/asm-analysis-9.7.jar.sha1

@@ -0,0 +1 @@
+e4a258b7eb96107106c0599f0061cfc1832fe07a

modules/lang-painless/licenses/asm-commons-9.7.jar.sha1

@@ -0,0 +1 @@
+e86dda4696d3c185fcc95d8d311904e7ce38a53f

modules/lang-painless/licenses/asm-tree-9.7.jar.sha1

@@ -0,0 +1 @@
+e446a17b175bfb733b87c5c2560ccb4e57d69f1a

modules/lang-painless/licenses/asm-util-9.7.jar.sha1

@@ -0,0 +1 @@
+c0655519f24d92af2202cb681cd7c1569df6ead6

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/71_context_api.yml

@@ -2,7 +2,7 @@
     - do:
         scripts_painless_context: {}
     - match: { contexts.0: aggregation_selector}
-    - match: { contexts.23: update}
+    - match: { contexts.24: update}
 ---
 
 "Action to get all API values for score context":

modules/transport-netty4/licenses/netty-buffer-4.1.108.Final.jar.sha1

@@ -0,0 +1 @@
+2a9d06026ed251705e6ab52fa6ebe5f4f15aab7a

modules/transport-netty4/licenses/netty-codec-4.1.108.Final.jar.sha1

@@ -0,0 +1 @@
+c2ef6018eecde345fcddb96e31f651df16dca4c2

modules/transport-netty4/licenses/netty-codec-http-4.1.108.Final.jar.sha1

@@ -0,0 +1 @@
+dd44733e94f3f6237c896f2bbe9927c1eba48543

modules/transport-netty4/licenses/netty-codec-http2-4.1.108.Final.jar.sha1

@@ -0,0 +1 @@
+ed90430e545529a2df7c1db6c94568ea00867a61

modules/transport-netty4/licenses/netty-common-4.1.108.Final.jar.sha1

@@ -0,0 +1 @@
+30617b39cc6f850ca3807459fe726fbcd63989f2

modules/transport-netty4/licenses/netty-handler-4.1.108.Final.jar.sha1

@@ -0,0 +1 @@
+d186a0be320e6a139c42d9b018596ef9d4a0b4ca

modules/transport-netty4/licenses/netty-resolver-4.1.108.Final.jar.sha1

@@ -0,0 +1 @@
+f3085568e45c2ca74118118f792d0d55968aeb13

modules/transport-netty4/licenses/netty-transport-4.1.108.Final.jar.sha1

@@ -0,0 +1 @@
+1fd80f714c85ca685a80f32e0a4e8fd3b866e310

modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.108.Final.jar.sha1

@@ -0,0 +1 @@
+0df31f1cd96df8b2882b1e0faf4409b0bd704541

plugins/repository-azure/licenses/netty-codec-dns-4.1.108.Final.jar.sha1

@@ -0,0 +1 @@
+93cc78652ed836ef950604139bfb4afb45e0bc7b

plugins/repository-azure/licenses/netty-codec-http2-4.1.108.Final.jar.sha1

@@ -0,0 +1 @@
+ed90430e545529a2df7c1db6c94568ea00867a61

plugins/repository-azure/licenses/netty-codec-socks-4.1.108.Final.jar.sha1

@@ -0,0 +1 @@
+3ad0af28e408092f0d12994802a9f3fe18d45f8c