Merge branch 'main' into fix_remote_shards_balancer

CHANGELOG.md

@@ -103,7 +103,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add search query categorizer ([#10255](https://github.com/opensearch-project/OpenSearch/pull/10255))
 - Per request phase latency ([#10351](https://github.com/opensearch-project/OpenSearch/issues/10351))
 - Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670))
-- Remove ingest processor supports excluding fields ([#10967](https://github.com/opensearch-project/OpenSearch/pull/10967))
+- Remove ingest processor supports excluding fields ([#10967](https://github.com/opensearch-project/OpenSearch/pull/10967), [#11983](https://github.com/opensearch-project/OpenSearch/pull/11983))
 - [Tiered caching] Enabling serialization for IndicesRequestCache key object ([#10275](https://github.com/opensearch-project/OpenSearch/pull/10275))
 - [Tiered caching] Defining interfaces, listeners and extending IndicesRequestCache with Tiered cache support ([#10753](https://github.com/opensearch-project/OpenSearch/pull/10753))
 - [Remote cluster state] Restore cluster state version during remote state auto restore ([#10853](https://github.com/opensearch-project/OpenSearch/pull/10853))
@@ -209,7 +209,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Use slice_size == shard_size heuristic in terms aggs for concurrent segment search and properly calculate the doc_count_error ([#11732](https://github.com/opensearch-project/OpenSearch/pull/11732))
 - Added Support for dynamically adding SearchRequestOperationsListeners with SearchRequestOperationsCompositeListenerFactory ([#11526](https://github.com/opensearch-project/OpenSearch/pull/11526))
 - Ensure Jackson default maximums introduced in 2.16.0 do not conflict with OpenSearch settings ([#11890](https://github.com/opensearch-project/OpenSearch/pull/11890))
-- Extract cluster management for integration tests into JUnit test rule out of OpenSearchIntegTestCase ([#11877](https://github.com/opensearch-project/OpenSearch/pull/11877))
+- Extract cluster management for integration tests into JUnit test rule out of OpenSearchIntegTestCase ([#11877](https://github.com/opensearch-project/OpenSearch/pull/11877)), ([#12000](https://github.com/opensearch-project/OpenSearch/pull/12000))
+- Workaround for https://bugs.openjdk.org/browse/JDK-8323659 regression, introduced in JDK-21.0.2 ([#11968](https://github.com/opensearch-project/OpenSearch/pull/11968))
 
 ### Deprecated
 

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -77,9 +77,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "17.0.9+9";
+    private static final String SYSTEM_JDK_VERSION = "21.0.2+13";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "17.0.9+9";
+    private static final String GRADLE_JDK_VERSION = "21.0.2+13";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

buildSrc/version.properties

@@ -2,7 +2,7 @@ opensearch        = 3.0.0
 lucene            = 9.9.1
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 21.0.1+12
+bundled_jdk = 21.0.2+13
 
 # optional dependencies
 spatial4j         = 0.7

modules/ingest-common/src/main/java/org/opensearch/ingest/common/RemoveProcessor.java

@@ -72,7 +72,7 @@ public final class RemoveProcessor extends AbstractProcessor {
     ) {
         super(tag, description);
         if (fields == null && excludeFields == null || fields != null && excludeFields != null) {
-            throw new IllegalArgumentException("ether fields and excludeFields must be set");
+            throw new IllegalArgumentException("either fields or excludeFields must be set");
         }
         if (fields != null) {
             this.fields = new ArrayList<>(fields);
@@ -188,7 +188,7 @@ public RemoveProcessor create(
             final Object excludeField = ConfigurationUtils.readOptionalObject(config, "exclude_field");
 
             if (field == null && excludeField == null || field != null && excludeField != null) {
-                throw newConfigurationException(TYPE, processorTag, "field", "ether field or exclude_field must be set");
+                throw newConfigurationException(TYPE, processorTag, "field", "either field or exclude_field must be set");
             }
 
             boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);

modules/ingest-common/src/test/java/org/opensearch/ingest/common/CopyProcessorTests.java

@@ -14,6 +14,9 @@
 import org.opensearch.ingest.TestTemplateService;
 import org.opensearch.test.OpenSearchTestCase;
 
+import java.util.List;
+import java.util.Map;
+
 import static org.hamcrest.Matchers.equalTo;
 
 public class CopyProcessorTests extends OpenSearchTestCase {
@@ -26,8 +29,7 @@ public void testCopyExistingField() throws Exception {
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(targetFieldName), equalTo(true));
         Object sourceValue = ingestDocument.getFieldValue(sourceFieldName, Object.class);
-        assertThat(ingestDocument.getFieldValue(targetFieldName, Object.class), equalTo(sourceValue));
-        assertThat(ingestDocument.getFieldValue(sourceFieldName, Object.class), equalTo(sourceValue));
+        assertDeepCopiedObjectEquals(ingestDocument.getFieldValue(targetFieldName, Object.class), sourceValue);
 
         Processor processorWithEmptyTarget = createCopyProcessor(sourceFieldName, "", false, false, false);
         assertThrows(
@@ -75,7 +77,7 @@ public void testCopyWithRemoveSource() throws Exception {
         Processor processor = createCopyProcessor(sourceFieldName, targetFieldName, false, true, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(targetFieldName), equalTo(true));
-        assertThat(ingestDocument.getFieldValue(targetFieldName, Object.class), equalTo(sourceValue));
+        assertDeepCopiedObjectEquals(ingestDocument.getFieldValue(targetFieldName, Object.class), sourceValue);
         assertThat(ingestDocument.hasField(sourceFieldName), equalTo(false));
     }
 
@@ -97,12 +99,30 @@ public void testCopyToExistingField() throws Exception {
         Processor processorWithTargetNullValue = createCopyProcessor(sourceFieldName, targetFieldWithNullValue, false, false, false);
         processorWithTargetNullValue.execute(ingestDocument);
         assertThat(ingestDocument.hasField(targetFieldWithNullValue), equalTo(true));
-        assertThat(ingestDocument.getFieldValue(targetFieldWithNullValue, Object.class), equalTo(sourceValue));
+        assertDeepCopiedObjectEquals(ingestDocument.getFieldValue(targetFieldWithNullValue, Object.class), sourceValue);
 
         Processor processorWithOverrideTargetIsTrue = createCopyProcessor(sourceFieldName, targetFieldName, false, false, true);
         processorWithOverrideTargetIsTrue.execute(ingestDocument);
         assertThat(ingestDocument.hasField(targetFieldName), equalTo(true));
-        assertThat(ingestDocument.getFieldValue(targetFieldName, Object.class), equalTo(sourceValue));
+        assertDeepCopiedObjectEquals(ingestDocument.getFieldValue(targetFieldName, Object.class), sourceValue);
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void assertDeepCopiedObjectEquals(Object expected, Object actual) {
+        if (expected instanceof Map) {
+            Map<String, Object> expectedMap = (Map<String, Object>) expected;
+            Map<String, Object> actualMap = (Map<String, Object>) actual;
+            assertEquals(expectedMap.size(), actualMap.size());
+            for (Map.Entry<String, Object> expectedEntry : expectedMap.entrySet()) {
+                assertDeepCopiedObjectEquals(expectedEntry.getValue(), actualMap.get(expectedEntry.getKey()));
+            }
+        } else if (expected instanceof List) {
+            assertArrayEquals(((List<?>) expected).toArray(), ((List<?>) actual).toArray());
+        } else if (expected instanceof byte[]) {
+            assertArrayEquals((byte[]) expected, (byte[]) actual);
+        } else {
+            assertEquals(expected, actual);
+        }
     }
 
     private static Processor createCopyProcessor(

modules/ingest-common/src/test/java/org/opensearch/ingest/common/RemoveProcessorFactoryTests.java

@@ -97,13 +97,13 @@ public void testCreateWithExcludeField() throws Exception {
             OpenSearchParseException.class,
             () -> factory.create(null, processorTag, null, config)
         );
-        assertThat(exception.getMessage(), equalTo("[field] ether field or exclude_field must be set"));
+        assertThat(exception.getMessage(), equalTo("[field] either field or exclude_field must be set"));
 
         Map<String, Object> config2 = new HashMap<>();
         config2.put("field", "field1");
         config2.put("exclude_field", "field2");
         exception = expectThrows(OpenSearchParseException.class, () -> factory.create(null, processorTag, null, config2));
-        assertThat(exception.getMessage(), equalTo("[field] ether field or exclude_field must be set"));
+        assertThat(exception.getMessage(), equalTo("[field] either field or exclude_field must be set"));
 
         Map<String, Object> config6 = new HashMap<>();
         config6.put("exclude_field", "exclude_field");

modules/ingest-common/src/test/java/org/opensearch/ingest/common/RemoveProcessorTests.java

@@ -203,7 +203,7 @@ public void testRemoveMetadataField() throws Exception {
 
     public void testCreateRemoveProcessorWithBothFieldsAndExcludeFields() throws Exception {
         assertThrows(
-            "ether fields and excludeFields must be set",
+            "either fields or excludeFields must be set",
             IllegalArgumentException.class,
             () -> new RemoveProcessor(randomAlphaOfLength(10), null, null, null, false)
         );
@@ -223,7 +223,7 @@ public void testCreateRemoveProcessorWithBothFieldsAndExcludeFields() throws Exc
         }
 
         assertThrows(
-            "ether fields and excludeFields must be set",
+            "either fields or excludeFields must be set",
             IllegalArgumentException.class,
             () -> new RemoveProcessor(randomAlphaOfLength(10), null, fields, excludeFields, false)
         );

server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsFixedDocCountErrorIT.java

@@ -23,6 +23,7 @@
 import java.util.Collection;
 
 import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.opensearch.index.IndexSettings.MINIMUM_REFRESH_INTERVAL;
 import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
 import static org.opensearch.search.aggregations.AggregationBuilders.terms;
 import static org.opensearch.test.OpenSearchIntegTestCase.Scope.TEST;
@@ -71,7 +72,10 @@ public void testSimpleAggErrorMultiShard() throws Exception {
         assertAcked(
             prepareCreate("idx_mshard_1").setMapping(STRING_FIELD_NAME, "type=keyword")
                 .setSettings(
-                    Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", MINIMUM_REFRESH_INTERVAL)
                 )
         );
         client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get();
@@ -89,7 +93,10 @@ public void testSimpleAggErrorMultiShard() throws Exception {
         assertAcked(
             prepareCreate("idx_mshard_2").setMapping(STRING_FIELD_NAME, "type=keyword")
                 .setSettings(
-                    Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", MINIMUM_REFRESH_INTERVAL)
                 )
         );
         client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get();
@@ -127,7 +134,10 @@ public void testSimpleAggErrorSingleShard() throws Exception {
         assertAcked(
             prepareCreate("idx_shard_error").setMapping(STRING_FIELD_NAME, "type=keyword")
                 .setSettings(
-                    Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", MINIMUM_REFRESH_INTERVAL)
                 )
         );
         client().prepareIndex("idx_shard_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get();
@@ -170,7 +180,10 @@ public void testSliceLevelDocCountErrorSingleShard() throws Exception {
         assertAcked(
             prepareCreate("idx_slice_error").setMapping(STRING_FIELD_NAME, "type=keyword")
                 .setSettings(
-                    Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", MINIMUM_REFRESH_INTERVAL)
                 )
         );
         client().prepareIndex("idx_slice_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get();
@@ -248,7 +261,10 @@ public void testSliceLevelDocCountErrorMultiShard() throws Exception {
         assertAcked(
             prepareCreate("idx_mshard_1").setMapping(STRING_FIELD_NAME, "type=keyword")
                 .setSettings(
-                    Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", MINIMUM_REFRESH_INTERVAL)
                 )
         );
         client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get();
@@ -288,7 +304,10 @@ public void testSliceLevelDocCountErrorMultiShard() throws Exception {
         assertAcked(
             prepareCreate("idx_mshard_2").setMapping(STRING_FIELD_NAME, "type=keyword")
                 .setSettings(
-                    Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", MINIMUM_REFRESH_INTERVAL)
                 )
         );
         client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get();

server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java

@@ -60,6 +60,7 @@
 import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.FS;
 import static org.opensearch.core.common.util.CollectionUtils.iterableAsArrayList;
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
@@ -789,6 +790,47 @@ public void testDefaultShardPreference() throws Exception {
         }
     }
 
+    public void testRestoreSearchableSnapshotWithIndexStoreTypeThrowsException() throws Exception {
+        final String snapshotName = "test-snap";
+        final String repoName = "test-repo";
+        final String indexName1 = "test-idx-1";
+        final int numReplicasIndex1 = randomIntBetween(1, 4);
+        final Client client = client();
+
+        internalCluster().ensureAtLeastNumDataNodes(numReplicasIndex1 + 1);
+        createIndexWithDocsAndEnsureGreen(numReplicasIndex1, 100, indexName1);
+
+        createRepositoryWithSettings(null, repoName);
+        takeSnapshot(client, snapshotName, repoName, indexName1);
+        deleteIndicesAndEnsureGreen(client, indexName1);
+
+        internalCluster().ensureAtLeastNumSearchNodes(numReplicasIndex1 + 1);
+
+        // set "index.store.type" to "remote_snapshot" in index settings of restore API and assert appropriate exception with error message
+        // is thrown.
+        final SnapshotRestoreException error = expectThrows(
+            SnapshotRestoreException.class,
+            () -> client.admin()
+                .cluster()
+                .prepareRestoreSnapshot(repoName, snapshotName)
+                .setRenamePattern("(.+)")
+                .setRenameReplacement("$1-copy")
+                .setIndexSettings(
+                    Settings.builder()
+                        .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), RestoreSnapshotRequest.StorageType.REMOTE_SNAPSHOT)
+                )
+                .setWaitForCompletion(true)
+                .execute()
+                .actionGet()
+        );
+        assertThat(
+            error.getMessage(),
+            containsString(
+                "cannot restore remote snapshot with index settings \"index.store.type\" set to \"remote_snapshot\". Instead use \"storage_type\": \"remote_snapshot\" as argument to restore."
+            )
+        );
+    }
+
     /**
      * Asserts the cache folder count to match the number of shards and the number of indices within the cache folder
      * as provided.

server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchExecutors.java

@@ -447,6 +447,30 @@ public boolean offer(E e) {
             }
         }
 
+        /**
+         * Workaround for https://bugs.openjdk.org/browse/JDK-8323659 regression, introduced in JDK-21.0.2.
+         */
+        @Override
+        public void put(E e) {
+            super.offer(e);
+        }
+
+        /**
+         * Workaround for https://bugs.openjdk.org/browse/JDK-8323659 regression, introduced in JDK-21.0.2.
+         */
+        @Override
+        public boolean offer(E e, long timeout, TimeUnit unit) {
+            return super.offer(e);
+        }
+
+        /**
+         * Workaround for https://bugs.openjdk.org/browse/JDK-8323659 regression, introduced in JDK-21.0.2.
+         */
+        @Override
+        public boolean add(E e) {
+            return super.offer(e);
+        }
+
     }
 
     /**

server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRemoteClusterInfoAction.java

@@ -53,7 +53,7 @@ public final class RestRemoteClusterInfoAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return singletonList(new Route(GET, "_remote/info"));
+        return singletonList(new Route(GET, "/_remote/info"));
     }
 
     @Override

server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutMappingAction.java

@@ -66,10 +66,10 @@ public class RestPutMappingAction extends BaseRestHandler {
     public List<Route> routes() {
         return unmodifiableList(
             asList(
-                new Route(POST, "/{index}/_mapping/"),
-                new Route(PUT, "/{index}/_mapping/"),
-                new Route(POST, "/{index}/_mappings/"),
-                new Route(PUT, "/{index}/_mappings/")
+                new Route(POST, "/{index}/_mapping"),
+                new Route(PUT, "/{index}/_mapping"),
+                new Route(POST, "/{index}/_mappings"),
+                new Route(PUT, "/{index}/_mappings")
             )
         );
     }