Merge branch 'main' into bug/wrr-test-fix

.ci/bwcVersions

@@ -53,4 +53,5 @@ BWC_VERSION:
   - "2.3.0"
   - "2.3.1"
   - "2.4.0"
+  - "2.4.1"
   - "2.5.0"

.github/workflows/backport.yml

@@ -27,4 +27,3 @@ jobs:
         with:
           github_token: ${{ steps.github_app_token.outputs.token }}
           head_template: backport/backport-<%= number %>-to-<%= base %>
-          files_to_skip: 'CHANGELOG.md'

.github/workflows/gradle-check.yml

@@ -67,7 +67,7 @@ jobs:
           files: ./codeCoverage.xml
 
       - name: Create Comment Success
-        if: ${{ github.event_name == 'pull_request_target' && success() }}
+        if: ${{ github.event_name == 'pull_request_target' && success() && env.result == 'SUCCESS' }}
         uses: peter-evans/create-or-update-comment@v2
         with:
           issue-number: ${{ env.pr_number }}
@@ -77,15 +77,42 @@ jobs:
               * **URL:** ${{ env.workflow_url }}
               * **CommitID:** ${{ env.pr_from_sha }}
 
+      - name: Extract Test Failure
+        if: ${{ github.event_name == 'pull_request_target' && env.result != 'SUCCESS' }}
+        run: |
+            TEST_FAILURES=`curl -s "${{ env.workflow_url }}/testReport/api/json?tree=suites\[cases\[status,className,name\]\]" | jq -r '.. | objects | select(.status=="FAILED",.status=="REGRESSION") | (.className + "." +  .name)' | uniq -c | sort -n -r | head -n 10`
+            if [[ "$TEST_FAILURES" != "" ]]
+            then
+              echo "test_failures<<EOF" >> $GITHUB_ENV
+              echo "" >> $GITHUB_ENV
+              echo "* **TEST FAILURES:**" >> $GITHUB_ENV
+              echo '```' >> $GITHUB_ENV
+              echo "$TEST_FAILURES" >> $GITHUB_ENV
+              echo '```' >> $GITHUB_ENV
+              echo "EOF" >> $GITHUB_ENV
+            fi
+
+      - name: Create Comment Flaky
+        if: ${{ github.event_name == 'pull_request_target' && success() && env.result != 'SUCCESS' }}
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          issue-number: ${{ env.pr_number }}
+          body: |
+            ### Gradle Check (Jenkins) Run Completed with:
+            * **RESULT:** ${{ env.result }} :grey_exclamation: ${{ env.test_failures }}
+            * **URL:** ${{ env.workflow_url }}
+            * **CommitID:** ${{ env.pr_from_sha }}
+            Please review all [flaky tests](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) that succeeded after retry and create an issue if one does not already exist to track the flaky failure.
+
       - name: Create Comment Failure
         if: ${{ github.event_name == 'pull_request_target' && failure() }}
         uses: peter-evans/create-or-update-comment@v2
         with:
           issue-number: ${{ env.pr_number }}
           body: |
               ### Gradle Check (Jenkins) Run Completed with:
-              * **RESULT:** ${{ env.result }} :x:
+              * **RESULT:** ${{ env.result }} :x: ${{ env.test_failures }}
               * **URL:** ${{ env.workflow_url }}
               * **CommitID:** ${{ env.pr_from_sha }}
-              Please examine the workflow log, locate, and copy-paste the failure below, then iterate to green.
+              Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green.
               Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?

CHANGELOG.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add project health badges to the README.md ([#4843](https://github.com/opensearch-project/OpenSearch/pull/4843))
 - [Test] Add IAE test for deprecated edgeNGram analyzer name ([#5040](https://github.com/opensearch-project/OpenSearch/pull/5040))
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
+- Add feature flag for extensions ([#5211](https://github.com/opensearch-project/OpenSearch/pull/5211))
 
 ### Dependencies
 - Bumps `log4j-core` from 2.18.0 to 2.19.0
@@ -35,11 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bumps `reactor-core` from 3.4.18 to 3.4.23 ([#4548](https://github.com/opensearch-project/OpenSearch/pull/4548))
 - Bumps `jempbox` from 1.8.16 to 1.8.17 ([#4550](https://github.com/opensearch-project/OpenSearch/pull/4550))
 - Update Apache Lucene to 9.5.0-snapshot-a4ef70f ([#4979](https://github.com/opensearch-project/OpenSearch/pull/4979))
-- Bumps `bcpg-fips` from 1.0.5.1 to 1.0.7.1
 - Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973))
-- Bumps `azure-storage-blob` from 12.16.1 to 12.20.0 ([#4995](https://github.com/opensearch-project/OpenSearch/pull/4995))
-- Bumps `commons-compress` from 1.21 to 1.22 ([#5104](https://github.com/opensearch-project/OpenSearch/pull/5104))
-- Bump `opencensus-contrib-http-util` from 0.18.0 to 0.31.1 ([#3633](https://github.com/opensearch-project/OpenSearch/pull/3633))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -68,12 +65,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Fixed
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
 - Fixed compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
-
+- Add jvm option to allow security manager ([#5194](https://github.com/opensearch-project/OpenSearch/pull/5194))
 ### Security
 
 ## [Unreleased 2.x]
 ### Added
 ### Dependencies
+- Bumps `bcpg-fips` from 1.0.5.1 to 1.0.7.1
+- Bumps `azure-storage-blob` from 12.16.1 to 12.20.0 ([#4995](https://github.com/opensearch-project/OpenSearch/pull/4995))
+- Bumps `commons-compress` from 1.21 to 1.22 ([#5104](https://github.com/opensearch-project/OpenSearch/pull/5104))
+- Bump `opencensus-contrib-http-util` from 0.18.0 to 0.31.1 ([#3633](https://github.com/opensearch-project/OpenSearch/pull/3633))
+- Bump `geoip2` from 3.0.1 to 3.0.2 ([#5103](https://github.com/opensearch-project/OpenSearch/pull/5103))
 ### Changed
 ### Deprecated
 ### Removed

buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java

@@ -174,7 +174,7 @@ private static void addIvyRepo(Project project, String name, String url, String
 
         project.getRepositories().exclusiveContent(exclusiveContentRepository -> {
             exclusiveContentRepository.filter(config -> config.includeGroup(group));
-            exclusiveContentRepository.forRepositories(repos.toArray(new IvyArtifactRepository[repos.size()]));
+            exclusiveContentRepository.forRepositories(repos.toArray(new IvyArtifactRepository[0]));
         });
     }
 

buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java

@@ -109,7 +109,12 @@ public void execute(Task t) {
                         test.systemProperty("java.locale.providers", "SPI,JRE");
                     } else {
                         test.systemProperty("java.locale.providers", "SPI,COMPAT");
-                        test.jvmArgs("--illegal-access=warn");
+                        if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) < 0) {
+                            test.jvmArgs("--illegal-access=warn");
+                        }
+                    }
+                    if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) > 0) {
+                        test.jvmArgs("-Djava.security.manager=allow");
                     }
                 }
             });

client/rest-high-level/src/main/java/org/opensearch/client/core/CountResponse.java

@@ -233,7 +233,7 @@ static ShardStats fromXContent(XContentParser parser) throws IOException {
                     parser.skipChildren();
                 }
             }
-            return new ShardStats(successfulShards, totalShards, skippedShards, failures.toArray(new ShardSearchFailure[failures.size()]));
+            return new ShardStats(successfulShards, totalShards, skippedShards, failures.toArray(new ShardSearchFailure[0]));
         }
 
         @Override

client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java

@@ -435,7 +435,7 @@ public PutIndexTemplateRequest alias(Alias alias) {
 
     @Override
     public String[] indices() {
-        return indexPatterns.toArray(new String[indexPatterns.size()]);
+        return indexPatterns.toArray(new String[0]);
     }
 
     @Override

client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientExtTests.java

@@ -153,7 +153,7 @@ static CustomResponseSection2 fromXContent(XContentParser parser) throws IOExcep
                 values.add(parser.text());
             }
             assertEquals(XContentParser.Token.END_ARRAY, parser.currentToken());
-            CustomResponseSection2 responseSection2 = new CustomResponseSection2(values.toArray(new String[values.size()]));
+            CustomResponseSection2 responseSection2 = new CustomResponseSection2(values.toArray(new String[0]));
             assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
             return responseSection2;
         }

client/rest/src/main/java/org/opensearch/client/Response.java

@@ -147,7 +147,7 @@ public HttpEntity getEntity() {
      * @return {@code true} if the input string matches the specification
      */
     private static boolean matchWarningHeaderPatternByPrefix(final String s) {
-        return s.startsWith("299 OpenSearch-") || s.startsWith("299 Elasticsearch-");
+        return s.startsWith("299 OpenSearch-");
     }
 
     /**

distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java

@@ -77,12 +77,21 @@ static List<String> systemJvmOptions() {
                 // log4j 2
                 "-Dlog4j.shutdownHookEnabled=false",
                 "-Dlog4j2.disable.jmx=true",
-
+                // security manager
+                allowSecurityManagerOption(),
                 javaLocaleProviders()
             )
         ).stream().filter(e -> e.isEmpty() == false).collect(Collectors.toList());
     }
 
+    private static String allowSecurityManagerOption() {
+        if (Runtime.version().feature() > 17) {
+            return "-Djava.security.manager=allow";
+        } else {
+            return "";
+        }
+    }
+
     private static String maybeShowCodeDetailsInExceptionMessages() {
         if (Runtime.version().feature() >= 14) {
             return "-XX:+ShowCodeDetailsInExceptionMessages";

distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java

@@ -1028,7 +1028,7 @@ private static void setFileAttributes(final Path path, final Set<PosixFilePermis
 
     @Override
     public void close() throws IOException {
-        IOUtils.rm(pathsToDeleteOnShutdown.toArray(new Path[pathsToDeleteOnShutdown.size()]));
+        IOUtils.rm(pathsToDeleteOnShutdown.toArray(new Path[0]));
     }
 
 }

distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/RemovePluginCommand.java

@@ -215,7 +215,7 @@ void execute(Terminal terminal, Environment env, String pluginName, boolean purg
         // finally, add the marker file
         pluginPaths.add(removing);
 
-        IOUtils.rm(pluginPaths.toArray(new Path[pluginPaths.size()]));
+        IOUtils.rm(pluginPaths.toArray(new Path[0]));
     }
 
 }

gradle/wrapper/gradle-wrapper.properties

@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-rc-2-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-rc-3-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=be5387477162265eac882b9c83d756d8d2db170380e36fba2fdbee83d87de0d7
+distributionSha256Sum=abc6de2653ec9befb00cc0d064ce1ca4e4dab2c91955e830661505189c0b2f08

libs/x-content/src/main/java/org/opensearch/common/ParseField.java

@@ -68,12 +68,12 @@ public ParseField(String name, String... deprecatedNames) {
         } else {
             final HashSet<String> set = new HashSet<>();
             Collections.addAll(set, deprecatedNames);
-            this.deprecatedNames = set.toArray(new String[set.size()]);
+            this.deprecatedNames = set.toArray(new String[0]);
         }
         Set<String> allNames = new HashSet<>();
         allNames.add(name);
         Collections.addAll(allNames, this.deprecatedNames);
-        this.allNames = allNames.toArray(new String[allNames.size()]);
+        this.allNames = allNames.toArray(new String[0]);
     }
 
     /**

libs/x-content/src/main/java/org/opensearch/common/xcontent/support/filtering/FilterPath.java

@@ -101,7 +101,7 @@ public static FilterPath[] compile(Set<String> filters) {
                 }
             }
         }
-        return paths.toArray(new FilterPath[paths.size()]);
+        return paths.toArray(new FilterPath[0]);
     }
 
     private static FilterPath parse(final String filter, final String segment) {

libs/x-content/src/main/java/org/opensearch/common/xcontent/support/filtering/FilterPathBasedFilter.java

@@ -95,7 +95,7 @@ private TokenFilter evaluate(String name, FilterPath[] filters) {
             }
 
             if ((nextFilters != null) && (nextFilters.isEmpty() == false)) {
-                return new FilterPathBasedFilter(nextFilters.toArray(new FilterPath[nextFilters.size()]), inclusive);
+                return new FilterPathBasedFilter(nextFilters.toArray(new FilterPath[0]), inclusive);
             }
         }
         return NO_MATCHING;

modules/ingest-geoip/build.gradle

@@ -39,7 +39,7 @@ opensearchplugin {
 }
 
 dependencies {
-  api('com.maxmind.geoip2:geoip2:3.0.1')
+  api('com.maxmind.geoip2:geoip2:3.0.2')
   // geoip2 dependencies:
   api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
   api("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}")

modules/ingest-geoip/licenses/geoip2-3.0.2.jar.sha1

@@ -0,0 +1 @@
+f0ab0a451309c93f0fb6bf3cb203ba19d452c800

modules/parent-join/src/internalClusterTest/java/org/opensearch/join/query/ChildQuerySearchIT.java

@@ -310,7 +310,7 @@ public void testHasParentFilter() throws Exception {
             }
             assertThat(parentToChildren.get(previousParentId).add(childId), is(true));
         }
-        indexRandom(true, builders.toArray(new IndexRequestBuilder[builders.size()]));
+        indexRandom(true, builders.toArray(new IndexRequestBuilder[0]));
 
         assertThat(parentToChildren.isEmpty(), equalTo(false));
         for (Map.Entry<String, Set<String>> parentToChildrenEntry : parentToChildren.entrySet()) {

modules/rank-eval/src/main/java/org/opensearch/index/rankeval/TransportRankEvalAction.java

@@ -148,7 +148,7 @@ protected void doExecute(Task task, RankEvalRequest request, ActionListener<Rank
             if (summaryFields.isEmpty()) {
                 evaluationRequest.fetchSource(false);
             } else {
-                evaluationRequest.fetchSource(summaryFields.toArray(new String[summaryFields.size()]), new String[0]);
+                evaluationRequest.fetchSource(summaryFields.toArray(new String[0]), new String[0]);
             }
             SearchRequest searchRequest = new SearchRequest(request.indices(), evaluationRequest);
             searchRequest.indicesOptions(request.indicesOptions());
@@ -158,12 +158,7 @@ protected void doExecute(Task task, RankEvalRequest request, ActionListener<Rank
         assert ratedRequestsInSearch.size() == msearchRequest.requests().size();
         client.multiSearch(
             msearchRequest,
-            new RankEvalActionListener(
-                listener,
-                metric,
-                ratedRequestsInSearch.toArray(new RatedRequest[ratedRequestsInSearch.size()]),
-                errors
-            )
+            new RankEvalActionListener(listener, metric, ratedRequestsInSearch.toArray(new RatedRequest[0]), errors)
         );
     }
 

modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java

@@ -517,7 +517,7 @@ void refreshAndFinish(List<Failure> indexingFailures, List<SearchFailure> search
             return;
         }
         RefreshRequest refresh = new RefreshRequest();
-        refresh.indices(destinationIndices.toArray(new String[destinationIndices.size()]));
+        refresh.indices(destinationIndices.toArray(new String[0]));
         logger.debug("[{}]: refreshing", task.getId());
         client.admin().indices().refresh(refresh, new ActionListener<RefreshResponse>() {
             @Override

modules/reindex/src/test/java/org/opensearch/index/reindex/DeleteByQueryBasicTests.java

@@ -362,7 +362,7 @@ public void testMultipleSources() throws Exception {
         int slices = randomSlices(1, 10);
         int expectedSlices = expectedSliceStatuses(slices, docs.keySet());
 
-        String[] sourceIndexNames = docs.keySet().toArray(new String[docs.size()]);
+        String[] sourceIndexNames = docs.keySet().toArray(new String[0]);
 
         assertThat(
             deleteByQuery().source(sourceIndexNames).filter(QueryBuilders.matchAllQuery()).refresh(true).setSlices(slices).get(),

modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexBasicTests.java

@@ -161,7 +161,7 @@ public void testMultipleSources() throws Exception {
         int slices = randomSlices(1, 10);
         int expectedSlices = expectedSliceStatuses(slices, docs.keySet());
 
-        String[] sourceIndexNames = docs.keySet().toArray(new String[docs.size()]);
+        String[] sourceIndexNames = docs.keySet().toArray(new String[0]);
         ReindexRequestBuilder request = reindex().source(sourceIndexNames).destination("dest").refresh(true).setSlices(slices);
 
         BulkByScrollResponse response = request.get();

modules/reindex/src/test/java/org/opensearch/index/reindex/UpdateByQueryBasicTests.java

@@ -151,7 +151,7 @@ public void testMultipleSources() throws Exception {
         int slices = randomSlices(1, 10);
         int expectedSlices = expectedSliceStatuses(slices, docs.keySet());
 
-        String[] sourceIndexNames = docs.keySet().toArray(new String[docs.size()]);
+        String[] sourceIndexNames = docs.keySet().toArray(new String[0]);
         BulkByScrollResponse response = updateByQuery().source(sourceIndexNames).refresh(true).setSlices(slices).get();
         assertThat(response, matcher().updated(allDocs.size()).slices(hasSize(expectedSlices)));
 

plugins/mapper-annotated-text/src/main/java/org/opensearch/search/fetch/subphase/highlight/AnnotatedPassageFormatter.java

@@ -237,7 +237,7 @@ public AnnotationToken[] getIntersectingAnnotations(int start, int end) {
             // add 1 for the fieldvalue separator character
             fieldValueOffset += fieldValueAnnotations.textMinusMarkup.length() + 1;
         }
-        return intersectingAnnotations.toArray(new AnnotationToken[intersectingAnnotations.size()]);
+        return intersectingAnnotations.toArray(new AnnotationToken[0]);
     }
 
     private void append(StringBuilder dest, String content, int start, int end) {

qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/RefreshVersionInClusterStateIT.java

@@ -18,7 +18,7 @@
 public class RefreshVersionInClusterStateIT extends AbstractRollingTestCase {
 
     /*
-    This test ensures that after the upgrade from ElasticSearch/ OpenSearch all nodes report the version on and after 1.0.0
+     * This test ensures that after the upgrade, all nodes report the current version
      */
     public void testRefresh() throws IOException {
         switch (CLUSTER_TYPE) {

server/build.gradle

@@ -155,22 +155,6 @@ tasks.named("internalClusterTest").configure {
   jvmArgs -= '-XX:TieredStopAtLevel=1'
 }
 
-// Until this project is always being formatted with spotless, we need to
-// guard against `spotless()` not existing.
-try {
-  spotless {
-    java {
-      // Contains large data tables that do not format well.
-      targetExclude 'src/main/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlus.java'
-    }
-  }
-}
-catch (Exception e) {
-  if (e.getMessage().contains("Could not find method spotless") == false) {
-    throw e;
-  }
-}
-
 tasks.named("forbiddenPatterns").configure {
     exclude '**/*.json'
     exclude '**/*.jmx'

server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java

@@ -708,7 +708,7 @@ private String[] randomUniqueIndices() {
         while (uniqueIndices.size() < count) {
             uniqueIndices.add(randomFrom(this.indices));
         }
-        return uniqueIndices.toArray(new String[uniqueIndices.size()]);
+        return uniqueIndices.toArray(new String[0]);
     }
 
     private static void assertAllRequestsHaveBeenConsumed() {

server/src/internalClusterTest/java/org/opensearch/action/admin/indices/get/GetIndexIT.java

@@ -163,7 +163,7 @@ public void testSimpleMixedFeatures() {
         }
         GetIndexResponse response = runWithRandomFeatureMethod(
             client().admin().indices().prepareGetIndex().addIndices("idx"),
-            features.toArray(new Feature[features.size()])
+            features.toArray(new Feature[0])
         );
         String[] indices = response.indices();
         assertThat(indices, notNullValue());
@@ -194,7 +194,7 @@ public void testEmptyMixedFeatures() {
         }
         GetIndexResponse response = runWithRandomFeatureMethod(
             client().admin().indices().prepareGetIndex().addIndices("empty_idx"),
-            features.toArray(new Feature[features.size()])
+            features.toArray(new Feature[0])
         );
         String[] indices = response.indices();
         assertThat(indices, notNullValue());