Merge branch 'main' into ec2-scheme

Signed-off-by: 何延龙 <[email protected]>

.github/workflows/maintainer-approval.yml

@@ -0,0 +1,33 @@
+name: Maintainers approval
+
+on:
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  maintainer-approved-check:
+    name: Minimum approval count
+    runs-on: ubuntu-latest
+    steps:
+      - id: find-maintainers
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-encoding: string
+          script: |
+            // Get the collaborators - filtered to maintainer permissions
+            const maintainersResponse = await github.request('GET /repos/{owner}/{repo}/collaborators', {
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                permission: 'maintain',
+                affiliation: 'all',
+                per_page: 100
+            });
+
+            return maintainersResponse.data.map(item => item.login).join(', ');
+
+      - uses: peternied/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          min-required: 1
+          required-approvers-list: ${{ steps.find-maintainers.outputs.result }}

.github/workflows/stalled.yml

@@ -17,7 +17,7 @@ jobs:
           private_key: ${{ secrets.APP_PRIVATE_KEY }}
           installation_id: 22958780
       - name: Stale PRs
-        uses: actions/stale@v8
+        uses: actions/stale@v9
         with:
           repo-token: ${{ steps.github_app_token.outputs.token }}
           stale-pr-label: 'stalled'

CHANGELOG.md

@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800))
 - Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625))
 - [Admission Control] Integrate CPU AC with ResourceUsageCollector and add CPU AC stats to nodes/stats ([#10887](https://github.com/opensearch-project/OpenSearch/pull/10887))
+- Maintainer approval check ([#11378](https://github.com/opensearch-project/OpenSearch/pull/11378))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -119,10 +120,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Allow changing number of replicas of searchable snapshot index ([#11317](https://github.com/opensearch-project/OpenSearch/pull/11317))
 - Adding slf4j license header to LoggerMessageFormat.java ([#11069](https://github.com/opensearch-project/OpenSearch/pull/11069))
 - [BWC and API enforcement] Introduce checks for enforcing the API restrictions ([#11175](https://github.com/opensearch-project/OpenSearch/pull/11175))
+- Create separate transport action for render search template action ([#11170](https://github.com/opensearch-project/OpenSearch/pull/11170))
 
 ### Dependencies
 - Bump Lucene from 9.7.0 to 9.8.0 ([10276](https://github.com/opensearch-project/OpenSearch/pull/10276))
-- Bump `commons-io:commons-io` from 2.13.0 to 2.15.1 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294), [#11001](https://github.com/opensearch-project/OpenSearch/pull/11001), [#11002](https://github.com/opensearch-project/OpenSearch/pull/11002), [#11446](https://github.com/opensearch-project/OpenSearch/pull/11446))
+- Bump `commons-io:commons-io` from 2.13.0 to 2.15.1 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294), [#11001](https://github.com/opensearch-project/OpenSearch/pull/11001), [#11002](https://github.com/opensearch-project/OpenSearch/pull/11002), [#11446](https://github.com/opensearch-project/OpenSearch/pull/11446), [#11554](https://github.com/opensearch-project/OpenSearch/pull/11554))
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
 - Bump `com.netflix.nebula.ospackage-base` from 11.4.0 to 11.5.0 ([#10295](https://github.com/opensearch-project/OpenSearch/pull/10295))
 - Bump `org.apache.zookeeper:zookeeper` from 3.9.0 to 3.9.1 ([#10506](https://github.com/opensearch-project/OpenSearch/pull/10506))
@@ -146,6 +148,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `commons-net:commons-net` from 3.9.0 to 3.10.0 ([#11450](https://github.com/opensearch-project/OpenSearch/pull/11450))
 - Bump `org.apache.maven:maven-model` from 3.9.4 to 3.9.6 ([#11445](https://github.com/opensearch-project/OpenSearch/pull/11445))
 - Bump `org.apache.xmlbeans:xmlbeans` from 5.1.1 to 5.2.0 ([#11448](https://github.com/opensearch-project/OpenSearch/pull/11448))
+- Bump `logback-core` and `logback-classic` to 1.2.13 ([#11521](https://github.com/opensearch-project/OpenSearch/pull/11521))
+- Bumps `jetty` version from 9.4.52.v20230823 to 9.4.53.v20231009 ([#11539](https://github.com/opensearch-project/OpenSearch/pull/11539))
+- Bump `org.wiremock:wiremock-standalone` from 3.1.0 to 3.3.1 ([#11555](https://github.com/opensearch-project/OpenSearch/pull/11555))
+- Bump `org.apache.commons:commons-compress` from 1.24.0 to 1.25.0 ([#11556](https://github.com/opensearch-project/OpenSearch/pull/11556))
+- Bump `actions/stale` from 8 to 9 ([#11557](https://github.com/opensearch-project/OpenSearch/pull/11557))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
@@ -166,6 +173,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Interpret byte array as primitive using VarHandles ([#11362](https://github.com/opensearch-project/OpenSearch/pull/11362))
 - Change error message when per shard document limit is breached ([#11312](https://github.com/opensearch-project/OpenSearch/pull/11312))
 - Automatically add scheme to discovery.ec2.endpoint ([#11512](https://github.com/opensearch-project/OpenSearch/pull/11512))
+- Restore support for Java 8 for RestClient ([#11562](https://github.com/opensearch-project/OpenSearch/pull/11562))
 
 ### Deprecated
 
@@ -182,6 +190,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Delegating CachingWeightWrapper#count to internal weight object ([#10543](https://github.com/opensearch-project/OpenSearch/pull/10543))
 - Fix per request latency last phase not tracked ([#10934](https://github.com/opensearch-project/OpenSearch/pull/10934))
 - Fix for stuck update action in a bulk with `retry_on_conflict` property ([#11152](https://github.com/opensearch-project/OpenSearch/issues/11152))
+- Fix the issue with DefaultSpanScope restoring wrong span in the TracerContextStorage upon detach ([#11316](https://github.com/opensearch-project/OpenSearch/issues/11316))
 - Remove shadowJar from `lang-painless` module publication ([#11369](https://github.com/opensearch-project/OpenSearch/issues/11369))
 - Fix remote shards balancer and remove unused variables ([#11167](https://github.com/opensearch-project/OpenSearch/pull/11167))
 - Fix bug where replication lag grows post primary relocation ([#11238](https://github.com/opensearch-project/OpenSearch/pull/11238))

buildSrc/build.gradle

@@ -103,7 +103,7 @@ dependencies {
   api localGroovy()
 
   api 'commons-codec:commons-codec:1.16.0'
-  api 'org.apache.commons:commons-compress:1.24.0'
+  api 'org.apache.commons:commons-compress:1.25.0'
   api 'org.apache.ant:ant:1.10.14'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0'
   api 'com.netflix.nebula:nebula-publishing-plugin:20.3.0'
@@ -122,21 +122,17 @@ dependencies {
   api 'org.jruby.jcodings:jcodings:1.0.58'
   api 'org.jruby.joni:joni:2.2.1'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
-  api "org.ajoberstar.grgit:grgit-core:5.2.0"
+  api "org.ajoberstar.grgit:grgit-core:5.2.1"
 
   testFixturesApi "junit:junit:${props.getProperty('junit')}"
   testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
   testFixturesApi gradleApi()
   testFixturesApi gradleTestKit()
-  testImplementation 'org.wiremock:wiremock-standalone:3.1.0'
+  testImplementation 'org.wiremock:wiremock-standalone:3.3.1'
   testImplementation "org.mockito:mockito-core:${props.getProperty('mockito')}"
   integTestImplementation('org.spockframework:spock-core:2.3-groovy-3.0') {
     exclude module: "groovy"
   }
-  implementation('org.ajoberstar.grgit:grgit-core:5.2.0') {
-    exclude group: 'org.eclipse.jgit', module: 'org.eclipse.jgit'
-  }
-  implementation 'org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r'
 }
 
 configurations.all {

client/rest/build.gradle

@@ -34,8 +34,8 @@ apply plugin: 'opensearch.build'
 apply plugin: 'opensearch.publish'
 
 java {
-  targetCompatibility = JavaVersion.VERSION_11
-  sourceCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_1_8
+  sourceCompatibility = JavaVersion.VERSION_1_8
 }
 
 base {
@@ -109,3 +109,10 @@ thirdPartyAudit.ignoreMissingClasses(
   'javax.servlet.ServletContextEvent',
   'javax.servlet.ServletContextListener'
 )
+
+tasks.withType(JavaCompile) {
+  // Suppressing '[options] target value 8 is obsolete and will be removed in a future release'
+  configure(options) {
+    options.compilerArgs << '-Xlint:-options'
+  }
+}

client/rest/src/main/java/org/opensearch/client/RestClient.java

@@ -1116,9 +1116,15 @@ public long getContentLength() {
                 if (chunkedEnabled.get()) {
                     return -1L;
                 } else {
-                    long size;
+                    long size = 0;
+                    final byte[] buf = new byte[8192];
+                    int nread = 0;
+
                     try (InputStream is = getContent()) {
-                        size = is.readAllBytes().length;
+                        // read to EOF which may read more or less than buffer size
+                        while ((nread = is.read(buf)) > 0) {
+                            size += nread;
+                        }
                     } catch (IOException ex) {
                         size = -1L;
                     }

client/rest/src/test/java/org/opensearch/client/nio/HeapBufferedAsyncEntityConsumerTests.java

@@ -35,34 +35,34 @@ public void tearDown() {
     }
 
     public void testConsumerAllocatesBufferLimit() throws IOException {
-        consumer.consume(randomByteBufferOfLength(1000).flip());
+        consumer.consume((ByteBuffer) randomByteBufferOfLength(1000).flip());
         assertThat(consumer.getBuffer().capacity(), equalTo(1000));
     }
 
     public void testConsumerAllocatesEmptyBuffer() throws IOException {
-        consumer.consume(ByteBuffer.allocate(0).flip());
+        consumer.consume((ByteBuffer) ByteBuffer.allocate(0).flip());
         assertThat(consumer.getBuffer().capacity(), equalTo(0));
     }
 
     public void testConsumerExpandsBufferLimits() throws IOException {
-        consumer.consume(randomByteBufferOfLength(1000).flip());
-        consumer.consume(randomByteBufferOfLength(2000).flip());
-        consumer.consume(randomByteBufferOfLength(3000).flip());
+        consumer.consume((ByteBuffer) randomByteBufferOfLength(1000).flip());
+        consumer.consume((ByteBuffer) randomByteBufferOfLength(2000).flip());
+        consumer.consume((ByteBuffer) randomByteBufferOfLength(3000).flip());
         assertThat(consumer.getBuffer().capacity(), equalTo(6000));
     }
 
     public void testConsumerAllocatesLimit() throws IOException {
-        consumer.consume(randomByteBufferOfLength(BUFFER_LIMIT).flip());
+        consumer.consume((ByteBuffer) randomByteBufferOfLength(BUFFER_LIMIT).flip());
         assertThat(consumer.getBuffer().capacity(), equalTo(BUFFER_LIMIT));
     }
 
     public void testConsumerFailsToAllocateOverLimit() throws IOException {
-        assertThrows(ContentTooLongException.class, () -> consumer.consume(randomByteBufferOfLength(BUFFER_LIMIT + 1).flip()));
+        assertThrows(ContentTooLongException.class, () -> consumer.consume((ByteBuffer) randomByteBufferOfLength(BUFFER_LIMIT + 1).flip()));
     }
 
     public void testConsumerFailsToExpandOverLimit() throws IOException {
-        consumer.consume(randomByteBufferOfLength(BUFFER_LIMIT).flip());
-        assertThrows(ContentTooLongException.class, () -> consumer.consume(randomByteBufferOfLength(1).flip()));
+        consumer.consume((ByteBuffer) randomByteBufferOfLength(BUFFER_LIMIT).flip());
+        assertThrows(ContentTooLongException.class, () -> consumer.consume((ByteBuffer) randomByteBufferOfLength(1).flip()));
     }
 
     private static ByteBuffer randomByteBufferOfLength(int length) {

client/test/build.gradle

@@ -30,8 +30,8 @@
 apply plugin: 'opensearch.build'
 
 java {
-  targetCompatibility = JavaVersion.VERSION_11
-  sourceCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_1_8
+  sourceCompatibility = JavaVersion.VERSION_1_8
 }
 
 base {
@@ -69,3 +69,10 @@ dependenciesInfo.enabled = false
 //we aren't releasing this jar
 thirdPartyAudit.enabled = false
 test.enabled = false
+
+tasks.withType(JavaCompile) {
+  // Suppressing '[options] target value 8 is obsolete and will be removed in a future release'
+  configure(options) {
+    options.compilerArgs << '-Xlint:-options'
+  }
+}

libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultSpanScope.java

@@ -21,6 +21,7 @@
 class DefaultSpanScope implements SpanScope {
     private final Span span;
     private final SpanScope previousSpanScope;
+    private final Span beforeSpan;
     private static final ThreadLocal<SpanScope> spanScopeThreadLocal = new ThreadLocal<>();
     private final TracerContextStorage<String, Span> tracerContextStorage;
 
@@ -29,8 +30,14 @@ class DefaultSpanScope implements SpanScope {
      * @param span span
      * @param previousSpanScope before attached span scope.
      */
-    private DefaultSpanScope(Span span, SpanScope previousSpanScope, TracerContextStorage<String, Span> tracerContextStorage) {
+    private DefaultSpanScope(
+        Span span,
+        final Span beforeSpan,
+        SpanScope previousSpanScope,
+        TracerContextStorage<String, Span> tracerContextStorage
+    ) {
         this.span = Objects.requireNonNull(span);
+        this.beforeSpan = beforeSpan;
         this.previousSpanScope = previousSpanScope;
         this.tracerContextStorage = tracerContextStorage;
     }
@@ -43,7 +50,8 @@ private DefaultSpanScope(Span span, SpanScope previousSpanScope, TracerContextSt
      */
     public static SpanScope create(Span span, TracerContextStorage<String, Span> tracerContextStorage) {
         final SpanScope beforeSpanScope = spanScopeThreadLocal.get();
-        SpanScope newSpanScope = new DefaultSpanScope(span, beforeSpanScope, tracerContextStorage);
+        final Span beforeSpan = tracerContextStorage.get(TracerContextStorage.CURRENT_SPAN);
+        SpanScope newSpanScope = new DefaultSpanScope(span, beforeSpan, beforeSpanScope, tracerContextStorage);
         return newSpanScope;
     }
 
@@ -61,8 +69,8 @@ public SpanScope attach() {
 
     private void detach() {
         spanScopeThreadLocal.set(previousSpanScope);
-        if (previousSpanScope != null) {
-            tracerContextStorage.put(TracerContextStorage.CURRENT_SPAN, previousSpanScope.getSpan());
+        if (beforeSpan != null) {
+            tracerContextStorage.put(TracerContextStorage.CURRENT_SPAN, beforeSpan);
         } else {
             tracerContextStorage.put(TracerContextStorage.CURRENT_SPAN, null);
         }

modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MustacheModulePlugin.java

@@ -65,6 +65,7 @@ public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return Arrays.asList(
             new ActionHandler<>(SearchTemplateAction.INSTANCE, TransportSearchTemplateAction.class),
+            new ActionHandler<>(RenderSearchTemplateAction.INSTANCE, TransportRenderSearchTemplateAction.class),
             new ActionHandler<>(MultiSearchTemplateAction.INSTANCE, TransportMultiSearchTemplateAction.class)
         );
     }

modules/lang-mustache/src/main/java/org/opensearch/script/mustache/RenderSearchTemplateAction.java

@@ -0,0 +1,21 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.script.mustache;
+
+import org.opensearch.action.ActionType;
+
+public class RenderSearchTemplateAction extends ActionType<SearchTemplateResponse> {
+
+    public static final RenderSearchTemplateAction INSTANCE = new RenderSearchTemplateAction();
+    public static final String NAME = "indices:data/read/search/template/render";
+
+    private RenderSearchTemplateAction() {
+        super(NAME, SearchTemplateResponse::new);
+    }
+}

modules/lang-mustache/src/main/java/org/opensearch/script/mustache/RestRenderSearchTemplateAction.java

@@ -81,6 +81,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
             renderRequest.setScript(id);
         }
 
-        return channel -> client.execute(SearchTemplateAction.INSTANCE, renderRequest, new RestToXContentListener<>(channel));
+        return channel -> client.execute(RenderSearchTemplateAction.INSTANCE, renderRequest, new RestToXContentListener<>(channel));
     }
 }

modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportRenderSearchTemplateAction.java

@@ -0,0 +1,30 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.script.mustache;
+
+import org.opensearch.action.support.ActionFilters;
+import org.opensearch.client.node.NodeClient;
+import org.opensearch.common.inject.Inject;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.script.ScriptService;
+import org.opensearch.transport.TransportService;
+
+public class TransportRenderSearchTemplateAction extends TransportSearchTemplateAction {
+
+    @Inject
+    public TransportRenderSearchTemplateAction(
+        TransportService transportService,
+        ActionFilters actionFilters,
+        ScriptService scriptService,
+        NamedXContentRegistry xContentRegistry,
+        NodeClient client
+    ) {
+        super(RenderSearchTemplateAction.NAME, transportService, actionFilters, scriptService, xContentRegistry, client);
+    }
+}

modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java

@@ -61,9 +61,9 @@ public class TransportSearchTemplateAction extends HandledTransportAction<Search
 
     private static final String TEMPLATE_LANG = MustacheScriptEngine.NAME;
 
-    private final ScriptService scriptService;
-    private final NamedXContentRegistry xContentRegistry;
-    private final NodeClient client;
+    protected final ScriptService scriptService;
+    protected final NamedXContentRegistry xContentRegistry;
+    protected final NodeClient client;
 
     @Inject
     public TransportSearchTemplateAction(
@@ -79,6 +79,20 @@ public TransportSearchTemplateAction(
         this.client = client;
     }
 
+    public TransportSearchTemplateAction(
+        String actionName,
+        TransportService transportService,
+        ActionFilters actionFilters,
+        ScriptService scriptService,
+        NamedXContentRegistry xContentRegistry,
+        NodeClient client
+    ) {
+        super(actionName, transportService, actionFilters, SearchTemplateRequest::new);
+        this.scriptService = scriptService;
+        this.xContentRegistry = xContentRegistry;
+        this.client = client;
+    }
+
     @Override
     protected void doExecute(Task task, SearchTemplateRequest request, ActionListener<SearchTemplateResponse> listener) {
         final SearchTemplateResponse response = new SearchTemplateResponse();

plugins/discovery-azure-classic/build.gradle

@@ -53,7 +53,7 @@ dependencies {
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api "commons-lang:commons-lang:2.6"
-  api "commons-io:commons-io:2.15.0"
+  api "commons-io:commons-io:2.15.1"
   api 'javax.mail:mail:1.4.7'
   api 'javax.inject:javax.inject:1'
   api "com.sun.jersey:jersey-client:${versions.jersey}"

plugins/discovery-azure-classic/licenses/commons-io-2.15.1.jar.sha1

@@ -0,0 +1 @@
+f11560da189ab563a5c8e351941415430e9304ea