merge 2.x

Signed-off-by: Gao Binlong <[email protected]>

CHANGELOG.md

@@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
 - [Streaming Indexing] Introduce new experimental server HTTP transport based on Netty 4 and Project Reactor (Reactor Netty) ([#9672](https://github.com/opensearch-project/OpenSearch/pull/9672))
 - Add back half_float BKD based sort query optimization ([#11024](https://github.com/opensearch-project/OpenSearch/pull/11024))
+- Request level coordinator slow logs ([#11246](https://github.com/opensearch-project/OpenSearch/pull/11246))
 
 ### Dependencies
 - Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822))
@@ -64,6 +65,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Fixed
 - Fix class_cast_exception when passing int to _version and other metadata fields in ingest simulate API ([#10101](https://github.com/opensearch-project/OpenSearch/pull/10101))
 - Fix passing wrong parameter when calling newConfigurationException() in DotExpanderProcessor ([#10737](https://github.com/opensearch-project/OpenSearch/pull/10737))
+- Fix per request latency last phase not tracked ([#10934](https://github.com/opensearch-project/OpenSearch/pull/10934))
 - Adding version condition while adding geoshape doc values to the index, to ensure backward compatibility.([#11095](https://github.com/opensearch-project/OpenSearch/pull/11095))
 - Fix SuggestSearch.testSkipDuplicates by forcing refresh when indexing its test documents ([#11068](https://github.com/opensearch-project/OpenSearch/pull/11068))
 

distribution/docker/src/docker/config/log4j2.properties

@@ -34,6 +34,16 @@ logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
 logger.deprecation.appenderRef.header_warning.ref = header_warning
 logger.deprecation.additivity = false
 
+appender.search_request_slowlog_json_appender.type = Console
+appender.search_request_slowlog_json_appender.name = search_request_slowlog_json_appender
+appender.search_request_slowlog_json_appender.layout.type = OpenSearchJsonLayout
+appender.search_request_slowlog_json_appender.layout.type_name = search_request_slowlog
+
+logger.search_request_slowlog_logger.name = cluster.search.request.slowlog
+logger.search_request_slowlog_logger.level = trace
+logger.search_request_slowlog_logger.appenderRef.search_request_slowlog_json_appender.ref = search_request_slowlog_json_appender
+logger.search_request_slowlog_logger.additivity = false
+
 appender.index_search_slowlog_rolling.type = Console
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
 appender.index_search_slowlog_rolling.layout.type = OpenSearchJsonLayout

distribution/src/config/log4j2.properties

@@ -113,6 +113,47 @@ logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling
 logger.deprecation.appenderRef.header_warning.ref = header_warning
 logger.deprecation.additivity = false
 
+######## Search Request Slowlog JSON ####################
+appender.search_request_slowlog_json_appender.type = RollingFile
+appender.search_request_slowlog_json_appender.name = search_request_slowlog_json_appender
+appender.search_request_slowlog_json_appender.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs\
+  .cluster_name}_index_search_slowlog.json
+appender.search_request_slowlog_json_appender.filePermissions = rw-r-----
+appender.search_request_slowlog_json_appender.layout.type = OpenSearchJsonLayout
+appender.search_request_slowlog_json_appender.layout.type_name = search_request_slowlog
+appender.search_request_slowlog_json_appender.layout.opensearchmessagefields=message,took,took_millis,phase_took,total_hits,search_type,shards,source,id
+
+appender.search_request_slowlog_json_appender.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs\
+  .cluster_name}_index_search_slowlog-%i.json.gz
+appender.search_request_slowlog_json_appender.policies.type = Policies
+appender.search_request_slowlog_json_appender.policies.size.type = SizeBasedTriggeringPolicy
+appender.search_request_slowlog_json_appender.policies.size.size = 1GB
+appender.search_request_slowlog_json_appender.strategy.type = DefaultRolloverStrategy
+appender.search_request_slowlog_json_appender.strategy.max = 4
+#################################################
+######## Search Request Slowlog Log File -  old style pattern ####
+appender.search_request_slowlog_log_appender.type = RollingFile
+appender.search_request_slowlog_log_appender.name = search_request_slowlog_log_appender
+appender.search_request_slowlog_log_appender.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\
+  _index_search_slowlog.log
+appender.search_request_slowlog_log_appender.filePermissions = rw-r-----
+appender.search_request_slowlog_log_appender.layout.type = PatternLayout
+appender.search_request_slowlog_log_appender.layout.pattern = [%d{ISO8601}][%-5p][%c{1.}] [%node_name]%marker %m%n
+
+appender.search_request_slowlog_log_appender.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\
+  _index_search_slowlog-%i.log.gz
+appender.search_request_slowlog_log_appender.policies.type = Policies
+appender.search_request_slowlog_log_appender.policies.size.type = SizeBasedTriggeringPolicy
+appender.search_request_slowlog_log_appender.policies.size.size = 1GB
+appender.search_request_slowlog_log_appender.strategy.type = DefaultRolloverStrategy
+appender.search_request_slowlog_log_appender.strategy.max = 4
+#################################################
+logger.search_request_slowlog_logger.name = cluster.search.request.slowlog
+logger.search_request_slowlog_logger.level = trace
+logger.search_request_slowlog_logger.appenderRef.search_request_slowlog_json_appender.ref = search_request_slowlog_json_appender
+logger.search_request_slowlog_logger.appenderRef.search_request_slowlog_log_appender.ref = search_request_slowlog_log_appender
+logger.search_request_slowlog_logger.additivity = false
+
 ######## Search slowlog JSON ####################
 appender.index_search_slowlog_rolling.type = RollingFile
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling

libs/common/src/main/java/org/opensearch/common/Explicit.java

@@ -32,6 +32,8 @@
 
 package org.opensearch.common;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.Objects;
 
 /**
@@ -43,8 +45,9 @@
  * field mapping settings it is preferable to preserve an explicit
  * choice rather than a choice made only made implicitly by defaults.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Explicit<T> {
 
     private final T value;

libs/common/src/main/java/org/opensearch/common/lease/Releasable.java

@@ -32,13 +32,16 @@
 
 package org.opensearch.common.lease;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.io.Closeable;
 
 /**
  * Specialization of {@link AutoCloseable} for calls that might not throw a checked exception.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface Releasable extends Closeable {
 
     @Override

libs/core/src/main/java/org/opensearch/core/action/ActionResponse.java

@@ -32,6 +32,7 @@
 
 package org.opensearch.core.action;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.transport.TransportResponse;
 
@@ -42,6 +43,7 @@
  *
  * @opensearch.internal
  */
+@PublicApi(since = "1.0.0")
 public abstract class ActionResponse extends TransportResponse {
 
     public ActionResponse() {}

libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java

@@ -32,6 +32,8 @@
 
 package org.opensearch.core.common.io.stream;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.io.IOException;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -41,8 +43,9 @@
  * across the wire" using OpenSearch's internal protocol. If the implementer also implements equals and hashCode then a copy made by
  * serializing and deserializing must be equal and have the same hashCode. It isn't required that such a copy be entirely unchanged.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.8.0")
 public interface Writeable {
     /**
      * A WriteableRegistry registers {@link Writer} methods for writing data types over a
@@ -135,8 +138,11 @@ public static Class<?> getCustomClassFromInstance(final Object value) {
      *     out.writeMapOfLists(someMap, StreamOutput::writeString, StreamOutput::writeString);
      * }
      * </code></pre>
+     *
+     * @opensearch.api
      */
     @FunctionalInterface
+    @PublicApi(since = "2.8.0")
     interface Writer<V> {
 
         /**
@@ -161,8 +167,11 @@ interface Writer<V> {
      *     this.someMap = in.readMapOfLists(StreamInput::readString, StreamInput::readString);
      * }
      * </code></pre>
+     *
+     * @opensearch.api
      */
     @FunctionalInterface
+    @PublicApi(since = "2.8.0")
     interface Reader<V> {
 
         /**

libs/core/src/main/java/org/opensearch/core/common/util/ByteArray.java

@@ -33,14 +33,16 @@
 package org.opensearch.core.common.util;
 
 import org.apache.lucene.util.BytesRef;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.nio.ByteBuffer;
 
 /**
  * Abstraction of an array of byte values.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ByteArray extends BigArray {
 
     /**

libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java

@@ -32,6 +32,7 @@
 
 package org.opensearch.core.indices.breaker;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -43,8 +44,9 @@
 /**
  * Stats class encapsulating all of the different circuit breaker stats
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class AllCircuitBreakerStats implements Writeable, ToXContentFragment {
 
     /** An array of all the circuit breaker stats */

libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerService.java

@@ -32,20 +32,18 @@
 
 package org.opensearch.core.indices.breaker;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
 import org.opensearch.core.common.breaker.CircuitBreaker;
 
 /**
  * Interface for Circuit Breaker services, which provide breakers to classes
  * that load field data.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class CircuitBreakerService extends AbstractLifecycleComponent {
-    private static final Logger logger = LogManager.getLogger(CircuitBreakerService.class);
-
     protected CircuitBreakerService() {}
 
     /**

libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java

@@ -32,6 +32,7 @@
 
 package org.opensearch.core.indices.breaker;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -45,8 +46,9 @@
 /**
  * Class encapsulating stats about the {@link org.opensearch.core.common.breaker.CircuitBreaker}
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CircuitBreakerStats implements Writeable, ToXContentObject {
 
     /** The name of the circuit breaker */

libs/core/src/main/java/org/opensearch/core/xcontent/ObjectParser.java

@@ -32,6 +32,7 @@
 package org.opensearch.core.xcontent;
 
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 
 import java.io.IOException;
@@ -83,8 +84,9 @@
  * It's highly recommended to use the high level declare methods like {@link #declareString(BiConsumer, ParseField)} instead of
  * {@link #declareField} which can be used to implement exceptional parsing operations not covered by the high level methods.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class ObjectParser<Value, Context> extends AbstractObjectParser<Value, Context>
     implements
         BiFunction<XContentParser, Context, Value>,

server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationSuiteIT.java

@@ -8,13 +8,15 @@
 
 package org.opensearch.indices.replication;
 
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.junit.Before;
 
+@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9499")
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, minNumDataNodes = 2)
 public class SegmentReplicationSuiteIT extends SegmentReplicationBaseIT {
 

server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java

@@ -169,7 +169,7 @@ private Settings.Builder settingsBuilder() {
         return Settings.builder().put(indexSettings());
     }
 
-    public void testFieldDataStats() {
+    public void testFieldDataStats() throws InterruptedException {
         assertAcked(
             client().admin()
                 .indices()
@@ -182,6 +182,7 @@ public void testFieldDataStats() {
         client().prepareIndex("test").setId("1").setSource("field", "value1", "field2", "value1").execute().actionGet();
         client().prepareIndex("test").setId("2").setSource("field", "value2", "field2", "value2").execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
+        indexRandomForConcurrentSearch("test");
 
         NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
         assertThat(
@@ -305,6 +306,7 @@ public void testClearAllCaches() throws Exception {
         client().prepareIndex("test").setId("1").setSource("field", "value1").execute().actionGet();
         client().prepareIndex("test").setId("2").setSource("field", "value2").execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
+        indexRandomForConcurrentSearch("test");
 
         NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
         assertThat(

server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java

@@ -14,14 +14,14 @@
 import org.opensearch.action.admin.indices.flush.FlushResponse;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
-import org.opensearch.common.util.concurrent.AbstractAsyncTask;
 import org.opensearch.common.util.concurrent.UncategorizedExecutionException;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.index.IndexService;
+import org.opensearch.index.IndexServiceTestUtils;
 import org.opensearch.index.remote.RemoteSegmentTransferTracker;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.indices.IndicesService;
@@ -175,7 +175,7 @@ public void testAsyncTrimTaskSucceeds() {
 
         logger.info("Increasing the frequency of async trim task to ensure it runs in background while indexing");
         IndexService indexService = internalCluster().getInstance(IndicesService.class, dataNodeName).iterator().next();
-        ((AbstractAsyncTask) indexService.getTrimTranslogTask()).setInterval(TimeValue.timeValueMillis(100));
+        IndexServiceTestUtils.setTrimTranslogTaskInterval(indexService, TimeValue.timeValueMillis(100));
 
         logger.info("--> Indexing data");
         indexData(randomIntBetween(2, 5), true);

server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java

@@ -93,6 +93,7 @@ public void testSimpleTimeout() throws Exception {
             client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").get();
         }
         refresh("test");
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch("test")
             .setTimeout(new TimeValue(5, TimeUnit.MILLISECONDS))
@@ -104,12 +105,11 @@ public void testSimpleTimeout() throws Exception {
     }
 
     public void testSimpleDoesNotTimeout() throws Exception {
-        final int numDocs = 10;
+        final int numDocs = 9;
         for (int i = 0; i < numDocs; i++) {
-            client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").get();
+            client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         }
-        refresh("test");
-
+        indexRandomForConcurrentSearch("test");
         SearchResponse searchResponse = client().prepareSearch("test")
             .setTimeout(new TimeValue(10000, TimeUnit.SECONDS))
             .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap())))
@@ -122,7 +122,7 @@ public void testSimpleDoesNotTimeout() throws Exception {
 
     public void testPartialResultsIntolerantTimeout() throws Exception {
         client().prepareIndex("test").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
-
+        indexRandomForConcurrentSearch("test");
         OpenSearchException ex = expectThrows(
             OpenSearchException.class,
             () -> client().prepareSearch("test")

server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NaNSortingIT.java

@@ -172,6 +172,7 @@ public void setupSuiteScopeCluster() throws Exception {
             client().prepareIndex("idx").setSource(source.endObject()).get();
         }
         refresh();
+        indexRandomForMultipleSlices("idx");
         ensureSearchable();
     }