From 2eed8a81363696ebe9a45e23ccbf4560078800b8 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Thu, 5 Jan 2023 18:23:50 +0000 Subject: [PATCH 01/25] Initial Draft for adding segment_replication API Signed-off-by: Rishikesh1159 --- .../api/cat.segment_replication.json | 99 +++++++++++++++ .../org/opensearch/action/ActionModule.java | 5 + .../SegmentReplicationAction.java | 20 +++ .../SegmentReplicationRequest.java | 89 +++++++++++++ .../SegmentReplicationRequestBuilder.java | 30 +++++ .../SegmentReplicationResponse.java | 94 ++++++++++++++ .../TransportSegmentReplicationAction.java | 119 ++++++++++++++++++ .../opensearch/client/IndicesAdminClient.java | 18 +++ .../client/support/AbstractClient.java | 19 +++ .../org/opensearch/index/IndexModule.java | 18 ++- .../org/opensearch/index/IndexService.java | 11 ++ .../opensearch/index/shard/IndexShard.java | 10 ++ .../opensearch/indices/IndicesService.java | 12 +- .../replication/SegmentReplicationState.java | 47 +++++-- .../SegmentReplicationStatsState.java | 118 +++++++++++++++++ .../replication/SegmentReplicationTarget.java | 1 + .../SegmentReplicationTargetService.java | 2 + .../main/java/org/opensearch/node/Node.java | 10 ++ .../opensearch/plugins/IndexStorePlugin.java | 12 ++ .../cat/RestCatSegmentReplicationAction.java | 84 +++++++++++++ 20 files changed, 805 insertions(+), 13 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java create mode 100644 server/src/main/java/org/opensearch/indices/replication/SegmentReplicationStatsState.java create mode 100644 server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json new file mode 100644 index 0000000000000..541467ca313ff --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json @@ -0,0 +1,99 @@ +{ + "cat.segment_replication":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-segment_replication.html", + "description":"Returns information about index Segment Replication events, both on-going and completed." + }, + "stability":"stable", + "url":{ + "paths":[ + { + "path":"/_cat/segment_replication", + "methods":[ + "GET" + ] + }, + { + "path":"/_cat/segment_replication/{index}", + "methods":[ + "GET" + ], + "parts":{ + "index":{ + "type":"list", + "description":"Comma-separated list or wildcard expression of index names to limit the returned information" + } + } + } + ] + }, + "params":{ + "format":{ + "type":"string", + "description":"a short version of the Accept header, e.g. json, yaml" + }, + "active_only":{ + "type":"boolean", + "description":"If `true`, the response only includes ongoing segment replications", + "default":false + }, + "bytes":{ + "type":"enum", + "description":"The unit in which to display byte values", + "options":[ + "b", + "k", + "kb", + "m", + "mb", + "g", + "gb", + "t", + "tb", + "p", + "pb" + ] + }, + "detailed":{ + "type":"boolean", + "description":"If `true`, the response includes detailed information about segment replications", + "default":false + }, + "h":{ + "type":"list", + "description":"Comma-separated list of column names to display" + }, + "help":{ + "type":"boolean", + "description":"Return help information", + "default":false + }, + "index":{ + "type":"list", + "description":"Comma-separated list or wildcard expression of index names to limit the returned information" + }, + "s":{ + "type":"list", + "description":"Comma-separated list of column names or column aliases to sort by" + }, + "time":{ + "type":"enum", + "description":"The unit in which to display time values", + "options":[ + "d", + "h", + "m", + "s", + "ms", + "micros", + "nanos" + ] + }, + "v":{ + "type":"boolean", + "description":"Verbose mode. Display column headers", + "default":false + } + } + } +} diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index bba3aabdd61f9..96354b579ebda 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -176,6 +176,8 @@ import org.opensearch.action.admin.indices.resolve.ResolveIndexAction; import org.opensearch.action.admin.indices.rollover.RolloverAction; import org.opensearch.action.admin.indices.rollover.TransportRolloverAction; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationAction; +import org.opensearch.action.admin.indices.segment_replication.TransportSegmentReplicationAction; import org.opensearch.action.admin.indices.segments.IndicesSegmentsAction; import org.opensearch.action.admin.indices.segments.PitSegmentsAction; import org.opensearch.action.admin.indices.segments.TransportIndicesSegmentsAction; @@ -397,6 +399,7 @@ import org.opensearch.rest.action.cat.RestAllocationAction; import org.opensearch.rest.action.cat.RestCatAction; import org.opensearch.rest.action.cat.RestCatRecoveryAction; +import org.opensearch.rest.action.cat.RestCatSegmentReplicationAction; import org.opensearch.rest.action.cat.RestFielddataAction; import org.opensearch.rest.action.cat.RestHealthAction; import org.opensearch.rest.action.cat.RestIndicesAction; @@ -649,6 +652,7 @@ public void reg actions.register(ExplainAction.INSTANCE, TransportExplainAction.class); actions.register(ClearScrollAction.INSTANCE, TransportClearScrollAction.class); actions.register(RecoveryAction.INSTANCE, TransportRecoveryAction.class); + actions.register(SegmentReplicationAction.INSTANCE, TransportSegmentReplicationAction.class); actions.register(NodesReloadSecureSettingsAction.INSTANCE, TransportNodesReloadSecureSettingsAction.class); actions.register(AutoCreateAction.INSTANCE, AutoCreateAction.TransportAction.class); @@ -870,6 +874,7 @@ public void initRestHandlers(Supplier nodesInCluster) { // CAT API registerHandler.accept(new RestAllocationAction()); + registerHandler.accept(new RestCatSegmentReplicationAction()); registerHandler.accept(new RestShardsAction()); registerHandler.accept(new RestClusterManagerAction()); registerHandler.accept(new RestNodesAction()); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationAction.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationAction.java new file mode 100644 index 0000000000000..3f56d264fe82e --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationAction.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.segment_replication; + +import org.opensearch.action.ActionType; + +public class SegmentReplicationAction extends ActionType { + public static final SegmentReplicationAction INSTANCE = new SegmentReplicationAction(); + public static final String NAME = "indices:monitor/segment_replication"; + + private SegmentReplicationAction() { + super(NAME, SegmentReplicationResponse::new); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java new file mode 100644 index 0000000000000..30befe8abfee0 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java @@ -0,0 +1,89 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.segment_replication; + +import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.broadcast.BroadcastRequest; +import org.opensearch.common.Strings; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +public class SegmentReplicationRequest extends BroadcastRequest { + private boolean detailed = false; // Provides extra details in the response + private boolean activeOnly = false; // Only reports on active recoveries + + /** + * Constructs a request for segment replication information for all shards + */ + public SegmentReplicationRequest() { + this(Strings.EMPTY_ARRAY); + } + + public SegmentReplicationRequest(StreamInput in) throws IOException { + super(in); + detailed = in.readBoolean(); + activeOnly = in.readBoolean(); + } + + /** + * Constructs a request for segment replication information for all shards for the given indices + * + * @param indices Comma-separated list of indices about which to gather segment replication information + */ + public SegmentReplicationRequest(String... indices) { + super(indices, IndicesOptions.STRICT_EXPAND_OPEN_CLOSED); + } + + /** + * True if detailed flag is set, false otherwise. This value if false by default. + * + * @return True if detailed flag is set, false otherwise + */ + public boolean detailed() { + return detailed; + } + + /** + * Set value of the detailed flag. Detailed requests will contain extra + * information. + * + * @param detailed Whether or not to set the detailed flag + */ + public void detailed(boolean detailed) { + this.detailed = detailed; + } + + /** + * True if activeOnly flag is set, false otherwise. This value is false by default. + * + * @return True if activeOnly flag is set, false otherwise + */ + public boolean activeOnly() { + return activeOnly; + } + + /** + * Set value of the activeOnly flag. If true, this request will only response with + * on-going recovery information. + * + * @param activeOnly Whether or not to set the activeOnly flag. + */ + public void activeOnly(boolean activeOnly) { + this.activeOnly = activeOnly; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(detailed); + out.writeBoolean(activeOnly); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java new file mode 100644 index 0000000000000..de040e0b4c352 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.segment_replication; + +import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder; +import org.opensearch.client.OpenSearchClient; + +public class SegmentReplicationRequestBuilder extends BroadcastOperationRequestBuilder { + + public SegmentReplicationRequestBuilder(OpenSearchClient client, SegmentReplicationAction action) { + super(client, action, new SegmentReplicationRequest()); + } + + public SegmentReplicationRequestBuilder setDetailed(boolean detailed) { + request.detailed(detailed); + return this; + } + + public SegmentReplicationRequestBuilder setActiveOnly(boolean activeOnly) { + request.activeOnly(activeOnly); + return this; + } + +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java new file mode 100644 index 0000000000000..8eb5c054120b1 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java @@ -0,0 +1,94 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.segment_replication; + +import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.action.support.broadcast.BroadcastResponse; +import org.opensearch.common.Strings; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.indices.replication.SegmentReplicationStatsState; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class SegmentReplicationResponse extends BroadcastResponse { + private final Map> shardSegmentReplicationStatsStates; + + public SegmentReplicationResponse(StreamInput in) throws IOException { + super(in); + shardSegmentReplicationStatsStates = in.readMapOfLists(StreamInput::readString, SegmentReplicationStatsState:: new); + } + + /** + * Constructs segment replication information for a collection of indices and associated shards. Keeps track of how many total shards + * were seen, and out of those how many were successfully processed and how many failed. + * + * @param totalShards Total count of shards seen + * @param successfulShards Count of shards successfully processed + * @param failedShards Count of shards which failed to process + * @param shardSegmentReplicationStatsStates Map of indices to shard recovery information + * @param shardFailures List of failures processing shards + */ + public SegmentReplicationResponse( + int totalShards, + int successfulShards, + int failedShards, + Map> shardSegmentReplicationStatsStates, + List shardFailures + ) { + super(totalShards, successfulShards, failedShards, shardFailures); + this.shardSegmentReplicationStatsStates = shardSegmentReplicationStatsStates; + } + + public boolean hasSegmentReplication() { + return shardSegmentReplicationStatsStates.size() > 0; + } + + public Map> shardSegmentReplicationStatsStates() { + return shardSegmentReplicationStatsStates; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (hasSegmentReplication()) { + for (String index : shardSegmentReplicationStatsStates.keySet()) { + List segmentReplicationStatsStates = shardSegmentReplicationStatsStates.get(index); + if (segmentReplicationStatsStates == null || segmentReplicationStatsStates.size() == 0) { + continue; + } + builder.startObject(index); + builder.startArray("shards"); + for (SegmentReplicationStatsState segmentReplicationStatsState : segmentReplicationStatsStates) { + builder.startObject(); + segmentReplicationStatsState.toXContent(builder, params); + builder.endObject(); + } + builder.endArray(); + builder.endObject(); + } + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeMapOfLists(shardSegmentReplicationStatsStates, StreamOutput::writeString, (o, v) -> v.writeTo(o)); + } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java new file mode 100644 index 0000000000000..7e2976a62b9a8 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java @@ -0,0 +1,119 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.segment_replication; + +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardsIterator; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.index.IndexService; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.indices.IndicesService; +import org.opensearch.indices.replication.SegmentReplicationStatsState; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class TransportSegmentReplicationAction extends TransportBroadcastByNodeAction { + + private final IndicesService indicesService; + + @Inject + public TransportSegmentReplicationAction( + ClusterService clusterService, + TransportService transportService, + IndicesService indicesService, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super( + SegmentReplicationAction.NAME, + clusterService, + transportService, + actionFilters, + indexNameExpressionResolver, + SegmentReplicationRequest::new, + ThreadPool.Names.MANAGEMENT + ); + this.indicesService = indicesService; + } + + @Override + protected SegmentReplicationStatsState readShardResult(StreamInput in) throws IOException { + return new SegmentReplicationStatsState(in); + } + + @Override + protected SegmentReplicationResponse newResponse( + SegmentReplicationRequest request, + int totalShards, + int successfulShards, + int failedShards, + List responses, + List shardFailures, + ClusterState clusterState + ) { + Map> shardResponses = new HashMap<>(); + for (SegmentReplicationStatsState segmentReplicationStatsState : responses) { + if (segmentReplicationStatsState == null) { + continue; + } + String indexName = segmentReplicationStatsState.getShardRouting().getIndexName(); + if (!shardResponses.containsKey(indexName)) { + shardResponses.put(indexName, new ArrayList<>()); + } + if (request.activeOnly()) { + shardResponses.get(indexName).add(segmentReplicationStatsState); + } else { + shardResponses.get(indexName).add(segmentReplicationStatsState); + } + } + return new SegmentReplicationResponse(totalShards, successfulShards, failedShards, shardResponses, shardFailures); + } + + @Override + protected SegmentReplicationRequest readRequestFrom(StreamInput in) throws IOException { + return new SegmentReplicationRequest(in); + } + + @Override + protected SegmentReplicationStatsState shardOperation(SegmentReplicationRequest request, ShardRouting shardRouting) { + IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); + IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); + return indexShard.getSegmentReplicationStatsState(); + } + + @Override + protected ShardsIterator shards(ClusterState state, SegmentReplicationRequest request, String[] concreteIndices) { + return state.routingTable().allShardsIncludingRelocationTargets(concreteIndices); + } + + @Override + protected ClusterBlockException checkGlobalBlock(ClusterState state, SegmentReplicationRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, SegmentReplicationRequest request, String[] concreteIndices) { + return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices); + } +} diff --git a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java index c9cd0d0900b5a..62f13560a8e89 100644 --- a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java +++ b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java @@ -91,6 +91,9 @@ import org.opensearch.action.admin.indices.rollover.RolloverRequest; import org.opensearch.action.admin.indices.rollover.RolloverRequestBuilder; import org.opensearch.action.admin.indices.rollover.RolloverResponse; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationRequest; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationRequestBuilder; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationResponse; import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse; import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequestBuilder; @@ -185,11 +188,26 @@ public interface IndicesAdminClient extends OpenSearchClient { */ void recoveries(RecoveryRequest request, ActionListener listener); + /** + *Indices segment replication + */ + ActionFuture segment_replication(SegmentReplicationRequest request); + + /** + *Indices segment replication + */ + void segment_replication(SegmentReplicationRequest request, ActionListener listener); + /** * Indices recoveries */ RecoveryRequestBuilder prepareRecoveries(String... indices); + /** + * Indices segment replication + */ + SegmentReplicationRequestBuilder prepareSegment_Replication(String... indices); + /** * The segments of one or more indices. * diff --git a/server/src/main/java/org/opensearch/client/support/AbstractClient.java b/server/src/main/java/org/opensearch/client/support/AbstractClient.java index 828ca5f8083ee..83cb72cd76cdc 100644 --- a/server/src/main/java/org/opensearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/opensearch/client/support/AbstractClient.java @@ -260,6 +260,10 @@ import org.opensearch.action.admin.indices.rollover.RolloverRequest; import org.opensearch.action.admin.indices.rollover.RolloverRequestBuilder; import org.opensearch.action.admin.indices.rollover.RolloverResponse; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationAction; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationRequest; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationRequestBuilder; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationResponse; import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse; import org.opensearch.action.admin.indices.segments.IndicesSegmentsAction; import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; @@ -1775,6 +1779,21 @@ public RecoveryRequestBuilder prepareRecoveries(String... indices) { return new RecoveryRequestBuilder(this, RecoveryAction.INSTANCE).setIndices(indices); } + @Override + public ActionFuture segment_replication(final SegmentReplicationRequest request) { + return execute(SegmentReplicationAction.INSTANCE, request); + } + + @Override + public void segment_replication(final SegmentReplicationRequest request, final ActionListener listener){ + execute(SegmentReplicationAction.INSTANCE, request, listener); + } + + @Override + public SegmentReplicationRequestBuilder prepareSegment_Replication(String... indices) { + return new SegmentReplicationRequestBuilder(this, SegmentReplicationAction.INSTANCE).setIndices(indices); + } + @Override public ActionFuture segments(final IndicesSegmentsRequest request) { return execute(IndicesSegmentsAction.INSTANCE, request); diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index 69543577f48b4..f0a395a023b18 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -76,6 +76,7 @@ import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; import org.opensearch.indices.recovery.RecoveryState; +import org.opensearch.indices.replication.SegmentReplicationStatsState; import org.opensearch.plugins.IndexStorePlugin; import org.opensearch.repositories.RepositoriesService; import org.opensearch.script.ScriptService; @@ -123,6 +124,8 @@ public final class IndexModule { private static final IndexStorePlugin.RecoveryStateFactory DEFAULT_RECOVERY_STATE_FACTORY = RecoveryState::new; + private static final IndexStorePlugin.SegmentReplicationStatsStateFactory DEFAULT_SEGMENT_REPLICATION_STATS_STATE_FACTORY = SegmentReplicationStatsState::new; + public static final Setting INDEX_STORE_TYPE_SETTING = new Setting<>( "index.store.type", "", @@ -197,6 +200,8 @@ public final class IndexModule { private final BooleanSupplier allowExpensiveQueries; private final Map recoveryStateFactories; + private final Map segmentReplicationStatsStateFactories; + /** * Construct the index module for the index with the specified index settings. The index module contains extension points for plugins * via {@link org.opensearch.plugins.PluginsService#onIndexModule(IndexModule)}. @@ -214,7 +219,8 @@ public IndexModule( final Map directoryFactories, final BooleanSupplier allowExpensiveQueries, final IndexNameExpressionResolver expressionResolver, - final Map recoveryStateFactories + final Map recoveryStateFactories, + final Map segmentReplicationStatsStateFactories ) { this.indexSettings = indexSettings; this.analysisRegistry = analysisRegistry; @@ -226,6 +232,7 @@ public IndexModule( this.allowExpensiveQueries = allowExpensiveQueries; this.expressionResolver = expressionResolver; this.recoveryStateFactories = recoveryStateFactories; + this.segmentReplicationStatsStateFactories = segmentReplicationStatsStateFactories; } /** @@ -501,6 +508,7 @@ public IndexService newIndexService( eventListener.beforeIndexCreated(indexSettings.getIndex(), indexSettings.getSettings()); final IndexStorePlugin.DirectoryFactory directoryFactory = getDirectoryFactory(indexSettings, directoryFactories); final IndexStorePlugin.RecoveryStateFactory recoveryStateFactory = getRecoveryStateFactory(indexSettings, recoveryStateFactories); + final IndexStorePlugin.SegmentReplicationStatsStateFactory segmentReplicationStatsStateFactory = getSegmentReplicationStateFactory(indexSettings, segmentReplicationStatsStateFactories); QueryCache queryCache = null; IndexAnalyzers indexAnalyzers = null; boolean success = false; @@ -549,6 +557,7 @@ public IndexService newIndexService( expressionResolver, valuesSourceRegistry, recoveryStateFactory, + segmentReplicationStatsStateFactory, repositoriesServiceSupplier ); success = true; @@ -609,6 +618,13 @@ private static IndexStorePlugin.RecoveryStateFactory getRecoveryStateFactory( return factory; } + private static IndexStorePlugin.SegmentReplicationStatsStateFactory getSegmentReplicationStateFactory( + final IndexSettings indexSettings, + final Map segmentReplicationStateFactories + ) { + return DEFAULT_SEGMENT_REPLICATION_STATS_STATE_FACTORY; + } + /** * creates a new mapper service to do administrative work like mapping updates. This *should not* be used for document parsing. * doing so will result in an exception. diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 36237b56987e6..83b897fb5b87f 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -97,6 +97,7 @@ import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; import org.opensearch.indices.recovery.RecoveryState; +import org.opensearch.indices.replication.SegmentReplicationStatsState; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.plugins.IndexStorePlugin; import org.opensearch.repositories.RepositoriesService; @@ -142,6 +143,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final IndexStorePlugin.DirectoryFactory directoryFactory; private final IndexStorePlugin.RemoteDirectoryFactory remoteDirectoryFactory; private final IndexStorePlugin.RecoveryStateFactory recoveryStateFactory; + + private final IndexStorePlugin.SegmentReplicationStatsStateFactory segmentReplicationStatsStateFactory; private final CheckedFunction readerWrapper; private final IndexCache indexCache; private final MapperService mapperService; @@ -209,6 +212,7 @@ public IndexService( IndexNameExpressionResolver expressionResolver, ValuesSourceRegistry valuesSourceRegistry, IndexStorePlugin.RecoveryStateFactory recoveryStateFactory, + IndexStorePlugin.SegmentReplicationStatsStateFactory segmentReplicationStatsStateFactory, Supplier repositoriesServiceSupplier ) { super(indexSettings); @@ -270,6 +274,7 @@ public IndexService( this.directoryFactory = directoryFactory; this.remoteDirectoryFactory = remoteDirectoryFactory; this.recoveryStateFactory = recoveryStateFactory; + this.segmentReplicationStatsStateFactory = segmentReplicationStatsStateFactory; this.engineFactory = Objects.requireNonNull(engineFactory); this.engineConfigFactory = Objects.requireNonNull(engineConfigFactory); // initialize this last -- otherwise if the wrapper requires any other member to be non-null we fail with an NPE @@ -541,6 +546,7 @@ public synchronized IndexShard createShard( new StoreCloseListener(shardId, () -> eventListener.onStoreClosed(shardId)) ); eventListener.onStoreCreated(shardId); + SegmentReplicationStatsState segmentReplicationStatsState = createSegmentReplicationStatsState(routing); indexShard = new IndexShard( routing, this.indexSettings, @@ -564,6 +570,7 @@ public synchronized IndexShard createShard( circuitBreakerService, translogFactory, this.indexSettings.isSegRepEnabled() ? checkpointPublisher : null, + segmentReplicationStatsState, remoteStore ); eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created"); @@ -662,6 +669,10 @@ public RecoveryState createRecoveryState(ShardRouting shardRouting, DiscoveryNod return recoveryStateFactory.newRecoveryState(shardRouting, targetNode, sourceNode); } + public SegmentReplicationStatsState createSegmentReplicationStatsState(ShardRouting shardRouting){ + return segmentReplicationStatsStateFactory.newSegmentReplicationStatsState(shardRouting); + } + @Override public IndexSettings getIndexSettings() { return indexSettings; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index ed27ad7edd7d2..0e10d250b027f 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -171,6 +171,7 @@ import org.opensearch.indices.recovery.RecoveryListener; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.recovery.RecoveryTarget; +import org.opensearch.indices.replication.SegmentReplicationStatsState; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.repositories.RepositoriesService; @@ -277,6 +278,9 @@ Runnable getGlobalCheckpointSyncer() { @Nullable private volatile RecoveryState recoveryState; + @Nullable + private volatile SegmentReplicationStatsState segmentReplicationStatsState; + private final RecoveryStats recoveryStats = new RecoveryStats(); private final MeanMetric refreshMetric = new MeanMetric(); private final MeanMetric externalRefreshMetric = new MeanMetric(); @@ -344,6 +348,7 @@ public IndexShard( final CircuitBreakerService circuitBreakerService, final TranslogFactory translogFactory, @Nullable final SegmentReplicationCheckpointPublisher checkpointPublisher, + @Nullable final SegmentReplicationStatsState segmentReplicationStatsState, @Nullable final Store remoteStore ) throws IOException { super(shardRouting.shardId(), indexSettings); @@ -428,6 +433,7 @@ public boolean shouldCache(Query query) { this.useRetentionLeasesInPeerRecovery = replicationTracker.hasAllPeerRecoveryRetentionLeases(); this.refreshPendingLocationListener = new RefreshPendingLocationListener(); this.checkpointPublisher = checkpointPublisher; + this.segmentReplicationStatsState = segmentReplicationStatsState; this.remoteStore = remoteStore; this.translogFactory = translogFactory; } @@ -1477,6 +1483,10 @@ public final boolean shouldProcessCheckpoint(ReplicationCheckpoint requestCheckp return true; } + public SegmentReplicationStatsState getSegmentReplicationStatsState(){ + return this.segmentReplicationStatsState; + } + /** * gets a {@link Store.MetadataSnapshot} for the current directory. This method is safe to call in all lifecycle of the index shard, * without having to worry about the current state of the engine and concurrent flushes. diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 2946411fc9238..28fc6c0752b17 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -259,6 +259,8 @@ public class IndicesService extends AbstractLifecycleComponent private final Collection>> engineFactoryProviders; private final Map directoryFactories; private final Map recoveryStateFactories; + + private final Map segmentReplicationStatsStateFactories; final AbstractRefCounted indicesRefCount; // pkg-private for testing private final CountDownLatch closeLatch = new CountDownLatch(1); private volatile boolean idFieldDataEnabled; @@ -299,6 +301,7 @@ public IndicesService( Map directoryFactories, ValuesSourceRegistry valuesSourceRegistry, Map recoveryStateFactories, + Map segmentReplicationStatsStateFactories, IndexStorePlugin.RemoteDirectoryFactory remoteDirectoryFactory, Supplier repositoriesServiceSupplier ) { @@ -347,6 +350,7 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.directoryFactories = directoryFactories; this.recoveryStateFactories = recoveryStateFactories; + this.segmentReplicationStatsStateFactories = segmentReplicationStatsStateFactories; // doClose() is called when shutting down a node, yet there might still be ongoing requests // that we need to wait for before closing some resources such as the caches. In order to // avoid closing these resources while ongoing requests are still being processed, we use a @@ -414,6 +418,7 @@ public IndicesService( Map directoryFactories, ValuesSourceRegistry valuesSourceRegistry, Map recoveryStateFactories, + Map segmentReplicationStatsStateFactories, IndexStorePlugin.RemoteDirectoryFactory remoteDirectoryFactory, Supplier repositoriesServiceSupplier ) { @@ -462,6 +467,7 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.directoryFactories = directoryFactories; this.recoveryStateFactories = recoveryStateFactories; + this.segmentReplicationStatsStateFactories = segmentReplicationStatsStateFactories; // doClose() is called when shutting down a node, yet there might still be ongoing requests // that we need to wait for before closing some resources such as the caches. In order to // avoid closing these resources while ongoing requests are still being processed, we use a @@ -838,7 +844,8 @@ private synchronized IndexService createIndexService( directoryFactories, () -> allowExpensiveQueries, indexNameExpressionResolver, - recoveryStateFactories + recoveryStateFactories, + segmentReplicationStatsStateFactories ); for (IndexingOperationListener operationListener : indexingOperationListeners) { indexModule.addIndexOperationListener(operationListener); @@ -928,7 +935,8 @@ public synchronized MapperService createIndexMapperService(IndexMetadata indexMe directoryFactories, () -> allowExpensiveQueries, indexNameExpressionResolver, - recoveryStateFactories + recoveryStateFactories, + segmentReplicationStatsStateFactories ); pluginsService.onIndexModule(indexModule); return indexModule.newIndexMapperService(xContentRegistry, mapperRegistry, scriptService); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index 2e2e6df007c5c..a39bb258ab571 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -8,20 +8,26 @@ package org.opensearch.indices.replication; -import org.opensearch.common.collect.Tuple; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; +import org.opensearch.common.xcontent.ToXContent; +import org.opensearch.common.xcontent.ToXContentFragment; +import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.replication.common.ReplicationState; import org.opensearch.indices.replication.common.ReplicationTimer; -import java.util.ArrayList; -import java.util.List; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; /** * ReplicationState implementation to track Segment Replication events. * * @opensearch.internal */ -public class SegmentReplicationState implements ReplicationState { +public class SegmentReplicationState implements ReplicationState, ToXContentFragment, Writeable { /** * The stage of the recovery state @@ -69,7 +75,7 @@ public static Stage fromId(byte id) { private final ReplicationLuceneIndex index; private final ReplicationTimer overallTimer; private final ReplicationTimer stageTimer; - private final List> timingData; + private final Map timingData; private long replicationId; public SegmentReplicationState(ReplicationLuceneIndex index) { @@ -77,7 +83,8 @@ public SegmentReplicationState(ReplicationLuceneIndex index) { this.index = index; // Timing data will have as many entries as stages, plus one // additional entry for the overall timer - timingData = new ArrayList<>(Stage.values().length + 1); + timingData = new HashMap<>(Stage.values().length + 1); + overallTimer = new ReplicationTimer(); stageTimer = new ReplicationTimer(); stageTimer.start(); @@ -90,6 +97,16 @@ public SegmentReplicationState(ReplicationLuceneIndex index, long replicationId) this.replicationId = replicationId; } + public SegmentReplicationState(StreamInput in) throws IOException { + this(new ReplicationLuceneIndex(in)); + replicationId = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(replicationId); + } + @Override public ReplicationLuceneIndex getIndex() { return index; @@ -104,7 +121,7 @@ public ReplicationTimer getTimer() { return overallTimer; } - public List> getTimingData() { + public Map getTimingData() { return timingData; } @@ -125,7 +142,7 @@ protected void validateAndSetStage(Stage expected, Stage next) { private void stopTimersAndSetStage(Stage next) { // save the timing data for the current step stageTimer.stop(); - timingData.add(new Tuple<>(stage.name(), stageTimer.time())); + timingData.put(stage.name().toString(), stageTimer.time()); // restart the step timer stageTimer.reset(); stageTimer.start(); @@ -158,7 +175,7 @@ public void setStage(Stage stage) { validateAndSetStage(Stage.FINALIZE_REPLICATION, stage); // add the overall timing data overallTimer.stop(); - timingData.add(new Tuple<>("OVERALL", overallTimer.time())); + timingData.put("OVERALL", overallTimer.time()); break; case CANCELLED: if (this.stage == Stage.DONE) { @@ -166,10 +183,20 @@ public void setStage(Stage stage) { } stopTimersAndSetStage(Stage.CANCELLED); overallTimer.stop(); - timingData.add(new Tuple<>("OVERALL", overallTimer.time())); + timingData.put("OVERALL", overallTimer.time()); break; default: throw new IllegalArgumentException("unknown SegmentReplicationState.Stage [" + stage + "]"); } } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + + builder.field("REPLICATION_ID", replicationId); + builder.field("OVERALL TIMER", getTimer()); + builder.field("STAGE", stage.toString()); + + return builder; + } } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationStatsState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationStatsState.java new file mode 100644 index 0000000000000..705ec7f32dd3b --- /dev/null +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationStatsState.java @@ -0,0 +1,118 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.indices.replication; + +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.ToXContentFragment; +import org.opensearch.common.xcontent.XContentBuilder; + + +import java.io.IOException; + + +public class SegmentReplicationStatsState implements ToXContentFragment, Writeable { + + private SegmentReplicationState segmentReplicationState; + + public SegmentReplicationStatsState(ShardRouting shardRouting) { + this.shardRouting = shardRouting; + } + + public long getNumberOfReplicationEvents() { + return numberOfReplicationEvents; + } + + public void incrementCountOfSegmentReplicationEvents() { + numberOfReplicationEvents++; + } + + public ShardRouting getShardRouting() { + return shardRouting; + } + + private ShardRouting shardRouting; + + private long numberOfReplicationEvents; + + public SegmentReplicationStatsState(StreamInput in) throws IOException { + shardRouting = new ShardRouting(in); + numberOfReplicationEvents = in.readLong(); + segmentReplicationState =new SegmentReplicationState(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + shardRouting.writeTo(out); + out.writeLong(numberOfReplicationEvents); + segmentReplicationState.writeTo(out); + } + + protected void setSegmentReplicationState(SegmentReplicationState segmentReplicationState) { + this.segmentReplicationState = segmentReplicationState; + } + + public SegmentReplicationState getSegmentReplicationState(){ + return segmentReplicationState; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + + + builder.field(Fields.ID, shardRouting.shardId().id()); + builder.field(Fields.STAGE, segmentReplicationState.getStage()); + builder.field(Fields.COUNT_OF_SEGMENT_REPLICATION_EVENTS, numberOfReplicationEvents); + builder.timeField(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, segmentReplicationState.getTimer().startTime()); + if (segmentReplicationState.getTimer().stopTime() > 0) { + builder.timeField(Fields.STOP_TIME_IN_MILLIS, Fields.STOP_TIME, segmentReplicationState.getTimer().stopTime()); + } + builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(segmentReplicationState.getTimer().time())); + builder.field(Fields.INIT_STAGE, segmentReplicationState.getTimingData().get(SegmentReplicationState.Stage.INIT.toString())); + builder.field(Fields.REPLICATING_STAGE, segmentReplicationState.getTimingData().get(SegmentReplicationState.Stage.REPLICATING.toString())); + builder.field(Fields.GET_CHECKPOINT_INFO_STAGE, segmentReplicationState.getTimingData().get(SegmentReplicationState.Stage.GET_CHECKPOINT_INFO.toString())); + builder.field(Fields.FILE_DIFF_STAGE, segmentReplicationState.getTimingData().get(SegmentReplicationState.Stage.FILE_DIFF.toString())); + builder.field(Fields.GET_FILES_STAGE, segmentReplicationState.getTimingData().get(SegmentReplicationState.Stage.GET_FILES.toString())); + builder.field(Fields.FINALIZE_REPLICATION_STAGE, segmentReplicationState.getTimingData().get(SegmentReplicationState.Stage.FINALIZE_REPLICATION.toString())); + return builder; + } + + /** + * Fields used in the segment replication stats state + * + * @opensearch.internal + */ + static final class Fields { + static final String ID = "id"; + static final String COUNT_OF_SEGMENT_REPLICATION_EVENTS = "count_of_segment_replication_events"; + static final String STAGE = "stage"; + static final String START_TIME = "start_time"; + static final String START_TIME_IN_MILLIS = "start_time_in_millis"; + static final String STOP_TIME = "stop_time"; + static final String STOP_TIME_IN_MILLIS = "stop_time_in_millis"; + static final String TOTAL_TIME = "total_time"; + static final String TOTAL_TIME_IN_MILLIS = "total_time_in_millis"; + static final String SOURCE = "source"; + static final String HOST = "host"; + static final String TRANSPORT_ADDRESS = "transport_address"; + static final String IP = "ip"; + static final String NAME = "name"; + static final String TARGET = "target"; + static final String INIT_STAGE = "init_stage"; + static final String REPLICATING_STAGE = "replicating_stage"; + static final String GET_CHECKPOINT_INFO_STAGE = "get_checkpoint_info_stage"; + static final String FILE_DIFF_STAGE = "file_diff_stage"; + static final String GET_FILES_STAGE = "get_files_stage"; + static final String FINALIZE_REPLICATION_STAGE = "finalize_replication_stage"; + } + +} diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index aadcb577f6174..d56384f902cd0 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -68,6 +68,7 @@ public SegmentReplicationTarget( this.source = source; this.state = new SegmentReplicationState(stateIndex, getId()); this.multiFileWriter = new MultiFileWriter(indexShard.store(), stateIndex, getPrefix(), logger, this::ensureRefCount); + indexShard.getSegmentReplicationStatsState().setSegmentReplicationState(state); } @Override diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index b633f0fa3b9a0..42e42be24c48f 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -182,6 +182,7 @@ public void onReplicationDone(SegmentReplicationState state) { state.getTimingData() ) ); + replicaShard.getSegmentReplicationStatsState().incrementCountOfSegmentReplicationEvents(); // if we received a checkpoint during the copy event that is ahead of this // try and process it. if (latestReceivedCheckpoint.get(replicaShard.shardId()).isAheadOf(replicaShard.getLatestReplicationCheckpoint())) { @@ -205,6 +206,7 @@ public void onReplicationFailure(SegmentReplicationState state, ReplicationFaile state.getTimingData() ) ); + replicaShard.getSegmentReplicationStatsState().incrementCountOfSegmentReplicationEvents(); if (sendShardFailure == true) { logger.error("replication failure", e); replicaShard.failShard("replication failure", e); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index eec05f778a8ca..662f97a516a1f 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -648,6 +648,14 @@ protected Node( .flatMap(m -> m.entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + final Map segmentReplicationStatsStateFactories = pluginsService.filterPlugins( + IndexStorePlugin.class + ) + .stream() + .map(IndexStorePlugin::getSegmentReplicationStatsStateFactory) + .flatMap(m -> m.entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + final Map> systemIndexDescriptorMap = Collections.unmodifiableMap( pluginsService.filterPlugins(SystemIndexPlugin.class) .stream() @@ -689,6 +697,7 @@ protected Node( Map.copyOf(directoryFactories), searchModule.getValuesSourceRegistry(), recoveryStateFactories, + segmentReplicationStatsStateFactories, remoteDirectoryFactory, repositoriesServiceReference::get ); @@ -714,6 +723,7 @@ protected Node( Map.copyOf(directoryFactories), searchModule.getValuesSourceRegistry(), recoveryStateFactories, + segmentReplicationStatsStateFactories, remoteDirectoryFactory, repositoriesServiceReference::get ); diff --git a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java index 1dc90a21c2f70..a2e0c9e008e48 100644 --- a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java @@ -39,6 +39,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; import org.opensearch.indices.recovery.RecoveryState; +import org.opensearch.indices.replication.SegmentReplicationStatsState; import java.io.IOException; import java.util.Collections; @@ -103,6 +104,15 @@ interface RecoveryStateFactory { RecoveryState newRecoveryState(ShardRouting shardRouting, DiscoveryNode targetNode, @Nullable DiscoveryNode sourceNode); } + @FunctionalInterface + interface SegmentReplicationStatsStateFactory { + /** + * Creates a new {@link RecoveryState} per shard. This method is called once per shard on shard creation. + * @return a new RecoveryState instance + */ + SegmentReplicationStatsState newSegmentReplicationStatsState(ShardRouting shardRouting); + } + /** * The {@link RecoveryStateFactory} mappings for this plugin. When an index is created the recovery type setting * {@link org.opensearch.index.IndexModule#INDEX_RECOVERY_TYPE_SETTING} on the index will be examined and either use the default @@ -113,4 +123,6 @@ interface RecoveryStateFactory { default Map getRecoveryStateFactories() { return Collections.emptyMap(); } + + default Map getSegmentReplicationStatsStateFactory() { return Collections.emptyMap(); } } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java new file mode 100644 index 0000000000000..223dc484e2956 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.cat; + +import org.apache.lucene.util.CollectionUtil; +import org.opensearch.action.admin.indices.recovery.RecoveryResponse; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationRequest; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationResponse; +import org.opensearch.action.support.IndicesOptions; +import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.routing.RecoverySource; +import org.opensearch.common.Strings; +import org.opensearch.common.Table; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentOpenSearchExtension; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; + +import java.util.Comparator; +import java.util.List; +import java.util.Locale; + +import static java.util.Arrays.asList; +import static java.util.Collections.unmodifiableList; +import static org.opensearch.rest.RestRequest.Method.GET; + +public class RestCatSegmentReplicationAction extends AbstractCatAction { + @Override + public List routes() { + return unmodifiableList(asList(new RestHandler.Route(GET, "/_cat/segment_replication"), new RestHandler.Route(GET, "/_cat/segment_replication/{index}"))); + } + + @Override + public String getName() { + return "cat_segment_replication_action"; + } + + @Override + protected void documentation(StringBuilder sb) { + sb.append("/_cat/segment_replication\n"); + sb.append("/_cat/segment_replication/{index}\n"); + } + + @Override + public BaseRestHandler.RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) { + final SegmentReplicationRequest segmentReplicationRequest = new SegmentReplicationRequest(Strings.splitStringByCommaToArray(request.param("index"))); + segmentReplicationRequest.timeout(request.param("timeout")); + segmentReplicationRequest.detailed(request.paramAsBoolean("detailed", false)); + segmentReplicationRequest.activeOnly(request.paramAsBoolean("active_only", false)); + segmentReplicationRequest.indicesOptions(IndicesOptions.fromRequest(request, segmentReplicationRequest.indicesOptions())); + + return channel -> client.admin().indices().segment_replication(segmentReplicationRequest, new RestToXContentListener<>(channel) { +// @Override +// public RestResponse buildResponse(final SegmentReplicationResponse response) throws Exception { +// return RestTable.buildResponse(buildSegmentReplicationTable(request, response), channel); +// } + }); + } + + @Override + protected Table getTableWithHeader(RestRequest request) { + return null; + } + + /** + * buildRecoveryTable will build a table of recovery information suitable + * for displaying at the command line. + * + * @param request A Rest request + * @param response A recovery status response + * @return A table containing index, shardId, node, target size, recovered size and percentage for each recovering replica + */ + public Table buildRecoveryTable(RestRequest request, SegmentReplicationResponse response) { + return null; + } +} From 88cf749a30033df21fb709702058d01bc8466f86 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Mon, 9 Jan 2023 15:50:33 +0000 Subject: [PATCH 02/25] Adding bytes transfered in each segrep events and additional metrics. Signed-off-by: Rishikesh1159 --- .../SegmentReplicationResponse.java | 30 +- .../TransportSegmentReplicationAction.java | 26 +- .../org/opensearch/index/IndexModule.java | 20 +- .../org/opensearch/index/IndexService.java | 14 +- .../opensearch/index/shard/IndexShard.java | 14 +- .../opensearch/indices/IndicesService.java | 17 +- .../cluster/IndicesClusterStateService.java | 4 +- .../replication/SegmentReplicationState.java | 388 ++++++++++++++++-- .../SegmentReplicationStatsState.java | 118 ------ .../replication/SegmentReplicationTarget.java | 5 +- .../SegmentReplicationTargetService.java | 6 +- .../main/java/org/opensearch/node/Node.java | 8 +- .../opensearch/plugins/IndexStorePlugin.java | 8 +- 13 files changed, 435 insertions(+), 223 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/indices/replication/SegmentReplicationStatsState.java diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java index 8eb5c054120b1..15abde0cd142f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java @@ -14,18 +14,18 @@ import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.indices.replication.SegmentReplicationStatsState; +import org.opensearch.indices.replication.SegmentReplicationState; import java.io.IOException; import java.util.List; import java.util.Map; public class SegmentReplicationResponse extends BroadcastResponse { - private final Map> shardSegmentReplicationStatsStates; + private final Map> shardSegmentReplicationStates; public SegmentReplicationResponse(StreamInput in) throws IOException { super(in); - shardSegmentReplicationStatsStates = in.readMapOfLists(StreamInput::readString, SegmentReplicationStatsState:: new); + shardSegmentReplicationStates = in.readMapOfLists(StreamInput::readString, SegmentReplicationState:: new); } /** @@ -35,42 +35,42 @@ public SegmentReplicationResponse(StreamInput in) throws IOException { * @param totalShards Total count of shards seen * @param successfulShards Count of shards successfully processed * @param failedShards Count of shards which failed to process - * @param shardSegmentReplicationStatsStates Map of indices to shard recovery information + * @param shardSegmentReplicationStates Map of indices to shard recovery information * @param shardFailures List of failures processing shards */ public SegmentReplicationResponse( int totalShards, int successfulShards, int failedShards, - Map> shardSegmentReplicationStatsStates, + Map> shardSegmentReplicationStates, List shardFailures ) { super(totalShards, successfulShards, failedShards, shardFailures); - this.shardSegmentReplicationStatsStates = shardSegmentReplicationStatsStates; + this.shardSegmentReplicationStates = shardSegmentReplicationStates; } public boolean hasSegmentReplication() { - return shardSegmentReplicationStatsStates.size() > 0; + return shardSegmentReplicationStates.size() > 0; } - public Map> shardSegmentReplicationStatsStates() { - return shardSegmentReplicationStatsStates; + public Map> shardSegmentReplicationStates() { + return shardSegmentReplicationStates; } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); if (hasSegmentReplication()) { - for (String index : shardSegmentReplicationStatsStates.keySet()) { - List segmentReplicationStatsStates = shardSegmentReplicationStatsStates.get(index); - if (segmentReplicationStatsStates == null || segmentReplicationStatsStates.size() == 0) { + for (String index : shardSegmentReplicationStates.keySet()) { + List segmentReplicationStates = shardSegmentReplicationStates.get(index); + if (segmentReplicationStates == null || segmentReplicationStates.size() == 0) { continue; } builder.startObject(index); builder.startArray("shards"); - for (SegmentReplicationStatsState segmentReplicationStatsState : segmentReplicationStatsStates) { + for (SegmentReplicationState segmentReplicationState : segmentReplicationStates) { builder.startObject(); - segmentReplicationStatsState.toXContent(builder, params); + segmentReplicationState.toXContent(builder, params); builder.endObject(); } builder.endArray(); @@ -84,7 +84,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeMapOfLists(shardSegmentReplicationStatsStates, StreamOutput::writeString, (o, v) -> v.writeTo(o)); + out.writeMapOfLists(shardSegmentReplicationStates, StreamOutput::writeString, (o, v) -> v.writeTo(o)); } @Override diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java index 7e2976a62b9a8..70a33d0b980b9 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java @@ -23,7 +23,7 @@ import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; -import org.opensearch.indices.replication.SegmentReplicationStatsState; +import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -33,7 +33,7 @@ import java.util.List; import java.util.Map; -public class TransportSegmentReplicationAction extends TransportBroadcastByNodeAction { +public class TransportSegmentReplicationAction extends TransportBroadcastByNodeAction { private final IndicesService indicesService; @@ -58,8 +58,8 @@ public TransportSegmentReplicationAction( } @Override - protected SegmentReplicationStatsState readShardResult(StreamInput in) throws IOException { - return new SegmentReplicationStatsState(in); + protected SegmentReplicationState readShardResult(StreamInput in) throws IOException { + return new SegmentReplicationState(in); } @Override @@ -68,23 +68,23 @@ protected SegmentReplicationResponse newResponse( int totalShards, int successfulShards, int failedShards, - List responses, + List responses, List shardFailures, ClusterState clusterState ) { - Map> shardResponses = new HashMap<>(); - for (SegmentReplicationStatsState segmentReplicationStatsState : responses) { - if (segmentReplicationStatsState == null) { + Map> shardResponses = new HashMap<>(); + for (SegmentReplicationState segmentReplicationState : responses) { + if (segmentReplicationState == null) { continue; } - String indexName = segmentReplicationStatsState.getShardRouting().getIndexName(); + String indexName = segmentReplicationState.getShardRouting().getIndexName(); if (!shardResponses.containsKey(indexName)) { shardResponses.put(indexName, new ArrayList<>()); } if (request.activeOnly()) { - shardResponses.get(indexName).add(segmentReplicationStatsState); + shardResponses.get(indexName).add(segmentReplicationState); } else { - shardResponses.get(indexName).add(segmentReplicationStatsState); + shardResponses.get(indexName).add(segmentReplicationState); } } return new SegmentReplicationResponse(totalShards, successfulShards, failedShards, shardResponses, shardFailures); @@ -96,10 +96,10 @@ protected SegmentReplicationRequest readRequestFrom(StreamInput in) throws IOExc } @Override - protected SegmentReplicationStatsState shardOperation(SegmentReplicationRequest request, ShardRouting shardRouting) { + protected SegmentReplicationState shardOperation(SegmentReplicationRequest request, ShardRouting shardRouting) { IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); - return indexShard.getSegmentReplicationStatsState(); + return indexShard.getSegmentReplicationState(); } @Override diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index f0a395a023b18..3e7b24e0d9770 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -76,7 +76,7 @@ import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; import org.opensearch.indices.recovery.RecoveryState; -import org.opensearch.indices.replication.SegmentReplicationStatsState; +import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.plugins.IndexStorePlugin; import org.opensearch.repositories.RepositoriesService; import org.opensearch.script.ScriptService; @@ -124,7 +124,7 @@ public final class IndexModule { private static final IndexStorePlugin.RecoveryStateFactory DEFAULT_RECOVERY_STATE_FACTORY = RecoveryState::new; - private static final IndexStorePlugin.SegmentReplicationStatsStateFactory DEFAULT_SEGMENT_REPLICATION_STATS_STATE_FACTORY = SegmentReplicationStatsState::new; + private static final IndexStorePlugin.SegmentReplicationStateFactory DEFAULT_SEGMENT_REPLICATION_STATE_FACTORY = SegmentReplicationState ::new; public static final Setting INDEX_STORE_TYPE_SETTING = new Setting<>( "index.store.type", @@ -200,7 +200,7 @@ public final class IndexModule { private final BooleanSupplier allowExpensiveQueries; private final Map recoveryStateFactories; - private final Map segmentReplicationStatsStateFactories; + private final Map segmentReplicationStateFactories; /** * Construct the index module for the index with the specified index settings. The index module contains extension points for plugins @@ -220,7 +220,7 @@ public IndexModule( final BooleanSupplier allowExpensiveQueries, final IndexNameExpressionResolver expressionResolver, final Map recoveryStateFactories, - final Map segmentReplicationStatsStateFactories + final Map segmentReplicationStateFactories ) { this.indexSettings = indexSettings; this.analysisRegistry = analysisRegistry; @@ -232,7 +232,7 @@ public IndexModule( this.allowExpensiveQueries = allowExpensiveQueries; this.expressionResolver = expressionResolver; this.recoveryStateFactories = recoveryStateFactories; - this.segmentReplicationStatsStateFactories = segmentReplicationStatsStateFactories; + this.segmentReplicationStateFactories = segmentReplicationStateFactories; } /** @@ -508,7 +508,7 @@ public IndexService newIndexService( eventListener.beforeIndexCreated(indexSettings.getIndex(), indexSettings.getSettings()); final IndexStorePlugin.DirectoryFactory directoryFactory = getDirectoryFactory(indexSettings, directoryFactories); final IndexStorePlugin.RecoveryStateFactory recoveryStateFactory = getRecoveryStateFactory(indexSettings, recoveryStateFactories); - final IndexStorePlugin.SegmentReplicationStatsStateFactory segmentReplicationStatsStateFactory = getSegmentReplicationStateFactory(indexSettings, segmentReplicationStatsStateFactories); + final IndexStorePlugin.SegmentReplicationStateFactory segmentReplicationStateFactory = getSegmentReplicationStateFactory(indexSettings, segmentReplicationStateFactories); QueryCache queryCache = null; IndexAnalyzers indexAnalyzers = null; boolean success = false; @@ -557,7 +557,7 @@ public IndexService newIndexService( expressionResolver, valuesSourceRegistry, recoveryStateFactory, - segmentReplicationStatsStateFactory, + segmentReplicationStateFactory, repositoriesServiceSupplier ); success = true; @@ -618,11 +618,11 @@ private static IndexStorePlugin.RecoveryStateFactory getRecoveryStateFactory( return factory; } - private static IndexStorePlugin.SegmentReplicationStatsStateFactory getSegmentReplicationStateFactory( + private static IndexStorePlugin.SegmentReplicationStateFactory getSegmentReplicationStateFactory( final IndexSettings indexSettings, - final Map segmentReplicationStateFactories + final Map segmentReplicationStateFactories ) { - return DEFAULT_SEGMENT_REPLICATION_STATS_STATE_FACTORY; + return DEFAULT_SEGMENT_REPLICATION_STATE_FACTORY; } /** diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 83b897fb5b87f..b7c1a802b03f3 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -97,7 +97,7 @@ import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; import org.opensearch.indices.recovery.RecoveryState; -import org.opensearch.indices.replication.SegmentReplicationStatsState; +import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.plugins.IndexStorePlugin; import org.opensearch.repositories.RepositoriesService; @@ -144,7 +144,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final IndexStorePlugin.RemoteDirectoryFactory remoteDirectoryFactory; private final IndexStorePlugin.RecoveryStateFactory recoveryStateFactory; - private final IndexStorePlugin.SegmentReplicationStatsStateFactory segmentReplicationStatsStateFactory; + private final IndexStorePlugin.SegmentReplicationStateFactory segmentReplicationStateFactory; private final CheckedFunction readerWrapper; private final IndexCache indexCache; private final MapperService mapperService; @@ -212,7 +212,7 @@ public IndexService( IndexNameExpressionResolver expressionResolver, ValuesSourceRegistry valuesSourceRegistry, IndexStorePlugin.RecoveryStateFactory recoveryStateFactory, - IndexStorePlugin.SegmentReplicationStatsStateFactory segmentReplicationStatsStateFactory, + IndexStorePlugin.SegmentReplicationStateFactory segmentReplicationStateFactory, Supplier repositoriesServiceSupplier ) { super(indexSettings); @@ -274,7 +274,7 @@ public IndexService( this.directoryFactory = directoryFactory; this.remoteDirectoryFactory = remoteDirectoryFactory; this.recoveryStateFactory = recoveryStateFactory; - this.segmentReplicationStatsStateFactory = segmentReplicationStatsStateFactory; + this.segmentReplicationStateFactory = segmentReplicationStateFactory; this.engineFactory = Objects.requireNonNull(engineFactory); this.engineConfigFactory = Objects.requireNonNull(engineConfigFactory); // initialize this last -- otherwise if the wrapper requires any other member to be non-null we fail with an NPE @@ -546,7 +546,6 @@ public synchronized IndexShard createShard( new StoreCloseListener(shardId, () -> eventListener.onStoreClosed(shardId)) ); eventListener.onStoreCreated(shardId); - SegmentReplicationStatsState segmentReplicationStatsState = createSegmentReplicationStatsState(routing); indexShard = new IndexShard( routing, this.indexSettings, @@ -570,7 +569,6 @@ public synchronized IndexShard createShard( circuitBreakerService, translogFactory, this.indexSettings.isSegRepEnabled() ? checkpointPublisher : null, - segmentReplicationStatsState, remoteStore ); eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created"); @@ -669,8 +667,8 @@ public RecoveryState createRecoveryState(ShardRouting shardRouting, DiscoveryNod return recoveryStateFactory.newRecoveryState(shardRouting, targetNode, sourceNode); } - public SegmentReplicationStatsState createSegmentReplicationStatsState(ShardRouting shardRouting){ - return segmentReplicationStatsStateFactory.newSegmentReplicationStatsState(shardRouting); + public SegmentReplicationState createSegmentReplicationState(ShardRouting shardRouting, DiscoveryNode node){ + return segmentReplicationStateFactory.newSegmentReplicationState(shardRouting, node); } @Override diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 0e10d250b027f..8085ed02754cc 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -171,7 +171,7 @@ import org.opensearch.indices.recovery.RecoveryListener; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.recovery.RecoveryTarget; -import org.opensearch.indices.replication.SegmentReplicationStatsState; +import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.repositories.RepositoriesService; @@ -279,7 +279,7 @@ Runnable getGlobalCheckpointSyncer() { private volatile RecoveryState recoveryState; @Nullable - private volatile SegmentReplicationStatsState segmentReplicationStatsState; + private volatile SegmentReplicationState segmentReplicationState; private final RecoveryStats recoveryStats = new RecoveryStats(); private final MeanMetric refreshMetric = new MeanMetric(); @@ -348,7 +348,6 @@ public IndexShard( final CircuitBreakerService circuitBreakerService, final TranslogFactory translogFactory, @Nullable final SegmentReplicationCheckpointPublisher checkpointPublisher, - @Nullable final SegmentReplicationStatsState segmentReplicationStatsState, @Nullable final Store remoteStore ) throws IOException { super(shardRouting.shardId(), indexSettings); @@ -433,7 +432,6 @@ public boolean shouldCache(Query query) { this.useRetentionLeasesInPeerRecovery = replicationTracker.hasAllPeerRecoveryRetentionLeases(); this.refreshPendingLocationListener = new RefreshPendingLocationListener(); this.checkpointPublisher = checkpointPublisher; - this.segmentReplicationStatsState = segmentReplicationStatsState; this.remoteStore = remoteStore; this.translogFactory = translogFactory; } @@ -1483,8 +1481,12 @@ public final boolean shouldProcessCheckpoint(ReplicationCheckpoint requestCheckp return true; } - public SegmentReplicationStatsState getSegmentReplicationStatsState(){ - return this.segmentReplicationStatsState; + public SegmentReplicationState getSegmentReplicationState(){ + return this.segmentReplicationState; + } + + public void setSegmentReplicationState(SegmentReplicationState segmentReplicationState){ + this.segmentReplicationState = segmentReplicationState; } /** diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 28fc6c0752b17..9f36bde68fce0 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -260,7 +260,7 @@ public class IndicesService extends AbstractLifecycleComponent private final Map directoryFactories; private final Map recoveryStateFactories; - private final Map segmentReplicationStatsStateFactories; + private final Map segmentReplicationStateFactories; final AbstractRefCounted indicesRefCount; // pkg-private for testing private final CountDownLatch closeLatch = new CountDownLatch(1); private volatile boolean idFieldDataEnabled; @@ -301,7 +301,7 @@ public IndicesService( Map directoryFactories, ValuesSourceRegistry valuesSourceRegistry, Map recoveryStateFactories, - Map segmentReplicationStatsStateFactories, + Map segmentReplicationStateFactories, IndexStorePlugin.RemoteDirectoryFactory remoteDirectoryFactory, Supplier repositoriesServiceSupplier ) { @@ -350,7 +350,7 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.directoryFactories = directoryFactories; this.recoveryStateFactories = recoveryStateFactories; - this.segmentReplicationStatsStateFactories = segmentReplicationStatsStateFactories; + this.segmentReplicationStateFactories = segmentReplicationStateFactories; // doClose() is called when shutting down a node, yet there might still be ongoing requests // that we need to wait for before closing some resources such as the caches. In order to // avoid closing these resources while ongoing requests are still being processed, we use a @@ -418,7 +418,7 @@ public IndicesService( Map directoryFactories, ValuesSourceRegistry valuesSourceRegistry, Map recoveryStateFactories, - Map segmentReplicationStatsStateFactories, + Map segmentReplicationStateFactories, IndexStorePlugin.RemoteDirectoryFactory remoteDirectoryFactory, Supplier repositoriesServiceSupplier ) { @@ -467,7 +467,7 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.directoryFactories = directoryFactories; this.recoveryStateFactories = recoveryStateFactories; - this.segmentReplicationStatsStateFactories = segmentReplicationStatsStateFactories; + this.segmentReplicationStateFactories = segmentReplicationStateFactories; // doClose() is called when shutting down a node, yet there might still be ongoing requests // that we need to wait for before closing some resources such as the caches. In order to // avoid closing these resources while ongoing requests are still being processed, we use a @@ -845,7 +845,7 @@ private synchronized IndexService createIndexService( () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories, - segmentReplicationStatsStateFactories + segmentReplicationStateFactories ); for (IndexingOperationListener operationListener : indexingOperationListeners) { indexModule.addIndexOperationListener(operationListener); @@ -936,7 +936,7 @@ public synchronized MapperService createIndexMapperService(IndexMetadata indexMe () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories, - segmentReplicationStatsStateFactories + segmentReplicationStateFactories ); pluginsService.onIndexModule(indexModule); return indexModule.newIndexMapperService(xContentRegistry, mapperRegistry, scriptService); @@ -1005,6 +1005,9 @@ public IndexShard createShard( .setSource(mapping.source().string(), XContentType.JSON) .get(); }, this); + if(indexService.getIndexSettings().isSegRepEnabled() && shardRouting.primary() == false){ + indexShard.setSegmentReplicationState(indexService.createSegmentReplicationState(shardRouting, targetNode)); + } return indexShard; } diff --git a/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java index 83f4e0c7cbed9..64e832a6236c0 100644 --- a/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java +++ b/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java @@ -827,7 +827,7 @@ public void onReplicationDone(SegmentReplicationState state) { "[shardId {}] [replication id {}] Replication complete, timing data: {}", indexShard.shardId().getId(), state.getReplicationId(), - state.getTimingData() + state.getOverallTimer() ) ); forceSegRepListener.onResponse(null); @@ -844,7 +844,7 @@ public void onReplicationFailure( "[shardId {}] [replication id {}] Replication failed, timing data: {}", indexShard.shardId().getId(), state.getReplicationId(), - state.getTimingData() + state.getOverallTimer() ) ); if (sendShardFailure == true) { diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index a39bb258ab571..7b3045dd9a8d9 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -8,12 +8,17 @@ package org.opensearch.indices.replication; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; +import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.ToXContent; import org.opensearch.common.xcontent.ToXContentFragment; import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.replication.common.ReplicationState; import org.opensearch.indices.replication.common.ReplicationTimer; @@ -72,39 +77,134 @@ public static Stage fromId(byte id) { } private Stage stage; - private final ReplicationLuceneIndex index; + private ReplicationLuceneIndex index; + + public ReplicationTimer getOverallTimer() { + return overallTimer; + } + + public ShardRouting getShardRouting() { + return shardRouting; + } + private final ReplicationTimer overallTimer; - private final ReplicationTimer stageTimer; - private final Map timingData; +// private final ReplicationTimer stageTimer; +// private final Map timingData; + + public Replicating getReplicating() { + return replicating; + } + + public GetCheckpointInfo getGetCheckpointInfo() { + return getCheckpointInfo; + } + + public FileDiff getFileDiff() { + return fileDiff; + } + + public GetFile getGetFile() { + return getFile; + } + + public FinalizeReplication getFinalizeReplication() { + return finalizeReplication; + } + + private final Replicating replicating; + + private final GetCheckpointInfo getCheckpointInfo; + + private final FileDiff fileDiff; + + private final GetFile getFile; + + private final FinalizeReplication finalizeReplication; private long replicationId; - public SegmentReplicationState(ReplicationLuceneIndex index) { + private final ShardRouting shardRouting; + + private DiscoveryNode sourceNode; + + private DiscoveryNode targetNode; + + public long getTotalNumberOfSegRepEvents() { + return totalNumberOfSegRepEvents; + } + + private long totalNumberOfSegRepEvents; + + public SegmentReplicationState(ShardRouting shardRouting, ReplicationLuceneIndex index, DiscoveryNode node) { stage = Stage.INIT; this.index = index; + this.shardRouting = shardRouting; // Timing data will have as many entries as stages, plus one // additional entry for the overall timer - timingData = new HashMap<>(Stage.values().length + 1); +// timingData = new HashMap<>(Stage.values().length + 1); overallTimer = new ReplicationTimer(); - stageTimer = new ReplicationTimer(); - stageTimer.start(); +// stageTimer = new ReplicationTimer(); + replicating = new Replicating(); + getCheckpointInfo = new GetCheckpointInfo(); + fileDiff = new FileDiff(); + getFile = new GetFile(); + finalizeReplication = new FinalizeReplication(); // set an invalid value by default this.replicationId = -1L; + this.sourceNode = node; + this.targetNode = node; } - public SegmentReplicationState(ReplicationLuceneIndex index, long replicationId) { - this(index); + public SegmentReplicationState(ShardRouting shardRouting, DiscoveryNode node) { + this(shardRouting, new ReplicationLuceneIndex(), node); + } + + public SegmentReplicationState onNewSegmentReplicationEvent(ReplicationLuceneIndex index, long replicationId, DiscoveryNode sourceNode, DiscoveryNode targetNode) { + this.index = index; this.replicationId = replicationId; + this.sourceNode = sourceNode; + this.targetNode = targetNode; + setStage(Stage.INIT); + totalNumberOfSegRepEvents++; + return this; } public SegmentReplicationState(StreamInput in) throws IOException { - this(new ReplicationLuceneIndex(in)); + index = new ReplicationLuceneIndex(in); + shardRouting = new ShardRouting(in); + stage = in.readEnum(Stage.class); replicationId = in.readLong(); + overallTimer = new ReplicationTimer(in); +// stageTimer = new ReplicationTimer(in); + replicating = new Replicating(in); + getCheckpointInfo = new GetCheckpointInfo(in); + fileDiff = new FileDiff(in); + getFile = new GetFile(in); + finalizeReplication = new FinalizeReplication(in); + sourceNode = new DiscoveryNode(in); + targetNode = new DiscoveryNode(in); + totalNumberOfSegRepEvents = in.readLong(); +// timingData = in.readMap(StreamInput::readString, StreamInput::readLong); } @Override public void writeTo(StreamOutput out) throws IOException { + index.writeTo(out); + shardRouting.writeTo(out); + out.writeEnum(stage); out.writeLong(replicationId); + overallTimer.writeTo(out); +// stageTimer.writeTo(out); + replicating.writeTo(out); + getCheckpointInfo.writeTo(out); + fileDiff.writeTo(out); + getFile.writeTo(out); + finalizeReplication.writeTo(out); + sourceNode.writeTo(out); + targetNode.writeTo(out); + out.writeLong(totalNumberOfSegRepEvents); +// out.writeMap((Map)timingData); + } @Override @@ -121,12 +221,12 @@ public ReplicationTimer getTimer() { return overallTimer; } - public Map getTimingData() { - return timingData; - } +// public Map getTimingData() { +// return timingData; +// } public Stage getStage() { - return stage; + return this.stage; } protected void validateAndSetStage(Stage expected, Stage next) { @@ -136,54 +236,72 @@ protected void validateAndSetStage(Stage expected, Stage next) { "can't move replication to stage [" + next + "]. current stage: [" + stage + "] (expected [" + expected + "])" ); } - stopTimersAndSetStage(next); - } - - private void stopTimersAndSetStage(Stage next) { - // save the timing data for the current step - stageTimer.stop(); - timingData.put(stage.name().toString(), stageTimer.time()); - // restart the step timer - stageTimer.reset(); - stageTimer.start(); +// stopTimersAndSetStage(next); stage = next; } +// private void stopTimersAndSetStage(Stage next) { +// // save the timing data for the current step +// stageTimer.stop(); +// timingData.put(stage.name().toString(), stageTimer.time()); +// // restart the step timer +// stageTimer.reset(); +// stageTimer.start(); +// stage = next; +// } + public void setStage(Stage stage) { switch (stage) { case INIT: this.stage = Stage.INIT; + overallTimer.reset(); + getReplicating().reset(); + getGetCheckpointInfo().reset(); + getFileDiff().reset(); + getGetFile().reset(); + getFinalizeReplication().reset(); break; case REPLICATING: validateAndSetStage(Stage.INIT, stage); // only start the overall timer once we've started replication overallTimer.start(); + getReplicating().start(); break; case GET_CHECKPOINT_INFO: validateAndSetStage(Stage.REPLICATING, stage); + getReplicating().stop(); + getGetCheckpointInfo().start(); break; case FILE_DIFF: validateAndSetStage(Stage.GET_CHECKPOINT_INFO, stage); + getGetCheckpointInfo().stop(); + getFileDiff().start(); break; case GET_FILES: validateAndSetStage(Stage.FILE_DIFF, stage); + getFileDiff().stop(); + getGetFile().start(); break; case FINALIZE_REPLICATION: validateAndSetStage(Stage.GET_FILES, stage); + getGetFile().stop(); + getFinalizeReplication().start(); break; case DONE: validateAndSetStage(Stage.FINALIZE_REPLICATION, stage); + getFinalizeReplication().stop(); // add the overall timing data overallTimer.stop(); - timingData.put("OVERALL", overallTimer.time()); +// timingData.put("OVERALL", overallTimer.time()); break; case CANCELLED: if (this.stage == Stage.DONE) { throw new IllegalStateException("can't move replication to Cancelled state from Done."); } - stopTimersAndSetStage(Stage.CANCELLED); +// stopTimersAndSetStage(Stage.CANCELLED); + this.stage = Stage.CANCELLED; overallTimer.stop(); - timingData.put("OVERALL", overallTimer.time()); +// timingData.put("OVERALL", overallTimer.time()); break; default: throw new IllegalArgumentException("unknown SegmentReplicationState.Stage [" + stage + "]"); @@ -193,10 +311,220 @@ public void setStage(Stage stage) { @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.field("REPLICATION_ID", replicationId); - builder.field("OVERALL TIMER", getTimer()); - builder.field("STAGE", stage.toString()); + builder.field(Fields.ID, shardRouting.shardId().id()); + builder.field(Fields.STAGE, getStage()); + builder.field(Fields.REPLICATION_ID, getReplicationId()); + builder.field(Fields.NUMBER_OF_SEGMENT_REPLICATION_EVENTS, getTotalNumberOfSegRepEvents()); + builder.timeField(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, getTimer().startTime()); + if (getTimer().stopTime() > 0) { + builder.timeField(Fields.STOP_TIME_IN_MILLIS, Fields.STOP_TIME, getTimer().stopTime()); + } + builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(getTimer().time())); +// builder.field(SegmentReplicationStatsState.Fields.INIT_STAGE, getReplicating().checkReplicatingTime); + if(sourceNode != null){ + builder.startObject(SegmentReplicationState.Fields.SOURCE); + builder.field(Fields.ID, sourceNode.getId()); + builder.field(Fields.HOST, sourceNode.getHostName()); + builder.field(Fields.TRANSPORT_ADDRESS, sourceNode.getAddress().toString()); + builder.field(SegmentReplicationState.Fields.IP, sourceNode.getHostAddress()); + builder.field(SegmentReplicationState.Fields.NAME, sourceNode.getName()); + builder.endObject(); + } + + if (targetNode != null){ + builder.startObject(Fields.TARGET); + builder.field(Fields.ID, targetNode.getId()); + builder.field(Fields.HOST, targetNode.getHostName()); + builder.field(Fields.TRANSPORT_ADDRESS, targetNode.getAddress().toString()); + builder.field(Fields.IP, targetNode.getHostAddress()); + builder.field(Fields.NAME, targetNode.getName()); + builder.endObject(); + } + builder.startObject(Fields.REPLICATING_STAGE); + replicating.toXContent(builder, params); + builder.endObject(); + + builder.startObject(Fields.GET_CHECKPOINT_INFO_STAGE); + getCheckpointInfo.toXContent(builder, params); + builder.endObject(); + + builder.startObject(Fields.FILE_DIFF_STAGE); + fileDiff.toXContent(builder, params); + builder.endObject(); + + builder.startObject(Fields.GET_FILES_STAGE); + index.toXContent(builder, params); + getFile.toXContent(builder, params); + builder.endObject(); + + builder.startObject(Fields.FINALIZE_REPLICATION_STAGE); + finalizeReplication.toXContent(builder, params); + builder.endObject(); return builder; } + + static final class Fields { + static final String ID = "id"; + static final String REPLICATION_ID = "replication_id"; + static final String STAGE = "stage"; + static final String START_TIME = "start_time"; + static final String START_TIME_IN_MILLIS = "start_time_in_millis"; + static final String STOP_TIME = "stop_time"; + static final String STOP_TIME_IN_MILLIS = "stop_time_in_millis"; + static final String TOTAL_TIME = "total_time"; + static final String TOTAL_TIME_IN_MILLIS = "total_time_in_millis"; + static final String SOURCE = "source"; + static final String HOST = "host"; + static final String TRANSPORT_ADDRESS = "transport_address"; + static final String IP = "ip"; + static final String NAME = "name"; + static final String TARGET = "target"; + static final String INIT_STAGE = "init_stage"; + + static final String SIZE_OF_FILES_IN_BYTES = "size_of_files_in_bytes"; + + static final String NUMBER_OF_SEGMENT_REPLICATION_EVENTS = "number_of_segment_replication_events"; + + static final String INDEX = "index"; + static final String TOTAL_GET_FILES_STAGE_TIME_IN_MILLIS = "total_get_files_stage_in_millis"; + static final String REPLICATING_STAGE = "replicating_stage"; + static final String GET_CHECKPOINT_INFO_STAGE = "get_checkpoint_info_stage"; + static final String FILE_DIFF_STAGE = "file_diff_stage"; + static final String GET_FILES_STAGE = "get_files_stage"; + static final String FINALIZE_REPLICATION_STAGE = "finalize_replication_stage"; + } + + public static class Replicating extends ReplicationTimer implements ToXContentFragment, Writeable { + + public Replicating() {} + + public Replicating(StreamInput in) throws IOException { + super(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + + public void reset() { + super.reset(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time())); + return builder; + } + } + + public static class GetCheckpointInfo extends ReplicationTimer implements ToXContentFragment, Writeable { + + public GetCheckpointInfo() {} + + public GetCheckpointInfo(StreamInput in) throws IOException { + super(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + + public void reset() { + super.reset(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time())); + return builder; + } + } + + public static class FileDiff extends ReplicationTimer implements ToXContentFragment, Writeable { + + private volatile long sizeOfFiles; + public FileDiff() {} + + public FileDiff(StreamInput in) throws IOException { + super(in); + sizeOfFiles = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeVLong(sizeOfFiles); + } + + public void reset() { + super.reset(); + sizeOfFiles = 0; + } + + public long getSizeOfFiles() { + return sizeOfFiles; + } + + public void setSizeOfFiles(long sizeOfFiles) { + this.sizeOfFiles = sizeOfFiles; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(Fields.SIZE_OF_FILES_IN_BYTES, getSizeOfFiles()); + builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time())); + return builder; + } + } + + public static class GetFile extends ReplicationTimer implements ToXContentFragment, Writeable { + + public GetFile() {} + + public GetFile(StreamInput in) throws IOException { + super(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + + public void reset() { + super.reset(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.humanReadableField(Fields.TOTAL_GET_FILES_STAGE_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time())); + return builder; + } + } + + public static class FinalizeReplication extends ReplicationTimer implements ToXContentFragment, Writeable { + + public FinalizeReplication() {} + + public FinalizeReplication(StreamInput in) throws IOException { + super(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + + public void reset() { + super.reset(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time())); + return builder; + } + } } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationStatsState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationStatsState.java deleted file mode 100644 index 705ec7f32dd3b..0000000000000 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationStatsState.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.indices.replication; - -import org.opensearch.cluster.routing.ShardRouting; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.xcontent.ToXContentFragment; -import org.opensearch.common.xcontent.XContentBuilder; - - -import java.io.IOException; - - -public class SegmentReplicationStatsState implements ToXContentFragment, Writeable { - - private SegmentReplicationState segmentReplicationState; - - public SegmentReplicationStatsState(ShardRouting shardRouting) { - this.shardRouting = shardRouting; - } - - public long getNumberOfReplicationEvents() { - return numberOfReplicationEvents; - } - - public void incrementCountOfSegmentReplicationEvents() { - numberOfReplicationEvents++; - } - - public ShardRouting getShardRouting() { - return shardRouting; - } - - private ShardRouting shardRouting; - - private long numberOfReplicationEvents; - - public SegmentReplicationStatsState(StreamInput in) throws IOException { - shardRouting = new ShardRouting(in); - numberOfReplicationEvents = in.readLong(); - segmentReplicationState =new SegmentReplicationState(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - shardRouting.writeTo(out); - out.writeLong(numberOfReplicationEvents); - segmentReplicationState.writeTo(out); - } - - protected void setSegmentReplicationState(SegmentReplicationState segmentReplicationState) { - this.segmentReplicationState = segmentReplicationState; - } - - public SegmentReplicationState getSegmentReplicationState(){ - return segmentReplicationState; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - - - builder.field(Fields.ID, shardRouting.shardId().id()); - builder.field(Fields.STAGE, segmentReplicationState.getStage()); - builder.field(Fields.COUNT_OF_SEGMENT_REPLICATION_EVENTS, numberOfReplicationEvents); - builder.timeField(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, segmentReplicationState.getTimer().startTime()); - if (segmentReplicationState.getTimer().stopTime() > 0) { - builder.timeField(Fields.STOP_TIME_IN_MILLIS, Fields.STOP_TIME, segmentReplicationState.getTimer().stopTime()); - } - builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(segmentReplicationState.getTimer().time())); - builder.field(Fields.INIT_STAGE, segmentReplicationState.getTimingData().get(SegmentReplicationState.Stage.INIT.toString())); - builder.field(Fields.REPLICATING_STAGE, segmentReplicationState.getTimingData().get(SegmentReplicationState.Stage.REPLICATING.toString())); - builder.field(Fields.GET_CHECKPOINT_INFO_STAGE, segmentReplicationState.getTimingData().get(SegmentReplicationState.Stage.GET_CHECKPOINT_INFO.toString())); - builder.field(Fields.FILE_DIFF_STAGE, segmentReplicationState.getTimingData().get(SegmentReplicationState.Stage.FILE_DIFF.toString())); - builder.field(Fields.GET_FILES_STAGE, segmentReplicationState.getTimingData().get(SegmentReplicationState.Stage.GET_FILES.toString())); - builder.field(Fields.FINALIZE_REPLICATION_STAGE, segmentReplicationState.getTimingData().get(SegmentReplicationState.Stage.FINALIZE_REPLICATION.toString())); - return builder; - } - - /** - * Fields used in the segment replication stats state - * - * @opensearch.internal - */ - static final class Fields { - static final String ID = "id"; - static final String COUNT_OF_SEGMENT_REPLICATION_EVENTS = "count_of_segment_replication_events"; - static final String STAGE = "stage"; - static final String START_TIME = "start_time"; - static final String START_TIME_IN_MILLIS = "start_time_in_millis"; - static final String STOP_TIME = "stop_time"; - static final String STOP_TIME_IN_MILLIS = "stop_time_in_millis"; - static final String TOTAL_TIME = "total_time"; - static final String TOTAL_TIME_IN_MILLIS = "total_time_in_millis"; - static final String SOURCE = "source"; - static final String HOST = "host"; - static final String TRANSPORT_ADDRESS = "transport_address"; - static final String IP = "ip"; - static final String NAME = "name"; - static final String TARGET = "target"; - static final String INIT_STAGE = "init_stage"; - static final String REPLICATING_STAGE = "replicating_stage"; - static final String GET_CHECKPOINT_INFO_STAGE = "get_checkpoint_info_stage"; - static final String FILE_DIFF_STAGE = "file_diff_stage"; - static final String GET_FILES_STAGE = "get_files_stage"; - static final String FINALIZE_REPLICATION_STAGE = "finalize_replication_stage"; - } - -} diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index d56384f902cd0..671d09a93e7e3 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -66,9 +66,8 @@ public SegmentReplicationTarget( super("replication_target", indexShard, new ReplicationLuceneIndex(), listener); this.checkpoint = checkpoint; this.source = source; - this.state = new SegmentReplicationState(stateIndex, getId()); + this.state = indexShard.getSegmentReplicationState().onNewSegmentReplicationEvent(stateIndex, getId(), indexShard.recoveryState().getSourceNode(), indexShard.recoveryState().getTargetNode()); this.multiFileWriter = new MultiFileWriter(indexShard.store(), stateIndex, getPrefix(), logger, this::ensureRefCount); - indexShard.getSegmentReplicationStatsState().setSegmentReplicationState(state); } @Override @@ -195,6 +194,8 @@ private void getFiles(CheckpointInfoResponse checkpointInfo, StepListener file.length()).sum(); + state.getFileDiff().setSizeOfFiles(sizeOfSegmentFiles); cancellableThreads.checkForCancel(); source.getSegmentFiles(getId(), checkpointInfo.getCheckpoint(), diff.missing, store, getFilesListener); } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index 42e42be24c48f..30d2fa12849ae 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -179,10 +179,9 @@ public void onReplicationDone(SegmentReplicationState state) { "[shardId {}] [replication id {}] Replication complete, timing data: {}", replicaShard.shardId().getId(), state.getReplicationId(), - state.getTimingData() + state.getOverallTimer() ) ); - replicaShard.getSegmentReplicationStatsState().incrementCountOfSegmentReplicationEvents(); // if we received a checkpoint during the copy event that is ahead of this // try and process it. if (latestReceivedCheckpoint.get(replicaShard.shardId()).isAheadOf(replicaShard.getLatestReplicationCheckpoint())) { @@ -203,10 +202,9 @@ public void onReplicationFailure(SegmentReplicationState state, ReplicationFaile "[shardId {}] [replication id {}] Replication failed, timing data: {}", replicaShard.shardId().getId(), state.getReplicationId(), - state.getTimingData() + state.getOverallTimer() ) ); - replicaShard.getSegmentReplicationStatsState().incrementCountOfSegmentReplicationEvents(); if (sendShardFailure == true) { logger.error("replication failure", e); replicaShard.failShard("replication failure", e); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 662f97a516a1f..7d9fbd01a1792 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -648,11 +648,11 @@ protected Node( .flatMap(m -> m.entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - final Map segmentReplicationStatsStateFactories = pluginsService.filterPlugins( + final Map segmentReplicationStateFactories = pluginsService.filterPlugins( IndexStorePlugin.class ) .stream() - .map(IndexStorePlugin::getSegmentReplicationStatsStateFactory) + .map(IndexStorePlugin::getSegmentReplicationStateFactory) .flatMap(m -> m.entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -697,7 +697,7 @@ protected Node( Map.copyOf(directoryFactories), searchModule.getValuesSourceRegistry(), recoveryStateFactories, - segmentReplicationStatsStateFactories, + segmentReplicationStateFactories, remoteDirectoryFactory, repositoriesServiceReference::get ); @@ -723,7 +723,7 @@ protected Node( Map.copyOf(directoryFactories), searchModule.getValuesSourceRegistry(), recoveryStateFactories, - segmentReplicationStatsStateFactories, + segmentReplicationStateFactories, remoteDirectoryFactory, repositoriesServiceReference::get ); diff --git a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java index a2e0c9e008e48..3ffe8b050f906 100644 --- a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java @@ -39,7 +39,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; import org.opensearch.indices.recovery.RecoveryState; -import org.opensearch.indices.replication.SegmentReplicationStatsState; +import org.opensearch.indices.replication.SegmentReplicationState; import java.io.IOException; import java.util.Collections; @@ -105,12 +105,12 @@ interface RecoveryStateFactory { } @FunctionalInterface - interface SegmentReplicationStatsStateFactory { + interface SegmentReplicationStateFactory { /** * Creates a new {@link RecoveryState} per shard. This method is called once per shard on shard creation. * @return a new RecoveryState instance */ - SegmentReplicationStatsState newSegmentReplicationStatsState(ShardRouting shardRouting); + SegmentReplicationState newSegmentReplicationState(ShardRouting shardRouting, DiscoveryNode node); } /** @@ -124,5 +124,5 @@ default Map getRecoveryStateFactories() { return Collections.emptyMap(); } - default Map getSegmentReplicationStatsStateFactory() { return Collections.emptyMap(); } + default Map getSegmentReplicationStateFactory() { return Collections.emptyMap(); } } From ba048a44d1cdacbf1a53b129964ce14732120ff7 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Tue, 24 Jan 2023 15:06:55 +0000 Subject: [PATCH 03/25] Fix broken tests. Signed-off-by: Rishikesh1159 --- .../org/opensearch/action/IndicesRequestIT.java | 13 +++++++++++++ .../action/cat/RestCatSegmentReplicationAction.java | 2 ++ .../extensions/ExtensionsManagerTests.java | 1 + .../java/org/opensearch/index/IndexModuleTests.java | 6 +++++- .../snapshots/SnapshotResiliencyTests.java | 2 ++ 5 files changed, 23 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java index c2eb563145d43..c65c1f6b3233f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java @@ -56,6 +56,8 @@ import org.opensearch.action.admin.indices.recovery.RecoveryRequest; import org.opensearch.action.admin.indices.refresh.RefreshRequest; import org.opensearch.action.admin.indices.refresh.TransportShardRefreshAction; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationAction; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationRequest; import org.opensearch.action.admin.indices.segments.IndicesSegmentsAction; import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsAction; @@ -468,6 +470,17 @@ public void testRecovery() { assertSameIndices(recoveryRequest, recoveryAction); } + public void testSegment_Replication() { + String segmentReplicationAction = SegmentReplicationAction.NAME + "[n]"; + interceptTransportActions(segmentReplicationAction); + + SegmentReplicationRequest segmentReplicationRequest = new SegmentReplicationRequest(randomIndicesOrAliases()); + internalCluster().coordOnlyNodeClient().admin().indices().segment_replication(segmentReplicationRequest).actionGet(); + + clearInterceptedActions(); + assertSameIndices(segmentReplicationRequest, segmentReplicationAction); + } + public void testSegments() { String segmentsAction = IndicesSegmentsAction.NAME + "[n]"; interceptTransportActions(segmentsAction); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java index 223dc484e2956..149f427fc0e3d 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java @@ -51,6 +51,8 @@ protected void documentation(StringBuilder sb) { @Override public BaseRestHandler.RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) { + String indexname = request.param("index"); + logger.info("index name is : {} ", indexname); final SegmentReplicationRequest segmentReplicationRequest = new SegmentReplicationRequest(Strings.splitStringByCommaToArray(request.param("index"))); segmentReplicationRequest.timeout(request.param("timeout")); segmentReplicationRequest.detailed(request.paramAsBoolean("detailed", false)); diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 47820ee739c49..6d1a57dcd446a 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -743,6 +743,7 @@ public void testOnIndexModule() throws Exception { Collections.emptyMap(), () -> true, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), + Collections.emptyMap(), Collections.emptyMap() ); diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 429c2126d9a00..ecb775123b532 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -252,6 +252,7 @@ public void testWrapperIsBound() throws IOException { Collections.emptyMap(), () -> true, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), + Collections.emptyMap(), Collections.emptyMap() ); module.setReaderWrapper(s -> new Wrapper()); @@ -278,6 +279,7 @@ public void testRegisterIndexStore() throws IOException { indexStoreFactories, () -> true, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), + Collections.emptyMap(), Collections.emptyMap() ); @@ -606,7 +608,8 @@ public void testRegisterCustomRecoveryStateFactory() throws IOException { Collections.emptyMap(), () -> true, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), - recoveryStateFactories + recoveryStateFactories, + Collections.emptyMap() ); final IndexService indexService = newIndexService(module); @@ -638,6 +641,7 @@ private static IndexModule createIndexModule(IndexSettings indexSettings, Analys Collections.emptyMap(), () -> true, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), + Collections.emptyMap(), Collections.emptyMap() ); } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 5732fc5bfa270..32ad57a5e7f8d 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -1833,6 +1833,7 @@ public void onFailure(final Exception e) { emptyMap(), null, emptyMap(), + emptyMap(), new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService), repositoriesServiceReference::get ); @@ -1869,6 +1870,7 @@ public void onFailure(final Exception e) { emptyMap(), null, emptyMap(), + emptyMap(), new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService), repositoriesServiceReference::get ); From b062420d8ed454edc0add0612a59cda4ea1a4a5d Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Tue, 24 Jan 2023 15:09:57 +0000 Subject: [PATCH 04/25] Fix compile errors Signed-off-by: Rishikesh1159 --- server/src/main/java/org/opensearch/index/IndexModule.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index 8d77b6cb0911c..3589bb0a68dce 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -568,7 +568,6 @@ public IndexService newIndexService( valuesSourceRegistry, recoveryStateFactory, segmentReplicationStateFactory, - repositoriesServiceSupplier translogFactorySupplier ); success = true; From 1728ae73877a1d53080ee826f51c6c4ce339ca3f Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Wed, 25 Jan 2023 17:50:26 +0000 Subject: [PATCH 05/25] Adding Tests and gating logic behind feature flag. Signed-off-by: Rishikesh1159 --- .../action/admin/ClientTimeoutIT.java | 42 ++++ .../SegmentReplicationRequestBuilder.java | 5 +- .../SegmentReplicationResponse.java | 2 +- .../TransportSegmentReplicationAction.java | 9 +- .../client/support/AbstractClient.java | 5 +- .../org/opensearch/index/IndexModule.java | 23 +- .../org/opensearch/index/IndexService.java | 7 +- .../opensearch/index/shard/IndexShard.java | 4 +- .../opensearch/indices/IndicesService.java | 13 +- .../replication/SegmentReplicationState.java | 53 ++--- .../replication/SegmentReplicationTarget.java | 8 +- .../SegmentReplicationTargetService.java | 4 +- .../main/java/org/opensearch/node/Node.java | 10 - .../opensearch/plugins/IndexStorePlugin.java | 6 +- .../cat/RestCatSegmentReplicationAction.java | 143 ++++++++++-- .../extensions/ExtensionsManagerTests.java | 1 - .../opensearch/index/IndexModuleTests.java | 6 +- .../RestSegment_ReplicationActionTests.java | 215 ++++++++++++++++++ .../snapshots/SnapshotResiliencyTests.java | 2 - 19 files changed, 444 insertions(+), 114 deletions(-) create mode 100644 server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java index 3b56c07cb10c8..978be79cf7eb6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java @@ -19,8 +19,10 @@ import org.opensearch.action.admin.indices.recovery.RecoveryResponse; import org.opensearch.action.admin.indices.stats.IndicesStatsAction; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; @@ -147,6 +149,46 @@ public void testRecoveriesWithTimeout() { assertThat(recoveryResponse.getShardFailures()[0].reason(), containsString("ReceiveTimeoutTransportException")); } + public void testSegment_ReplicationWithTimeout() { + internalCluster().startClusterManagerOnlyNode(); + String dataNode = internalCluster().startDataOnlyNode(); + String anotherDataNode = internalCluster().startDataOnlyNode(); + + int numShards = 4; + assertAcked( + prepareCreate( + "test-index", + 0, + Settings.builder() + .put("number_of_shards", numShards) + .put("number_of_replicas", 1) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + ) + ); + ensureGreen(); + final long numDocs = scaledRandomIntBetween(50, 100); + for (int i = 0; i < numDocs; i++) { + index("test-index", "doc", Integer.toString(i)); + } + refresh("test-index"); + ensureSearchable("test-index"); + + // Happy case + RecoveryResponse recoveryResponse = dataNodeClient().admin().indices().prepareRecoveries().get(); + assertThat(recoveryResponse.getTotalShards(), equalTo(numShards * 2)); + assertThat(recoveryResponse.getSuccessfulShards(), equalTo(numShards * 2)); + + // simulate timeout on bad node. + simulateTimeoutAtTransport(dataNode, anotherDataNode, RecoveryAction.NAME); + + // verify response with bad node. + recoveryResponse = dataNodeClient().admin().indices().prepareRecoveries().get(); + assertThat(recoveryResponse.getTotalShards(), equalTo(numShards * 2)); + assertThat(recoveryResponse.getSuccessfulShards(), equalTo(numShards)); + assertThat(recoveryResponse.getFailedShards(), equalTo(numShards)); + assertThat(recoveryResponse.getShardFailures()[0].reason(), containsString("ReceiveTimeoutTransportException")); + } + public void testStatsWithTimeout() { internalCluster().startClusterManagerOnlyNode(); String dataNode = internalCluster().startDataOnlyNode(); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java index de040e0b4c352..042ff326e894f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java @@ -11,7 +11,10 @@ import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder; import org.opensearch.client.OpenSearchClient; -public class SegmentReplicationRequestBuilder extends BroadcastOperationRequestBuilder { +public class SegmentReplicationRequestBuilder extends BroadcastOperationRequestBuilder< + SegmentReplicationRequest, + SegmentReplicationResponse, + SegmentReplicationRequestBuilder> { public SegmentReplicationRequestBuilder(OpenSearchClient client, SegmentReplicationAction action) { super(client, action, new SegmentReplicationRequest()); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java index 15abde0cd142f..47ba28d8cc0f7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java @@ -25,7 +25,7 @@ public class SegmentReplicationResponse extends BroadcastResponse { public SegmentReplicationResponse(StreamInput in) throws IOException { super(in); - shardSegmentReplicationStates = in.readMapOfLists(StreamInput::readString, SegmentReplicationState:: new); + shardSegmentReplicationStates = in.readMapOfLists(StreamInput::readString, SegmentReplicationState::new); } /** diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java index 70a33d0b980b9..5279c6fed587a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java @@ -33,7 +33,10 @@ import java.util.List; import java.util.Map; -public class TransportSegmentReplicationAction extends TransportBroadcastByNodeAction { +public class TransportSegmentReplicationAction extends TransportBroadcastByNodeAction< + SegmentReplicationRequest, + SegmentReplicationResponse, + SegmentReplicationState> { private final IndicesService indicesService; @@ -82,7 +85,9 @@ protected SegmentReplicationResponse newResponse( shardResponses.put(indexName, new ArrayList<>()); } if (request.activeOnly()) { - shardResponses.get(indexName).add(segmentReplicationState); + if (segmentReplicationState.getStage() != SegmentReplicationState.Stage.DONE) { + shardResponses.get(indexName).add(segmentReplicationState); + } } else { shardResponses.get(indexName).add(segmentReplicationState); } diff --git a/server/src/main/java/org/opensearch/client/support/AbstractClient.java b/server/src/main/java/org/opensearch/client/support/AbstractClient.java index 83cb72cd76cdc..f17898e08ccb5 100644 --- a/server/src/main/java/org/opensearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/opensearch/client/support/AbstractClient.java @@ -1785,7 +1785,10 @@ public ActionFuture segment_replication(final Segmen } @Override - public void segment_replication(final SegmentReplicationRequest request, final ActionListener listener){ + public void segment_replication( + final SegmentReplicationRequest request, + final ActionListener listener + ) { execute(SegmentReplicationAction.INSTANCE, request, listener); } diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index 3589bb0a68dce..b86c270ec7514 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -54,6 +54,7 @@ import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.NamedXContentRegistry; import org.opensearch.core.internal.io.IOUtils; import org.opensearch.env.NodeEnvironment; @@ -126,7 +127,8 @@ public final class IndexModule { private static final IndexStorePlugin.RecoveryStateFactory DEFAULT_RECOVERY_STATE_FACTORY = RecoveryState::new; - private static final IndexStorePlugin.SegmentReplicationStateFactory DEFAULT_SEGMENT_REPLICATION_STATE_FACTORY = SegmentReplicationState ::new; + private static final IndexStorePlugin.SegmentReplicationStateFactory DEFAULT_SEGMENT_REPLICATION_STATE_FACTORY = + SegmentReplicationState::new; public static final Setting INDEX_STORE_TYPE_SETTING = new Setting<>( "index.store.type", @@ -202,8 +204,6 @@ public final class IndexModule { private final BooleanSupplier allowExpensiveQueries; private final Map recoveryStateFactories; - private final Map segmentReplicationStateFactories; - /** * Construct the index module for the index with the specified index settings. The index module contains extension points for plugins * via {@link org.opensearch.plugins.PluginsService#onIndexModule(IndexModule)}. @@ -221,8 +221,7 @@ public IndexModule( final Map directoryFactories, final BooleanSupplier allowExpensiveQueries, final IndexNameExpressionResolver expressionResolver, - final Map recoveryStateFactories, - final Map segmentReplicationStateFactories + final Map recoveryStateFactories ) { this.indexSettings = indexSettings; this.analysisRegistry = analysisRegistry; @@ -234,7 +233,6 @@ public IndexModule( this.allowExpensiveQueries = allowExpensiveQueries; this.expressionResolver = expressionResolver; this.recoveryStateFactories = recoveryStateFactories; - this.segmentReplicationStateFactories = segmentReplicationStateFactories; } /** @@ -518,7 +516,7 @@ public IndexService newIndexService( eventListener.beforeIndexCreated(indexSettings.getIndex(), indexSettings.getSettings()); final IndexStorePlugin.DirectoryFactory directoryFactory = getDirectoryFactory(indexSettings, directoryFactories); final IndexStorePlugin.RecoveryStateFactory recoveryStateFactory = getRecoveryStateFactory(indexSettings, recoveryStateFactories); - final IndexStorePlugin.SegmentReplicationStateFactory segmentReplicationStateFactory = getSegmentReplicationStateFactory(indexSettings, segmentReplicationStateFactories); + final IndexStorePlugin.SegmentReplicationStateFactory segmentReplicationStateFactory = DEFAULT_SEGMENT_REPLICATION_STATE_FACTORY; QueryCache queryCache = null; IndexAnalyzers indexAnalyzers = null; boolean success = false; @@ -567,7 +565,9 @@ public IndexService newIndexService( expressionResolver, valuesSourceRegistry, recoveryStateFactory, - segmentReplicationStateFactory, + FeatureFlags.isEnabled(FeatureFlags.REPLICATION_TYPE) && indexSettings.isSegRepEnabled() + ? segmentReplicationStateFactory + : null, translogFactorySupplier ); success = true; @@ -628,13 +628,6 @@ private static IndexStorePlugin.RecoveryStateFactory getRecoveryStateFactory( return factory; } - private static IndexStorePlugin.SegmentReplicationStateFactory getSegmentReplicationStateFactory( - final IndexSettings indexSettings, - final Map segmentReplicationStateFactories - ) { - return DEFAULT_SEGMENT_REPLICATION_STATE_FACTORY; - } - /** * creates a new mapper service to do administrative work like mapping updates. This *should not* be used for document parsing. * doing so will result in an exception. diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index a0f6f95a5d504..b7fda1543662e 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -142,6 +142,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final IndexStorePlugin.RemoteDirectoryFactory remoteDirectoryFactory; private final IndexStorePlugin.RecoveryStateFactory recoveryStateFactory; + @Nullable private final IndexStorePlugin.SegmentReplicationStateFactory segmentReplicationStateFactory; private final CheckedFunction readerWrapper; private final IndexCache indexCache; @@ -657,8 +658,10 @@ public RecoveryState createRecoveryState(ShardRouting shardRouting, DiscoveryNod return recoveryStateFactory.newRecoveryState(shardRouting, targetNode, sourceNode); } - public SegmentReplicationState createSegmentReplicationState(ShardRouting shardRouting, DiscoveryNode node){ - return segmentReplicationStateFactory.newSegmentReplicationState(shardRouting, node); + public SegmentReplicationState createSegmentReplicationState(ShardRouting shardRouting, DiscoveryNode node) { + if (segmentReplicationStateFactory != null && indexSettings.isSegRepEnabled() && shardRouting.primary() == false) { + return segmentReplicationStateFactory.newSegmentReplicationState(shardRouting, node); + } else return null; } @Override diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 853ec024d188c..eb2a7f2d82a52 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -1548,11 +1548,11 @@ public final boolean shouldProcessCheckpoint(ReplicationCheckpoint requestCheckp return true; } - public SegmentReplicationState getSegmentReplicationState(){ + public SegmentReplicationState getSegmentReplicationState() { return this.segmentReplicationState; } - public void setSegmentReplicationState(SegmentReplicationState segmentReplicationState){ + public void setSegmentReplicationState(SegmentReplicationState segmentReplicationState) { this.segmentReplicationState = segmentReplicationState; } diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index b57b75271b340..8ca8e7cf7095a 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -264,7 +264,6 @@ public class IndicesService extends AbstractLifecycleComponent private final Map directoryFactories; private final Map recoveryStateFactories; - private final Map segmentReplicationStateFactories; final AbstractRefCounted indicesRefCount; // pkg-private for testing private final CountDownLatch closeLatch = new CountDownLatch(1); private volatile boolean idFieldDataEnabled; @@ -305,7 +304,6 @@ public IndicesService( Map directoryFactories, ValuesSourceRegistry valuesSourceRegistry, Map recoveryStateFactories, - Map segmentReplicationStateFactories, IndexStorePlugin.RemoteDirectoryFactory remoteDirectoryFactory, Supplier repositoriesServiceSupplier ) { @@ -354,7 +352,6 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.directoryFactories = directoryFactories; this.recoveryStateFactories = recoveryStateFactories; - this.segmentReplicationStateFactories = segmentReplicationStateFactories; // doClose() is called when shutting down a node, yet there might still be ongoing requests // that we need to wait for before closing some resources such as the caches. In order to // avoid closing these resources while ongoing requests are still being processed, we use a @@ -422,7 +419,6 @@ public IndicesService( Map directoryFactories, ValuesSourceRegistry valuesSourceRegistry, Map recoveryStateFactories, - Map segmentReplicationStateFactories, IndexStorePlugin.RemoteDirectoryFactory remoteDirectoryFactory, Supplier repositoriesServiceSupplier ) { @@ -471,7 +467,6 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.directoryFactories = directoryFactories; this.recoveryStateFactories = recoveryStateFactories; - this.segmentReplicationStateFactories = segmentReplicationStateFactories; // doClose() is called when shutting down a node, yet there might still be ongoing requests // that we need to wait for before closing some resources such as the caches. In order to // avoid closing these resources while ongoing requests are still being processed, we use a @@ -864,8 +859,7 @@ private synchronized IndexService createIndexService( directoryFactories, () -> allowExpensiveQueries, indexNameExpressionResolver, - recoveryStateFactories, - segmentReplicationStateFactories + recoveryStateFactories ); for (IndexingOperationListener operationListener : indexingOperationListeners) { indexModule.addIndexOperationListener(operationListener); @@ -955,8 +949,7 @@ public synchronized MapperService createIndexMapperService(IndexMetadata indexMe directoryFactories, () -> allowExpensiveQueries, indexNameExpressionResolver, - recoveryStateFactories, - segmentReplicationStateFactories + recoveryStateFactories ); pluginsService.onIndexModule(indexModule); return indexModule.newIndexMapperService(xContentRegistry, mapperRegistry, scriptService); @@ -1025,7 +1018,7 @@ public IndexShard createShard( .setSource(mapping.source().string(), XContentType.JSON) .get(); }, this); - if(indexService.getIndexSettings().isSegRepEnabled() && shardRouting.primary() == false){ + if (indexService.getIndexSettings().isSegRepEnabled() && shardRouting.primary() == false) { indexShard.setSegmentReplicationState(indexService.createSegmentReplicationState(shardRouting, targetNode)); } return indexShard; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index 7b3045dd9a8d9..3c5c3de62429a 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -17,15 +17,11 @@ import org.opensearch.common.xcontent.ToXContent; import org.opensearch.common.xcontent.ToXContentFragment; import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.index.shard.IndexShard; -import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.replication.common.ReplicationState; import org.opensearch.indices.replication.common.ReplicationTimer; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; /** * ReplicationState implementation to track Segment Replication events. @@ -88,8 +84,6 @@ public ShardRouting getShardRouting() { } private final ReplicationTimer overallTimer; -// private final ReplicationTimer stageTimer; -// private final Map timingData; public Replicating getReplicating() { return replicating; @@ -124,6 +118,14 @@ public FinalizeReplication getFinalizeReplication() { private final ShardRouting shardRouting; + public DiscoveryNode getSourceNode() { + return sourceNode; + } + + public DiscoveryNode getTargetNode() { + return targetNode; + } + private DiscoveryNode sourceNode; private DiscoveryNode targetNode; @@ -139,11 +141,8 @@ public SegmentReplicationState(ShardRouting shardRouting, ReplicationLuceneIndex this.index = index; this.shardRouting = shardRouting; // Timing data will have as many entries as stages, plus one - // additional entry for the overall timer -// timingData = new HashMap<>(Stage.values().length + 1); overallTimer = new ReplicationTimer(); -// stageTimer = new ReplicationTimer(); replicating = new Replicating(); getCheckpointInfo = new GetCheckpointInfo(); fileDiff = new FileDiff(); @@ -159,7 +158,12 @@ public SegmentReplicationState(ShardRouting shardRouting, DiscoveryNode node) { this(shardRouting, new ReplicationLuceneIndex(), node); } - public SegmentReplicationState onNewSegmentReplicationEvent(ReplicationLuceneIndex index, long replicationId, DiscoveryNode sourceNode, DiscoveryNode targetNode) { + public SegmentReplicationState onNewSegmentReplicationEvent( + ReplicationLuceneIndex index, + long replicationId, + DiscoveryNode sourceNode, + DiscoveryNode targetNode + ) { this.index = index; this.replicationId = replicationId; this.sourceNode = sourceNode; @@ -175,7 +179,6 @@ public SegmentReplicationState(StreamInput in) throws IOException { stage = in.readEnum(Stage.class); replicationId = in.readLong(); overallTimer = new ReplicationTimer(in); -// stageTimer = new ReplicationTimer(in); replicating = new Replicating(in); getCheckpointInfo = new GetCheckpointInfo(in); fileDiff = new FileDiff(in); @@ -184,7 +187,6 @@ public SegmentReplicationState(StreamInput in) throws IOException { sourceNode = new DiscoveryNode(in); targetNode = new DiscoveryNode(in); totalNumberOfSegRepEvents = in.readLong(); -// timingData = in.readMap(StreamInput::readString, StreamInput::readLong); } @Override @@ -194,7 +196,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeEnum(stage); out.writeLong(replicationId); overallTimer.writeTo(out); -// stageTimer.writeTo(out); + // stageTimer.writeTo(out); replicating.writeTo(out); getCheckpointInfo.writeTo(out); fileDiff.writeTo(out); @@ -203,7 +205,6 @@ public void writeTo(StreamOutput out) throws IOException { sourceNode.writeTo(out); targetNode.writeTo(out); out.writeLong(totalNumberOfSegRepEvents); -// out.writeMap((Map)timingData); } @@ -221,10 +222,6 @@ public ReplicationTimer getTimer() { return overallTimer; } -// public Map getTimingData() { -// return timingData; -// } - public Stage getStage() { return this.stage; } @@ -236,20 +233,9 @@ protected void validateAndSetStage(Stage expected, Stage next) { "can't move replication to stage [" + next + "]. current stage: [" + stage + "] (expected [" + expected + "])" ); } -// stopTimersAndSetStage(next); stage = next; } -// private void stopTimersAndSetStage(Stage next) { -// // save the timing data for the current step -// stageTimer.stop(); -// timingData.put(stage.name().toString(), stageTimer.time()); -// // restart the step timer -// stageTimer.reset(); -// stageTimer.start(); -// stage = next; -// } - public void setStage(Stage stage) { switch (stage) { case INIT: @@ -292,16 +278,13 @@ public void setStage(Stage stage) { getFinalizeReplication().stop(); // add the overall timing data overallTimer.stop(); -// timingData.put("OVERALL", overallTimer.time()); break; case CANCELLED: if (this.stage == Stage.DONE) { throw new IllegalStateException("can't move replication to Cancelled state from Done."); } -// stopTimersAndSetStage(Stage.CANCELLED); this.stage = Stage.CANCELLED; overallTimer.stop(); -// timingData.put("OVERALL", overallTimer.time()); break; default: throw new IllegalArgumentException("unknown SegmentReplicationState.Stage [" + stage + "]"); @@ -320,8 +303,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.timeField(Fields.STOP_TIME_IN_MILLIS, Fields.STOP_TIME, getTimer().stopTime()); } builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(getTimer().time())); -// builder.field(SegmentReplicationStatsState.Fields.INIT_STAGE, getReplicating().checkReplicatingTime); - if(sourceNode != null){ + if (sourceNode != null) { builder.startObject(SegmentReplicationState.Fields.SOURCE); builder.field(Fields.ID, sourceNode.getId()); builder.field(Fields.HOST, sourceNode.getHostName()); @@ -331,7 +313,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.endObject(); } - if (targetNode != null){ + if (targetNode != null) { builder.startObject(Fields.TARGET); builder.field(Fields.ID, targetNode.getId()); builder.field(Fields.HOST, targetNode.getHostName()); @@ -446,6 +428,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public static class FileDiff extends ReplicationTimer implements ToXContentFragment, Writeable { private volatile long sizeOfFiles; + public FileDiff() {} public FileDiff(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 671d09a93e7e3..661fa97c7cf55 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -66,7 +66,13 @@ public SegmentReplicationTarget( super("replication_target", indexShard, new ReplicationLuceneIndex(), listener); this.checkpoint = checkpoint; this.source = source; - this.state = indexShard.getSegmentReplicationState().onNewSegmentReplicationEvent(stateIndex, getId(), indexShard.recoveryState().getSourceNode(), indexShard.recoveryState().getTargetNode()); + this.state = indexShard.getSegmentReplicationState() + .onNewSegmentReplicationEvent( + stateIndex, + getId(), + indexShard.recoveryState().getSourceNode(), + indexShard.recoveryState().getTargetNode() + ); this.multiFileWriter = new MultiFileWriter(indexShard.store(), stateIndex, getPrefix(), logger, this::ensureRefCount); } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index 7653d03cd45de..c95cb1a0a5377 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -363,7 +363,7 @@ public void onReplicationDone(SegmentReplicationState state) { "[shardId {}] [replication id {}] Replication complete, timing data: {}", indexShard.shardId().getId(), state.getReplicationId(), - state.getTimingData() + state.getOverallTimer() ) ); try { @@ -385,7 +385,7 @@ public void onReplicationFailure( "[shardId {}] [replication id {}] Replication failed, timing data: {}", indexShard.shardId().getId(), state.getReplicationId(), - state.getTimingData() + state.getOverallTimer() ) ); if (sendShardFailure == true) { diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index ce980783aff67..662a44a81ef85 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -648,14 +648,6 @@ protected Node( .flatMap(m -> m.entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - final Map segmentReplicationStateFactories = pluginsService.filterPlugins( - IndexStorePlugin.class - ) - .stream() - .map(IndexStorePlugin::getSegmentReplicationStateFactory) - .flatMap(m -> m.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - final Map> systemIndexDescriptorMap = Collections.unmodifiableMap( pluginsService.filterPlugins(SystemIndexPlugin.class) .stream() @@ -697,7 +689,6 @@ protected Node( Map.copyOf(directoryFactories), searchModule.getValuesSourceRegistry(), recoveryStateFactories, - segmentReplicationStateFactories, remoteDirectoryFactory, repositoriesServiceReference::get ); @@ -723,7 +714,6 @@ protected Node( Map.copyOf(directoryFactories), searchModule.getValuesSourceRegistry(), recoveryStateFactories, - segmentReplicationStateFactories, remoteDirectoryFactory, repositoriesServiceReference::get ); diff --git a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java index 3ffe8b050f906..3009a6e8db0ac 100644 --- a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java @@ -107,8 +107,8 @@ interface RecoveryStateFactory { @FunctionalInterface interface SegmentReplicationStateFactory { /** - * Creates a new {@link RecoveryState} per shard. This method is called once per shard on shard creation. - * @return a new RecoveryState instance + * Creates a new {@link SegmentReplicationState} per shard. This method is called once per shard on shard creation. + * @return a new SegmentReplicationState instance */ SegmentReplicationState newSegmentReplicationState(ShardRouting shardRouting, DiscoveryNode node); } @@ -123,6 +123,4 @@ interface SegmentReplicationStateFactory { default Map getRecoveryStateFactories() { return Collections.emptyMap(); } - - default Map getSegmentReplicationStateFactory() { return Collections.emptyMap(); } } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java index 149f427fc0e3d..dbffa3606c1c8 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java @@ -9,20 +9,20 @@ package org.opensearch.rest.action.cat; import org.apache.lucene.util.CollectionUtil; -import org.opensearch.action.admin.indices.recovery.RecoveryResponse; import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationRequest; import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationResponse; import org.opensearch.action.support.IndicesOptions; import org.opensearch.client.node.NodeClient; -import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.common.Strings; import org.opensearch.common.Table; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentOpenSearchExtension; +import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestHandler; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; import java.util.Comparator; import java.util.List; @@ -32,10 +32,12 @@ import static java.util.Collections.unmodifiableList; import static org.opensearch.rest.RestRequest.Method.GET; -public class RestCatSegmentReplicationAction extends AbstractCatAction { +public class RestCatSegmentReplicationAction extends AbstractCatAction { @Override public List routes() { - return unmodifiableList(asList(new RestHandler.Route(GET, "/_cat/segment_replication"), new RestHandler.Route(GET, "/_cat/segment_replication/{index}"))); + return unmodifiableList( + asList(new RestHandler.Route(GET, "/_cat/segment_replication"), new RestHandler.Route(GET, "/_cat/segment_replication/{index}")) + ); } @Override @@ -51,36 +53,137 @@ protected void documentation(StringBuilder sb) { @Override public BaseRestHandler.RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) { - String indexname = request.param("index"); - logger.info("index name is : {} ", indexname); - final SegmentReplicationRequest segmentReplicationRequest = new SegmentReplicationRequest(Strings.splitStringByCommaToArray(request.param("index"))); + final SegmentReplicationRequest segmentReplicationRequest = new SegmentReplicationRequest( + Strings.splitStringByCommaToArray(request.param("index")) + ); segmentReplicationRequest.timeout(request.param("timeout")); segmentReplicationRequest.detailed(request.paramAsBoolean("detailed", false)); segmentReplicationRequest.activeOnly(request.paramAsBoolean("active_only", false)); segmentReplicationRequest.indicesOptions(IndicesOptions.fromRequest(request, segmentReplicationRequest.indicesOptions())); - return channel -> client.admin().indices().segment_replication(segmentReplicationRequest, new RestToXContentListener<>(channel) { -// @Override -// public RestResponse buildResponse(final SegmentReplicationResponse response) throws Exception { -// return RestTable.buildResponse(buildSegmentReplicationTable(request, response), channel); -// } - }); + return channel -> client.admin() + .indices() + .segment_replication(segmentReplicationRequest, new RestResponseListener(channel) { + @Override + public RestResponse buildResponse(final SegmentReplicationResponse response) throws Exception { + return RestTable.buildResponse(buildSegmentReplicationTable(request, response), channel); + } + }); } @Override protected Table getTableWithHeader(RestRequest request) { - return null; + + Table t = new Table(); + t.startHeaders() + .addCell("index", "alias:i,idx;desc:index name") + .addCell("shardId", "desc: shard Id") + .addCell("replication_id", "desc: replication Id") + .addCell("start_time", "default:false;alias:start;desc:segment replication start time") + .addCell("start_time_millis", "default:false;alias:start_millis;desc:segment replication start time in epoch milliseconds") + .addCell("stop_time", "default:false;alias:stop;desc:segment replication stop time") + .addCell("stop_time_millis", "default:false;alias:stop_millis;desc:segment replication stop time in epoch milliseconds") + .addCell("time", "alias:t,ti;desc:segment replication time") + .addCell("stage", "alias:st;desc:segment replication stage") + .addCell("source_host", "alias:shost;desc:source host") + .addCell("source_node", "alias:snode;desc:source node name") + .addCell("target_host", "alias:thost;desc:target host") + .addCell("target_node", "alias:tnode;desc:target node name") + .addCell("files", "alias:f;desc:number of files to fetch") + .addCell("files_fetched", "alias:ff;desc:files fetched") + .addCell("files_percent", "alias:fp;desc:percent of files fetched") + .addCell("files_total", "alias:tf;desc:total number of files") + .addCell("bytes", "alias:b;desc:number of bytes to fetch") + .addCell("bytes_fetched", "alias:bf;desc:bytes fetched") + .addCell("bytes_percent", "alias:bp;desc:percent of bytes fetched") + .addCell("bytes_total", "alias:tb;desc:total number of bytes") + .addCell("replicating_stage_time_taken", "alias:rstt;desc:time taken in replicating stage") + .addCell("get_checkpoint_info_stage_time_taken", "alias:gcistt;desc:time taken in get checkpoint info stage") + .addCell("file_diff_stage_time_taken", "alias:fdstt;desc:time taken in file diff stage") + .addCell("get_files_stage_time_taken", "alias:gfstt;desc:time taken in get files stage") + .addCell("finalize_replication_stage_time_taken", "alias:frstt;desc:time taken in finalize replication stage") + .endHeaders(); + return t; } /** - * buildRecoveryTable will build a table of recovery information suitable + * buildSegmentReplicationTable will build a table of SegmentReplication information suitable * for displaying at the command line. * * @param request A Rest request - * @param response A recovery status response - * @return A table containing index, shardId, node, target size, recovered size and percentage for each recovering replica + * @param response A SegmentReplication status response + * @return A table containing index, shardId, node, target size, recovered size and percentage for each fetching replica */ - public Table buildRecoveryTable(RestRequest request, SegmentReplicationResponse response) { - return null; + public Table buildSegmentReplicationTable(RestRequest request, SegmentReplicationResponse response) { + String indexname = ""; + if (request != null) { + indexname = request.param("index"); + } + if (request != null && indexname != null && !response.shardSegmentReplicationStates().containsKey(indexname)) { + Table t = new Table(); + t.startHeaders().addCell("Reason", "desc: Reason for API response").endHeaders(); + t.startRow(); + t.addCell("{ Segment Replication is not enabled on index: " + indexname + " }"); + t.endRow(); + return t; + } + Table t = getTableWithHeader(request); + + for (String index : response.shardSegmentReplicationStates().keySet()) { + + List shardSegmentReplicationStates = response.shardSegmentReplicationStates().get(index); + if (shardSegmentReplicationStates.size() == 0) { + continue; + } + + // Sort ascending by shard id for readability + CollectionUtil.introSort(shardSegmentReplicationStates, new Comparator() { + @Override + public int compare(SegmentReplicationState o1, SegmentReplicationState o2) { + int id1 = o1.getShardRouting().shardId().id(); + int id2 = o2.getShardRouting().shardId().id(); + if (id1 < id2) { + return -1; + } else if (id1 > id2) { + return 1; + } else { + return 0; + } + } + }); + + for (SegmentReplicationState state : shardSegmentReplicationStates) { + t.startRow(); + t.addCell(index); + t.addCell(state.getShardRouting().shardId().id()); + t.addCell(state.getReplicationId()); + t.addCell(XContentOpenSearchExtension.DEFAULT_DATE_PRINTER.print(state.getTimer().startTime())); + t.addCell(state.getTimer().startTime()); + t.addCell(XContentOpenSearchExtension.DEFAULT_DATE_PRINTER.print(state.getTimer().stopTime())); + t.addCell(state.getTimer().stopTime()); + t.addCell(new TimeValue(state.getTimer().time())); + t.addCell(state.getStage().toString().toLowerCase(Locale.ROOT)); + t.addCell(state.getSourceNode().getHostName()); + t.addCell(state.getSourceNode().getName()); + t.addCell(state.getTargetNode().getHostName()); + t.addCell(state.getTargetNode().getName()); + t.addCell(state.getIndex().totalRecoverFiles()); + t.addCell(state.getIndex().recoveredFileCount()); + t.addCell(String.format(Locale.ROOT, "%1.1f%%", state.getIndex().recoveredFilesPercent())); + t.addCell(state.getIndex().totalFileCount()); + t.addCell(state.getIndex().totalRecoverBytes()); + t.addCell(state.getIndex().recoveredBytes()); + t.addCell(String.format(Locale.ROOT, "%1.1f%%", state.getIndex().recoveredBytesPercent())); + t.addCell(state.getIndex().totalBytes()); + t.addCell(new TimeValue(state.getReplicating().time())); + t.addCell(new TimeValue(state.getGetCheckpointInfo().time())); + t.addCell(new TimeValue(state.getFileDiff().time())); + t.addCell(new TimeValue(state.getGetFile().time())); + t.addCell(new TimeValue(state.getFinalizeReplication().time())); + t.endRow(); + } + } + + return t; } } diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 8db14760a3c77..5de2113672ca5 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -831,7 +831,6 @@ public void testOnIndexModule() throws Exception { Collections.emptyMap(), () -> true, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), - Collections.emptyMap(), Collections.emptyMap() ); expectThrows(NodeNotConnectedException.class, () -> extensionsManager.onIndexModule(indexModule)); diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index a84eb47ac1d14..565ae5a16a384 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -266,7 +266,6 @@ public void testWrapperIsBound() throws IOException { Collections.emptyMap(), () -> true, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), - Collections.emptyMap(), Collections.emptyMap() ); module.setReaderWrapper(s -> new Wrapper()); @@ -293,7 +292,6 @@ public void testRegisterIndexStore() throws IOException { indexStoreFactories, () -> true, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), - Collections.emptyMap(), Collections.emptyMap() ); @@ -622,8 +620,7 @@ public void testRegisterCustomRecoveryStateFactory() throws IOException { Collections.emptyMap(), () -> true, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), - recoveryStateFactories, - Collections.emptyMap() + recoveryStateFactories ); final IndexService indexService = newIndexService(module); @@ -655,7 +652,6 @@ private static IndexModule createIndexModule(IndexSettings indexSettings, Analys Collections.emptyMap(), () -> true, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), - Collections.emptyMap(), Collections.emptyMap() ); } diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java new file mode 100644 index 0000000000000..e1096dc222cd3 --- /dev/null +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java @@ -0,0 +1,215 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.cat; + +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationResponse; +import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.common.Randomness; +import org.opensearch.common.Table; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentOpenSearchExtension; +import org.opensearch.index.Index; +import org.opensearch.index.shard.ShardId; +import org.opensearch.indices.replication.SegmentReplicationState; +import org.opensearch.indices.replication.common.ReplicationLuceneIndex; +import org.opensearch.indices.replication.common.ReplicationTimer; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RestSegment_ReplicationActionTests extends OpenSearchTestCase { + public void testSegment_ReplicationActionAction() { + final RestCatSegmentReplicationAction action = new RestCatSegmentReplicationAction(); + final int totalShards = randomIntBetween(1, 32); + final int successfulShards = Math.max(0, totalShards - randomIntBetween(1, 2)); + final int failedShards = totalShards - successfulShards; + final Map> shardSegmentReplicationStates = new HashMap<>(); + final List segmentReplicationStates = new ArrayList<>(); + + for (int i = 0; i < successfulShards; i++) { + final SegmentReplicationState state = mock(SegmentReplicationState.class); + final ShardRouting shardRouting = mock(ShardRouting.class); + when(state.getShardRouting()).thenReturn(shardRouting); + when(shardRouting.shardId()).thenReturn(new ShardId(new Index("index", "_na_"), i)); + when(state.getReplicationId()).thenReturn(randomLongBetween(0, 1000)); + final ReplicationTimer timer = mock(ReplicationTimer.class); + final long startTime = randomLongBetween(0, new Date().getTime()); + when(timer.startTime()).thenReturn(startTime); + final long time = randomLongBetween(1000000, 10 * 1000000); + when(timer.time()).thenReturn(time); + when(timer.stopTime()).thenReturn(startTime + time); + when(state.getTimer()).thenReturn(timer); + when(state.getStage()).thenReturn(randomFrom(SegmentReplicationState.Stage.values())); + final DiscoveryNode sourceNode = mock(DiscoveryNode.class); + when(sourceNode.getHostName()).thenReturn(randomAlphaOfLength(8)); + when(state.getSourceNode()).thenReturn(sourceNode); + final DiscoveryNode targetNode = mock(DiscoveryNode.class); + when(targetNode.getHostName()).thenReturn(randomAlphaOfLength(8)); + when(state.getTargetNode()).thenReturn(targetNode); + + ReplicationLuceneIndex index = createTestIndex(); + when(state.getIndex()).thenReturn(index); + + final SegmentReplicationState.Replicating replicating = mock(SegmentReplicationState.Replicating.class); + final SegmentReplicationState.GetCheckpointInfo getCheckpointInfo = mock(SegmentReplicationState.GetCheckpointInfo.class); + final SegmentReplicationState.FileDiff fileDiff = mock(SegmentReplicationState.FileDiff.class); + final SegmentReplicationState.GetFile getFile = mock(SegmentReplicationState.GetFile.class); + final SegmentReplicationState.FinalizeReplication finalizeReplication = mock(SegmentReplicationState.FinalizeReplication.class); + + when(state.getReplicating()).thenReturn(replicating); + when(replicating.time()).thenReturn(randomLongBetween(0, 10)); + when(state.getGetCheckpointInfo()).thenReturn(getCheckpointInfo); + when(getCheckpointInfo.time()).thenReturn(randomLongBetween(0, 10)); + when(state.getFileDiff()).thenReturn(fileDiff); + when(fileDiff.time()).thenReturn(randomLongBetween(0, 10)); + when(state.getGetFile()).thenReturn(getFile); + when(getFile.time()).thenReturn(randomLongBetween(0, 10)); + when(state.getFinalizeReplication()).thenReturn(finalizeReplication); + when(finalizeReplication.time()).thenReturn(randomLongBetween(0, 10)); + + segmentReplicationStates.add(state); + } + + final List shuffle = new ArrayList<>(segmentReplicationStates); + Randomness.shuffle(shuffle); + shardSegmentReplicationStates.put("index", shuffle); + + final List shardFailures = new ArrayList<>(); + final SegmentReplicationResponse response = new SegmentReplicationResponse( + totalShards, + successfulShards, + failedShards, + shardSegmentReplicationStates, + shardFailures + ); + final Table table = action.buildSegmentReplicationTable(null, response); + + assertNotNull(table); + + List headers = table.getHeaders(); + + final List expectedHeaders = Arrays.asList( + "index", + "shardId", + "replication_id", + "start_time", + "start_time_millis", + "stop_time", + "stop_time_millis", + "time", + "stage", + "source_host", + "source_node", + "target_host", + "target_node", + "files", + "files_fetched", + "files_percent", + "files_total", + "bytes", + "bytes_fetched", + "bytes_percent", + "bytes_total", + "replicating_stage_time_taken", + "get_checkpoint_info_stage_time_taken", + "file_diff_stage_time_taken", + "get_files_stage_time_taken", + "finalize_replication_stage_time_taken" + ); + + for (int i = 0; i < expectedHeaders.size(); i++) { + assertThat(headers.get(i).value, equalTo(expectedHeaders.get(i))); + } + + assertThat(table.getRows().size(), equalTo(successfulShards)); + + for (int i = 0; i < successfulShards; i++) { + final SegmentReplicationState state = segmentReplicationStates.get(i); + final List expectedValues = Arrays.asList( + "index", + i, + state.getReplicationId(), + XContentOpenSearchExtension.DEFAULT_DATE_PRINTER.print(state.getTimer().startTime()), + state.getTimer().startTime(), + XContentOpenSearchExtension.DEFAULT_DATE_PRINTER.print(state.getTimer().stopTime()), + state.getTimer().stopTime(), + new TimeValue(state.getTimer().time()), + state.getStage().name().toLowerCase(Locale.ROOT), + state.getSourceNode().getHostName(), + state.getSourceNode().getName(), + state.getTargetNode().getHostName(), + state.getTargetNode().getName(), + state.getIndex().totalRecoverFiles(), + state.getIndex().recoveredFileCount(), + percent(state.getIndex().recoveredFilesPercent()), + state.getIndex().totalFileCount(), + state.getIndex().totalRecoverBytes(), + state.getIndex().recoveredBytes(), + percent(state.getIndex().recoveredBytesPercent()), + state.getIndex().totalBytes(), + new TimeValue(state.getReplicating().time()), + new TimeValue(state.getGetCheckpointInfo().time()), + new TimeValue(state.getFileDiff().time()), + new TimeValue(state.getGetFile().time()), + new TimeValue(state.getFinalizeReplication().time()) + ); + + final List cells = table.getRows().get(i); + for (int j = 0; j < expectedValues.size(); j++) { + assertThat(cells.get(j).value, equalTo(expectedValues.get(j))); + } + } + } + + private ReplicationLuceneIndex createTestIndex() { + ReplicationLuceneIndex index = new ReplicationLuceneIndex(); + final int filesToRecoverCount = randomIntBetween(1, 64); + final int recoveredFilesCount = randomIntBetween(0, filesToRecoverCount); + addTestFileMetadata(index, 0, recoveredFilesCount, false, true); + addTestFileMetadata(index, recoveredFilesCount, filesToRecoverCount, false, false); + + final int totalFilesCount = randomIntBetween(filesToRecoverCount, 2 * filesToRecoverCount); + addTestFileMetadata(index, filesToRecoverCount, totalFilesCount, true, false); + return index; + } + + private void addTestFileMetadata(ReplicationLuceneIndex index, int startIndex, int endIndex, boolean reused, boolean isFullyRecovered) { + for (int i = startIndex; i < endIndex; i++) { + final int completeFileSize = randomIntBetween(1, 1024); + index.addFileDetail(String.valueOf(i), completeFileSize, reused); + + if (!reused) { + final int recoveredFileSize; + if (isFullyRecovered) { + recoveredFileSize = completeFileSize; + + } else { + recoveredFileSize = randomIntBetween(0, completeFileSize); + } + index.addRecoveredBytesToFile(String.valueOf(i), recoveredFileSize); + } + } + } + + private static String percent(float percent) { + return String.format(Locale.ROOT, "%1.1f%%", percent); + } +} diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 627d958027f88..a6677fbc0c99b 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -1833,7 +1833,6 @@ public void onFailure(final Exception e) { emptyMap(), null, emptyMap(), - emptyMap(), new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService), repositoriesServiceReference::get ); @@ -1870,7 +1869,6 @@ public void onFailure(final Exception e) { emptyMap(), null, emptyMap(), - emptyMap(), new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService), repositoriesServiceReference::get ); From b10519e54ad3d7223a8c4c0334c93ddc7ada3781 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Thu, 26 Jan 2023 09:27:11 +0000 Subject: [PATCH 06/25] Add java docs and enable query parameter detailed. Signed-off-by: Rishikesh1159 --- CHANGELOG.md | 3 +- .../api/cat.segment_replication.json | 2 +- .../SegmentReplicationAction.java | 5 +++ .../SegmentReplicationRequest.java | 13 ++++-- .../SegmentReplicationRequestBuilder.java | 5 +++ .../SegmentReplicationResponse.java | 5 +++ .../TransportSegmentReplicationAction.java | 6 +++ .../segment_replication/package-info.java | 10 +++++ .../replication/SegmentReplicationState.java | 25 +++++++++++ .../opensearch/plugins/IndexStorePlugin.java | 3 ++ .../cat/RestCatSegmentReplicationAction.java | 42 +++++++++++++------ .../RestSegment_ReplicationActionTests.java | 31 ++------------ 12 files changed, 104 insertions(+), 46 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/segment_replication/package-info.java diff --git a/CHANGELOG.md b/CHANGELOG.md index aaae7ea3c1a67..796c6de2a5bad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Added support for feature flags in opensearch.yml ([#4959](https://github.com/opensearch-project/OpenSearch/pull/4959)) - Add query for initialized extensions ([#5658](https://github.com/opensearch-project/OpenSearch/pull/5658)) - Add update-index-settings allowlist for searchable snapshot ([#5907](https://github.com/opensearch-project/OpenSearch/pull/5907)) +- Add new cat/segment_replication API to surface Segment Replication metrics ([#5718](https://github.com/opensearch-project/OpenSearch/pull/5718)). ### Dependencies - Update nebula-publishing-plugin to 19.2.0 ([#5704](https://github.com/opensearch-project/OpenSearch/pull/5704)) @@ -97,4 +98,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.5...2.x \ No newline at end of file +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.5...2.x diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json index 541467ca313ff..e90908488729c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json @@ -1,7 +1,7 @@ { "cat.segment_replication":{ "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-segment_replication.html", + "url":"TO DO", "description":"Returns information about index Segment Replication events, both on-going and completed." }, "stability":"stable", diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationAction.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationAction.java index 3f56d264fe82e..d22c24d7fbeef 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationAction.java @@ -10,6 +10,11 @@ import org.opensearch.action.ActionType; +/** + * Segment Replication information action + * + * @opensearch.internal + */ public class SegmentReplicationAction extends ActionType { public static final SegmentReplicationAction INSTANCE = new SegmentReplicationAction(); public static final String NAME = "indices:monitor/segment_replication"; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java index 30befe8abfee0..809e0728137d5 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java @@ -16,9 +16,14 @@ import java.io.IOException; +/** + * Request for Segment Replication information + * + * @opensearch.internal + */ public class SegmentReplicationRequest extends BroadcastRequest { private boolean detailed = false; // Provides extra details in the response - private boolean activeOnly = false; // Only reports on active recoveries + private boolean activeOnly = false; // Only reports on active segment replication events /** * Constructs a request for segment replication information for all shards @@ -53,7 +58,7 @@ public boolean detailed() { /** * Set value of the detailed flag. Detailed requests will contain extra - * information. + * information like timing metric of each stage of segment replication event. * * @param detailed Whether or not to set the detailed flag */ @@ -71,8 +76,8 @@ public boolean activeOnly() { } /** - * Set value of the activeOnly flag. If true, this request will only response with - * on-going recovery information. + * Set value of the activeOnly flag. If true, this request will only respond with + * on-going segment replication event information. * * @param activeOnly Whether or not to set the activeOnly flag. */ diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java index 042ff326e894f..d58cb45643773 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java @@ -11,6 +11,11 @@ import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder; import org.opensearch.client.OpenSearchClient; +/** + * Segment Replication information request builder. + * + * @opensearch.internal + */ public class SegmentReplicationRequestBuilder extends BroadcastOperationRequestBuilder< SegmentReplicationRequest, SegmentReplicationResponse, diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java index 47ba28d8cc0f7..d96a0afdf2172 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java @@ -20,6 +20,11 @@ import java.util.List; import java.util.Map; +/** + * Information regarding the Segment Replication state of indices and their associated shards. + * + * @opensearch.internal + */ public class SegmentReplicationResponse extends BroadcastResponse { private final Map> shardSegmentReplicationStates; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java index 5279c6fed587a..b34c4223e7ee7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java @@ -33,6 +33,12 @@ import java.util.List; import java.util.Map; +/** + * Transport action for shard segment replication operation. This transport action does not actually + * perform segment replication, it only reports on metrics of segment replication event (both active and complete). + * + * @opensearch.internal + */ public class TransportSegmentReplicationAction extends TransportBroadcastByNodeAction< SegmentReplicationRequest, SegmentReplicationResponse, diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/package-info.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/package-info.java new file mode 100644 index 0000000000000..ae37f03a63a28 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Segment Replication transport handlers. */ +package org.opensearch.action.admin.indices.segment_replication; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index 3c5c3de62429a..183bf62599c2c 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -377,6 +377,11 @@ static final class Fields { static final String FINALIZE_REPLICATION_STAGE = "finalize_replication_stage"; } + /** + * Starts Replicating + * + * @opensearch.internal + */ public static class Replicating extends ReplicationTimer implements ToXContentFragment, Writeable { public Replicating() {} @@ -401,6 +406,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } } + /** + * Gets checkpoint info from source + * + * @opensearch.internal + */ public static class GetCheckpointInfo extends ReplicationTimer implements ToXContentFragment, Writeable { public GetCheckpointInfo() {} @@ -425,6 +435,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } } + /** + * Computes File Diff + * + * @opensearch.internal + */ public static class FileDiff extends ReplicationTimer implements ToXContentFragment, Writeable { private volatile long sizeOfFiles; @@ -463,6 +478,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } } + /** + * Gets necessary files from source + * + * @opensearch.internal + */ public static class GetFile extends ReplicationTimer implements ToXContentFragment, Writeable { public GetFile() {} @@ -487,6 +507,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } } + /** + * Finalizes Replication + * + * @opensearch.internal + */ public static class FinalizeReplication extends ReplicationTimer implements ToXContentFragment, Writeable { public FinalizeReplication() {} diff --git a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java index 3009a6e8db0ac..ca38246766fd8 100644 --- a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java @@ -104,6 +104,9 @@ interface RecoveryStateFactory { RecoveryState newRecoveryState(ShardRouting shardRouting, DiscoveryNode targetNode, @Nullable DiscoveryNode sourceNode); } + /** + * An interface that allows to create a new {@link SegmentReplicationState} per shard. + */ @FunctionalInterface interface SegmentReplicationStateFactory { /** diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java index dbffa3606c1c8..63b638c1f77bd 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java @@ -32,6 +32,13 @@ import static java.util.Collections.unmodifiableList; import static org.opensearch.rest.RestRequest.Method.GET; +/** + * RestCatSegmentReplicationAction provides information about the status of replica's segment replication event + * in a string format, designed to be used at the command line. An Index can + * be specified to limit output to a particular index or indices. + * + * @opensearch.api + */ public class RestCatSegmentReplicationAction extends AbstractCatAction { @Override public List routes() { @@ -74,6 +81,11 @@ public RestResponse buildResponse(final SegmentReplicationResponse response) thr @Override protected Table getTableWithHeader(RestRequest request) { + boolean detailed = false; + if (request != null) { + detailed = Boolean.parseBoolean(request.param("detailed")); + } + Table t = new Table(); t.startHeaders() .addCell("index", "alias:i,idx;desc:index name") @@ -96,13 +108,15 @@ protected Table getTableWithHeader(RestRequest request) { .addCell("bytes", "alias:b;desc:number of bytes to fetch") .addCell("bytes_fetched", "alias:bf;desc:bytes fetched") .addCell("bytes_percent", "alias:bp;desc:percent of bytes fetched") - .addCell("bytes_total", "alias:tb;desc:total number of bytes") - .addCell("replicating_stage_time_taken", "alias:rstt;desc:time taken in replicating stage") - .addCell("get_checkpoint_info_stage_time_taken", "alias:gcistt;desc:time taken in get checkpoint info stage") - .addCell("file_diff_stage_time_taken", "alias:fdstt;desc:time taken in file diff stage") - .addCell("get_files_stage_time_taken", "alias:gfstt;desc:time taken in get files stage") - .addCell("finalize_replication_stage_time_taken", "alias:frstt;desc:time taken in finalize replication stage") - .endHeaders(); + .addCell("bytes_total", "alias:tb;desc:total number of bytes"); + if (detailed) { + t.addCell("replicating_stage_time_taken", "alias:rstt;desc:time taken in replicating stage") + .addCell("get_checkpoint_info_stage_time_taken", "alias:gcistt;desc:time taken in get checkpoint info stage") + .addCell("file_diff_stage_time_taken", "alias:fdstt;desc:time taken in file diff stage") + .addCell("get_files_stage_time_taken", "alias:gfstt;desc:time taken in get files stage") + .addCell("finalize_replication_stage_time_taken", "alias:frstt;desc:time taken in finalize replication stage"); + } + t.endHeaders(); return t; } @@ -116,8 +130,10 @@ protected Table getTableWithHeader(RestRequest request) { */ public Table buildSegmentReplicationTable(RestRequest request, SegmentReplicationResponse response) { String indexname = ""; + boolean detailed = false; if (request != null) { indexname = request.param("index"); + detailed = Boolean.parseBoolean(request.param("detailed")); } if (request != null && indexname != null && !response.shardSegmentReplicationStates().containsKey(indexname)) { Table t = new Table(); @@ -175,11 +191,13 @@ public int compare(SegmentReplicationState o1, SegmentReplicationState o2) { t.addCell(state.getIndex().recoveredBytes()); t.addCell(String.format(Locale.ROOT, "%1.1f%%", state.getIndex().recoveredBytesPercent())); t.addCell(state.getIndex().totalBytes()); - t.addCell(new TimeValue(state.getReplicating().time())); - t.addCell(new TimeValue(state.getGetCheckpointInfo().time())); - t.addCell(new TimeValue(state.getFileDiff().time())); - t.addCell(new TimeValue(state.getGetFile().time())); - t.addCell(new TimeValue(state.getFinalizeReplication().time())); + if (detailed) { + t.addCell(new TimeValue(state.getReplicating().time())); + t.addCell(new TimeValue(state.getGetCheckpointInfo().time())); + t.addCell(new TimeValue(state.getFileDiff().time())); + t.addCell(new TimeValue(state.getGetFile().time())); + t.addCell(new TimeValue(state.getFinalizeReplication().time())); + } t.endRow(); } } diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java index e1096dc222cd3..bb0223b05068a 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java @@ -68,22 +68,7 @@ public void testSegment_ReplicationActionAction() { ReplicationLuceneIndex index = createTestIndex(); when(state.getIndex()).thenReturn(index); - final SegmentReplicationState.Replicating replicating = mock(SegmentReplicationState.Replicating.class); - final SegmentReplicationState.GetCheckpointInfo getCheckpointInfo = mock(SegmentReplicationState.GetCheckpointInfo.class); - final SegmentReplicationState.FileDiff fileDiff = mock(SegmentReplicationState.FileDiff.class); - final SegmentReplicationState.GetFile getFile = mock(SegmentReplicationState.GetFile.class); - final SegmentReplicationState.FinalizeReplication finalizeReplication = mock(SegmentReplicationState.FinalizeReplication.class); - - when(state.getReplicating()).thenReturn(replicating); - when(replicating.time()).thenReturn(randomLongBetween(0, 10)); - when(state.getGetCheckpointInfo()).thenReturn(getCheckpointInfo); - when(getCheckpointInfo.time()).thenReturn(randomLongBetween(0, 10)); - when(state.getFileDiff()).thenReturn(fileDiff); - when(fileDiff.time()).thenReturn(randomLongBetween(0, 10)); - when(state.getGetFile()).thenReturn(getFile); - when(getFile.time()).thenReturn(randomLongBetween(0, 10)); - when(state.getFinalizeReplication()).thenReturn(finalizeReplication); - when(finalizeReplication.time()).thenReturn(randomLongBetween(0, 10)); + // segmentReplicationStates.add(state); } @@ -127,12 +112,7 @@ public void testSegment_ReplicationActionAction() { "bytes", "bytes_fetched", "bytes_percent", - "bytes_total", - "replicating_stage_time_taken", - "get_checkpoint_info_stage_time_taken", - "file_diff_stage_time_taken", - "get_files_stage_time_taken", - "finalize_replication_stage_time_taken" + "bytes_total" ); for (int i = 0; i < expectedHeaders.size(); i++) { @@ -164,12 +144,7 @@ public void testSegment_ReplicationActionAction() { state.getIndex().totalRecoverBytes(), state.getIndex().recoveredBytes(), percent(state.getIndex().recoveredBytesPercent()), - state.getIndex().totalBytes(), - new TimeValue(state.getReplicating().time()), - new TimeValue(state.getGetCheckpointInfo().time()), - new TimeValue(state.getFileDiff().time()), - new TimeValue(state.getGetFile().time()), - new TimeValue(state.getFinalizeReplication().time()) + state.getIndex().totalBytes() ); final List cells = table.getRows().get(i); From 74a41b0ed5e425092830643f3906231f9ce4c2fd Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Thu, 26 Jan 2023 09:53:41 +0000 Subject: [PATCH 07/25] Add temporary documentation URL Signed-off-by: Rishikesh1159 --- .../resources/rest-api-spec/api/cat.segment_replication.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json index e90908488729c..2fca24e3f8fde 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json @@ -1,7 +1,7 @@ { "cat.segment_replication":{ "documentation":{ - "url":"TO DO", + "url":"https://TODO.html", "description":"Returns information about index Segment Replication events, both on-going and completed." }, "stability":"stable", From 7752a000193c83ec3b05ee496c851e79d3b710a8 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Fri, 27 Jan 2023 18:16:27 +0000 Subject: [PATCH 08/25] Fixing failing tests. Signed-off-by: Rishikesh1159 --- .../api/cat.segment_replication.json | 2 +- .../opensearch/index/shard/IndexShard.java | 5 ++- .../replication/SegmentReplicationTarget.java | 2 + ...overyWithRemoteTranslogOnPrimaryTests.java | 14 ++++++ .../SegmentReplicationIndexShardTests.java | 45 ++++++++++++++++++- ...teStorePeerRecoverySourceHandlerTests.java | 12 +++++ .../SegmentReplicationTargetServiceTests.java | 10 +++++ .../SegmentReplicationTargetTests.java | 10 +++++ 8 files changed, 97 insertions(+), 3 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json index 2fca24e3f8fde..9ec9fb05d66dc 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json @@ -1,7 +1,7 @@ { "cat.segment_replication":{ "documentation":{ - "url":"https://TODO.html", + "url":"https://opensearch.org/docs/1.0/opensearch/rest-api/cat/cat-recovery/", "description":"Returns information about index Segment Replication events, both on-going and completed." }, "stability":"stable", diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index eb2a7f2d82a52..3f6337bee5429 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -1553,7 +1553,10 @@ public SegmentReplicationState getSegmentReplicationState() { } public void setSegmentReplicationState(SegmentReplicationState segmentReplicationState) { - this.segmentReplicationState = segmentReplicationState; + if(this.shardRouting.primary() == false || getReplicationEngine().isEmpty() == false){ + this.segmentReplicationState = segmentReplicationState; + } + else throw new OpenSearchException("Cannot set Segment Replication State on a primary shard"); } /** diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 661fa97c7cf55..d28895ba0574f 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -66,6 +66,8 @@ public SegmentReplicationTarget( super("replication_target", indexShard, new ReplicationLuceneIndex(), listener); this.checkpoint = checkpoint; this.source = source; + logger.info("shard is: {}", indexShard.routingEntry().primary()); + logger.info("state is: {}", indexShard.getSegmentReplicationState()); this.state = indexShard.getSegmentReplicationState() .onNewSegmentReplicationEvent( stateIndex, diff --git a/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java b/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java index ba707cc30e6b8..144607c29e56d 100644 --- a/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java @@ -10,10 +10,12 @@ import org.junit.Assert; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.common.settings.Settings; +import org.opensearch.common.transport.TransportAddress; import org.opensearch.index.engine.DocIdSeqNoAndSource; import org.opensearch.index.engine.NRTReplicationEngine; import org.opensearch.index.engine.NRTReplicationEngineFactory; @@ -21,7 +23,9 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.translog.WriteOnlyTranslogManager; import org.opensearch.indices.recovery.RecoveryTarget; +import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.VersionUtils; import java.io.IOException; import java.util.List; @@ -38,6 +42,12 @@ public class ReplicaRecoveryWithRemoteTranslogOnPrimaryTests extends OpenSearchI .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "translog-repo") .build(); + DiscoveryNode node = new DiscoveryNode( + "101", + new TransportAddress(TransportAddress.META_ADDRESS, randomInt(0xFFFF)), + VersionUtils.randomVersion(random()) + ); + public void testStartSequenceForReplicaRecovery() throws Exception { try (ReplicationGroup shards = createGroup(0, settings, new NRTReplicationEngineFactory())) { @@ -102,6 +112,9 @@ public IndexShard indexShard() { return idxShard; } }); + for(IndexShard replicaShard:shards.getReplicas()){ + replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); + } shards.flush(); replicateSegments(primary, shards.getReplicas()); @@ -123,6 +136,7 @@ public void testNoTranslogHistoryTransferred() throws Exception { // Step 2 - Start replica, recovery happens, check docs recovered till last flush final IndexShard replica = shards.addReplica(); + replica.setSegmentReplicationState(new SegmentReplicationState(replica.routingEntry(), node)); shards.startAll(); assertEquals(docIdAndSeqNosAfterFlush, getDocIdAndSeqNos(replica)); assertDocCount(replica, numDocs); diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index 44771faf36871..9affd62adf594 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -16,11 +16,13 @@ import org.opensearch.action.delete.DeleteRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.xcontent.XContentType; @@ -50,6 +52,7 @@ import org.opensearch.indices.replication.common.ReplicationListener; import org.opensearch.indices.replication.common.ReplicationState; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.VersionUtils; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -78,6 +81,12 @@ public class SegmentReplicationIndexShardTests extends OpenSearchIndexLevelRepli .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(); + private DiscoveryNode node = new DiscoveryNode( + "101", + new TransportAddress(TransportAddress.META_ADDRESS, randomInt(0xFFFF)), + VersionUtils.randomVersion(random()) + ); + /** * Test that latestReplicationCheckpoint returns null only for docrep enabled indices */ @@ -116,6 +125,9 @@ public void testSegmentReplication_Index_Update_Delete() throws Exception { } primaryShard.refresh("Test"); + for(IndexShard replicaShard:shards.getReplicas()){ + replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); + } replicateSegments(primaryShard, shards.getReplicas()); shards.assertAllEqual(numDocs); @@ -144,6 +156,9 @@ public void testSegmentReplication_Index_Update_Delete() throws Exception { } } primaryShard.refresh("Test"); + for(IndexShard replicaShard:shards.getReplicas()){ + replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); + } replicateSegments(primaryShard, shards.getReplicas()); final List docsAfterDelete = getDocIdAndSeqNos(primaryShard); for (IndexShard shard : shards.getReplicas()) { @@ -157,6 +172,7 @@ public void testIgnoreShardIdle() throws Exception { shards.startAll(); final IndexShard primary = shards.getPrimary(); final IndexShard replica = shards.getReplicas().get(0); + replica.setSegmentReplicationState(new SegmentReplicationState(replica.routingEntry(), node)); final int numDocs = shards.indexDocs(randomInt(10)); primary.refresh("test"); @@ -258,6 +274,7 @@ public void testReplicaReceivesGenIncrease() throws Exception { shards.startAll(); final IndexShard primary = shards.getPrimary(); final IndexShard replica = shards.getReplicas().get(0); + replica.setSegmentReplicationState(new SegmentReplicationState(replica.routingEntry(), node)); final int numDocs = randomIntBetween(10, 100); shards.indexDocs(numDocs); assertEquals(numDocs, primary.translogStats().estimatedNumberOfOperations()); @@ -313,7 +330,11 @@ public void testPrimaryRelocation() throws Exception { try { assert shardList.size() >= 2; final IndexShard primary = shardList.get(0); - return replicateSegments(primary, shardList.subList(1, shardList.size()), listener); + List replicaShards = shardList.subList(1, shardList.size()); + for(IndexShard replicaShard:replicaShards){ + replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); + } + return replicateSegments(primary, replicaShards, listener); } catch (IOException | InterruptedException e) { listener.onFailure(e); throw new RuntimeException(e); @@ -394,6 +415,9 @@ public void testReplicaReceivesLowerGeneration() throws Exception { final IndexShard primary = shards.getPrimary(); final IndexShard replica_1 = shards.getReplicas().get(0); final IndexShard replica_2 = shards.getReplicas().get(1); + for(IndexShard replicaShard:shards.getReplicas()){ + replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); + } int numDocs = randomIntBetween(10, 100); shards.indexDocs(numDocs); flushShard(primary, false); @@ -421,6 +445,9 @@ public void testReplicaReceivesLowerGeneration() throws Exception { numDocs = randomIntBetween(numDocs + 1, numDocs + 10); shards.indexDocs(numDocs); flushShard(replica_2, false); + for(IndexShard replicaShard:shards.getReplicas()){ + replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); + } replicateSegments(replica_2, shards.getReplicas()); assertEqualCommittedSegments(replica_2, oldPrimary, replica_1); } @@ -438,6 +465,9 @@ public void testReplicaRestarts() throws Exception { flushShard(primary); } primary.refresh("Test"); + for(IndexShard replicaShard:shards.getReplicas()){ + replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); + } replicateSegments(primary, shards.getReplicas()); // at this point both shards should have numDocs persisted and searchable. @@ -466,6 +496,9 @@ public void testReplicaRestarts() throws Exception { failAndPromoteRandomReplica(shards); } flushShard(shards.getPrimary()); + for(IndexShard replicaShard:shards.getReplicas()){ + replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); + } replicateSegments(shards.getPrimary(), shards.getReplicas()); } primary = shards.getPrimary(); @@ -579,6 +612,9 @@ public void testNRTReplicaPromotedAsPrimary() throws Exception { IndexShard oldPrimary = shards.getPrimary(); final IndexShard nextPrimary = shards.getReplicas().get(0); final IndexShard replica = shards.getReplicas().get(1); + for(IndexShard replicaShard:shards.getReplicas()){ + replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); + } // 1. Create ops that are in the index and xlog of both shards but not yet part of a commit point. final int numDocs = shards.indexDocs(randomInt(10)); @@ -642,6 +678,7 @@ public void testReplicaPromotedWhileReplicating() throws Exception { shards.startAll(); final IndexShard oldPrimary = shards.getPrimary(); final IndexShard nextPrimary = shards.getReplicas().get(0); + nextPrimary.setSegmentReplicationState(new SegmentReplicationState(nextPrimary.routingEntry(), node)); final int numDocs = shards.indexDocs(randomInt(10)); oldPrimary.refresh("Test"); @@ -705,6 +742,9 @@ public void onFailure(Exception e) { assertDocCount(newReplica, numDocs); nextPrimary.refresh("test"); + for(IndexShard replicaShard:shards.getReplicas()){ + replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); + } replicateSegments(nextPrimary, shards.getReplicas()); final List docsAfterRecovery = getDocIdAndSeqNos(shards.getPrimary()); for (IndexShard shard : shards.getReplicas()) { @@ -718,6 +758,7 @@ public void testReplicaClosesWhileReplicating_AfterGetCheckpoint() throws Except shards.startAll(); IndexShard primary = shards.getPrimary(); final IndexShard replica = shards.getReplicas().get(0); + replica.setSegmentReplicationState(new SegmentReplicationState(replica.routingEntry(), node)); final int numDocs = shards.indexDocs(randomInt(10)); primary.refresh("Test"); @@ -760,6 +801,7 @@ public void testReplicaClosesWhileReplicating_AfterGetSegmentFiles() throws Exce shards.startAll(); IndexShard primary = shards.getPrimary(); final IndexShard replica = shards.getReplicas().get(0); + replica.setSegmentReplicationState(new SegmentReplicationState(replica.routingEntry(), node)); final int numDocs = shards.indexDocs(randomInt(10)); primary.refresh("Test"); @@ -802,6 +844,7 @@ public void testPrimaryCancelsExecution() throws Exception { shards.startAll(); IndexShard primary = shards.getPrimary(); final IndexShard replica = shards.getReplicas().get(0); + replica.setSegmentReplicationState(new SegmentReplicationState(replica.routingEntry(), node)); final int numDocs = shards.indexDocs(randomInt(10)); primary.refresh("Test"); diff --git a/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java index 465629406b54b..76236f7e23d28 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java @@ -9,12 +9,16 @@ package org.opensearch.indices.recovery; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.Settings; +import org.opensearch.common.transport.TransportAddress; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase; import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.index.shard.IndexShard; +import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.VersionUtils; public class RemoteStorePeerRecoverySourceHandlerTests extends OpenSearchIndexLevelReplicationTestCase { @@ -25,6 +29,12 @@ public class RemoteStorePeerRecoverySourceHandlerTests extends OpenSearchIndexLe .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "translog-repo") .build(); + DiscoveryNode node = new DiscoveryNode( + "101", + new TransportAddress(TransportAddress.META_ADDRESS, randomInt(0xFFFF)), + VersionUtils.randomVersion(random()) + ); + public void testReplicaShardRecoveryUptoLastFlushedCommit() throws Exception { try (ReplicationGroup shards = createGroup(0, settings, new NRTReplicationEngineFactory())) { @@ -36,6 +46,7 @@ public void testReplicaShardRecoveryUptoLastFlushedCommit() throws Exception { // Step 2 - Start replica for recovery to happen, check both has same number of docs final IndexShard replica1 = shards.addReplica(); + replica1.setSegmentReplicationState(new SegmentReplicationState(replica1.routingEntry(), node)); shards.startAll(); assertEquals(getDocIdAndSeqNos(primary), getDocIdAndSeqNos(replica1)); @@ -55,6 +66,7 @@ public void testReplicaShardRecoveryUptoLastFlushedCommit() throws Exception { // Step 6 - Start new replica, recovery happens, and check that new replica has docs upto last flush final IndexShard replica2 = shards.addReplica(); + replica2.setSegmentReplicationState(new SegmentReplicationState(replica2.routingEntry(), node)); shards.startAll(); assertDocCount(replica2, numDocs); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index 25625fbf68a64..9abca703fc6f3 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -13,8 +13,10 @@ import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.util.CancellableThreads; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.shard.IndexShard; @@ -23,7 +25,9 @@ import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationFailedException; +import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.VersionUtils; import java.io.IOException; import java.util.List; @@ -70,6 +74,12 @@ public void setUp() throws Exception { SegmentReplicationSourceFactory replicationSourceFactory = mock(SegmentReplicationSourceFactory.class); replicationSource = mock(SegmentReplicationSource.class); when(replicationSourceFactory.get(replicaShard)).thenReturn(replicationSource); + DiscoveryNode node = new DiscoveryNode( + "101", + new TransportAddress(TransportAddress.META_ADDRESS, randomInt(0xFFFF)), + VersionUtils.randomVersion(random()) + ); + replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); sut = prepareForReplication(primaryShard, null); initialCheckpoint = replicaShard.getLatestReplicationCheckpoint(); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java index ffea4aaf6b7c4..df03dc2b634da 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java @@ -29,7 +29,9 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionListener; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.Settings; +import org.opensearch.common.transport.TransportAddress; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.shard.IndexShard; @@ -43,6 +45,7 @@ import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.DummyShardLock; import org.opensearch.test.IndexSettingsModule; +import org.opensearch.test.VersionUtils; import java.io.FileNotFoundException; import java.io.IOException; @@ -95,6 +98,13 @@ public void setUp() throws Exception { indexShard = newStartedShard(false, indexSettings, new NRTReplicationEngineFactory()); spyIndexShard = spy(indexShard); + DiscoveryNode node = new DiscoveryNode( + "101", + new TransportAddress(TransportAddress.META_ADDRESS, randomInt(0xFFFF)), + VersionUtils.randomVersion(random()) + ); + when(spyIndexShard.getSegmentReplicationState()).thenReturn(new SegmentReplicationState(indexShard.routingEntry(), node)); + indexShard.setSegmentReplicationState(new SegmentReplicationState(indexShard.routingEntry(), node)); Mockito.doNothing().when(spyIndexShard).finalizeReplication(any(SegmentInfos.class), anyLong()); testSegmentInfos = spyIndexShard.store().readLastCommittedSegmentsInfo(); From b5432bd4ceb2b0aaff4f8879b51150cac8534a75 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Fri, 27 Jan 2023 18:22:27 +0000 Subject: [PATCH 09/25] Spotless Apply. Signed-off-by: Rishikesh1159 --- .../org/opensearch/index/shard/IndexShard.java | 5 ++--- ...coveryWithRemoteTranslogOnPrimaryTests.java | 2 +- .../SegmentReplicationIndexShardTests.java | 18 +++++++++--------- .../SegmentReplicationTargetServiceTests.java | 1 - 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index d947c790c9813..0c13854aebbaf 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -1553,10 +1553,9 @@ public SegmentReplicationState getSegmentReplicationState() { } public void setSegmentReplicationState(SegmentReplicationState segmentReplicationState) { - if(this.shardRouting.primary() == false || getReplicationEngine().isEmpty() == false){ + if (this.shardRouting.primary() == false || getReplicationEngine().isEmpty() == false) { this.segmentReplicationState = segmentReplicationState; - } - else throw new OpenSearchException("Cannot set Segment Replication State on a primary shard"); + } else throw new OpenSearchException("Cannot set Segment Replication State on a primary shard"); } /** diff --git a/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java b/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java index 144607c29e56d..3821a533c0850 100644 --- a/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java @@ -112,7 +112,7 @@ public IndexShard indexShard() { return idxShard; } }); - for(IndexShard replicaShard:shards.getReplicas()){ + for (IndexShard replicaShard : shards.getReplicas()) { replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); } diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index 9affd62adf594..f4e6ced551428 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -125,7 +125,7 @@ public void testSegmentReplication_Index_Update_Delete() throws Exception { } primaryShard.refresh("Test"); - for(IndexShard replicaShard:shards.getReplicas()){ + for (IndexShard replicaShard : shards.getReplicas()) { replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); } replicateSegments(primaryShard, shards.getReplicas()); @@ -156,7 +156,7 @@ public void testSegmentReplication_Index_Update_Delete() throws Exception { } } primaryShard.refresh("Test"); - for(IndexShard replicaShard:shards.getReplicas()){ + for (IndexShard replicaShard : shards.getReplicas()) { replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); } replicateSegments(primaryShard, shards.getReplicas()); @@ -331,7 +331,7 @@ public void testPrimaryRelocation() throws Exception { assert shardList.size() >= 2; final IndexShard primary = shardList.get(0); List replicaShards = shardList.subList(1, shardList.size()); - for(IndexShard replicaShard:replicaShards){ + for (IndexShard replicaShard : replicaShards) { replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); } return replicateSegments(primary, replicaShards, listener); @@ -415,7 +415,7 @@ public void testReplicaReceivesLowerGeneration() throws Exception { final IndexShard primary = shards.getPrimary(); final IndexShard replica_1 = shards.getReplicas().get(0); final IndexShard replica_2 = shards.getReplicas().get(1); - for(IndexShard replicaShard:shards.getReplicas()){ + for (IndexShard replicaShard : shards.getReplicas()) { replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); } int numDocs = randomIntBetween(10, 100); @@ -445,7 +445,7 @@ public void testReplicaReceivesLowerGeneration() throws Exception { numDocs = randomIntBetween(numDocs + 1, numDocs + 10); shards.indexDocs(numDocs); flushShard(replica_2, false); - for(IndexShard replicaShard:shards.getReplicas()){ + for (IndexShard replicaShard : shards.getReplicas()) { replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); } replicateSegments(replica_2, shards.getReplicas()); @@ -465,7 +465,7 @@ public void testReplicaRestarts() throws Exception { flushShard(primary); } primary.refresh("Test"); - for(IndexShard replicaShard:shards.getReplicas()){ + for (IndexShard replicaShard : shards.getReplicas()) { replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); } replicateSegments(primary, shards.getReplicas()); @@ -496,7 +496,7 @@ public void testReplicaRestarts() throws Exception { failAndPromoteRandomReplica(shards); } flushShard(shards.getPrimary()); - for(IndexShard replicaShard:shards.getReplicas()){ + for (IndexShard replicaShard : shards.getReplicas()) { replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); } replicateSegments(shards.getPrimary(), shards.getReplicas()); @@ -612,7 +612,7 @@ public void testNRTReplicaPromotedAsPrimary() throws Exception { IndexShard oldPrimary = shards.getPrimary(); final IndexShard nextPrimary = shards.getReplicas().get(0); final IndexShard replica = shards.getReplicas().get(1); - for(IndexShard replicaShard:shards.getReplicas()){ + for (IndexShard replicaShard : shards.getReplicas()) { replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); } @@ -742,7 +742,7 @@ public void onFailure(Exception e) { assertDocCount(newReplica, numDocs); nextPrimary.refresh("test"); - for(IndexShard replicaShard:shards.getReplicas()){ + for (IndexShard replicaShard : shards.getReplicas()) { replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); } replicateSegments(nextPrimary, shards.getReplicas()); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index 9abca703fc6f3..d59f7cb903dda 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -25,7 +25,6 @@ import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationFailedException; -import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.VersionUtils; From 2b92c81dab1818a7b97d80b71539ffb0162cb569 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Fri, 27 Jan 2023 22:19:40 +0000 Subject: [PATCH 10/25] Fix media type copile check. Signed-off-by: Rishikesh1159 --- .../segment_replication/SegmentReplicationResponse.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java index d96a0afdf2172..fcd56c339709c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java @@ -14,6 +14,7 @@ import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.indices.replication.SegmentReplicationState; import java.io.IOException; @@ -94,6 +95,6 @@ public void writeTo(StreamOutput out) throws IOException { @Override public String toString() { - return Strings.toString(this, true, true); + return Strings.toString(XContentType.JSON, this, true, true); } } From 5c9f8a7b9ec3eaf76d3a14e05da7c6ebe435f8de Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Sun, 29 Jan 2023 19:18:52 +0000 Subject: [PATCH 11/25] Revert previous changes and fix failing tests. Signed-off-by: Rishikesh1159 --- .../action/admin/ClientTimeoutIT.java | 49 +++++++++++++++---- .../opensearch/indices/IndicesService.java | 6 +-- .../replication/SegmentReplicationTarget.java | 5 +- ...overyWithRemoteTranslogOnPrimaryTests.java | 14 ------ .../SegmentReplicationIndexShardTests.java | 45 +---------------- ...teStorePeerRecoverySourceHandlerTests.java | 12 ----- .../SegmentReplicationTargetServiceTests.java | 9 ---- .../SegmentReplicationTargetTests.java | 10 ---- 8 files changed, 47 insertions(+), 103 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java index 978be79cf7eb6..7d0eee5087deb 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java @@ -17,11 +17,14 @@ import org.opensearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.opensearch.action.admin.indices.recovery.RecoveryAction; import org.opensearch.action.admin.indices.recovery.RecoveryResponse; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationAction; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationResponse; import org.opensearch.action.admin.indices.stats.IndicesStatsAction; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; @@ -47,6 +50,30 @@ public class ClientTimeoutIT extends OpenSearchIntegTestCase { protected Collection> nodePlugins() { return Collections.singletonList(MockTransportService.TestPlugin.class); } + private static String featureFlagSetting = "NONE"; + + @Override + protected Settings featureFlagSettings() { + if(!featureFlagSetting.equals("NONE")){ + return Settings.builder() + .put(super.featureFlagSettings()) + .put(featureFlagSetting, "true") + .build(); + } + else{ + return Settings.builder() + .put(super.featureFlagSettings()) + .build(); + } + } + + private void enableFeatureFlag(String featureFlagSetting){ + this.featureFlagSetting = featureFlagSetting; + } + + private void disableFeatureFlag(){ + this.featureFlagSetting = "NONE"; + } public void testNodesInfoTimeout() { String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); @@ -150,6 +177,7 @@ public void testRecoveriesWithTimeout() { } public void testSegment_ReplicationWithTimeout() { + enableFeatureFlag(FeatureFlags.REPLICATION_TYPE); internalCluster().startClusterManagerOnlyNode(); String dataNode = internalCluster().startDataOnlyNode(); String anotherDataNode = internalCluster().startDataOnlyNode(); @@ -174,21 +202,24 @@ public void testSegment_ReplicationWithTimeout() { ensureSearchable("test-index"); // Happy case - RecoveryResponse recoveryResponse = dataNodeClient().admin().indices().prepareRecoveries().get(); - assertThat(recoveryResponse.getTotalShards(), equalTo(numShards * 2)); - assertThat(recoveryResponse.getSuccessfulShards(), equalTo(numShards * 2)); + SegmentReplicationResponse segmentReplicationResponse = dataNodeClient().admin().indices().prepareSegment_Replication().get(); + assertThat(segmentReplicationResponse.getTotalShards(), equalTo(numShards * 2)); + assertThat(segmentReplicationResponse.getSuccessfulShards(), equalTo(numShards * 2)); // simulate timeout on bad node. - simulateTimeoutAtTransport(dataNode, anotherDataNode, RecoveryAction.NAME); + simulateTimeoutAtTransport(dataNode, anotherDataNode, SegmentReplicationAction.NAME); // verify response with bad node. - recoveryResponse = dataNodeClient().admin().indices().prepareRecoveries().get(); - assertThat(recoveryResponse.getTotalShards(), equalTo(numShards * 2)); - assertThat(recoveryResponse.getSuccessfulShards(), equalTo(numShards)); - assertThat(recoveryResponse.getFailedShards(), equalTo(numShards)); - assertThat(recoveryResponse.getShardFailures()[0].reason(), containsString("ReceiveTimeoutTransportException")); + segmentReplicationResponse = dataNodeClient().admin().indices().prepareSegment_Replication().get(); + assertThat(segmentReplicationResponse.getTotalShards(), equalTo(numShards * 2)); + assertThat(segmentReplicationResponse.getSuccessfulShards(), equalTo(numShards)); + assertThat(segmentReplicationResponse.getFailedShards(), equalTo(numShards)); + assertThat(segmentReplicationResponse.getShardFailures()[0].reason(), containsString("ReceiveTimeoutTransportException")); + disableFeatureFlag(); } + + public void testStatsWithTimeout() { internalCluster().startClusterManagerOnlyNode(); String dataNode = internalCluster().startDataOnlyNode(); diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 8ca8e7cf7095a..b98a88bacbdb9 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -1018,9 +1018,9 @@ public IndexShard createShard( .setSource(mapping.source().string(), XContentType.JSON) .get(); }, this); - if (indexService.getIndexSettings().isSegRepEnabled() && shardRouting.primary() == false) { - indexShard.setSegmentReplicationState(indexService.createSegmentReplicationState(shardRouting, targetNode)); - } +// if (indexService.getIndexSettings().isSegRepEnabled() && shardRouting.primary() == false) { +// indexShard.setSegmentReplicationState(indexService.createSegmentReplicationState(shardRouting, targetNode)); +// } return indexShard; } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 948eb50f972d0..468bf0ab14f3e 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -63,8 +63,9 @@ public SegmentReplicationTarget( super("replication_target", indexShard, new ReplicationLuceneIndex(), listener); this.checkpoint = checkpoint; this.source = source; - logger.info("shard is: {}", indexShard.routingEntry().primary()); - logger.info("state is: {}", indexShard.getSegmentReplicationState()); + if(indexShard.getSegmentReplicationState() == null){ + indexShard.setSegmentReplicationState(new SegmentReplicationState(indexShard.routingEntry(), indexShard.recoveryState().getSourceNode())); + } this.state = indexShard.getSegmentReplicationState() .onNewSegmentReplicationEvent( stateIndex, diff --git a/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java b/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java index 3821a533c0850..ba707cc30e6b8 100644 --- a/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java @@ -10,12 +10,10 @@ import org.junit.Assert; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.common.settings.Settings; -import org.opensearch.common.transport.TransportAddress; import org.opensearch.index.engine.DocIdSeqNoAndSource; import org.opensearch.index.engine.NRTReplicationEngine; import org.opensearch.index.engine.NRTReplicationEngineFactory; @@ -23,9 +21,7 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.translog.WriteOnlyTranslogManager; import org.opensearch.indices.recovery.RecoveryTarget; -import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.test.VersionUtils; import java.io.IOException; import java.util.List; @@ -42,12 +38,6 @@ public class ReplicaRecoveryWithRemoteTranslogOnPrimaryTests extends OpenSearchI .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "translog-repo") .build(); - DiscoveryNode node = new DiscoveryNode( - "101", - new TransportAddress(TransportAddress.META_ADDRESS, randomInt(0xFFFF)), - VersionUtils.randomVersion(random()) - ); - public void testStartSequenceForReplicaRecovery() throws Exception { try (ReplicationGroup shards = createGroup(0, settings, new NRTReplicationEngineFactory())) { @@ -112,9 +102,6 @@ public IndexShard indexShard() { return idxShard; } }); - for (IndexShard replicaShard : shards.getReplicas()) { - replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); - } shards.flush(); replicateSegments(primary, shards.getReplicas()); @@ -136,7 +123,6 @@ public void testNoTranslogHistoryTransferred() throws Exception { // Step 2 - Start replica, recovery happens, check docs recovered till last flush final IndexShard replica = shards.addReplica(); - replica.setSegmentReplicationState(new SegmentReplicationState(replica.routingEntry(), node)); shards.startAll(); assertEquals(docIdAndSeqNosAfterFlush, getDocIdAndSeqNos(replica)); assertDocCount(replica, numDocs); diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index f4e6ced551428..44771faf36871 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -16,13 +16,11 @@ import org.opensearch.action.delete.DeleteRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.xcontent.XContentType; @@ -52,7 +50,6 @@ import org.opensearch.indices.replication.common.ReplicationListener; import org.opensearch.indices.replication.common.ReplicationState; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.test.VersionUtils; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -81,12 +78,6 @@ public class SegmentReplicationIndexShardTests extends OpenSearchIndexLevelRepli .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(); - private DiscoveryNode node = new DiscoveryNode( - "101", - new TransportAddress(TransportAddress.META_ADDRESS, randomInt(0xFFFF)), - VersionUtils.randomVersion(random()) - ); - /** * Test that latestReplicationCheckpoint returns null only for docrep enabled indices */ @@ -125,9 +116,6 @@ public void testSegmentReplication_Index_Update_Delete() throws Exception { } primaryShard.refresh("Test"); - for (IndexShard replicaShard : shards.getReplicas()) { - replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); - } replicateSegments(primaryShard, shards.getReplicas()); shards.assertAllEqual(numDocs); @@ -156,9 +144,6 @@ public void testSegmentReplication_Index_Update_Delete() throws Exception { } } primaryShard.refresh("Test"); - for (IndexShard replicaShard : shards.getReplicas()) { - replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); - } replicateSegments(primaryShard, shards.getReplicas()); final List docsAfterDelete = getDocIdAndSeqNos(primaryShard); for (IndexShard shard : shards.getReplicas()) { @@ -172,7 +157,6 @@ public void testIgnoreShardIdle() throws Exception { shards.startAll(); final IndexShard primary = shards.getPrimary(); final IndexShard replica = shards.getReplicas().get(0); - replica.setSegmentReplicationState(new SegmentReplicationState(replica.routingEntry(), node)); final int numDocs = shards.indexDocs(randomInt(10)); primary.refresh("test"); @@ -274,7 +258,6 @@ public void testReplicaReceivesGenIncrease() throws Exception { shards.startAll(); final IndexShard primary = shards.getPrimary(); final IndexShard replica = shards.getReplicas().get(0); - replica.setSegmentReplicationState(new SegmentReplicationState(replica.routingEntry(), node)); final int numDocs = randomIntBetween(10, 100); shards.indexDocs(numDocs); assertEquals(numDocs, primary.translogStats().estimatedNumberOfOperations()); @@ -330,11 +313,7 @@ public void testPrimaryRelocation() throws Exception { try { assert shardList.size() >= 2; final IndexShard primary = shardList.get(0); - List replicaShards = shardList.subList(1, shardList.size()); - for (IndexShard replicaShard : replicaShards) { - replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); - } - return replicateSegments(primary, replicaShards, listener); + return replicateSegments(primary, shardList.subList(1, shardList.size()), listener); } catch (IOException | InterruptedException e) { listener.onFailure(e); throw new RuntimeException(e); @@ -415,9 +394,6 @@ public void testReplicaReceivesLowerGeneration() throws Exception { final IndexShard primary = shards.getPrimary(); final IndexShard replica_1 = shards.getReplicas().get(0); final IndexShard replica_2 = shards.getReplicas().get(1); - for (IndexShard replicaShard : shards.getReplicas()) { - replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); - } int numDocs = randomIntBetween(10, 100); shards.indexDocs(numDocs); flushShard(primary, false); @@ -445,9 +421,6 @@ public void testReplicaReceivesLowerGeneration() throws Exception { numDocs = randomIntBetween(numDocs + 1, numDocs + 10); shards.indexDocs(numDocs); flushShard(replica_2, false); - for (IndexShard replicaShard : shards.getReplicas()) { - replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); - } replicateSegments(replica_2, shards.getReplicas()); assertEqualCommittedSegments(replica_2, oldPrimary, replica_1); } @@ -465,9 +438,6 @@ public void testReplicaRestarts() throws Exception { flushShard(primary); } primary.refresh("Test"); - for (IndexShard replicaShard : shards.getReplicas()) { - replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); - } replicateSegments(primary, shards.getReplicas()); // at this point both shards should have numDocs persisted and searchable. @@ -496,9 +466,6 @@ public void testReplicaRestarts() throws Exception { failAndPromoteRandomReplica(shards); } flushShard(shards.getPrimary()); - for (IndexShard replicaShard : shards.getReplicas()) { - replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); - } replicateSegments(shards.getPrimary(), shards.getReplicas()); } primary = shards.getPrimary(); @@ -612,9 +579,6 @@ public void testNRTReplicaPromotedAsPrimary() throws Exception { IndexShard oldPrimary = shards.getPrimary(); final IndexShard nextPrimary = shards.getReplicas().get(0); final IndexShard replica = shards.getReplicas().get(1); - for (IndexShard replicaShard : shards.getReplicas()) { - replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); - } // 1. Create ops that are in the index and xlog of both shards but not yet part of a commit point. final int numDocs = shards.indexDocs(randomInt(10)); @@ -678,7 +642,6 @@ public void testReplicaPromotedWhileReplicating() throws Exception { shards.startAll(); final IndexShard oldPrimary = shards.getPrimary(); final IndexShard nextPrimary = shards.getReplicas().get(0); - nextPrimary.setSegmentReplicationState(new SegmentReplicationState(nextPrimary.routingEntry(), node)); final int numDocs = shards.indexDocs(randomInt(10)); oldPrimary.refresh("Test"); @@ -742,9 +705,6 @@ public void onFailure(Exception e) { assertDocCount(newReplica, numDocs); nextPrimary.refresh("test"); - for (IndexShard replicaShard : shards.getReplicas()) { - replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); - } replicateSegments(nextPrimary, shards.getReplicas()); final List docsAfterRecovery = getDocIdAndSeqNos(shards.getPrimary()); for (IndexShard shard : shards.getReplicas()) { @@ -758,7 +718,6 @@ public void testReplicaClosesWhileReplicating_AfterGetCheckpoint() throws Except shards.startAll(); IndexShard primary = shards.getPrimary(); final IndexShard replica = shards.getReplicas().get(0); - replica.setSegmentReplicationState(new SegmentReplicationState(replica.routingEntry(), node)); final int numDocs = shards.indexDocs(randomInt(10)); primary.refresh("Test"); @@ -801,7 +760,6 @@ public void testReplicaClosesWhileReplicating_AfterGetSegmentFiles() throws Exce shards.startAll(); IndexShard primary = shards.getPrimary(); final IndexShard replica = shards.getReplicas().get(0); - replica.setSegmentReplicationState(new SegmentReplicationState(replica.routingEntry(), node)); final int numDocs = shards.indexDocs(randomInt(10)); primary.refresh("Test"); @@ -844,7 +802,6 @@ public void testPrimaryCancelsExecution() throws Exception { shards.startAll(); IndexShard primary = shards.getPrimary(); final IndexShard replica = shards.getReplicas().get(0); - replica.setSegmentReplicationState(new SegmentReplicationState(replica.routingEntry(), node)); final int numDocs = shards.indexDocs(randomInt(10)); primary.refresh("Test"); diff --git a/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java index 76236f7e23d28..465629406b54b 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java @@ -9,16 +9,12 @@ package org.opensearch.indices.recovery; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.Settings; -import org.opensearch.common.transport.TransportAddress; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase; import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.index.shard.IndexShard; -import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.test.VersionUtils; public class RemoteStorePeerRecoverySourceHandlerTests extends OpenSearchIndexLevelReplicationTestCase { @@ -29,12 +25,6 @@ public class RemoteStorePeerRecoverySourceHandlerTests extends OpenSearchIndexLe .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "translog-repo") .build(); - DiscoveryNode node = new DiscoveryNode( - "101", - new TransportAddress(TransportAddress.META_ADDRESS, randomInt(0xFFFF)), - VersionUtils.randomVersion(random()) - ); - public void testReplicaShardRecoveryUptoLastFlushedCommit() throws Exception { try (ReplicationGroup shards = createGroup(0, settings, new NRTReplicationEngineFactory())) { @@ -46,7 +36,6 @@ public void testReplicaShardRecoveryUptoLastFlushedCommit() throws Exception { // Step 2 - Start replica for recovery to happen, check both has same number of docs final IndexShard replica1 = shards.addReplica(); - replica1.setSegmentReplicationState(new SegmentReplicationState(replica1.routingEntry(), node)); shards.startAll(); assertEquals(getDocIdAndSeqNos(primary), getDocIdAndSeqNos(replica1)); @@ -66,7 +55,6 @@ public void testReplicaShardRecoveryUptoLastFlushedCommit() throws Exception { // Step 6 - Start new replica, recovery happens, and check that new replica has docs upto last flush final IndexShard replica2 = shards.addReplica(); - replica2.setSegmentReplicationState(new SegmentReplicationState(replica2.routingEntry(), node)); shards.startAll(); assertDocCount(replica2, numDocs); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index d59f7cb903dda..25625fbf68a64 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -13,10 +13,8 @@ import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.util.CancellableThreads; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.shard.IndexShard; @@ -26,7 +24,6 @@ import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationFailedException; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.test.VersionUtils; import java.io.IOException; import java.util.List; @@ -73,12 +70,6 @@ public void setUp() throws Exception { SegmentReplicationSourceFactory replicationSourceFactory = mock(SegmentReplicationSourceFactory.class); replicationSource = mock(SegmentReplicationSource.class); when(replicationSourceFactory.get(replicaShard)).thenReturn(replicationSource); - DiscoveryNode node = new DiscoveryNode( - "101", - new TransportAddress(TransportAddress.META_ADDRESS, randomInt(0xFFFF)), - VersionUtils.randomVersion(random()) - ); - replicaShard.setSegmentReplicationState(new SegmentReplicationState(replicaShard.routingEntry(), node)); sut = prepareForReplication(primaryShard, null); initialCheckpoint = replicaShard.getLatestReplicationCheckpoint(); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java index 35c9ec1ddb9f2..5900ac3ad29f3 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java @@ -29,9 +29,7 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionListener; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.Settings; -import org.opensearch.common.transport.TransportAddress; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.shard.IndexShard; @@ -45,7 +43,6 @@ import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.DummyShardLock; import org.opensearch.test.IndexSettingsModule; -import org.opensearch.test.VersionUtils; import java.io.FileNotFoundException; import java.io.IOException; @@ -98,13 +95,6 @@ public void setUp() throws Exception { indexShard = newStartedShard(false, indexSettings, new NRTReplicationEngineFactory()); spyIndexShard = spy(indexShard); - DiscoveryNode node = new DiscoveryNode( - "101", - new TransportAddress(TransportAddress.META_ADDRESS, randomInt(0xFFFF)), - VersionUtils.randomVersion(random()) - ); - when(spyIndexShard.getSegmentReplicationState()).thenReturn(new SegmentReplicationState(indexShard.routingEntry(), node)); - indexShard.setSegmentReplicationState(new SegmentReplicationState(indexShard.routingEntry(), node)); Mockito.doNothing().when(spyIndexShard).finalizeReplication(any(SegmentInfos.class), anyLong()); testSegmentInfos = spyIndexShard.store().readLastCommittedSegmentsInfo(); From e022a7e72f547bdacb25cdbc29e84dbeeb6cb149 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Sun, 29 Jan 2023 19:32:24 +0000 Subject: [PATCH 12/25] Apply spotless check. Signed-off-by: Rishikesh1159 --- .../action/admin/ClientTimeoutIT.java | 21 +++++++------------ .../opensearch/indices/IndicesService.java | 6 +++--- .../replication/SegmentReplicationTarget.java | 6 ++++-- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java index 7d0eee5087deb..2f13939ae0663 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java @@ -50,28 +50,23 @@ public class ClientTimeoutIT extends OpenSearchIntegTestCase { protected Collection> nodePlugins() { return Collections.singletonList(MockTransportService.TestPlugin.class); } + private static String featureFlagSetting = "NONE"; @Override protected Settings featureFlagSettings() { - if(!featureFlagSetting.equals("NONE")){ - return Settings.builder() - .put(super.featureFlagSettings()) - .put(featureFlagSetting, "true") - .build(); - } - else{ - return Settings.builder() - .put(super.featureFlagSettings()) - .build(); + if (!featureFlagSetting.equals("NONE")) { + return Settings.builder().put(super.featureFlagSettings()).put(featureFlagSetting, "true").build(); + } else { + return Settings.builder().put(super.featureFlagSettings()).build(); } } - private void enableFeatureFlag(String featureFlagSetting){ + private void enableFeatureFlag(String featureFlagSetting) { this.featureFlagSetting = featureFlagSetting; } - private void disableFeatureFlag(){ + private void disableFeatureFlag() { this.featureFlagSetting = "NONE"; } @@ -218,8 +213,6 @@ public void testSegment_ReplicationWithTimeout() { disableFeatureFlag(); } - - public void testStatsWithTimeout() { internalCluster().startClusterManagerOnlyNode(); String dataNode = internalCluster().startDataOnlyNode(); diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index b98a88bacbdb9..de214ef4f8da7 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -1018,9 +1018,9 @@ public IndexShard createShard( .setSource(mapping.source().string(), XContentType.JSON) .get(); }, this); -// if (indexService.getIndexSettings().isSegRepEnabled() && shardRouting.primary() == false) { -// indexShard.setSegmentReplicationState(indexService.createSegmentReplicationState(shardRouting, targetNode)); -// } + // if (indexService.getIndexSettings().isSegRepEnabled() && shardRouting.primary() == false) { + // indexShard.setSegmentReplicationState(indexService.createSegmentReplicationState(shardRouting, targetNode)); + // } return indexShard; } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 468bf0ab14f3e..f5415ee0ed79d 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -63,8 +63,10 @@ public SegmentReplicationTarget( super("replication_target", indexShard, new ReplicationLuceneIndex(), listener); this.checkpoint = checkpoint; this.source = source; - if(indexShard.getSegmentReplicationState() == null){ - indexShard.setSegmentReplicationState(new SegmentReplicationState(indexShard.routingEntry(), indexShard.recoveryState().getSourceNode())); + if (indexShard.getSegmentReplicationState() == null) { + indexShard.setSegmentReplicationState( + new SegmentReplicationState(indexShard.routingEntry(), indexShard.recoveryState().getSourceNode()) + ); } this.state = indexShard.getSegmentReplicationState() .onNewSegmentReplicationEvent( From 8b26e2efe6fb6f6066ae8bbcf36629d6be88bf9b Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Sun, 29 Jan 2023 21:53:30 +0000 Subject: [PATCH 13/25] Refactoring call to segmentreplicationstate. Signed-off-by: Rishikesh1159 --- .../action/admin/ClientTimeoutIT.java | 2 +- .../org/opensearch/index/IndexModule.java | 9 ------ .../org/opensearch/index/IndexService.java | 12 -------- .../opensearch/indices/IndicesService.java | 4 --- .../replication/SegmentReplicationState.java | 28 +++---------------- .../replication/SegmentReplicationTarget.java | 13 +++------ .../opensearch/plugins/IndexStorePlugin.java | 12 -------- 7 files changed, 9 insertions(+), 71 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java index 2f13939ae0663..cb1a87e869237 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java @@ -51,7 +51,7 @@ protected Collection> nodePlugins() { return Collections.singletonList(MockTransportService.TestPlugin.class); } - private static String featureFlagSetting = "NONE"; + private String featureFlagSetting = "NONE"; @Override protected Settings featureFlagSettings() { diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index b86c270ec7514..f2675f018d0e6 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -54,7 +54,6 @@ import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.NamedXContentRegistry; import org.opensearch.core.internal.io.IOUtils; import org.opensearch.env.NodeEnvironment; @@ -79,7 +78,6 @@ import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; import org.opensearch.indices.recovery.RecoveryState; -import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.plugins.IndexStorePlugin; import org.opensearch.repositories.RepositoriesService; import org.opensearch.script.ScriptService; @@ -127,9 +125,6 @@ public final class IndexModule { private static final IndexStorePlugin.RecoveryStateFactory DEFAULT_RECOVERY_STATE_FACTORY = RecoveryState::new; - private static final IndexStorePlugin.SegmentReplicationStateFactory DEFAULT_SEGMENT_REPLICATION_STATE_FACTORY = - SegmentReplicationState::new; - public static final Setting INDEX_STORE_TYPE_SETTING = new Setting<>( "index.store.type", "", @@ -516,7 +511,6 @@ public IndexService newIndexService( eventListener.beforeIndexCreated(indexSettings.getIndex(), indexSettings.getSettings()); final IndexStorePlugin.DirectoryFactory directoryFactory = getDirectoryFactory(indexSettings, directoryFactories); final IndexStorePlugin.RecoveryStateFactory recoveryStateFactory = getRecoveryStateFactory(indexSettings, recoveryStateFactories); - final IndexStorePlugin.SegmentReplicationStateFactory segmentReplicationStateFactory = DEFAULT_SEGMENT_REPLICATION_STATE_FACTORY; QueryCache queryCache = null; IndexAnalyzers indexAnalyzers = null; boolean success = false; @@ -565,9 +559,6 @@ public IndexService newIndexService( expressionResolver, valuesSourceRegistry, recoveryStateFactory, - FeatureFlags.isEnabled(FeatureFlags.REPLICATION_TYPE) && indexSettings.isSegRepEnabled() - ? segmentReplicationStateFactory - : null, translogFactorySupplier ); success = true; diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index b7fda1543662e..78211f12f71ad 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -95,7 +95,6 @@ import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; import org.opensearch.indices.recovery.RecoveryState; -import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.plugins.IndexStorePlugin; import org.opensearch.script.ScriptService; @@ -141,9 +140,6 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final IndexStorePlugin.DirectoryFactory directoryFactory; private final IndexStorePlugin.RemoteDirectoryFactory remoteDirectoryFactory; private final IndexStorePlugin.RecoveryStateFactory recoveryStateFactory; - - @Nullable - private final IndexStorePlugin.SegmentReplicationStateFactory segmentReplicationStateFactory; private final CheckedFunction readerWrapper; private final IndexCache indexCache; private final MapperService mapperService; @@ -211,7 +207,6 @@ public IndexService( IndexNameExpressionResolver expressionResolver, ValuesSourceRegistry valuesSourceRegistry, IndexStorePlugin.RecoveryStateFactory recoveryStateFactory, - IndexStorePlugin.SegmentReplicationStateFactory segmentReplicationStateFactory, BiFunction translogFactorySupplier ) { super(indexSettings); @@ -273,7 +268,6 @@ public IndexService( this.directoryFactory = directoryFactory; this.remoteDirectoryFactory = remoteDirectoryFactory; this.recoveryStateFactory = recoveryStateFactory; - this.segmentReplicationStateFactory = segmentReplicationStateFactory; this.engineFactory = Objects.requireNonNull(engineFactory); this.engineConfigFactory = Objects.requireNonNull(engineConfigFactory); // initialize this last -- otherwise if the wrapper requires any other member to be non-null we fail with an NPE @@ -658,12 +652,6 @@ public RecoveryState createRecoveryState(ShardRouting shardRouting, DiscoveryNod return recoveryStateFactory.newRecoveryState(shardRouting, targetNode, sourceNode); } - public SegmentReplicationState createSegmentReplicationState(ShardRouting shardRouting, DiscoveryNode node) { - if (segmentReplicationStateFactory != null && indexSettings.isSegRepEnabled() && shardRouting.primary() == false) { - return segmentReplicationStateFactory.newSegmentReplicationState(shardRouting, node); - } else return null; - } - @Override public IndexSettings getIndexSettings() { return indexSettings; diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index de214ef4f8da7..4644a09f3e03c 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -263,7 +263,6 @@ public class IndicesService extends AbstractLifecycleComponent private final Collection>> engineFactoryProviders; private final Map directoryFactories; private final Map recoveryStateFactories; - final AbstractRefCounted indicesRefCount; // pkg-private for testing private final CountDownLatch closeLatch = new CountDownLatch(1); private volatile boolean idFieldDataEnabled; @@ -1018,9 +1017,6 @@ public IndexShard createShard( .setSource(mapping.source().string(), XContentType.JSON) .get(); }, this); - // if (indexService.getIndexSettings().isSegRepEnabled() && shardRouting.primary() == false) { - // indexShard.setSegmentReplicationState(indexService.createSegmentReplicationState(shardRouting, targetNode)); - // } return indexShard; } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index 183bf62599c2c..30aaa3e0590bc 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -136,41 +136,21 @@ public long getTotalNumberOfSegRepEvents() { private long totalNumberOfSegRepEvents; - public SegmentReplicationState(ShardRouting shardRouting, ReplicationLuceneIndex index, DiscoveryNode node) { - stage = Stage.INIT; + public SegmentReplicationState(ShardRouting shardRouting, ReplicationLuceneIndex index, long replicationId, DiscoveryNode sourceNode, DiscoveryNode targetNode) { this.index = index; this.shardRouting = shardRouting; + this.replicationId = replicationId; + this.sourceNode = sourceNode; + this.targetNode = targetNode; // Timing data will have as many entries as stages, plus one - overallTimer = new ReplicationTimer(); replicating = new Replicating(); getCheckpointInfo = new GetCheckpointInfo(); fileDiff = new FileDiff(); getFile = new GetFile(); finalizeReplication = new FinalizeReplication(); - // set an invalid value by default - this.replicationId = -1L; - this.sourceNode = node; - this.targetNode = node; - } - - public SegmentReplicationState(ShardRouting shardRouting, DiscoveryNode node) { - this(shardRouting, new ReplicationLuceneIndex(), node); - } - - public SegmentReplicationState onNewSegmentReplicationEvent( - ReplicationLuceneIndex index, - long replicationId, - DiscoveryNode sourceNode, - DiscoveryNode targetNode - ) { - this.index = index; - this.replicationId = replicationId; - this.sourceNode = sourceNode; - this.targetNode = targetNode; setStage(Stage.INIT); totalNumberOfSegRepEvents++; - return this; } public SegmentReplicationState(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index f5415ee0ed79d..74de7943ca784 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -63,18 +63,13 @@ public SegmentReplicationTarget( super("replication_target", indexShard, new ReplicationLuceneIndex(), listener); this.checkpoint = checkpoint; this.source = source; - if (indexShard.getSegmentReplicationState() == null) { - indexShard.setSegmentReplicationState( - new SegmentReplicationState(indexShard.routingEntry(), indexShard.recoveryState().getSourceNode()) - ); - } - this.state = indexShard.getSegmentReplicationState() - .onNewSegmentReplicationEvent( - stateIndex, + indexShard.setSegmentReplicationState( + new SegmentReplicationState(indexShard.routingEntry(), stateIndex, getId(), indexShard.recoveryState().getSourceNode(), indexShard.recoveryState().getTargetNode() - ); + )); + this.state = indexShard.getSegmentReplicationState(); this.multiFileWriter = new MultiFileWriter(indexShard.store(), stateIndex, getPrefix(), logger, this::ensureRefCount); } diff --git a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java index ca38246766fd8..5a73a6fdb1aa8 100644 --- a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java @@ -104,18 +104,6 @@ interface RecoveryStateFactory { RecoveryState newRecoveryState(ShardRouting shardRouting, DiscoveryNode targetNode, @Nullable DiscoveryNode sourceNode); } - /** - * An interface that allows to create a new {@link SegmentReplicationState} per shard. - */ - @FunctionalInterface - interface SegmentReplicationStateFactory { - /** - * Creates a new {@link SegmentReplicationState} per shard. This method is called once per shard on shard creation. - * @return a new SegmentReplicationState instance - */ - SegmentReplicationState newSegmentReplicationState(ShardRouting shardRouting, DiscoveryNode node); - } - /** * The {@link RecoveryStateFactory} mappings for this plugin. When an index is created the recovery type setting * {@link org.opensearch.index.IndexModule#INDEX_RECOVERY_TYPE_SETTING} on the index will be examined and either use the default From dd4f5b168b02e02385ce401d75428e3fa2e3bc80 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Sun, 29 Jan 2023 21:55:27 +0000 Subject: [PATCH 14/25] spotless check Signed-off-by: Rishikesh1159 --- .../indices/replication/SegmentReplicationState.java | 8 +++++++- .../indices/replication/SegmentReplicationTarget.java | 7 +++++-- .../java/org/opensearch/plugins/IndexStorePlugin.java | 1 - 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index 30aaa3e0590bc..f003e5aa7d624 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -136,7 +136,13 @@ public long getTotalNumberOfSegRepEvents() { private long totalNumberOfSegRepEvents; - public SegmentReplicationState(ShardRouting shardRouting, ReplicationLuceneIndex index, long replicationId, DiscoveryNode sourceNode, DiscoveryNode targetNode) { + public SegmentReplicationState( + ShardRouting shardRouting, + ReplicationLuceneIndex index, + long replicationId, + DiscoveryNode sourceNode, + DiscoveryNode targetNode + ) { this.index = index; this.shardRouting = shardRouting; this.replicationId = replicationId; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 74de7943ca784..236eb24d75b27 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -64,11 +64,14 @@ public SegmentReplicationTarget( this.checkpoint = checkpoint; this.source = source; indexShard.setSegmentReplicationState( - new SegmentReplicationState(indexShard.routingEntry(), stateIndex, + new SegmentReplicationState( + indexShard.routingEntry(), + stateIndex, getId(), indexShard.recoveryState().getSourceNode(), indexShard.recoveryState().getTargetNode() - )); + ) + ); this.state = indexShard.getSegmentReplicationState(); this.multiFileWriter = new MultiFileWriter(indexShard.store(), stateIndex, getPrefix(), logger, this::ensureRefCount); } diff --git a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java index 5a73a6fdb1aa8..1dc90a21c2f70 100644 --- a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java @@ -39,7 +39,6 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; import org.opensearch.indices.recovery.RecoveryState; -import org.opensearch.indices.replication.SegmentReplicationState; import java.io.IOException; import java.util.Collections; From c85e38a8cbccb9a85e0ea7dd96021e2efe53a328 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Tue, 31 Jan 2023 16:47:47 +0000 Subject: [PATCH 15/25] Changing invokation of segment replication shard and filtering API response by shard id Signed-off-by: Rishikesh1159 --- gradle/run.gradle | 2 +- .../api/cat.segment_replication.json | 4 ++ .../SegmentReplicationRequest.java | 21 ++++++ .../opensearch/common/util/FeatureFlags.java | 11 +-- .../opensearch/index/shard/IndexShard.java | 15 ++++- .../opensearch/indices/IndicesService.java | 4 ++ .../replication/SegmentReplicationState.java | 31 +++++---- .../replication/SegmentReplicationTarget.java | 18 +++-- .../cat/RestCatSegmentReplicationAction.java | 67 +++++++++++++------ .../RestSegment_ReplicationActionTests.java | 14 +--- 10 files changed, 126 insertions(+), 61 deletions(-) diff --git a/gradle/run.gradle b/gradle/run.gradle index 639479e97d28f..b7ca1129fe083 100644 --- a/gradle/run.gradle +++ b/gradle/run.gradle @@ -31,7 +31,7 @@ import org.opensearch.gradle.testclusters.RunTask apply plugin: 'opensearch.testclusters' -def numNodes = findProperty('numNodes') as Integer ?: 1 +def numNodes = findProperty('numNodes') as Integer ?: 3 def numZones = findProperty('numZones') as Integer ?: 1 testClusters { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json index 9ec9fb05d66dc..35bc196b5b7d1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json @@ -59,6 +59,10 @@ "description":"If `true`, the response includes detailed information about segment replications", "default":false }, + "shards":{ + "type":"list", + "description":"Comma-separated list of shards to display" + }, "h":{ "type":"list", "description":"Comma-separated list of column names to display" diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java index 809e0728137d5..d592e6b0b8079 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java @@ -25,6 +25,8 @@ public class SegmentReplicationRequest extends BroadcastRequest REPLICATION_TYPE_SETTING = Setting.boolSetting(REPLICATION_TYPE, false, Property.NodeScope); diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 0c13854aebbaf..d0627c421c3be 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -1549,15 +1549,26 @@ public final boolean shouldProcessCheckpoint(ReplicationCheckpoint requestCheckp } public SegmentReplicationState getSegmentReplicationState() { - return this.segmentReplicationState; + if (checkSegmentReplicationAllowed(segmentReplicationState)) { + return this.segmentReplicationState; + } + if (this.segmentReplicationState != null) this.segmentReplicationState = null; + return null; } public void setSegmentReplicationState(SegmentReplicationState segmentReplicationState) { - if (this.shardRouting.primary() == false || getReplicationEngine().isEmpty() == false) { + if (checkSegmentReplicationAllowed(segmentReplicationState)) { this.segmentReplicationState = segmentReplicationState; } else throw new OpenSearchException("Cannot set Segment Replication State on a primary shard"); } + private boolean checkSegmentReplicationAllowed(SegmentReplicationState segmentReplicationState) { + if (indexSettings.isSegRepEnabled() && (this.shardRouting.primary() == false || getReplicationEngine().isEmpty() == false)) { + return true; + } + return false; + } + /** * gets a {@link Store.MetadataSnapshot} for the current directory. This method is safe to call in all lifecycle of the index shard, * without having to worry about the current state of the engine and concurrent flushes. diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 4644a09f3e03c..8af3512b76a77 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -143,6 +143,7 @@ import org.opensearch.indices.recovery.PeerRecoveryTargetService; import org.opensearch.indices.recovery.RecoveryListener; import org.opensearch.indices.recovery.RecoveryState; +import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.node.Node; import org.opensearch.plugins.IndexStorePlugin; @@ -1017,6 +1018,9 @@ public IndexShard createShard( .setSource(mapping.source().string(), XContentType.JSON) .get(); }, this); + if (indexService.getIndexSettings().isSegRepEnabled() && shardRouting.primary() == false) { + indexShard.setSegmentReplicationState(new SegmentReplicationState(indexShard.routingEntry(), sourceNode, targetNode)); + } return indexShard; } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index f003e5aa7d624..1695487798e3e 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -130,12 +130,6 @@ public DiscoveryNode getTargetNode() { private DiscoveryNode targetNode; - public long getTotalNumberOfSegRepEvents() { - return totalNumberOfSegRepEvents; - } - - private long totalNumberOfSegRepEvents; - public SegmentReplicationState( ShardRouting shardRouting, ReplicationLuceneIndex index, @@ -155,8 +149,24 @@ public SegmentReplicationState( fileDiff = new FileDiff(); getFile = new GetFile(); finalizeReplication = new FinalizeReplication(); + } + + public SegmentReplicationState(ShardRouting shardRouting, DiscoveryNode sourceNode, DiscoveryNode targetNode) { + this(shardRouting, new ReplicationLuceneIndex(), -1, sourceNode, targetNode); + } + + public SegmentReplicationState onNewSegmentReplicationEvent( + ReplicationLuceneIndex index, + long replicationId, + DiscoveryNode sourceNode, + DiscoveryNode targetNode + ) { + this.index = index; + this.replicationId = replicationId; + this.sourceNode = sourceNode; + this.targetNode = targetNode; setStage(Stage.INIT); - totalNumberOfSegRepEvents++; + return this; } public SegmentReplicationState(StreamInput in) throws IOException { @@ -172,7 +182,6 @@ public SegmentReplicationState(StreamInput in) throws IOException { finalizeReplication = new FinalizeReplication(in); sourceNode = new DiscoveryNode(in); targetNode = new DiscoveryNode(in); - totalNumberOfSegRepEvents = in.readLong(); } @Override @@ -182,7 +191,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeEnum(stage); out.writeLong(replicationId); overallTimer.writeTo(out); - // stageTimer.writeTo(out); replicating.writeTo(out); getCheckpointInfo.writeTo(out); fileDiff.writeTo(out); @@ -190,8 +198,6 @@ public void writeTo(StreamOutput out) throws IOException { finalizeReplication.writeTo(out); sourceNode.writeTo(out); targetNode.writeTo(out); - out.writeLong(totalNumberOfSegRepEvents); - } @Override @@ -283,7 +289,6 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.field(Fields.ID, shardRouting.shardId().id()); builder.field(Fields.STAGE, getStage()); builder.field(Fields.REPLICATION_ID, getReplicationId()); - builder.field(Fields.NUMBER_OF_SEGMENT_REPLICATION_EVENTS, getTotalNumberOfSegRepEvents()); builder.timeField(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, getTimer().startTime()); if (getTimer().stopTime() > 0) { builder.timeField(Fields.STOP_TIME_IN_MILLIS, Fields.STOP_TIME, getTimer().stopTime()); @@ -352,8 +357,6 @@ static final class Fields { static final String SIZE_OF_FILES_IN_BYTES = "size_of_files_in_bytes"; - static final String NUMBER_OF_SEGMENT_REPLICATION_EVENTS = "number_of_segment_replication_events"; - static final String INDEX = "index"; static final String TOTAL_GET_FILES_STAGE_TIME_IN_MILLIS = "total_get_files_stage_in_millis"; static final String REPLICATING_STAGE = "replicating_stage"; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 236eb24d75b27..3ff21ac01db6e 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -63,16 +63,22 @@ public SegmentReplicationTarget( super("replication_target", indexShard, new ReplicationLuceneIndex(), listener); this.checkpoint = checkpoint; this.source = source; - indexShard.setSegmentReplicationState( - new SegmentReplicationState( - indexShard.routingEntry(), + if (indexShard.getSegmentReplicationState() == null) { + indexShard.setSegmentReplicationState( + new SegmentReplicationState( + indexShard.routingEntry(), + indexShard.recoveryState().getSourceNode(), + indexShard.recoveryState().getTargetNode() + ) + ); + } + this.state = indexShard.getSegmentReplicationState() + .onNewSegmentReplicationEvent( stateIndex, getId(), indexShard.recoveryState().getSourceNode(), indexShard.recoveryState().getTargetNode() - ) - ); - this.state = indexShard.getSegmentReplicationState(); + ); this.multiFileWriter = new MultiFileWriter(indexShard.store(), stateIndex, getPrefix(), logger, this::ensureRefCount); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java index 63b638c1f77bd..78473a5ae6061 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java @@ -27,6 +27,8 @@ import java.util.Comparator; import java.util.List; import java.util.Locale; +import java.util.Set; +import java.util.HashSet; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; @@ -65,6 +67,7 @@ public BaseRestHandler.RestChannelConsumer doCatRequest(final RestRequest reques ); segmentReplicationRequest.timeout(request.param("timeout")); segmentReplicationRequest.detailed(request.paramAsBoolean("detailed", false)); + segmentReplicationRequest.shards(Strings.splitStringByCommaToArray(request.param("shards"))); segmentReplicationRequest.activeOnly(request.paramAsBoolean("active_only", false)); segmentReplicationRequest.indicesOptions(IndicesOptions.fromRequest(request, segmentReplicationRequest.indicesOptions())); @@ -90,7 +93,6 @@ protected Table getTableWithHeader(RestRequest request) { t.startHeaders() .addCell("index", "alias:i,idx;desc:index name") .addCell("shardId", "desc: shard Id") - .addCell("replication_id", "desc: replication Id") .addCell("start_time", "default:false;alias:start;desc:segment replication start time") .addCell("start_time_millis", "default:false;alias:start_millis;desc:segment replication start time in epoch milliseconds") .addCell("stop_time", "default:false;alias:stop;desc:segment replication stop time") @@ -101,16 +103,17 @@ protected Table getTableWithHeader(RestRequest request) { .addCell("source_node", "alias:snode;desc:source node name") .addCell("target_host", "alias:thost;desc:target host") .addCell("target_node", "alias:tnode;desc:target node name") - .addCell("files", "alias:f;desc:number of files to fetch") .addCell("files_fetched", "alias:ff;desc:files fetched") .addCell("files_percent", "alias:fp;desc:percent of files fetched") - .addCell("files_total", "alias:tf;desc:total number of files") - .addCell("bytes", "alias:b;desc:number of bytes to fetch") .addCell("bytes_fetched", "alias:bf;desc:bytes fetched") - .addCell("bytes_percent", "alias:bp;desc:percent of bytes fetched") - .addCell("bytes_total", "alias:tb;desc:total number of bytes"); + .addCell("bytes_percent", "alias:bp;desc:percent of bytes fetched"); if (detailed) { - t.addCell("replicating_stage_time_taken", "alias:rstt;desc:time taken in replicating stage") + t.addCell("files", "alias:f;desc:number of files to fetch") + .addCell("files_total", "alias:tf;desc:total number of files") + .addCell("bytes", "alias:b;desc:number of bytes to fetch") + .addCell("bytes_total", "alias:tb;desc:total number of bytes") + .addCell("replication_id", "desc: replication Id") + .addCell("replicating_stage_time_taken", "alias:rstt;desc:time taken in replicating stage") .addCell("get_checkpoint_info_stage_time_taken", "alias:gcistt;desc:time taken in get checkpoint info stage") .addCell("file_diff_stage_time_taken", "alias:fdstt;desc:time taken in file diff stage") .addCell("get_files_stage_time_taken", "alias:gfstt;desc:time taken in get files stage") @@ -120,6 +123,15 @@ protected Table getTableWithHeader(RestRequest request) { return t; } + protected Table buildCustomResponseTable(String reason) { + Table t = new Table(); + t.startHeaders().addCell("Reason", "desc: Reason for API response").endHeaders(); + t.startRow(); + t.addCell(reason); + t.endRow(); + return t; + } + /** * buildSegmentReplicationTable will build a table of SegmentReplication information suitable * for displaying at the command line. @@ -129,19 +141,28 @@ protected Table getTableWithHeader(RestRequest request) { * @return A table containing index, shardId, node, target size, recovered size and percentage for each fetching replica */ public Table buildSegmentReplicationTable(RestRequest request, SegmentReplicationResponse response) { - String indexname = ""; + String[] indexNames = new String[0]; boolean detailed = false; + String[] shards = new String[0]; + Set set = new HashSet<>(); if (request != null) { - indexname = request.param("index"); + indexNames = Strings.splitStringByCommaToArray(request.param("index")); detailed = Boolean.parseBoolean(request.param("detailed")); + shards = Strings.splitStringByCommaToArray(request.param("shards")); + } + if (request != null && indexNames.length > 0) { + for (String index : indexNames) { + if (!response.shardSegmentReplicationStates().containsKey(index)) { + String reason = "error:" + "{Segment Replication is not enabled on index: " + index + " }"; + return buildCustomResponseTable(reason); + } + } + } - if (request != null && indexname != null && !response.shardSegmentReplicationStates().containsKey(indexname)) { - Table t = new Table(); - t.startHeaders().addCell("Reason", "desc: Reason for API response").endHeaders(); - t.startRow(); - t.addCell("{ Segment Replication is not enabled on index: " + indexname + " }"); - t.endRow(); - return t; + if (shards.length > 0) { + for (String shard : shards) { + set.add(shard); + } } Table t = getTableWithHeader(request); @@ -169,10 +190,13 @@ public int compare(SegmentReplicationState o1, SegmentReplicationState o2) { }); for (SegmentReplicationState state : shardSegmentReplicationStates) { + int shardId = state.getShardRouting().shardId().id(); + if (shards.length > 0 && !set.contains(Integer.toString(shardId))) { + continue; + } t.startRow(); t.addCell(index); t.addCell(state.getShardRouting().shardId().id()); - t.addCell(state.getReplicationId()); t.addCell(XContentOpenSearchExtension.DEFAULT_DATE_PRINTER.print(state.getTimer().startTime())); t.addCell(state.getTimer().startTime()); t.addCell(XContentOpenSearchExtension.DEFAULT_DATE_PRINTER.print(state.getTimer().stopTime())); @@ -183,15 +207,16 @@ public int compare(SegmentReplicationState o1, SegmentReplicationState o2) { t.addCell(state.getSourceNode().getName()); t.addCell(state.getTargetNode().getHostName()); t.addCell(state.getTargetNode().getName()); - t.addCell(state.getIndex().totalRecoverFiles()); t.addCell(state.getIndex().recoveredFileCount()); t.addCell(String.format(Locale.ROOT, "%1.1f%%", state.getIndex().recoveredFilesPercent())); - t.addCell(state.getIndex().totalFileCount()); - t.addCell(state.getIndex().totalRecoverBytes()); t.addCell(state.getIndex().recoveredBytes()); t.addCell(String.format(Locale.ROOT, "%1.1f%%", state.getIndex().recoveredBytesPercent())); - t.addCell(state.getIndex().totalBytes()); if (detailed) { + t.addCell(state.getIndex().totalRecoverFiles()); + t.addCell(state.getIndex().totalFileCount()); + t.addCell(state.getIndex().totalRecoverBytes()); + t.addCell(state.getIndex().totalBytes()); + t.addCell(state.getReplicationId()); t.addCell(new TimeValue(state.getReplicating().time())); t.addCell(new TimeValue(state.getGetCheckpointInfo().time())); t.addCell(new TimeValue(state.getFileDiff().time())); diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java index bb0223b05068a..acda16d1d0f02 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java @@ -94,7 +94,6 @@ public void testSegment_ReplicationActionAction() { final List expectedHeaders = Arrays.asList( "index", "shardId", - "replication_id", "start_time", "start_time_millis", "stop_time", @@ -105,14 +104,10 @@ public void testSegment_ReplicationActionAction() { "source_node", "target_host", "target_node", - "files", "files_fetched", "files_percent", - "files_total", - "bytes", "bytes_fetched", - "bytes_percent", - "bytes_total" + "bytes_percent" ); for (int i = 0; i < expectedHeaders.size(); i++) { @@ -126,7 +121,6 @@ public void testSegment_ReplicationActionAction() { final List expectedValues = Arrays.asList( "index", i, - state.getReplicationId(), XContentOpenSearchExtension.DEFAULT_DATE_PRINTER.print(state.getTimer().startTime()), state.getTimer().startTime(), XContentOpenSearchExtension.DEFAULT_DATE_PRINTER.print(state.getTimer().stopTime()), @@ -137,14 +131,10 @@ public void testSegment_ReplicationActionAction() { state.getSourceNode().getName(), state.getTargetNode().getHostName(), state.getTargetNode().getName(), - state.getIndex().totalRecoverFiles(), state.getIndex().recoveredFileCount(), percent(state.getIndex().recoveredFilesPercent()), - state.getIndex().totalFileCount(), - state.getIndex().totalRecoverBytes(), state.getIndex().recoveredBytes(), - percent(state.getIndex().recoveredBytesPercent()), - state.getIndex().totalBytes() + percent(state.getIndex().recoveredBytesPercent()) ); final List cells = table.getRows().get(i); From 2ef7a9fa741440bf7530dc5da3c2774fd77aa0d9 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Tue, 31 Jan 2023 16:51:50 +0000 Subject: [PATCH 16/25] disable feature flag by default. Signed-off-by: Rishikesh1159 --- gradle/run.gradle | 2 +- .../java/org/opensearch/common/util/FeatureFlags.java | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/gradle/run.gradle b/gradle/run.gradle index b7ca1129fe083..639479e97d28f 100644 --- a/gradle/run.gradle +++ b/gradle/run.gradle @@ -31,7 +31,7 @@ import org.opensearch.gradle.testclusters.RunTask apply plugin: 'opensearch.testclusters' -def numNodes = findProperty('numNodes') as Integer ?: 3 +def numNodes = findProperty('numNodes') as Integer ?: 1 def numZones = findProperty('numZones') as Integer ?: 1 testClusters { diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 362fbb1661127..69509bccf5684 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -75,12 +75,11 @@ public static void initializeFeatureFlags(Settings openSearchSettings) { * and false otherwise. */ public static boolean isEnabled(String featureFlagName) { - // if ("true".equalsIgnoreCase(System.getProperty(featureFlagName))) { - // // TODO: Remove the if condition once FeatureFlags are only supported via opensearch.yml - // return true; - // } - // return settings != null && settings.getAsBoolean(featureFlagName, false); - return true; + if ("true".equalsIgnoreCase(System.getProperty(featureFlagName))) { + // TODO: Remove the if condition once FeatureFlags are only supported via opensearch.yml + return true; + } + return settings != null && settings.getAsBoolean(featureFlagName, false); } public static final Setting REPLICATION_TYPE_SETTING = Setting.boolSetting(REPLICATION_TYPE, false, Property.NodeScope); From 9a4f09d928b4852e7edd8cfc91a9f2e912e30fe6 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Tue, 31 Jan 2023 17:07:04 +0000 Subject: [PATCH 17/25] Apply spotless Signed-off-by: Rishikesh1159 --- .../java/org/opensearch/common/util/FeatureFlags.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 69509bccf5684..72b7349180bad 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -75,11 +75,11 @@ public static void initializeFeatureFlags(Settings openSearchSettings) { * and false otherwise. */ public static boolean isEnabled(String featureFlagName) { - if ("true".equalsIgnoreCase(System.getProperty(featureFlagName))) { - // TODO: Remove the if condition once FeatureFlags are only supported via opensearch.yml - return true; - } - return settings != null && settings.getAsBoolean(featureFlagName, false); + if ("true".equalsIgnoreCase(System.getProperty(featureFlagName))) { + // TODO: Remove the if condition once FeatureFlags are only supported via opensearch.yml + return true; + } + return settings != null && settings.getAsBoolean(featureFlagName, false); } public static final Setting REPLICATION_TYPE_SETTING = Setting.boolSetting(REPLICATION_TYPE, false, Property.NodeScope); From 1080536f53888932df564dc1872b5ff7faf92f87 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Fri, 3 Feb 2023 18:07:18 +0000 Subject: [PATCH 18/25] Address comments on PR. Signed-off-by: Rishikesh1159 --- .../api/cat.segment_replication.json | 11 +- .../opensearch/action/IndicesRequestIT.java | 12 +- .../action/admin/ClientTimeoutIT.java | 58 ++- .../org/opensearch/action/ActionModule.java | 6 +- ...ava => SegmentReplicationStatsAction.java} | 8 +- ...va => SegmentReplicationStatsRequest.java} | 32 +- ...egmentReplicationStatsRequestBuilder.java} | 16 +- ...a => SegmentReplicationStatsResponse.java} | 8 +- ...ansportSegmentReplicationStatsAction.java} | 94 +++-- .../opensearch/client/IndicesAdminClient.java | 12 +- .../client/support/AbstractClient.java | 22 +- .../opensearch/index/shard/IndexShard.java | 21 - .../opensearch/indices/IndicesService.java | 4 - .../PrimaryShardReplicationSource.java | 9 + .../replication/SegmentReplicationSource.java | 7 + .../replication/SegmentReplicationState.java | 362 ++++-------------- .../replication/SegmentReplicationTarget.java | 25 +- .../SegmentReplicationTargetService.java | 33 ++ .../cat/RestCatSegmentReplicationAction.java | 66 +--- .../RestSegment_ReplicationActionTests.java | 12 +- 20 files changed, 323 insertions(+), 495 deletions(-) rename server/src/main/java/org/opensearch/action/admin/indices/segment_replication/{SegmentReplicationAction.java => SegmentReplicationStatsAction.java} (61%) rename server/src/main/java/org/opensearch/action/admin/indices/segment_replication/{SegmentReplicationRequest.java => SegmentReplicationStatsRequest.java} (76%) rename server/src/main/java/org/opensearch/action/admin/indices/segment_replication/{SegmentReplicationRequestBuilder.java => SegmentReplicationStatsRequestBuilder.java} (52%) rename server/src/main/java/org/opensearch/action/admin/indices/segment_replication/{SegmentReplicationResponse.java => SegmentReplicationStatsResponse.java} (94%) rename server/src/main/java/org/opensearch/action/admin/indices/segment_replication/{TransportSegmentReplicationAction.java => TransportSegmentReplicationStatsAction.java} (53%) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json index 35bc196b5b7d1..581e7a3a235a7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json @@ -1,8 +1,8 @@ { "cat.segment_replication":{ "documentation":{ - "url":"https://opensearch.org/docs/1.0/opensearch/rest-api/cat/cat-recovery/", - "description":"Returns information about index Segment Replication events, both on-going and completed." + "url":"https://github.com/opensearch-project/documentation-website/issues/2627", + "description":"Returns information about both on-going and latest completed Segment Replication events" }, "stability":"stable", "url":{ @@ -34,7 +34,12 @@ }, "active_only":{ "type":"boolean", - "description":"If `true`, the response only includes ongoing segment replications", + "description":"If `true`, the response only includes ongoing segment replication events", + "default":false + }, + "active_only":{ + "type":"boolean", + "description":"If `true`, the response only includes latest completed segment replication events", "default":false }, "bytes":{ diff --git a/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java index c65c1f6b3233f..9b2423f7029ff 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java @@ -56,8 +56,8 @@ import org.opensearch.action.admin.indices.recovery.RecoveryRequest; import org.opensearch.action.admin.indices.refresh.RefreshRequest; import org.opensearch.action.admin.indices.refresh.TransportShardRefreshAction; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationAction; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationRequest; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsAction; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsRequest; import org.opensearch.action.admin.indices.segments.IndicesSegmentsAction; import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsAction; @@ -471,14 +471,14 @@ public void testRecovery() { } public void testSegment_Replication() { - String segmentReplicationAction = SegmentReplicationAction.NAME + "[n]"; + String segmentReplicationAction = SegmentReplicationStatsAction.NAME + "[n]"; interceptTransportActions(segmentReplicationAction); - SegmentReplicationRequest segmentReplicationRequest = new SegmentReplicationRequest(randomIndicesOrAliases()); - internalCluster().coordOnlyNodeClient().admin().indices().segment_replication(segmentReplicationRequest).actionGet(); + SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(randomIndicesOrAliases()); + internalCluster().coordOnlyNodeClient().admin().indices().segment_replication(segmentReplicationStatsRequest).actionGet(); clearInterceptedActions(); - assertSameIndices(segmentReplicationRequest, segmentReplicationAction); + assertSameIndices(segmentReplicationStatsRequest, segmentReplicationAction); } public void testSegments() { diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java index cb1a87e869237..98d1524de8115 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java @@ -17,8 +17,8 @@ import org.opensearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.opensearch.action.admin.indices.recovery.RecoveryAction; import org.opensearch.action.admin.indices.recovery.RecoveryResponse; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationAction; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationResponse; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsAction; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsResponse; import org.opensearch.action.admin.indices.stats.IndicesStatsAction; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.cluster.metadata.IndexMetadata; @@ -51,25 +51,6 @@ protected Collection> nodePlugins() { return Collections.singletonList(MockTransportService.TestPlugin.class); } - private String featureFlagSetting = "NONE"; - - @Override - protected Settings featureFlagSettings() { - if (!featureFlagSetting.equals("NONE")) { - return Settings.builder().put(super.featureFlagSettings()).put(featureFlagSetting, "true").build(); - } else { - return Settings.builder().put(super.featureFlagSettings()).build(); - } - } - - private void enableFeatureFlag(String featureFlagSetting) { - this.featureFlagSetting = featureFlagSetting; - } - - private void disableFeatureFlag() { - this.featureFlagSetting = "NONE"; - } - public void testNodesInfoTimeout() { String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); String dataNode = internalCluster().startDataOnlyNode(); @@ -172,10 +153,15 @@ public void testRecoveriesWithTimeout() { } public void testSegment_ReplicationWithTimeout() { - enableFeatureFlag(FeatureFlags.REPLICATION_TYPE); - internalCluster().startClusterManagerOnlyNode(); - String dataNode = internalCluster().startDataOnlyNode(); - String anotherDataNode = internalCluster().startDataOnlyNode(); + internalCluster().startClusterManagerOnlyNode( + Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REPLICATION_TYPE, "true").build() + ); + String dataNode = internalCluster().startDataOnlyNode( + Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REPLICATION_TYPE, "true").build() + ); + String anotherDataNode = internalCluster().startDataOnlyNode( + Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REPLICATION_TYPE, "true").build() + ); int numShards = 4; assertAcked( @@ -197,20 +183,22 @@ public void testSegment_ReplicationWithTimeout() { ensureSearchable("test-index"); // Happy case - SegmentReplicationResponse segmentReplicationResponse = dataNodeClient().admin().indices().prepareSegment_Replication().get(); - assertThat(segmentReplicationResponse.getTotalShards(), equalTo(numShards * 2)); - assertThat(segmentReplicationResponse.getSuccessfulShards(), equalTo(numShards * 2)); + SegmentReplicationStatsResponse segmentReplicationStatsResponse = dataNodeClient().admin() + .indices() + .prepareSegmentReplication() + .get(); + assertThat(segmentReplicationStatsResponse.getTotalShards(), equalTo(numShards * 2)); + assertThat(segmentReplicationStatsResponse.getSuccessfulShards(), equalTo(numShards * 2)); // simulate timeout on bad node. - simulateTimeoutAtTransport(dataNode, anotherDataNode, SegmentReplicationAction.NAME); + simulateTimeoutAtTransport(dataNode, anotherDataNode, SegmentReplicationStatsAction.NAME); // verify response with bad node. - segmentReplicationResponse = dataNodeClient().admin().indices().prepareSegment_Replication().get(); - assertThat(segmentReplicationResponse.getTotalShards(), equalTo(numShards * 2)); - assertThat(segmentReplicationResponse.getSuccessfulShards(), equalTo(numShards)); - assertThat(segmentReplicationResponse.getFailedShards(), equalTo(numShards)); - assertThat(segmentReplicationResponse.getShardFailures()[0].reason(), containsString("ReceiveTimeoutTransportException")); - disableFeatureFlag(); + segmentReplicationStatsResponse = dataNodeClient().admin().indices().prepareSegmentReplication().get(); + assertThat(segmentReplicationStatsResponse.getTotalShards(), equalTo(numShards * 2)); + assertThat(segmentReplicationStatsResponse.getSuccessfulShards(), equalTo(numShards)); + assertThat(segmentReplicationStatsResponse.getFailedShards(), equalTo(numShards)); + assertThat(segmentReplicationStatsResponse.getShardFailures()[0].reason(), containsString("ReceiveTimeoutTransportException")); } public void testStatsWithTimeout() { diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 96354b579ebda..832196285311e 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -176,8 +176,8 @@ import org.opensearch.action.admin.indices.resolve.ResolveIndexAction; import org.opensearch.action.admin.indices.rollover.RolloverAction; import org.opensearch.action.admin.indices.rollover.TransportRolloverAction; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationAction; -import org.opensearch.action.admin.indices.segment_replication.TransportSegmentReplicationAction; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsAction; +import org.opensearch.action.admin.indices.segment_replication.TransportSegmentReplicationStatsAction; import org.opensearch.action.admin.indices.segments.IndicesSegmentsAction; import org.opensearch.action.admin.indices.segments.PitSegmentsAction; import org.opensearch.action.admin.indices.segments.TransportIndicesSegmentsAction; @@ -652,7 +652,7 @@ public void reg actions.register(ExplainAction.INSTANCE, TransportExplainAction.class); actions.register(ClearScrollAction.INSTANCE, TransportClearScrollAction.class); actions.register(RecoveryAction.INSTANCE, TransportRecoveryAction.class); - actions.register(SegmentReplicationAction.INSTANCE, TransportSegmentReplicationAction.class); + actions.register(SegmentReplicationStatsAction.INSTANCE, TransportSegmentReplicationStatsAction.class); actions.register(NodesReloadSecureSettingsAction.INSTANCE, TransportNodesReloadSecureSettingsAction.class); actions.register(AutoCreateAction.INSTANCE, AutoCreateAction.TransportAction.class); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationAction.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsAction.java similarity index 61% rename from server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationAction.java rename to server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsAction.java index d22c24d7fbeef..3810ea23ca30e 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsAction.java @@ -15,11 +15,11 @@ * * @opensearch.internal */ -public class SegmentReplicationAction extends ActionType { - public static final SegmentReplicationAction INSTANCE = new SegmentReplicationAction(); +public class SegmentReplicationStatsAction extends ActionType { + public static final SegmentReplicationStatsAction INSTANCE = new SegmentReplicationStatsAction(); public static final String NAME = "indices:monitor/segment_replication"; - private SegmentReplicationAction() { - super(NAME, SegmentReplicationResponse::new); + private SegmentReplicationStatsAction() { + super(NAME, SegmentReplicationStatsResponse::new); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequest.java similarity index 76% rename from server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java rename to server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequest.java index d592e6b0b8079..ce66e27e766de 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequest.java @@ -21,23 +21,27 @@ * * @opensearch.internal */ -public class SegmentReplicationRequest extends BroadcastRequest { +public class SegmentReplicationStatsRequest extends BroadcastRequest { private boolean detailed = false; // Provides extra details in the response private boolean activeOnly = false; // Only reports on active segment replication events private String[] shards = new String[0]; + private boolean completedOnly = false; + /** * Constructs a request for segment replication information for all shards */ - public SegmentReplicationRequest() { + public SegmentReplicationStatsRequest() { this(Strings.EMPTY_ARRAY); } - public SegmentReplicationRequest(StreamInput in) throws IOException { + public SegmentReplicationStatsRequest(StreamInput in) throws IOException { super(in); detailed = in.readBoolean(); activeOnly = in.readBoolean(); + completedOnly = in.readBoolean(); + } /** @@ -45,7 +49,7 @@ public SegmentReplicationRequest(StreamInput in) throws IOException { * * @param indices Comma-separated list of indices about which to gather segment replication information */ - public SegmentReplicationRequest(String... indices) { + public SegmentReplicationStatsRequest(String... indices) { super(indices, IndicesOptions.STRICT_EXPAND_OPEN_CLOSED); } @@ -87,6 +91,25 @@ public void activeOnly(boolean activeOnly) { this.activeOnly = activeOnly; } + /** + * True if completedOnly flag is set, false otherwise. This value is false by default. + * + * @return True if completedOnly flag is set, false otherwise + */ + public boolean completedOnly() { + return completedOnly; + } + + /** + * Set value of the completedOnly flag. If true, this request will only respond with + * latest completed segment replication event information. + * + * @param completedOnly Whether or not to set the completedOnly flag. + */ + public void completedOnly(boolean completedOnly) { + this.completedOnly = completedOnly; + } + /** * Contains list of shard id's if shards are passed, empty otherwise. Array is empty by default. * @@ -111,5 +134,6 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeBoolean(detailed); out.writeBoolean(activeOnly); + out.writeBoolean(completedOnly); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequestBuilder.java similarity index 52% rename from server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java rename to server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequestBuilder.java index d58cb45643773..e05c307eb2b9e 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequestBuilder.java @@ -16,21 +16,21 @@ * * @opensearch.internal */ -public class SegmentReplicationRequestBuilder extends BroadcastOperationRequestBuilder< - SegmentReplicationRequest, - SegmentReplicationResponse, - SegmentReplicationRequestBuilder> { +public class SegmentReplicationStatsRequestBuilder extends BroadcastOperationRequestBuilder< + SegmentReplicationStatsRequest, + SegmentReplicationStatsResponse, + SegmentReplicationStatsRequestBuilder> { - public SegmentReplicationRequestBuilder(OpenSearchClient client, SegmentReplicationAction action) { - super(client, action, new SegmentReplicationRequest()); + public SegmentReplicationStatsRequestBuilder(OpenSearchClient client, SegmentReplicationStatsAction action) { + super(client, action, new SegmentReplicationStatsRequest()); } - public SegmentReplicationRequestBuilder setDetailed(boolean detailed) { + public SegmentReplicationStatsRequestBuilder setDetailed(boolean detailed) { request.detailed(detailed); return this; } - public SegmentReplicationRequestBuilder setActiveOnly(boolean activeOnly) { + public SegmentReplicationStatsRequestBuilder setActiveOnly(boolean activeOnly) { request.activeOnly(activeOnly); return this; } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsResponse.java similarity index 94% rename from server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java rename to server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsResponse.java index fcd56c339709c..20af0170f321b 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsResponse.java @@ -26,10 +26,10 @@ * * @opensearch.internal */ -public class SegmentReplicationResponse extends BroadcastResponse { +public class SegmentReplicationStatsResponse extends BroadcastResponse { private final Map> shardSegmentReplicationStates; - public SegmentReplicationResponse(StreamInput in) throws IOException { + public SegmentReplicationStatsResponse(StreamInput in) throws IOException { super(in); shardSegmentReplicationStates = in.readMapOfLists(StreamInput::readString, SegmentReplicationState::new); } @@ -41,10 +41,10 @@ public SegmentReplicationResponse(StreamInput in) throws IOException { * @param totalShards Total count of shards seen * @param successfulShards Count of shards successfully processed * @param failedShards Count of shards which failed to process - * @param shardSegmentReplicationStates Map of indices to shard recovery information + * @param shardSegmentReplicationStates Map of indices to shard replication information * @param shardFailures List of failures processing shards */ - public SegmentReplicationResponse( + public SegmentReplicationStatsResponse( int totalShards, int successfulShards, int failedShards, diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationStatsAction.java similarity index 53% rename from server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java rename to server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationStatsAction.java index b34c4223e7ee7..cdecafd1691a2 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationStatsAction.java @@ -8,6 +8,7 @@ package org.opensearch.action.admin.indices.segment_replication; +import org.opensearch.ResourceNotFoundException; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; @@ -24,14 +25,17 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.SegmentReplicationState; +import org.opensearch.indices.replication.SegmentReplicationTargetService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; import java.util.List; +import java.util.ArrayList; import java.util.Map; +import java.util.HashMap; /** * Transport action for shard segment replication operation. This transport action does not actually @@ -39,31 +43,35 @@ * * @opensearch.internal */ -public class TransportSegmentReplicationAction extends TransportBroadcastByNodeAction< - SegmentReplicationRequest, - SegmentReplicationResponse, +public class TransportSegmentReplicationStatsAction extends TransportBroadcastByNodeAction< + SegmentReplicationStatsRequest, + SegmentReplicationStatsResponse, SegmentReplicationState> { + private final SegmentReplicationTargetService targetService; private final IndicesService indicesService; + private String singleIndexWithSegmentReplicationDisabled = null; @Inject - public TransportSegmentReplicationAction( + public TransportSegmentReplicationStatsAction( ClusterService clusterService, TransportService transportService, IndicesService indicesService, + SegmentReplicationTargetService targetService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver ) { super( - SegmentReplicationAction.NAME, + SegmentReplicationStatsAction.NAME, clusterService, transportService, actionFilters, indexNameExpressionResolver, - SegmentReplicationRequest::new, + SegmentReplicationStatsRequest::new, ThreadPool.Names.MANAGEMENT ); this.indicesService = indicesService; + this.targetService = targetService; } @Override @@ -72,8 +80,8 @@ protected SegmentReplicationState readShardResult(StreamInput in) throws IOExcep } @Override - protected SegmentReplicationResponse newResponse( - SegmentReplicationRequest request, + protected SegmentReplicationStatsResponse newResponse( + SegmentReplicationStatsRequest request, int totalShards, int successfulShards, int failedShards, @@ -81,50 +89,86 @@ protected SegmentReplicationResponse newResponse( List shardFailures, ClusterState clusterState ) { + // throw exception if API call is made on single index with segment replication disabled. + if (singleIndexWithSegmentReplicationDisabled != null) { + String index = singleIndexWithSegmentReplicationDisabled; + singleIndexWithSegmentReplicationDisabled = null; + throw new ResourceNotFoundException("Segment Replication is not enabled on Index: " + index); + } + String[] shards = request.shards(); + Set set = new HashSet<>(); + if (shards.length > 0) { + for (String shard : shards) { + set.add(shard); + } + } Map> shardResponses = new HashMap<>(); for (SegmentReplicationState segmentReplicationState : responses) { if (segmentReplicationState == null) { continue; } + + // Limit responses to only specific shard id's passed in query paramter shards. + int shardId = segmentReplicationState.getShardRouting().shardId().id(); + if (shards.length > 0 && set.contains(Integer.toString(shardId)) == false) { + continue; + } String indexName = segmentReplicationState.getShardRouting().getIndexName(); if (!shardResponses.containsKey(indexName)) { shardResponses.put(indexName, new ArrayList<>()); } - if (request.activeOnly()) { - if (segmentReplicationState.getStage() != SegmentReplicationState.Stage.DONE) { - shardResponses.get(indexName).add(segmentReplicationState); - } - } else { - shardResponses.get(indexName).add(segmentReplicationState); - } + shardResponses.get(indexName).add(segmentReplicationState); } - return new SegmentReplicationResponse(totalShards, successfulShards, failedShards, shardResponses, shardFailures); + return new SegmentReplicationStatsResponse(totalShards, successfulShards, failedShards, shardResponses, shardFailures); } @Override - protected SegmentReplicationRequest readRequestFrom(StreamInput in) throws IOException { - return new SegmentReplicationRequest(in); + protected SegmentReplicationStatsRequest readRequestFrom(StreamInput in) throws IOException { + return new SegmentReplicationStatsRequest(in); } @Override - protected SegmentReplicationState shardOperation(SegmentReplicationRequest request, ShardRouting shardRouting) { + protected SegmentReplicationState shardOperation(SegmentReplicationStatsRequest request, ShardRouting shardRouting) { IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); - return indexShard.getSegmentReplicationState(); + + // check if API call is made on single index with segment replication disabled. + if (request.indices().length == 1 && indexShard.indexSettings().isSegRepEnabled() == false) { + singleIndexWithSegmentReplicationDisabled = shardRouting.getIndexName(); + return null; + } + if (indexShard.indexSettings().isSegRepEnabled() == false) { + return null; + } + + // return information about only on-going segment replication events. + if (request.activeOnly()) { + return targetService.getOngoingEventSegmentReplicationState(shardRouting); + } + + // return information about only latest completed segment replication events. + if (request.completedOnly()) { + return targetService.getlatestCompletedEventSegmentReplicationState(shardRouting); + } + return targetService.getSegmentReplicationState(shardRouting); } @Override - protected ShardsIterator shards(ClusterState state, SegmentReplicationRequest request, String[] concreteIndices) { + protected ShardsIterator shards(ClusterState state, SegmentReplicationStatsRequest request, String[] concreteIndices) { return state.routingTable().allShardsIncludingRelocationTargets(concreteIndices); } @Override - protected ClusterBlockException checkGlobalBlock(ClusterState state, SegmentReplicationRequest request) { + protected ClusterBlockException checkGlobalBlock(ClusterState state, SegmentReplicationStatsRequest request) { return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); } @Override - protected ClusterBlockException checkRequestBlock(ClusterState state, SegmentReplicationRequest request, String[] concreteIndices) { + protected ClusterBlockException checkRequestBlock( + ClusterState state, + SegmentReplicationStatsRequest request, + String[] concreteIndices + ) { return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices); } } diff --git a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java index 62f13560a8e89..f63a483a6b356 100644 --- a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java +++ b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java @@ -91,9 +91,9 @@ import org.opensearch.action.admin.indices.rollover.RolloverRequest; import org.opensearch.action.admin.indices.rollover.RolloverRequestBuilder; import org.opensearch.action.admin.indices.rollover.RolloverResponse; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationRequest; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationRequestBuilder; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationResponse; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsRequest; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsRequestBuilder; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsResponse; import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse; import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequestBuilder; @@ -191,12 +191,12 @@ public interface IndicesAdminClient extends OpenSearchClient { /** *Indices segment replication */ - ActionFuture segment_replication(SegmentReplicationRequest request); + ActionFuture segment_replication(SegmentReplicationStatsRequest request); /** *Indices segment replication */ - void segment_replication(SegmentReplicationRequest request, ActionListener listener); + void segment_replication(SegmentReplicationStatsRequest request, ActionListener listener); /** * Indices recoveries @@ -206,7 +206,7 @@ public interface IndicesAdminClient extends OpenSearchClient { /** * Indices segment replication */ - SegmentReplicationRequestBuilder prepareSegment_Replication(String... indices); + SegmentReplicationStatsRequestBuilder prepareSegmentReplication(String... indices); /** * The segments of one or more indices. diff --git a/server/src/main/java/org/opensearch/client/support/AbstractClient.java b/server/src/main/java/org/opensearch/client/support/AbstractClient.java index f17898e08ccb5..c4df2f2e63e75 100644 --- a/server/src/main/java/org/opensearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/opensearch/client/support/AbstractClient.java @@ -260,10 +260,10 @@ import org.opensearch.action.admin.indices.rollover.RolloverRequest; import org.opensearch.action.admin.indices.rollover.RolloverRequestBuilder; import org.opensearch.action.admin.indices.rollover.RolloverResponse; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationAction; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationRequest; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationRequestBuilder; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationResponse; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsAction; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsRequest; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsRequestBuilder; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsResponse; import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse; import org.opensearch.action.admin.indices.segments.IndicesSegmentsAction; import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; @@ -1780,21 +1780,21 @@ public RecoveryRequestBuilder prepareRecoveries(String... indices) { } @Override - public ActionFuture segment_replication(final SegmentReplicationRequest request) { - return execute(SegmentReplicationAction.INSTANCE, request); + public ActionFuture segment_replication(final SegmentReplicationStatsRequest request) { + return execute(SegmentReplicationStatsAction.INSTANCE, request); } @Override public void segment_replication( - final SegmentReplicationRequest request, - final ActionListener listener + final SegmentReplicationStatsRequest request, + final ActionListener listener ) { - execute(SegmentReplicationAction.INSTANCE, request, listener); + execute(SegmentReplicationStatsAction.INSTANCE, request, listener); } @Override - public SegmentReplicationRequestBuilder prepareSegment_Replication(String... indices) { - return new SegmentReplicationRequestBuilder(this, SegmentReplicationAction.INSTANCE).setIndices(indices); + public SegmentReplicationStatsRequestBuilder prepareSegmentReplication(String... indices) { + return new SegmentReplicationStatsRequestBuilder(this, SegmentReplicationStatsAction.INSTANCE).setIndices(indices); } @Override diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 14e6590a116e9..d5dd724524654 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -1548,27 +1548,6 @@ public final boolean shouldProcessCheckpoint(ReplicationCheckpoint requestCheckp return true; } - public SegmentReplicationState getSegmentReplicationState() { - if (checkSegmentReplicationAllowed(segmentReplicationState)) { - return this.segmentReplicationState; - } - if (this.segmentReplicationState != null) this.segmentReplicationState = null; - return null; - } - - public void setSegmentReplicationState(SegmentReplicationState segmentReplicationState) { - if (checkSegmentReplicationAllowed(segmentReplicationState)) { - this.segmentReplicationState = segmentReplicationState; - } else throw new OpenSearchException("Cannot set Segment Replication State on a primary shard"); - } - - private boolean checkSegmentReplicationAllowed(SegmentReplicationState segmentReplicationState) { - if (indexSettings.isSegRepEnabled() && (this.shardRouting.primary() == false || getReplicationEngine().isEmpty() == false)) { - return true; - } - return false; - } - /** * gets a {@link Store.MetadataSnapshot} for the current directory. This method is safe to call in all lifecycle of the index shard, * without having to worry about the current state of the engine and concurrent flushes. diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 8af3512b76a77..4644a09f3e03c 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -143,7 +143,6 @@ import org.opensearch.indices.recovery.PeerRecoveryTargetService; import org.opensearch.indices.recovery.RecoveryListener; import org.opensearch.indices.recovery.RecoveryState; -import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.node.Node; import org.opensearch.plugins.IndexStorePlugin; @@ -1018,9 +1017,6 @@ public IndexShard createShard( .setSource(mapping.source().string(), XContentType.JSON) .get(); }, this); - if (indexService.getIndexSettings().isSegRepEnabled() && shardRouting.primary() == false) { - indexShard.setSegmentReplicationState(new SegmentReplicationState(indexShard.routingEntry(), sourceNode, targetNode)); - } return indexShard; } diff --git a/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java index 8107f99723eaf..3b65a75fad233 100644 --- a/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java +++ b/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java @@ -38,6 +38,8 @@ public class PrimaryShardReplicationSource implements SegmentReplicationSource { private static final Logger logger = LogManager.getLogger(PrimaryShardReplicationSource.class); private final RetryableTransportClient transportClient; + + private final DiscoveryNode sourceNode; private final DiscoveryNode targetNode; private final String targetAllocationId; @@ -55,6 +57,7 @@ public PrimaryShardReplicationSource( recoverySettings.internalActionRetryTimeout(), logger ); + this.sourceNode = sourceNode; this.targetNode = targetNode; } @@ -104,6 +107,12 @@ public void getSegmentFiles( transportClient.executeRetryableAction(GET_SEGMENT_FILES, request, options, responseListener, reader); } + @Override + public String getSourceDescription() { + String description = "Host:" + this.sourceNode.getHostName() + ", Node:" + this.sourceNode.getName(); + return description; + } + @Override public void cancel() { transportClient.cancel(); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java index b2e7487fff4b2..1270dd5b6fb76 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java @@ -49,6 +49,13 @@ void getSegmentFiles( ActionListener listener ); + /** + * Get the source description + */ + default String getSourceDescription() { + return null; + } + /** * Cancel any ongoing requests, should resolve any ongoing listeners with onFailure with a {@link ExecutionCancelledException}. */ diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index 1695487798e3e..fd5cd02025085 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -22,6 +22,8 @@ import org.opensearch.indices.replication.common.ReplicationTimer; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; /** * ReplicationState implementation to track Segment Replication events. @@ -75,6 +77,14 @@ public static Stage fromId(byte id) { private Stage stage; private ReplicationLuceneIndex index; + private final ReplicationTimer overallTimer; + private final Map timingData; + private final ReplicationTimer stageTimer; + private long replicationId; + private final ShardRouting shardRouting; + private String sourceDescription; + private DiscoveryNode targetNode; + public ReplicationTimer getOverallTimer() { return overallTimer; } @@ -83,90 +93,71 @@ public ShardRouting getShardRouting() { return shardRouting; } - private final ReplicationTimer overallTimer; - - public Replicating getReplicating() { - return replicating; - } - - public GetCheckpointInfo getGetCheckpointInfo() { - return getCheckpointInfo; + @Override + public ReplicationLuceneIndex getIndex() { + return index; } - public FileDiff getFileDiff() { - return fileDiff; + public long getReplicationId() { + return replicationId; } - public GetFile getGetFile() { - return getFile; + @Override + public ReplicationTimer getTimer() { + return overallTimer; } - public FinalizeReplication getFinalizeReplication() { - return finalizeReplication; + public Stage getStage() { + return this.stage; } - private final Replicating replicating; - - private final GetCheckpointInfo getCheckpointInfo; + public String getSourceDescription() { - private final FileDiff fileDiff; - - private final GetFile getFile; + return sourceDescription; + } - private final FinalizeReplication finalizeReplication; - private long replicationId; + public DiscoveryNode getTargetNode() { + return targetNode; + } - private final ShardRouting shardRouting; + public long getReplicatingStageTime() { + return timingData.get(Stage.REPLICATING.toString()); + } - public DiscoveryNode getSourceNode() { - return sourceNode; + public long getGetCheckpointInfoStageTime() { + return timingData.get(Stage.GET_CHECKPOINT_INFO.toString()); } - public DiscoveryNode getTargetNode() { - return targetNode; + public long getFileDiffStageTime() { + return timingData.get(Stage.FILE_DIFF.toString()); } - private DiscoveryNode sourceNode; + public long getGetFileStageTime() { + return timingData.get(Stage.GET_FILES.toString()); + } - private DiscoveryNode targetNode; + public long getFinalizeReplicationStageTime() { + return timingData.get(Stage.FINALIZE_REPLICATION.toString()); + } public SegmentReplicationState( ShardRouting shardRouting, ReplicationLuceneIndex index, long replicationId, - DiscoveryNode sourceNode, + String sourceDescription, DiscoveryNode targetNode ) { this.index = index; this.shardRouting = shardRouting; this.replicationId = replicationId; - this.sourceNode = sourceNode; + this.sourceDescription = sourceDescription; this.targetNode = targetNode; // Timing data will have as many entries as stages, plus one + timingData = new HashMap<>(Stage.values().length + 1); overallTimer = new ReplicationTimer(); - replicating = new Replicating(); - getCheckpointInfo = new GetCheckpointInfo(); - fileDiff = new FileDiff(); - getFile = new GetFile(); - finalizeReplication = new FinalizeReplication(); - } - - public SegmentReplicationState(ShardRouting shardRouting, DiscoveryNode sourceNode, DiscoveryNode targetNode) { - this(shardRouting, new ReplicationLuceneIndex(), -1, sourceNode, targetNode); - } - - public SegmentReplicationState onNewSegmentReplicationEvent( - ReplicationLuceneIndex index, - long replicationId, - DiscoveryNode sourceNode, - DiscoveryNode targetNode - ) { - this.index = index; - this.replicationId = replicationId; - this.sourceNode = sourceNode; - this.targetNode = targetNode; + stageTimer = new ReplicationTimer(); setStage(Stage.INIT); - return this; + stageTimer.start(); } public SegmentReplicationState(StreamInput in) throws IOException { @@ -175,12 +166,9 @@ public SegmentReplicationState(StreamInput in) throws IOException { stage = in.readEnum(Stage.class); replicationId = in.readLong(); overallTimer = new ReplicationTimer(in); - replicating = new Replicating(in); - getCheckpointInfo = new GetCheckpointInfo(in); - fileDiff = new FileDiff(in); - getFile = new GetFile(in); - finalizeReplication = new FinalizeReplication(in); - sourceNode = new DiscoveryNode(in); + stageTimer = new ReplicationTimer(in); + timingData = in.readMap(StreamInput::readString, StreamInput::readLong); + sourceDescription = in.readString(); targetNode = new DiscoveryNode(in); } @@ -191,33 +179,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeEnum(stage); out.writeLong(replicationId); overallTimer.writeTo(out); - replicating.writeTo(out); - getCheckpointInfo.writeTo(out); - fileDiff.writeTo(out); - getFile.writeTo(out); - finalizeReplication.writeTo(out); - sourceNode.writeTo(out); + stageTimer.writeTo(out); + out.writeMap(timingData, StreamOutput::writeString, StreamOutput::writeLong); + out.writeString(sourceDescription); targetNode.writeTo(out); } - @Override - public ReplicationLuceneIndex getIndex() { - return index; - } - - public long getReplicationId() { - return replicationId; - } - - @Override - public ReplicationTimer getTimer() { - return overallTimer; - } - - public Stage getStage() { - return this.stage; - } - protected void validateAndSetStage(Stage expected, Stage next) { if (stage != expected) { assert false : "can't move replication to stage [" + next + "]. current stage: [" + stage + "] (expected [" + expected + "])"; @@ -225,6 +192,16 @@ protected void validateAndSetStage(Stage expected, Stage next) { "can't move replication to stage [" + next + "]. current stage: [" + stage + "] (expected [" + expected + "])" ); } + stopTimersAndSetStage(next); + } + + private void stopTimersAndSetStage(Stage next) { + // save the timing data for the current step + stageTimer.stop(); + timingData.put(stage.name(), stageTimer.time()); + // restart the step timer + stageTimer.reset(); + stageTimer.start(); stage = next; } @@ -232,44 +209,29 @@ public void setStage(Stage stage) { switch (stage) { case INIT: this.stage = Stage.INIT; - overallTimer.reset(); - getReplicating().reset(); - getGetCheckpointInfo().reset(); - getFileDiff().reset(); - getGetFile().reset(); - getFinalizeReplication().reset(); break; case REPLICATING: validateAndSetStage(Stage.INIT, stage); // only start the overall timer once we've started replication overallTimer.start(); - getReplicating().start(); break; case GET_CHECKPOINT_INFO: validateAndSetStage(Stage.REPLICATING, stage); - getReplicating().stop(); - getGetCheckpointInfo().start(); break; case FILE_DIFF: validateAndSetStage(Stage.GET_CHECKPOINT_INFO, stage); - getGetCheckpointInfo().stop(); - getFileDiff().start(); break; case GET_FILES: validateAndSetStage(Stage.FILE_DIFF, stage); - getFileDiff().stop(); - getGetFile().start(); break; case FINALIZE_REPLICATION: validateAndSetStage(Stage.GET_FILES, stage); - getGetFile().stop(); - getFinalizeReplication().start(); break; case DONE: validateAndSetStage(Stage.FINALIZE_REPLICATION, stage); - getFinalizeReplication().stop(); // add the overall timing data overallTimer.stop(); + timingData.put("OVERALL", overallTimer.time()); break; case CANCELLED: if (this.stage == Stage.DONE) { @@ -277,6 +239,7 @@ public void setStage(Stage stage) { } this.stage = Stage.CANCELLED; overallTimer.stop(); + timingData.put("OVERALL", overallTimer.time()); break; default: throw new IllegalArgumentException("unknown SegmentReplicationState.Stage [" + stage + "]"); @@ -286,22 +249,16 @@ public void setStage(Stage stage) { @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.field(Fields.INDEX_NAME, shardRouting.index().getName()); builder.field(Fields.ID, shardRouting.shardId().id()); builder.field(Fields.STAGE, getStage()); - builder.field(Fields.REPLICATION_ID, getReplicationId()); builder.timeField(Fields.START_TIME_IN_MILLIS, Fields.START_TIME, getTimer().startTime()); if (getTimer().stopTime() > 0) { builder.timeField(Fields.STOP_TIME_IN_MILLIS, Fields.STOP_TIME, getTimer().stopTime()); } builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(getTimer().time())); - if (sourceNode != null) { - builder.startObject(SegmentReplicationState.Fields.SOURCE); - builder.field(Fields.ID, sourceNode.getId()); - builder.field(Fields.HOST, sourceNode.getHostName()); - builder.field(Fields.TRANSPORT_ADDRESS, sourceNode.getAddress().toString()); - builder.field(SegmentReplicationState.Fields.IP, sourceNode.getHostAddress()); - builder.field(SegmentReplicationState.Fields.NAME, sourceNode.getName()); - builder.endObject(); + if (sourceDescription != null) { + builder.field(Fields.SOURCE, getSourceDescription()); } if (targetNode != null) { @@ -313,33 +270,20 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.field(Fields.NAME, targetNode.getName()); builder.endObject(); } - builder.startObject(Fields.REPLICATING_STAGE); - replicating.toXContent(builder, params); - builder.endObject(); - - builder.startObject(Fields.GET_CHECKPOINT_INFO_STAGE); - getCheckpointInfo.toXContent(builder, params); - builder.endObject(); - - builder.startObject(Fields.FILE_DIFF_STAGE); - fileDiff.toXContent(builder, params); - builder.endObject(); - - builder.startObject(Fields.GET_FILES_STAGE); + builder.startObject(SegmentReplicationState.Fields.INDEX); index.toXContent(builder, params); - getFile.toXContent(builder, params); - builder.endObject(); - - builder.startObject(Fields.FINALIZE_REPLICATION_STAGE); - finalizeReplication.toXContent(builder, params); builder.endObject(); + builder.field(Fields.REPLICATING_STAGE, new TimeValue(timingData.get("REPLICATING"))); + builder.field(Fields.GET_CHECKPOINT_INFO_STAGE, new TimeValue(timingData.get("GET_CHECKPOINT_INFO"))); + builder.field(Fields.FILE_DIFF_STAGE, new TimeValue(timingData.get("FILE_DIFF"))); + builder.field(Fields.GET_FILES_STAGE, new TimeValue(timingData.get("GET_FILES"))); + builder.field(Fields.FINALIZE_REPLICATION_STAGE, new TimeValue(timingData.get("FINALIZE_REPLICATION"))); return builder; } static final class Fields { static final String ID = "id"; - static final String REPLICATION_ID = "replication_id"; static final String STAGE = "stage"; static final String START_TIME = "start_time"; static final String START_TIME_IN_MILLIS = "start_time_in_millis"; @@ -353,11 +297,10 @@ static final class Fields { static final String IP = "ip"; static final String NAME = "name"; static final String TARGET = "target"; - static final String INIT_STAGE = "init_stage"; - - static final String SIZE_OF_FILES_IN_BYTES = "size_of_files_in_bytes"; static final String INDEX = "index"; + + static final String INDEX_NAME = "index_name"; static final String TOTAL_GET_FILES_STAGE_TIME_IN_MILLIS = "total_get_files_stage_in_millis"; static final String REPLICATING_STAGE = "replicating_stage"; static final String GET_CHECKPOINT_INFO_STAGE = "get_checkpoint_info_stage"; @@ -365,163 +308,4 @@ static final class Fields { static final String GET_FILES_STAGE = "get_files_stage"; static final String FINALIZE_REPLICATION_STAGE = "finalize_replication_stage"; } - - /** - * Starts Replicating - * - * @opensearch.internal - */ - public static class Replicating extends ReplicationTimer implements ToXContentFragment, Writeable { - - public Replicating() {} - - public Replicating(StreamInput in) throws IOException { - super(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } - - public void reset() { - super.reset(); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time())); - return builder; - } - } - - /** - * Gets checkpoint info from source - * - * @opensearch.internal - */ - public static class GetCheckpointInfo extends ReplicationTimer implements ToXContentFragment, Writeable { - - public GetCheckpointInfo() {} - - public GetCheckpointInfo(StreamInput in) throws IOException { - super(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } - - public void reset() { - super.reset(); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time())); - return builder; - } - } - - /** - * Computes File Diff - * - * @opensearch.internal - */ - public static class FileDiff extends ReplicationTimer implements ToXContentFragment, Writeable { - - private volatile long sizeOfFiles; - - public FileDiff() {} - - public FileDiff(StreamInput in) throws IOException { - super(in); - sizeOfFiles = in.readVLong(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeVLong(sizeOfFiles); - } - - public void reset() { - super.reset(); - sizeOfFiles = 0; - } - - public long getSizeOfFiles() { - return sizeOfFiles; - } - - public void setSizeOfFiles(long sizeOfFiles) { - this.sizeOfFiles = sizeOfFiles; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(Fields.SIZE_OF_FILES_IN_BYTES, getSizeOfFiles()); - builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time())); - return builder; - } - } - - /** - * Gets necessary files from source - * - * @opensearch.internal - */ - public static class GetFile extends ReplicationTimer implements ToXContentFragment, Writeable { - - public GetFile() {} - - public GetFile(StreamInput in) throws IOException { - super(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } - - public void reset() { - super.reset(); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.humanReadableField(Fields.TOTAL_GET_FILES_STAGE_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time())); - return builder; - } - } - - /** - * Finalizes Replication - * - * @opensearch.internal - */ - public static class FinalizeReplication extends ReplicationTimer implements ToXContentFragment, Writeable { - - public FinalizeReplication() {} - - public FinalizeReplication(StreamInput in) throws IOException { - super(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } - - public void reset() { - super.reset(); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time())); - return builder; - } - } } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 16b166bc93a81..d14907a8a963b 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -63,22 +63,13 @@ public SegmentReplicationTarget( super("replication_target", indexShard, new ReplicationLuceneIndex(), listener); this.checkpoint = checkpoint; this.source = source; - if (indexShard.getSegmentReplicationState() == null) { - indexShard.setSegmentReplicationState( - new SegmentReplicationState( - indexShard.routingEntry(), - indexShard.recoveryState().getSourceNode(), - indexShard.recoveryState().getTargetNode() - ) - ); - } - this.state = indexShard.getSegmentReplicationState() - .onNewSegmentReplicationEvent( - stateIndex, - getId(), - indexShard.recoveryState().getSourceNode(), - indexShard.recoveryState().getTargetNode() - ); + this.state = new SegmentReplicationState( + indexShard.routingEntry(), + stateIndex, + getId(), + source.getSourceDescription(), + indexShard.recoveryState().getTargetNode() + ); this.multiFileWriter = new MultiFileWriter(indexShard.store(), stateIndex, getPrefix(), logger, this::ensureRefCount); } @@ -209,8 +200,6 @@ private void getFiles(CheckpointInfoResponse checkpointInfo, StepListener file.length()).sum(); - state.getFileDiff().setSizeOfFiles(sizeOfSegmentFiles); cancellableThreads.checkForCancel(); source.getSegmentFiles(getId(), checkpointInfo.getCheckpoint(), diff.missing, store, getFilesListener); } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index c95cb1a0a5377..56f4edcc3766a 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -58,6 +58,8 @@ public class SegmentReplicationTargetService implements IndexEventListener { private final ReplicationCollection onGoingReplications; + private final Map completedReplications = ConcurrentCollections.newConcurrentMap(); + private final SegmentReplicationSourceFactory sourceFactory; private final Map latestReceivedCheckpoint = ConcurrentCollections.newConcurrentMap(); @@ -152,6 +154,32 @@ public void shardRoutingChanged(IndexShard indexShard, @Nullable ShardRouting ol } } + /** + * returns SegmentReplicationState of on-going segment replication events. + */ + public SegmentReplicationState getOngoingEventSegmentReplicationState(ShardRouting shardRouting) { + SegmentReplicationTarget target = onGoingReplications.getOngoingReplicationTarget(shardRouting.shardId()); + return target != null ? target.state() : null; + } + + /** + * returns SegmentReplicationState of latest completed segment replication events. + */ + public SegmentReplicationState getlatestCompletedEventSegmentReplicationState(ShardRouting shardRouting) { + SegmentReplicationTarget target = completedReplications.get(shardRouting.shardId()); + return target != null ? target.state() : null; + } + + /** + * returns SegmentReplicationState of on-going if present or completed segment replication events. + */ + public SegmentReplicationState getSegmentReplicationState(ShardRouting shardRouting) { + if (getOngoingEventSegmentReplicationState(shardRouting) == null) { + return getlatestCompletedEventSegmentReplicationState(shardRouting); + } + return getOngoingEventSegmentReplicationState(shardRouting); + } + /** * Invoked when a new checkpoint is received from a primary shard. * It checks if a new checkpoint should be processed or not and starts replication if needed. @@ -179,6 +207,7 @@ public synchronized void onNewCheckpoint(final ReplicationCheckpoint receivedChe ongoingReplicationTarget.getCheckpoint().getPrimaryTerm() ); onGoingReplications.cancel(ongoingReplicationTarget.getId(), "Cancelling stuck target after new primary"); + completedReplications.put(replicaShard.shardId(), ongoingReplicationTarget); } else { logger.trace( () -> new ParameterizedMessage( @@ -307,10 +336,13 @@ private void start(final long replicationId) { if (replicationRef == null) { return; } + SegmentReplicationTarget target = onGoingReplications.getTarget(replicationId); replicationRef.get().startReplication(new ActionListener<>() { @Override public void onResponse(Void o) { onGoingReplications.markAsDone(replicationId); + completedReplications.put(target.shardId(), target); + } @Override @@ -323,6 +355,7 @@ public void onFailure(Exception e) { // but do not fail the shard. Cancellations initiated by this node from Index events will be removed with // onGoingReplications.cancel and not appear in the collection when this listener resolves. onGoingReplications.fail(replicationId, new ReplicationFailedException(indexShard, cause), false); + completedReplications.put(target.shardId(), target); } } else { onGoingReplications.fail(replicationId, new ReplicationFailedException("Segment Replication failed", e), true); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java index 78473a5ae6061..509741c23563d 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java @@ -9,8 +9,8 @@ package org.opensearch.rest.action.cat; import org.apache.lucene.util.CollectionUtil; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationRequest; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationResponse; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsRequest; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsResponse; import org.opensearch.action.support.IndicesOptions; import org.opensearch.client.node.NodeClient; import org.opensearch.common.Strings; @@ -27,8 +27,6 @@ import java.util.Comparator; import java.util.List; import java.util.Locale; -import java.util.Set; -import java.util.HashSet; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; @@ -62,20 +60,21 @@ protected void documentation(StringBuilder sb) { @Override public BaseRestHandler.RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) { - final SegmentReplicationRequest segmentReplicationRequest = new SegmentReplicationRequest( + final SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest( Strings.splitStringByCommaToArray(request.param("index")) ); - segmentReplicationRequest.timeout(request.param("timeout")); - segmentReplicationRequest.detailed(request.paramAsBoolean("detailed", false)); - segmentReplicationRequest.shards(Strings.splitStringByCommaToArray(request.param("shards"))); - segmentReplicationRequest.activeOnly(request.paramAsBoolean("active_only", false)); - segmentReplicationRequest.indicesOptions(IndicesOptions.fromRequest(request, segmentReplicationRequest.indicesOptions())); + segmentReplicationStatsRequest.timeout(request.param("timeout")); + segmentReplicationStatsRequest.detailed(request.paramAsBoolean("detailed", false)); + segmentReplicationStatsRequest.shards(Strings.splitStringByCommaToArray(request.param("shards"))); + segmentReplicationStatsRequest.activeOnly(request.paramAsBoolean("active_only", false)); + segmentReplicationStatsRequest.completedOnly(request.paramAsBoolean("completed_only", false)); + segmentReplicationStatsRequest.indicesOptions(IndicesOptions.fromRequest(request, segmentReplicationStatsRequest.indicesOptions())); return channel -> client.admin() .indices() - .segment_replication(segmentReplicationRequest, new RestResponseListener(channel) { + .segment_replication(segmentReplicationStatsRequest, new RestResponseListener(channel) { @Override - public RestResponse buildResponse(final SegmentReplicationResponse response) throws Exception { + public RestResponse buildResponse(final SegmentReplicationStatsResponse response) throws Exception { return RestTable.buildResponse(buildSegmentReplicationTable(request, response), channel); } }); @@ -99,8 +98,7 @@ protected Table getTableWithHeader(RestRequest request) { .addCell("stop_time_millis", "default:false;alias:stop_millis;desc:segment replication stop time in epoch milliseconds") .addCell("time", "alias:t,ti;desc:segment replication time") .addCell("stage", "alias:st;desc:segment replication stage") - .addCell("source_host", "alias:shost;desc:source host") - .addCell("source_node", "alias:snode;desc:source node name") + .addCell("source_description", "alias:sdesc;desc:source description") .addCell("target_host", "alias:thost;desc:target host") .addCell("target_node", "alias:tnode;desc:target node name") .addCell("files_fetched", "alias:ff;desc:files fetched") @@ -112,7 +110,6 @@ protected Table getTableWithHeader(RestRequest request) { .addCell("files_total", "alias:tf;desc:total number of files") .addCell("bytes", "alias:b;desc:number of bytes to fetch") .addCell("bytes_total", "alias:tb;desc:total number of bytes") - .addCell("replication_id", "desc: replication Id") .addCell("replicating_stage_time_taken", "alias:rstt;desc:time taken in replicating stage") .addCell("get_checkpoint_info_stage_time_taken", "alias:gcistt;desc:time taken in get checkpoint info stage") .addCell("file_diff_stage_time_taken", "alias:fdstt;desc:time taken in file diff stage") @@ -140,29 +137,10 @@ protected Table buildCustomResponseTable(String reason) { * @param response A SegmentReplication status response * @return A table containing index, shardId, node, target size, recovered size and percentage for each fetching replica */ - public Table buildSegmentReplicationTable(RestRequest request, SegmentReplicationResponse response) { - String[] indexNames = new String[0]; + public Table buildSegmentReplicationTable(RestRequest request, SegmentReplicationStatsResponse response) { boolean detailed = false; - String[] shards = new String[0]; - Set set = new HashSet<>(); if (request != null) { - indexNames = Strings.splitStringByCommaToArray(request.param("index")); detailed = Boolean.parseBoolean(request.param("detailed")); - shards = Strings.splitStringByCommaToArray(request.param("shards")); - } - if (request != null && indexNames.length > 0) { - for (String index : indexNames) { - if (!response.shardSegmentReplicationStates().containsKey(index)) { - String reason = "error:" + "{Segment Replication is not enabled on index: " + index + " }"; - return buildCustomResponseTable(reason); - } - } - - } - if (shards.length > 0) { - for (String shard : shards) { - set.add(shard); - } } Table t = getTableWithHeader(request); @@ -190,10 +168,6 @@ public int compare(SegmentReplicationState o1, SegmentReplicationState o2) { }); for (SegmentReplicationState state : shardSegmentReplicationStates) { - int shardId = state.getShardRouting().shardId().id(); - if (shards.length > 0 && !set.contains(Integer.toString(shardId))) { - continue; - } t.startRow(); t.addCell(index); t.addCell(state.getShardRouting().shardId().id()); @@ -203,8 +177,7 @@ public int compare(SegmentReplicationState o1, SegmentReplicationState o2) { t.addCell(state.getTimer().stopTime()); t.addCell(new TimeValue(state.getTimer().time())); t.addCell(state.getStage().toString().toLowerCase(Locale.ROOT)); - t.addCell(state.getSourceNode().getHostName()); - t.addCell(state.getSourceNode().getName()); + t.addCell(state.getSourceDescription()); t.addCell(state.getTargetNode().getHostName()); t.addCell(state.getTargetNode().getName()); t.addCell(state.getIndex().recoveredFileCount()); @@ -216,12 +189,11 @@ public int compare(SegmentReplicationState o1, SegmentReplicationState o2) { t.addCell(state.getIndex().totalFileCount()); t.addCell(state.getIndex().totalRecoverBytes()); t.addCell(state.getIndex().totalBytes()); - t.addCell(state.getReplicationId()); - t.addCell(new TimeValue(state.getReplicating().time())); - t.addCell(new TimeValue(state.getGetCheckpointInfo().time())); - t.addCell(new TimeValue(state.getFileDiff().time())); - t.addCell(new TimeValue(state.getGetFile().time())); - t.addCell(new TimeValue(state.getFinalizeReplication().time())); + t.addCell(new TimeValue(state.getReplicatingStageTime())); + t.addCell(new TimeValue(state.getGetCheckpointInfoStageTime())); + t.addCell(new TimeValue(state.getFileDiffStageTime())); + t.addCell(new TimeValue(state.getGetFileStageTime())); + t.addCell(new TimeValue(state.getFinalizeReplicationStageTime())); } t.endRow(); } diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java index acda16d1d0f02..75142b88212f3 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java @@ -8,7 +8,7 @@ package org.opensearch.rest.action.cat; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationResponse; +import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsResponse; import org.opensearch.action.support.DefaultShardOperationFailedException; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; @@ -58,9 +58,7 @@ public void testSegment_ReplicationActionAction() { when(timer.stopTime()).thenReturn(startTime + time); when(state.getTimer()).thenReturn(timer); when(state.getStage()).thenReturn(randomFrom(SegmentReplicationState.Stage.values())); - final DiscoveryNode sourceNode = mock(DiscoveryNode.class); - when(sourceNode.getHostName()).thenReturn(randomAlphaOfLength(8)); - when(state.getSourceNode()).thenReturn(sourceNode); + when(state.getSourceDescription()).thenReturn("Source"); final DiscoveryNode targetNode = mock(DiscoveryNode.class); when(targetNode.getHostName()).thenReturn(randomAlphaOfLength(8)); when(state.getTargetNode()).thenReturn(targetNode); @@ -78,7 +76,7 @@ public void testSegment_ReplicationActionAction() { shardSegmentReplicationStates.put("index", shuffle); final List shardFailures = new ArrayList<>(); - final SegmentReplicationResponse response = new SegmentReplicationResponse( + final SegmentReplicationStatsResponse response = new SegmentReplicationStatsResponse( totalShards, successfulShards, failedShards, @@ -127,8 +125,8 @@ public void testSegment_ReplicationActionAction() { state.getTimer().stopTime(), new TimeValue(state.getTimer().time()), state.getStage().name().toLowerCase(Locale.ROOT), - state.getSourceNode().getHostName(), - state.getSourceNode().getName(), + // state.getSourceDescription().getHostName(), + // state.getSourceDescription().getName(), state.getTargetNode().getHostName(), state.getTargetNode().getName(), state.getIndex().recoveredFileCount(), From b8f7b718e3f32de44254ee7360c99acc5ee5633c Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Fri, 3 Feb 2023 20:36:52 +0000 Subject: [PATCH 19/25] Fix gradle check failures Signed-off-by: Rishikesh1159 --- .../rest-api-spec/api/cat.segment_replication.json | 2 +- .../TransportSegmentReplicationStatsAction.java | 5 +++-- .../main/java/org/opensearch/index/shard/IndexShard.java | 4 ---- .../indices/replication/SegmentReplicationState.java | 8 ++++---- .../replication/SegmentReplicationTargetService.java | 8 ++++---- .../rest/action/cat/RestCatSegmentReplicationAction.java | 2 +- 6 files changed, 13 insertions(+), 16 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json index 581e7a3a235a7..3c2195c7765a0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json @@ -37,7 +37,7 @@ "description":"If `true`, the response only includes ongoing segment replication events", "default":false }, - "active_only":{ + "completed_only":{ "type":"boolean", "description":"If `true`, the response only includes latest completed segment replication events", "default":false diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationStatsAction.java index cdecafd1691a2..c470f7243ad91 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationStatsAction.java @@ -8,7 +8,7 @@ package org.opensearch.action.admin.indices.segment_replication; -import org.opensearch.ResourceNotFoundException; +import org.opensearch.OpenSearchStatusException; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; @@ -26,6 +26,7 @@ import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.SegmentReplicationTargetService; +import org.opensearch.rest.RestStatus; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -93,7 +94,7 @@ protected SegmentReplicationStatsResponse newResponse( if (singleIndexWithSegmentReplicationDisabled != null) { String index = singleIndexWithSegmentReplicationDisabled; singleIndexWithSegmentReplicationDisabled = null; - throw new ResourceNotFoundException("Segment Replication is not enabled on Index: " + index); + throw new OpenSearchStatusException("Segment Replication is not enabled on Index: " + index, RestStatus.BAD_REQUEST); } String[] shards = request.shards(); Set set = new HashSet<>(); diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 233256b9ca538..4e345b05ab267 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -173,7 +173,6 @@ import org.opensearch.indices.recovery.RecoveryListener; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.recovery.RecoveryTarget; -import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.repositories.RepositoriesService; @@ -282,9 +281,6 @@ Runnable getGlobalCheckpointSyncer() { @Nullable private volatile RecoveryState recoveryState; - @Nullable - private volatile SegmentReplicationState segmentReplicationState; - private final RecoveryStats recoveryStats = new RecoveryStats(); private final MeanMetric refreshMetric = new MeanMetric(); private final MeanMetric externalRefreshMetric = new MeanMetric(); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index fd5cd02025085..282b3fb85a49b 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -85,10 +85,6 @@ public static Stage fromId(byte id) { private String sourceDescription; private DiscoveryNode targetNode; - public ReplicationTimer getOverallTimer() { - return overallTimer; - } - public ShardRouting getShardRouting() { return shardRouting; } @@ -124,6 +120,10 @@ public long getReplicatingStageTime() { return timingData.get(Stage.REPLICATING.toString()); } + public Map getTimingData() { + return timingData; + } + public long getGetCheckpointInfoStageTime() { return timingData.get(Stage.GET_CHECKPOINT_INFO.toString()); } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index fc67fbc16c222..7317d4db1d2b5 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -228,7 +228,7 @@ public void onReplicationDone(SegmentReplicationState state) { "[shardId {}] [replication id {}] Replication complete, timing data: {}", replicaShard.shardId().getId(), state.getReplicationId(), - state.getOverallTimer() + state.getTimingData() ) ); // if we received a checkpoint during the copy event that is ahead of this @@ -255,7 +255,7 @@ public void onReplicationFailure( "[shardId {}] [replication id {}] Replication failed, timing data: {}", replicaShard.shardId().getId(), state.getReplicationId(), - state.getOverallTimer() + state.getTimingData() ) ); if (sendShardFailure == true) { @@ -396,7 +396,7 @@ public void onReplicationDone(SegmentReplicationState state) { "[shardId {}] [replication id {}] Replication complete, timing data: {}", indexShard.shardId().getId(), state.getReplicationId(), - state.getOverallTimer() + state.getTimingData() ) ); try { @@ -421,7 +421,7 @@ public void onReplicationFailure( "[shardId {}] [replication id {}] Replication failed, timing data: {}", indexShard.shardId().getId(), state.getReplicationId(), - state.getOverallTimer() + state.getTimingData() ) ); if (sendShardFailure == true) { diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java index 509741c23563d..2fcc7db4e52ac 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java @@ -91,7 +91,7 @@ protected Table getTableWithHeader(RestRequest request) { Table t = new Table(); t.startHeaders() .addCell("index", "alias:i,idx;desc:index name") - .addCell("shardId", "desc: shard Id") + .addCell("shardId", "alias:s;desc: shard Id") .addCell("start_time", "default:false;alias:start;desc:segment replication start time") .addCell("start_time_millis", "default:false;alias:start_millis;desc:segment replication start time in epoch milliseconds") .addCell("stop_time", "default:false;alias:stop;desc:segment replication stop time") From c1ef1d4d286ab67a33c7982948773359cd9fc1bc Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Fri, 3 Feb 2023 21:18:55 +0000 Subject: [PATCH 20/25] fix failing testSegment_ReplicationActionAction() Signed-off-by: Rishikesh1159 --- .../segment_replication/SegmentReplicationStatsAction.java | 2 +- .../segment_replication/SegmentReplicationStatsRequest.java | 6 +++--- .../SegmentReplicationStatsRequestBuilder.java | 2 +- .../SegmentReplicationStatsResponse.java | 4 ++-- .../TransportSegmentReplicationStatsAction.java | 2 +- .../rest/action/cat/RestSegment_ReplicationActionTests.java | 6 ++---- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsAction.java index 3810ea23ca30e..f7caab0223559 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsAction.java @@ -11,7 +11,7 @@ import org.opensearch.action.ActionType; /** - * Segment Replication information action + * Segment Replication stats information action * * @opensearch.internal */ diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequest.java index ce66e27e766de..f6d0f1f882399 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequest.java @@ -17,7 +17,7 @@ import java.io.IOException; /** - * Request for Segment Replication information + * Request for Segment Replication stats information * * @opensearch.internal */ @@ -30,7 +30,7 @@ public class SegmentReplicationStatsRequest extends BroadcastRequest Date: Mon, 6 Feb 2023 17:49:31 +0000 Subject: [PATCH 21/25] Exclude empty segment replication events in API response. Signed-off-by: Rishikesh1159 --- .../replication/SegmentReplicationTargetService.java | 5 ++++- .../action/cat/RestCatSegmentReplicationAction.java | 11 +---------- ...java => RestCatSegmentReplicationActionTests.java} | 2 +- 3 files changed, 6 insertions(+), 12 deletions(-) rename server/src/test/java/org/opensearch/rest/action/cat/{RestSegment_ReplicationActionTests.java => RestCatSegmentReplicationActionTests.java} (98%) diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index 7317d4db1d2b5..673e4a21e756e 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -341,7 +341,10 @@ private void start(final long replicationId) { @Override public void onResponse(Void o) { onGoingReplications.markAsDone(replicationId); - completedReplications.put(target.shardId(), target); + if(target.state().getIndex().recoveredFileCount() != 0 + && target.state().getIndex().recoveredBytes() != 0) { + completedReplications.put(target.shardId(), target); + } } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java index 2fcc7db4e52ac..8202b680745dd 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java @@ -120,22 +120,13 @@ protected Table getTableWithHeader(RestRequest request) { return t; } - protected Table buildCustomResponseTable(String reason) { - Table t = new Table(); - t.startHeaders().addCell("Reason", "desc: Reason for API response").endHeaders(); - t.startRow(); - t.addCell(reason); - t.endRow(); - return t; - } - /** * buildSegmentReplicationTable will build a table of SegmentReplication information suitable * for displaying at the command line. * * @param request A Rest request * @param response A SegmentReplication status response - * @return A table containing index, shardId, node, target size, recovered size and percentage for each fetching replica + * @return A table containing index, shardId, node, target size, fetched size and percentage for each fetching replica */ public Table buildSegmentReplicationTable(RestRequest request, SegmentReplicationStatsResponse response) { boolean detailed = false; diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java similarity index 98% rename from server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java rename to server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java index 347cf807630b7..17ea33c86e5bb 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestSegment_ReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java @@ -35,7 +35,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class RestSegment_ReplicationActionTests extends OpenSearchTestCase { +public class RestCatSegmentReplicationActionTests extends OpenSearchTestCase { public void testSegment_ReplicationActionAction() { final RestCatSegmentReplicationAction action = new RestCatSegmentReplicationAction(); final int totalShards = randomIntBetween(1, 32); From 537964f86fedc2e15ac5651619fb883e062e8519 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Mon, 6 Feb 2023 17:50:53 +0000 Subject: [PATCH 22/25] Apply spotless. Signed-off-by: Rishikesh1159 --- .../indices/replication/SegmentReplicationTargetService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index 673e4a21e756e..d708c85afea7a 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -341,8 +341,7 @@ private void start(final long replicationId) { @Override public void onResponse(Void o) { onGoingReplications.markAsDone(replicationId); - if(target.state().getIndex().recoveredFileCount() != 0 - && target.state().getIndex().recoveredBytes() != 0) { + if (target.state().getIndex().recoveredFileCount() != 0 && target.state().getIndex().recoveredBytes() != 0) { completedReplications.put(target.shardId(), target); } From f120a92617ad1926b8d811b2b16e7281ffa1cb13 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Tue, 7 Feb 2023 16:49:02 +0000 Subject: [PATCH 23/25] Address PR comments and add Integ Tests. Signed-off-by: Rishikesh1159 --- .../opensearch/action/IndicesRequestIT.java | 8 +- .../action/admin/ClientTimeoutIT.java | 6 +- .../replication/SegmentReplicationApiIT.java | 149 ++++++++++++++++++ .../org/opensearch/action/ActionModule.java | 4 +- .../SegmentReplicationStatsAction.java | 2 +- .../SegmentReplicationStatsRequest.java | 2 +- ...SegmentReplicationStatsRequestBuilder.java | 12 +- .../SegmentReplicationStatsResponse.java | 2 +- ...ransportSegmentReplicationStatsAction.java | 10 +- .../package-info.java | 2 +- .../opensearch/client/IndicesAdminClient.java | 10 +- .../client/support/AbstractClient.java | 12 +- .../PrimaryShardReplicationSource.java | 5 +- .../replication/SegmentReplicationSource.java | 4 +- .../replication/SegmentReplicationState.java | 35 ++-- .../replication/SegmentReplicationTarget.java | 2 +- .../SegmentReplicationTargetService.java | 24 +-- .../cat/RestCatSegmentReplicationAction.java | 16 +- .../SegmentReplicationIndexShardTests.java | 20 +++ .../SegmentReplicationTargetServiceTests.java | 5 + .../SegmentReplicationTargetTests.java | 35 ++++ .../RestCatSegmentReplicationActionTests.java | 4 +- .../index/shard/IndexShardTestCase.java | 5 + 23 files changed, 299 insertions(+), 75 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationApiIT.java rename server/src/main/java/org/opensearch/action/admin/indices/{segment_replication => replication}/SegmentReplicationStatsAction.java (91%) rename server/src/main/java/org/opensearch/action/admin/indices/{segment_replication => replication}/SegmentReplicationStatsRequest.java (98%) rename server/src/main/java/org/opensearch/action/admin/indices/{segment_replication => replication}/SegmentReplicationStatsRequestBuilder.java (75%) rename server/src/main/java/org/opensearch/action/admin/indices/{segment_replication => replication}/SegmentReplicationStatsResponse.java (98%) rename server/src/main/java/org/opensearch/action/admin/indices/{segment_replication => replication}/TransportSegmentReplicationStatsAction.java (96%) rename server/src/main/java/org/opensearch/action/admin/indices/{segment_replication => replication}/package-info.java (79%) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java index 9b2423f7029ff..d956b0e7da1e1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java @@ -56,8 +56,8 @@ import org.opensearch.action.admin.indices.recovery.RecoveryRequest; import org.opensearch.action.admin.indices.refresh.RefreshRequest; import org.opensearch.action.admin.indices.refresh.TransportShardRefreshAction; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsAction; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsRequest; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsAction; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsRequest; import org.opensearch.action.admin.indices.segments.IndicesSegmentsAction; import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsAction; @@ -470,12 +470,12 @@ public void testRecovery() { assertSameIndices(recoveryRequest, recoveryAction); } - public void testSegment_Replication() { + public void testSegmentReplication() { String segmentReplicationAction = SegmentReplicationStatsAction.NAME + "[n]"; interceptTransportActions(segmentReplicationAction); SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(randomIndicesOrAliases()); - internalCluster().coordOnlyNodeClient().admin().indices().segment_replication(segmentReplicationStatsRequest).actionGet(); + internalCluster().coordOnlyNodeClient().admin().indices().segmentReplication(segmentReplicationStatsRequest).actionGet(); clearInterceptedActions(); assertSameIndices(segmentReplicationStatsRequest, segmentReplicationAction); diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java index 98d1524de8115..c40bc5d914ff6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java @@ -17,8 +17,8 @@ import org.opensearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.opensearch.action.admin.indices.recovery.RecoveryAction; import org.opensearch.action.admin.indices.recovery.RecoveryResponse; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsAction; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsResponse; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsAction; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsResponse; import org.opensearch.action.admin.indices.stats.IndicesStatsAction; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.cluster.metadata.IndexMetadata; @@ -152,7 +152,7 @@ public void testRecoveriesWithTimeout() { assertThat(recoveryResponse.getShardFailures()[0].reason(), containsString("ReceiveTimeoutTransportException")); } - public void testSegment_ReplicationWithTimeout() { + public void testSegmentReplicationWithTimeout() { internalCluster().startClusterManagerOnlyNode( Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REPLICATION_TYPE, "true").build() ); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationApiIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationApiIT.java new file mode 100644 index 0000000000000..86ceae9285cfe --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationApiIT.java @@ -0,0 +1,149 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.indices.replication; + +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsResponse; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.transport.MockTransportService; +import org.opensearch.transport.TransportService; + +import java.util.concurrent.CountDownLatch; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class SegmentReplicationApiIT extends SegmentReplicationBaseIT { + + public void testSegmentReplicationApiResponse() { + logger.info("--> starting [Primary Node] ..."); + final String primaryNode = internalCluster().startNode(); + createIndex(INDEX_NAME); + ensureYellowAndNoInitializingShards(INDEX_NAME); + logger.info("--> start empty node to add replica shard"); + final String replicaNode = internalCluster().startNode(); + ensureGreen(INDEX_NAME); + logger.info("--> index 10 docs"); + for (int i = 0; i < 10; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); + } + refresh(INDEX_NAME); + logger.info("--> verifying count"); + assertEquals(client().prepareSearch(INDEX_NAME).setSize(0).execute().actionGet().getHits().getTotalHits().value, 10L); + + SegmentReplicationStatsResponse response = client().admin().indices().prepareSegmentReplication(INDEX_NAME).execute().actionGet(); + // Verify API Response + assertThat(response.shardSegmentReplicationStates().size(), equalTo(SHARD_COUNT)); + assertThat(response.shardSegmentReplicationStates().get(INDEX_NAME).get(0).getStage(), equalTo(SegmentReplicationState.Stage.DONE)); + assertThat(response.shardSegmentReplicationStates().get(INDEX_NAME).get(0).getIndex().recoveredFileCount(), greaterThan(0)); + } + + public void testSegmentReplicationApiResponseForActiveAndCompletedOnly() { + logger.info("--> starting [Primary Node] ..."); + final String primaryNode = internalCluster().startNode(); + createIndex(INDEX_NAME); + ensureYellowAndNoInitializingShards(INDEX_NAME); + logger.info("--> starting [Replica Node] ..."); + final String replicaNode = internalCluster().startNode(); + ensureGreen(INDEX_NAME); + logger.info("--> index 10 docs"); + for (int i = 0; i < 10; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); + } + refresh(INDEX_NAME); + for (int i = 10; i < 20; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); + } + final CountDownLatch waitForReplication = new CountDownLatch(1); + + final CountDownLatch waitForAssertions = new CountDownLatch(1); + // Mock transport service to add behaviour of waiting in GET_SEGMENT_FILES Stage of a segment replication event. + MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance( + TransportService.class, + replicaNode + )); + mockTransportService.addSendBehavior( + internalCluster().getInstance(TransportService.class, primaryNode), + (connection, requestId, action, request, options) -> { + if (action.equals(SegmentReplicationSourceService.Actions.GET_SEGMENT_FILES)) { + waitForReplication.countDown(); + try { + waitForAssertions.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + connection.sendRequest(requestId, action, request, options); + } + ); + refresh(INDEX_NAME); + try { + waitForReplication.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + logger.info("--> verifying active_only by checking if current stage is GET_FILES STAGE"); + SegmentReplicationStatsResponse activeOnlyResponse = client().admin() + .indices() + .prepareSegmentReplication(INDEX_NAME) + .setActiveOnly(true) + .execute() + .actionGet(); + assertThat( + activeOnlyResponse.shardSegmentReplicationStates().get(INDEX_NAME).get(0).getStage(), + equalTo(SegmentReplicationState.Stage.GET_FILES) + ); + + logger.info("--> verifying completed_only by checking if current stage is DONE"); + SegmentReplicationStatsResponse completedOnlyResponse = client().admin() + .indices() + .prepareSegmentReplication(INDEX_NAME) + .setCompletedOnly(true) + .execute() + .actionGet(); + assertThat(completedOnlyResponse.shardSegmentReplicationStates().size(), equalTo(SHARD_COUNT)); + assertThat( + completedOnlyResponse.shardSegmentReplicationStates().get(INDEX_NAME).get(0).getStage(), + equalTo(SegmentReplicationState.Stage.DONE) + ); + waitForAssertions.countDown(); + } + + public void testSegmentReplicationApiResponseOnDocumentReplicationIndex() { + logger.info("--> starting [Primary Node] ..."); + final String primaryNode = internalCluster().startNode(); + prepareCreate( + INDEX_NAME, + Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) + + ).get(); + ensureYellowAndNoInitializingShards(INDEX_NAME); + logger.info("--> start empty node to add replica shard"); + final String replicaNode = internalCluster().startNode(); + ensureGreen(INDEX_NAME); + logger.info("--> index 10 docs"); + for (int i = 0; i < 10; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); + } + refresh(INDEX_NAME); + OpenSearchStatusException exception = assertThrows( + OpenSearchStatusException.class, + () -> client().admin().indices().prepareSegmentReplication(INDEX_NAME).execute().actionGet() + ); + // Verify exception message + String expectedMessage = "Segment Replication is not enabled on Index: test-idx-1"; + assertEquals(expectedMessage, exception.getMessage()); + + } + +} diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 832196285311e..2cb11a0586c98 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -176,8 +176,8 @@ import org.opensearch.action.admin.indices.resolve.ResolveIndexAction; import org.opensearch.action.admin.indices.rollover.RolloverAction; import org.opensearch.action.admin.indices.rollover.TransportRolloverAction; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsAction; -import org.opensearch.action.admin.indices.segment_replication.TransportSegmentReplicationStatsAction; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsAction; +import org.opensearch.action.admin.indices.replication.TransportSegmentReplicationStatsAction; import org.opensearch.action.admin.indices.segments.IndicesSegmentsAction; import org.opensearch.action.admin.indices.segments.PitSegmentsAction; import org.opensearch.action.admin.indices.segments.TransportIndicesSegmentsAction; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsAction.java similarity index 91% rename from server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsAction.java rename to server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsAction.java index f7caab0223559..9d1de20a8ff37 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsAction.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.action.admin.indices.segment_replication; +package org.opensearch.action.admin.indices.replication; import org.opensearch.action.ActionType; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java similarity index 98% rename from server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequest.java rename to server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java index f6d0f1f882399..fdd29990fb446 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.action.admin.indices.segment_replication; +package org.opensearch.action.admin.indices.replication; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.broadcast.BroadcastRequest; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequestBuilder.java similarity index 75% rename from server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequestBuilder.java rename to server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequestBuilder.java index b20bc53a6a430..abd48cfe0ba4f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequestBuilder.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.action.admin.indices.segment_replication; +package org.opensearch.action.admin.indices.replication; import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder; import org.opensearch.client.OpenSearchClient; @@ -35,4 +35,14 @@ public SegmentReplicationStatsRequestBuilder setActiveOnly(boolean activeOnly) { return this; } + public SegmentReplicationStatsRequestBuilder setCompletedOnly(boolean completedOnly) { + request.completedOnly(completedOnly); + return this; + } + + public SegmentReplicationStatsRequestBuilder shards(String... indices) { + request.shards(indices); + return this; + } + } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java similarity index 98% rename from server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsResponse.java rename to server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java index 09c62fd61ee19..759e35b37bbc7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/SegmentReplicationStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.action.admin.indices.segment_replication; +package org.opensearch.action.admin.indices.replication; import org.opensearch.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java similarity index 96% rename from server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationStatsAction.java rename to server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java index f73b8ebd9ae20..d32a9df6d0d77 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/TransportSegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.action.admin.indices.segment_replication; +package org.opensearch.action.admin.indices.replication; import org.opensearch.OpenSearchStatusException; import org.opensearch.action.support.ActionFilters; @@ -23,6 +23,7 @@ import org.opensearch.common.io.stream.StreamInput; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.shard.ShardId; import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.SegmentReplicationTargetService; @@ -132,6 +133,7 @@ protected SegmentReplicationStatsRequest readRequestFrom(StreamInput in) throws protected SegmentReplicationState shardOperation(SegmentReplicationStatsRequest request, ShardRouting shardRouting) { IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); + ShardId shardId = shardRouting.shardId(); // check if API call is made on single index with segment replication disabled. if (request.indices().length == 1 && indexShard.indexSettings().isSegRepEnabled() == false) { @@ -144,14 +146,14 @@ protected SegmentReplicationState shardOperation(SegmentReplicationStatsRequest // return information about only on-going segment replication events. if (request.activeOnly()) { - return targetService.getOngoingEventSegmentReplicationState(shardRouting); + return targetService.getOngoingEventSegmentReplicationState(shardId); } // return information about only latest completed segment replication events. if (request.completedOnly()) { - return targetService.getlatestCompletedEventSegmentReplicationState(shardRouting); + return targetService.getlatestCompletedEventSegmentReplicationState(shardId); } - return targetService.getSegmentReplicationState(shardRouting); + return targetService.getSegmentReplicationState(shardId); } @Override diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/package-info.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/package-info.java similarity index 79% rename from server/src/main/java/org/opensearch/action/admin/indices/segment_replication/package-info.java rename to server/src/main/java/org/opensearch/action/admin/indices/replication/package-info.java index ae37f03a63a28..f801f2f288797 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segment_replication/package-info.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/package-info.java @@ -7,4 +7,4 @@ */ /** Segment Replication transport handlers. */ -package org.opensearch.action.admin.indices.segment_replication; +package org.opensearch.action.admin.indices.replication; diff --git a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java index f63a483a6b356..fd32695262100 100644 --- a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java +++ b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java @@ -91,9 +91,9 @@ import org.opensearch.action.admin.indices.rollover.RolloverRequest; import org.opensearch.action.admin.indices.rollover.RolloverRequestBuilder; import org.opensearch.action.admin.indices.rollover.RolloverResponse; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsRequest; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsRequestBuilder; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsResponse; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsRequest; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsRequestBuilder; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsResponse; import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse; import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequestBuilder; @@ -191,12 +191,12 @@ public interface IndicesAdminClient extends OpenSearchClient { /** *Indices segment replication */ - ActionFuture segment_replication(SegmentReplicationStatsRequest request); + ActionFuture segmentReplication(SegmentReplicationStatsRequest request); /** *Indices segment replication */ - void segment_replication(SegmentReplicationStatsRequest request, ActionListener listener); + void segmentReplication(SegmentReplicationStatsRequest request, ActionListener listener); /** * Indices recoveries diff --git a/server/src/main/java/org/opensearch/client/support/AbstractClient.java b/server/src/main/java/org/opensearch/client/support/AbstractClient.java index c4df2f2e63e75..e4e2f03464321 100644 --- a/server/src/main/java/org/opensearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/opensearch/client/support/AbstractClient.java @@ -260,10 +260,10 @@ import org.opensearch.action.admin.indices.rollover.RolloverRequest; import org.opensearch.action.admin.indices.rollover.RolloverRequestBuilder; import org.opensearch.action.admin.indices.rollover.RolloverResponse; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsAction; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsRequest; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsRequestBuilder; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsResponse; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsAction; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsRequest; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsRequestBuilder; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsResponse; import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse; import org.opensearch.action.admin.indices.segments.IndicesSegmentsAction; import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; @@ -1780,12 +1780,12 @@ public RecoveryRequestBuilder prepareRecoveries(String... indices) { } @Override - public ActionFuture segment_replication(final SegmentReplicationStatsRequest request) { + public ActionFuture segmentReplication(final SegmentReplicationStatsRequest request) { return execute(SegmentReplicationStatsAction.INSTANCE, request); } @Override - public void segment_replication( + public void segmentReplication( final SegmentReplicationStatsRequest request, final ActionListener listener ) { diff --git a/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java index 3b65a75fad233..941f749ac17d1 100644 --- a/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java +++ b/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java @@ -108,9 +108,8 @@ public void getSegmentFiles( } @Override - public String getSourceDescription() { - String description = "Host:" + this.sourceNode.getHostName() + ", Node:" + this.sourceNode.getName(); - return description; + public String getDescription() { + return sourceNode.getName(); } @Override diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java index 1270dd5b6fb76..2fa74819fe4de 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java @@ -52,9 +52,7 @@ void getSegmentFiles( /** * Get the source description */ - default String getSourceDescription() { - return null; - } + String getDescription(); /** * Cancel any ongoing requests, should resolve any ongoing listeners with onFailure with a {@link ExecutionCancelledException}. diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index 282b3fb85a49b..def4e65042ca4 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -116,28 +116,28 @@ public DiscoveryNode getTargetNode() { return targetNode; } - public long getReplicatingStageTime() { - return timingData.get(Stage.REPLICATING.toString()); - } - public Map getTimingData() { return timingData; } - public long getGetCheckpointInfoStageTime() { - return timingData.get(Stage.GET_CHECKPOINT_INFO.toString()); + public TimeValue getReplicatingStageTime() { + return new TimeValue(timingData.get(Stage.REPLICATING.toString())); + } + + public TimeValue getGetCheckpointInfoStageTime() { + return new TimeValue(timingData.get(Stage.GET_CHECKPOINT_INFO.toString())); } - public long getFileDiffStageTime() { - return timingData.get(Stage.FILE_DIFF.toString()); + public TimeValue getFileDiffStageTime() { + return new TimeValue(timingData.get(Stage.FILE_DIFF.toString())); } - public long getGetFileStageTime() { - return timingData.get(Stage.GET_FILES.toString()); + public TimeValue getGetFileStageTime() { + return new TimeValue(timingData.get(Stage.GET_FILES.toString())); } - public long getFinalizeReplicationStageTime() { - return timingData.get(Stage.FINALIZE_REPLICATION.toString()); + public TimeValue getFinalizeReplicationStageTime() { + return new TimeValue(timingData.get(Stage.FINALIZE_REPLICATION.toString())); } public SegmentReplicationState( @@ -273,11 +273,11 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.startObject(SegmentReplicationState.Fields.INDEX); index.toXContent(builder, params); builder.endObject(); - builder.field(Fields.REPLICATING_STAGE, new TimeValue(timingData.get("REPLICATING"))); - builder.field(Fields.GET_CHECKPOINT_INFO_STAGE, new TimeValue(timingData.get("GET_CHECKPOINT_INFO"))); - builder.field(Fields.FILE_DIFF_STAGE, new TimeValue(timingData.get("FILE_DIFF"))); - builder.field(Fields.GET_FILES_STAGE, new TimeValue(timingData.get("GET_FILES"))); - builder.field(Fields.FINALIZE_REPLICATION_STAGE, new TimeValue(timingData.get("FINALIZE_REPLICATION"))); + builder.field(Fields.REPLICATING_STAGE, getReplicatingStageTime()); + builder.field(Fields.GET_CHECKPOINT_INFO_STAGE, getGetCheckpointInfoStageTime()); + builder.field(Fields.FILE_DIFF_STAGE, getFileDiffStageTime()); + builder.field(Fields.GET_FILES_STAGE, getGetFileStageTime()); + builder.field(Fields.FINALIZE_REPLICATION_STAGE, getFinalizeReplicationStageTime()); return builder; } @@ -301,7 +301,6 @@ static final class Fields { static final String INDEX = "index"; static final String INDEX_NAME = "index_name"; - static final String TOTAL_GET_FILES_STAGE_TIME_IN_MILLIS = "total_get_files_stage_in_millis"; static final String REPLICATING_STAGE = "replicating_stage"; static final String GET_CHECKPOINT_INFO_STAGE = "get_checkpoint_info_stage"; static final String FILE_DIFF_STAGE = "file_diff_stage"; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index d14907a8a963b..6f46fe8398388 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -67,7 +67,7 @@ public SegmentReplicationTarget( indexShard.routingEntry(), stateIndex, getId(), - source.getSourceDescription(), + source.getDescription(), indexShard.recoveryState().getTargetNode() ); this.multiFileWriter = new MultiFileWriter(indexShard.store(), stateIndex, getPrefix(), logger, this::ensureRefCount); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index d708c85afea7a..9cb10a2c9699a 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -41,6 +41,7 @@ import java.io.IOException; import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; @@ -157,27 +158,28 @@ public void shardRoutingChanged(IndexShard indexShard, @Nullable ShardRouting ol /** * returns SegmentReplicationState of on-going segment replication events. */ - public SegmentReplicationState getOngoingEventSegmentReplicationState(ShardRouting shardRouting) { - SegmentReplicationTarget target = onGoingReplications.getOngoingReplicationTarget(shardRouting.shardId()); - return target != null ? target.state() : null; + @Nullable + public SegmentReplicationState getOngoingEventSegmentReplicationState(ShardId shardId) { + return Optional.ofNullable(onGoingReplications.getOngoingReplicationTarget(shardId)) + .map(SegmentReplicationTarget::state) + .orElse(null); } /** * returns SegmentReplicationState of latest completed segment replication events. */ - public SegmentReplicationState getlatestCompletedEventSegmentReplicationState(ShardRouting shardRouting) { - SegmentReplicationTarget target = completedReplications.get(shardRouting.shardId()); - return target != null ? target.state() : null; + @Nullable + public SegmentReplicationState getlatestCompletedEventSegmentReplicationState(ShardId shardId) { + return Optional.ofNullable(completedReplications.get(shardId)).map(SegmentReplicationTarget::state).orElse(null); } /** * returns SegmentReplicationState of on-going if present or completed segment replication events. */ - public SegmentReplicationState getSegmentReplicationState(ShardRouting shardRouting) { - if (getOngoingEventSegmentReplicationState(shardRouting) == null) { - return getlatestCompletedEventSegmentReplicationState(shardRouting); - } - return getOngoingEventSegmentReplicationState(shardRouting); + @Nullable + public SegmentReplicationState getSegmentReplicationState(ShardId shardId) { + return Optional.ofNullable(getOngoingEventSegmentReplicationState(shardId)) + .orElseGet(() -> getlatestCompletedEventSegmentReplicationState(shardId)); } /** diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java index 8202b680745dd..ea88f447c8a0b 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java @@ -9,8 +9,8 @@ package org.opensearch.rest.action.cat; import org.apache.lucene.util.CollectionUtil; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsRequest; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsResponse; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsRequest; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsResponse; import org.opensearch.action.support.IndicesOptions; import org.opensearch.client.node.NodeClient; import org.opensearch.common.Strings; @@ -72,7 +72,7 @@ public BaseRestHandler.RestChannelConsumer doCatRequest(final RestRequest reques return channel -> client.admin() .indices() - .segment_replication(segmentReplicationStatsRequest, new RestResponseListener(channel) { + .segmentReplication(segmentReplicationStatsRequest, new RestResponseListener(channel) { @Override public RestResponse buildResponse(final SegmentReplicationStatsResponse response) throws Exception { return RestTable.buildResponse(buildSegmentReplicationTable(request, response), channel); @@ -180,11 +180,11 @@ public int compare(SegmentReplicationState o1, SegmentReplicationState o2) { t.addCell(state.getIndex().totalFileCount()); t.addCell(state.getIndex().totalRecoverBytes()); t.addCell(state.getIndex().totalBytes()); - t.addCell(new TimeValue(state.getReplicatingStageTime())); - t.addCell(new TimeValue(state.getGetCheckpointInfoStageTime())); - t.addCell(new TimeValue(state.getFileDiffStageTime())); - t.addCell(new TimeValue(state.getGetFileStageTime())); - t.addCell(new TimeValue(state.getFinalizeReplicationStageTime())); + t.addCell(state.getReplicatingStageTime()); + t.addCell(state.getGetCheckpointInfoStageTime()); + t.addCell(state.getFileDiffStageTime()); + t.addCell(state.getGetFileStageTime()); + t.addCell(state.getFinalizeReplicationStageTime()); } t.endRow(); } diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index 1dcfabda4d92d..99abb98f34716 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -845,6 +845,11 @@ public void getSegmentFiles( ) { listener.onResponse(new GetSegmentFilesResponse(Collections.emptyList())); } + + @Override + public String getDescription() { + return ""; + } }; when(sourceFactory.get(any())).thenReturn(source); startReplicationAndAssertCancellation(nextPrimary, targetService); @@ -915,6 +920,11 @@ public void getSegmentFiles( ) { Assert.fail("Should not be reached"); } + + @Override + public String getDescription() { + return ""; + } }; when(sourceFactory.get(any())).thenReturn(source); startReplicationAndAssertCancellation(replica, targetService); @@ -957,6 +967,11 @@ public void getSegmentFiles( listener.onResponse(new GetSegmentFilesResponse(Collections.emptyList())); targetService.beforeIndexShardClosed(replica.shardId, replica, Settings.EMPTY); } + + @Override + public String getDescription() { + return ""; + } }; when(sourceFactory.get(any())).thenReturn(source); startReplicationAndAssertCancellation(replica, targetService); @@ -995,6 +1010,11 @@ public void getSegmentFiles( Store store, ActionListener listener ) {} + + @Override + public String getDescription() { + return ""; + } }; when(sourceFactory.get(any())).thenReturn(source); startReplicationAndAssertCancellation(replica, targetService); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index f6185cc071941..ed76265cc15a7 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -136,6 +136,11 @@ public void getSegmentFiles( ) { Assert.fail("Should not be called"); } + + @Override + public String getDescription() { + return ""; + } }; final SegmentReplicationTarget target = new SegmentReplicationTarget( checkpoint, diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java index 5900ac3ad29f3..d3b0e0c747468 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java @@ -135,6 +135,11 @@ public void getSegmentFiles( assert (filesToFetch.contains(SEGMENT_FILE)); listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } + + @Override + public String getDescription() { + return ""; + } }; SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( @@ -184,6 +189,11 @@ public void getSegmentFiles( ) { listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } + + @Override + public String getDescription() { + return ""; + } }; SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( SegmentReplicationTargetService.SegmentReplicationListener.class @@ -227,6 +237,11 @@ public void getSegmentFiles( ) { listener.onFailure(exception); } + + @Override + public String getDescription() { + return ""; + } }; SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( SegmentReplicationTargetService.SegmentReplicationListener.class @@ -270,6 +285,11 @@ public void getSegmentFiles( ) { listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } + + @Override + public String getDescription() { + return ""; + } }; SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( SegmentReplicationTargetService.SegmentReplicationListener.class @@ -315,6 +335,11 @@ public void getSegmentFiles( ) { listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } + + @Override + public String getDescription() { + return ""; + } }; SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( SegmentReplicationTargetService.SegmentReplicationListener.class @@ -359,6 +384,11 @@ public void getSegmentFiles( ) { listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } + + @Override + public String getDescription() { + return ""; + } }; SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( SegmentReplicationTargetService.SegmentReplicationListener.class @@ -410,6 +440,11 @@ public void getSegmentFiles( ) { listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } + + @Override + public String getDescription() { + return ""; + } }; SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( SegmentReplicationTargetService.SegmentReplicationListener.class diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java index 17ea33c86e5bb..abf0613d0d23e 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java @@ -8,7 +8,7 @@ package org.opensearch.rest.action.cat; -import org.opensearch.action.admin.indices.segment_replication.SegmentReplicationStatsResponse; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsResponse; import org.opensearch.action.support.DefaultShardOperationFailedException; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; @@ -36,7 +36,7 @@ import static org.mockito.Mockito.when; public class RestCatSegmentReplicationActionTests extends OpenSearchTestCase { - public void testSegment_ReplicationActionAction() { + public void testSegmentReplicationAction() { final RestCatSegmentReplicationAction action = new RestCatSegmentReplicationAction(); final int totalShards = randomIntBetween(1, 32); final int successfulShards = Math.max(0, totalShards - randomIntBetween(1, 2)); diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 1fa619012ff16..5553a5e2fda9a 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -1326,6 +1326,11 @@ public void getSegmentFiles( } listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } + + @Override + public String getDescription() { + return ""; + } }; when(sourceFactory.get(any())).thenReturn(replicationSource); when(indicesService.getShardOrNull(any())).thenReturn(target); From fc24661a4afed5c70fe1eeaa8863c03c9cfa3ee7 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Tue, 7 Feb 2023 17:48:36 +0000 Subject: [PATCH 24/25] Fix failing testSegmentReplicationApiResponse(). Signed-off-by: Rishikesh1159 --- .../indices/replication/SegmentReplicationApiIT.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationApiIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationApiIT.java index 86ceae9285cfe..cd10ca6132666 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationApiIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationApiIT.java @@ -19,13 +19,14 @@ import java.util.concurrent.CountDownLatch; +import static java.util.Arrays.asList; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class SegmentReplicationApiIT extends SegmentReplicationBaseIT { - public void testSegmentReplicationApiResponse() { + public void testSegmentReplicationApiResponse() throws Exception { logger.info("--> starting [Primary Node] ..."); final String primaryNode = internalCluster().startNode(); createIndex(INDEX_NAME); @@ -38,8 +39,7 @@ public void testSegmentReplicationApiResponse() { client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); } refresh(INDEX_NAME); - logger.info("--> verifying count"); - assertEquals(client().prepareSearch(INDEX_NAME).setSize(0).execute().actionGet().getHits().getTotalHits().value, 10L); + waitForSearchableDocs(10L, asList(primaryNode, replicaNode)); SegmentReplicationStatsResponse response = client().admin().indices().prepareSegmentReplication(INDEX_NAME).execute().actionGet(); // Verify API Response @@ -48,7 +48,7 @@ public void testSegmentReplicationApiResponse() { assertThat(response.shardSegmentReplicationStates().get(INDEX_NAME).get(0).getIndex().recoveredFileCount(), greaterThan(0)); } - public void testSegmentReplicationApiResponseForActiveAndCompletedOnly() { + public void testSegmentReplicationApiResponseForActiveAndCompletedOnly() throws Exception { logger.info("--> starting [Primary Node] ..."); final String primaryNode = internalCluster().startNode(); createIndex(INDEX_NAME); @@ -61,6 +61,7 @@ public void testSegmentReplicationApiResponseForActiveAndCompletedOnly() { client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); } refresh(INDEX_NAME); + waitForSearchableDocs(10L, asList(primaryNode, replicaNode)); for (int i = 10; i < 20; i++) { client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); } From 7d07dc6b29e65d59ffe669ac5c713e5d934bc70e Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Wed, 8 Feb 2023 01:39:40 +0000 Subject: [PATCH 25/25] Refactoring code. Signed-off-by: Rishikesh1159 --- .../opensearch/action/IndicesRequestIT.java | 4 +- .../action/admin/ClientTimeoutIT.java | 6 +-- ...IT.java => SegmentReplicationStatsIT.java} | 42 ++++++++------- .../SegmentReplicationStatsResponse.java | 4 +- .../indices/replication/package-info.java | 2 +- .../opensearch/client/IndicesAdminClient.java | 6 +-- .../client/support/AbstractClient.java | 6 +-- .../cat/RestCatSegmentReplicationAction.java | 2 +- .../SegmentReplicationIndexShardTests.java | 29 ++--------- .../SegmentReplicationTargetServiceTests.java | 8 +-- .../SegmentReplicationTargetTests.java | 51 ++++--------------- .../replication/TestReplicationSource.java | 46 +++++++++++++++++ .../index/shard/IndexShardTestCase.java | 8 +-- 13 files changed, 102 insertions(+), 112 deletions(-) rename server/src/internalClusterTest/java/org/opensearch/indices/replication/{SegmentReplicationApiIT.java => SegmentReplicationStatsIT.java} (82%) create mode 100644 test/framework/src/main/java/org/opensearch/index/replication/TestReplicationSource.java diff --git a/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java index d956b0e7da1e1..02b4f0a216ac7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/IndicesRequestIT.java @@ -470,12 +470,12 @@ public void testRecovery() { assertSameIndices(recoveryRequest, recoveryAction); } - public void testSegmentReplication() { + public void testSegmentReplicationStats() { String segmentReplicationAction = SegmentReplicationStatsAction.NAME + "[n]"; interceptTransportActions(segmentReplicationAction); SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(randomIndicesOrAliases()); - internalCluster().coordOnlyNodeClient().admin().indices().segmentReplication(segmentReplicationStatsRequest).actionGet(); + internalCluster().coordOnlyNodeClient().admin().indices().segmentReplicationStats(segmentReplicationStatsRequest).actionGet(); clearInterceptedActions(); assertSameIndices(segmentReplicationStatsRequest, segmentReplicationAction); diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java index c40bc5d914ff6..356db8b919a85 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/ClientTimeoutIT.java @@ -152,7 +152,7 @@ public void testRecoveriesWithTimeout() { assertThat(recoveryResponse.getShardFailures()[0].reason(), containsString("ReceiveTimeoutTransportException")); } - public void testSegmentReplicationWithTimeout() { + public void testSegmentReplicationStatsWithTimeout() { internalCluster().startClusterManagerOnlyNode( Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REPLICATION_TYPE, "true").build() ); @@ -185,7 +185,7 @@ public void testSegmentReplicationWithTimeout() { // Happy case SegmentReplicationStatsResponse segmentReplicationStatsResponse = dataNodeClient().admin() .indices() - .prepareSegmentReplication() + .prepareSegmentReplicationStats() .get(); assertThat(segmentReplicationStatsResponse.getTotalShards(), equalTo(numShards * 2)); assertThat(segmentReplicationStatsResponse.getSuccessfulShards(), equalTo(numShards * 2)); @@ -194,7 +194,7 @@ public void testSegmentReplicationWithTimeout() { simulateTimeoutAtTransport(dataNode, anotherDataNode, SegmentReplicationStatsAction.NAME); // verify response with bad node. - segmentReplicationStatsResponse = dataNodeClient().admin().indices().prepareSegmentReplication().get(); + segmentReplicationStatsResponse = dataNodeClient().admin().indices().prepareSegmentReplicationStats().get(); assertThat(segmentReplicationStatsResponse.getTotalShards(), equalTo(numShards * 2)); assertThat(segmentReplicationStatsResponse.getSuccessfulShards(), equalTo(numShards)); assertThat(segmentReplicationStatsResponse.getFailedShards(), equalTo(numShards)); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationApiIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java similarity index 82% rename from server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationApiIT.java rename to server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java index cd10ca6132666..3dbaed9e03c80 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationApiIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java @@ -24,43 +24,47 @@ import static org.hamcrest.Matchers.greaterThan; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -public class SegmentReplicationApiIT extends SegmentReplicationBaseIT { +public class SegmentReplicationStatsIT extends SegmentReplicationBaseIT { - public void testSegmentReplicationApiResponse() throws Exception { - logger.info("--> starting [Primary Node] ..."); + public void testSegmentReplicationStatsResponse() throws Exception { final String primaryNode = internalCluster().startNode(); createIndex(INDEX_NAME); ensureYellowAndNoInitializingShards(INDEX_NAME); - logger.info("--> start empty node to add replica shard"); final String replicaNode = internalCluster().startNode(); ensureGreen(INDEX_NAME); - logger.info("--> index 10 docs"); + + // index 10 docs for (int i = 0; i < 10; i++) { client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); } refresh(INDEX_NAME); waitForSearchableDocs(10L, asList(primaryNode, replicaNode)); - SegmentReplicationStatsResponse response = client().admin().indices().prepareSegmentReplication(INDEX_NAME).execute().actionGet(); + SegmentReplicationStatsResponse response = client().admin() + .indices() + .prepareSegmentReplicationStats(INDEX_NAME) + .execute() + .actionGet(); // Verify API Response assertThat(response.shardSegmentReplicationStates().size(), equalTo(SHARD_COUNT)); assertThat(response.shardSegmentReplicationStates().get(INDEX_NAME).get(0).getStage(), equalTo(SegmentReplicationState.Stage.DONE)); assertThat(response.shardSegmentReplicationStates().get(INDEX_NAME).get(0).getIndex().recoveredFileCount(), greaterThan(0)); } - public void testSegmentReplicationApiResponseForActiveAndCompletedOnly() throws Exception { - logger.info("--> starting [Primary Node] ..."); + public void testSegmentReplicationStatsResponseForActiveAndCompletedOnly() throws Exception { final String primaryNode = internalCluster().startNode(); createIndex(INDEX_NAME); ensureYellowAndNoInitializingShards(INDEX_NAME); - logger.info("--> starting [Replica Node] ..."); final String replicaNode = internalCluster().startNode(); ensureGreen(INDEX_NAME); - logger.info("--> index 10 docs"); + + // index 10 docs for (int i = 0; i < 10; i++) { client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); } refresh(INDEX_NAME); + + // index 10 more docs waitForSearchableDocs(10L, asList(primaryNode, replicaNode)); for (int i = 10; i < 20; i++) { client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); @@ -93,10 +97,11 @@ public void testSegmentReplicationApiResponseForActiveAndCompletedOnly() throws } catch (InterruptedException e) { throw new RuntimeException(e); } - logger.info("--> verifying active_only by checking if current stage is GET_FILES STAGE"); + + // verifying active_only by checking if current stage is GET_FILES STAGE SegmentReplicationStatsResponse activeOnlyResponse = client().admin() .indices() - .prepareSegmentReplication(INDEX_NAME) + .prepareSegmentReplicationStats(INDEX_NAME) .setActiveOnly(true) .execute() .actionGet(); @@ -105,10 +110,10 @@ public void testSegmentReplicationApiResponseForActiveAndCompletedOnly() throws equalTo(SegmentReplicationState.Stage.GET_FILES) ); - logger.info("--> verifying completed_only by checking if current stage is DONE"); + // verifying completed_only by checking if current stage is DONE SegmentReplicationStatsResponse completedOnlyResponse = client().admin() .indices() - .prepareSegmentReplication(INDEX_NAME) + .prepareSegmentReplicationStats(INDEX_NAME) .setCompletedOnly(true) .execute() .actionGet(); @@ -120,8 +125,7 @@ public void testSegmentReplicationApiResponseForActiveAndCompletedOnly() throws waitForAssertions.countDown(); } - public void testSegmentReplicationApiResponseOnDocumentReplicationIndex() { - logger.info("--> starting [Primary Node] ..."); + public void testSegmentReplicationStatsResponseOnDocumentReplicationIndex() { final String primaryNode = internalCluster().startNode(); prepareCreate( INDEX_NAME, @@ -129,17 +133,17 @@ public void testSegmentReplicationApiResponseOnDocumentReplicationIndex() { ).get(); ensureYellowAndNoInitializingShards(INDEX_NAME); - logger.info("--> start empty node to add replica shard"); final String replicaNode = internalCluster().startNode(); ensureGreen(INDEX_NAME); - logger.info("--> index 10 docs"); + + // index 10 docs for (int i = 0; i < 10; i++) { client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); } refresh(INDEX_NAME); OpenSearchStatusException exception = assertThrows( OpenSearchStatusException.class, - () -> client().admin().indices().prepareSegmentReplication(INDEX_NAME).execute().actionGet() + () -> client().admin().indices().prepareSegmentReplicationStats(INDEX_NAME).execute().actionGet() ); // Verify exception message String expectedMessage = "Segment Replication is not enabled on Index: test-idx-1"; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java index 759e35b37bbc7..2f72d7dd3e544 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java @@ -55,7 +55,7 @@ public SegmentReplicationStatsResponse( this.shardSegmentReplicationStates = shardSegmentReplicationStates; } - public boolean hasSegmentReplication() { + public boolean hasSegmentReplicationStats() { return shardSegmentReplicationStates.size() > 0; } @@ -66,7 +66,7 @@ public Map> shardSegmentReplicationStates( @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (hasSegmentReplication()) { + if (hasSegmentReplicationStats()) { for (String index : shardSegmentReplicationStates.keySet()) { List segmentReplicationStates = shardSegmentReplicationStates.get(index); if (segmentReplicationStates == null || segmentReplicationStates.size() == 0) { diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/package-info.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/package-info.java index f801f2f288797..db69c19825199 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/package-info.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/package-info.java @@ -6,5 +6,5 @@ * compatible open source license. */ -/** Segment Replication transport handlers. */ +/** Segment Replication Stats transport handlers. */ package org.opensearch.action.admin.indices.replication; diff --git a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java index fd32695262100..217902a2600e8 100644 --- a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java +++ b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java @@ -191,12 +191,12 @@ public interface IndicesAdminClient extends OpenSearchClient { /** *Indices segment replication */ - ActionFuture segmentReplication(SegmentReplicationStatsRequest request); + ActionFuture segmentReplicationStats(SegmentReplicationStatsRequest request); /** *Indices segment replication */ - void segmentReplication(SegmentReplicationStatsRequest request, ActionListener listener); + void segmentReplicationStats(SegmentReplicationStatsRequest request, ActionListener listener); /** * Indices recoveries @@ -206,7 +206,7 @@ public interface IndicesAdminClient extends OpenSearchClient { /** * Indices segment replication */ - SegmentReplicationStatsRequestBuilder prepareSegmentReplication(String... indices); + SegmentReplicationStatsRequestBuilder prepareSegmentReplicationStats(String... indices); /** * The segments of one or more indices. diff --git a/server/src/main/java/org/opensearch/client/support/AbstractClient.java b/server/src/main/java/org/opensearch/client/support/AbstractClient.java index e4e2f03464321..261709394e531 100644 --- a/server/src/main/java/org/opensearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/opensearch/client/support/AbstractClient.java @@ -1780,12 +1780,12 @@ public RecoveryRequestBuilder prepareRecoveries(String... indices) { } @Override - public ActionFuture segmentReplication(final SegmentReplicationStatsRequest request) { + public ActionFuture segmentReplicationStats(final SegmentReplicationStatsRequest request) { return execute(SegmentReplicationStatsAction.INSTANCE, request); } @Override - public void segmentReplication( + public void segmentReplicationStats( final SegmentReplicationStatsRequest request, final ActionListener listener ) { @@ -1793,7 +1793,7 @@ public void segmentReplication( } @Override - public SegmentReplicationStatsRequestBuilder prepareSegmentReplication(String... indices) { + public SegmentReplicationStatsRequestBuilder prepareSegmentReplicationStats(String... indices) { return new SegmentReplicationStatsRequestBuilder(this, SegmentReplicationStatsAction.INSTANCE).setIndices(indices); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java index ea88f447c8a0b..f4da8bfc8adcb 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java @@ -72,7 +72,7 @@ public BaseRestHandler.RestChannelConsumer doCatRequest(final RestRequest reques return channel -> client.admin() .indices() - .segmentReplication(segmentReplicationStatsRequest, new RestResponseListener(channel) { + .segmentReplicationStats(segmentReplicationStatsRequest, new RestResponseListener(channel) { @Override public RestResponse buildResponse(final SegmentReplicationStatsResponse response) throws Exception { return RestTable.buildResponse(buildSegmentReplicationTable(request, response), channel); diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index 99abb98f34716..d554af0ffc488 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -35,6 +35,7 @@ import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase; +import org.opensearch.index.replication.TestReplicationSource; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.recovery.RecoverySettings; @@ -818,7 +819,7 @@ public void testReplicaPromotedWhileReplicating() throws Exception { final SegmentReplicationSourceFactory sourceFactory = mock(SegmentReplicationSourceFactory.class); final SegmentReplicationTargetService targetService = newTargetService(sourceFactory); - SegmentReplicationSource source = new SegmentReplicationSource() { + SegmentReplicationSource source = new TestReplicationSource() { @Override public void getCheckpointMetadata( long replicationId, @@ -845,11 +846,6 @@ public void getSegmentFiles( ) { listener.onResponse(new GetSegmentFilesResponse(Collections.emptyList())); } - - @Override - public String getDescription() { - return ""; - } }; when(sourceFactory.get(any())).thenReturn(source); startReplicationAndAssertCancellation(nextPrimary, targetService); @@ -898,7 +894,7 @@ public void testReplicaClosesWhileReplicating_AfterGetCheckpoint() throws Except final SegmentReplicationSourceFactory sourceFactory = mock(SegmentReplicationSourceFactory.class); final SegmentReplicationTargetService targetService = newTargetService(sourceFactory); - SegmentReplicationSource source = new SegmentReplicationSource() { + SegmentReplicationSource source = new TestReplicationSource() { @Override public void getCheckpointMetadata( long replicationId, @@ -920,11 +916,6 @@ public void getSegmentFiles( ) { Assert.fail("Should not be reached"); } - - @Override - public String getDescription() { - return ""; - } }; when(sourceFactory.get(any())).thenReturn(source); startReplicationAndAssertCancellation(replica, targetService); @@ -945,7 +936,7 @@ public void testReplicaClosesWhileReplicating_AfterGetSegmentFiles() throws Exce final SegmentReplicationSourceFactory sourceFactory = mock(SegmentReplicationSourceFactory.class); final SegmentReplicationTargetService targetService = newTargetService(sourceFactory); - SegmentReplicationSource source = new SegmentReplicationSource() { + SegmentReplicationSource source = new TestReplicationSource() { @Override public void getCheckpointMetadata( long replicationId, @@ -967,11 +958,6 @@ public void getSegmentFiles( listener.onResponse(new GetSegmentFilesResponse(Collections.emptyList())); targetService.beforeIndexShardClosed(replica.shardId, replica, Settings.EMPTY); } - - @Override - public String getDescription() { - return ""; - } }; when(sourceFactory.get(any())).thenReturn(source); startReplicationAndAssertCancellation(replica, targetService); @@ -992,7 +978,7 @@ public void testPrimaryCancelsExecution() throws Exception { final SegmentReplicationSourceFactory sourceFactory = mock(SegmentReplicationSourceFactory.class); final SegmentReplicationTargetService targetService = newTargetService(sourceFactory); - SegmentReplicationSource source = new SegmentReplicationSource() { + SegmentReplicationSource source = new TestReplicationSource() { @Override public void getCheckpointMetadata( long replicationId, @@ -1010,11 +996,6 @@ public void getSegmentFiles( Store store, ActionListener listener ) {} - - @Override - public String getDescription() { - return ""; - } }; when(sourceFactory.get(any())).thenReturn(source); startReplicationAndAssertCancellation(replica, targetService); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index ed76265cc15a7..69e1e6f8de09b 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -16,6 +16,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.CancellableThreads; import org.opensearch.index.engine.NRTReplicationEngineFactory; +import org.opensearch.index.replication.TestReplicationSource; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.index.store.Store; @@ -115,7 +116,7 @@ public void onReplicationFailure(SegmentReplicationState state, ReplicationFaile public void testReplicationFails() throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); final OpenSearchException expectedError = new OpenSearchException("Fail"); - SegmentReplicationSource source = new SegmentReplicationSource() { + SegmentReplicationSource source = new TestReplicationSource() { @Override public void getCheckpointMetadata( @@ -136,11 +137,6 @@ public void getSegmentFiles( ) { Assert.fail("Should not be called"); } - - @Override - public String getDescription() { - return ""; - } }; final SegmentReplicationTarget target = new SegmentReplicationTarget( checkpoint, diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java index d3b0e0c747468..81edd9665d502 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java @@ -32,6 +32,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.NRTReplicationEngineFactory; +import org.opensearch.index.replication.TestReplicationSource; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.index.shard.ShardId; @@ -113,7 +114,7 @@ public void setUp() throws Exception { public void testSuccessfulResponse_startReplication() { - SegmentReplicationSource segrepSource = new SegmentReplicationSource() { + SegmentReplicationSource segrepSource = new TestReplicationSource() { @Override public void getCheckpointMetadata( long replicationId, @@ -135,11 +136,6 @@ public void getSegmentFiles( assert (filesToFetch.contains(SEGMENT_FILE)); listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } - - @Override - public String getDescription() { - return ""; - } }; SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( @@ -169,7 +165,7 @@ public void onFailure(Exception e) { public void testFailureResponse_getCheckpointMetadata() { Exception exception = new Exception("dummy failure"); - SegmentReplicationSource segrepSource = new SegmentReplicationSource() { + SegmentReplicationSource segrepSource = new TestReplicationSource() { @Override public void getCheckpointMetadata( long replicationId, @@ -189,11 +185,6 @@ public void getSegmentFiles( ) { listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } - - @Override - public String getDescription() { - return ""; - } }; SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( SegmentReplicationTargetService.SegmentReplicationListener.class @@ -217,7 +208,7 @@ public void onFailure(Exception e) { public void testFailureResponse_getSegmentFiles() { Exception exception = new Exception("dummy failure"); - SegmentReplicationSource segrepSource = new SegmentReplicationSource() { + SegmentReplicationSource segrepSource = new TestReplicationSource() { @Override public void getCheckpointMetadata( long replicationId, @@ -237,11 +228,6 @@ public void getSegmentFiles( ) { listener.onFailure(exception); } - - @Override - public String getDescription() { - return ""; - } }; SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( SegmentReplicationTargetService.SegmentReplicationListener.class @@ -265,7 +251,7 @@ public void onFailure(Exception e) { public void testFailure_finalizeReplication_IOException() throws IOException { IOException exception = new IOException("dummy failure"); - SegmentReplicationSource segrepSource = new SegmentReplicationSource() { + SegmentReplicationSource segrepSource = new TestReplicationSource() { @Override public void getCheckpointMetadata( long replicationId, @@ -285,11 +271,6 @@ public void getSegmentFiles( ) { listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } - - @Override - public String getDescription() { - return ""; - } }; SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( SegmentReplicationTargetService.SegmentReplicationListener.class @@ -315,7 +296,7 @@ public void onFailure(Exception e) { public void testFailure_finalizeReplication_IndexFormatException() throws IOException { IndexFormatTooNewException exception = new IndexFormatTooNewException("string", 1, 2, 1); - SegmentReplicationSource segrepSource = new SegmentReplicationSource() { + SegmentReplicationSource segrepSource = new TestReplicationSource() { @Override public void getCheckpointMetadata( long replicationId, @@ -335,11 +316,6 @@ public void getSegmentFiles( ) { listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } - - @Override - public String getDescription() { - return ""; - } }; SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( SegmentReplicationTargetService.SegmentReplicationListener.class @@ -364,7 +340,7 @@ public void onFailure(Exception e) { public void testFailure_differentSegmentFiles() throws IOException { - SegmentReplicationSource segrepSource = new SegmentReplicationSource() { + SegmentReplicationSource segrepSource = new TestReplicationSource() { @Override public void getCheckpointMetadata( long replicationId, @@ -384,11 +360,6 @@ public void getSegmentFiles( ) { listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } - - @Override - public String getDescription() { - return ""; - } }; SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( SegmentReplicationTargetService.SegmentReplicationListener.class @@ -420,7 +391,7 @@ public void test_MissingFiles_NotCausingFailure() throws IOException { // snapshot (2nd element which contains delete operations) and replica's existing snapshot (1st element). List storeMetadataSnapshots = generateStoreMetadataSnapshot(docCount); - SegmentReplicationSource segrepSource = new SegmentReplicationSource() { + SegmentReplicationSource segrepSource = new TestReplicationSource() { @Override public void getCheckpointMetadata( long replicationId, @@ -440,11 +411,6 @@ public void getSegmentFiles( ) { listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } - - @Override - public String getDescription() { - return ""; - } }; SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( SegmentReplicationTargetService.SegmentReplicationListener.class @@ -526,4 +492,5 @@ public void tearDown() throws Exception { super.tearDown(); closeShards(spyIndexShard, indexShard); } + } diff --git a/test/framework/src/main/java/org/opensearch/index/replication/TestReplicationSource.java b/test/framework/src/main/java/org/opensearch/index/replication/TestReplicationSource.java new file mode 100644 index 0000000000000..a3adedcbdef86 --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/index/replication/TestReplicationSource.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.replication; + +import org.opensearch.action.ActionListener; +import org.opensearch.index.store.Store; +import org.opensearch.index.store.StoreFileMetadata; +import org.opensearch.indices.replication.CheckpointInfoResponse; +import org.opensearch.indices.replication.GetSegmentFilesResponse; +import org.opensearch.indices.replication.SegmentReplicationSource; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; + +import java.util.List; + +/** + * This class is used by unit tests implementing SegmentReplicationSource + */ +public abstract class TestReplicationSource implements SegmentReplicationSource { + + @Override + public abstract void getCheckpointMetadata( + long replicationId, + ReplicationCheckpoint checkpoint, + ActionListener listener + ); + + @Override + public abstract void getSegmentFiles( + long replicationId, + ReplicationCheckpoint checkpoint, + List filesToFetch, + Store store, + ActionListener listener + ); + + @Override + public String getDescription() { + return "This is a test description"; + } +} diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 5553a5e2fda9a..0ce86f8ae5661 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -89,6 +89,7 @@ import org.opensearch.index.engine.InternalEngineFactory; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.SourceToParse; +import org.opensearch.index.replication.TestReplicationSource; import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.index.seqno.RetentionLeaseSyncer; import org.opensearch.index.seqno.SequenceNumbers; @@ -1290,7 +1291,7 @@ public final SegmentReplicationTargetService prepareForReplication(IndexShard pr sourceFactory, indicesService ); - final SegmentReplicationSource replicationSource = new SegmentReplicationSource() { + final SegmentReplicationSource replicationSource = new TestReplicationSource() { @Override public void getCheckpointMetadata( long replicationId, @@ -1326,11 +1327,6 @@ public void getSegmentFiles( } listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } - - @Override - public String getDescription() { - return ""; - } }; when(sourceFactory.get(any())).thenReturn(replicationSource); when(indicesService.getShardOrNull(any())).thenReturn(target);