Skip to content

Commit

Permalink
Mark restored indices as remote backed during migration
Browse files Browse the repository at this point in the history 
Signed-off-by: Lakshya Taragi <[email protected]>
  • Loading branch information
@ltaragi
ltaragi committed Mar 19, 2024
1 parent f9cd3e3 commit 501b052
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 6 deletions.
6 changes: 4 additions & 2 deletions ...rnalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationAllocationIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
public class RemoteStoreMigrationAllocationIT extends MigrationBaseTestCase {

private static final String TEST_INDEX = "test_index";
public static final String TEST_INDEX = "test_index";
protected static final String NAME = "remote_store_migration";

private final ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
Expand Down Expand Up @@ -364,6 +364,7 @@ public void testAlwaysAllocateNewShardForStrictMode() throws Exception {
}

// test for remote store backed index

public void testDontAllocateToNonRemoteNodeForRemoteStoreBackedIndex() throws Exception {
logger.info(" --> initialize cluster with remote master node");
initializeCluster(true);
Expand Down Expand Up @@ -419,6 +420,7 @@ public void testDontAllocateToNonRemoteNodeForRemoteStoreBackedIndex() throws Ex
assertNonAllocation(!isReplicaAllocation);
}


// bootstrap a cluster
public void initializeCluster(boolean remoteClusterManager) {
addRemote = remoteClusterManager;
Expand Down Expand Up @@ -457,7 +459,7 @@ public DiscoveryNode assertNodeInCluster(String nodeName) {
}

// returns a comma-separated list of node names excluding `except`
private String allNodesExcept(String except) {
public String allNodesExcept(String except) {
StringBuilder exclude = new StringBuilder();
DiscoveryNodes allNodes = client.admin().cluster().prepareState().get().getState().nodes();
for (DiscoveryNode node : allNodes) {
Expand Down
95 changes: 93 additions & 2 deletions ...ClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,19 @@

package org.opensearch.remotemigration;

import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.settings.SettingsException;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.index.IndexSettings;
import org.opensearch.indices.replication.common.ReplicationType;
import org.opensearch.snapshots.SnapshotInfo;
import org.opensearch.snapshots.SnapshotState;
import org.opensearch.test.InternalTestCluster;
import org.opensearch.test.OpenSearchIntegTestCase;

import java.nio.file.Path;
import java.util.Optional;

import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
Expand All @@ -25,6 +31,7 @@
import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.STRICT;
import static org.opensearch.node.remotestore.RemoteStoreNodeService.Direction.REMOTE_STORE;
import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.MIXED;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;


@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
Expand All @@ -44,8 +51,9 @@ public void testNewIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode()

logger.info(" --> add non-remote node");
addRemote = false;
internalCluster().startNode();
String remoteNodeName = internalCluster().startNode();
internalCluster().validateClusterFormed();
assertNodeInCluster(remoteNodeName);

logger.info(" --> create an index");
prepareIndexWithoutReplica(Optional.of(indexName1));
Expand All @@ -59,8 +67,9 @@ public void testNewIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode()

logger.info(" --> add remote node");
addRemote = true;
internalCluster().startNode();
String nonRemoteNodeName = internalCluster().startNode();
internalCluster().validateClusterFormed();
assertNodeInCluster(nonRemoteNodeName);

logger.info(" --> create another index");
prepareIndexWithoutReplica(Optional.of(indexName2));
Expand All @@ -69,6 +78,71 @@ public void testNewIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode()
assertRemoteStoreBackedIndex(indexName2);
}

public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode() throws Exception {
logger.info(" --> initialize cluster: gives non remote cluster manager");
initializeCluster(false);

logger.info(" --> add remote and non-remote nodes");
setClusterMode(MIXED.mode);
addRemote = false;
String nonRemoteNodeName = internalCluster().startNode();
addRemote = true;
String remoteNodeName = internalCluster().startNode();
internalCluster().validateClusterFormed();
assertNodeInCluster(nonRemoteNodeName);
assertNodeInCluster(remoteNodeName);

logger.info(" --> create a non remote-backed index");
internalCluster().client().admin()
.indices()
.prepareCreate(TEST_INDEX)
.setSettings(
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.build()
)
.get();

logger.info(" --> verify that non remote stored backed index is created");
assertNonRemoteStoreBackedIndex(TEST_INDEX);

logger.info(" --> create repository");
String snapshotName = "test-snapshot";
String snapshotRepoName = "test-restore-snapshot-repo";
Path snapshotRepoNameAbsolutePath = randomRepoPath().toAbsolutePath();
assertAcked(clusterAdmin().preparePutRepository(snapshotRepoName).setType("fs").setSettings(Settings.builder().put("location", snapshotRepoNameAbsolutePath)));

logger.info(" --> create snapshot of non remote stored backed index");

SnapshotInfo snapshotInfo = client().admin()
.cluster()
.prepareCreateSnapshot(snapshotRepoName, snapshotName)
.setIndices(TEST_INDEX)
.setWaitForCompletion(true)
.get()
.getSnapshotInfo();

assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
assertTrue(snapshotInfo.successfulShards() > 0);
assertEquals(0, snapshotInfo.failedShards());

logger.info(" --> restore index from snapshot under NONE direction");
String restoredIndexName1 = TEST_INDEX + "-restored1";
restoreSnapshot(snapshotRepoName, snapshotName, restoredIndexName1);

logger.info(" --> verify that restored index is non remote-backed");
assertNonRemoteStoreBackedIndex(restoredIndexName1);

logger.info(" --> restore index from snapshot under REMOTE_STORE direction");
setDirection(REMOTE_STORE.direction);
String restoredIndexName2 = TEST_INDEX + "-restored2";
restoreSnapshot(snapshotRepoName, snapshotName, restoredIndexName2);

logger.info(" --> verify that restored index is non remote-backed");
assertRemoteStoreBackedIndex(restoredIndexName2);
}


// compatibility mode setting test
public void testSwitchToStrictMode() throws Exception {
Expand All @@ -82,6 +156,8 @@ public void testSwitchToStrictMode() throws Exception {
addRemote = false;
String nonRemoteNodeName = internalCluster().startNode();
internalCluster().validateClusterFormed();
assertNodeInCluster(remoteNodeName);
assertNodeInCluster(nonRemoteNodeName);

logger.info(" --> attempt switching to strict mode");
SettingsException exception = assertThrows(
Expand Down Expand Up @@ -120,4 +196,19 @@ private void assertRemoteStoreBackedIndex(String indexName) {
assertEquals(IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL, INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings));
}

// restore indices from a snapshot
private void restoreSnapshot(String snapshotRepoName, String snapshotName, String restoredIndexName) {
RestoreSnapshotResponse restoreSnapshotResponse = internalCluster().client().admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepoName, snapshotName)
.setWaitForCompletion(false)
.setIndices(TEST_INDEX)
.setRenamePattern(TEST_INDEX)
.setRenameReplacement(restoredIndexName)
.get();

assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED);
ensureGreen(restoredIndexName);
}

}
6 changes: 4 additions & 2 deletions server/src/main/java/org/opensearch/snapshots/RestoreService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -671,9 +671,11 @@ private Settings getOverrideSettingsInternal() {
clusterService.getSettings(),
clusterService.getClusterSettings()
);
// remote store settings needs to be overridden if the remote store feature is enabled in the
// cluster where snapshot is being restored.
// remote store settings needs to be overridden in the cluster where snapshot is being restored if:
// 1. the remote store feature is enabled
// 2. cluster is undergoing remote store migration
MetadataCreateIndexService.updateRemoteStoreSettings(settingsBuilder, clusterService.getSettings());
MetadataCreateIndexService.updateRemoteStoreSettingsForMigration(settingsBuilder, clusterService.state(), clusterService.getClusterSettings());
return settingsBuilder.build();
}

Expand Down

0 comments on commit 501b052

Please sign in to comment.