Skip to content

Commit

Permalink
remove remote translog setting and restrict to remote only index crea…
Browse files Browse the repository at this point in the history 
…tion

Signed-off-by: bansvaru <[email protected]>
  • Loading branch information
@linuxpi
linuxpi committed Jul 19, 2023
1 parent 1d3b006 commit 7bac123
Show file tree
Hide file tree
Showing 31 changed files with 151 additions and 624 deletions.
6 changes: 6 additions & 0 deletions ...rTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
public abstract class AbstractRemoteStoreMockRepositoryIntegTestCase extends AbstractSnapshotIntegTestCase {

protected static final String REPOSITORY_NAME = "my-segment-repo-1";
protected static final String TRANSLOG_REPOSITORY_NAME = "my-translog-repo-1";
protected static final String INDEX_NAME = "remote-store-test-idx-1";

@Override
Expand Down Expand Up @@ -63,12 +64,15 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas) {
.put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
.put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true)
.put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, REPOSITORY_NAME)
.put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, TRANSLOG_REPOSITORY_NAME)
.build();
}

protected void deleteRepo() {
logger.info("--> Deleting the repository={}", REPOSITORY_NAME);
assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME));
logger.info("--> Deleting the repository={}", TRANSLOG_REPOSITORY_NAME);
assertAcked(clusterAdmin().prepareDeleteRepository(TRANSLOG_REPOSITORY_NAME));
}

protected String setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) {
Expand All @@ -88,6 +92,8 @@ protected String setup(Path repoLocation, double ioFailureRate, String skipExcep
.put("skip_exception_on_blobs", skipExceptionBlobList)
.put("max_failure_number", maxFailure)
);
logger.info("--> Creating repository={} at the path={}", TRANSLOG_REPOSITORY_NAME, repoLocation);
createRepository(TRANSLOG_REPOSITORY_NAME, "mock", Settings.builder().put("location", repoLocation));

String dataNodeName = internalCluster().startDataOnlyNodes(1).get(0);
createIndex(INDEX_NAME);
Expand Down
27 changes: 0 additions & 27 deletions ...nalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import org.opensearch.test.OpenSearchIntegTestCase;

import static org.hamcrest.Matchers.containsString;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED;
import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;

Expand All @@ -33,31 +32,6 @@ protected Settings nodeSettings(int nodeOriginal) {
return builder.build();
}

@Override
public void testRemoteStoreTranslogDisabledByUser() throws Exception {
Settings settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
.put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false)
.build();
assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get());
GetIndexResponse getIndexResponse = client().admin()
.indices()
.getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true))
.get();
Settings indexSettings = getIndexResponse.settings().get("test-idx-1");
verifyRemoteStoreIndexSettings(
indexSettings,
"true",
"my-segment-repo-1",
"false",
null,
ReplicationType.SEGMENT.toString(),
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL
);
}

@Override
public void testDefaultRemoteStoreNoUserOverride() throws Exception {
Settings settings = Settings.builder()
Expand Down Expand Up @@ -90,7 +64,6 @@ public void testDefaultRemoteStoreNoUserOverrideExceptReplicationTypeSegment() t
indexSettings,
"true",
"my-segment-repo-1",
"true",
"my-translog-repo-1",
ReplicationType.SEGMENT.toString(),
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL
Expand Down
150 changes: 2 additions & 148 deletions server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,11 @@
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY;
import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_REPOSITORY_SETTING;
import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING;
import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING;
import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING;
import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING;
import static org.opensearch.indices.IndicesService.CLUSTER_SETTING_REPLICATION_TYPE;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
Expand All @@ -53,7 +51,6 @@ protected Settings nodeSettings(int nodeOriginal) {
.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT)
.put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true)
.put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1")
.put(CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING.getKey(), true)
.put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1")
.put(settings);
return builder.build();
Expand Down Expand Up @@ -100,7 +97,6 @@ public void testDefaultRemoteStoreNoUserOverride() throws Exception {
indexSettings,
"true",
"my-segment-repo-1",
"true",
"my-translog-repo-1",
ReplicationType.SEGMENT.toString(),
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL
Expand All @@ -124,7 +120,6 @@ public void testRemoteStoreDisabledByUser() throws Exception {
"false",
null,
null,
null,
client().settings().get(CLUSTER_SETTING_REPLICATION_TYPE),
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL
);
Expand Down Expand Up @@ -159,10 +154,7 @@ public void testRemoteStoreEnabledByUserWithoutRemoteRepoIllegalArgumentExceptio
IllegalArgumentException.class,
() -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()
);
assertThat(
exc.getMessage(),
containsString("Setting index.remote_store.repository should be provided with non-empty repository ID")
);
assertThat(exc.getMessage(), containsString("Setting index.remote_store.repository should be provided with non-empty repository ID"));
}

public void testReplicationTypeDocumentByUser() throws Exception {
Expand All @@ -182,7 +174,6 @@ public void testReplicationTypeDocumentByUser() throws Exception {
null,
null,
null,
null,
ReplicationType.DOCUMENT.toString(),
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL
);
Expand Down Expand Up @@ -223,36 +214,12 @@ public void testRemoteStoreEnabledByUserWithRemoteRepo() throws Exception {
indexSettings,
"true",
"my-custom-repo",
"true",
"my-translog-repo-1",
ReplicationType.SEGMENT.toString(),
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL
);
}

public void testRemoteStoreTranslogDisabledByUser() throws Exception {
Settings settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false)
.build();
assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get());
GetIndexResponse getIndexResponse = client().admin()
.indices()
.getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true))
.get();
Settings indexSettings = getIndexResponse.settings().get("test-idx-1");
verifyRemoteStoreIndexSettings(
indexSettings,
"true",
"my-segment-repo-1",
"false",
null,
ReplicationType.SEGMENT.toString(),
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL
);
}

public void testRemoteStoreOverrideOnlyTranslogRepoIllegalArgumentException() throws Exception {
Settings settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
Expand All @@ -266,115 +233,7 @@ public void testRemoteStoreOverrideOnlyTranslogRepoIllegalArgumentException() th
assertThat(
exc.getMessage(),
containsString(
"Settings index.remote_store.translog.repository can only be set/enabled when index.remote_store.translog.enabled is set to true"
)
);
}

public void testRemoteStoreOverrideOnlyTranslogEnabled() throws Exception {
Settings settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true)
.build();

IllegalArgumentException exc = expectThrows(
IllegalArgumentException.class,
() -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()
);
assertThat(
exc.getMessage(),
containsString(
"Settings index.remote_store.translog.enabled can only be set/enabled when index.remote_store.enabled is set to true"
)
);
}

public void testRemoteStoreOverrideOnlyTranslogEnabledAndRepo() throws Exception {
Settings settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true)
.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo")
.build();

IllegalArgumentException exc = expectThrows(
IllegalArgumentException.class,
() -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()
);
assertThat(
exc.getMessage(),
containsString(
"Settings index.remote_store.translog.enabled can only be set/enabled when index.remote_store.enabled is set to true"
)
);
}

public void testRemoteStoreOverrideTranslogDisabledCorrectly() throws Exception {
Settings settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
.put(SETTING_REMOTE_STORE_ENABLED, true)
.put(SETTING_REMOTE_STORE_REPOSITORY, "my-custom-repo")
.put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false)
.build();
assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get());
GetIndexResponse getIndexResponse = client().admin()
.indices()
.getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true))
.get();
Settings indexSettings = getIndexResponse.settings().get("test-idx-1");
verifyRemoteStoreIndexSettings(
indexSettings,
"true",
"my-custom-repo",
"false",
null,
ReplicationType.SEGMENT.toString(),
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL
);
}

public void testRemoteStoreOverrideTranslogDisabledWithTranslogRepoIllegalArgumentException() throws Exception {
Settings settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
.put(SETTING_REMOTE_STORE_ENABLED, true)
.put(SETTING_REMOTE_STORE_REPOSITORY, "my-custom-repo")
.put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false)
.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo")
.build();
IllegalArgumentException exc = expectThrows(
IllegalArgumentException.class,
() -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()
);
assertThat(
exc.getMessage(),
containsString(
"Settings index.remote_store.translog.repository can only be set/enabled when index.remote_store.translog.enabled is set to true"
)
);
}

public void testRemoteStoreOverrideOnlyTranslogRepoWithRemoteStoreEnabledIllegalArgumentException() throws Exception {
Settings settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
.put(SETTING_REMOTE_STORE_ENABLED, true)
.put(SETTING_REMOTE_STORE_REPOSITORY, "my-custom-repo")
.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo")
.build();
IllegalArgumentException exc = expectThrows(
IllegalArgumentException.class,
() -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()
);
assertThat(
exc.getMessage(),
containsString(
"Settings index.remote_store.translog.repository can only be set/enabled when index.remote_store.translog.enabled is set to true"
"Settings index.remote_store.translog.repository can only be set/enabled when index.remote_store.enabled is set to true"
)
);
}
Expand All @@ -386,7 +245,6 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception {
.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
.put(SETTING_REMOTE_STORE_ENABLED, true)
.put(SETTING_REMOTE_STORE_REPOSITORY, "my-custom-repo")
.put(SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true)
.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo")
.build();
assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get());
Expand All @@ -399,7 +257,6 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception {
indexSettings,
"true",
"my-custom-repo",
"true",
"my-custom-repo",
ReplicationType.SEGMENT.toString(),
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL
Expand All @@ -423,7 +280,6 @@ public void testRemoteStoreOverrideReplicationTypeIndexSettings() throws Excepti
null,
null,
null,
null,
ReplicationType.DOCUMENT.toString(),
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL
);
Expand All @@ -433,15 +289,13 @@ protected void verifyRemoteStoreIndexSettings(
Settings indexSettings,
String isRemoteSegmentEnabled,
String remoteSegmentRepo,
String isRemoteTranslogEnabled,
String remoteTranslogRepo,
String replicationType,
TimeValue translogBufferInterval
) {
assertEquals(replicationType, indexSettings.get(SETTING_REPLICATION_TYPE));
assertEquals(isRemoteSegmentEnabled, indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_STORE_REPOSITORY));
assertEquals(isRemoteTranslogEnabled, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_ENABLED));
assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
assertEquals(translogBufferInterval, INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings));
}
Expand Down
Loading

0 comments on commit 7bac123

Please sign in to comment.