From f9b7e1bc26980a80b816c6c4bcd335d29109d57d Mon Sep 17 00:00:00 2001 From: Mohit Godwani Date: Mon, 2 Sep 2024 16:45:50 +0530 Subject: [PATCH 1/6] Add remote publication test with context Signed-off-by: Mohit Godwani --- .../remote/RemoteStatePublicationIT.java | 7 +++ .../RemoteStoreBaseIntegTestCase.java | 51 ++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index f237a81942e1a..eac4f39da48e3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -13,6 +13,8 @@ import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.client.Client; +import org.opensearch.cluster.applicationtemplates.SystemTemplatesService; +import org.opensearch.cluster.metadata.Context; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; @@ -67,6 +69,8 @@ public void setup() { protected Settings featureFlagSettings() { return Settings.builder() .put(super.featureFlagSettings()) + .put(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES, Boolean.TRUE.toString()) + .put(SystemTemplatesService.SETTING_APPLICATION_BASED_CONFIGURATION_TEMPLATES_ENABLED.getKey(), Boolean.TRUE.toString()) .put(FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL, isRemotePublicationEnabled) .build(); } @@ -142,6 +146,9 @@ public void testPublication() throws Exception { .metadata() .settings() .get(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()); + + assertEquals(new Context("testcontext"), response.getState() + .metadata().indices().get(INDEX_NAME).context()); assertEquals("10mb", refreshSetting); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index ba06bb463e5a8..045ffce4d94f7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -21,6 +21,11 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.WriteRequest; +import org.opensearch.cluster.applicationtemplates.ClusterStateSystemTemplateLoader; +import org.opensearch.cluster.applicationtemplates.SystemTemplate; +import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata; +import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata; +import org.opensearch.cluster.metadata.Context; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; @@ -28,6 +33,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexService; @@ -46,6 +52,8 @@ import org.junit.After; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; @@ -56,6 +64,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -76,6 +85,7 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { protected static final String REFRESHED_OR_FLUSHED_OPERATIONS = "refreshed-or-flushed-operations"; protected static final String MAX_SEQ_NO_TOTAL = "max-seq-no-total"; protected static final String MAX_SEQ_NO_REFRESHED_OR_FLUSHED = "max-seq-no-refreshed-or-flushed"; + protected static final String CONTEXT_NAME = "testcontext"; protected Path segmentRepoPath; protected Path translogRepoPath; @@ -352,9 +362,17 @@ protected void restore(boolean restoreAllShards, String... indices) { protected void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) { internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); + + // Adding context template to the cluster + + addTemplateForContext(CONTEXT_NAME); internalCluster().startDataOnlyNodes(numDataOnlyNodes); for (String index : indices.split(",")) { - createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); + // Ensure index is created with additional metadata field. + assertAcked(prepareCreate(index) + .setSettings(remoteStoreIndexSettings(replicaCount, shardCount)) + .setContext(new Context("testcontext"))); + ensureYellowAndNoInitializingShards(index); ensureGreen(index); } @@ -375,4 +393,35 @@ protected void prepareCluster( ensureGreen(index); } } + + private void addTemplateForContext(String contextName) { + try { + String templateContent = "{\n" + + " \"template\": {\n" + + " \"settings\": {\n" + + " \"index.merge.policy\": \"log_byte_size\"\n" + + " }\n" + + " },\n" + + " \"_meta\": {\n" + + " \"_type\": \"@abc_template\",\n" + + " \"_version\": 1\n" + + " },\n" + + " \"version\": 1\n" + + "}\n"; + + ClusterStateSystemTemplateLoader loader = new ClusterStateSystemTemplateLoader( + internalCluster().clusterManagerClient(), + () -> internalCluster().getInstance(ClusterService.class).state() + ); + loader.loadTemplate( + new SystemTemplate( + BytesReference.fromByteBuffer(ByteBuffer.wrap(templateContent.getBytes(StandardCharsets.UTF_8))), + SystemTemplateMetadata.fromComponentTemplateInfo(contextName, 1L), + new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L) + ) + ); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } From 62d10e7c12e271d744176ef3e7a2fd1f6829c49a Mon Sep 17 00:00:00 2001 From: Mohit Godwani Date: Tue, 3 Sep 2024 11:29:35 +0530 Subject: [PATCH 2/6] Add remote state test for context field in metadata Signed-off-by: Mohit Godwani --- .../gateway/remote/RemoteStatePublicationIT.java | 15 +++++++++++---- .../remotestore/RemoteStoreBaseIntegTestCase.java | 13 ++++++++++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index eac4f39da48e3..c692f1cc3925b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -99,7 +99,7 @@ protected Settings nodeSettings(int nodeOrdinal) { public void testPublication() throws Exception { // create cluster with multi node (3 master + 2 data) - prepareCluster(3, 2, INDEX_NAME, 1, 2); + prepareClusterWithDefaultContext(3, 2, INDEX_NAME, 1, 2); ensureStableCluster(5); ensureGreen(INDEX_NAME); // update settings on a random node @@ -141,15 +141,22 @@ public void testPublication() throws Exception { Settings settings = clusterService().getSettings(); logger.info("settings : {}", settings); for (Client client : clients()) { - ClusterStateResponse response = client.admin().cluster().prepareState().clear().setMetadata(true).get(); + ClusterStateResponse response = client.admin().cluster().prepareState().clear() + .setMetadata(true) + .setLocal(true) + .get(); + String refreshSetting = response.getState() .metadata() .settings() .get(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()); - assertEquals(new Context("testcontext"), response.getState() - .metadata().indices().get(INDEX_NAME).context()); assertEquals("10mb", refreshSetting); + + + // Verify context is present in metadata + assertEquals(new Context(CONTEXT_NAME), response.getState() + .metadata().indices().get(INDEX_NAME).context()); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 045ffce4d94f7..690b249897886 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -362,11 +362,22 @@ protected void restore(boolean restoreAllShards, String... indices) { protected void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) { internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); + internalCluster().startDataOnlyNodes(numDataOnlyNodes); + for (String index : indices.split(",")) { + createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); + ensureYellowAndNoInitializingShards(index); + ensureGreen(index); + } + } - // Adding context template to the cluster + protected void prepareClusterWithDefaultContext(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) { + internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); + // Adding context template to the cluster addTemplateForContext(CONTEXT_NAME); + internalCluster().startDataOnlyNodes(numDataOnlyNodes); + for (String index : indices.split(",")) { // Ensure index is created with additional metadata field. assertAcked(prepareCreate(index) From 4d13589db4d1b3f2b2c0774d6ace2dd611c9927c Mon Sep 17 00:00:00 2001 From: Mohit Godwani Date: Tue, 3 Sep 2024 12:12:51 +0530 Subject: [PATCH 3/6] Apply spotless Signed-off-by: Mohit Godwani --- .../gateway/remote/RemoteStatePublicationIT.java | 9 ++------- .../remotestore/RemoteStoreBaseIntegTestCase.java | 14 ++++++++++---- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index c692f1cc3925b..708ea9ea9b08e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -141,10 +141,7 @@ public void testPublication() throws Exception { Settings settings = clusterService().getSettings(); logger.info("settings : {}", settings); for (Client client : clients()) { - ClusterStateResponse response = client.admin().cluster().prepareState().clear() - .setMetadata(true) - .setLocal(true) - .get(); + ClusterStateResponse response = client.admin().cluster().prepareState().clear().setMetadata(true).setLocal(true).get(); String refreshSetting = response.getState() .metadata() @@ -153,10 +150,8 @@ public void testPublication() throws Exception { assertEquals("10mb", refreshSetting); - // Verify context is present in metadata - assertEquals(new Context(CONTEXT_NAME), response.getState() - .metadata().indices().get(INDEX_NAME).context()); + assertEquals(new Context(CONTEXT_NAME), response.getState().metadata().indices().get(INDEX_NAME).context()); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 690b249897886..faf58a9ec82ee 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -370,7 +370,13 @@ protected void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, } } - protected void prepareClusterWithDefaultContext(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) { + protected void prepareClusterWithDefaultContext( + int numClusterManagerNodes, + int numDataOnlyNodes, + String indices, + int replicaCount, + int shardCount + ) { internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); // Adding context template to the cluster @@ -380,9 +386,9 @@ protected void prepareClusterWithDefaultContext(int numClusterManagerNodes, int for (String index : indices.split(",")) { // Ensure index is created with additional metadata field. - assertAcked(prepareCreate(index) - .setSettings(remoteStoreIndexSettings(replicaCount, shardCount)) - .setContext(new Context("testcontext"))); + assertAcked( + prepareCreate(index).setSettings(remoteStoreIndexSettings(replicaCount, shardCount)).setContext(new Context("testcontext")) + ); ensureYellowAndNoInitializingShards(index); ensureGreen(index); From d67fafaaa48da609ea44aa50e44f446fe7d5790c Mon Sep 17 00:00:00 2001 From: Mohit Godwani Date: Tue, 3 Sep 2024 14:55:03 +0530 Subject: [PATCH 4/6] Trigger Build Signed-off-by: Mohit Godwani From a13b19707643fe9db1010229419a5325a81a8825 Mon Sep 17 00:00:00 2001 From: Mohit Godwani Date: Tue, 3 Sep 2024 15:06:34 +0530 Subject: [PATCH 5/6] Trigger Build Signed-off-by: Mohit Godwani From c53b798c2153cfa1d3f739f26ee73cb5fa59876b Mon Sep 17 00:00:00 2001 From: Mohit Godwani Date: Tue, 3 Sep 2024 16:31:56 +0530 Subject: [PATCH 6/6] Trigger Build Signed-off-by: Mohit Godwani