Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add test for remote state publication with new context field in metadata #15593

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
import org.opensearch.action.admin.cluster.state.ClusterStateResponse;
import org.opensearch.client.Client;
import org.opensearch.cluster.applicationtemplates.SystemTemplatesService;
import org.opensearch.cluster.metadata.Context;
import org.opensearch.common.blobstore.BlobPath;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.util.FeatureFlags;
Expand Down Expand Up @@ -67,6 +69,8 @@ public void setup() {
protected Settings featureFlagSettings() {
return Settings.builder()
.put(super.featureFlagSettings())
.put(FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES, Boolean.TRUE.toString())
.put(SystemTemplatesService.SETTING_APPLICATION_BASED_CONFIGURATION_TEMPLATES_ENABLED.getKey(), Boolean.TRUE.toString())
.put(FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL, isRemotePublicationEnabled)
.build();
}
Expand Down Expand Up @@ -95,7 +99,7 @@ protected Settings nodeSettings(int nodeOrdinal) {

public void testPublication() throws Exception {
// create cluster with multi node (3 master + 2 data)
prepareCluster(3, 2, INDEX_NAME, 1, 2);
prepareClusterWithDefaultContext(3, 2, INDEX_NAME, 1, 2);
ensureStableCluster(5);
ensureGreen(INDEX_NAME);
// update settings on a random node
Expand Down Expand Up @@ -137,12 +141,17 @@ public void testPublication() throws Exception {
Settings settings = clusterService().getSettings();
logger.info("settings : {}", settings);
for (Client client : clients()) {
ClusterStateResponse response = client.admin().cluster().prepareState().clear().setMetadata(true).get();
ClusterStateResponse response = client.admin().cluster().prepareState().clear().setMetadata(true).setLocal(true).get();

String refreshSetting = response.getState()
.metadata()
.settings()
.get(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey());

assertEquals("10mb", refreshSetting);

// Verify context is present in metadata
assertEquals(new Context(CONTEXT_NAME), response.getState().metadata().indices().get(INDEX_NAME).context());
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,19 @@
import org.opensearch.action.index.IndexResponse;
import org.opensearch.action.support.PlainActionFuture;
import org.opensearch.action.support.WriteRequest;
import org.opensearch.cluster.applicationtemplates.ClusterStateSystemTemplateLoader;
import org.opensearch.cluster.applicationtemplates.SystemTemplate;
import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata;
import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata;
import org.opensearch.cluster.metadata.Context;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.RepositoriesMetadata;
import org.opensearch.cluster.metadata.RepositoryMetadata;
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.UUIDs;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.index.Index;
import org.opensearch.index.IndexModule;
import org.opensearch.index.IndexService;
Expand All @@ -46,6 +52,8 @@
import org.junit.After;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
Expand All @@ -56,6 +64,7 @@
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
Expand All @@ -76,6 +85,7 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase {
protected static final String REFRESHED_OR_FLUSHED_OPERATIONS = "refreshed-or-flushed-operations";
protected static final String MAX_SEQ_NO_TOTAL = "max-seq-no-total";
protected static final String MAX_SEQ_NO_REFRESHED_OR_FLUSHED = "max-seq-no-refreshed-or-flushed";
protected static final String CONTEXT_NAME = "testcontext";

protected Path segmentRepoPath;
protected Path translogRepoPath;
Expand Down Expand Up @@ -360,6 +370,31 @@ protected void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes,
}
}

protected void prepareClusterWithDefaultContext(
int numClusterManagerNodes,
int numDataOnlyNodes,
String indices,
int replicaCount,
int shardCount
) {
internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes);

// Adding context template to the cluster
addTemplateForContext(CONTEXT_NAME);

internalCluster().startDataOnlyNodes(numDataOnlyNodes);

for (String index : indices.split(",")) {
// Ensure index is created with additional metadata field.
assertAcked(
prepareCreate(index).setSettings(remoteStoreIndexSettings(replicaCount, shardCount)).setContext(new Context("testcontext"))
);

ensureYellowAndNoInitializingShards(index);
ensureGreen(index);
}
}

protected void prepareCluster(
int numClusterManagerNodes,
int numDataOnlyNodes,
Expand All @@ -375,4 +410,35 @@ protected void prepareCluster(
ensureGreen(index);
}
}

private void addTemplateForContext(String contextName) {
try {
String templateContent = "{\n"
+ " \"template\": {\n"
+ " \"settings\": {\n"
+ " \"index.merge.policy\": \"log_byte_size\"\n"
+ " }\n"
+ " },\n"
+ " \"_meta\": {\n"
+ " \"_type\": \"@abc_template\",\n"
+ " \"_version\": 1\n"
+ " },\n"
+ " \"version\": 1\n"
+ "}\n";

ClusterStateSystemTemplateLoader loader = new ClusterStateSystemTemplateLoader(
internalCluster().clusterManagerClient(),
() -> internalCluster().getInstance(ClusterService.class).state()
);
loader.loadTemplate(
new SystemTemplate(
BytesReference.fromByteBuffer(ByteBuffer.wrap(templateContent.getBytes(StandardCharsets.UTF_8))),
SystemTemplateMetadata.fromComponentTemplateInfo(contextName, 1L),
new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L)
)
);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
Loading