From 670b02f32cea04eff7dbcbd9a21c2ce3c632b3f1 Mon Sep 17 00:00:00 2001 From: bansvaru Date: Wed, 7 Jun 2023 18:58:02 +0530 Subject: [PATCH] fix translog path and UTs --- .idea/inspectionProfiles/Project_Default.xml | 3 +++ .idea/runConfigurations/Debug_OpenSearch.xml | 6 ++++- gradle/run.gradle | 5 ++++ .../index/translog/RemoteFsTranslog.java | 2 +- .../transfer/TranslogTransferManager.java | 4 +-- .../index/translog/RemoteFSTranslogTests.java | 25 +++++++++++++------ .../bootstrap/BootstrapForTesting.java | 4 +-- 7 files changed, 36 insertions(+), 13 deletions(-) diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml index 89e81c64fe205..ebdef86202ac4 100644 --- a/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -1,6 +1,9 @@ + \ No newline at end of file diff --git a/gradle/run.gradle b/gradle/run.gradle index 639479e97d28f..42450946e172e 100644 --- a/gradle/run.gradle +++ b/gradle/run.gradle @@ -39,6 +39,11 @@ testClusters { testDistribution = 'archive' if (numZones > 1) numberOfZones = numZones if (numNodes > 1) numberOfNodes = numNodes + systemProperty 'opensearch.experimental.feature.replication_type.enabled', 'true' + systemProperty 'opensearch.experimental.feature.remote_store.enabled', 'true' + systemProperty 'opensearch.experimental.feature.segment_replication_experimental.enabled', 'true' + setting 'remote_store.segment.pressure.enabled', 'true' + } } diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 5d760fac28f92..190ca6948f42a 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -65,7 +65,7 @@ public class RemoteFsTranslog extends Translog { private final SetOnce olderPrimaryCleaned = new SetOnce<>(); private static final int REMOTE_DELETION_PERMITS = 2; - private static final String TRANSLOG = "translog"; + public static final String TRANSLOG = "translog"; // Semaphore used to allow only single remote generation to happen at a time private final Semaphore remoteGenerationDeletionPermits = new Semaphore(REMOTE_DELETION_PERMITS); diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index 2facd0e0c2214..fc4192a4efe0e 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -68,7 +68,7 @@ public TranslogTransferManager( ) { this.shardId = shardId; this.transferService = transferService; - this.remoteBaseTransferPath = remoteBaseTransferPath; + this.remoteBaseTransferPath = remoteBaseTransferPath.add("data"); this.remoteMetadataTransferPath = remoteBaseTransferPath.add(METADATA_DIR); this.fileTransferTracker = fileTransferTracker; } @@ -110,7 +110,7 @@ public boolean transferSnapshot(TransferSnapshot transferSnapshot, TranslogTrans fileSnapshot -> transferService.uploadBlobAsync( ThreadPool.Names.TRANSLOG_TRANSFER, fileSnapshot, - remoteBaseTransferPath.add("data").add(String.valueOf(fileSnapshot.getPrimaryTerm())), + remoteBaseTransferPath.add(String.valueOf(fileSnapshot.getPrimaryTerm())), latchedActionListener ) ); diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java index 24dae6f5be9ab..57e193d8b886e 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java @@ -93,6 +93,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.opensearch.common.util.BigArrays.NON_RECYCLING_INSTANCE; +import static org.opensearch.index.translog.RemoteFsTranslog.TRANSLOG; import static org.opensearch.index.translog.SnapshotMatchers.containsOperationsInAnyOrder; import static org.opensearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy; @@ -484,13 +485,13 @@ public void testSimpleOperationsUpload() throws Exception { assertEquals(6, translog.allUploaded().size()); Set mdFiles = blobStoreTransferService.listAll( - repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add("metadata") + getTranslogDirectory().add("metadata") ); assertEquals(2, mdFiles.size()); logger.info("All md files {}", mdFiles); Set tlogFiles = blobStoreTransferService.listAll( - repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(String.valueOf(primaryTerm.get())) + getTranslogDirectory().add("data").add(String.valueOf(primaryTerm.get())) ); logger.info("All data files {}", tlogFiles); @@ -498,6 +499,8 @@ public void testSimpleOperationsUpload() throws Exception { BlobPath path = repository.basePath() .add(shardId.getIndex().getUUID()) .add(String.valueOf(shardId.id())) + .add(TRANSLOG) + .add("data") .add(String.valueOf(primaryTerm.get())); for (TranslogReader reader : translog.readers) { final long readerGeneration = reader.getGeneration(); @@ -537,6 +540,8 @@ public void testSimpleOperationsUpload() throws Exception { repository.basePath() .add(shardId.getIndex().getUUID()) .add(String.valueOf(shardId.id())) + .add(TRANSLOG) + .add("data") .add(String.valueOf(primaryTerm.get())) ).size() ); @@ -555,6 +560,8 @@ public void testSimpleOperationsUpload() throws Exception { repository.basePath() .add(shardId.getIndex().getUUID()) .add(String.valueOf(shardId.id())) + .add(TRANSLOG) + .add("data") .add(String.valueOf(primaryTerm.get())) ).size() ); @@ -587,7 +594,7 @@ public void testMetadataFileDeletion() throws Exception { () -> assertEquals( 2, blobStoreTransferService.listAll( - repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR) + getTranslogDirectory().add(METADATA_DIR) ).size() ) ); @@ -603,7 +610,7 @@ public void testMetadataFileDeletion() throws Exception { () -> assertEquals( 1 + moreDocs, blobStoreTransferService.listAll( - repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR) + getTranslogDirectory().add(METADATA_DIR) ).size() ) ); @@ -623,7 +630,7 @@ public void testMetadataFileDeletion() throws Exception { () -> assertEquals( 2, blobStoreTransferService.listAll( - repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR) + getTranslogDirectory().add(METADATA_DIR) ).size() ) ); @@ -643,7 +650,7 @@ public void testMetadataFileDeletion() throws Exception { // Check all metadata files corresponds to old primary term Set mdFileNames = blobStoreTransferService.listAll( - repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR) + getTranslogDirectory().add(METADATA_DIR) ); assertTrue(mdFileNames.stream().allMatch(name -> name.startsWith(String.valueOf(oldPrimaryTerm).concat("__")))); @@ -659,7 +666,7 @@ public void testMetadataFileDeletion() throws Exception { // Check that all metadata files are belonging now to the new primary mdFileNames = blobStoreTransferService.listAll( - repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR) + getTranslogDirectory().add(METADATA_DIR) ); assertTrue(mdFileNames.stream().allMatch(name -> name.startsWith(String.valueOf(newPrimaryTerm).concat("__")))); @@ -671,6 +678,10 @@ public void testMetadataFileDeletion() throws Exception { } } + private BlobPath getTranslogDirectory() { + return repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(TRANSLOG); + } + private Long populateTranslogOps(boolean withMissingOps) throws IOException { long minSeqNo = SequenceNumbers.NO_OPS_PERFORMED; long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED; diff --git a/test/framework/src/main/java/org/opensearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/opensearch/bootstrap/BootstrapForTesting.java index d46fcb224e8c2..841cc92eed0db 100644 --- a/test/framework/src/main/java/org/opensearch/bootstrap/BootstrapForTesting.java +++ b/test/framework/src/main/java/org/opensearch/bootstrap/BootstrapForTesting.java @@ -170,8 +170,8 @@ public boolean implies(ProtectionDomain domain, Permission permission) { }); // Create access control context for mocking PriviledgedMockMaker.createAccessControlContext(); - System.setSecurityManager(SecureSM.createTestSecureSM(getTrustedHosts())); - Security.selfTest(); +// System.setSecurityManager(SecureSM.createTestSecureSM(getTrustedHosts())); +// Security.selfTest(); // guarantee plugin classes are initialized first, in case they have one-time hacks. // this just makes unit testing more realistic