From 97731589e9c2f3c45321d5172cb7f8a5718c4097 Mon Sep 17 00:00:00 2001 From: bansvaru Date: Wed, 7 Jun 2023 20:49:40 +0530 Subject: [PATCH] fix translog base path and update tests --- .../index/translog/RemoteFsTranslog.java | 2 +- .../transfer/TranslogTransferManager.java | 4 +-- .../index/translog/RemoteFSTranslogTests.java | 25 +++++++++++++------ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 5d760fac28f92..190ca6948f42a 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -65,7 +65,7 @@ public class RemoteFsTranslog extends Translog { private final SetOnce olderPrimaryCleaned = new SetOnce<>(); private static final int REMOTE_DELETION_PERMITS = 2; - private static final String TRANSLOG = "translog"; + public static final String TRANSLOG = "translog"; // Semaphore used to allow only single remote generation to happen at a time private final Semaphore remoteGenerationDeletionPermits = new Semaphore(REMOTE_DELETION_PERMITS); diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index 2facd0e0c2214..fc4192a4efe0e 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -68,7 +68,7 @@ public TranslogTransferManager( ) { this.shardId = shardId; this.transferService = transferService; - this.remoteBaseTransferPath = remoteBaseTransferPath; + this.remoteBaseTransferPath = remoteBaseTransferPath.add("data"); this.remoteMetadataTransferPath = remoteBaseTransferPath.add(METADATA_DIR); this.fileTransferTracker = fileTransferTracker; } @@ -110,7 +110,7 @@ public boolean transferSnapshot(TransferSnapshot transferSnapshot, TranslogTrans fileSnapshot -> transferService.uploadBlobAsync( ThreadPool.Names.TRANSLOG_TRANSFER, fileSnapshot, - remoteBaseTransferPath.add("data").add(String.valueOf(fileSnapshot.getPrimaryTerm())), + remoteBaseTransferPath.add(String.valueOf(fileSnapshot.getPrimaryTerm())), latchedActionListener ) ); diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java index 24dae6f5be9ab..57e193d8b886e 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java @@ -93,6 +93,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.opensearch.common.util.BigArrays.NON_RECYCLING_INSTANCE; +import static org.opensearch.index.translog.RemoteFsTranslog.TRANSLOG; import static org.opensearch.index.translog.SnapshotMatchers.containsOperationsInAnyOrder; import static org.opensearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy; @@ -484,13 +485,13 @@ public void testSimpleOperationsUpload() throws Exception { assertEquals(6, translog.allUploaded().size()); Set mdFiles = blobStoreTransferService.listAll( - repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add("metadata") + getTranslogDirectory().add("metadata") ); assertEquals(2, mdFiles.size()); logger.info("All md files {}", mdFiles); Set tlogFiles = blobStoreTransferService.listAll( - repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(String.valueOf(primaryTerm.get())) + getTranslogDirectory().add("data").add(String.valueOf(primaryTerm.get())) ); logger.info("All data files {}", tlogFiles); @@ -498,6 +499,8 @@ public void testSimpleOperationsUpload() throws Exception { BlobPath path = repository.basePath() .add(shardId.getIndex().getUUID()) .add(String.valueOf(shardId.id())) + .add(TRANSLOG) + .add("data") .add(String.valueOf(primaryTerm.get())); for (TranslogReader reader : translog.readers) { final long readerGeneration = reader.getGeneration(); @@ -537,6 +540,8 @@ public void testSimpleOperationsUpload() throws Exception { repository.basePath() .add(shardId.getIndex().getUUID()) .add(String.valueOf(shardId.id())) + .add(TRANSLOG) + .add("data") .add(String.valueOf(primaryTerm.get())) ).size() ); @@ -555,6 +560,8 @@ public void testSimpleOperationsUpload() throws Exception { repository.basePath() .add(shardId.getIndex().getUUID()) .add(String.valueOf(shardId.id())) + .add(TRANSLOG) + .add("data") .add(String.valueOf(primaryTerm.get())) ).size() ); @@ -587,7 +594,7 @@ public void testMetadataFileDeletion() throws Exception { () -> assertEquals( 2, blobStoreTransferService.listAll( - repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR) + getTranslogDirectory().add(METADATA_DIR) ).size() ) ); @@ -603,7 +610,7 @@ public void testMetadataFileDeletion() throws Exception { () -> assertEquals( 1 + moreDocs, blobStoreTransferService.listAll( - repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR) + getTranslogDirectory().add(METADATA_DIR) ).size() ) ); @@ -623,7 +630,7 @@ public void testMetadataFileDeletion() throws Exception { () -> assertEquals( 2, blobStoreTransferService.listAll( - repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR) + getTranslogDirectory().add(METADATA_DIR) ).size() ) ); @@ -643,7 +650,7 @@ public void testMetadataFileDeletion() throws Exception { // Check all metadata files corresponds to old primary term Set mdFileNames = blobStoreTransferService.listAll( - repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR) + getTranslogDirectory().add(METADATA_DIR) ); assertTrue(mdFileNames.stream().allMatch(name -> name.startsWith(String.valueOf(oldPrimaryTerm).concat("__")))); @@ -659,7 +666,7 @@ public void testMetadataFileDeletion() throws Exception { // Check that all metadata files are belonging now to the new primary mdFileNames = blobStoreTransferService.listAll( - repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR) + getTranslogDirectory().add(METADATA_DIR) ); assertTrue(mdFileNames.stream().allMatch(name -> name.startsWith(String.valueOf(newPrimaryTerm).concat("__")))); @@ -671,6 +678,10 @@ public void testMetadataFileDeletion() throws Exception { } } + private BlobPath getTranslogDirectory() { + return repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(TRANSLOG); + } + private Long populateTranslogOps(boolean withMissingOps) throws IOException { long minSeqNo = SequenceNumbers.NO_OPS_PERFORMED; long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;