fix translog base path and update tests

opensearch-project · Jun 7, 2023 · 9773158 · 9773158
1 parent 2ced9e2
commit 9773158
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 10 deletions.
diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
@@ -65,7 +65,7 @@ public class RemoteFsTranslog extends Translog {
     private final SetOnce<Boolean> olderPrimaryCleaned = new SetOnce<>();
 
     private static final int REMOTE_DELETION_PERMITS = 2;
-    private static final String TRANSLOG = "translog";
+    public static final String TRANSLOG = "translog";
 
     // Semaphore used to allow only single remote generation to happen at a time
     private final Semaphore remoteGenerationDeletionPermits = new Semaphore(REMOTE_DELETION_PERMITS);

diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
@@ -68,7 +68,7 @@ public TranslogTransferManager(
     ) {
         this.shardId = shardId;
         this.transferService = transferService;
-        this.remoteBaseTransferPath = remoteBaseTransferPath;
+        this.remoteBaseTransferPath = remoteBaseTransferPath.add("data");
         this.remoteMetadataTransferPath = remoteBaseTransferPath.add(METADATA_DIR);
         this.fileTransferTracker = fileTransferTracker;
     }
@@ -110,7 +110,7 @@ public boolean transferSnapshot(TransferSnapshot transferSnapshot, TranslogTrans
                 fileSnapshot -> transferService.uploadBlobAsync(
                     ThreadPool.Names.TRANSLOG_TRANSFER,
                     fileSnapshot,
-                    remoteBaseTransferPath.add("data").add(String.valueOf(fileSnapshot.getPrimaryTerm())),
+                    remoteBaseTransferPath.add(String.valueOf(fileSnapshot.getPrimaryTerm())),
                     latchedActionListener
                 )
             );

diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java
@@ -93,6 +93,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.opensearch.common.util.BigArrays.NON_RECYCLING_INSTANCE;
+import static org.opensearch.index.translog.RemoteFsTranslog.TRANSLOG;
 import static org.opensearch.index.translog.SnapshotMatchers.containsOperationsInAnyOrder;
 import static org.opensearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy;
 
@@ -484,20 +485,22 @@ public void testSimpleOperationsUpload() throws Exception {
         assertEquals(6, translog.allUploaded().size());
 
         Set<String> mdFiles = blobStoreTransferService.listAll(
-            repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add("metadata")
+            getTranslogDirectory().add("metadata")
         );
         assertEquals(2, mdFiles.size());
         logger.info("All md files {}", mdFiles);
 
         Set<String> tlogFiles = blobStoreTransferService.listAll(
-            repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(String.valueOf(primaryTerm.get()))
+            getTranslogDirectory().add("data").add(String.valueOf(primaryTerm.get()))
         );
         logger.info("All data files {}", tlogFiles);
 
         // assert content of ckp and tlog files
         BlobPath path = repository.basePath()
             .add(shardId.getIndex().getUUID())
             .add(String.valueOf(shardId.id()))
+            .add(TRANSLOG)
+            .add("data")
             .add(String.valueOf(primaryTerm.get()));
         for (TranslogReader reader : translog.readers) {
             final long readerGeneration = reader.getGeneration();
@@ -537,6 +540,8 @@ public void testSimpleOperationsUpload() throws Exception {
                     repository.basePath()
                         .add(shardId.getIndex().getUUID())
                         .add(String.valueOf(shardId.id()))
+                        .add(TRANSLOG)
+                        .add("data")
                         .add(String.valueOf(primaryTerm.get()))
                 ).size()
             );
@@ -555,6 +560,8 @@ public void testSimpleOperationsUpload() throws Exception {
                     repository.basePath()
                         .add(shardId.getIndex().getUUID())
                         .add(String.valueOf(shardId.id()))
+                        .add(TRANSLOG)
+                        .add("data")
                         .add(String.valueOf(primaryTerm.get()))
                 ).size()
             );
@@ -587,7 +594,7 @@ public void testMetadataFileDeletion() throws Exception {
             () -> assertEquals(
                 2,
                 blobStoreTransferService.listAll(
-                    repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR)
+                    getTranslogDirectory().add(METADATA_DIR)
                 ).size()
             )
         );
@@ -603,7 +610,7 @@ public void testMetadataFileDeletion() throws Exception {
             () -> assertEquals(
                 1 + moreDocs,
                 blobStoreTransferService.listAll(
-                    repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR)
+                    getTranslogDirectory().add(METADATA_DIR)
                 ).size()
             )
         );
@@ -623,7 +630,7 @@ public void testMetadataFileDeletion() throws Exception {
             () -> assertEquals(
                 2,
                 blobStoreTransferService.listAll(
-                    repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR)
+                    getTranslogDirectory().add(METADATA_DIR)
                 ).size()
             )
         );
@@ -643,7 +650,7 @@ public void testMetadataFileDeletion() throws Exception {
 
         // Check all metadata files corresponds to old primary term
         Set<String> mdFileNames = blobStoreTransferService.listAll(
-            repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR)
+            getTranslogDirectory().add(METADATA_DIR)
         );
         assertTrue(mdFileNames.stream().allMatch(name -> name.startsWith(String.valueOf(oldPrimaryTerm).concat("__"))));
 
@@ -659,7 +666,7 @@ public void testMetadataFileDeletion() throws Exception {
 
         // Check that all metadata files are belonging now to the new primary
         mdFileNames = blobStoreTransferService.listAll(
-            repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR)
+            getTranslogDirectory().add(METADATA_DIR)
         );
         assertTrue(mdFileNames.stream().allMatch(name -> name.startsWith(String.valueOf(newPrimaryTerm).concat("__"))));
 
@@ -671,6 +678,10 @@ public void testMetadataFileDeletion() throws Exception {
         }
     }
 
+    private BlobPath getTranslogDirectory() {
+        return repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(TRANSLOG);
+    }
+
     private Long populateTranslogOps(boolean withMissingOps) throws IOException {
         long minSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
         long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;