fix unit tests

Signed-off-by: bansvaru <[email protected]>
opensearch-project · Jul 19, 2023 · 01809a0 · 01809a0
1 parent 4999d3e
commit 01809a0
Show file tree

Hide file tree

Showing 8 changed files with 40 additions and 28 deletions.
diff --git a/...rTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/...rTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java
@@ -62,6 +62,7 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas) {
             .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
             .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true)
             .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, REPOSITORY_NAME)
+            .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME)
             .build();
     }
 

diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -4585,7 +4585,7 @@ public void close() throws IOException {
         onSettingsChanged();
     }
 
-    private void syncRemoteTranslogAndUpdateGlobalCheckpoint() throws IOException {
+    public void syncRemoteTranslogAndUpdateGlobalCheckpoint() throws IOException {
         syncTranslogFilesFromRemoteTranslog();
         loadGlobalCheckpointToReplicationTracker();
     }

diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
@@ -97,7 +97,7 @@ public boolean transferSnapshot(TransferSnapshot transferSnapshot, TranslogTrans
         Set<TransferFileSnapshot> toUpload = new HashSet<>(transferSnapshot.getTranslogTransferMetadata().getCount());
         try {
             toUpload.addAll(fileTransferTracker.exclusionFilter(transferSnapshot.getTranslogFileSnapshots()));
-            toUpload.addAll(fileTransferTracker.exclusionFilter((transferSnapshot.getCheckpointFileSnapshots())));
+            // toUpload.addAll(fileTransferTracker.exclusionFilter((transferSnapshot.getCheckpointFileSnapshots())));
             if (toUpload.isEmpty()) {
                 logger.trace("Nothing to upload for transfer");
                 translogTransferListener.onUploadComplete(transferSnapshot);
@@ -363,32 +363,29 @@ public void onFailure(Exception e) {
     }
 
     public void deleteStaleTranslogMetadataFilesAsync(Runnable onCompletion) {
-        try {
-            transferService.listAllInSortedOrderAsync(
-                ThreadPool.Names.REMOTE_PURGE,
-                remoteMetadataTransferPath,
-                Integer.MAX_VALUE,
-                new ActionListener<>() {
-                    @Override
-                    public void onResponse(List<BlobMetadata> blobMetadata) {
-                        List<String> sortedMetadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList());
-                        if (sortedMetadataFiles.size() <= 1) {
-                            logger.trace("Remote Metadata file count is {}, so skipping deletion", sortedMetadataFiles.size());
-                            onCompletion.run();
-                            return;
-                        }
-                        List<String> metadataFilesToDelete = sortedMetadataFiles.subList(1, sortedMetadataFiles.size());
-                        logger.trace("Deleting remote translog metadata files {}", metadataFilesToDelete);
-                        deleteMetadataFilesAsync(metadataFilesToDelete, onCompletion);
-                    }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        logger.error("Exception occurred while listing translog metadata files from remote store", e);
-                        onCompletion.run();
-                    }
+        ActionListener<List<BlobMetadata>> al = new ActionListener<>() {
+            @Override
+            public void onResponse(List<BlobMetadata> blobMetadata) {
+                List<String> sortedMetadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList());
+                if (sortedMetadataFiles.size() <= 1) {
+                    logger.trace("Remote Metadata file count is {}, so skipping deletion", sortedMetadataFiles.size());
+                    onCompletion.run();
+                    return;
                 }
-            );
+                List<String> metadataFilesToDelete = sortedMetadataFiles.subList(1, sortedMetadataFiles.size());
+                logger.trace("Deleting remote translog metadata files {}", metadataFilesToDelete);
+                deleteMetadataFilesAsync(metadataFilesToDelete, onCompletion);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                logger.error("Exception occurred while listing translog metadata files from remote store", e);
+                onCompletion.run();
+            }
+        };
+
+        try {
+            transferService.listAllInSortedOrderAsync(ThreadPool.Names.REMOTE_PURGE, remoteMetadataTransferPath, Integer.MAX_VALUE, al);
         } catch (Exception e) {
             logger.error("Exception occurred while listing translog metadata files from remote store", e);
             onCompletion.run();

diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
@@ -2809,6 +2809,7 @@ public void testSyncSegmentsFromGivenRemoteSegmentStore() throws IOException {
                 .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
                 .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true)
                 .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, remoteStorePath + "__test")
+                .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStorePath + "__test")
                 .build(),
             new InternalEngineFactory()
         );
@@ -2847,6 +2848,8 @@ public void testSyncSegmentsFromGivenRemoteSegmentStore() throws IOException {
             Settings.builder()
                 .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
                 .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true)
+                .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, remoteStorePath + "__test1")
+                .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStorePath + "__test1")
                 .build(),
             new InternalEngineFactory()
         );
@@ -2887,6 +2890,8 @@ public void testRestoreShardFromRemoteStore(boolean performFlush) throws IOExcep
             Settings.builder()
                 .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
                 .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true)
+                .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, "temp-fs")
+                .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "temp-fs")
                 .build(),
             new InternalEngineFactory()
         );

diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java
@@ -58,6 +58,8 @@ public void setup(boolean primary, int numberOfDocs) throws IOException {
             primary,
             Settings.builder()
                 .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true)
+                .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, "temp-fs")
+                .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "temp-fs")
                 .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
                 .build(),
             new InternalEngineFactory()
@@ -386,6 +388,8 @@ private Tuple<RemoteStoreRefreshListener, RemoteRefreshSegmentPressureService> m
             true,
             Settings.builder()
                 .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true)
+                .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, "temp-fs")
+                .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "temp-fs")
                 .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
                 .build(),
             new InternalEngineFactory()

diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
@@ -892,6 +892,8 @@ private void testNRTReplicaWithRemoteStorePromotedAsPrimary(boolean performFlush
         Settings settings = Settings.builder()
             .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
             .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true)
+            .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, "temp-fs")
+            .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "temp-fs")
             .build();
 
         try (ReplicationGroup shards = createGroup(1, settings, indexMapping, new NRTReplicationEngineFactory(), createTempDir())) {

diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java
@@ -217,6 +217,7 @@ private Settings getRemoteStoreBackedIndexSettings(String remoteStoreRepo) {
             .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
             .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true)
             .put(IndexMetadata.SETTING_REMOTE_STORE_REPOSITORY, remoteStoreRepo)
+            .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStoreRepo + "translog")
             .build();
     }
 
@@ -365,6 +366,7 @@ public void testGetRemoteStoreShallowCopyShardMetadata() throws IOException {
             .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings()))
             .build();
         createRepository(client, remoteStoreRepositoryName, remoteStoreRepoSettings);
+        createRepository(client, remoteStoreRepositoryName + "translog", remoteStoreRepoSettings);
 
         logger.info("--> creating a remote store enabled index and indexing documents");
         final String remoteStoreIndexName = "test-rs-idx";
@@ -438,6 +440,7 @@ public void testRetrieveShallowCopySnapshotCase2() throws IOException {
 
         logger.info("-->  creating remote store repository");
         createRepository(client, remoteStoreRepositoryName);
+        createRepository(client, remoteStoreRepositoryName + "translog");
 
         logger.info("--> creating an index and indexing documents");
         final String indexName = "test-idx";

diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java
@@ -651,7 +651,7 @@ protected RepositoriesService createRepositoriesService() {
         BlobStore blobStore = Mockito.mock(BlobStore.class);
         BlobContainer blobContainer = Mockito.mock(BlobContainer.class);
         doAnswer(invocation -> {
-            LatchedActionListener<List<BlobMetadata>> listener = invocation.getArgument(3);
+            ActionListener<List<BlobMetadata>> listener = invocation.getArgument(3);
             listener.onResponse(new ArrayList<>());
             return null;
         }).when(blobContainer)