Fix deletion permits flow in RemoteFsTimestampAwareTranslog

server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java

@@ -215,21 +215,42 @@
                     logger.debug(() -> "generationsToBeDeleted = " + generationsToBeDeleted);
                     if (generationsToBeDeleted.isEmpty() == false) {
                         // Delete stale generations
-                        translogTransferManager.deleteGenerationAsync(
-                            primaryTermSupplier.getAsLong(),
-                            generationsToBeDeleted,
-                            remoteGenerationDeletionPermits::release
-                        );
+                        try {
+                            translogTransferManager.deleteGenerationAsync(
+                                primaryTermSupplier.getAsLong(),
+                                generationsToBeDeleted,
+                                remoteGenerationDeletionPermits::release
+                            );
+                        } catch (Exception e) {
+                            logger.error("Exception in delete generations flow", e);
+                            // Release permit that is meant for metadata files and return
+                            remoteGenerationDeletionPermits.release();
+                            assert remoteGenerationDeletionPermits.availablePermits() == REMOTE_DELETION_PERMITS : "Available permits "
+                                + remoteGenerationDeletionPermits.availablePermits()
+                                + " is not equal to "
+                                + REMOTE_DELETION_PERMITS;
+                            return;
+                        }
                     } else {
                         remoteGenerationDeletionPermits.release();
                     }
 
                     if (metadataFilesToBeDeleted.isEmpty() == false) {
                         // Delete stale metadata files
-                        translogTransferManager.deleteMetadataFilesAsync(
-                            metadataFilesToBeDeleted,
-                            remoteGenerationDeletionPermits::release
-                        );
+                        try {
+                            translogTransferManager.deleteMetadataFilesAsync(
+                                metadataFilesToBeDeleted,
+                                remoteGenerationDeletionPermits::release
+                            );
+                        } catch (Exception e) {
+                            logger.error("Exception in delete metadata files flow", e);
+                            // Permits is already released by deleteMetadataFilesAsync
+                            assert remoteGenerationDeletionPermits.availablePermits() == REMOTE_DELETION_PERMITS : "Available permits "
+                                + remoteGenerationDeletionPermits.availablePermits()
+                                + " is not equal to "
+                                + REMOTE_DELETION_PERMITS;
+                            return;
+                        }
 
                         // Update cache to keep only those metadata files that are not getting deleted
                         oldFormatMetadataFileGenerationMap.keySet().retainAll(metadataFilesNotToBeDeleted);
@@ -240,7 +261,12 @@
                         remoteGenerationDeletionPermits.release();
                     }
                 } catch (Exception e) {
+                    logger.error("Exception in trimUnreferencedReaders", e);
                     remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS);
+                    assert remoteGenerationDeletionPermits.availablePermits() == REMOTE_DELETION_PERMITS : "Available permits "
+                        + remoteGenerationDeletionPermits.availablePermits()
+                        + " is not equal to "
+                        + REMOTE_DELETION_PERMITS;
                 }
             }
 
@@ -441,7 +467,8 @@
         }
         Optional<Long> minPrimaryTermFromMetadataFiles = metadataFilesNotToBeDeleted.stream().map(file -> {
             try {
-                return getMinMaxPrimaryTermFromMetadataFile(file, translogTransferManager, oldFormatMetadataFilePrimaryTermMap).v1();
+                return getMinMaxPrimaryTermFromMetadataFile(file, translogTransferManager, oldFormatMetadataFilePrimaryTermMap, logger)
+                    .v1();
             } catch (IOException e) {
                 return Long.MIN_VALUE;
             }
@@ -482,7 +509,8 @@
     protected static Tuple<Long, Long> getMinMaxPrimaryTermFromMetadataFile(
         String metadataFile,
         TranslogTransferManager translogTransferManager,
-        Map<String, Tuple<Long, Long>> oldFormatMetadataFilePrimaryTermMap
+        Map<String, Tuple<Long, Long>> oldFormatMetadataFilePrimaryTermMap,
+        Logger logger
     ) throws IOException {
         Tuple<Long, Long> minMaxPrimaryTermFromFileName = TranslogTransferMetadata.getMinMaxPrimaryTermFromFilename(metadataFile);
         if (minMaxPrimaryTermFromFileName != null) {
@@ -504,6 +532,8 @@
                     if (primaryTerm.isPresent()) {
                         minPrimaryTem = primaryTerm.get();
                     }
+                } else {
+                    logger.warn("No primary term found from GenerationToPrimaryTermMap for file [{}]", metadataFile);
                 }
                 Tuple<Long, Long> minMaxPrimaryTermTuple = new Tuple<>(minPrimaryTem, maxPrimaryTem);
                 oldFormatMetadataFilePrimaryTermMap.put(metadataFile, minMaxPrimaryTermTuple);

server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java

@@ -590,7 +590,14 @@
             generationsToDelete.add(generation);
         }
         if (generationsToDelete.isEmpty() == false) {
-            deleteRemoteGeneration(generationsToDelete);
+            try {
+                deleteRemoteGeneration(generationsToDelete);
+            } catch (Exception e) {
+                logger.error("Exception in delete generations flow", e);
+                // Release permit that is meant for metadata files and return
+                remoteGenerationDeletionPermits.release();
+                return;
+            }
             translogTransferManager.deleteStaleTranslogMetadataFilesAsync(remoteGenerationDeletionPermits::release);
             deleteStaleRemotePrimaryTerms();
         } else {