Increase max keys limit, raise logging levels

slackhq · Sep 18, 2023 · 850cd90 · 850cd90
1 parent 7497778
commit 850cd90
Showing 1 changed file with 5 additions and 3 deletions.
diff --git a/kaldb/src/main/java/com/slack/kaldb/blobfs/s3/S3CrtBlobFs.java b/kaldb/src/main/java/com/slack/kaldb/blobfs/s3/S3CrtBlobFs.java
@@ -440,7 +440,9 @@ public String[] listFiles(URI fileUri, boolean recursive) throws IOException {
       String prefix = normalizeToDirectoryPrefix(fileUri);
       while (!isDone) {
         ListObjectsV2Request.Builder listObjectsV2RequestBuilder =
-            ListObjectsV2Request.builder().bucket(fileUri.getHost());
+            ListObjectsV2Request.builder()
+                    .maxKeys(2500)
+                    .bucket(fileUri.getHost());
         if (!prefix.equals(DELIMITER)) {
           listObjectsV2RequestBuilder = listObjectsV2RequestBuilder.prefix(prefix);
         }
@@ -451,7 +453,7 @@ public String[] listFiles(URI fileUri, boolean recursive) throws IOException {
           listObjectsV2RequestBuilder.continuationToken(continuationToken);
         }
         ListObjectsV2Request listObjectsV2Request = listObjectsV2RequestBuilder.build();
-        LOG.debug("Trying to send ListObjectsV2Request {}", listObjectsV2Request);
+        LOG.info("Trying to send ListObjectsV2Request {}", listObjectsV2Request);
         ListObjectsV2Response listObjectsV2Response =
             s3AsyncClient.listObjectsV2(listObjectsV2Request).get(15, TimeUnit.SECONDS);
         LOG.debug("Getting ListObjectsV2Response: {}", listObjectsV2Response);
@@ -473,7 +475,7 @@ public String[] listFiles(URI fileUri, boolean recursive) throws IOException {
         continuationToken = listObjectsV2Response.nextContinuationToken();
       }
       String[] listedFiles = builder.build().toArray(new String[0]);
-      LOG.debug(
+      LOG.info(
           "Listed {} files from URI: {}, is recursive: {}", listedFiles.length, fileUri, recursive);
       return listedFiles;
     } catch (Throwable t) {