-
Notifications
You must be signed in to change notification settings - Fork 643
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fix math overflow when copying large AWS S3 files
Signed-off-by: Paolo Di Tommaso <[email protected]>
- Loading branch information
1 parent
66f4669
commit e2b4a93
Showing
7 changed files
with
256 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
101 changes: 101 additions & 0 deletions
101
plugins/nf-amazon/src/main/com/upplication/s3fs/util/S3UploadHelper.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| /* | ||
| * Copyright 2020-2022, Seqera Labs | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| * | ||
| */ | ||
|
|
||
| package com.upplication.s3fs.util; | ||
|
|
||
| /** | ||
| * | ||
| * @author Paolo Di Tommaso <[email protected]> | ||
| */ | ||
| public class S3UploadHelper { | ||
|
|
||
| private static final long _1_KiB = 1024; | ||
| private static final long _1_MiB = _1_KiB * _1_KiB; | ||
| private static final long _1_GiB = _1_KiB * _1_KiB * _1_KiB; | ||
| private static final long _1_TiB = _1_KiB * _1_KiB * _1_KiB * _1_KiB; | ||
|
|
||
| /** | ||
| * AWS S3 max part size | ||
| * https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html | ||
| */ | ||
| public static final long MIN_PART_SIZE = 5 * _1_MiB; | ||
|
|
||
| /** | ||
| * AWS S3 min part size | ||
| * https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html | ||
| */ | ||
| public static final long MAX_PART_SIZE = 5 * _1_GiB; | ||
|
|
||
| /** | ||
| * AWS S3 max object size | ||
| * https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html | ||
| */ | ||
| public static final long MAX_OBJECT_SIZE = 5 * _1_TiB; | ||
|
|
||
| /** | ||
| * AWS S3 max parts in multi-part upload and copy request | ||
| */ | ||
| public static final int MAX_PARTS_COUNT = 10_000; | ||
|
|
||
| static public long computePartSize( long objectSize, long chunkSize ) { | ||
| if( objectSize<0 ) throw new IllegalArgumentException("Argument 'objectSize' cannot be less than zero"); | ||
| if( chunkSize<MIN_PART_SIZE ) throw new IllegalArgumentException("Argument 'chunkSize' cannot be less than " + MIN_PART_SIZE); | ||
| // Multipart upload and copy allows max 10_000 parts | ||
| // each part can be up to 5 GB | ||
| // Max file size is 5 TB | ||
| // See https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html | ||
| long numOfParts = objectSize / chunkSize; | ||
| if( numOfParts > MAX_PARTS_COUNT) { | ||
| final long x = ceilDiv(objectSize, MAX_PARTS_COUNT); | ||
| return ceilDiv(x, 10* _1_MiB) *10* _1_MiB; | ||
| } | ||
| return chunkSize; | ||
| } | ||
|
|
||
|
|
||
| private static long ceilDiv(long x, long y){ | ||
| return -Math.floorDiv(-x,y); | ||
| } | ||
|
|
||
| private static long ceilDiv(long x, int y){ | ||
| return -Math.floorDiv(-x,y); | ||
| } | ||
|
|
||
| static public void checkPartSize(long partSize) { | ||
| if( partSize<MIN_PART_SIZE ) { | ||
| String msg = String.format("The minimum part size for S3 multipart copy and upload operation cannot be less than 5 MiB -- offending value: %d", partSize); | ||
| throw new IllegalArgumentException(msg); | ||
| } | ||
|
|
||
| if( partSize>MAX_PART_SIZE ) { | ||
| String msg = String.format("The minimum part size for S3 multipart copy and upload operation cannot be less than 5 GiB -- offending value: %d", partSize); | ||
| throw new IllegalArgumentException(msg); | ||
| } | ||
| } | ||
|
|
||
| static public void checkPartIndex(int i, String path, long fileSize, long chunkSize) { | ||
| if( i < 1 ) { | ||
| String msg = String.format("S3 multipart copy request index cannot less than 1 -- offending value: %d; file: '%s'; size: %d; part-size: %d", i, path, fileSize, chunkSize); | ||
| throw new IllegalArgumentException(msg); | ||
| } | ||
| if( i > MAX_PARTS_COUNT) { | ||
| String msg = String.format("S3 multipart copy request exceed the number of max allowed parts -- offending value: %d; file: '%s'; size: %d; part-size: %d", i, path, fileSize, chunkSize); | ||
| throw new IllegalArgumentException(msg); | ||
| } | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
128 changes: 128 additions & 0 deletions
128
plugins/nf-amazon/src/test/com/upplication/s3fs/util/S3UploadHelperTest.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| /* | ||
| * Copyright 2020-2022, Seqera Labs | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| * | ||
| */ | ||
|
|
||
| package com.upplication.s3fs.util | ||
|
|
||
| import com.amazonaws.services.s3.AmazonS3 | ||
| import com.upplication.s3fs.AmazonS3Client | ||
| import spock.lang.Shared | ||
| import spock.lang.Specification | ||
| import spock.lang.Unroll | ||
| /** | ||
| * | ||
| * @author Paolo Di Tommaso <[email protected]> | ||
| */ | ||
| class S3UploadHelperTest extends Specification { | ||
|
|
||
| @Shared final long _1_KiB = 1_024 | ||
| @Shared final long _1_MiB = _1_KiB **2 | ||
| @Shared final long _1_GiB = _1_KiB **3 | ||
| @Shared final long _1_TiB = _1_KiB **4 | ||
|
|
||
| @Shared final long _10_MiB = _1_MiB * 10 | ||
| @Shared final long _100_MiB = _1_MiB * 100 | ||
|
|
||
| @Unroll | ||
| def 'should compute s3 file chunk size' () { | ||
|
|
||
| expect: | ||
| S3UploadHelper.computePartSize(FILE_SIZE, CHUNK_SIZE) == EXPECTED_CHUNK_SIZE | ||
| and: | ||
| def parts = FILE_SIZE / EXPECTED_CHUNK_SIZE | ||
| parts <= S3UploadHelper.MAX_PARTS_COUNT | ||
| parts > 0 | ||
|
|
||
| where: | ||
| FILE_SIZE | EXPECTED_CHUNK_SIZE | CHUNK_SIZE | ||
| _1_KiB | _10_MiB | _10_MiB | ||
| _1_MiB | _10_MiB | _10_MiB | ||
| _1_GiB | _10_MiB | _10_MiB | ||
| _1_TiB | 110 * _1_MiB | _10_MiB | ||
| 5 * _1_TiB | 530 * _1_MiB | _10_MiB | ||
| 10 * _1_TiB | 1050 * _1_MiB | _10_MiB | ||
| and: | ||
| _1_KiB | _100_MiB | _100_MiB | ||
| _1_MiB | _100_MiB | _100_MiB | ||
| _1_GiB | _100_MiB | _100_MiB | ||
| _1_TiB | 110 * _1_MiB | _100_MiB | ||
| 5 * _1_TiB | 530 * _1_MiB | _100_MiB | ||
| 10 * _1_TiB | 1050 * _1_MiB | _100_MiB | ||
|
|
||
| } | ||
|
|
||
|
|
||
| def 'should check s3 part size' () { | ||
| when: | ||
| S3UploadHelper.checkPartSize(S3UploadHelper.MIN_PART_SIZE) | ||
| then: | ||
| noExceptionThrown() | ||
|
|
||
| when: | ||
| S3UploadHelper.checkPartSize(S3UploadHelper.MIN_PART_SIZE+1) | ||
| then: | ||
| noExceptionThrown() | ||
|
|
||
| when: | ||
| S3UploadHelper.checkPartSize(S3UploadHelper.MAX_PART_SIZE-1) | ||
| then: | ||
| noExceptionThrown() | ||
|
|
||
| when: | ||
| S3UploadHelper.checkPartSize(S3UploadHelper.MAX_PART_SIZE) | ||
| then: | ||
| noExceptionThrown() | ||
|
|
||
| when: | ||
| S3UploadHelper.checkPartSize(S3UploadHelper.MAX_PART_SIZE+1) | ||
| then: | ||
| thrown(IllegalArgumentException) | ||
|
|
||
| when: | ||
| S3UploadHelper.checkPartSize(S3UploadHelper.MIN_PART_SIZE-1) | ||
| then: | ||
| thrown(IllegalArgumentException) | ||
| } | ||
|
|
||
| def 'should check part index' () { | ||
| given: | ||
| def client = new AmazonS3Client(Mock(AmazonS3)) | ||
|
|
||
| when: | ||
| client.checkPartIndex(1, 's3://foo', 1000, 100) | ||
| then: | ||
| noExceptionThrown() | ||
|
|
||
| when: | ||
| client.checkPartIndex(S3MultipartOptions.MAX_PARTS_COUNT, 's3://foo', 1000, 100) | ||
| then: | ||
| noExceptionThrown() | ||
|
|
||
| when: | ||
| client.checkPartIndex(S3MultipartOptions.MAX_PARTS_COUNT+1, 's3://foo', 1000, 100) | ||
| then: | ||
| def e1 = thrown(IllegalArgumentException) | ||
| e1.message == "S3 multipart copy request exceed the number of max allowed parts -- offending value: 10001; file: 's3://foo'; size: 1000; part-size: 100" | ||
|
|
||
| when: | ||
| client.checkPartIndex(0, 's3://foo', 1000, 100) | ||
| then: | ||
| def e2 = thrown(IllegalArgumentException) | ||
| e2.message == "S3 multipart copy request index cannot less than 1 -- offending value: 0; file: 's3://foo'; size: 1000; part-size: 100" | ||
|
|
||
|
|
||
| } | ||
| } |