From f3e1374f1a7211e3488ce4e0bddfdf077a19b4e2 Mon Sep 17 00:00:00 2001 From: Samiwel Thomas Date: Fri, 22 Nov 2024 12:06:39 +0000 Subject: [PATCH 01/35] Tune S3 for performance improvements. --- source/dea-backend/src/constructs/dea-backend-stack.ts | 1 + source/dea-ui/ui/package.json | 1 + .../ui/src/components/upload-files/UploadFilesForm.tsx | 10 ++++++++++ 3 files changed, 12 insertions(+) diff --git a/source/dea-backend/src/constructs/dea-backend-stack.ts b/source/dea-backend/src/constructs/dea-backend-stack.ts index 49d8240..f8371ca 100644 --- a/source/dea-backend/src/constructs/dea-backend-stack.ts +++ b/source/dea-backend/src/constructs/dea-backend-stack.ts @@ -196,6 +196,7 @@ export class DeaBackendConstruct extends Construct { const datasetsBucket = new Bucket(this, 'S3DatasetsBucket', { blockPublicAccess: BlockPublicAccess.BLOCK_ALL, bucketKeyEnabled: true, + transferAcceleration: true, encryption: BucketEncryption.KMS, encryptionKey: key, enforceSSL: true, diff --git a/source/dea-ui/ui/package.json b/source/dea-ui/ui/package.json index 792698f..8274b50 100644 --- a/source/dea-ui/ui/package.json +++ b/source/dea-ui/ui/package.json @@ -56,6 +56,7 @@ "@aws-sdk/client-cognito-identity": "~3.474.0", "@aws-sdk/client-cognito-identity-provider": "~3.474.0", "@aws-sdk/client-s3": "~3.474.0", + "@aws-sdk/node-http-handler": "^3.374.0", "@aws/dea-app": "workspace:*", "@cloudscape-design/collection-hooks": "^1.0.20", "@cloudscape-design/component-toolkit": "~1.0.0-beta.25", diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 7caadaf..724944a 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -11,6 +11,7 @@ import { UploadPartCommandInput, UploadPartCommandOutput, } from '@aws-sdk/client-s3'; +import { NodeHttpHandler } from '@aws-sdk/node-http-handler'; import { Alert, Box, @@ -119,9 +120,14 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { async function uploadFilePartsAndComplete(activeFileUpload: ActiveFileUpload, chunkSizeBytes: number) { const initiatedCaseFile = await initiateUpload(activeFileUpload.upoadDto); + const MAX_CONCURRENT_REQUESTS = 100; + let federationS3Client = new S3Client({ credentials: initiatedCaseFile.federationCredentials, region: initiatedCaseFile.region, + requestHandler: new NodeHttpHandler({ + maxConcurrentRequests: MAX_CONCURRENT_REQUESTS, + }), }); const credentialsInterval = setInterval(async () => { @@ -133,6 +139,10 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { federationS3Client = new S3Client({ credentials: refreshRequest.federationCredentials, region: initiatedCaseFile.region, + useAccelerateEndpoint: true, + requestHandler: new NodeHttpHandler({ + maxConcurrentRequests: MAX_CONCURRENT_REQUESTS, + }), }); }, 20 * MINUTES_TO_MILLISECONDS); From ef519bc4b9fea1a1f60bb05b93a22ec0d8a51b51 Mon Sep 17 00:00:00 2001 From: Samiwel Thomas Date: Fri, 22 Nov 2024 12:31:06 +0000 Subject: [PATCH 02/35] Update http handler package. --- source/dea-ui/ui/package.json | 2 +- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dea-ui/ui/package.json b/source/dea-ui/ui/package.json index 8274b50..f20fbd2 100644 --- a/source/dea-ui/ui/package.json +++ b/source/dea-ui/ui/package.json @@ -56,7 +56,7 @@ "@aws-sdk/client-cognito-identity": "~3.474.0", "@aws-sdk/client-cognito-identity-provider": "~3.474.0", "@aws-sdk/client-s3": "~3.474.0", - "@aws-sdk/node-http-handler": "^3.374.0", + "@smithy/node-http-handler": "^3.3.1", "@aws/dea-app": "workspace:*", "@cloudscape-design/collection-hooks": "^1.0.20", "@cloudscape-design/component-toolkit": "~1.0.0-beta.25", diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 724944a..ad5240d 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -11,7 +11,7 @@ import { UploadPartCommandInput, UploadPartCommandOutput, } from '@aws-sdk/client-s3'; -import { NodeHttpHandler } from '@aws-sdk/node-http-handler'; +import { NodeHttpHandler } from '@smithy/node-http-handler'; import { Alert, Box, From c4387d757f7e7acf9c3efcfa5e2e6bdc56053ae7 Mon Sep 17 00:00:00 2001 From: Samiwel Thomas Date: Fri, 22 Nov 2024 12:54:21 +0000 Subject: [PATCH 03/35] Remove max concurrent connection. --- .../ui/src/components/upload-files/UploadFilesForm.tsx | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index ad5240d..39e78bf 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -120,14 +120,10 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { async function uploadFilePartsAndComplete(activeFileUpload: ActiveFileUpload, chunkSizeBytes: number) { const initiatedCaseFile = await initiateUpload(activeFileUpload.upoadDto); - const MAX_CONCURRENT_REQUESTS = 100; - let federationS3Client = new S3Client({ credentials: initiatedCaseFile.federationCredentials, region: initiatedCaseFile.region, - requestHandler: new NodeHttpHandler({ - maxConcurrentRequests: MAX_CONCURRENT_REQUESTS, - }), + requestHandler: new NodeHttpHandler(), }); const credentialsInterval = setInterval(async () => { @@ -140,9 +136,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { credentials: refreshRequest.federationCredentials, region: initiatedCaseFile.region, useAccelerateEndpoint: true, - requestHandler: new NodeHttpHandler({ - maxConcurrentRequests: MAX_CONCURRENT_REQUESTS, - }), + requestHandler: new NodeHttpHandler(), }); }, 20 * MINUTES_TO_MILLISECONDS); From a84a721c4236788dbaee3cdf7eaeea38a094b00b Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Sun, 24 Nov 2024 11:41:53 +0000 Subject: [PATCH 04/35] reoder import --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 39e78bf..7c9410a 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -11,7 +11,6 @@ import { UploadPartCommandInput, UploadPartCommandOutput, } from '@aws-sdk/client-s3'; -import { NodeHttpHandler } from '@smithy/node-http-handler'; import { Alert, Box, @@ -28,6 +27,7 @@ import { Table, Textarea, } from '@cloudscape-design/components'; +import { NodeHttpHandler } from '@smithy/node-http-handler'; import { useRouter } from 'next/router'; import { useState } from 'react'; import { completeUpload, initiateUpload } from '../../api/cases'; From c3ce57b50fd42870a0f2f351b699729b47d933b6 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Sun, 24 Nov 2024 12:03:40 +0000 Subject: [PATCH 05/35] remove import --- source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 7c9410a..c2864ac 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -27,7 +27,6 @@ import { Table, Textarea, } from '@cloudscape-design/components'; -import { NodeHttpHandler } from '@smithy/node-http-handler'; import { useRouter } from 'next/router'; import { useState } from 'react'; import { completeUpload, initiateUpload } from '../../api/cases'; From 8ed6a28e30d3c40020ea095f41f361959ca47b6d Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Sun, 24 Nov 2024 12:04:31 +0000 Subject: [PATCH 06/35] reorder import --- source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index c2864ac..7c9410a 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -27,6 +27,7 @@ import { Table, Textarea, } from '@cloudscape-design/components'; +import { NodeHttpHandler } from '@smithy/node-http-handler'; import { useRouter } from 'next/router'; import { useState } from 'react'; import { completeUpload, initiateUpload } from '../../api/cases'; From 4a2bed49a73905f5e669c1bdcd1d5f36f3586598 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Sun, 24 Nov 2024 12:13:28 +0000 Subject: [PATCH 07/35] reorder import --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 7c9410a..076af8c 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -27,7 +27,6 @@ import { Table, Textarea, } from '@cloudscape-design/components'; -import { NodeHttpHandler } from '@smithy/node-http-handler'; import { useRouter } from 'next/router'; import { useState } from 'react'; import { completeUpload, initiateUpload } from '../../api/cases'; @@ -37,6 +36,7 @@ import { FileWithPath, formatFileSize } from '../../helpers/fileHelper'; import { InitiateUploadForm } from '../../models/CaseFiles'; import FileUpload from '../common-components/FileUpload'; import { UploadFilesProps } from './UploadFilesBody'; +import { NodeHttpHandler } from '@smithy/node-http-handler'; const MINUTES_TO_MILLISECONDS = 60 * 1000; From 5dbc06062820ff3844a8f9288f82c2f44f150012 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Sun, 24 Nov 2024 12:18:10 +0000 Subject: [PATCH 08/35] reorder import --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 076af8c..a26dcaf 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -27,6 +27,7 @@ import { Table, Textarea, } from '@cloudscape-design/components'; +import { NodeHttpHandler } from '@smithy/node-http-handler'; import { useRouter } from 'next/router'; import { useState } from 'react'; import { completeUpload, initiateUpload } from '../../api/cases'; @@ -36,7 +37,7 @@ import { FileWithPath, formatFileSize } from '../../helpers/fileHelper'; import { InitiateUploadForm } from '../../models/CaseFiles'; import FileUpload from '../common-components/FileUpload'; import { UploadFilesProps } from './UploadFilesBody'; -import { NodeHttpHandler } from '@smithy/node-http-handler'; + const MINUTES_TO_MILLISECONDS = 60 * 1000; From 03edd6df8f2c37548176b3a84e8213454ff20e50 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Sun, 24 Nov 2024 13:09:41 +0000 Subject: [PATCH 09/35] remove @smithy/node-http-handler --- source/dea-ui/ui/package.json | 1 - .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 3 --- 2 files changed, 4 deletions(-) diff --git a/source/dea-ui/ui/package.json b/source/dea-ui/ui/package.json index f20fbd2..792698f 100644 --- a/source/dea-ui/ui/package.json +++ b/source/dea-ui/ui/package.json @@ -56,7 +56,6 @@ "@aws-sdk/client-cognito-identity": "~3.474.0", "@aws-sdk/client-cognito-identity-provider": "~3.474.0", "@aws-sdk/client-s3": "~3.474.0", - "@smithy/node-http-handler": "^3.3.1", "@aws/dea-app": "workspace:*", "@cloudscape-design/collection-hooks": "^1.0.20", "@cloudscape-design/component-toolkit": "~1.0.0-beta.25", diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index a26dcaf..709fc09 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -27,7 +27,6 @@ import { Table, Textarea, } from '@cloudscape-design/components'; -import { NodeHttpHandler } from '@smithy/node-http-handler'; import { useRouter } from 'next/router'; import { useState } from 'react'; import { completeUpload, initiateUpload } from '../../api/cases'; @@ -124,7 +123,6 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { let federationS3Client = new S3Client({ credentials: initiatedCaseFile.federationCredentials, region: initiatedCaseFile.region, - requestHandler: new NodeHttpHandler(), }); const credentialsInterval = setInterval(async () => { @@ -137,7 +135,6 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { credentials: refreshRequest.federationCredentials, region: initiatedCaseFile.region, useAccelerateEndpoint: true, - requestHandler: new NodeHttpHandler(), }); }, 20 * MINUTES_TO_MILLISECONDS); From 21f35e14c452f55a4e4c8877c36a3485688ad9a3 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Tue, 26 Nov 2024 13:07:08 +0000 Subject: [PATCH 10/35] increase the chunkSizeBytes --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 709fc09..9327411 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -190,7 +190,8 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { // Maximum object size 5 TiB // Maximum number of parts per upload 10,000 // 5 MiB to 5 GiB. There is no minimum size limit on the last part of your multipart upload. - const chunkSizeBytes = Math.max(selectedFile.size / 10_000, 50 * ONE_MB); + //const chunkSizeBytes = Math.max(selectedFile.size / 10_000, 50 * ONE_MB); + const chunkSizeBytes = 300 * ONE_MB; // per file try/finally state to initiate uploads try { const contentType = selectedFile.type ? selectedFile.type : 'text/plain'; From e546296aa82bb48ee8cd06083f4be8536a4acbab Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Tue, 26 Nov 2024 13:47:47 +0000 Subject: [PATCH 11/35] set useAccelerateEndpoint to true --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 9327411..6c20955 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -123,6 +123,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { let federationS3Client = new S3Client({ credentials: initiatedCaseFile.federationCredentials, region: initiatedCaseFile.region, + useAccelerateEndpoint: true, }); const credentialsInterval = setInterval(async () => { @@ -194,7 +195,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { const chunkSizeBytes = 300 * ONE_MB; // per file try/finally state to initiate uploads try { - const contentType = selectedFile.type ? selectedFile.type : 'text/plain'; + const contentType = selectedFile.type ? selectedFile.type : pro'text/plain'; const activeFileUpload = { file: selectedFile, upoadDto: { From 791c239ddd89639cb17daa25eec226d2b0ce9787 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Tue, 26 Nov 2024 14:25:37 +0000 Subject: [PATCH 12/35] set useAccelerateEndpoint to true --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 6c20955..db816a9 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -195,7 +195,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { const chunkSizeBytes = 300 * ONE_MB; // per file try/finally state to initiate uploads try { - const contentType = selectedFile.type ? selectedFile.type : pro'text/plain'; + const contentType = selectedFile.type ? selectedFile.type : 'text/plain'; const activeFileUpload = { file: selectedFile, upoadDto: { From 5cf0f32abda5f3f50fb4f39f424b73dc64452f2b Mon Sep 17 00:00:00 2001 From: Samiwel Thomas Date: Wed, 27 Nov 2024 09:41:13 +0000 Subject: [PATCH 13/35] Add accelerate endpoint to S3 client. --- source/dea-app/src/storage/datasets.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dea-app/src/storage/datasets.ts b/source/dea-app/src/storage/datasets.ts index 7e4583c..951d099 100644 --- a/source/dea-app/src/storage/datasets.ts +++ b/source/dea-app/src/storage/datasets.ts @@ -73,7 +73,7 @@ const stsClient = new STSClient({ region, customUserAgent: getCustomUserAgent() const sqsClient = new SQSClient({ region, customUserAgent: getCustomUserAgent() }); export const defaultDatasetsProvider = { - s3Client: new S3Client({ region, customUserAgent: getCustomUserAgent() }), + s3Client: new S3Client({ region, customUserAgent: getCustomUserAgent(), useAccelerateEndpoint: true }), region: region, bucketName: getRequiredEnv('DATASETS_BUCKET_NAME', 'DATASETS_BUCKET_NAME is not set in your lambda!'), s3BatchDeleteCaseFileLambdaArn: getRequiredEnv( @@ -556,6 +556,7 @@ async function getDownloadPresignedUrlClient( sessionToken: credentials.SessionToken, expiration: credentials.Expiration, }, + useAccelerateEndpoint: true, }); } From 314a1e278e1142576373991ed72cdb74029fbc39 Mon Sep 17 00:00:00 2001 From: Samiwel Thomas Date: Wed, 27 Nov 2024 10:05:15 +0000 Subject: [PATCH 14/35] Update CSP header. --- source/dea-ui/infrastructure/src/dea-ui-stack.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dea-ui/infrastructure/src/dea-ui-stack.ts b/source/dea-ui/infrastructure/src/dea-ui-stack.ts index 7c0f068..42afd1a 100644 --- a/source/dea-ui/infrastructure/src/dea-ui-stack.ts +++ b/source/dea-ui/infrastructure/src/dea-ui-stack.ts @@ -190,7 +190,7 @@ export class DeaUiConstruct extends NestedStack { `style-src 'unsafe-inline' 'self';` + `connect-src 'self' https://${deaConfig.cognitoDomain()}.${this.authSubdomain()}.${this.cognitoRegion()}.amazoncognito.com https://*.s3.${ Aws.REGION - }.amazonaws.com https://cognito-identity.${this.cognitoRegion()}.amazonaws.com https://cognito-idp.${this.cognitoRegion()}.amazonaws.com;` + + }.amazonaws.com https://*.s3-accelerate.amazonaws.com https://cognito-identity.${this.cognitoRegion()}.amazonaws.com https://cognito-idp.${this.cognitoRegion()}.amazonaws.com;` + `script-src 'strict-dynamic' '${this.sriString}';` + `font-src 'self' data:;` + `base-uri 'self';` + From 9347eca0d3332e947acd4bf6cba3ab7583d69f52 Mon Sep 17 00:00:00 2001 From: Samiwel Thomas Date: Wed, 27 Nov 2024 10:32:42 +0000 Subject: [PATCH 15/35] Increase in flight upload size. --- .../ui/src/components/upload-files/UploadFilesForm.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index db816a9..a37c1cd 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -37,7 +37,6 @@ import { InitiateUploadForm } from '../../models/CaseFiles'; import FileUpload from '../common-components/FileUpload'; import { UploadFilesProps } from './UploadFilesBody'; - const MINUTES_TO_MILLISECONDS = 60 * 1000; interface FileUploadProgressRow { @@ -164,8 +163,8 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { uploadPromises.push(federationS3Client.send(uploadCommand)); promisesSize += chunkSizeBytes; - // flush promises if we've got over 500MB queued - if (promisesSize > ONE_MB * 500) { + // flush promises if we've got over 1GB queued + if (promisesSize > ONE_MB * 1000) { await Promise.all(uploadPromises); promisesSize = 0; uploadPromises.length = 0; From 9d5e3048fbc260543c4e6928b825bf947efc8600 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Wed, 27 Nov 2024 12:08:48 +0000 Subject: [PATCH 16/35] set MAX_PARALLEL_UPLOADS to 2 --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index a37c1cd..c7e4767 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -59,7 +59,7 @@ interface ActiveFileUpload { export const ONE_MB = 1024 * 1024; export const ONE_GB = ONE_MB * 1024; -const MAX_PARALLEL_UPLOADS = 1; // One file concurrently for now. The backend requires a code refactor to deal with the TransactionConflictException thrown ocassionally. +const MAX_PARALLEL_UPLOADS = 2; // One file concurrently for now. The backend requires a code refactor to deal with the TransactionConflictException thrown ocassionally. function UploadFilesForm(props: UploadFilesProps): JSX.Element { const [selectedFiles, setSelectedFiles] = useState([]); From c9ce24632106be570a783ed4335be7baa1e8090e Mon Sep 17 00:00:00 2001 From: Samiwel Thomas Date: Wed, 27 Nov 2024 12:43:06 +0000 Subject: [PATCH 17/35] Increase to 6 parallel uploads. --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index c7e4767..234e08f 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -59,7 +59,7 @@ interface ActiveFileUpload { export const ONE_MB = 1024 * 1024; export const ONE_GB = ONE_MB * 1024; -const MAX_PARALLEL_UPLOADS = 2; // One file concurrently for now. The backend requires a code refactor to deal with the TransactionConflictException thrown ocassionally. +const MAX_PARALLEL_UPLOADS = 6; // One file concurrently for now. The backend requires a code refactor to deal with the TransactionConflictException thrown ocassionally. function UploadFilesForm(props: UploadFilesProps): JSX.Element { const [selectedFiles, setSelectedFiles] = useState([]); From 5f8056c0fe526121a9e6cb9774fd83a3c40f0270 Mon Sep 17 00:00:00 2001 From: Samiwel Thomas Date: Wed, 27 Nov 2024 13:46:42 +0000 Subject: [PATCH 18/35] Remove DEA rate limiting. --- .../ui/src/components/upload-files/UploadFilesForm.tsx | 7 ------- 1 file changed, 7 deletions(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 234e08f..0a747b4 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -162,13 +162,6 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { uploadPromises.push(federationS3Client.send(uploadCommand)); promisesSize += chunkSizeBytes; - - // flush promises if we've got over 1GB queued - if (promisesSize > ONE_MB * 1000) { - await Promise.all(uploadPromises); - promisesSize = 0; - uploadPromises.length = 0; - } } await Promise.all(uploadPromises); From 4145d31de434a6fccae9a505f7135bf605a82797 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Wed, 27 Nov 2024 13:57:08 +0000 Subject: [PATCH 19/35] remove promisesSize code --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 0a747b4..36a6ede 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -142,7 +142,6 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { try { const totalChunks = Math.ceil(activeFileUpload.file.size / chunkSizeBytes); - let promisesSize = 0; for (let i = 0; i < totalChunks; i++) { const chunkBlob = activeFileUpload.file.slice(i * chunkSizeBytes, (i + 1) * chunkSizeBytes); @@ -161,7 +160,6 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { const uploadCommand = new UploadPartCommand(uploadInput); uploadPromises.push(federationS3Client.send(uploadCommand)); - promisesSize += chunkSizeBytes; } await Promise.all(uploadPromises); From 319bbf26897b7da394bad80e9c1c0d2985fd95b0 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Thu, 28 Nov 2024 11:41:24 +0000 Subject: [PATCH 20/35] increase MAX_PARALLEL_UPLOADS 10 and chunkSizeBytes 500 --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 36a6ede..5d0c0f8 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -59,7 +59,7 @@ interface ActiveFileUpload { export const ONE_MB = 1024 * 1024; export const ONE_GB = ONE_MB * 1024; -const MAX_PARALLEL_UPLOADS = 6; // One file concurrently for now. The backend requires a code refactor to deal with the TransactionConflictException thrown ocassionally. +const MAX_PARALLEL_UPLOADS = 10; // One file concurrently for now. The backend requires a code refactor to deal with the TransactionConflictException thrown ocassionally. function UploadFilesForm(props: UploadFilesProps): JSX.Element { const [selectedFiles, setSelectedFiles] = useState([]); @@ -182,7 +182,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { // Maximum number of parts per upload 10,000 // 5 MiB to 5 GiB. There is no minimum size limit on the last part of your multipart upload. //const chunkSizeBytes = Math.max(selectedFile.size / 10_000, 50 * ONE_MB); - const chunkSizeBytes = 300 * ONE_MB; + const chunkSizeBytes = 500 * ONE_MB; // per file try/finally state to initiate uploads try { const contentType = selectedFile.type ? selectedFile.type : 'text/plain'; From 6eaf3057274eb84449e3bc864950e676793c1939 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Thu, 28 Nov 2024 12:29:28 +0000 Subject: [PATCH 21/35] set chunkSizeBytes 400 --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 5d0c0f8..3b63d57 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -182,7 +182,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { // Maximum number of parts per upload 10,000 // 5 MiB to 5 GiB. There is no minimum size limit on the last part of your multipart upload. //const chunkSizeBytes = Math.max(selectedFile.size / 10_000, 50 * ONE_MB); - const chunkSizeBytes = 500 * ONE_MB; + const chunkSizeBytes = 400 * ONE_MB; // per file try/finally state to initiate uploads try { const contentType = selectedFile.type ? selectedFile.type : 'text/plain'; From ce332a662a2cc2502bcadeb133f237e193a911d5 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Thu, 28 Nov 2024 12:55:07 +0000 Subject: [PATCH 22/35] set chunkSizeBytes 300 --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 3b63d57..f74bae8 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -182,7 +182,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { // Maximum number of parts per upload 10,000 // 5 MiB to 5 GiB. There is no minimum size limit on the last part of your multipart upload. //const chunkSizeBytes = Math.max(selectedFile.size / 10_000, 50 * ONE_MB); - const chunkSizeBytes = 400 * ONE_MB; + const chunkSizeBytes = 300 * ONE_MB; // per file try/finally state to initiate uploads try { const contentType = selectedFile.type ? selectedFile.type : 'text/plain'; From fd8db2203c87bfc4764643a75f880150ff6d56ba Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Thu, 28 Nov 2024 13:19:18 +0000 Subject: [PATCH 23/35] set parallel to 6 --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index f74bae8..36a6ede 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -59,7 +59,7 @@ interface ActiveFileUpload { export const ONE_MB = 1024 * 1024; export const ONE_GB = ONE_MB * 1024; -const MAX_PARALLEL_UPLOADS = 10; // One file concurrently for now. The backend requires a code refactor to deal with the TransactionConflictException thrown ocassionally. +const MAX_PARALLEL_UPLOADS = 6; // One file concurrently for now. The backend requires a code refactor to deal with the TransactionConflictException thrown ocassionally. function UploadFilesForm(props: UploadFilesProps): JSX.Element { const [selectedFiles, setSelectedFiles] = useState([]); From 0b1f74beeb1f96eb39a2955aad26cb9089b823c4 Mon Sep 17 00:00:00 2001 From: Samiwel Thomas Date: Thu, 28 Nov 2024 13:39:10 +0000 Subject: [PATCH 24/35] Send upload command immediately. --- .../src/components/upload-files/UploadFilesForm.tsx | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 36a6ede..640f60f 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -138,10 +138,9 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { }); }, 20 * MINUTES_TO_MILLISECONDS); - const uploadPromises: Promise[] = []; - try { const totalChunks = Math.ceil(activeFileUpload.file.size / chunkSizeBytes); + let promisesSize = 0; for (let i = 0; i < totalChunks; i++) { const chunkBlob = activeFileUpload.file.slice(i * chunkSizeBytes, (i + 1) * chunkSizeBytes); @@ -158,15 +157,15 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { ChecksumAlgorithm: ChecksumAlgorithm.SHA256, }; const uploadCommand = new UploadPartCommand(uploadInput); - - uploadPromises.push(federationS3Client.send(uploadCommand)); + await federationS3Client.send(uploadCommand); } - - await Promise.all(uploadPromises); } finally { clearInterval(credentialsInterval); } + // @ts-ignore + delete uploadPromises; + await completeUpload({ caseUlid: props.caseId, ulid: initiatedCaseFile.ulid, @@ -182,7 +181,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { // Maximum number of parts per upload 10,000 // 5 MiB to 5 GiB. There is no minimum size limit on the last part of your multipart upload. //const chunkSizeBytes = Math.max(selectedFile.size / 10_000, 50 * ONE_MB); - const chunkSizeBytes = 300 * ONE_MB; + const chunkSizeBytes = 100 * ONE_MB; // per file try/finally state to initiate uploads try { const contentType = selectedFile.type ? selectedFile.type : 'text/plain'; From 1d282edb423e6adcac8ad4f16d2fffff314a4a2f Mon Sep 17 00:00:00 2001 From: Samiwel Thomas Date: Thu, 28 Nov 2024 13:46:09 +0000 Subject: [PATCH 25/35] Fix linting errors. --- .../src/components/upload-files/UploadFilesForm.tsx | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 640f60f..c7960de 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -4,13 +4,7 @@ */ import crypto from 'crypto'; -import { - ChecksumAlgorithm, - S3Client, - UploadPartCommand, - UploadPartCommandInput, - UploadPartCommandOutput, -} from '@aws-sdk/client-s3'; +import { ChecksumAlgorithm, S3Client, UploadPartCommand, UploadPartCommandInput } from '@aws-sdk/client-s3'; import { Alert, Box, @@ -140,7 +134,6 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { try { const totalChunks = Math.ceil(activeFileUpload.file.size / chunkSizeBytes); - let promisesSize = 0; for (let i = 0; i < totalChunks; i++) { const chunkBlob = activeFileUpload.file.slice(i * chunkSizeBytes, (i + 1) * chunkSizeBytes); @@ -163,9 +156,6 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { clearInterval(credentialsInterval); } - // @ts-ignore - delete uploadPromises; - await completeUpload({ caseUlid: props.caseId, ulid: initiatedCaseFile.ulid, From a7ae30bbac760265389741c5e262565c96094c4d Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Thu, 28 Nov 2024 15:56:26 +0000 Subject: [PATCH 26/35] set chunkSizeBytes 300 --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index c7960de..374aa78 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -171,7 +171,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { // Maximum number of parts per upload 10,000 // 5 MiB to 5 GiB. There is no minimum size limit on the last part of your multipart upload. //const chunkSizeBytes = Math.max(selectedFile.size / 10_000, 50 * ONE_MB); - const chunkSizeBytes = 100 * ONE_MB; + const chunkSizeBytes = 300 * ONE_MB; // per file try/finally state to initiate uploads try { const contentType = selectedFile.type ? selectedFile.type : 'text/plain'; From e4b17be92740b410bbb216981a55e44442ba8550 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Thu, 28 Nov 2024 17:31:14 +0000 Subject: [PATCH 27/35] axios test --- .../upload-files/UploadFilesForm.tsx | 86 +++++++++---------- 1 file changed, 39 insertions(+), 47 deletions(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 374aa78..5f87d2b 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -1,10 +1,3 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -import crypto from 'crypto'; -import { ChecksumAlgorithm, S3Client, UploadPartCommand, UploadPartCommandInput } from '@aws-sdk/client-s3'; import { Alert, Box, @@ -21,9 +14,10 @@ import { Table, Textarea, } from '@cloudscape-design/components'; +import axios from 'axios'; import { useRouter } from 'next/router'; import { useState } from 'react'; -import { completeUpload, initiateUpload } from '../../api/cases'; +import { completeUpload, initiateUpload } from '../../api/cases'; // Existing API request functions import { commonLabels, commonTableLabels, fileOperationsLabels } from '../../common/labels'; import { refreshCredentials } from '../../helpers/authService'; import { FileWithPath, formatFileSize } from '../../helpers/fileHelper'; @@ -53,7 +47,7 @@ interface ActiveFileUpload { export const ONE_MB = 1024 * 1024; export const ONE_GB = ONE_MB * 1024; -const MAX_PARALLEL_UPLOADS = 6; // One file concurrently for now. The backend requires a code refactor to deal with the TransactionConflictException thrown ocassionally. +const MAX_PARALLEL_UPLOADS = 6; // One file concurrently for now. The backend requires a code refactor to deal with the TransactionConflictException thrown occasionally. function UploadFilesForm(props: UploadFilesProps): JSX.Element { const [selectedFiles, setSelectedFiles] = useState([]); @@ -65,7 +59,6 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { const router = useRouter(); async function onSubmitHandler() { - // top level try/finally to set uploadInProgress bool state try { setUploadInProgress(true); @@ -113,11 +106,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { async function uploadFilePartsAndComplete(activeFileUpload: ActiveFileUpload, chunkSizeBytes: number) { const initiatedCaseFile = await initiateUpload(activeFileUpload.upoadDto); - let federationS3Client = new S3Client({ - credentials: initiatedCaseFile.federationCredentials, - region: initiatedCaseFile.region, - useAccelerateEndpoint: true, - }); + let credentials = initiatedCaseFile.federationCredentials; const credentialsInterval = setInterval(async () => { await refreshCredentials(); @@ -125,32 +114,14 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { ...activeFileUpload.upoadDto, uploadId: initiatedCaseFile.uploadId, }); - federationS3Client = new S3Client({ - credentials: refreshRequest.federationCredentials, - region: initiatedCaseFile.region, - useAccelerateEndpoint: true, - }); + credentials = refreshRequest.federationCredentials; }, 20 * MINUTES_TO_MILLISECONDS); try { const totalChunks = Math.ceil(activeFileUpload.file.size / chunkSizeBytes); for (let i = 0; i < totalChunks; i++) { const chunkBlob = activeFileUpload.file.slice(i * chunkSizeBytes, (i + 1) * chunkSizeBytes); - - const arrayFromBlob = new Uint8Array(await blobToArrayBuffer(chunkBlob)); - const partHash = crypto.createHash('sha256').update(arrayFromBlob).digest('base64'); - - const uploadInput: UploadPartCommandInput = { - Bucket: initiatedCaseFile.bucket, - Key: `${initiatedCaseFile.caseUlid}/${initiatedCaseFile.ulid}`, - PartNumber: i + 1, - UploadId: initiatedCaseFile.uploadId, - Body: arrayFromBlob, - ChecksumSHA256: partHash, - ChecksumAlgorithm: ChecksumAlgorithm.SHA256, - }; - const uploadCommand = new UploadPartCommand(uploadInput); - await federationS3Client.send(uploadCommand); + await uploadChunk(chunkBlob, initiatedCaseFile, i + 1, credentials); } } finally { clearInterval(credentialsInterval); @@ -164,18 +135,38 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { updateFileProgress(activeFileUpload.file, UploadStatus.complete); } + async function uploadChunk(chunkBlob: Blob, initiatedCaseFile: any, partNumber: number, credentials: any) { + const arrayBuffer = await blobToArrayBuffer(chunkBlob); + + // Calculate SHA-256 hash using Web Crypto API + const hashBuffer = await crypto.subtle.digest('SHA-256', arrayBuffer); + const hashArray = Array.from(new Uint8Array(hashBuffer)); + const partHash = btoa(String.fromCharCode(...hashArray)); + + const url = initiatedCaseFile.preSignedUrls[partNumber - 1]; // Assuming initiateUpload provides pre-signed URLs for each part. + + try { + await axios.put(url, arrayBuffer, { + headers: { + 'Content-Type': 'application/octet-stream', + 'x-amz-checksum-sha256': partHash, + 'x-amz-algorithm': 'AWS4-HMAC-SHA256', + 'x-amz-credential': `${credentials.accessKeyId}/${credentials.sessionToken}`, + 'x-amz-security-token': credentials.sessionToken, + }, + }); + } catch (error) { + console.error(`Upload of part ${partNumber} failed`, error); + throw error; + } + } + async function uploadFile(selectedFile: FileWithPath) { const fileSizeBytes = Math.max(selectedFile.size, 1); - // Trying to use small chunk size (50MB) to reduce memory use. - // Maximum object size 5 TiB - // Maximum number of parts per upload 10,000 - // 5 MiB to 5 GiB. There is no minimum size limit on the last part of your multipart upload. - //const chunkSizeBytes = Math.max(selectedFile.size / 10_000, 50 * ONE_MB); - const chunkSizeBytes = 300 * ONE_MB; - // per file try/finally state to initiate uploads + const chunkSizeBytes = 300 * ONE_MB; // Set chunk size to 300MB + try { - const contentType = selectedFile.type ? selectedFile.type : 'text/plain'; - const activeFileUpload = { + const activeFileUpload: ActiveFileUpload = { file: selectedFile, upoadDto: { caseUlid: props.caseId, @@ -183,15 +174,16 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { filePath: selectedFile.relativePath, fileSizeBytes, chunkSizeBytes, - contentType, + contentType: selectedFile.type || 'text/plain', reason, details, }, }; + await uploadFilePartsAndComplete(activeFileUpload, chunkSizeBytes); - } catch (e) { + } catch (error) { updateFileProgress(selectedFile, UploadStatus.failed); - console.log('Upload failed', e); + console.error('Upload failed:', error); } } From a708d915ec2478b8e73b974cf95a474b6919661e Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Thu, 28 Nov 2024 18:51:28 +0000 Subject: [PATCH 28/35] axios test --- .../upload-files/UploadFilesForm.tsx | 65 +++++++++++++------ 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 5f87d2b..267adbe 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -17,7 +17,7 @@ import { import axios from 'axios'; import { useRouter } from 'next/router'; import { useState } from 'react'; -import { completeUpload, initiateUpload } from '../../api/cases'; // Existing API request functions +import { completeUpload, initiateUpload } from '../../api/cases'; import { commonLabels, commonTableLabels, fileOperationsLabels } from '../../common/labels'; import { refreshCredentials } from '../../helpers/authService'; import { FileWithPath, formatFileSize } from '../../helpers/fileHelper'; @@ -47,7 +47,7 @@ interface ActiveFileUpload { export const ONE_MB = 1024 * 1024; export const ONE_GB = ONE_MB * 1024; -const MAX_PARALLEL_UPLOADS = 6; // One file concurrently for now. The backend requires a code refactor to deal with the TransactionConflictException thrown occasionally. +const MAX_PARALLEL_UPLOADS = 6; function UploadFilesForm(props: UploadFilesProps): JSX.Element { const [selectedFiles, setSelectedFiles] = useState([]); @@ -75,17 +75,21 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { let position = 0; while (position < selectedFiles.length) { const itemsForBatch = selectedFiles.slice(position, position + MAX_PARALLEL_UPLOADS); + console.log(`Uploading batch starting from position ${position}:`, itemsForBatch); await Promise.all(itemsForBatch.map((item) => uploadFile(item))); position += MAX_PARALLEL_UPLOADS; } setSelectedFiles([]); + } catch (error) { + console.error('Error during file upload submission:', error); } finally { setUploadInProgress(false); } } async function blobToArrayBuffer(blob: Blob) { + console.log('Converting blob to ArrayBuffer...'); if ('arrayBuffer' in blob) { return await blob.arrayBuffer(); } @@ -96,54 +100,72 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { if (reader.result && typeof reader.result !== 'string') { resolve(reader.result); } - reject(); + reject(new Error('Failed to convert blob to ArrayBuffer.')); }; - reader.onerror = () => reject(); + reader.onerror = () => reject(new Error('Error reading blob.')); reader.readAsArrayBuffer(blob); }); } async function uploadFilePartsAndComplete(activeFileUpload: ActiveFileUpload, chunkSizeBytes: number) { + console.log('Initiating upload for file:', activeFileUpload.file.name); const initiatedCaseFile = await initiateUpload(activeFileUpload.upoadDto); let credentials = initiatedCaseFile.federationCredentials; + console.log('Upload initiated. Case details:', initiatedCaseFile); const credentialsInterval = setInterval(async () => { - await refreshCredentials(); - const refreshRequest = await initiateUpload({ - ...activeFileUpload.upoadDto, - uploadId: initiatedCaseFile.uploadId, - }); - credentials = refreshRequest.federationCredentials; + try { + console.log('Refreshing credentials...'); + await refreshCredentials(); + const refreshRequest = await initiateUpload({ + ...activeFileUpload.upoadDto, + uploadId: initiatedCaseFile.uploadId, + }); + credentials = refreshRequest.federationCredentials; + console.log('Credentials refreshed successfully.'); + } catch (error) { + console.error('Error refreshing credentials:', error); + } }, 20 * MINUTES_TO_MILLISECONDS); try { const totalChunks = Math.ceil(activeFileUpload.file.size / chunkSizeBytes); for (let i = 0; i < totalChunks; i++) { const chunkBlob = activeFileUpload.file.slice(i * chunkSizeBytes, (i + 1) * chunkSizeBytes); + console.log(`Uploading chunk ${i + 1}/${totalChunks} for file:`, activeFileUpload.file.name); await uploadChunk(chunkBlob, initiatedCaseFile, i + 1, credentials); } + } catch (error) { + console.error('Error during file part upload:', error); } finally { clearInterval(credentialsInterval); } - await completeUpload({ - caseUlid: props.caseId, - ulid: initiatedCaseFile.ulid, - uploadId: initiatedCaseFile.uploadId, - }); - updateFileProgress(activeFileUpload.file, UploadStatus.complete); + try { + console.log('Completing upload for file:', activeFileUpload.file.name); + await completeUpload({ + caseUlid: props.caseId, + ulid: initiatedCaseFile.ulid, + uploadId: initiatedCaseFile.uploadId, + }); + updateFileProgress(activeFileUpload.file, UploadStatus.complete); + } catch (error) { + console.error('Error completing upload:', error); + } } async function uploadChunk(chunkBlob: Blob, initiatedCaseFile: any, partNumber: number, credentials: any) { const arrayBuffer = await blobToArrayBuffer(chunkBlob); + console.log(`Calculated ArrayBuffer for part ${partNumber}`); // Calculate SHA-256 hash using Web Crypto API const hashBuffer = await crypto.subtle.digest('SHA-256', arrayBuffer); const hashArray = Array.from(new Uint8Array(hashBuffer)); const partHash = btoa(String.fromCharCode(...hashArray)); - const url = initiatedCaseFile.preSignedUrls[partNumber - 1]; // Assuming initiateUpload provides pre-signed URLs for each part. + const url = initiatedCaseFile.preSignedUrls[partNumber - 1]; + console.log(`Uploading part ${partNumber} to URL:`, url); try { await axios.put(url, arrayBuffer, { @@ -155,6 +177,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { 'x-amz-security-token': credentials.sessionToken, }, }); + console.log(`Successfully uploaded part ${partNumber}`); } catch (error) { console.error(`Upload of part ${partNumber} failed`, error); throw error; @@ -163,7 +186,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { async function uploadFile(selectedFile: FileWithPath) { const fileSizeBytes = Math.max(selectedFile.size, 1); - const chunkSizeBytes = 300 * ONE_MB; // Set chunk size to 300MB + const chunkSizeBytes = 300 * ONE_MB; try { const activeFileUpload: ActiveFileUpload = { @@ -179,11 +202,11 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { details, }, }; - + console.log('Uploading file:', selectedFile.name); await uploadFilePartsAndComplete(activeFileUpload, chunkSizeBytes); } catch (error) { updateFileProgress(selectedFile, UploadStatus.failed); - console.error('Upload failed:', error); + console.error('Upload failed for file:', selectedFile.name, error); } } @@ -230,7 +253,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { - {uploadProgress.status} + {uploadProgress.status} ); From 2623985743531c596afd1df66fe1bc21a0290dec Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Thu, 28 Nov 2024 19:22:29 +0000 Subject: [PATCH 29/35] axios test --- .../components/upload-files/UploadFilesForm.tsx | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 267adbe..60e9ba8 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -1,3 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + import { Alert, Box, @@ -133,6 +138,15 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { const totalChunks = Math.ceil(activeFileUpload.file.size / chunkSizeBytes); for (let i = 0; i < totalChunks; i++) { const chunkBlob = activeFileUpload.file.slice(i * chunkSizeBytes, (i + 1) * chunkSizeBytes); + + // Add the console.log statement here to debug the parameters used for each part upload. + console.log('Upload Part Params:', { + Bucket: initiatedCaseFile.bucket, + Key: `${initiatedCaseFile.caseUlid}/${initiatedCaseFile.ulid}`, + PartNumber: i + 1, + UploadId: initiatedCaseFile.uploadId, + }); + console.log(`Uploading chunk ${i + 1}/${totalChunks} for file:`, activeFileUpload.file.name); await uploadChunk(chunkBlob, initiatedCaseFile, i + 1, credentials); } From 81a5c5adc5c7d7161c2dbddeae95047085ccda32 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Thu, 28 Nov 2024 21:04:01 +0000 Subject: [PATCH 30/35] axios test --- .../upload-files/UploadFilesForm.tsx | 173 ++++++------------ 1 file changed, 51 insertions(+), 122 deletions(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 60e9ba8..1bf71f8 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -24,14 +24,11 @@ import { useRouter } from 'next/router'; import { useState } from 'react'; import { completeUpload, initiateUpload } from '../../api/cases'; import { commonLabels, commonTableLabels, fileOperationsLabels } from '../../common/labels'; -import { refreshCredentials } from '../../helpers/authService'; import { FileWithPath, formatFileSize } from '../../helpers/fileHelper'; import { InitiateUploadForm } from '../../models/CaseFiles'; import FileUpload from '../common-components/FileUpload'; import { UploadFilesProps } from './UploadFilesBody'; -const MINUTES_TO_MILLISECONDS = 60 * 1000; - interface FileUploadProgressRow { fileName: string; status: UploadStatus; @@ -47,11 +44,9 @@ enum UploadStatus { interface ActiveFileUpload { file: FileWithPath; - upoadDto: InitiateUploadForm; + uploadDto: InitiateUploadForm; } -export const ONE_MB = 1024 * 1024; -export const ONE_GB = ONE_MB * 1024; const MAX_PARALLEL_UPLOADS = 6; function UploadFilesForm(props: UploadFilesProps): JSX.Element { @@ -80,150 +75,84 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { let position = 0; while (position < selectedFiles.length) { const itemsForBatch = selectedFiles.slice(position, position + MAX_PARALLEL_UPLOADS); - console.log(`Uploading batch starting from position ${position}:`, itemsForBatch); await Promise.all(itemsForBatch.map((item) => uploadFile(item))); position += MAX_PARALLEL_UPLOADS; } setSelectedFiles([]); - } catch (error) { - console.error('Error during file upload submission:', error); } finally { setUploadInProgress(false); } } - async function blobToArrayBuffer(blob: Blob) { - console.log('Converting blob to ArrayBuffer...'); - if ('arrayBuffer' in blob) { - return await blob.arrayBuffer(); - } - - return new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => { - if (reader.result && typeof reader.result !== 'string') { - resolve(reader.result); - } - reject(new Error('Failed to convert blob to ArrayBuffer.')); - }; - reader.onerror = () => reject(new Error('Error reading blob.')); - reader.readAsArrayBuffer(blob); - }); - } - - async function uploadFilePartsAndComplete(activeFileUpload: ActiveFileUpload, chunkSizeBytes: number) { - console.log('Initiating upload for file:', activeFileUpload.file.name); - const initiatedCaseFile = await initiateUpload(activeFileUpload.upoadDto); - - let credentials = initiatedCaseFile.federationCredentials; - console.log('Upload initiated. Case details:', initiatedCaseFile); - - const credentialsInterval = setInterval(async () => { - try { - console.log('Refreshing credentials...'); - await refreshCredentials(); - const refreshRequest = await initiateUpload({ - ...activeFileUpload.upoadDto, - uploadId: initiatedCaseFile.uploadId, - }); - credentials = refreshRequest.federationCredentials; - console.log('Credentials refreshed successfully.'); - } catch (error) { - console.error('Error refreshing credentials:', error); - } - }, 20 * MINUTES_TO_MILLISECONDS); - - try { - const totalChunks = Math.ceil(activeFileUpload.file.size / chunkSizeBytes); - for (let i = 0; i < totalChunks; i++) { - const chunkBlob = activeFileUpload.file.slice(i * chunkSizeBytes, (i + 1) * chunkSizeBytes); - - // Add the console.log statement here to debug the parameters used for each part upload. - console.log('Upload Part Params:', { - Bucket: initiatedCaseFile.bucket, - Key: `${initiatedCaseFile.caseUlid}/${initiatedCaseFile.ulid}`, - PartNumber: i + 1, - UploadId: initiatedCaseFile.uploadId, - }); - - console.log(`Uploading chunk ${i + 1}/${totalChunks} for file:`, activeFileUpload.file.name); - await uploadChunk(chunkBlob, initiatedCaseFile, i + 1, credentials); - } - } catch (error) { - console.error('Error during file part upload:', error); - } finally { - clearInterval(credentialsInterval); - } - - try { - console.log('Completing upload for file:', activeFileUpload.file.name); - await completeUpload({ - caseUlid: props.caseId, - ulid: initiatedCaseFile.ulid, - uploadId: initiatedCaseFile.uploadId, - }); - updateFileProgress(activeFileUpload.file, UploadStatus.complete); - } catch (error) { - console.error('Error completing upload:', error); - } - } - - async function uploadChunk(chunkBlob: Blob, initiatedCaseFile: any, partNumber: number, credentials: any) { - const arrayBuffer = await blobToArrayBuffer(chunkBlob); - console.log(`Calculated ArrayBuffer for part ${partNumber}`); - - // Calculate SHA-256 hash using Web Crypto API - const hashBuffer = await crypto.subtle.digest('SHA-256', arrayBuffer); - const hashArray = Array.from(new Uint8Array(hashBuffer)); - const partHash = btoa(String.fromCharCode(...hashArray)); - - const url = initiatedCaseFile.preSignedUrls[partNumber - 1]; - console.log(`Uploading part ${partNumber} to URL:`, url); - - try { - await axios.put(url, arrayBuffer, { - headers: { - 'Content-Type': 'application/octet-stream', - 'x-amz-checksum-sha256': partHash, - 'x-amz-algorithm': 'AWS4-HMAC-SHA256', - 'x-amz-credential': `${credentials.accessKeyId}/${credentials.sessionToken}`, - 'x-amz-security-token': credentials.sessionToken, - }, - }); - console.log(`Successfully uploaded part ${partNumber}`); - } catch (error) { - console.error(`Upload of part ${partNumber} failed`, error); - throw error; - } - } - async function uploadFile(selectedFile: FileWithPath) { const fileSizeBytes = Math.max(selectedFile.size, 1); - const chunkSizeBytes = 300 * ONE_MB; + const chunkSizeBytes = 300 * 1024 * 1024; // 300MB try { - const activeFileUpload: ActiveFileUpload = { + const contentType = selectedFile.type ? selectedFile.type : 'text/plain'; + const activeFileUpload = { file: selectedFile, - upoadDto: { + uploadDto: { caseUlid: props.caseId, fileName: selectedFile.name, filePath: selectedFile.relativePath, fileSizeBytes, chunkSizeBytes, - contentType: selectedFile.type || 'text/plain', + contentType, reason, details, }, }; - console.log('Uploading file:', selectedFile.name); await uploadFilePartsAndComplete(activeFileUpload, chunkSizeBytes); - } catch (error) { + } catch (e) { updateFileProgress(selectedFile, UploadStatus.failed); - console.error('Upload failed for file:', selectedFile.name, error); + console.log('Upload failed', e); } } + async function uploadFilePartsAndComplete(activeFileUpload: ActiveFileUpload, chunkSizeBytes: number) { + const initiatedCaseFile = await initiateUpload(activeFileUpload.uploadDto); + + if ( + !initiatedCaseFile || + !initiatedCaseFile.uploadId || + !initiatedCaseFile.bucket || + !initiatedCaseFile.region + ) { + throw new Error('Invalid upload parameters'); + } + + const totalChunks = Math.ceil(activeFileUpload.file.size / chunkSizeBytes); + for (let i = 0; i < totalChunks; i++) { + const chunkBlob = activeFileUpload.file.slice(i * chunkSizeBytes, (i + 1) * chunkSizeBytes); + const formData = new FormData(); + formData.append('file', chunkBlob); + formData.append('partNumber', (i + 1).toString()); + formData.append('uploadId', initiatedCaseFile.uploadId); + formData.append('bucket', initiatedCaseFile.bucket); + formData.append('key', `${initiatedCaseFile.caseUlid}/${initiatedCaseFile.ulid}`); + + try { + const uploadUrl = `${initiatedCaseFile.bucket}/${initiatedCaseFile.uploadId}/part/${i + 1}`; // Simulated URL for each part upload + await axios.put(uploadUrl, chunkBlob, { + headers: { + 'Content-Type': 'application/octet-stream', + }, + }); + } catch (error) { + throw new Error(`Failed to upload part ${i + 1}: ${error}`); + } + } + + await completeUpload({ + caseUlid: props.caseId, + ulid: initiatedCaseFile.ulid, + uploadId: initiatedCaseFile.uploadId, + }); + updateFileProgress(activeFileUpload.file, UploadStatus.complete); + } + function updateFileProgress(selectedFile: FileWithPath, status: UploadStatus) { setUploadedFiles((prev) => { const newList = [...prev]; @@ -267,7 +196,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { - {uploadProgress.status} + {uploadProgress.status} ); From 5fcf1a3bf358a0fed0ec111914e9003898c27106 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Thu, 28 Nov 2024 21:34:42 +0000 Subject: [PATCH 31/35] axios test --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index 1bf71f8..c2bb0ef 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -87,7 +87,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { async function uploadFile(selectedFile: FileWithPath) { const fileSizeBytes = Math.max(selectedFile.size, 1); - const chunkSizeBytes = 300 * 1024 * 1024; // 300MB + const chunkSizeBytes = 50 * 1024 * 1024; // 50MB try { const contentType = selectedFile.type ? selectedFile.type : 'text/plain'; From 041dfa349dd86acdbe9a128047a88684421c1d1f Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Thu, 28 Nov 2024 21:54:09 +0000 Subject: [PATCH 32/35] axios test --- .../dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index c2bb0ef..f65b2d1 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -87,7 +87,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { async function uploadFile(selectedFile: FileWithPath) { const fileSizeBytes = Math.max(selectedFile.size, 1); - const chunkSizeBytes = 50 * 1024 * 1024; // 50MB + const chunkSizeBytes = 10 * 1024 * 1024; // 10MB try { const contentType = selectedFile.type ? selectedFile.type : 'text/plain'; From 92a840db2a8181b7a653c1cf50fd9dcd105fbaea Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Thu, 28 Nov 2024 22:29:19 +0000 Subject: [PATCH 33/35] axios test --- .../ui/src/components/upload-files/UploadFilesForm.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index f65b2d1..eeb6ae2 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -118,7 +118,8 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { !initiatedCaseFile || !initiatedCaseFile.uploadId || !initiatedCaseFile.bucket || - !initiatedCaseFile.region + !initiatedCaseFile.region || + !initiatedCaseFile.federationCredentials ) { throw new Error('Invalid upload parameters'); } @@ -134,8 +135,9 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { formData.append('key', `${initiatedCaseFile.caseUlid}/${initiatedCaseFile.ulid}`); try { - const uploadUrl = `${initiatedCaseFile.bucket}/${initiatedCaseFile.uploadId}/part/${i + 1}`; // Simulated URL for each part upload - await axios.put(uploadUrl, chunkBlob, { + // Assuming the upload URL is part of initiated upload response + // Use appropriate method to get the upload URL based on your implementation + await axios.put(initiatedCaseFile.bucket, chunkBlob, { headers: { 'Content-Type': 'application/octet-stream', }, From a389efcb5a229e46290db3d1682752b0904e69d0 Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Fri, 29 Nov 2024 00:02:43 +0000 Subject: [PATCH 34/35] revert changes and update client-s3 sdk --- source/dea-ui/ui/package.json | 2 +- .../upload-files/UploadFilesForm.tsx | 140 +++++++++++------- 2 files changed, 91 insertions(+), 51 deletions(-) diff --git a/source/dea-ui/ui/package.json b/source/dea-ui/ui/package.json index 792698f..75817a0 100644 --- a/source/dea-ui/ui/package.json +++ b/source/dea-ui/ui/package.json @@ -55,7 +55,7 @@ "@aws-sdk/client-athena": "~3.474.0", "@aws-sdk/client-cognito-identity": "~3.474.0", "@aws-sdk/client-cognito-identity-provider": "~3.474.0", - "@aws-sdk/client-s3": "~3.474.0", + "@aws-sdk/client-s3": "^3.700.0", "@aws/dea-app": "workspace:*", "@cloudscape-design/collection-hooks": "^1.0.20", "@cloudscape-design/component-toolkit": "~1.0.0-beta.25", diff --git a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx index eeb6ae2..374aa78 100644 --- a/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx +++ b/source/dea-ui/ui/src/components/upload-files/UploadFilesForm.tsx @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +import crypto from 'crypto'; +import { ChecksumAlgorithm, S3Client, UploadPartCommand, UploadPartCommandInput } from '@aws-sdk/client-s3'; import { Alert, Box, @@ -19,16 +21,18 @@ import { Table, Textarea, } from '@cloudscape-design/components'; -import axios from 'axios'; import { useRouter } from 'next/router'; import { useState } from 'react'; import { completeUpload, initiateUpload } from '../../api/cases'; import { commonLabels, commonTableLabels, fileOperationsLabels } from '../../common/labels'; +import { refreshCredentials } from '../../helpers/authService'; import { FileWithPath, formatFileSize } from '../../helpers/fileHelper'; import { InitiateUploadForm } from '../../models/CaseFiles'; import FileUpload from '../common-components/FileUpload'; import { UploadFilesProps } from './UploadFilesBody'; +const MINUTES_TO_MILLISECONDS = 60 * 1000; + interface FileUploadProgressRow { fileName: string; status: UploadStatus; @@ -44,10 +48,12 @@ enum UploadStatus { interface ActiveFileUpload { file: FileWithPath; - uploadDto: InitiateUploadForm; + upoadDto: InitiateUploadForm; } -const MAX_PARALLEL_UPLOADS = 6; +export const ONE_MB = 1024 * 1024; +export const ONE_GB = ONE_MB * 1024; +const MAX_PARALLEL_UPLOADS = 6; // One file concurrently for now. The backend requires a code refactor to deal with the TransactionConflictException thrown ocassionally. function UploadFilesForm(props: UploadFilesProps): JSX.Element { const [selectedFiles, setSelectedFiles] = useState([]); @@ -59,6 +65,7 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { const router = useRouter(); async function onSubmitHandler() { + // top level try/finally to set uploadInProgress bool state try { setUploadInProgress(true); @@ -85,15 +92,92 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { } } + async function blobToArrayBuffer(blob: Blob) { + if ('arrayBuffer' in blob) { + return await blob.arrayBuffer(); + } + + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + if (reader.result && typeof reader.result !== 'string') { + resolve(reader.result); + } + reject(); + }; + reader.onerror = () => reject(); + reader.readAsArrayBuffer(blob); + }); + } + + async function uploadFilePartsAndComplete(activeFileUpload: ActiveFileUpload, chunkSizeBytes: number) { + const initiatedCaseFile = await initiateUpload(activeFileUpload.upoadDto); + + let federationS3Client = new S3Client({ + credentials: initiatedCaseFile.federationCredentials, + region: initiatedCaseFile.region, + useAccelerateEndpoint: true, + }); + + const credentialsInterval = setInterval(async () => { + await refreshCredentials(); + const refreshRequest = await initiateUpload({ + ...activeFileUpload.upoadDto, + uploadId: initiatedCaseFile.uploadId, + }); + federationS3Client = new S3Client({ + credentials: refreshRequest.federationCredentials, + region: initiatedCaseFile.region, + useAccelerateEndpoint: true, + }); + }, 20 * MINUTES_TO_MILLISECONDS); + + try { + const totalChunks = Math.ceil(activeFileUpload.file.size / chunkSizeBytes); + for (let i = 0; i < totalChunks; i++) { + const chunkBlob = activeFileUpload.file.slice(i * chunkSizeBytes, (i + 1) * chunkSizeBytes); + + const arrayFromBlob = new Uint8Array(await blobToArrayBuffer(chunkBlob)); + const partHash = crypto.createHash('sha256').update(arrayFromBlob).digest('base64'); + + const uploadInput: UploadPartCommandInput = { + Bucket: initiatedCaseFile.bucket, + Key: `${initiatedCaseFile.caseUlid}/${initiatedCaseFile.ulid}`, + PartNumber: i + 1, + UploadId: initiatedCaseFile.uploadId, + Body: arrayFromBlob, + ChecksumSHA256: partHash, + ChecksumAlgorithm: ChecksumAlgorithm.SHA256, + }; + const uploadCommand = new UploadPartCommand(uploadInput); + await federationS3Client.send(uploadCommand); + } + } finally { + clearInterval(credentialsInterval); + } + + await completeUpload({ + caseUlid: props.caseId, + ulid: initiatedCaseFile.ulid, + uploadId: initiatedCaseFile.uploadId, + }); + updateFileProgress(activeFileUpload.file, UploadStatus.complete); + } + async function uploadFile(selectedFile: FileWithPath) { const fileSizeBytes = Math.max(selectedFile.size, 1); - const chunkSizeBytes = 10 * 1024 * 1024; // 10MB - + // Trying to use small chunk size (50MB) to reduce memory use. + // Maximum object size 5 TiB + // Maximum number of parts per upload 10,000 + // 5 MiB to 5 GiB. There is no minimum size limit on the last part of your multipart upload. + //const chunkSizeBytes = Math.max(selectedFile.size / 10_000, 50 * ONE_MB); + const chunkSizeBytes = 300 * ONE_MB; + // per file try/finally state to initiate uploads try { const contentType = selectedFile.type ? selectedFile.type : 'text/plain'; const activeFileUpload = { file: selectedFile, - uploadDto: { + upoadDto: { caseUlid: props.caseId, fileName: selectedFile.name, filePath: selectedFile.relativePath, @@ -111,50 +195,6 @@ function UploadFilesForm(props: UploadFilesProps): JSX.Element { } } - async function uploadFilePartsAndComplete(activeFileUpload: ActiveFileUpload, chunkSizeBytes: number) { - const initiatedCaseFile = await initiateUpload(activeFileUpload.uploadDto); - - if ( - !initiatedCaseFile || - !initiatedCaseFile.uploadId || - !initiatedCaseFile.bucket || - !initiatedCaseFile.region || - !initiatedCaseFile.federationCredentials - ) { - throw new Error('Invalid upload parameters'); - } - - const totalChunks = Math.ceil(activeFileUpload.file.size / chunkSizeBytes); - for (let i = 0; i < totalChunks; i++) { - const chunkBlob = activeFileUpload.file.slice(i * chunkSizeBytes, (i + 1) * chunkSizeBytes); - const formData = new FormData(); - formData.append('file', chunkBlob); - formData.append('partNumber', (i + 1).toString()); - formData.append('uploadId', initiatedCaseFile.uploadId); - formData.append('bucket', initiatedCaseFile.bucket); - formData.append('key', `${initiatedCaseFile.caseUlid}/${initiatedCaseFile.ulid}`); - - try { - // Assuming the upload URL is part of initiated upload response - // Use appropriate method to get the upload URL based on your implementation - await axios.put(initiatedCaseFile.bucket, chunkBlob, { - headers: { - 'Content-Type': 'application/octet-stream', - }, - }); - } catch (error) { - throw new Error(`Failed to upload part ${i + 1}: ${error}`); - } - } - - await completeUpload({ - caseUlid: props.caseId, - ulid: initiatedCaseFile.ulid, - uploadId: initiatedCaseFile.uploadId, - }); - updateFileProgress(activeFileUpload.file, UploadStatus.complete); - } - function updateFileProgress(selectedFile: FileWithPath, status: UploadStatus) { setUploadedFiles((prev) => { const newList = [...prev]; From 4a1824ba5173da1b6f1e0a90f309e341cb3c236f Mon Sep 17 00:00:00 2001 From: Mahruf Iqbal Date: Fri, 29 Nov 2024 00:16:49 +0000 Subject: [PATCH 35/35] revert client-s3 sdk --- source/dea-ui/ui/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dea-ui/ui/package.json b/source/dea-ui/ui/package.json index 75817a0..792698f 100644 --- a/source/dea-ui/ui/package.json +++ b/source/dea-ui/ui/package.json @@ -55,7 +55,7 @@ "@aws-sdk/client-athena": "~3.474.0", "@aws-sdk/client-cognito-identity": "~3.474.0", "@aws-sdk/client-cognito-identity-provider": "~3.474.0", - "@aws-sdk/client-s3": "^3.700.0", + "@aws-sdk/client-s3": "~3.474.0", "@aws/dea-app": "workspace:*", "@cloudscape-design/collection-hooks": "^1.0.20", "@cloudscape-design/component-toolkit": "~1.0.0-beta.25",