From 72448229d78251c86499643f29a28390f86ef80e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 7 Jun 2023 15:29:04 +0300 Subject: [PATCH 01/37] :rocket: Release 0.232.0 (#6399) Co-authored-by: Alex Grozav --- CHANGELOG.md | 31 +++++++++++++++++++++++++++++ package.json | 2 +- packages/cli/package.json | 2 +- packages/core/package.json | 2 +- packages/design-system/package.json | 2 +- packages/editor-ui/package.json | 2 +- packages/node-dev/package.json | 2 +- packages/nodes-base/package.json | 2 +- packages/workflow/package.json | 2 +- 9 files changed, 39 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96c57ce8d5cfb..dc482d324d2f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,34 @@ +# [0.232.0](https://github.com/n8n-io/n8n/compare/n8n@0.231.0...n8n@0.232.0) (2023-06-07) + + +### Bug Fixes + +* **core:** RMC boolean value fix ([#6397](https://github.com/n8n-io/n8n/issues/6397)) ([28bb797](https://github.com/n8n-io/n8n/commit/28bb797bb0ea59b66a7641fc116f47c25564c21a)) +* **Date & Time Node:** Reset responseData at end of loop ([#6385](https://github.com/n8n-io/n8n/issues/6385)) ([eaa8648](https://github.com/n8n-io/n8n/commit/eaa8648f2bf61074eae6dcd7355f8f107a31388e)) +* **editor:** Add button to refresh branches ([#6387](https://github.com/n8n-io/n8n/issues/6387)) ([ce57816](https://github.com/n8n-io/n8n/commit/ce578162f4e44a6cc1774ab217967110b254ab3f)) +* **editor:** Add secondary icon to menu items ([#6351](https://github.com/n8n-io/n8n/issues/6351)) ([3dd2601](https://github.com/n8n-io/n8n/commit/3dd260168eb627fd7fbed740bc97fa7f6289628f)) +* **editor:** Add Set up version control CTA ([#6356](https://github.com/n8n-io/n8n/issues/6356)) ([e72521d](https://github.com/n8n-io/n8n/commit/e72521d5ec7a5e57dc311defb70f1fe19054b0f0)) +* **editor:** Adding branch color ([#6380](https://github.com/n8n-io/n8n/issues/6380)) ([dba3f44](https://github.com/n8n-io/n8n/commit/dba3f44bc00de68113cc98db9afc6267f56ec04c)) +* **editor:** Fix an issue with connections breaking during renaming ([#6358](https://github.com/n8n-io/n8n/issues/6358)) ([0f2bc6b](https://github.com/n8n-io/n8n/commit/0f2bc6b73711597fdf008ee54665d9bed82a1a9e)) +* **editor:** Fix hard-coded parameter names for code editors ([#6372](https://github.com/n8n-io/n8n/issues/6372)) ([f61b776](https://github.com/n8n-io/n8n/commit/f61b776beac961fa58c6c69371c69ae1e74ef83e)) +* **editor:** Fix typing `$` in inline expression field reloading node parameters form ([#6374](https://github.com/n8n-io/n8n/issues/6374)) ([4c0d4eb](https://github.com/n8n-io/n8n/commit/4c0d4ebd9917e52512e85a5cad2c93b554e0e212)) +* **editor:** Pin all data regardless of pagination ([#6346](https://github.com/n8n-io/n8n/issues/6346)) ([f88029f](https://github.com/n8n-io/n8n/commit/f88029f308356c1c8271d7345ecbbd6e91c41b3d)) +* **editor:** Remove explicit parameter name scanning for code editors ([#6390](https://github.com/n8n-io/n8n/issues/6390)) ([97295f6](https://github.com/n8n-io/n8n/commit/97295f67f0f8509ac6ba0d4ce38ce12582dff074)) +* **editor:** Remove root level tag selector from css module to avoid making it a global style ([#6392](https://github.com/n8n-io/n8n/issues/6392)) ([cc37f21](https://github.com/n8n-io/n8n/commit/cc37f21aa27f3536f2043b5ff5da944388ac5504)) +* **editor:** Update version control setup CTA tooltip ([#6393](https://github.com/n8n-io/n8n/issues/6393)) ([385b3e8](https://github.com/n8n-io/n8n/commit/385b3e871a9307c36428f8239a5db318d71948c1)) +* Improve executions list polling performance ([#6355](https://github.com/n8n-io/n8n/issues/6355)) ([b5cabfe](https://github.com/n8n-io/n8n/commit/b5cabfef54e186f59580112a90566099bb79305e)) +* **Ldap Node:** Add DN field to update operation ([#6371](https://github.com/n8n-io/n8n/issues/6371)) ([9396e7e](https://github.com/n8n-io/n8n/commit/9396e7eb585ab9d6fda742b0d234c4262570af93)) +* Show actual execution data for production executions even if pin data exists ([#6302](https://github.com/n8n-io/n8n/issues/6302)) ([4eb8437](https://github.com/n8n-io/n8n/commit/4eb8437196a298a64f039aff51ba030dc21abb08)) + + +### Features + +* **Crypto Node:** Add support for hash and hmac on binary data ([#6359](https://github.com/n8n-io/n8n/issues/6359)) ([406a405](https://github.com/n8n-io/n8n/commit/406a405dd153833057286a27d04278ef71ceef3d)) +* **editor:** Make WF name a link on /executions ([#6354](https://github.com/n8n-io/n8n/issues/6354)) ([6ddf161](https://github.com/n8n-io/n8n/commit/6ddf16128b4ab47db716eeab89f7526558073f56)) +* New trigger PostgreSQL ([#5495](https://github.com/n8n-io/n8n/issues/5495)) ([4488f93](https://github.com/n8n-io/n8n/commit/4488f93c39b0ec0b4a0eff98391b46db6a2eed78)) +* Version control mvp ([#6271](https://github.com/n8n-io/n8n/issues/6271)) ([1b32141](https://github.com/n8n-io/n8n/commit/1b321416c0ba5371e0016398ae660ce298b8cdd6)) + + # [0.231.0](https://github.com/n8n-io/n8n/compare/n8n@0.230.0...n8n@0.231.0) (2023-05-31) diff --git a/package.json b/package.json index fee3fcbe49f3c..8eea8b61a408b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "n8n", - "version": "0.231.0", + "version": "0.232.0", "private": true, "homepage": "https://n8n.io", "engines": { diff --git a/packages/cli/package.json b/packages/cli/package.json index f5c36aff4c838..831dd674402cb 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "n8n", - "version": "0.231.0", + "version": "0.232.0", "description": "n8n Workflow Automation Tool", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", diff --git a/packages/core/package.json b/packages/core/package.json index 5624965d8443e..fc0e8fe3f26aa 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "n8n-core", - "version": "0.170.0", + "version": "0.171.0", "description": "Core functionality of n8n", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", diff --git a/packages/design-system/package.json b/packages/design-system/package.json index ec3248f1e167e..83e913fb7cfc1 100644 --- a/packages/design-system/package.json +++ b/packages/design-system/package.json @@ -1,6 +1,6 @@ { "name": "n8n-design-system", - "version": "0.67.0", + "version": "0.68.0", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", "author": { diff --git a/packages/editor-ui/package.json b/packages/editor-ui/package.json index 20e26b0adb082..7d8276d54b02a 100644 --- a/packages/editor-ui/package.json +++ b/packages/editor-ui/package.json @@ -1,6 +1,6 @@ { "name": "n8n-editor-ui", - "version": "0.197.0", + "version": "0.198.0", "description": "Workflow Editor UI for n8n", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", diff --git a/packages/node-dev/package.json b/packages/node-dev/package.json index eb3b3c6d32542..3dc7399fb8b98 100644 --- a/packages/node-dev/package.json +++ b/packages/node-dev/package.json @@ -1,6 +1,6 @@ { "name": "n8n-node-dev", - "version": "0.109.0", + "version": "0.110.0", "description": "CLI to simplify n8n credentials/node development", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index e54c094a7e3e3..6081b8d335993 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -1,6 +1,6 @@ { "name": "n8n-nodes-base", - "version": "0.229.0", + "version": "0.230.0", "description": "Base nodes of n8n", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", diff --git a/packages/workflow/package.json b/packages/workflow/package.json index 4cbc0f27ec768..26521ec5f2ae0 100644 --- a/packages/workflow/package.json +++ b/packages/workflow/package.json @@ -1,6 +1,6 @@ { "name": "n8n-workflow", - "version": "0.151.0", + "version": "0.152.0", "description": "Workflow base code of n8n", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", From fecf62296b92e2085ecb06a43e9ef15677f81a2c Mon Sep 17 00:00:00 2001 From: Deborah Date: Tue, 13 Jun 2023 15:06:28 +0100 Subject: [PATCH 02/37] docs: Fix Postgres trigger docs URL (#6403) fix postgres trigger docs URL --- packages/nodes-base/nodes/Postgres/PostgresTrigger.node.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nodes-base/nodes/Postgres/PostgresTrigger.node.json b/packages/nodes-base/nodes/Postgres/PostgresTrigger.node.json index 5f733549cc849..49f18daf8e899 100644 --- a/packages/nodes-base/nodes/Postgres/PostgresTrigger.node.json +++ b/packages/nodes-base/nodes/Postgres/PostgresTrigger.node.json @@ -11,7 +11,7 @@ ], "primaryDocumentation": [ { - "url": "https://docs.n8n.io/integrations/builtin/app-nodes/n8n-nodes-base.postgres/" + "url": "https://docs.n8n.io/integrations/builtin/trigger-nodes/n8n-nodes-base.postgrestrigger/" } ] } From d041602754a694e67e81c1bbff87c13bf91bdd5d Mon Sep 17 00:00:00 2001 From: Jon Date: Tue, 13 Jun 2023 15:44:35 +0100 Subject: [PATCH 03/37] fix(LinkedIn Node): Fix issue with posting as user or organization (#6414) --- .../nodes-base/credentials/LinkedInOAuth2Api.credentials.ts | 2 +- packages/nodes-base/nodes/LinkedIn/GenericFunctions.ts | 2 +- packages/nodes-base/nodes/LinkedIn/LinkedIn.node.ts | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/nodes-base/credentials/LinkedInOAuth2Api.credentials.ts b/packages/nodes-base/credentials/LinkedInOAuth2Api.credentials.ts index c87c2999baf15..065115a69822b 100644 --- a/packages/nodes-base/credentials/LinkedInOAuth2Api.credentials.ts +++ b/packages/nodes-base/credentials/LinkedInOAuth2Api.credentials.ts @@ -42,7 +42,7 @@ export class LinkedInOAuth2Api implements ICredentialType { name: 'scope', type: 'hidden', default: - '=r_liteprofile,r_emailaddress,w_member_social{{$self["organizationSupport"] === true ? ",w_organization_social":""}}', + '=w_member_social{{$self["organizationSupport"] === true ? ",w_organization_social":",r_liteprofile,r_emailaddress"}}', description: 'Standard scopes for posting on behalf of a user or organization. See this resource .', }, diff --git a/packages/nodes-base/nodes/LinkedIn/GenericFunctions.ts b/packages/nodes-base/nodes/LinkedIn/GenericFunctions.ts index 1c1caf9047b9e..093356a14ba1f 100644 --- a/packages/nodes-base/nodes/LinkedIn/GenericFunctions.ts +++ b/packages/nodes-base/nodes/LinkedIn/GenericFunctions.ts @@ -29,7 +29,7 @@ export async function linkedInApiRequest( headers: { Accept: 'application/json', 'X-Restli-Protocol-Version': '2.0.0', - 'LinkedIn-Version': '202301', + 'LinkedIn-Version': '202304', }, method, body, diff --git a/packages/nodes-base/nodes/LinkedIn/LinkedIn.node.ts b/packages/nodes-base/nodes/LinkedIn/LinkedIn.node.ts index 434fa2cc8c8f9..3b0571cd93a7a 100644 --- a/packages/nodes-base/nodes/LinkedIn/LinkedIn.node.ts +++ b/packages/nodes-base/nodes/LinkedIn/LinkedIn.node.ts @@ -107,7 +107,6 @@ export class LinkedIn implements INodeType { lifecycleState: 'PUBLISHED', distribution: { feedDistribution: 'MAIN_FEED', - targetEnties: [], thirdPartyDistributionChannels: [], }, visibility, From 75c0ab03f8379bb5426cef87045b67b458c97d70 Mon Sep 17 00:00:00 2001 From: Csaba Tuncsik Date: Tue, 13 Jun 2023 17:09:17 +0200 Subject: [PATCH 04/37] fix(editor): Hide version control main menu component if no feature flag (#6419) * fix(editor): Hide version control main menu component if no feature flag * fix(editor): Update unit tes * test(editor): Test for feature flag --- .../src/components/MainSidebarVersionControl.vue | 3 ++- .../__tests__/MainSidebarVersionControl.test.ts | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/editor-ui/src/components/MainSidebarVersionControl.vue b/packages/editor-ui/src/components/MainSidebarVersionControl.vue index 14496ccb61a4f..86e47b9607168 100644 --- a/packages/editor-ui/src/components/MainSidebarVersionControl.vue +++ b/packages/editor-ui/src/components/MainSidebarVersionControl.vue @@ -25,7 +25,7 @@ const tooltipOpenDelay = ref(300); const currentBranch = computed(() => { return versionControlStore.preferences.branchName; }); - +const featureEnabled = computed(() => window.localStorage.getItem('version-control')); const setupButtonTooltipPlacement = computed(() => (props.isCollapsed ? 'right' : 'top')); async function pushWorkfolder() { @@ -80,6 +80,7 @@ const goToVersionControlSetup = async () => { From 109442f38f9868d51d85ba4f88d185910f4f2688 Mon Sep 17 00:00:00 2001 From: agobrech <45268029+agobrech@users.noreply.github.com> Date: Thu, 15 Jun 2023 13:19:22 +0200 Subject: [PATCH 14/37] feat(AwsS3 Node): Small overhaul of the node with multipart uploading (#6017) * Create new version for S3 * Update S3 to new aws s3 methods * Switch from SAOP to Rest api * Add multipart request * Seperate stream into chunks and send the multipart * Fix chunk into buffer * Fix wrong sha256 mismatch * Add abort multipart on error * Complete multipart and list parts * Change format to xml and add a minmum size of 5MB for each part * Fix returned data for uploading a file * Remove console.logs * Seperate needed headers and multipart headers * Throw error on aborting, remove console.logs * Remove soap request from generic function * Keep buffer * Add unit test for V2 * fix upload file content body * removed unused import * Fix bug where the object was too smal and used only one part * Fix naming for bucket name * Fix issue with file name not returning data * Add parent name * Remove console.logs * Add content type * fix headears for other upload mode --------- Co-authored-by: Marcus --- .../nodes-base/credentials/Aws.credentials.ts | 1 - .../nodes-base/nodes/Aws/S3/AwsS3.node.ts | 931 +-------------- .../nodes/Aws/S3/V1/AwsS3V1.node.ts | 915 ++++++++++++++ .../Aws/S3/{ => V1}/BucketDescription.ts | 0 .../nodes/Aws/S3/{ => V1}/FileDescription.ts | 0 .../Aws/S3/{ => V1}/FolderDescription.ts | 0 .../nodes/Aws/S3/{ => V1}/GenericFunctions.ts | 0 .../nodes/Aws/S3/V2/AwsS3V2.node.ts | 1058 +++++++++++++++++ .../nodes/Aws/S3/V2/BucketDescription.ts | 321 +++++ .../nodes/Aws/S3/V2/FileDescription.ts | 841 +++++++++++++ .../nodes/Aws/S3/V2/FolderDescription.ts | 241 ++++ .../nodes/Aws/S3/V2/GenericFunctions.ts | 132 ++ .../AwsS3.file.upload.V1.workflow.json} | 0 .../Aws/S3/test/{ => V1}/AwsS3.node.test.ts | 2 +- .../V2/AwsS3.file.upload.V2.workflow.json | 97 ++ .../nodes/Aws/S3/test/V2/AwsS3.node.test.ts | 48 + packages/nodes-base/nodes/S3/S3.node.ts | 6 +- 17 files changed, 3682 insertions(+), 911 deletions(-) create mode 100644 packages/nodes-base/nodes/Aws/S3/V1/AwsS3V1.node.ts rename packages/nodes-base/nodes/Aws/S3/{ => V1}/BucketDescription.ts (100%) rename packages/nodes-base/nodes/Aws/S3/{ => V1}/FileDescription.ts (100%) rename packages/nodes-base/nodes/Aws/S3/{ => V1}/FolderDescription.ts (100%) rename packages/nodes-base/nodes/Aws/S3/{ => V1}/GenericFunctions.ts (100%) create mode 100644 packages/nodes-base/nodes/Aws/S3/V2/AwsS3V2.node.ts create mode 100644 packages/nodes-base/nodes/Aws/S3/V2/BucketDescription.ts create mode 100644 packages/nodes-base/nodes/Aws/S3/V2/FileDescription.ts create mode 100644 packages/nodes-base/nodes/Aws/S3/V2/FolderDescription.ts create mode 100644 packages/nodes-base/nodes/Aws/S3/V2/GenericFunctions.ts rename packages/nodes-base/nodes/Aws/S3/test/{AwsS3.file.upload.workflow.json => V1/AwsS3.file.upload.V1.workflow.json} (100%) rename packages/nodes-base/nodes/Aws/S3/test/{ => V1}/AwsS3.node.test.ts (96%) create mode 100644 packages/nodes-base/nodes/Aws/S3/test/V2/AwsS3.file.upload.V2.workflow.json create mode 100644 packages/nodes-base/nodes/Aws/S3/test/V2/AwsS3.node.test.ts diff --git a/packages/nodes-base/credentials/Aws.credentials.ts b/packages/nodes-base/credentials/Aws.credentials.ts index 4a7fd4efae5fc..b7e8067d11cef 100644 --- a/packages/nodes-base/credentials/Aws.credentials.ts +++ b/packages/nodes-base/credentials/Aws.credentials.ts @@ -287,7 +287,6 @@ export class Aws implements ICredentialType { let body = requestOptions.body; let region = credentials.region; let query = requestOptions.qs?.query as IDataObject; - // ! Workaround as we still use the OptionsWithUri interface which uses uri instead of url // ! To change when we replace the interface with IHttpRequestOptions const requestWithUri = requestOptions as unknown as OptionsWithUri; diff --git a/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts b/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts index b39940377e52f..50a4c1738a62d 100644 --- a/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts +++ b/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts @@ -1,908 +1,27 @@ -import { paramCase, snakeCase } from 'change-case'; - -import { createHash } from 'crypto'; - -import { Builder } from 'xml2js'; - -import type { - IDataObject, - IExecuteFunctions, - INodeExecutionData, - INodeType, - INodeTypeDescription, -} from 'n8n-workflow'; -import { NodeOperationError } from 'n8n-workflow'; - -import { bucketFields, bucketOperations } from './BucketDescription'; - -import { folderFields, folderOperations } from './FolderDescription'; - -import { fileFields, fileOperations } from './FileDescription'; - -import { - awsApiRequestREST, - awsApiRequestSOAP, - awsApiRequestSOAPAllItems, -} from './GenericFunctions'; - -export class AwsS3 implements INodeType { - description: INodeTypeDescription = { - displayName: 'AWS S3', - name: 'awsS3', - icon: 'file:s3.svg', - group: ['output'], - version: 1, - subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', - description: 'Sends data to AWS S3', - defaults: { - name: 'AWS S3', - }, - inputs: ['main'], - outputs: ['main'], - credentials: [ - { - name: 'aws', - required: true, - }, - ], - properties: [ - { - displayName: 'Resource', - name: 'resource', - type: 'options', - noDataExpression: true, - options: [ - { - name: 'Bucket', - value: 'bucket', - }, - { - name: 'File', - value: 'file', - }, - { - name: 'Folder', - value: 'folder', - }, - ], - default: 'file', - }, - // BUCKET - ...bucketOperations, - ...bucketFields, - // FOLDER - ...folderOperations, - ...folderFields, - // UPLOAD - ...fileOperations, - ...fileFields, - ], - }; - - async execute(this: IExecuteFunctions): Promise { - const items = this.getInputData(); - const returnData: INodeExecutionData[] = []; - const qs: IDataObject = {}; - let responseData; - const resource = this.getNodeParameter('resource', 0); - const operation = this.getNodeParameter('operation', 0); - for (let i = 0; i < items.length; i++) { - const headers: IDataObject = {}; - try { - if (resource === 'bucket') { - //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html - if (operation === 'create') { - const credentials = await this.getCredentials('aws'); - const name = this.getNodeParameter('name', i) as string; - const additionalFields = this.getNodeParameter('additionalFields', i); - if (additionalFields.acl) { - headers['x-amz-acl'] = paramCase(additionalFields.acl as string); - } - if (additionalFields.bucketObjectLockEnabled) { - headers['x-amz-bucket-object-lock-enabled'] = - additionalFields.bucketObjectLockEnabled as boolean; - } - if (additionalFields.grantFullControl) { - headers['x-amz-grant-full-control'] = ''; - } - if (additionalFields.grantRead) { - headers['x-amz-grant-read'] = ''; - } - if (additionalFields.grantReadAcp) { - headers['x-amz-grant-read-acp'] = ''; - } - if (additionalFields.grantWrite) { - headers['x-amz-grant-write'] = ''; - } - if (additionalFields.grantWriteAcp) { - headers['x-amz-grant-write-acp'] = ''; - } - let region = credentials.region as string; - - if (additionalFields.region) { - region = additionalFields.region as string; - } - - const body: IDataObject = { - CreateBucketConfiguration: { - $: { - xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/', - }, - }, - }; - let data = ''; - // if credentials has the S3 defaul region (us-east-1) the body (XML) does not have to be sent. - if (region !== 'us-east-1') { - // @ts-ignore - body.CreateBucketConfiguration.LocationConstraint = [region]; - const builder = new Builder(); - data = builder.buildObject(body); - } - responseData = await awsApiRequestSOAP.call( - this, - `${name}.s3`, - 'PUT', - '', - data, - qs, - headers, - ); - const executionData = this.helpers.constructExecutionMetaData( - this.helpers.returnJsonArray({ success: true }), - { itemData: { item: i } }, - ); - returnData.push(...executionData); - } - - // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucket.html - if (operation === 'delete') { - const name = this.getNodeParameter('name', i) as string; - - responseData = await awsApiRequestSOAP.call( - this, - `${name}.s3`, - 'DELETE', - '', - '', - {}, - headers, - ); - const executionData = this.helpers.constructExecutionMetaData( - this.helpers.returnJsonArray({ success: true }), - { itemData: { item: i } }, - ); - returnData.push(...executionData); - } - - //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html - if (operation === 'getAll') { - const returnAll = this.getNodeParameter('returnAll', 0); - if (returnAll) { - responseData = await awsApiRequestSOAPAllItems.call( - this, - 'ListAllMyBucketsResult.Buckets.Bucket', - 's3', - 'GET', - '', - ); - } else { - qs.limit = this.getNodeParameter('limit', 0); - responseData = await awsApiRequestSOAPAllItems.call( - this, - 'ListAllMyBucketsResult.Buckets.Bucket', - 's3', - 'GET', - '', - '', - qs, - ); - responseData = responseData.slice(0, qs.limit); - } - const executionData = this.helpers.constructExecutionMetaData( - this.helpers.returnJsonArray(responseData as IDataObject[]), - { itemData: { item: i } }, - ); - returnData.push(...executionData); - } - - //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html - if (operation === 'search') { - const bucketName = this.getNodeParameter('bucketName', i) as string; - const returnAll = this.getNodeParameter('returnAll', 0); - const additionalFields = this.getNodeParameter('additionalFields', 0); - - if (additionalFields.prefix) { - qs.prefix = additionalFields.prefix as string; - } - - if (additionalFields.encodingType) { - qs['encoding-type'] = additionalFields.encodingType as string; - } - - if (additionalFields.delimiter) { - qs.delimiter = additionalFields.delimiter as string; - } - - if (additionalFields.fetchOwner) { - qs['fetch-owner'] = additionalFields.fetchOwner as string; - } - - if (additionalFields.startAfter) { - qs['start-after'] = additionalFields.startAfter as string; - } - - if (additionalFields.requesterPays) { - qs['x-amz-request-payer'] = 'requester'; - } - - qs['list-type'] = 2; - - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { - location: '', - }); - - const region = responseData.LocationConstraint._ as string; - - if (returnAll) { - responseData = await awsApiRequestSOAPAllItems.call( - this, - 'ListBucketResult.Contents', - `${bucketName}.s3`, - 'GET', - '', - '', - qs, - {}, - {}, - region, - ); - } else { - qs['max-keys'] = this.getNodeParameter('limit', 0); - responseData = await awsApiRequestSOAP.call( - this, - `${bucketName}.s3`, - 'GET', - '', - '', - qs, - {}, - {}, - region, - ); - responseData = responseData.ListBucketResult.Contents; - } - const executionData = this.helpers.constructExecutionMetaData( - this.helpers.returnJsonArray(responseData as IDataObject[]), - { itemData: { item: i } }, - ); - returnData.push(...executionData); - } - } - if (resource === 'folder') { - //https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html - if (operation === 'create') { - const bucketName = this.getNodeParameter('bucketName', i) as string; - const folderName = this.getNodeParameter('folderName', i) as string; - const additionalFields = this.getNodeParameter('additionalFields', i); - let path = `/${folderName}/`; - - if (additionalFields.requesterPays) { - headers['x-amz-request-payer'] = 'requester'; - } - if (additionalFields.parentFolderKey) { - path = `/${additionalFields.parentFolderKey}${folderName}/`; - } - if (additionalFields.storageClass) { - headers['x-amz-storage-class'] = snakeCase( - additionalFields.storageClass as string, - ).toUpperCase(); - } - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { - location: '', - }); - - const region = responseData.LocationConstraint._; - - responseData = await awsApiRequestSOAP.call( - this, - `${bucketName}.s3`, - 'PUT', - path, - '', - qs, - headers, - {}, - region as string, - ); - const executionData = this.helpers.constructExecutionMetaData( - this.helpers.returnJsonArray({ success: true }), - { itemData: { item: i } }, - ); - returnData.push(...executionData); - } - //https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html - if (operation === 'delete') { - const bucketName = this.getNodeParameter('bucketName', i) as string; - const folderKey = this.getNodeParameter('folderKey', i) as string; - - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { - location: '', - }); - - const region = responseData.LocationConstraint._; - - responseData = await awsApiRequestSOAPAllItems.call( - this, - 'ListBucketResult.Contents', - `${bucketName}.s3`, - 'GET', - '/', - '', - { 'list-type': 2, prefix: folderKey }, - {}, - {}, - region as string, - ); - - // folder empty then just delete it - if (responseData.length === 0) { - responseData = await awsApiRequestSOAP.call( - this, - `${bucketName}.s3`, - 'DELETE', - `/${folderKey}`, - '', - qs, - {}, - {}, - region as string, - ); - - responseData = { deleted: [{ Key: folderKey }] }; - } else { - // delete everything inside the folder - const body: IDataObject = { - Delete: { - $: { - xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/', - }, - Object: [], - }, - }; - - for (const childObject of responseData) { - //@ts-ignore - (body.Delete.Object as IDataObject[]).push({ - Key: childObject.Key as string, - }); - } - - const builder = new Builder(); - const data = builder.buildObject(body); - - headers['Content-MD5'] = createHash('md5').update(data).digest('base64'); - - headers['Content-Type'] = 'application/xml'; - - responseData = await awsApiRequestSOAP.call( - this, - `${bucketName}.s3`, - 'POST', - '/', - data, - { delete: '' }, - headers, - {}, - region as string, - ); - - responseData = { deleted: responseData.DeleteResult.Deleted }; - } - const executionData = this.helpers.constructExecutionMetaData( - this.helpers.returnJsonArray(responseData), - { itemData: { item: i } }, - ); - returnData.push(...executionData); - } - //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html - if (operation === 'getAll') { - const bucketName = this.getNodeParameter('bucketName', i) as string; - const returnAll = this.getNodeParameter('returnAll', 0); - const options = this.getNodeParameter('options', 0); - - if (options.folderKey) { - qs.prefix = options.folderKey as string; - } - - if (options.fetchOwner) { - qs['fetch-owner'] = options.fetchOwner as string; - } - - qs['list-type'] = 2; - - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { - location: '', - }); - - const region = responseData.LocationConstraint._; - - if (returnAll) { - responseData = await awsApiRequestSOAPAllItems.call( - this, - 'ListBucketResult.Contents', - `${bucketName}.s3`, - 'GET', - '', - '', - qs, - {}, - {}, - region as string, - ); - } else { - qs.limit = this.getNodeParameter('limit', 0); - responseData = await awsApiRequestSOAPAllItems.call( - this, - 'ListBucketResult.Contents', - `${bucketName}.s3`, - 'GET', - '', - '', - qs, - {}, - {}, - region as string, - ); - } - if (Array.isArray(responseData)) { - responseData = responseData.filter( - (e: IDataObject) => - (e.Key as string).endsWith('/') && e.Size === '0' && e.Key !== options.folderKey, - ); - if (qs.limit) { - responseData = responseData.splice(0, qs.limit as number); - } - const executionData = this.helpers.constructExecutionMetaData( - this.helpers.returnJsonArray(responseData), - { itemData: { item: i } }, - ); - returnData.push(...executionData); - } - } - } - if (resource === 'file') { - //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html - if (operation === 'copy') { - const sourcePath = this.getNodeParameter('sourcePath', i) as string; - const destinationPath = this.getNodeParameter('destinationPath', i) as string; - const additionalFields = this.getNodeParameter('additionalFields', i); - - headers['x-amz-copy-source'] = sourcePath; - - if (additionalFields.requesterPays) { - headers['x-amz-request-payer'] = 'requester'; - } - if (additionalFields.storageClass) { - headers['x-amz-storage-class'] = snakeCase( - additionalFields.storageClass as string, - ).toUpperCase(); - } - if (additionalFields.acl) { - headers['x-amz-acl'] = paramCase(additionalFields.acl as string); - } - if (additionalFields.grantFullControl) { - headers['x-amz-grant-full-control'] = ''; - } - if (additionalFields.grantRead) { - headers['x-amz-grant-read'] = ''; - } - if (additionalFields.grantReadAcp) { - headers['x-amz-grant-read-acp'] = ''; - } - if (additionalFields.grantWriteAcp) { - headers['x-amz-grant-write-acp'] = ''; - } - if (additionalFields.lockLegalHold) { - headers['x-amz-object-lock-legal-hold'] = (additionalFields.lockLegalHold as boolean) - ? 'ON' - : 'OFF'; - } - if (additionalFields.lockMode) { - headers['x-amz-object-lock-mode'] = ( - additionalFields.lockMode as string - ).toUpperCase(); - } - if (additionalFields.lockRetainUntilDate) { - headers['x-amz-object-lock-retain-until-date'] = - additionalFields.lockRetainUntilDate as string; - } - if (additionalFields.serverSideEncryption) { - headers['x-amz-server-side-encryption'] = - additionalFields.serverSideEncryption as string; - } - if (additionalFields.encryptionAwsKmsKeyId) { - headers['x-amz-server-side-encryption-aws-kms-key-id'] = - additionalFields.encryptionAwsKmsKeyId as string; - } - if (additionalFields.serverSideEncryptionContext) { - headers['x-amz-server-side-encryption-context'] = - additionalFields.serverSideEncryptionContext as string; - } - if (additionalFields.serversideEncryptionCustomerAlgorithm) { - headers['x-amz-server-side-encryption-customer-algorithm'] = - additionalFields.serversideEncryptionCustomerAlgorithm as string; - } - if (additionalFields.serversideEncryptionCustomerKey) { - headers['x-amz-server-side-encryption-customer-key'] = - additionalFields.serversideEncryptionCustomerKey as string; - } - if (additionalFields.serversideEncryptionCustomerKeyMD5) { - headers['x-amz-server-side-encryption-customer-key-MD5'] = - additionalFields.serversideEncryptionCustomerKeyMD5 as string; - } - if (additionalFields.taggingDirective) { - headers['x-amz-tagging-directive'] = ( - additionalFields.taggingDirective as string - ).toUpperCase(); - } - if (additionalFields.metadataDirective) { - headers['x-amz-metadata-directive'] = ( - additionalFields.metadataDirective as string - ).toUpperCase(); - } - - const destinationParts = destinationPath.split('/'); - - const bucketName = destinationParts[1]; - - const destination = `/${destinationParts.slice(2, destinationParts.length).join('/')}`; - - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { - location: '', - }); - - const region = responseData.LocationConstraint._; - - responseData = await awsApiRequestSOAP.call( - this, - `${bucketName}.s3`, - 'PUT', - destination, - '', - qs, - headers, - {}, - region as string, - ); - const executionData = this.helpers.constructExecutionMetaData( - this.helpers.returnJsonArray(responseData.CopyObjectResult as IDataObject[]), - { itemData: { item: i } }, - ); - returnData.push(...executionData); - } - //https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html - if (operation === 'download') { - const bucketName = this.getNodeParameter('bucketName', i) as string; - - const fileKey = this.getNodeParameter('fileKey', i) as string; - - const fileName = fileKey.split('/')[fileKey.split('/').length - 1]; - - if (fileKey.substring(fileKey.length - 1) === '/') { - throw new NodeOperationError( - this.getNode(), - 'Downloading a whole directory is not yet supported, please provide a file key', - ); - } - - let region = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { - location: '', - }); - - region = region.LocationConstraint._; - - const response = await awsApiRequestREST.call( - this, - `${bucketName}.s3`, - 'GET', - `/${fileKey}`, - '', - qs, - {}, - { encoding: null, resolveWithFullResponse: true }, - region as string, - ); - - let mimeType: string | undefined; - if (response.headers['content-type']) { - mimeType = response.headers['content-type']; - } - - const newItem: INodeExecutionData = { - json: items[i].json, - binary: {}, - }; - - if (items[i].binary !== undefined && newItem.binary) { - // Create a shallow copy of the binary data so that the old - // data references which do not get changed still stay behind - // but the incoming data does not get changed. - Object.assign(newItem.binary, items[i].binary); - } - - items[i] = newItem; - - const dataPropertyNameDownload = this.getNodeParameter('binaryPropertyName', i); - - const data = Buffer.from(response.body as string, 'utf8'); - - items[i].binary![dataPropertyNameDownload] = await this.helpers.prepareBinaryData( - data as unknown as Buffer, - fileName, - mimeType, - ); - } - //https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html - if (operation === 'delete') { - const bucketName = this.getNodeParameter('bucketName', i) as string; - - const fileKey = this.getNodeParameter('fileKey', i) as string; - - const options = this.getNodeParameter('options', i); - - if (options.versionId) { - qs.versionId = options.versionId as string; - } - - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { - location: '', - }); - - const region = responseData.LocationConstraint._; - - responseData = await awsApiRequestSOAP.call( - this, - `${bucketName}.s3`, - 'DELETE', - `/${fileKey}`, - '', - qs, - {}, - {}, - region as string, - ); - const executionData = this.helpers.constructExecutionMetaData( - this.helpers.returnJsonArray({ success: true }), - { itemData: { item: i } }, - ); - returnData.push(...executionData); - } - //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html - if (operation === 'getAll') { - const bucketName = this.getNodeParameter('bucketName', i) as string; - const returnAll = this.getNodeParameter('returnAll', 0); - const options = this.getNodeParameter('options', 0); - - if (options.folderKey) { - qs.prefix = options.folderKey as string; - } - - if (options.fetchOwner) { - qs['fetch-owner'] = options.fetchOwner as string; - } - - qs.delimiter = '/'; - - qs['list-type'] = 2; - - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { - location: '', - }); - - const region = responseData.LocationConstraint._; - - if (returnAll) { - responseData = await awsApiRequestSOAPAllItems.call( - this, - 'ListBucketResult.Contents', - `${bucketName}.s3`, - 'GET', - '', - '', - qs, - {}, - {}, - region as string, - ); - } else { - qs.limit = this.getNodeParameter('limit', 0); - responseData = await awsApiRequestSOAPAllItems.call( - this, - 'ListBucketResult.Contents', - `${bucketName}.s3`, - 'GET', - '', - '', - qs, - {}, - {}, - region as string, - ); - responseData = responseData.splice(0, qs.limit); - } - if (Array.isArray(responseData)) { - responseData = responseData.filter( - (e: IDataObject) => !(e.Key as string).endsWith('/') && e.Size !== '0', - ); - if (qs.limit) { - responseData = responseData.splice(0, qs.limit as number); - } - const executionData = this.helpers.constructExecutionMetaData( - this.helpers.returnJsonArray(responseData), - { itemData: { item: i } }, - ); - returnData.push(...executionData); - } - } - //https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html - if (operation === 'upload') { - const bucketName = this.getNodeParameter('bucketName', i) as string; - const fileName = this.getNodeParameter('fileName', i) as string; - const isBinaryData = this.getNodeParameter('binaryData', i); - const additionalFields = this.getNodeParameter('additionalFields', i); - const tagsValues = (this.getNodeParameter('tagsUi', i) as IDataObject) - .tagsValues as IDataObject[]; - let path = '/'; - let body; - - if (additionalFields.requesterPays) { - headers['x-amz-request-payer'] = 'requester'; - } - if (additionalFields.parentFolderKey) { - path = `/${additionalFields.parentFolderKey}/`; - } - if (additionalFields.storageClass) { - headers['x-amz-storage-class'] = snakeCase( - additionalFields.storageClass as string, - ).toUpperCase(); - } - if (additionalFields.acl) { - headers['x-amz-acl'] = paramCase(additionalFields.acl as string); - } - if (additionalFields.grantFullControl) { - headers['x-amz-grant-full-control'] = ''; - } - if (additionalFields.grantRead) { - headers['x-amz-grant-read'] = ''; - } - if (additionalFields.grantReadAcp) { - headers['x-amz-grant-read-acp'] = ''; - } - if (additionalFields.grantWriteAcp) { - headers['x-amz-grant-write-acp'] = ''; - } - if (additionalFields.lockLegalHold) { - headers['x-amz-object-lock-legal-hold'] = (additionalFields.lockLegalHold as boolean) - ? 'ON' - : 'OFF'; - } - if (additionalFields.lockMode) { - headers['x-amz-object-lock-mode'] = ( - additionalFields.lockMode as string - ).toUpperCase(); - } - if (additionalFields.lockRetainUntilDate) { - headers['x-amz-object-lock-retain-until-date'] = - additionalFields.lockRetainUntilDate as string; - } - if (additionalFields.serverSideEncryption) { - headers['x-amz-server-side-encryption'] = - additionalFields.serverSideEncryption as string; - } - if (additionalFields.encryptionAwsKmsKeyId) { - headers['x-amz-server-side-encryption-aws-kms-key-id'] = - additionalFields.encryptionAwsKmsKeyId as string; - } - if (additionalFields.serverSideEncryptionContext) { - headers['x-amz-server-side-encryption-context'] = - additionalFields.serverSideEncryptionContext as string; - } - if (additionalFields.serversideEncryptionCustomerAlgorithm) { - headers['x-amz-server-side-encryption-customer-algorithm'] = - additionalFields.serversideEncryptionCustomerAlgorithm as string; - } - if (additionalFields.serversideEncryptionCustomerKey) { - headers['x-amz-server-side-encryption-customer-key'] = - additionalFields.serversideEncryptionCustomerKey as string; - } - if (additionalFields.serversideEncryptionCustomerKeyMD5) { - headers['x-amz-server-side-encryption-customer-key-MD5'] = - additionalFields.serversideEncryptionCustomerKeyMD5 as string; - } - if (tagsValues) { - const tags: string[] = []; - tagsValues.forEach((o: IDataObject) => { - tags.push(`${o.key}=${o.value}`); - }); - headers['x-amz-tagging'] = tags.join('&'); - } - - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { - location: '', - }); - - const region = responseData.LocationConstraint._; - - if (isBinaryData) { - const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i); - const binaryPropertyData = this.helpers.assertBinaryData(i, binaryPropertyName); - const binaryDataBuffer = await this.helpers.getBinaryDataBuffer( - i, - binaryPropertyName, - ); - - body = binaryDataBuffer; - - headers['Content-Type'] = binaryPropertyData.mimeType; - - headers['Content-MD5'] = createHash('md5').update(body).digest('base64'); - - responseData = await awsApiRequestSOAP.call( - this, - `${bucketName}.s3`, - 'PUT', - `${path}${fileName || binaryPropertyData.fileName}`, - body, - qs, - headers, - {}, - region as string, - ); - } else { - const fileContent = this.getNodeParameter('fileContent', i) as string; - - body = Buffer.from(fileContent, 'utf8'); - - headers['Content-Type'] = 'text/html'; - - headers['Content-MD5'] = createHash('md5').update(fileContent).digest('base64'); - - responseData = await awsApiRequestSOAP.call( - this, - `${bucketName}.s3`, - 'PUT', - `${path}${fileName}`, - body, - qs, - headers, - {}, - region as string, - ); - } - const executionData = this.helpers.constructExecutionMetaData( - this.helpers.returnJsonArray({ success: true }), - { itemData: { item: i } }, - ); - returnData.push(...executionData); - } - } - } catch (error) { - if (this.continueOnFail()) { - const executionData = this.helpers.constructExecutionMetaData( - this.helpers.returnJsonArray({ error: error.message }), - { itemData: { item: i } }, - ); - returnData.push(...executionData); - continue; - } - throw error; - } - } - if (resource === 'file' && operation === 'download') { - // For file downloads the files get attached to the existing items - return this.prepareOutputData(items); - } else { - return this.prepareOutputData(returnData); - } +import type { INodeTypeBaseDescription, IVersionedNodeType } from 'n8n-workflow'; +import { VersionedNodeType } from 'n8n-workflow'; + +import { AwsS3V1 } from './V1/AwsS3V1.node'; + +import { AwsS3V2 } from './V2/AwsS3V2.node'; + +export class AwsS3 extends VersionedNodeType { + constructor() { + const baseDescription: INodeTypeBaseDescription = { + displayName: 'AwsS3', + name: 'awsS3', + icon: 'file:s3.svg', + group: ['output'], + subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', + description: 'Sends data to AWS S3', + defaultVersion: 2, + }; + + const nodeVersions: IVersionedNodeType['nodeVersions'] = { + 1: new AwsS3V1(baseDescription), + 2: new AwsS3V2(baseDescription), + }; + + super(nodeVersions, baseDescription); } } diff --git a/packages/nodes-base/nodes/Aws/S3/V1/AwsS3V1.node.ts b/packages/nodes-base/nodes/Aws/S3/V1/AwsS3V1.node.ts new file mode 100644 index 0000000000000..d619404ad17c6 --- /dev/null +++ b/packages/nodes-base/nodes/Aws/S3/V1/AwsS3V1.node.ts @@ -0,0 +1,915 @@ +/* eslint-disable n8n-nodes-base/node-filename-against-convention */ +import { paramCase, snakeCase } from 'change-case'; + +import { createHash } from 'crypto'; + +import { Builder } from 'xml2js'; + +import type { + IDataObject, + IExecuteFunctions, + INodeExecutionData, + INodeType, + INodeTypeBaseDescription, + INodeTypeDescription, +} from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import { bucketFields, bucketOperations } from './BucketDescription'; + +import { folderFields, folderOperations } from './FolderDescription'; + +import { fileFields, fileOperations } from './FileDescription'; + +import { + awsApiRequestREST, + awsApiRequestSOAP, + awsApiRequestSOAPAllItems, +} from './GenericFunctions'; + +export class AwsS3V1 implements INodeType { + description: INodeTypeDescription; + + constructor(baseDescription: INodeTypeBaseDescription) { + this.description = { + ...baseDescription, + displayName: 'AWS S3', + name: 'awsS3', + icon: 'file:s3.svg', + group: ['output'], + version: 1, + subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', + description: 'Sends data to AWS S3', + defaults: { + name: 'AWS S3', + }, + inputs: ['main'], + outputs: ['main'], + credentials: [ + { + name: 'aws', + required: true, + }, + ], + properties: [ + { + displayName: 'Resource', + name: 'resource', + type: 'options', + noDataExpression: true, + options: [ + { + name: 'Bucket', + value: 'bucket', + }, + { + name: 'File', + value: 'file', + }, + { + name: 'Folder', + value: 'folder', + }, + ], + default: 'file', + }, + // BUCKET + ...bucketOperations, + ...bucketFields, + // FOLDER + ...folderOperations, + ...folderFields, + // UPLOAD + ...fileOperations, + ...fileFields, + ], + }; + } + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + const returnData: INodeExecutionData[] = []; + const qs: IDataObject = {}; + let responseData; + const resource = this.getNodeParameter('resource', 0); + const operation = this.getNodeParameter('operation', 0); + for (let i = 0; i < items.length; i++) { + const headers: IDataObject = {}; + try { + if (resource === 'bucket') { + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html + if (operation === 'create') { + const credentials = await this.getCredentials('aws'); + const name = this.getNodeParameter('name', i) as string; + const additionalFields = this.getNodeParameter('additionalFields', i); + if (additionalFields.acl) { + headers['x-amz-acl'] = paramCase(additionalFields.acl as string); + } + if (additionalFields.bucketObjectLockEnabled) { + headers['x-amz-bucket-object-lock-enabled'] = + additionalFields.bucketObjectLockEnabled as boolean; + } + if (additionalFields.grantFullControl) { + headers['x-amz-grant-full-control'] = ''; + } + if (additionalFields.grantRead) { + headers['x-amz-grant-read'] = ''; + } + if (additionalFields.grantReadAcp) { + headers['x-amz-grant-read-acp'] = ''; + } + if (additionalFields.grantWrite) { + headers['x-amz-grant-write'] = ''; + } + if (additionalFields.grantWriteAcp) { + headers['x-amz-grant-write-acp'] = ''; + } + let region = credentials.region as string; + + if (additionalFields.region) { + region = additionalFields.region as string; + } + + const body: IDataObject = { + CreateBucketConfiguration: { + $: { + xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/', + }, + }, + }; + let data = ''; + // if credentials has the S3 defaul region (us-east-1) the body (XML) does not have to be sent. + if (region !== 'us-east-1') { + // @ts-ignore + body.CreateBucketConfiguration.LocationConstraint = [region]; + const builder = new Builder(); + data = builder.buildObject(body); + } + responseData = await awsApiRequestSOAP.call( + this, + `${name}.s3`, + 'PUT', + '', + data, + qs, + headers, + ); + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray({ success: true }), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucket.html + if (operation === 'delete') { + const name = this.getNodeParameter('name', i) as string; + + responseData = await awsApiRequestSOAP.call( + this, + `${name}.s3`, + 'DELETE', + '', + '', + {}, + headers, + ); + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray({ success: true }), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html + if (operation === 'getAll') { + const returnAll = this.getNodeParameter('returnAll', 0); + if (returnAll) { + responseData = await awsApiRequestSOAPAllItems.call( + this, + 'ListAllMyBucketsResult.Buckets.Bucket', + 's3', + 'GET', + '', + ); + } else { + qs.limit = this.getNodeParameter('limit', 0); + responseData = await awsApiRequestSOAPAllItems.call( + this, + 'ListAllMyBucketsResult.Buckets.Bucket', + 's3', + 'GET', + '', + '', + qs, + ); + responseData = responseData.slice(0, qs.limit); + } + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray(responseData as IDataObject[]), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + if (operation === 'search') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const returnAll = this.getNodeParameter('returnAll', 0); + const additionalFields = this.getNodeParameter('additionalFields', 0); + + if (additionalFields.prefix) { + qs.prefix = additionalFields.prefix as string; + } + + if (additionalFields.encodingType) { + qs['encoding-type'] = additionalFields.encodingType as string; + } + + if (additionalFields.delimiter) { + qs.delimiter = additionalFields.delimiter as string; + } + + if (additionalFields.fetchOwner) { + qs['fetch-owner'] = additionalFields.fetchOwner as string; + } + + if (additionalFields.startAfter) { + qs['start-after'] = additionalFields.startAfter as string; + } + + if (additionalFields.requesterPays) { + qs['x-amz-request-payer'] = 'requester'; + } + + qs['list-type'] = 2; + + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._ as string; + + if (returnAll) { + responseData = await awsApiRequestSOAPAllItems.call( + this, + 'ListBucketResult.Contents', + `${bucketName}.s3`, + 'GET', + '', + '', + qs, + {}, + {}, + region, + ); + } else { + qs['max-keys'] = this.getNodeParameter('limit', 0); + responseData = await awsApiRequestSOAP.call( + this, + `${bucketName}.s3`, + 'GET', + '', + '', + qs, + {}, + {}, + region, + ); + responseData = responseData.ListBucketResult.Contents; + } + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray(responseData as IDataObject[]), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + } + if (resource === 'folder') { + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html + if (operation === 'create') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const folderName = this.getNodeParameter('folderName', i) as string; + const additionalFields = this.getNodeParameter('additionalFields', i); + let path = `/${folderName}/`; + + if (additionalFields.requesterPays) { + headers['x-amz-request-payer'] = 'requester'; + } + if (additionalFields.parentFolderKey) { + path = `/${additionalFields.parentFolderKey}${folderName}/`; + } + if (additionalFields.storageClass) { + headers['x-amz-storage-class'] = snakeCase( + additionalFields.storageClass as string, + ).toUpperCase(); + } + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._; + + responseData = await awsApiRequestSOAP.call( + this, + `${bucketName}.s3`, + 'PUT', + path, + '', + qs, + headers, + {}, + region as string, + ); + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray({ success: true }), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html + if (operation === 'delete') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const folderKey = this.getNodeParameter('folderKey', i) as string; + + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._; + + responseData = await awsApiRequestSOAPAllItems.call( + this, + 'ListBucketResult.Contents', + `${bucketName}.s3`, + 'GET', + '/', + '', + { 'list-type': 2, prefix: folderKey }, + {}, + {}, + region as string, + ); + + // folder empty then just delete it + if (responseData.length === 0) { + responseData = await awsApiRequestSOAP.call( + this, + `${bucketName}.s3`, + 'DELETE', + `/${folderKey}`, + '', + qs, + {}, + {}, + region as string, + ); + + responseData = { deleted: [{ Key: folderKey }] }; + } else { + // delete everything inside the folder + const body: IDataObject = { + Delete: { + $: { + xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/', + }, + Object: [], + }, + }; + + for (const childObject of responseData) { + //@ts-ignore + (body.Delete.Object as IDataObject[]).push({ + Key: childObject.Key as string, + }); + } + + const builder = new Builder(); + const data = builder.buildObject(body); + + headers['Content-MD5'] = createHash('md5').update(data).digest('base64'); + + headers['Content-Type'] = 'application/xml'; + + responseData = await awsApiRequestSOAP.call( + this, + `${bucketName}.s3`, + 'POST', + '/', + data, + { delete: '' }, + headers, + {}, + region as string, + ); + + responseData = { deleted: responseData.DeleteResult.Deleted }; + } + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray(responseData), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + if (operation === 'getAll') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const returnAll = this.getNodeParameter('returnAll', 0); + const options = this.getNodeParameter('options', 0); + + if (options.folderKey) { + qs.prefix = options.folderKey as string; + } + + if (options.fetchOwner) { + qs['fetch-owner'] = options.fetchOwner as string; + } + + qs['list-type'] = 2; + + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._; + + if (returnAll) { + responseData = await awsApiRequestSOAPAllItems.call( + this, + 'ListBucketResult.Contents', + `${bucketName}.s3`, + 'GET', + '', + '', + qs, + {}, + {}, + region as string, + ); + } else { + qs.limit = this.getNodeParameter('limit', 0); + responseData = await awsApiRequestSOAPAllItems.call( + this, + 'ListBucketResult.Contents', + `${bucketName}.s3`, + 'GET', + '', + '', + qs, + {}, + {}, + region as string, + ); + } + if (Array.isArray(responseData)) { + responseData = responseData.filter( + (e: IDataObject) => + (e.Key as string).endsWith('/') && e.Size === '0' && e.Key !== options.folderKey, + ); + if (qs.limit) { + responseData = responseData.splice(0, qs.limit as number); + } + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray(responseData), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + } + } + if (resource === 'file') { + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html + if (operation === 'copy') { + const sourcePath = this.getNodeParameter('sourcePath', i) as string; + const destinationPath = this.getNodeParameter('destinationPath', i) as string; + const additionalFields = this.getNodeParameter('additionalFields', i); + + headers['x-amz-copy-source'] = sourcePath; + + if (additionalFields.requesterPays) { + headers['x-amz-request-payer'] = 'requester'; + } + if (additionalFields.storageClass) { + headers['x-amz-storage-class'] = snakeCase( + additionalFields.storageClass as string, + ).toUpperCase(); + } + if (additionalFields.acl) { + headers['x-amz-acl'] = paramCase(additionalFields.acl as string); + } + if (additionalFields.grantFullControl) { + headers['x-amz-grant-full-control'] = ''; + } + if (additionalFields.grantRead) { + headers['x-amz-grant-read'] = ''; + } + if (additionalFields.grantReadAcp) { + headers['x-amz-grant-read-acp'] = ''; + } + if (additionalFields.grantWriteAcp) { + headers['x-amz-grant-write-acp'] = ''; + } + if (additionalFields.lockLegalHold) { + headers['x-amz-object-lock-legal-hold'] = (additionalFields.lockLegalHold as boolean) + ? 'ON' + : 'OFF'; + } + if (additionalFields.lockMode) { + headers['x-amz-object-lock-mode'] = ( + additionalFields.lockMode as string + ).toUpperCase(); + } + if (additionalFields.lockRetainUntilDate) { + headers['x-amz-object-lock-retain-until-date'] = + additionalFields.lockRetainUntilDate as string; + } + if (additionalFields.serverSideEncryption) { + headers['x-amz-server-side-encryption'] = + additionalFields.serverSideEncryption as string; + } + if (additionalFields.encryptionAwsKmsKeyId) { + headers['x-amz-server-side-encryption-aws-kms-key-id'] = + additionalFields.encryptionAwsKmsKeyId as string; + } + if (additionalFields.serverSideEncryptionContext) { + headers['x-amz-server-side-encryption-context'] = + additionalFields.serverSideEncryptionContext as string; + } + if (additionalFields.serversideEncryptionCustomerAlgorithm) { + headers['x-amz-server-side-encryption-customer-algorithm'] = + additionalFields.serversideEncryptionCustomerAlgorithm as string; + } + if (additionalFields.serversideEncryptionCustomerKey) { + headers['x-amz-server-side-encryption-customer-key'] = + additionalFields.serversideEncryptionCustomerKey as string; + } + if (additionalFields.serversideEncryptionCustomerKeyMD5) { + headers['x-amz-server-side-encryption-customer-key-MD5'] = + additionalFields.serversideEncryptionCustomerKeyMD5 as string; + } + if (additionalFields.taggingDirective) { + headers['x-amz-tagging-directive'] = ( + additionalFields.taggingDirective as string + ).toUpperCase(); + } + if (additionalFields.metadataDirective) { + headers['x-amz-metadata-directive'] = ( + additionalFields.metadataDirective as string + ).toUpperCase(); + } + + const destinationParts = destinationPath.split('/'); + + const bucketName = destinationParts[1]; + + const destination = `/${destinationParts.slice(2, destinationParts.length).join('/')}`; + + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._; + + responseData = await awsApiRequestSOAP.call( + this, + `${bucketName}.s3`, + 'PUT', + destination, + '', + qs, + headers, + {}, + region as string, + ); + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray(responseData.CopyObjectResult as IDataObject[]), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html + if (operation === 'download') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + + const fileKey = this.getNodeParameter('fileKey', i) as string; + + const fileName = fileKey.split('/')[fileKey.split('/').length - 1]; + + if (fileKey.substring(fileKey.length - 1) === '/') { + throw new NodeOperationError( + this.getNode(), + 'Downloading a whole directory is not yet supported, please provide a file key', + ); + } + + let region = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + region = region.LocationConstraint._; + + const response = await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'GET', + `/${fileKey}`, + '', + qs, + {}, + { encoding: null, resolveWithFullResponse: true }, + region as string, + ); + + let mimeType: string | undefined; + if (response.headers['content-type']) { + mimeType = response.headers['content-type']; + } + + const newItem: INodeExecutionData = { + json: items[i].json, + binary: {}, + }; + + if (items[i].binary !== undefined && newItem.binary) { + // Create a shallow copy of the binary data so that the old + // data references which do not get changed still stay behind + // but the incoming data does not get changed. + Object.assign(newItem.binary, items[i].binary); + } + + items[i] = newItem; + + const dataPropertyNameDownload = this.getNodeParameter('binaryPropertyName', i); + + const data = Buffer.from(response.body as string, 'utf8'); + + items[i].binary![dataPropertyNameDownload] = await this.helpers.prepareBinaryData( + data as unknown as Buffer, + fileName, + mimeType, + ); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html + if (operation === 'delete') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + + const fileKey = this.getNodeParameter('fileKey', i) as string; + + const options = this.getNodeParameter('options', i); + + if (options.versionId) { + qs.versionId = options.versionId as string; + } + + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._; + + responseData = await awsApiRequestSOAP.call( + this, + `${bucketName}.s3`, + 'DELETE', + `/${fileKey}`, + '', + qs, + {}, + {}, + region as string, + ); + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray({ success: true }), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + if (operation === 'getAll') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const returnAll = this.getNodeParameter('returnAll', 0); + const options = this.getNodeParameter('options', 0); + + if (options.folderKey) { + qs.prefix = options.folderKey as string; + } + + if (options.fetchOwner) { + qs['fetch-owner'] = options.fetchOwner as string; + } + + qs.delimiter = '/'; + + qs['list-type'] = 2; + + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._; + + if (returnAll) { + responseData = await awsApiRequestSOAPAllItems.call( + this, + 'ListBucketResult.Contents', + `${bucketName}.s3`, + 'GET', + '', + '', + qs, + {}, + {}, + region as string, + ); + } else { + qs.limit = this.getNodeParameter('limit', 0); + responseData = await awsApiRequestSOAPAllItems.call( + this, + 'ListBucketResult.Contents', + `${bucketName}.s3`, + 'GET', + '', + '', + qs, + {}, + {}, + region as string, + ); + responseData = responseData.splice(0, qs.limit); + } + if (Array.isArray(responseData)) { + responseData = responseData.filter( + (e: IDataObject) => !(e.Key as string).endsWith('/') && e.Size !== '0', + ); + if (qs.limit) { + responseData = responseData.splice(0, qs.limit as number); + } + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray(responseData), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html + if (operation === 'upload') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const fileName = this.getNodeParameter('fileName', i) as string; + const isBinaryData = this.getNodeParameter('binaryData', i); + const additionalFields = this.getNodeParameter('additionalFields', i); + const tagsValues = (this.getNodeParameter('tagsUi', i) as IDataObject) + .tagsValues as IDataObject[]; + let path = '/'; + let body; + + if (additionalFields.requesterPays) { + headers['x-amz-request-payer'] = 'requester'; + } + if (additionalFields.parentFolderKey) { + path = `/${additionalFields.parentFolderKey}/`; + } + if (additionalFields.storageClass) { + headers['x-amz-storage-class'] = snakeCase( + additionalFields.storageClass as string, + ).toUpperCase(); + } + if (additionalFields.acl) { + headers['x-amz-acl'] = paramCase(additionalFields.acl as string); + } + if (additionalFields.grantFullControl) { + headers['x-amz-grant-full-control'] = ''; + } + if (additionalFields.grantRead) { + headers['x-amz-grant-read'] = ''; + } + if (additionalFields.grantReadAcp) { + headers['x-amz-grant-read-acp'] = ''; + } + if (additionalFields.grantWriteAcp) { + headers['x-amz-grant-write-acp'] = ''; + } + if (additionalFields.lockLegalHold) { + headers['x-amz-object-lock-legal-hold'] = (additionalFields.lockLegalHold as boolean) + ? 'ON' + : 'OFF'; + } + if (additionalFields.lockMode) { + headers['x-amz-object-lock-mode'] = ( + additionalFields.lockMode as string + ).toUpperCase(); + } + if (additionalFields.lockRetainUntilDate) { + headers['x-amz-object-lock-retain-until-date'] = + additionalFields.lockRetainUntilDate as string; + } + if (additionalFields.serverSideEncryption) { + headers['x-amz-server-side-encryption'] = + additionalFields.serverSideEncryption as string; + } + if (additionalFields.encryptionAwsKmsKeyId) { + headers['x-amz-server-side-encryption-aws-kms-key-id'] = + additionalFields.encryptionAwsKmsKeyId as string; + } + if (additionalFields.serverSideEncryptionContext) { + headers['x-amz-server-side-encryption-context'] = + additionalFields.serverSideEncryptionContext as string; + } + if (additionalFields.serversideEncryptionCustomerAlgorithm) { + headers['x-amz-server-side-encryption-customer-algorithm'] = + additionalFields.serversideEncryptionCustomerAlgorithm as string; + } + if (additionalFields.serversideEncryptionCustomerKey) { + headers['x-amz-server-side-encryption-customer-key'] = + additionalFields.serversideEncryptionCustomerKey as string; + } + if (additionalFields.serversideEncryptionCustomerKeyMD5) { + headers['x-amz-server-side-encryption-customer-key-MD5'] = + additionalFields.serversideEncryptionCustomerKeyMD5 as string; + } + if (tagsValues) { + const tags: string[] = []; + tagsValues.forEach((o: IDataObject) => { + tags.push(`${o.key}=${o.value}`); + }); + headers['x-amz-tagging'] = tags.join('&'); + } + + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._; + + if (isBinaryData) { + const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i); + const binaryPropertyData = this.helpers.assertBinaryData(i, binaryPropertyName); + const binaryDataBuffer = await this.helpers.getBinaryDataBuffer( + i, + binaryPropertyName, + ); + + body = binaryDataBuffer; + + headers['Content-Type'] = binaryPropertyData.mimeType; + + headers['Content-MD5'] = createHash('md5').update(body).digest('base64'); + + responseData = await awsApiRequestSOAP.call( + this, + `${bucketName}.s3`, + 'PUT', + `${path}${fileName || binaryPropertyData.fileName}`, + body, + qs, + headers, + {}, + region as string, + ); + } else { + const fileContent = this.getNodeParameter('fileContent', i) as string; + + body = Buffer.from(fileContent, 'utf8'); + + headers['Content-Type'] = 'text/html'; + + headers['Content-MD5'] = createHash('md5').update(fileContent).digest('base64'); + + responseData = await awsApiRequestSOAP.call( + this, + `${bucketName}.s3`, + 'PUT', + `${path}${fileName}`, + body, + qs, + headers, + {}, + region as string, + ); + } + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray({ success: true }), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + } + } catch (error) { + if (this.continueOnFail()) { + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray({ error: error.message }), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + continue; + } + throw error; + } + } + if (resource === 'file' && operation === 'download') { + // For file downloads the files get attached to the existing items + return this.prepareOutputData(items); + } else { + return this.prepareOutputData(returnData); + } + } +} diff --git a/packages/nodes-base/nodes/Aws/S3/BucketDescription.ts b/packages/nodes-base/nodes/Aws/S3/V1/BucketDescription.ts similarity index 100% rename from packages/nodes-base/nodes/Aws/S3/BucketDescription.ts rename to packages/nodes-base/nodes/Aws/S3/V1/BucketDescription.ts diff --git a/packages/nodes-base/nodes/Aws/S3/FileDescription.ts b/packages/nodes-base/nodes/Aws/S3/V1/FileDescription.ts similarity index 100% rename from packages/nodes-base/nodes/Aws/S3/FileDescription.ts rename to packages/nodes-base/nodes/Aws/S3/V1/FileDescription.ts diff --git a/packages/nodes-base/nodes/Aws/S3/FolderDescription.ts b/packages/nodes-base/nodes/Aws/S3/V1/FolderDescription.ts similarity index 100% rename from packages/nodes-base/nodes/Aws/S3/FolderDescription.ts rename to packages/nodes-base/nodes/Aws/S3/V1/FolderDescription.ts diff --git a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts b/packages/nodes-base/nodes/Aws/S3/V1/GenericFunctions.ts similarity index 100% rename from packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts rename to packages/nodes-base/nodes/Aws/S3/V1/GenericFunctions.ts diff --git a/packages/nodes-base/nodes/Aws/S3/V2/AwsS3V2.node.ts b/packages/nodes-base/nodes/Aws/S3/V2/AwsS3V2.node.ts new file mode 100644 index 0000000000000..67352e1bdc736 --- /dev/null +++ b/packages/nodes-base/nodes/Aws/S3/V2/AwsS3V2.node.ts @@ -0,0 +1,1058 @@ +/* eslint-disable n8n-nodes-base/node-filename-against-convention */ +import { paramCase, snakeCase } from 'change-case'; + +import { createHash } from 'crypto'; + +import { Builder } from 'xml2js'; + +import type { + IDataObject, + IExecuteFunctions, + INodeExecutionData, + INodeType, + INodeTypeBaseDescription, + INodeTypeDescription, +} from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import { bucketFields, bucketOperations } from './BucketDescription'; + +import { folderFields, folderOperations } from './FolderDescription'; + +import { fileFields, fileOperations } from './FileDescription'; + +import { awsApiRequestREST, awsApiRequestRESTAllItems } from './GenericFunctions'; +import type { Readable } from 'stream'; + +// Minimum size 5MB for multipart upload in S3 +const UPLOAD_CHUNK_SIZE = 5120 * 1024; + +export class AwsS3V2 implements INodeType { + description: INodeTypeDescription; + + constructor(baseDescription: INodeTypeBaseDescription) { + this.description = { + ...baseDescription, + displayName: 'AWS S3', + name: 'awsS3', + icon: 'file:s3.svg', + group: ['output'], + version: 2, + subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', + description: 'Sends data to AWS S3', + defaults: { + name: 'AWS S3', + }, + inputs: ['main'], + outputs: ['main'], + credentials: [ + { + name: 'aws', + required: true, + }, + ], + properties: [ + { + displayName: 'Resource', + name: 'resource', + type: 'options', + noDataExpression: true, + options: [ + { + name: 'Bucket', + value: 'bucket', + }, + { + name: 'File', + value: 'file', + }, + { + name: 'Folder', + value: 'folder', + }, + ], + default: 'file', + }, + // BUCKET + ...bucketOperations, + ...bucketFields, + // FOLDER + ...folderOperations, + ...folderFields, + // UPLOAD + ...fileOperations, + ...fileFields, + ], + }; + } + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + const returnData: INodeExecutionData[] = []; + const qs: IDataObject = {}; + let responseData; + const resource = this.getNodeParameter('resource', 0); + const operation = this.getNodeParameter('operation', 0); + for (let i = 0; i < items.length; i++) { + let headers: IDataObject = {}; + try { + if (resource === 'bucket') { + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html + if (operation === 'create') { + const credentials = await this.getCredentials('aws'); + const name = this.getNodeParameter('name', i) as string; + const additionalFields = this.getNodeParameter('additionalFields', i); + if (additionalFields.acl) { + headers['x-amz-acl'] = paramCase(additionalFields.acl as string); + } + if (additionalFields.bucketObjectLockEnabled) { + headers['x-amz-bucket-object-lock-enabled'] = + additionalFields.bucketObjectLockEnabled as boolean; + } + if (additionalFields.grantFullControl) { + headers['x-amz-grant-full-control'] = ''; + } + if (additionalFields.grantRead) { + headers['x-amz-grant-read'] = ''; + } + if (additionalFields.grantReadAcp) { + headers['x-amz-grant-read-acp'] = ''; + } + if (additionalFields.grantWrite) { + headers['x-amz-grant-write'] = ''; + } + if (additionalFields.grantWriteAcp) { + headers['x-amz-grant-write-acp'] = ''; + } + let region = credentials.region as string; + + if (additionalFields.region) { + region = additionalFields.region as string; + } + + const body: IDataObject = { + CreateBucketConfiguration: { + $: { + xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/', + }, + }, + }; + let data = ''; + // if credentials has the S3 defaul region (us-east-1) the body (XML) does not have to be sent. + if (region !== 'us-east-1') { + // @ts-ignore + body.CreateBucketConfiguration.LocationConstraint = [region]; + const builder = new Builder(); + data = builder.buildObject(body); + } + responseData = await awsApiRequestREST.call( + this, + `${name}.s3`, + 'PUT', + '', + data, + qs, + headers, + ); + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray({ success: true }), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucket.html + if (operation === 'delete') { + const name = this.getNodeParameter('name', i) as string; + + responseData = await awsApiRequestREST.call( + this, + `${name}.s3`, + 'DELETE', + '', + '', + {}, + headers, + ); + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray({ success: true }), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html + if (operation === 'getAll') { + const returnAll = this.getNodeParameter('returnAll', 0); + if (returnAll) { + responseData = await awsApiRequestRESTAllItems.call( + this, + 'ListAllMyBucketsResult.Buckets.Bucket', + 's3', + 'GET', + '', + ); + } else { + qs.limit = this.getNodeParameter('limit', 0); + responseData = await awsApiRequestRESTAllItems.call( + this, + 'ListAllMyBucketsResult.Buckets.Bucket', + 's3', + 'GET', + '', + '', + qs, + ); + responseData = responseData.slice(0, qs.limit); + } + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray(responseData as IDataObject[]), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + if (operation === 'search') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const returnAll = this.getNodeParameter('returnAll', 0); + const additionalFields = this.getNodeParameter('additionalFields', 0); + + if (additionalFields.prefix) { + qs.prefix = additionalFields.prefix as string; + } + + if (additionalFields.encodingType) { + qs['encoding-type'] = additionalFields.encodingType as string; + } + + if (additionalFields.delimiter) { + qs.delimiter = additionalFields.delimiter as string; + } + + if (additionalFields.fetchOwner) { + qs['fetch-owner'] = additionalFields.fetchOwner as string; + } + + if (additionalFields.startAfter) { + qs['start-after'] = additionalFields.startAfter as string; + } + + if (additionalFields.requesterPays) { + qs['x-amz-request-payer'] = 'requester'; + } + + qs['list-type'] = 2; + + responseData = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._ as string; + + if (returnAll) { + responseData = await awsApiRequestRESTAllItems.call( + this, + 'ListBucketResult.Contents', + `${bucketName}.s3`, + 'GET', + '', + '', + qs, + {}, + {}, + region, + ); + } else { + qs['max-keys'] = this.getNodeParameter('limit', 0); + responseData = await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'GET', + '', + '', + qs, + {}, + {}, + region, + ); + responseData = responseData.ListBucketResult.Contents; + } + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray(responseData as IDataObject[]), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + } + if (resource === 'folder') { + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html + if (operation === 'create') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const folderName = this.getNodeParameter('folderName', i) as string; + const additionalFields = this.getNodeParameter('additionalFields', i); + let path = `/${folderName}/`; + + if (additionalFields.requesterPays) { + headers['x-amz-request-payer'] = 'requester'; + } + if (additionalFields.parentFolderKey) { + path = `/${additionalFields.parentFolderKey}${folderName}/`; + } + if (additionalFields.storageClass) { + headers['x-amz-storage-class'] = snakeCase( + additionalFields.storageClass as string, + ).toUpperCase(); + } + responseData = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._; + + responseData = await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'PUT', + path, + '', + qs, + headers, + {}, + region as string, + ); + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray({ success: true }), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html + if (operation === 'delete') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const folderKey = this.getNodeParameter('folderKey', i) as string; + + responseData = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._; + + responseData = await awsApiRequestRESTAllItems.call( + this, + 'ListBucketResult.Contents', + `${bucketName}.s3`, + 'GET', + '/', + '', + { 'list-type': 2, prefix: folderKey }, + {}, + {}, + region as string, + ); + + // folder empty then just delete it + if (responseData.length === 0) { + responseData = await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'DELETE', + `/${folderKey}`, + '', + qs, + {}, + {}, + region as string, + ); + + responseData = { deleted: [{ Key: folderKey }] }; + } else { + // delete everything inside the folder + const body: IDataObject = { + Delete: { + $: { + xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/', + }, + Object: [], + }, + }; + + for (const childObject of responseData) { + //@ts-ignore + (body.Delete.Object as IDataObject[]).push({ + Key: childObject.Key as string, + }); + } + + const builder = new Builder(); + const data = builder.buildObject(body); + + headers['Content-MD5'] = createHash('md5').update(data).digest('base64'); + + headers['Content-Type'] = 'application/xml'; + + responseData = await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'POST', + '/', + data, + { delete: '' }, + headers, + {}, + region as string, + ); + + responseData = { deleted: responseData.DeleteResult.Deleted }; + } + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray(responseData), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + if (operation === 'getAll') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const returnAll = this.getNodeParameter('returnAll', 0); + const options = this.getNodeParameter('options', 0); + + if (options.folderKey) { + qs.prefix = options.folderKey as string; + } + + if (options.fetchOwner) { + qs['fetch-owner'] = options.fetchOwner as string; + } + + qs['list-type'] = 2; + + responseData = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._; + + if (returnAll) { + responseData = await awsApiRequestRESTAllItems.call( + this, + 'ListBucketResult.Contents', + `${bucketName}.s3`, + 'GET', + '', + '', + qs, + {}, + {}, + region as string, + ); + } else { + qs.limit = this.getNodeParameter('limit', 0); + responseData = await awsApiRequestRESTAllItems.call( + this, + 'ListBucketResult.Contents', + `${bucketName}.s3`, + 'GET', + '', + '', + qs, + {}, + {}, + region as string, + ); + } + if (Array.isArray(responseData)) { + responseData = responseData.filter( + (e: IDataObject) => + (e.Key as string).endsWith('/') && e.Size === '0' && e.Key !== options.folderKey, + ); + if (qs.limit) { + responseData = responseData.splice(0, qs.limit as number); + } + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray(responseData), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + } + } + if (resource === 'file') { + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html + if (operation === 'copy') { + const sourcePath = this.getNodeParameter('sourcePath', i) as string; + const destinationPath = this.getNodeParameter('destinationPath', i) as string; + const additionalFields = this.getNodeParameter('additionalFields', i); + + headers['x-amz-copy-source'] = sourcePath; + + if (additionalFields.requesterPays) { + headers['x-amz-request-payer'] = 'requester'; + } + if (additionalFields.storageClass) { + headers['x-amz-storage-class'] = snakeCase( + additionalFields.storageClass as string, + ).toUpperCase(); + } + if (additionalFields.acl) { + headers['x-amz-acl'] = paramCase(additionalFields.acl as string); + } + if (additionalFields.grantFullControl) { + headers['x-amz-grant-full-control'] = ''; + } + if (additionalFields.grantRead) { + headers['x-amz-grant-read'] = ''; + } + if (additionalFields.grantReadAcp) { + headers['x-amz-grant-read-acp'] = ''; + } + if (additionalFields.grantWriteAcp) { + headers['x-amz-grant-write-acp'] = ''; + } + if (additionalFields.lockLegalHold) { + headers['x-amz-object-lock-legal-hold'] = (additionalFields.lockLegalHold as boolean) + ? 'ON' + : 'OFF'; + } + if (additionalFields.lockMode) { + headers['x-amz-object-lock-mode'] = ( + additionalFields.lockMode as string + ).toUpperCase(); + } + if (additionalFields.lockRetainUntilDate) { + headers['x-amz-object-lock-retain-until-date'] = + additionalFields.lockRetainUntilDate as string; + } + if (additionalFields.serverSideEncryption) { + headers['x-amz-server-side-encryption'] = + additionalFields.serverSideEncryption as string; + } + if (additionalFields.encryptionAwsKmsKeyId) { + headers['x-amz-server-side-encryption-aws-kms-key-id'] = + additionalFields.encryptionAwsKmsKeyId as string; + } + if (additionalFields.serverSideEncryptionContext) { + headers['x-amz-server-side-encryption-context'] = + additionalFields.serverSideEncryptionContext as string; + } + if (additionalFields.serversideEncryptionCustomerAlgorithm) { + headers['x-amz-server-side-encryption-customer-algorithm'] = + additionalFields.serversideEncryptionCustomerAlgorithm as string; + } + if (additionalFields.serversideEncryptionCustomerKey) { + headers['x-amz-server-side-encryption-customer-key'] = + additionalFields.serversideEncryptionCustomerKey as string; + } + if (additionalFields.serversideEncryptionCustomerKeyMD5) { + headers['x-amz-server-side-encryption-customer-key-MD5'] = + additionalFields.serversideEncryptionCustomerKeyMD5 as string; + } + if (additionalFields.taggingDirective) { + headers['x-amz-tagging-directive'] = ( + additionalFields.taggingDirective as string + ).toUpperCase(); + } + if (additionalFields.metadataDirective) { + headers['x-amz-metadata-directive'] = ( + additionalFields.metadataDirective as string + ).toUpperCase(); + } + + const destinationParts = destinationPath.split('/'); + + const bucketName = destinationParts[1]; + + const destination = `/${destinationParts.slice(2, destinationParts.length).join('/')}`; + + responseData = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._; + + responseData = await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'PUT', + destination, + '', + qs, + headers, + {}, + region as string, + ); + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray(responseData.CopyObjectResult as IDataObject[]), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html + if (operation === 'download') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + + const fileKey = this.getNodeParameter('fileKey', i) as string; + + const fileName = fileKey.split('/')[fileKey.split('/').length - 1]; + + if (fileKey.substring(fileKey.length - 1) === '/') { + throw new NodeOperationError( + this.getNode(), + 'Downloading a whole directory is not yet supported, please provide a file key', + ); + } + + let region = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + region = region.LocationConstraint._; + + const response = await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'GET', + `/${fileKey}`, + '', + qs, + {}, + { encoding: null, resolveWithFullResponse: true }, + region as string, + ); + + let mimeType: string | undefined; + if (response.headers['content-type']) { + mimeType = response.headers['content-type']; + } + + const newItem: INodeExecutionData = { + json: items[i].json, + binary: {}, + }; + + if (items[i].binary !== undefined && newItem.binary) { + // Create a shallow copy of the binary data so that the old + // data references which do not get changed still stay behind + // but the incoming data does not get changed. + Object.assign(newItem.binary, items[i].binary); + } + + items[i] = newItem; + + const dataPropertyNameDownload = this.getNodeParameter('binaryPropertyName', i); + + const data = Buffer.from(response.body as string, 'utf8'); + + items[i].binary![dataPropertyNameDownload] = await this.helpers.prepareBinaryData( + data as unknown as Buffer, + fileName, + mimeType, + ); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html + if (operation === 'delete') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + + const fileKey = this.getNodeParameter('fileKey', i) as string; + + const options = this.getNodeParameter('options', i); + + if (options.versionId) { + qs.versionId = options.versionId as string; + } + + responseData = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._; + + responseData = await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'DELETE', + `/${fileKey}`, + '', + qs, + {}, + {}, + region as string, + ); + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray({ success: true }), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + if (operation === 'getAll') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const returnAll = this.getNodeParameter('returnAll', 0); + const options = this.getNodeParameter('options', 0); + + if (options.folderKey) { + qs.prefix = options.folderKey as string; + } + + if (options.fetchOwner) { + qs['fetch-owner'] = options.fetchOwner as string; + } + + qs.delimiter = '/'; + + qs['list-type'] = 2; + + responseData = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + + const region = responseData.LocationConstraint._; + + if (returnAll) { + responseData = await awsApiRequestRESTAllItems.call( + this, + 'ListBucketResult.Contents', + `${bucketName}.s3`, + 'GET', + '', + '', + qs, + {}, + {}, + region as string, + ); + } else { + qs.limit = this.getNodeParameter('limit', 0); + responseData = await awsApiRequestRESTAllItems.call( + this, + 'ListBucketResult.Contents', + `${bucketName}.s3`, + 'GET', + '', + '', + qs, + {}, + {}, + region as string, + ); + responseData = responseData.splice(0, qs.limit); + } + if (Array.isArray(responseData)) { + responseData = responseData.filter( + (e: IDataObject) => !(e.Key as string).endsWith('/') && e.Size !== '0', + ); + if (qs.limit) { + responseData = responseData.splice(0, qs.limit as number); + } + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray(responseData), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html + if (operation === 'upload') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const fileName = this.getNodeParameter('fileName', i) as string; + const isBinaryData = this.getNodeParameter('binaryData', i); + const additionalFields = this.getNodeParameter('additionalFields', i); + const tagsValues = (this.getNodeParameter('tagsUi', i) as IDataObject) + .tagsValues as IDataObject[]; + let path = ''; + let body; + + const multipartHeaders: IDataObject = {}; + const neededHeaders: IDataObject = {}; + + if (additionalFields.requesterPays) { + neededHeaders['x-amz-request-payer'] = 'requester'; + } + if (additionalFields.parentFolderKey) { + path = `${additionalFields.parentFolderKey}/${fileName}`; + } else { + path = `${fileName}`; + } + if (additionalFields.storageClass) { + multipartHeaders['x-amz-storage-class'] = snakeCase( + additionalFields.storageClass as string, + ).toUpperCase(); + } + + if (additionalFields.acl) { + multipartHeaders['x-amz-acl'] = paramCase(additionalFields.acl as string); + } + if (additionalFields.grantFullControl) { + multipartHeaders['x-amz-grant-full-control'] = ''; + } + if (additionalFields.grantRead) { + multipartHeaders['x-amz-grant-read'] = ''; + } + if (additionalFields.grantReadAcp) { + multipartHeaders['x-amz-grant-read-acp'] = ''; + } + if (additionalFields.grantWriteAcp) { + multipartHeaders['x-amz-grant-write-acp'] = ''; + } + if (additionalFields.lockLegalHold) { + multipartHeaders['x-amz-object-lock-legal-hold'] = + (additionalFields.lockLegalHold as boolean) ? 'ON' : 'OFF'; + } + if (additionalFields.lockMode) { + multipartHeaders['x-amz-object-lock-mode'] = ( + additionalFields.lockMode as string + ).toUpperCase(); + } + if (additionalFields.lockRetainUntilDate) { + multipartHeaders['x-amz-object-lock-retain-until-date'] = + additionalFields.lockRetainUntilDate as string; + } + if (additionalFields.serverSideEncryption) { + neededHeaders['x-amz-server-side-encryption'] = + additionalFields.serverSideEncryption as string; + } + if (additionalFields.encryptionAwsKmsKeyId) { + neededHeaders['x-amz-server-side-encryption-aws-kms-key-id'] = + additionalFields.encryptionAwsKmsKeyId as string; + } + if (additionalFields.serverSideEncryptionContext) { + neededHeaders['x-amz-server-side-encryption-context'] = + additionalFields.serverSideEncryptionContext as string; + } + if (additionalFields.serversideEncryptionCustomerAlgorithm) { + neededHeaders['x-amz-server-side-encryption-customer-algorithm'] = + additionalFields.serversideEncryptionCustomerAlgorithm as string; + } + if (additionalFields.serversideEncryptionCustomerKey) { + neededHeaders['x-amz-server-side-encryption-customer-key'] = + additionalFields.serversideEncryptionCustomerKey as string; + } + if (additionalFields.serversideEncryptionCustomerKeyMD5) { + neededHeaders['x-amz-server-side-encryption-customer-key-MD5'] = + additionalFields.serversideEncryptionCustomerKeyMD5 as string; + } + if (tagsValues) { + const tags: string[] = []; + tagsValues.forEach((o: IDataObject) => { + tags.push(`${o.key}=${o.value}`); + }); + multipartHeaders['x-amz-tagging'] = tags.join('&'); + } + // Get the region of the bucket + responseData = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', '', '', { + location: '', + }); + const region = responseData.LocationConstraint._; + + if (isBinaryData) { + const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i); + const binaryPropertyData = this.helpers.assertBinaryData(i, binaryPropertyName); + let uploadData: Buffer | Readable; + multipartHeaders['Content-Type'] = binaryPropertyData.mimeType; + if (binaryPropertyData.id) { + uploadData = this.helpers.getBinaryStream(binaryPropertyData.id, UPLOAD_CHUNK_SIZE); + const createMultiPartUpload = await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'POST', + `/${path}?uploads`, + body, + qs, + { ...neededHeaders, ...multipartHeaders }, + {}, + region as string, + ); + + const uploadId = createMultiPartUpload.InitiateMultipartUploadResult.UploadId; + let part = 1; + for await (const chunk of uploadData) { + const chunkBuffer = await this.helpers.binaryToBuffer(chunk as Readable); + const listHeaders: IDataObject = { + 'Content-Length': chunk.length, + 'Content-MD5': createHash('MD5').update(chunkBuffer).digest('base64'), + ...neededHeaders, + }; + try { + await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'PUT', + `/${path}?partNumber=${part}&uploadId=${uploadId}`, + chunk, + qs, + listHeaders, + {}, + region as string, + ); + part++; + } catch (error) { + try { + await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'DELETE', + `/${path}?uploadId=${uploadId}`, + ); + } catch (err) { + throw new NodeOperationError(this.getNode(), err as Error); + } + if (error.response?.status !== 308) throw error; + } + } + + const listParts = (await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'GET', + `/${path}?max-parts=${900}&part-number-marker=0&uploadId=${uploadId}`, + '', + qs, + { ...neededHeaders }, + {}, + region as string, + )) as { + ListPartsResult: { + Part: + | Array<{ + ETag: string; + PartNumber: number; + }> + | { + ETag: string; + PartNumber: number; + }; + }; + }; + if (!Array.isArray(listParts.ListPartsResult.Part)) { + body = { + CompleteMultipartUpload: { + $: { + xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/', + }, + Part: { + ETag: listParts.ListPartsResult.Part.ETag, + PartNumber: listParts.ListPartsResult.Part.PartNumber, + }, + }, + }; + } else { + body = { + CompleteMultipartUpload: { + $: { + xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/', + }, + Part: listParts.ListPartsResult.Part.map((Part) => { + return { + ETag: Part.ETag, + PartNumber: Part.PartNumber, + }; + }), + }, + }; + } + const builder = new Builder(); + const data = builder.buildObject(body); + const completeUpload = (await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'POST', + `/${path}?uploadId=${uploadId}`, + data, + qs, + { + ...neededHeaders, + 'Content-MD5': createHash('md5').update(data).digest('base64'), + 'Content-Type': 'application/xml', + }, + {}, + region as string, + )) as { + CompleteMultipartUploadResult: { + Location: string; + Bucket: string; + Key: string; + ETag: string; + }; + }; + responseData = { + ...completeUpload.CompleteMultipartUploadResult, + }; + } else { + const binaryDataBuffer = await this.helpers.getBinaryDataBuffer( + i, + binaryPropertyName, + ); + + body = binaryDataBuffer; + headers = { ...neededHeaders, ...multipartHeaders }; + headers['Content-Type'] = binaryPropertyData.mimeType; + + headers['Content-MD5'] = createHash('md5').update(body).digest('base64'); + + responseData = await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'PUT', + `/${path || binaryPropertyData.fileName}`, + body, + qs, + headers, + {}, + region as string, + ); + } + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray(responseData as IDataObject), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } else { + const fileContent = this.getNodeParameter('fileContent', i) as string; + + body = Buffer.from(fileContent, 'utf8'); + + headers = { ...neededHeaders, ...multipartHeaders }; + + headers['Content-Type'] = 'text/html'; + + headers['Content-MD5'] = createHash('md5').update(fileContent).digest('base64'); + + responseData = await awsApiRequestREST.call( + this, + `${bucketName}.s3`, + 'PUT', + `/${path}`, + body, + qs, + { ...headers }, + {}, + region as string, + ); + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray({ success: true }), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + } + } + } + } catch (error) { + if (this.continueOnFail()) { + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray({ error: error.message }), + { itemData: { item: i } }, + ); + returnData.push(...executionData); + continue; + } + throw error; + } + } + if (resource === 'file' && operation === 'download') { + // For file downloads the files get attached to the existing items + return this.prepareOutputData(items); + } else { + return this.prepareOutputData(returnData); + } + } +} diff --git a/packages/nodes-base/nodes/Aws/S3/V2/BucketDescription.ts b/packages/nodes-base/nodes/Aws/S3/V2/BucketDescription.ts new file mode 100644 index 0000000000000..1fba6ba5074a8 --- /dev/null +++ b/packages/nodes-base/nodes/Aws/S3/V2/BucketDescription.ts @@ -0,0 +1,321 @@ +import type { INodeProperties } from 'n8n-workflow'; + +export const bucketOperations: INodeProperties[] = [ + { + displayName: 'Operation', + name: 'operation', + type: 'options', + noDataExpression: true, + displayOptions: { + show: { + resource: ['bucket'], + }, + }, + options: [ + { + name: 'Create', + value: 'create', + description: 'Create a bucket', + action: 'Create a bucket', + }, + { + name: 'Delete', + value: 'delete', + description: 'Delete a bucket', + action: 'Delete a bucket', + }, + { + name: 'Get Many', + value: 'getAll', + description: 'Get many buckets', + action: 'Get many buckets', + }, + { + name: 'Search', + value: 'search', + description: 'Search within a bucket', + action: 'Search a bucket', + }, + ], + default: 'create', + }, +]; + +export const bucketFields: INodeProperties[] = [ + /* -------------------------------------------------------------------------- */ + /* bucket:create */ + /* -------------------------------------------------------------------------- */ + { + displayName: 'Name', + name: 'name', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: ['bucket'], + operation: ['create'], + }, + }, + description: 'A succinct description of the nature, symptoms, cause, or effect of the bucket', + }, + { + displayName: 'Additional Fields', + name: 'additionalFields', + type: 'collection', + placeholder: 'Add Field', + displayOptions: { + show: { + resource: ['bucket'], + operation: ['create'], + }, + }, + default: {}, + options: [ + { + displayName: 'ACL', + name: 'acl', + type: 'options', + options: [ + { + name: 'Authenticated Read', + value: 'authenticatedRead', + }, + { + name: 'Private', + value: 'Private', + }, + { + name: 'Public Read', + value: 'publicRead', + }, + { + name: 'Public Read Write', + value: 'publicReadWrite', + }, + ], + default: '', + description: 'The canned ACL to apply to the bucket', + }, + { + displayName: 'Bucket Object Lock Enabled', + name: 'bucketObjectLockEnabled', + type: 'boolean', + default: false, + description: 'Whether you want S3 Object Lock to be enabled for the new bucket', + }, + { + displayName: 'Grant Full Control', + name: 'grantFullControl', + type: 'boolean', + default: false, + description: + 'Whether to allow grantee the read, write, read ACP, and write ACP permissions on the bucket', + }, + { + displayName: 'Grant Read', + name: 'grantRead', + type: 'boolean', + default: false, + description: 'Whether to allow grantee to list the objects in the bucket', + }, + { + displayName: 'Grant Read ACP', + name: 'grantReadAcp', + type: 'boolean', + default: false, + description: 'Whether to allow grantee to read the bucket ACL', + }, + { + displayName: 'Grant Write', + name: 'grantWrite', + type: 'boolean', + default: false, + description: + 'Whether to allow grantee to create, overwrite, and delete any object in the bucket', + }, + { + displayName: 'Grant Write ACP', + name: 'grantWriteAcp', + type: 'boolean', + default: false, + description: 'Whether to allow grantee to write the ACL for the applicable bucket', + }, + { + displayName: 'Region', + name: 'region', + type: 'string', + default: '', + description: + 'Region you want to create the bucket in, by default the buckets are created on the region defined on the credentials', + }, + ], + }, + + /* -------------------------------------------------------------------------- */ + /* bucket:delete */ + /* -------------------------------------------------------------------------- */ + { + displayName: 'Name', + name: 'name', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: ['bucket'], + operation: ['delete'], + }, + }, + description: 'Name of the AWS S3 bucket to delete', + }, + + /* -------------------------------------------------------------------------- */ + /* bucket:getAll */ + /* -------------------------------------------------------------------------- */ + { + displayName: 'Return All', + name: 'returnAll', + type: 'boolean', + displayOptions: { + show: { + operation: ['getAll'], + resource: ['bucket'], + }, + }, + default: false, + description: 'Whether to return all results or only up to a given limit', + }, + { + displayName: 'Limit', + name: 'limit', + type: 'number', + displayOptions: { + show: { + operation: ['getAll'], + resource: ['bucket'], + returnAll: [false], + }, + }, + typeOptions: { + minValue: 1, + maxValue: 500, + }, + default: 100, + description: 'Max number of results to return', + }, + /* -------------------------------------------------------------------------- */ + /* bucket:search */ + /* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: ['bucket'], + operation: ['search'], + }, + }, + }, + { + displayName: 'Return All', + name: 'returnAll', + type: 'boolean', + displayOptions: { + show: { + operation: ['search'], + resource: ['bucket'], + }, + }, + default: false, + description: 'Whether to return all results or only up to a given limit', + }, + { + displayName: 'Limit', + name: 'limit', + type: 'number', + displayOptions: { + show: { + operation: ['search'], + resource: ['bucket'], + returnAll: [false], + }, + }, + typeOptions: { + minValue: 1, + maxValue: 500, + }, + default: 100, + description: 'Max number of results to return', + }, + { + displayName: 'Additional Fields', + name: 'additionalFields', + type: 'collection', + placeholder: 'Add Field', + displayOptions: { + show: { + resource: ['bucket'], + operation: ['search'], + }, + }, + default: {}, + options: [ + { + displayName: 'Delimiter', + name: 'delimiter', + type: 'string', + default: '', + description: 'A delimiter is a character you use to group keys', + }, + { + displayName: 'Encoding Type', + name: 'encodingType', + type: 'options', + options: [ + { + name: 'URL', + value: 'url', + }, + ], + default: '', + description: 'Encoding type used by Amazon S3 to encode object keys in the response', + }, + { + displayName: 'Fetch Owner', + name: 'fetchOwner', + type: 'boolean', + default: false, + // eslint-disable-next-line n8n-nodes-base/node-param-description-boolean-without-whether + description: + 'The owner field is not present in listV2 by default, if you want to return owner field with each key in the result then set the fetch owner field to true', + }, + { + displayName: 'Prefix', + name: 'prefix', + type: 'string', + default: '', + description: 'Limits the response to keys that begin with the specified prefix', + }, + { + displayName: 'Requester Pays', + name: 'requesterPays', + type: 'boolean', + default: false, + description: + 'Whether the requester will pay for requests and data transfer. While Requester Pays is enabled, anonymous access to this bucket is disabled.', + }, + { + displayName: 'Start After', + name: 'startAfter', + type: 'string', + default: '', + description: + 'StartAfter is where you want Amazon S3 to start listing from. Amazon S3 starts listing after this specified key.', + }, + ], + }, +]; diff --git a/packages/nodes-base/nodes/Aws/S3/V2/FileDescription.ts b/packages/nodes-base/nodes/Aws/S3/V2/FileDescription.ts new file mode 100644 index 0000000000000..be5146777efed --- /dev/null +++ b/packages/nodes-base/nodes/Aws/S3/V2/FileDescription.ts @@ -0,0 +1,841 @@ +import type { INodeProperties } from 'n8n-workflow'; + +export const fileOperations: INodeProperties[] = [ + { + displayName: 'Operation', + name: 'operation', + type: 'options', + noDataExpression: true, + displayOptions: { + show: { + resource: ['file'], + }, + }, + options: [ + { + name: 'Copy', + value: 'copy', + description: 'Copy a file', + action: 'Copy a file', + }, + { + name: 'Delete', + value: 'delete', + description: 'Delete a file', + action: 'Delete a file', + }, + { + name: 'Download', + value: 'download', + description: 'Download a file', + action: 'Download a file', + }, + { + name: 'Get Many', + value: 'getAll', + description: 'Get many files', + action: 'Get many files', + }, + { + name: 'Upload', + value: 'upload', + description: 'Upload a file', + action: 'Upload a file', + }, + ], + default: 'download', + }, +]; + +export const fileFields: INodeProperties[] = [ + /* -------------------------------------------------------------------------- */ + /* file:copy */ + /* -------------------------------------------------------------------------- */ + { + displayName: 'Source Path', + name: 'sourcePath', + type: 'string', + required: true, + default: '', + placeholder: '/bucket/my-image.jpg', + displayOptions: { + show: { + resource: ['file'], + operation: ['copy'], + }, + }, + description: + 'The name of the source bucket and key name of the source object, separated by a slash (/)', + }, + { + displayName: 'Destination Path', + name: 'destinationPath', + type: 'string', + required: true, + default: '', + placeholder: '/bucket/my-second-image.jpg', + displayOptions: { + show: { + resource: ['file'], + operation: ['copy'], + }, + }, + description: + 'The name of the destination bucket and key name of the destination object, separated by a slash (/)', + }, + { + displayName: 'Additional Fields', + name: 'additionalFields', + type: 'collection', + placeholder: 'Add Field', + displayOptions: { + show: { + resource: ['file'], + operation: ['copy'], + }, + }, + default: {}, + options: [ + { + displayName: 'ACL', + name: 'acl', + type: 'options', + options: [ + { + name: 'Authenticated Read', + value: 'authenticatedRead', + }, + { + name: 'AWS Exec Read', + value: 'awsExecRead', + }, + { + name: 'Bucket Owner Full Control', + value: 'bucketOwnerFullControl', + }, + { + name: 'Bucket Owner Read', + value: 'bucketOwnerRead', + }, + { + name: 'Private', + value: 'private', + }, + { + name: 'Public Read', + value: 'publicRead', + }, + { + name: 'Public Read Write', + value: 'publicReadWrite', + }, + ], + default: 'private', + description: 'The canned ACL to apply to the object', + }, + { + displayName: 'Grant Full Control', + name: 'grantFullControl', + type: 'boolean', + default: false, + description: + 'Whether to give the grantee READ, READ_ACP, and WRITE_ACP permissions on the object', + }, + { + displayName: 'Grant Read', + name: 'grantRead', + type: 'boolean', + default: false, + description: 'Whether to allow grantee to read the object data and its metadata', + }, + { + displayName: 'Grant Read ACP', + name: 'grantReadAcp', + type: 'boolean', + default: false, + description: 'Whether to allow grantee to read the object ACL', + }, + { + displayName: 'Grant Write ACP', + name: 'grantWriteAcp', + type: 'boolean', + default: false, + description: 'Whether to allow grantee to write the ACL for the applicable object', + }, + { + displayName: 'Lock Legal Hold', + name: 'lockLegalHold', + type: 'boolean', + default: false, + description: 'Whether a legal hold will be applied to this object', + }, + { + displayName: 'Lock Mode', + name: 'lockMode', + type: 'options', + options: [ + { + name: 'Governance', + value: 'governance', + }, + { + name: 'Compliance', + value: 'compliance', + }, + ], + default: '', + description: 'The Object Lock mode that you want to apply to this object', + }, + { + displayName: 'Lock Retain Until Date', + name: 'lockRetainUntilDate', + type: 'dateTime', + default: '', + description: "The date and time when you want this object's Object Lock to expire", + }, + { + displayName: 'Metadata Directive', + name: 'metadataDirective', + type: 'options', + options: [ + { + name: 'Copy', + value: 'copy', + }, + { + name: 'Replace', + value: 'replace', + }, + ], + default: '', + description: + 'Specifies whether the metadata is copied from the source object or replaced with metadata provided in the request', + }, + { + displayName: 'Requester Pays', + name: 'requesterPays', + type: 'boolean', + default: false, + description: + 'Whether the requester will pay for requests and data transfer. While Requester Pays is enabled, anonymous access to this bucket is disabled.', + }, + { + displayName: 'Server Side Encryption', + name: 'serverSideEncryption', + type: 'options', + options: [ + { + name: 'AES256', + value: 'AES256', + }, + { + name: 'AWS:KMS', + value: 'aws:kms', + }, + ], + default: '', + description: + 'The server-side encryption algorithm used when storing this object in Amazon S3', + }, + { + displayName: 'Server Side Encryption Context', + name: 'serverSideEncryptionContext', + type: 'string', + default: '', + description: 'Specifies the AWS KMS Encryption Context to use for object encryption', + }, + { + displayName: 'Server Side Encryption AWS KMS Key ID', + name: 'encryptionAwsKmsKeyId', + type: 'string', + default: '', + description: 'If x-amz-server-side-encryption is present and has the value of aws:kms', + }, + { + displayName: 'Server Side Encryption Customer Algorithm', + name: 'serversideEncryptionCustomerAlgorithm', + type: 'string', + default: '', + description: + 'Specifies the algorithm to use to when encrypting the object (for example, AES256)', + }, + { + displayName: 'Server Side Encryption Customer Key', + name: 'serversideEncryptionCustomerKey', + type: 'string', + default: '', + description: + 'Specifies the customer-provided encryption key for Amazon S3 to use in encrypting data', + }, + { + displayName: 'Server Side Encryption Customer Key MD5', + name: 'serversideEncryptionCustomerKeyMD5', + type: 'string', + default: '', + description: 'Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321', + }, + { + displayName: 'Storage Class', + name: 'storageClass', + type: 'options', + options: [ + { + name: 'Deep Archive', + value: 'deepArchive', + }, + { + name: 'Glacier', + value: 'glacier', + }, + { + name: 'Intelligent Tiering', + value: 'intelligentTiering', + }, + { + name: 'One Zone IA', + value: 'onezoneIA', + }, + { + name: 'Standard', + value: 'standard', + }, + { + name: 'Standard IA', + value: 'standardIA', + }, + ], + default: 'standard', + description: 'Amazon S3 storage classes', + }, + { + displayName: 'Tagging Directive', + name: 'taggingDirective', + type: 'options', + options: [ + { + name: 'Copy', + value: 'copy', + }, + { + name: 'Replace', + value: 'replace', + }, + ], + default: '', + description: + 'Specifies whether the metadata is copied from the source object or replaced with metadata provided in the request', + }, + ], + }, + /* -------------------------------------------------------------------------- */ + /* file:upload */ + /* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: ['file'], + operation: ['upload'], + }, + }, + }, + { + displayName: 'File Name', + name: 'fileName', + type: 'string', + default: '', + placeholder: 'hello.txt', + required: true, + displayOptions: { + show: { + resource: ['file'], + operation: ['upload'], + binaryData: [false], + }, + }, + }, + { + displayName: 'File Name', + name: 'fileName', + type: 'string', + default: '', + displayOptions: { + show: { + resource: ['file'], + operation: ['upload'], + binaryData: [true], + }, + }, + description: 'If not set the binary data filename will be used', + }, + { + displayName: 'Binary Data', + name: 'binaryData', + type: 'boolean', + default: true, + displayOptions: { + show: { + operation: ['upload'], + resource: ['file'], + }, + }, + description: 'Whether the data to upload should be taken from binary field', + }, + { + displayName: 'File Content', + name: 'fileContent', + type: 'string', + default: '', + displayOptions: { + show: { + operation: ['upload'], + resource: ['file'], + binaryData: [false], + }, + }, + placeholder: '', + description: 'The text content of the file to upload', + }, + { + displayName: 'Binary Property', + name: 'binaryPropertyName', + type: 'string', + default: 'data', + required: true, + displayOptions: { + show: { + operation: ['upload'], + resource: ['file'], + binaryData: [true], + }, + }, + placeholder: '', + description: 'Name of the binary property which contains the data for the file to be uploaded', + }, + { + displayName: 'Additional Fields', + name: 'additionalFields', + type: 'collection', + placeholder: 'Add Field', + displayOptions: { + show: { + resource: ['file'], + operation: ['upload'], + }, + }, + default: {}, + options: [ + { + displayName: 'ACL', + name: 'acl', + type: 'options', + options: [ + { + name: 'Authenticated Read', + value: 'authenticatedRead', + }, + { + name: 'AWS Exec Read', + value: 'awsExecRead', + }, + { + name: 'Bucket Owner Full Control', + value: 'bucketOwnerFullControl', + }, + { + name: 'Bucket Owner Read', + value: 'bucketOwnerRead', + }, + { + name: 'Private', + value: 'private', + }, + { + name: 'Public Read', + value: 'publicRead', + }, + { + name: 'Public Read Write', + value: 'publicReadWrite', + }, + ], + default: 'private', + description: 'The canned ACL to apply to the object', + }, + { + displayName: 'Grant Full Control', + name: 'grantFullControl', + type: 'boolean', + default: false, + description: + 'Whether to give the grantee READ, READ_ACP, and WRITE_ACP permissions on the object', + }, + { + displayName: 'Grant Read', + name: 'grantRead', + type: 'boolean', + default: false, + description: 'Whether to allow grantee to read the object data and its metadata', + }, + { + displayName: 'Grant Read ACP', + name: 'grantReadAcp', + type: 'boolean', + default: false, + description: 'Whether to allow grantee to read the object ACL', + }, + { + displayName: 'Grant Write ACP', + name: 'grantWriteAcp', + type: 'boolean', + default: false, + description: 'Whether to allow grantee to write the ACL for the applicable object', + }, + { + displayName: 'Lock Legal Hold', + name: 'lockLegalHold', + type: 'boolean', + default: false, + description: 'Whether a legal hold will be applied to this object', + }, + { + displayName: 'Lock Mode', + name: 'lockMode', + type: 'options', + options: [ + { + name: 'Governance', + value: 'governance', + }, + { + name: 'Compliance', + value: 'compliance', + }, + ], + default: '', + description: 'The Object Lock mode that you want to apply to this object', + }, + { + displayName: 'Lock Retain Until Date', + name: 'lockRetainUntilDate', + type: 'dateTime', + default: '', + description: "The date and time when you want this object's Object Lock to expire", + }, + { + displayName: 'Parent Folder Key', + name: 'parentFolderKey', + type: 'string', + default: '', + description: 'Parent folder you want to create the file in', + }, + { + displayName: 'Requester Pays', + name: 'requesterPays', + type: 'boolean', + default: false, + description: + 'Whether the requester will pay for requests and data transfer. While Requester Pays is enabled, anonymous access to this bucket is disabled.', + }, + { + displayName: 'Server Side Encryption', + name: 'serverSideEncryption', + type: 'options', + options: [ + { + name: 'AES256', + value: 'AES256', + }, + { + name: 'AWS:KMS', + value: 'aws:kms', + }, + ], + default: '', + description: + 'The server-side encryption algorithm used when storing this object in Amazon S3', + }, + { + displayName: 'Server Side Encryption Context', + name: 'serverSideEncryptionContext', + type: 'string', + default: '', + description: 'Specifies the AWS KMS Encryption Context to use for object encryption', + }, + { + displayName: 'Server Side Encryption AWS KMS Key ID', + name: 'encryptionAwsKmsKeyId', + type: 'string', + default: '', + description: 'If x-amz-server-side-encryption is present and has the value of aws:kms', + }, + { + displayName: 'Server Side Encryption Customer Algorithm', + name: 'serversideEncryptionCustomerAlgorithm', + type: 'string', + default: '', + description: + 'Specifies the algorithm to use to when encrypting the object (for example, AES256)', + }, + { + displayName: 'Server Side Encryption Customer Key', + name: 'serversideEncryptionCustomerKey', + type: 'string', + default: '', + description: + 'Specifies the customer-provided encryption key for Amazon S3 to use in encrypting data', + }, + { + displayName: 'Server Side Encryption Customer Key MD5', + name: 'serversideEncryptionCustomerKeyMD5', + type: 'string', + default: '', + description: 'Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321', + }, + { + displayName: 'Storage Class', + name: 'storageClass', + type: 'options', + options: [ + { + name: 'Deep Archive', + value: 'deepArchive', + }, + { + name: 'Glacier', + value: 'glacier', + }, + { + name: 'Intelligent Tiering', + value: 'intelligentTiering', + }, + { + name: 'One Zone IA', + value: 'onezoneIA', + }, + { + name: 'Standard', + value: 'standard', + }, + { + name: 'Standard IA', + value: 'standardIA', + }, + ], + default: 'standard', + description: 'Amazon S3 storage classes', + }, + ], + }, + { + displayName: 'Tags', + name: 'tagsUi', + placeholder: 'Add Tag', + type: 'fixedCollection', + default: {}, + typeOptions: { + multipleValues: true, + }, + displayOptions: { + show: { + resource: ['file'], + operation: ['upload'], + }, + }, + options: [ + { + name: 'tagsValues', + displayName: 'Tag', + values: [ + { + displayName: 'Key', + name: 'key', + type: 'string', + default: '', + }, + { + displayName: 'Value', + name: 'value', + type: 'string', + default: '', + }, + ], + }, + ], + description: 'Optional extra headers to add to the message (most headers are allowed)', + }, + /* -------------------------------------------------------------------------- */ + /* file:download */ + /* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: ['file'], + operation: ['download'], + }, + }, + }, + { + displayName: 'File Key', + name: 'fileKey', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: ['file'], + operation: ['download'], + }, + }, + }, + { + displayName: 'Binary Property', + name: 'binaryPropertyName', + type: 'string', + required: true, + default: 'data', + displayOptions: { + show: { + operation: ['download'], + resource: ['file'], + }, + }, + description: 'Name of the binary property to which to write the data of the read file', + }, + /* -------------------------------------------------------------------------- */ + /* file:delete */ + /* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: ['file'], + operation: ['delete'], + }, + }, + }, + { + displayName: 'File Key', + name: 'fileKey', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: ['file'], + operation: ['delete'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['file'], + operation: ['delete'], + }, + }, + options: [ + { + displayName: 'Version ID', + name: 'versionId', + type: 'string', + default: '', + }, + ], + }, + /* -------------------------------------------------------------------------- */ + /* file:getAll */ + /* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: ['file'], + operation: ['getAll'], + }, + }, + }, + { + displayName: 'Return All', + name: 'returnAll', + type: 'boolean', + displayOptions: { + show: { + operation: ['getAll'], + resource: ['file'], + }, + }, + default: false, + description: 'Whether to return all results or only up to a given limit', + }, + { + displayName: 'Limit', + name: 'limit', + type: 'number', + displayOptions: { + show: { + operation: ['getAll'], + resource: ['file'], + returnAll: [false], + }, + }, + typeOptions: { + minValue: 1, + maxValue: 500, + }, + default: 100, + description: 'Max number of results to return', + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['file'], + operation: ['getAll'], + }, + }, + options: [ + { + displayName: 'Fetch Owner', + name: 'fetchOwner', + type: 'boolean', + default: false, + // eslint-disable-next-line n8n-nodes-base/node-param-description-boolean-without-whether + description: + 'The owner field is not present in listV2 by default, if you want to return owner field with each key in the result then set the fetch owner field to true', + }, + { + displayName: 'Folder Key', + name: 'folderKey', + type: 'string', + default: '', + }, + ], + }, +]; diff --git a/packages/nodes-base/nodes/Aws/S3/V2/FolderDescription.ts b/packages/nodes-base/nodes/Aws/S3/V2/FolderDescription.ts new file mode 100644 index 0000000000000..db7db64ced245 --- /dev/null +++ b/packages/nodes-base/nodes/Aws/S3/V2/FolderDescription.ts @@ -0,0 +1,241 @@ +import type { INodeProperties } from 'n8n-workflow'; + +export const folderOperations: INodeProperties[] = [ + { + displayName: 'Operation', + name: 'operation', + type: 'options', + noDataExpression: true, + displayOptions: { + show: { + resource: ['folder'], + }, + }, + options: [ + { + name: 'Create', + value: 'create', + description: 'Create a folder', + action: 'Create a folder', + }, + { + name: 'Delete', + value: 'delete', + description: 'Delete a folder', + action: 'Delete a folder', + }, + { + name: 'Get Many', + value: 'getAll', + description: 'Get many folders', + action: 'Get many folders', + }, + ], + default: 'create', + }, +]; + +export const folderFields: INodeProperties[] = [ + /* -------------------------------------------------------------------------- */ + /* folder:create */ + /* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: ['folder'], + operation: ['create'], + }, + }, + }, + { + displayName: 'Folder Name', + name: 'folderName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: ['folder'], + operation: ['create'], + }, + }, + }, + { + displayName: 'Additional Fields', + name: 'additionalFields', + type: 'collection', + placeholder: 'Add Field', + displayOptions: { + show: { + resource: ['folder'], + operation: ['create'], + }, + }, + default: {}, + options: [ + { + displayName: 'Parent Folder Key', + name: 'parentFolderKey', + type: 'string', + default: '', + description: 'Parent folder you want to create the folder in', + }, + { + displayName: 'Requester Pays', + name: 'requesterPays', + type: 'boolean', + default: false, + description: + 'Whether the requester will pay for requests and data transfer. While Requester Pays is enabled, anonymous access to this bucket is disabled.', + }, + { + displayName: 'Storage Class', + name: 'storageClass', + type: 'options', + options: [ + { + name: 'Deep Archive', + value: 'deepArchive', + }, + { + name: 'Glacier', + value: 'glacier', + }, + { + name: 'Intelligent Tiering', + value: 'intelligentTiering', + }, + { + name: 'One Zone IA', + value: 'onezoneIA', + }, + { + name: 'Reduced Redundancy', + value: 'RecudedRedundancy', + }, + { + name: 'Standard', + value: 'standard', + }, + { + name: 'Standard IA', + value: 'standardIA', + }, + ], + default: 'standard', + description: 'Amazon S3 storage classes', + }, + ], + }, + /* -------------------------------------------------------------------------- */ + /* folder:delete */ + /* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: ['folder'], + operation: ['delete'], + }, + }, + }, + { + displayName: 'Folder Key', + name: 'folderKey', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: ['folder'], + operation: ['delete'], + }, + }, + }, + /* -------------------------------------------------------------------------- */ + /* folder:getAll */ + /* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: ['folder'], + operation: ['getAll'], + }, + }, + }, + { + displayName: 'Return All', + name: 'returnAll', + type: 'boolean', + displayOptions: { + show: { + operation: ['getAll'], + resource: ['folder'], + }, + }, + default: false, + description: 'Whether to return all results or only up to a given limit', + }, + { + displayName: 'Limit', + name: 'limit', + type: 'number', + displayOptions: { + show: { + operation: ['getAll'], + resource: ['folder'], + returnAll: [false], + }, + }, + typeOptions: { + minValue: 1, + maxValue: 500, + }, + default: 100, + description: 'Max number of results to return', + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['folder'], + operation: ['getAll'], + }, + }, + options: [ + { + displayName: 'Fetch Owner', + name: 'fetchOwner', + type: 'boolean', + default: false, + // eslint-disable-next-line n8n-nodes-base/node-param-description-boolean-without-whether + description: + 'The owner field is not present in listV2 by default, if you want to return owner field with each key in the result then set the fetch owner field to true', + }, + { + displayName: 'Folder Key', + name: 'folderKey', + type: 'string', + default: '', + }, + ], + }, +]; diff --git a/packages/nodes-base/nodes/Aws/S3/V2/GenericFunctions.ts b/packages/nodes-base/nodes/Aws/S3/V2/GenericFunctions.ts new file mode 100644 index 0000000000000..d7c95079c2678 --- /dev/null +++ b/packages/nodes-base/nodes/Aws/S3/V2/GenericFunctions.ts @@ -0,0 +1,132 @@ +import get from 'lodash.get'; + +import { parseString } from 'xml2js'; + +import type { + IDataObject, + IExecuteFunctions, + IHookFunctions, + ILoadOptionsFunctions, + IWebhookFunctions, + IHttpRequestOptions, +} from 'n8n-workflow'; + +export async function awsApiRequest( + this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, + service: string, + method: string, + path: string, + body?: string | Buffer | any, + query: IDataObject = {}, + headers?: object, + option: IDataObject = {}, + _region?: string, +): Promise { + const requestOptions = { + qs: { + ...query, + service, + path, + query, + }, + method, + body, + url: '', + headers, + } as IHttpRequestOptions; + if (Object.keys(option).length !== 0) { + Object.assign(requestOptions, option); + } + return this.helpers.requestWithAuthentication.call(this, 'aws', requestOptions); +} + +export async function awsApiRequestREST( + this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions, + service: string, + method: string, + path: string, + body?: string | Buffer | any, + query: IDataObject = {}, + headers?: object, + options: IDataObject = {}, + region?: string, +): Promise { + const response = await awsApiRequest.call( + this, + service, + method, + path, + body, + query, + headers, + options, + region, + ); + try { + if (response.includes('')) { + return await new Promise((resolve, reject) => { + parseString(response as string, { explicitArray: false }, (err, data) => { + if (err) { + return reject(err); + } + resolve(data); + }); + }); + } + return JSON.parse(response as string); + } catch (error) { + return response; + } +} + +export async function awsApiRequestRESTAllItems( + this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions, + propertyName: string, + service: string, + method: string, + path: string, + body?: string, + query: IDataObject = {}, + headers?: object, + option: IDataObject = {}, + region?: string, +): Promise { + const returnData: IDataObject[] = []; + + let responseData; + do { + responseData = await awsApiRequestREST.call( + this, + service, + method, + path, + body, + query, + headers, + option, + region, + ); + //https://forums.aws.amazon.com/thread.jspa?threadID=55746 + if (get(responseData, `${propertyName.split('.')[0]}.NextContinuationToken`)) { + query['continuation-token'] = get( + responseData, + `${propertyName.split('.')[0]}.NextContinuationToken`, + ); + } + if (get(responseData, propertyName)) { + if (Array.isArray(get(responseData, propertyName))) { + returnData.push.apply(returnData, get(responseData, propertyName) as IDataObject[]); + } else { + returnData.push(get(responseData, propertyName) as IDataObject); + } + } + const limit = query.limit as number | undefined; + if (limit && limit <= returnData.length) { + return returnData; + } + } while ( + get(responseData, `${propertyName.split('.')[0]}.IsTruncated`) !== undefined && + get(responseData, `${propertyName.split('.')[0]}.IsTruncated`) !== 'false' + ); + return returnData; +} diff --git a/packages/nodes-base/nodes/Aws/S3/test/AwsS3.file.upload.workflow.json b/packages/nodes-base/nodes/Aws/S3/test/V1/AwsS3.file.upload.V1.workflow.json similarity index 100% rename from packages/nodes-base/nodes/Aws/S3/test/AwsS3.file.upload.workflow.json rename to packages/nodes-base/nodes/Aws/S3/test/V1/AwsS3.file.upload.V1.workflow.json diff --git a/packages/nodes-base/nodes/Aws/S3/test/AwsS3.node.test.ts b/packages/nodes-base/nodes/Aws/S3/test/V1/AwsS3.node.test.ts similarity index 96% rename from packages/nodes-base/nodes/Aws/S3/test/AwsS3.node.test.ts rename to packages/nodes-base/nodes/Aws/S3/test/V1/AwsS3.node.test.ts index 17faedca2895a..91150c8c86d2d 100644 --- a/packages/nodes-base/nodes/Aws/S3/test/AwsS3.node.test.ts +++ b/packages/nodes-base/nodes/Aws/S3/test/V1/AwsS3.node.test.ts @@ -3,7 +3,7 @@ import { getWorkflowFilenames, initBinaryDataManager, testWorkflows } from '@tes const workflows = getWorkflowFilenames(__dirname); -describe('Test S3 Node', () => { +describe('Test S3 V1 Node', () => { describe('File Upload', () => { let mock: nock.Scope; const now = 1683028800000; diff --git a/packages/nodes-base/nodes/Aws/S3/test/V2/AwsS3.file.upload.V2.workflow.json b/packages/nodes-base/nodes/Aws/S3/test/V2/AwsS3.file.upload.V2.workflow.json new file mode 100644 index 0000000000000..a54d819ef6584 --- /dev/null +++ b/packages/nodes-base/nodes/Aws/S3/test/V2/AwsS3.file.upload.V2.workflow.json @@ -0,0 +1,97 @@ +{ + "name": "Test S3 upload", + "nodes": [ + { + "parameters": {}, + "id": "8f35d24b-1493-43a4-846f-bacb577bfcb2", + "name": "When clicking \"Execute Workflow\"", + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [540, 340] + }, + { + "parameters": { + "mode": "jsonToBinary", + "options": {} + }, + "id": "eae2946a-1a1e-47e9-9fd6-e32119b13ec0", + "name": "Move Binary Data", + "type": "n8n-nodes-base.moveBinaryData", + "typeVersion": 1, + "position": [900, 340] + }, + { + "parameters": { + "operation": "upload", + "bucketName": "bucket", + "fileName": "binary.json", + "additionalFields": {} + }, + "id": "6f21fa3f-ede1-44b1-8182-a2c07152f666", + "name": "AWS S3", + "type": "n8n-nodes-base.awsS3", + "typeVersion": 2, + "position": [1080, 340], + "credentials": { + "aws": { + "id": "1", + "name": "AWS account" + } + } + }, + { + "parameters": { + "jsCode": "return [{ key: \"value\" }];" + }, + "id": "e12f1876-cfd1-47a4-a21b-d478452683bc", + "name": "Code", + "type": "n8n-nodes-base.code", + "typeVersion": 1, + "position": [720, 340] + } + ], + "connections": { + "When clicking \"Execute Workflow\"": { + "main": [ + [ + { + "node": "Code", + "type": "main", + "index": 0 + } + ] + ] + }, + "Move Binary Data": { + "main": [ + [ + { + "node": "AWS S3", + "type": "main", + "index": 0 + } + ] + ] + }, + "Code": { + "main": [ + [ + { + "node": "Move Binary Data", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "pinData": { + "AWS S3": [ + { + "json": { + "success": true + } + } + ] + } +} diff --git a/packages/nodes-base/nodes/Aws/S3/test/V2/AwsS3.node.test.ts b/packages/nodes-base/nodes/Aws/S3/test/V2/AwsS3.node.test.ts new file mode 100644 index 0000000000000..b8a9a3d6c0a22 --- /dev/null +++ b/packages/nodes-base/nodes/Aws/S3/test/V2/AwsS3.node.test.ts @@ -0,0 +1,48 @@ +import nock from 'nock'; +import { getWorkflowFilenames, initBinaryDataManager, testWorkflows } from '@test/nodes/Helpers'; + +const workflows = getWorkflowFilenames(__dirname); + +describe('Test S3 V2 Node', () => { + describe('File Upload', () => { + let mock: nock.Scope; + const now = 1683028800000; + + beforeAll(async () => { + jest.useFakeTimers({ doNotFake: ['nextTick'], now }); + + await initBinaryDataManager(); + + nock.disableNetConnect(); + mock = nock('https://bucket.s3.eu-central-1.amazonaws.com'); + }); + + beforeEach(async () => { + mock.get('/?location').reply( + 200, + ` + + eu-central-1 + `, + { + 'content-type': 'application/xml', + }, + ); + + mock + .put('/binary.json') + .matchHeader( + 'X-Amz-Content-Sha256', + 'e43abcf3375244839c012f9633f95862d232a95b00d5bc7348b3098b9fed7f32', + ) + .once() + .reply(200, { success: true }); + }); + + afterAll(() => { + nock.restore(); + }); + + testWorkflows(workflows); + }); +}); diff --git a/packages/nodes-base/nodes/S3/S3.node.ts b/packages/nodes-base/nodes/S3/S3.node.ts index de9f08825c16d..5ddce6ac5ef12 100644 --- a/packages/nodes-base/nodes/S3/S3.node.ts +++ b/packages/nodes-base/nodes/S3/S3.node.ts @@ -14,11 +14,11 @@ import type { } from 'n8n-workflow'; import { NodeApiError, NodeOperationError } from 'n8n-workflow'; -import { bucketFields, bucketOperations } from '../Aws/S3/BucketDescription'; +import { bucketFields, bucketOperations } from '../Aws/S3/V1/BucketDescription'; -import { folderFields, folderOperations } from '../Aws/S3/FolderDescription'; +import { folderFields, folderOperations } from '../Aws/S3/V1/FolderDescription'; -import { fileFields, fileOperations } from '../Aws/S3/FileDescription'; +import { fileFields, fileOperations } from '../Aws/S3/V1/FileDescription'; import { s3ApiRequestREST, s3ApiRequestSOAP, s3ApiRequestSOAPAllItems } from './GenericFunctions'; From c2afed4ca189b133b4281e951280114a586d0e85 Mon Sep 17 00:00:00 2001 From: Alex Grozav Date: Thu, 15 Jun 2023 14:40:23 +0300 Subject: [PATCH 15/37] fix: Fix randomly failing scheduler node e2e tests (no-changelog) (#6430) * fix: fix randomly failing scheduler node e2e tests (no-changelog) * chore: rename variable name * fix: update all cy.request calls to use backend base url * fix: add back mistkenly removed workflowId code * fix: remove unnecessary .then * fix: update how workflowId is retrieved --- cypress/constants.ts | 2 + cypress/e2e/15-scheduler-node.cy.ts | 63 +++++++++++++---------------- cypress/e2e/16-webhook-node.cy.ts | 31 +++++++------- 3 files changed, 44 insertions(+), 52 deletions(-) diff --git a/cypress/constants.ts b/cypress/constants.ts index 5118696bc81d3..a7e29665774de 100644 --- a/cypress/constants.ts +++ b/cypress/constants.ts @@ -1,3 +1,5 @@ +export const BACKEND_BASE_URL = 'http://localhost:5678'; + export const N8N_AUTH_COOKIE = 'n8n-auth'; export const DEFAULT_USER_EMAIL = 'nathan@n8n.io'; diff --git a/cypress/e2e/15-scheduler-node.cy.ts b/cypress/e2e/15-scheduler-node.cy.ts index b3ffd430e1631..cc4b3d775809a 100644 --- a/cypress/e2e/15-scheduler-node.cy.ts +++ b/cypress/e2e/15-scheduler-node.cy.ts @@ -1,4 +1,5 @@ import { WorkflowPage, WorkflowsPage, NDV } from '../pages'; +import { BACKEND_BASE_URL } from '../constants'; const workflowsPage = new WorkflowsPage(); const workflowPage = new WorkflowPage(); @@ -39,44 +40,34 @@ describe('Schedule Trigger node', async () => { workflowPage.actions.activateWorkflow(); workflowPage.getters.activatorSwitch().should('have.class', 'is-checked'); - cy.request('GET', '/rest/workflows') - .then((response) => { + cy.url().then((url) => { + const workflowId = url.split('/').pop(); + + cy.wait(1200); + cy.request('GET', `${BACKEND_BASE_URL}/rest/executions`).then((response) => { expect(response.status).to.eq(200); - expect(response.body.data).to.have.length(1); - const workflowId = response.body.data[0].id.toString(); - expect(workflowId).to.not.be.empty; - return workflowId; - }) - .then((workflowId) => { + expect(workflowId).to.not.be.undefined; + expect(response.body.data.results.length).to.be.greaterThan(0); + const matchingExecutions = response.body.data.results.filter( + (execution: any) => execution.workflowId === workflowId, + ); + expect(matchingExecutions).to.have.length(1); + cy.wait(1200); - cy.request('GET', '/rest/executions') - .then((response) => { - expect(response.status).to.eq(200); - expect(response.body.data.results.length).to.be.greaterThan(0); - const matchingExecutions = response.body.data.results.filter( - (execution: any) => execution.workflowId === workflowId, - ); - expect(matchingExecutions).to.have.length(1); - return workflowId; - }) - .then((workflowId) => { - cy.wait(1200); - cy.request('GET', '/rest/executions') - .then((response) => { - expect(response.status).to.eq(200); - expect(response.body.data.results.length).to.be.greaterThan(0); - const matchingExecutions = response.body.data.results.filter( - (execution: any) => execution.workflowId === workflowId, - ); - expect(matchingExecutions).to.have.length(2); - }) - .then(() => { - workflowPage.actions.activateWorkflow(); - workflowPage.getters.activatorSwitch().should('not.have.class', 'is-checked'); - cy.visit(workflowsPage.url); - workflowsPage.actions.deleteWorkFlow('Schedule Trigger Workflow'); - }); - }); + cy.request('GET', `${BACKEND_BASE_URL}/rest/executions`).then((response) => { + expect(response.status).to.eq(200); + expect(response.body.data.results.length).to.be.greaterThan(0); + const matchingExecutions = response.body.data.results.filter( + (execution: any) => execution.workflowId === workflowId, + ); + expect(matchingExecutions).to.have.length(2); + + workflowPage.actions.activateWorkflow(); + workflowPage.getters.activatorSwitch().should('not.have.class', 'is-checked'); + cy.visit(workflowsPage.url); + workflowsPage.actions.deleteWorkFlow('Schedule Trigger Workflow'); + }); }); + }); }); }); diff --git a/cypress/e2e/16-webhook-node.cy.ts b/cypress/e2e/16-webhook-node.cy.ts index 64556037d8b9c..178350a32b05a 100644 --- a/cypress/e2e/16-webhook-node.cy.ts +++ b/cypress/e2e/16-webhook-node.cy.ts @@ -1,6 +1,7 @@ import { WorkflowPage, NDV, CredentialsModal } from '../pages'; import { v4 as uuid } from 'uuid'; import { cowBase64 } from '../support/binaryTestFiles'; +import { BACKEND_BASE_URL } from '../constants'; const workflowPage = new WorkflowPage(); const ndv = new NDV(); @@ -83,7 +84,7 @@ const simpleWebhookCall = (options: SimpleWebhookCallOptions) => { ndv.actions.execute(); cy.wait(waitForWebhook); - cy.request(method, '/webhook-test/' + webhookPath).then((response) => { + cy.request(method, `${BACKEND_BASE_URL}/webhook-test/${webhookPath}`).then((response) => { expect(response.status).to.eq(200); ndv.getters.outputPanel().contains('headers'); }); @@ -98,12 +99,10 @@ describe('Webhook Trigger node', async () => { beforeEach(() => { workflowPage.actions.visit(); - cy.window().then( - (win) => { - // @ts-ignore - win.preventNodeViewBeforeUnload = true; - }, - ); + cy.window().then((win) => { + // @ts-ignore + win.preventNodeViewBeforeUnload = true; + }); }); it('should listen for a GET request', () => { @@ -154,7 +153,7 @@ describe('Webhook Trigger node', async () => { workflowPage.actions.executeWorkflow(); cy.wait(waitForWebhook); - cy.request('GET', '/webhook-test/' + webhookPath).then((response) => { + cy.request('GET', `${BACKEND_BASE_URL}/webhook-test/${webhookPath}`).then((response) => { expect(response.status).to.eq(200); expect(response.body.MyValue).to.eq(1234); }); @@ -172,7 +171,7 @@ describe('Webhook Trigger node', async () => { ndv.actions.execute(); cy.wait(waitForWebhook); - cy.request('GET', '/webhook-test/' + webhookPath).then((response) => { + cy.request('GET', `${BACKEND_BASE_URL}/webhook-test/${webhookPath}`).then((response) => { expect(response.status).to.eq(201); }); }); @@ -201,7 +200,7 @@ describe('Webhook Trigger node', async () => { workflowPage.actions.executeWorkflow(); cy.wait(waitForWebhook); - cy.request('GET', '/webhook-test/' + webhookPath).then((response) => { + cy.request('GET', `${BACKEND_BASE_URL}/webhook-test/${webhookPath}`).then((response) => { expect(response.status).to.eq(200); expect(response.body.MyValue).to.eq(1234); }); @@ -246,7 +245,7 @@ describe('Webhook Trigger node', async () => { workflowPage.actions.executeWorkflow(); cy.wait(waitForWebhook); - cy.request('GET', '/webhook-test/' + webhookPath).then((response) => { + cy.request('GET', `${BACKEND_BASE_URL}/webhook-test/${webhookPath}`).then((response) => { expect(response.status).to.eq(200); expect(Object.keys(response.body).includes('data')).to.be.true; }); @@ -263,7 +262,7 @@ describe('Webhook Trigger node', async () => { }); ndv.actions.execute(); cy.wait(waitForWebhook); - cy.request('GET', '/webhook-test/' + webhookPath).then((response) => { + cy.request('GET', `${BACKEND_BASE_URL}/webhook-test/${webhookPath}`).then((response) => { expect(response.status).to.eq(200); expect(response.body.MyValue).to.be.undefined; }); @@ -287,7 +286,7 @@ describe('Webhook Trigger node', async () => { cy.wait(waitForWebhook); cy.request({ method: 'GET', - url: '/webhook-test/' + webhookPath, + url: `${BACKEND_BASE_URL}/webhook-test/${webhookPath}`, auth: { user: 'username', pass: 'password', @@ -300,7 +299,7 @@ describe('Webhook Trigger node', async () => { .then(() => { cy.request({ method: 'GET', - url: '/webhook-test/' + webhookPath, + url: `${BACKEND_BASE_URL}/webhook-test/${webhookPath}`, auth: { user: 'test', pass: 'test', @@ -330,7 +329,7 @@ describe('Webhook Trigger node', async () => { cy.wait(waitForWebhook); cy.request({ method: 'GET', - url: '/webhook-test/' + webhookPath, + url: `${BACKEND_BASE_URL}/webhook-test/${webhookPath}`, headers: { test: 'wrong', }, @@ -342,7 +341,7 @@ describe('Webhook Trigger node', async () => { .then(() => { cy.request({ method: 'GET', - url: '/webhook-test/' + webhookPath, + url: `${BACKEND_BASE_URL}/webhook-test/${webhookPath}`, headers: { test: 'test', }, From 596cf07e42155338d6997edc1aff66db77d1edc2 Mon Sep 17 00:00:00 2001 From: Alex Grozav Date: Thu, 15 Jun 2023 15:30:05 +0300 Subject: [PATCH 16/37] feat: Replace all Vue.set usages with direct assignment and spread operator (no-changelog) (#6280) * refactor: replace all Vue.set usages with direct assignment and spread operator * chore: fix linting issue * fix: fix updateNodeAtIndex function * fix: various post-refactoring fixes * fix: refactor recently added Vue.set directive --- cypress/e2e/12-canvas.cy.ts | 3 +- cypress/e2e/14-mapping.cy.ts | 33 ++- cypress/pages/ndv.ts | 36 ++- packages/editor-ui/src/Interface.ts | 11 +- .../CredentialEdit/AuthTypeSelector.vue | 13 +- .../CredentialEdit/CredentialConfig.vue | 2 +- .../CredentialEdit/CredentialEdit.vue | 55 +++- .../src/components/ExecutionsList.vue | 23 +- .../src/components/NodeDetailsView.vue | 4 +- .../editor-ui/src/components/NodeSettings.vue | 86 ++++-- .../EventDestinationSettingsModal.ee.vue | 10 +- .../src/components/TagsContainer.vue | 4 +- .../src/components/WorkflowSettings.vue | 10 +- .../editor-ui/src/stores/credentials.store.ts | 42 +-- .../editor-ui/src/stores/n8nRoot.store.ts | 27 +- packages/editor-ui/src/stores/ndv.store.ts | 64 +++-- .../editor-ui/src/stores/nodeTypes.store.ts | 3 +- .../editor-ui/src/stores/settings.store.ts | 19 +- packages/editor-ui/src/stores/tags.store.ts | 10 +- .../editor-ui/src/stores/templates.store.ts | 74 +++-- packages/editor-ui/src/stores/ui.store.ts | 79 ++++-- packages/editor-ui/src/stores/users.store.ts | 19 +- .../src/stores/workflows.ee.store.ts | 39 ++- .../editor-ui/src/stores/workflows.store.ts | 261 +++++++++++++----- 24 files changed, 620 insertions(+), 307 deletions(-) diff --git a/cypress/e2e/12-canvas.cy.ts b/cypress/e2e/12-canvas.cy.ts index 29869e4434a52..65428acda8944 100644 --- a/cypress/e2e/12-canvas.cy.ts +++ b/cypress/e2e/12-canvas.cy.ts @@ -29,7 +29,6 @@ describe('Canvas Node Manipulation and Navigation', () => { WorkflowPage.actions.visit(); }); - it('should add switch node and test connections', () => { WorkflowPage.actions.addNodeToCanvas(SWITCH_NODE_NAME, true); @@ -114,7 +113,7 @@ describe('Canvas Node Manipulation and Navigation', () => { WorkflowPage.actions.zoomToFit(); cy.get('.plus-draggable-endpoint').filter(':visible').should('not.have.class', 'ep-success'); - cy.get('.jtk-connector.success').should('have.length', 3); + cy.get('.jtk-connector.success').should('have.length', 4); cy.get('.jtk-connector').should('have.length', 4); }); diff --git a/cypress/e2e/14-mapping.cy.ts b/cypress/e2e/14-mapping.cy.ts index b66e909b59586..c7035396d3b34 100644 --- a/cypress/e2e/14-mapping.cy.ts +++ b/cypress/e2e/14-mapping.cy.ts @@ -16,12 +16,10 @@ describe('Data mapping', () => { beforeEach(() => { workflowPage.actions.visit(); - cy.window().then( - (win) => { - // @ts-ignore - win.preventNodeViewBeforeUnload = true; - }, - ); + cy.window().then((win) => { + // @ts-ignore + win.preventNodeViewBeforeUnload = true; + }); }); it('maps expressions from table header', () => { @@ -303,19 +301,28 @@ describe('Data mapping', () => { ndv.getters.parameterInput('keepOnlySet').find('input[type="checkbox"]').should('exist'); ndv.getters.parameterInput('keepOnlySet').find('input[type="text"]').should('not.exist'); - ndv.getters.inputDataContainer().should('exist').find('span').contains('count').realMouseDown().realMouseMove(100, 100); + ndv.getters + .inputDataContainer() + .should('exist') + .find('span') + .contains('count') + .realMouseDown() + .realMouseMove(100, 100); cy.wait(50); ndv.getters.parameterInput('keepOnlySet').find('input[type="checkbox"]').should('not.exist'); - ndv.getters.parameterInput('keepOnlySet').find('input[type="text"]') + ndv.getters + .parameterInput('keepOnlySet') + .find('input[type="text"]') .should('exist') .invoke('css', 'border') .then((border) => expect(border).to.include('dashed rgb(90, 76, 194)')); - ndv.getters.parameterInput('value').find('input[type="text"]') - .should('exist') - .invoke('css', 'border') - .then((border) => expect(border).to.include('dashed rgb(90, 76, 194)')); + ndv.getters + .parameterInput('value') + .find('input[type="text"]') + .should('exist') + .invoke('css', 'border') + .then((border) => expect(border).to.include('dashed rgb(90, 76, 194)')); }); - }); diff --git a/cypress/pages/ndv.ts b/cypress/pages/ndv.ts index f23eb97f4f29f..2a9b6edc4f026 100644 --- a/cypress/pages/ndv.ts +++ b/cypress/pages/ndv.ts @@ -13,14 +13,17 @@ export class NDV extends BasePage { outputPanel: () => cy.getByTestId('output-panel'), executingLoader: () => cy.getByTestId('ndv-executing'), inputDataContainer: () => this.getters.inputPanel().findChildByTestId('ndv-data-container'), - inputDisplayMode: () => this.getters.inputPanel().findChildByTestId('ndv-run-data-display-mode').first(), + inputDisplayMode: () => + this.getters.inputPanel().findChildByTestId('ndv-run-data-display-mode').first(), outputDataContainer: () => this.getters.outputPanel().findChildByTestId('ndv-data-container'), - outputDisplayMode: () => this.getters.outputPanel().findChildByTestId('ndv-run-data-display-mode').first(), + outputDisplayMode: () => + this.getters.outputPanel().findChildByTestId('ndv-run-data-display-mode').first(), pinDataButton: () => cy.getByTestId('ndv-pin-data'), editPinnedDataButton: () => cy.getByTestId('ndv-edit-pinned-data'), pinnedDataEditor: () => this.getters.outputPanel().find('.cm-editor .cm-scroller'), runDataPaneHeader: () => cy.getByTestId('run-data-pane-header'), - savePinnedDataButton: () => this.getters.runDataPaneHeader().find('button').filter(':visible').contains('Save'), + savePinnedDataButton: () => + this.getters.runDataPaneHeader().find('button').filter(':visible').contains('Save'), outputTableRows: () => this.getters.outputDataContainer().find('table tr'), outputTableHeaders: () => this.getters.outputDataContainer().find('table thead th'), outputTableRow: (row: number) => this.getters.outputTableRows().eq(row), @@ -52,10 +55,13 @@ export class NDV extends BasePage { outputBranches: () => this.getters.outputPanel().findChildByTestId('branches'), inputBranches: () => this.getters.inputPanel().findChildByTestId('branches'), resourceLocator: (paramName: string) => cy.getByTestId(`resource-locator-${paramName}`), - resourceLocatorInput: (paramName: string) => this.getters.resourceLocator(paramName).find('[data-test-id="rlc-input-container"]'), - resourceLocatorDropdown: (paramName: string) => this.getters.resourceLocator(paramName).find('[data-test-id="resource-locator-dropdown"]'), + resourceLocatorInput: (paramName: string) => + this.getters.resourceLocator(paramName).find('[data-test-id="rlc-input-container"]'), + resourceLocatorDropdown: (paramName: string) => + this.getters.resourceLocator(paramName).find('[data-test-id="resource-locator-dropdown"]'), resourceLocatorErrorMessage: () => cy.getByTestId('rlc-error-container'), - resourceLocatorModeSelector: (paramName: string) => this.getters.resourceLocator(paramName).find('[data-test-id="rlc-mode-selector"]'), + resourceLocatorModeSelector: (paramName: string) => + this.getters.resourceLocator(paramName).find('[data-test-id="rlc-mode-selector"]'), }; actions = { @@ -82,7 +88,9 @@ export class NDV extends BasePage { this.getters.editPinnedDataButton().click(); this.getters.pinnedDataEditor().click(); - this.getters.pinnedDataEditor().type(`{selectall}{backspace}${JSON.stringify(data).replace(new RegExp('{', 'g'), '{{}')}`); + this.getters + .pinnedDataEditor() + .type(`{selectall}{backspace}${JSON.stringify(data).replace(new RegExp('{', 'g'), '{{}')}`); this.actions.savePinnedData(); }, @@ -131,15 +139,11 @@ export class NDV extends BasePage { }, changeInputRunSelector: (runName: string) => { this.getters.inputRunSelector().click(); - cy.get('.el-select-dropdown:visible .el-select-dropdown__item') - .contains(runName) - .click(); + cy.get('.el-select-dropdown:visible .el-select-dropdown__item').contains(runName).click(); }, changeOutputRunSelector: (runName: string) => { this.getters.outputRunSelector().click(); - cy.get('.el-select-dropdown:visible .el-select-dropdown__item') - .contains(runName) - .click(); + cy.get('.el-select-dropdown:visible .el-select-dropdown__item').contains(runName).click(); }, toggleOutputRunLinking: () => { this.getters.outputRunSelector().find('button').click(); @@ -159,7 +163,10 @@ export class NDV extends BasePage { this.getters.resourceLocatorInput(paramName).type(value); }, validateExpressionPreview: (paramName: string, value: string) => { - this.getters.parameterExpressionPreview(paramName).find('span').should('include.html', asEncodedHTML(value)); + this.getters + .parameterExpressionPreview(paramName) + .find('span') + .should('include.html', asEncodedHTML(value)); }, }; } @@ -172,4 +179,3 @@ function asEncodedHTML(str: string): string { .replace(/"/g, '"') .replace(/ /g, ' '); } - diff --git a/packages/editor-ui/src/Interface.ts b/packages/editor-ui/src/Interface.ts index 6b4e872bec718..b79ca557a0454 100644 --- a/packages/editor-ui/src/Interface.ts +++ b/packages/editor-ui/src/Interface.ts @@ -975,13 +975,10 @@ export interface ITagsState { fetchedUsageCount: boolean; } -export type Modals = - | { - [key: string]: ModalState; - } - | { - [CREDENTIAL_EDIT_MODAL_KEY]: NewCredentialsModal; - }; +export type Modals = { + [CREDENTIAL_EDIT_MODAL_KEY]: NewCredentialsModal; + [key: string]: ModalState; +}; export type ModalState = { open: boolean; diff --git a/packages/editor-ui/src/components/CredentialEdit/AuthTypeSelector.vue b/packages/editor-ui/src/components/CredentialEdit/AuthTypeSelector.vue index b38dba6d490e1..8a74217fb877f 100644 --- a/packages/editor-ui/src/components/CredentialEdit/AuthTypeSelector.vue +++ b/packages/editor-ui/src/components/CredentialEdit/AuthTypeSelector.vue @@ -11,7 +11,6 @@ import { } from '@/utils'; import type { INodeProperties, INodeTypeDescription, NodeParameterValue } from 'n8n-workflow'; import { computed, onMounted, ref } from 'vue'; -import Vue from 'vue'; export interface Props { credentialType: Object; @@ -27,7 +26,7 @@ const ndvStore = useNDVStore(); const props = defineProps(); const selected = ref(''); -const authRelatedFieldsValues = ref({} as { [key: string]: NodeParameterValue }); +const authRelatedFieldsValues = ref<{ [key: string]: NodeParameterValue }>({}); onMounted(() => { if (activeNodeType.value?.credentials) { @@ -43,7 +42,10 @@ onMounted(() => { // Populate default values of related fields authRelatedFields.value.forEach((field) => { - Vue.set(authRelatedFieldsValues.value, field.name, field.default); + authRelatedFieldsValues.value = { + ...authRelatedFieldsValues.value, + [field.name]: field.default as NodeParameterValue, + }; }); }); @@ -102,7 +104,10 @@ function onAuthTypeChange(newType: string): void { } function valueChanged(data: IUpdateInformation): void { - Vue.set(authRelatedFieldsValues.value, data.name, data.value); + authRelatedFieldsValues.value = { + ...authRelatedFieldsValues.value, + [data.name]: data.value as NodeParameterValue, + }; } defineExpose({ diff --git a/packages/editor-ui/src/components/CredentialEdit/CredentialConfig.vue b/packages/editor-ui/src/components/CredentialEdit/CredentialConfig.vue index 4071970fca1df..e33117ea65a40 100644 --- a/packages/editor-ui/src/components/CredentialEdit/CredentialConfig.vue +++ b/packages/editor-ui/src/components/CredentialEdit/CredentialConfig.vue @@ -264,7 +264,7 @@ export default defineComponent({ ); }, credentialTypeName(): string { - return (this.credentialType as ICredentialType).name; + return (this.credentialType as ICredentialType)?.name; }, credentialOwnerName(): string { return this.credentialsStore.getCredentialOwnerNameById(`${this.credentialId}`); diff --git a/packages/editor-ui/src/components/CredentialEdit/CredentialEdit.vue b/packages/editor-ui/src/components/CredentialEdit/CredentialEdit.vue index 28e7c2ea3db0d..37202c4eee523 100644 --- a/packages/editor-ui/src/components/CredentialEdit/CredentialEdit.vue +++ b/packages/editor-ui/src/components/CredentialEdit/CredentialEdit.vue @@ -109,7 +109,6 @@